import csv
import subprocess
from pathlib import Path
import os
import cv2
import dlib
import glob
from imutils import face_utils
import shutil

ROOT_PATH = '/home/bortoletti/projects/mastership/FaceLandmarksExtractor'

class LandMarkPoints:

    def open_face(self, final_image_path, output_folder, save_image=True):
        exec_path = str(Path(ROOT_PATH, "exec/openFace/FeatureExtraction"))
        print('OPEN FACE')
        print('filename: ' + final_image_path)
        subprocess.run(
            [exec_path + " -f " + str(final_image_path) + " -of " +
             output_folder + "/points.csv " + " -no3Dfp " + " -noMparams " +
             " -noPose " + " -noAUs " + " -noGaze "], shell=True)
        points_dict = {}
        output_file = output_folder + "/" + "points.csv"
        points_file = open(output_file, 'r')
        reader = csv.reader(points_file)
        headers = next(reader)
        for row in reader:
            for h, v in zip(headers, row):
                points_dict[h.strip()] = float(v)
        points_file.close()
        # shutil.rmtree(output_folder)
        if save_image:
            img = cv2.imread(final_image_path)
            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            output_filename = output_folder + '_openFace.jpg'
            self.__save_image(gray, points_dict, output_filename)
        return points_dict

    def open_cv_haar_face_detector(self, final_image_path, output_folder, save_image=True):
        exec_path = str(Path(ROOT_PATH, "exec/haarcascade_frontalface_alt2.xml"))
        img = cv2.imread(final_image_path)
        print('HAAR')
        print('filename: ' + final_image_path)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        face_cascade = cv2.CascadeClassifier(exec_path)
        faces = face_cascade.detectMultiScale(gray, 1.3, 5)
        points = None
        if len(faces) == 1:
            points = self.__extract_land_marks(faces[0], gray)
        if save_image:
            output_filename = output_folder + '_openCvHaar.jpg'
            self.__save_image(img, points, output_filename, faces[0])
        return points

    def open_cv_dnn_face_detector(self, final_image_path, output_folder, save_image=True):
        config_file = str(Path(ROOT_PATH, "exec/opencv_face_detector.pbtxt"))
        model = str(Path(ROOT_PATH, "exec/opencv_face_detector_uint8.pb"))
        img = cv2.imread(final_image_path)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        height, width, channels = img.shape
        print('DNN')
        print('filename: ' + final_image_path)
        net = cv2.dnn.readNetFromTensorflow(model, config_file)
        blob = cv2.dnn.blobFromImage(img, 1.0, (300, 300), [104, 117, 123], False, False)
        net.setInput(blob)
        detections = net.forward()
        points = None
        face = None
        for i in range(detections.shape[2]):
            confidence = detections[0, 0, i, 2]
            if confidence > 0.9:  # 0.9 threshold
                x1 = int(detections[0, 0, i, 3] * width)
                y1 = int(detections[0, 0, i, 4] * height)
                x2 = int(detections[0, 0, i, 5] * width)
                y2 = int(detections[0, 0, i, 6] * height)
                w = x2 - x1
                h = y2 - y1
                face = x1, y1, w, h
                points = self.__extract_land_marks(face, gray)
        if save_image:
            output_filename = output_folder + '_openCvDNN.jpg'
            self.__save_image(img, points, output_filename, face)
        return points

    def dlib_hog(self, final_image_path, output_folder, save_image=True):
        img = cv2.imread(final_image_path)
        print('HOG')
        print('filename: ' + final_image_path)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        hogFaceDetector = dlib.get_frontal_face_detector()
        faceRects = hogFaceDetector(img, 0)
        points = None
        face = None
        for faceRect in faceRects:
            x1 = faceRect.left()
            y1 = faceRect.top()
            x2 = faceRect.right()
            y2 = faceRect.bottom()
            w = x2 - x1
            h = y2 - y1
            face = x1, y1, w, h
            points = self.__extract_land_marks(face, gray)
        if save_image:
            output_filename = output_folder + '_dlibHOG.jpg'
            print(face)
            self.__save_image(img, points, output_filename, face)
        return points

    def dlib_cnn(self, final_image_path, output_folder, save_image=True):
        face_detector = str(Path(ROOT_PATH, "exec/mmod_human_face_detector.dat"))
        img = cv2.imread(final_image_path)
        scale_percent = 50  # percent of original size
        width = int(img.shape[1] * scale_percent / 100)
        height = int(img.shape[0] * scale_percent / 100)
        dim = (width, height)
        # resize image
        img = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
        print('CNN')
        print('filename: ' + final_image_path)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        dnnFaceDetector = dlib.cnn_face_detection_model_v1(face_detector)
        faceRects = dnnFaceDetector(img, 0)
        points = None
        face = None
        for faceRect in faceRects:
            x1 = faceRect.rect.left()
            y1 = faceRect.rect.top()
            x2 = faceRect.rect.right()
            y2 = faceRect.rect.bottom()
            w = x2 - x1
            h = y2 - y1
            face = x1, y1, w, h
            points = self.__extract_land_marks(face, gray)
        if save_image:
            output_filename = output_folder + '_dlibCNN.jpg'
            self.__save_image(img, points, output_filename, face)
        return points

    def __save_image(self, gray, points_dict, output_filename, face=None):
        for i in range(0, 68):
            x = int(points_dict['x_' + str(i)])
            y = int(points_dict['y_' + str(i)])
            cv2.circle(gray, (x, y), 7, (255, 255, 0), -1)
            cv2.putText(gray, str(i), (x, y), fontFace=cv2.FONT_HERSHEY_TRIPLEX, fontScale=0.7,
                        color=(0, 0, 0))
        base_path = os.path.dirname(output_filename)
        if not os.path.exists(base_path):
            os.makedirs(base_path, exist_ok=True)
        print(base_path)
        if face is not None:
            x, y, w, h = face
            gray = gray[y - 300:y + h + 300, x - 300:x + w + 300]
        cv2.imwrite(output_filename, gray)

    def __extract_land_marks(self, face, gray):
        shape_predictor = str(Path(ROOT_PATH, "exec/shape_predictor_68_face_landmarks.dat"))
        predictor = dlib.shape_predictor(shape_predictor)
        x, y, w, h = face
        dlib_rect = dlib.rectangle(int(x), int(y), int(x + w), int(y + h))
        shape = predictor(gray, dlib_rect)
        shape = face_utils.shape_to_np(shape)
        points = {}
        for (i, (x, y)) in enumerate(shape):
            points['x_' + str(i)] = x
            points['y_' + str(i)] = y
        return points

if __name__ == '__main__':
    teste = LandMarkPoints()
    cropped_images = os.listdir(ROOT_PATH + "/cropped_images")
    for i in cropped_images:
        try:
            src = ROOT_PATH + "/cropped_images/" + i
            # teste.dlib_hog(src, ROOT_PATH + "/points_in_face/" + i.split('_')[0])
            # teste.open_face(src, ROOT_PATH + "/points_in_face/" + i.split('_')[0])
            teste.open_cv_dnn_face_detector(src, ROOT_PATH + "/points_in_face/" + i.split('_')[0])
            teste.open_cv_haar_face_detector(src, ROOT_PATH + "/points_in_face/" + i.split('_')[0])
        except:
            print("")
    # print(teste.dlib_hog('/home/bortoletti/Downloads/eu.jpg', ''))
    # print(teste.dlib_hog('/home/bortoletti/projects/mastership/oxford_images/22q11/image_10001_jpg.jpg', ''))
    # print(teste.open_cv_dnn_face_detector('/home/bortoletti/projects/mastership/oxford_images/22q11/image_10001_jpg.jpg', ''))
    # print(teste.open_cv_haar_face_detector('/home/bortoletti/projects/mastership/oxford_images/22q11/image_10001_jpg.jpg', ''))
    # print(teste.open_face('/home/bortoletti/projects/mastership/oxford_images/22q11/image_10001_jpg.jpg',teste.ROOT_PATH))


import sqlite3
import tempfile

import pytest

from gretel_trainer.relational.connectors import sqlite_conn
from gretel_trainer.relational.core import MultiTableException, Scope

def test_extract_subsets_of_relational_data(example_dbs, tmpdir):
    with tempfile.NamedTemporaryFile() as f:
        con = sqlite3.connect(f.name)
        cur = con.cursor()
        with open(example_dbs / "ecom.sql") as sql_script:
            cur.executescript(sql_script.read())

        connector = sqlite_conn(f.name)

        with pytest.raises(MultiTableException):
            connector.extract(only={"users"}, ignore={"events"}, storage_dir=tmpdir)

        only = connector.extract(
            only={"users", "events", "products"}, storage_dir=tmpdir
        )
        ignore = connector.extract(
            ignore={"distribution_center", "order_items", "inventory_items"},
            storage_dir=tmpdir,
        )

    expected_tables = {"users", "events", "products"}
    assert set(only.list_all_tables(Scope.ALL)) == expected_tables
    assert set(ignore.list_all_tables(Scope.ALL)) == expected_tables

    # `products` has a foreign key to `distribution_center` in the source, but because the
    # latter table was not extracted, the relationship is not recognized
    assert only.get_parents("products") == []
    assert ignore.get_parents("products") == []


import numpy as np
import torch
from utils import ChunkDataManager_finetune

class CNNDMDataset(torch.utils.data.Dataset):
    """ Feeder for skeleton-based action recognition
    Arguments:
        data_path: data path
        data_type: 'pair_leadn' for BERT-imp, 'pair' for BERT-sim, 'single'
        sep_token: [SEP] token id
        logger: logger instance
        max_seq_len: max. sequence length for input
        debug: If true, only use the first 100 samples
    """

    def __init__(self,
                 data_path,
                 data_type,
                 sep_token,
                 logger,
                 max_seq_len=512,
                 debug=False
                 ):
        self.debug = debug
        self.data_path = data_path
        self.data_type = data_type
        self.sep_token = sep_token
        self.max_seq_len = max_seq_len
        self.logger = logger

        if 'train' in self.data_path:
            self.mode = 'train'
        elif 'val' in self.data_path:
            self.mode = 'val'
        elif 'test' in self.data_path:
            self.mode = 'test'

        print('sep token id', self.sep_token)
        self.load_data()

    def load_data(self):
        # load data
        cdm_data = ChunkDataManager_finetune(load_data_path=self.data_path)
        self.data = cdm_data.load(search_key=self.data_type)

        if self.debug:
            self.data = self.data[:100]

        self.logger.write('{}-{} {} data is loaded from {}'.format(self.mode,
                                                                   self.data_type, len(self.data), self.data_path))

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        input_ids, label, src_idx = self.data[index]

        if len(input_ids) > self.max_seq_len:
            input_ids = input_ids[:self.max_seq_len - 1]
            input_ids[-1] = self.sep_token

        segment_ids = [0] * len(input_ids)
        input_mask = [1] * len(input_ids)

        segment_b = False
        for i, ids in enumerate(input_ids):
            if segment_b:
                segment_ids[i] = 1
            if ids == self.sep_token:
                segment_b = True

        # Zero-pad up to the sequence length.
        padding = [0] * (self.max_seq_len - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == self.max_seq_len
        assert len(input_mask) == self.max_seq_len
        assert len(segment_ids) == self.max_seq_len

        # tensorize
        input_ids = torch.from_numpy(
            np.array(input_ids)).type(torch.LongTensor)
        input_mask = torch.from_numpy(
            np.array(input_mask)).type(torch.LongTensor)
        segment_ids = torch.from_numpy(
            np.array(segment_ids)).type(torch.LongTensor)
        label = torch.tensor(label, dtype=torch.float)

        return input_ids, input_mask, segment_ids, label


import os
import numpy as np
from scipy.spatial.distance import cdist
from tqdm import tqdm
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

import torch
from torch.optim import lr_scheduler

from opt import opt
from data import Data
from network5 import MGN
from loss import Loss
from utils import get_optimizer,extract_feature
from metrics import mean_ap, cmc, re_ranking
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

import visdom

class Main():
    def __init__(self, model, loss, data):
        self.train_loader = data.train_loader
        self.test_loader = data.test_loader
        self.query_loader = data.query_loader
        self.testset = data.testset
        self.queryset = data.queryset

        self.model = model.to('cuda')
        self.loss = loss
        self.optimizer = get_optimizer(model)
        self.scheduler = lr_scheduler.MultiStepLR(self.optimizer, milestones=opt.lr_scheduler, gamma=0.1)

        self.trackloss=0

        # self.track_loss = 0
        # self.global_step = 0
        # self.vis = visdom.Visdom(env=u"train_loss")
        # self.win = self.vis.line(X=np.array([self.global_step]), Y=np.array([self.track_loss]))

    def train(self):

        self.scheduler.step()

        self.model.train()
        for batch, (inputs, labels) in enumerate(self.train_loader):
            inputs = inputs.to('cuda')
            labels = labels.to('cuda')
            self.optimizer.zero_grad()
            outputs = self.model(inputs)
            loss = self.loss(outputs, labels)
            self.trackloss=loss
            # if self.global_step%100==0:
            #     self.vis.line(X=np.array([self.global_step]), Y=np.array([loss.data[0]]), win=self.win,update='append')
            # self.global_step += 1
            loss.backward()
            self.optimizer.step()

    def evaluate(self):

        self.model.eval()

        print('extract features, this may take a few minutes')
        qf = extract_feature(self.model, tqdm(self.query_loader)).numpy()
        gf = extract_feature(self.model, tqdm(self.test_loader)).numpy()

        def rank(dist):
            r = cmc(dist, self.queryset.ids, self.testset.ids, self.queryset.cameras, self.testset.cameras,
                    separate_camera_set=False,
                    single_gallery_shot=False,
                    first_match_break=True)
            m_ap = mean_ap(dist, self.queryset.ids, self.testset.ids, self.queryset.cameras, self.testset.cameras)

            return r, m_ap

        #########################   re rank##########################
        q_g_dist = np.dot(qf, np.transpose(gf))
        q_q_dist = np.dot(qf, np.transpose(qf))
        g_g_dist = np.dot(gf, np.transpose(gf))
        dist = re_ranking(q_g_dist, q_q_dist, g_g_dist)

        r, m_ap = rank(dist)

        print('[With    Re-Ranking] mAP: {:.4f} rank1: {:.4f} rank3: {:.4f} rank5: {:.4f} rank10: {:.4f}'
              .format(m_ap, r[0], r[2], r[4], r[9]))

        #########################no re rank##########################
        dist = cdist(qf, gf)

        r, m_ap = rank(dist)

        print('[Without Re-Ranking] mAP: {:.4f} rank1: {:.4f} rank3: {:.4f} rank5: {:.4f} rank10: {:.4f}'
              .format(m_ap, r[0], r[2], r[4], r[9]))

    def vis(self):

        self.model.eval()

        gallery_path = data.testset.imgs
        query_path=data.queryset.imgs

        # Extract feature
        print('extract features, this may take a few minutes')
        #query_feature = extract_feature(model, tqdm([(torch.unsqueeze(data.query_image, 0), 1)]))
        query_feature=extract_feature(model, tqdm(data.query_loader))
        gallery_feature = extract_feature(model, tqdm(data.test_loader))
        print(query_feature.size())

        for query_Index in range(query_feature.size()[0]):
            # sort images
            query_img_path=query_path[query_Index]
            query_feature_now = query_feature[query_Index].unsqueeze(1)
            print(query_feature_now.size())
            #query_feature = query_feature.view(-1, 1)
            #print(query_feature.size())
            score = torch.mm(gallery_feature, query_feature_now)
            score = score.squeeze(1).cpu()
            score = score.numpy()

            index = np.argsort(score)  # from small to large
            index = index[::-1]  # from large to small

            # # Remove junk images
            # junk_index = np.argwhere(gallery_label == -1)
            # mask = np.in1d(index, junk_index, invert=True)
            # index = index[mask]

            # Visualize the rank result
            fig = plt.figure(figsize=(16, 4))

            ax = plt.subplot(1, 11, 1)
            ax.axis('off')
            plt.imshow(plt.imread(query_img_path))
            ax.set_title('query')

            #print('Top 10 images are as follow:')

            for i in range(10):
                img_path = gallery_path[index[i]]
                #print(img_path)

                ax = plt.subplot(1, 11, i + 2)
                ax.axis('off')
                plt.imshow(plt.imread(img_path))
                ax.set_title(img_path.split('/')[-1][:9])

            fig.savefig("/home/wangminjie/Desktop/wmj/projects/Part-reID_2/result/detect/{}.png".format(query_img_path.split('/')[-1][:9]))
            print('result saved to show{}.png'.format(query_img_path.split('/')[-1][:9]))
        print("task end")

if __name__ == '__main__':

    data = Data()
    model = MGN()
    loss = Loss()
    reid = Main(model, loss, data)

    # track_loss = 0
    # global_step = 0
    # vis = visdom.Visdom(env=u"train_loss")
    # win = vis.line(X=np.array([global_step]), Y=np.array([track_loss]))

    if opt.mode == 'train':

        for epoch in range(1, opt.epoch+1):
            print('\nepoch', epoch)
            reid.train()
            # vis.line(X=np.array([global_step]), Y=np.array([reid.trackloss.data[0]]), win=win,update='append')
            # global_step += 1
            if epoch % 100 == 0:
                print('\nstart evaluate')
                reid.evaluate()
                os.makedirs('weights',exist_ok=True)
                torch.save(model.state_dict(), ('weights/model_{}.pt'.format(epoch)))

    if opt.mode == 'evaluate':
        print('start evaluate')
        model.load_state_dict(torch.load(opt.weight))
        reid.evaluate()

    if opt.mode == 'vis':
        print('visualize')
        model.load_state_dict(torch.load(opt.weight))
        reid.vis()


import aiohttp

from ..types import *
from ..exceptions import *

class AioCryptoPay(BaseCryptoPay):
    base_url = "https://pay.crypt.bot/api/"

    def __init__(self, token: str):
        super().__init__(token)
        self.session = aiohttp.ClientSession(
            headers={
                "Host": "pay.crypt.bot",
                "Crypto-Pay-API-Token": token
            }
        )

    async def close(self):
        await self.session.close()

    async def _make_request(self, method, method_type, **kwargs) -> dict:
        request = getattr(self.session, method_type)

        response = await request(self.base_url + method, json=kwargs)
        response = await response.json()

        if response.get("ok"):
            return response.get("result")

        else:
            error = response.get("error")
            raise CryptoPayException(error)

    async def get_me(self) -> App:
        response = await self._make_request("getMe", "get")
        return App(**response)

    async def create_invoice(
            self,
            asset: str,
            amount: float,
            description: Optional[str] = None,
            hidden_message: Optional[str] = None,
            paid_btn_name: Optional[str] = None,
            paid_btn_url: Optional[str] = None,
            payload: Optional[str] = None,
            allow_comments: Optional[bool] = True,
            allow_anonymous: Optional[bool] = True,
            expires_in: Optional[int] = None
    ) -> Invoice:
        amount = str(amount)

        params = self._get_params(locals(), "createInvoice")

        response = await self._make_request(
            "createInvoice",
            "get",
            **params
        )

        return Invoice(**response)

    async def transfer(
            self,
            user_id: int,
            asset: str,
            amount: float,
            spend_id: str,
            comment: Optional[str] = None,
            disable_send_notification: Optional[bool] = False
    ) -> Transfer:
        amount = str(amount)

        params = self._get_params(locals(), "transfer")

        response = await self._make_request(
            "transfer",
            "get",
            **params
        )

        return Transfer(**response)

    async def get_invoices(
            self,
            asset: Optional[str] = None,
            invoice_ids: Optional[Union[int, List[int]]] = "",
            status: Optional[str] = None,
            offset: Optional[str] = None,
            count: Optional[str] = None
    ) -> List[Invoice]:

        if invoice_ids:
            if type(invoice_ids) == int:
                invoice_ids = str(invoice_ids)

        params = self._get_params(locals(), "createInvoice")

        response = await self._make_request(
            "getInvoices",
            "get",
            **params
        )

        invoices = [Invoice(**invoice) for invoice in response.get("items")]

        return invoices

    async def get_balance(self) -> List[Wallet]:
        response = await self._make_request("getBalance", "get")
        wallets = [Wallet(**wallet) for wallet in response]

        return wallets

    async def get_exchange_rates(self) -> List[ExchangeRate]:
        response = await self._make_request("getExchangeRates", "get")
        exchange_rates = [ExchangeRate(**exchange_rate) for exchange_rate in response]

        return exchange_rates

    async def get_currencies(self) -> List[Currency]:
        response = await self._make_request("getCurrencies", "get")
        print(response)
        currencies = [Currency(**currency) for currency in response]

        return currencies

'''
Передаю привет:
https://t.me/LordS_LOLZ
https://t.me/MOLUTBA
https://t.me/derkonw
https://t.me/mrrooooopert
https://t.me/zen220
https://t.me/Warkraftboy
https://t.me/Reboler
https://t.me/rage_nosoft
https://t.me/crypt0xTG
https://t.me/lolzwork
Horosh(oleg383 вроде хз убрал юзернейм)
https://t.me/raptinside

Ascii-арт с валему (https://t.me/walemu1337 (пес)):

┈╱╲┏┓▕╲▂▂▂╱▏
╱┏┓╲▏┈▏┻╯▃┻╲
╭━━╮╲┈▏╭┳┻┳╮▏
┃╭╮┃▕┈▏┣╮┈┈┃▏
┃┃╰╯▕╱╲╰┻┻┻╱
┃┃╱▔▔▕▕▔▕▔▔
┃╰▏▔╲▕▕╱▏▏
╰━╲▂▂▕▂▏▂▏

'''


import os
import os.path as osp
import numpy as np
import pandas as pd
from tqdm import tqdm
import argparse
from PIL import Image

def resize_img(img_name, img_w=256, img_h=128):
    with Image.open(img_name) as img:
        width, height = img.size
        ideal_w = height * 2
        if width > ideal_w:
            l, r = (width - ideal_w) // 2, (width + ideal_w) // 2
            img = img.crop((l, 0, r, height))
        img.thumbnail((img_w, img_h))
        array = np.array(img, dtype=np.float32)

    assert array.shape == (img_h, img_w, 3), \
        f'Image shape {array.shape} mismatch, expecting ({img_h}, {img_w}, 3)'

    return array

def main(args):
    labels = pd.read_csv(osp.join(args.input_dir, 'images.csv'), header=0, index_col=None)
    labels = labels.drop(11059).reset_index(drop=True) # hard-coded for single missing item
    labels.lat /= 90
    labels.lng /= 180

    img_dir = osp.join(args.input_dir, 'images')

    print(f'Split training and testing data with ratio {args.train_ratio}')
    train_labels = labels.sample(frac=args.train_ratio, random_state=args.seed)
    test_labels = labels.drop(train_labels.index)
    num_train, num_test = train_labels.shape[0], test_labels.shape[0]
    print(f'training data: {num_train}, testing data: {num_test}')

    train_images = np.array([
        resize_img(
            osp.join(img_dir, f'{train_id}.jpeg'),
            args.img_w, args.img_h
        ) for train_id in tqdm(
            train_labels.id, desc='processing training images', position=0, leave=True
        )
    ])
    test_images = np.array([
        resize_img(
            osp.join(img_dir, f'{test_id}.jpeg'),
            args.img_w, args.img_h
        ) for test_id in tqdm(
            test_labels.id, desc='processing testing images', position=0, leave=True
        )
    ])

    assert train_images.shape == (num_train, args.img_h, args.img_w, 3), \
        f'train_images shape {train_images.shape} mismatch, expecting ({num_train}, {args.img_h}, {args.img_w}, 3)'
    assert test_images.shape == (num_test, args.img_h, args.img_w, 3), \
        f'test_images shape {test_images.shape} mismatch, expecting ({num_test}, {args.img_h}, {args.img_w}, 3)'

    np.save(osp.join(args.out_dir, 'train_images.npy'), train_images)
    np.save(osp.join(args.out_dir, 'test_images.npy'), test_images)
    train_labels.to_csv(osp.join(args.out_dir, 'train_images.csv'), header=True, index=False)
    test_labels.to_csv(osp.join(args.out_dir, 'test_images.csv'), header=True, index=False)

    print(f'Preprocessing complete. Files saved to {args.out_dir}')

    return

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_dir', type=str, required=True)
    parser.add_argument('--out_dir', type=str, required=True)
    parser.add_argument('--img_w', type=int, default=256)
    parser.add_argument('--img_h', type=int, default=128)
    parser.add_argument('--train_ratio', type=float, default=0.8)
    parser.add_argument('--seed', type=int, default=977)

    args = parser.parse_args()

    main(args)


import json
import os
from argparse import ArgumentParser
from pathlib import Path

import networkx as nx
from collections import defaultdict
import sys
import matplotlib.pyplot as plt
from connected_components import read_nodeadjlist, findCommunities

def drawGraph(G):
    pos = nx.shell_layout(G)
    nx.draw(G, pos)

def common_friends(args):
    egofile = open(args.eponet_file)
    result = dict()
    for line in egofile:
        v_i, v = line.split(":")
        result[v_i] = len(v.split())
    return result

def circle_sizes(circle_file_path):
    "Returns a dict, indexed by circle_id, of each circle size."
    circle_file = open(circle_file_path)
    result = dict()
    for line in circle_file:
        circle_id, friends = line.split(':')
        result[circle_id] = len(friends.split())
    return result

def display_dist_with_circles(user_id, bins=20):
    plt.hist(common_friends('egonets/' + str(user_id) + '.egonet').values(), bins=bins)
    plt.title("User " + str(user_id))
    for circle_size in sorted(circle_sizes('Training/' + str(user_id) + '.circles').values()):
        plt.axvline(x=circle_size, color='r')

def get_friend_count_matrix(egofile_path):
    """Return a dict of dicts giving a matrix (indexed by ID) of friends in common.
    Should be a symmetric matrix with the values of common_friends(egofile_path) as
    the diagonal elements."""

    egofile = open(egofile_path)
    friend_sets = dict()
    for line in egofile:
        v_i, v = line.split(":")
        friend_sets[v_i] = set(v.split())
    result = dict()
    for v_i in friend_sets.keys():
        result[v_i] = dict()
        for v_j in friend_sets.keys():
            result[v_i][v_j] = len(friend_sets[v_i].intersection(friend_sets[v_j]))
    return result

def main(args):
    G = read_nodeadjlist(args.eponet_file)
    G2 = nx.subgraph(G, G.neighbors(87))
    drawGraph(G2)

    ccs = dict()
    for friend in G.nodes():
        ffriends = G.neighbors(friend)
        subgraph = nx.subgraph(G, ffriends)
        fcc = nx.connected_components(subgraph)
        ccs[friend] = len(fcc)
    print(ccs)

    G = read_nodeadjlist(args.eponet_file)
    common_friends = []
    for friend in G.nodes():
        common_friends.append(len(G.neighbors(friend)))
    plt.hist(common_friends)

    # print(nx.triangles(G))

    # cf = common_friends(args.eponet_file)
    # plt.hist(cf.values())

    # print(sum([x == 0 for x in cf.values()]))
    # print(circle_sizes('Training/239.circles'))
    # display_dist_with_circles(239)
    # display_dist_with_circles(345)

    # circle_files = os.listdir('Training')
    # plt.figure(plt.figsize(18,60))
    # for (n, circle_file) in enumerate(circle_files):
    #     user_id = int(os.path.splitext(circle_file)[0])
    #     plt.subplot(20, 3, n+1)
    #     display_dist_with_circles(user_id)

    # ccf239 = get_friend_count_matrix('egonets/239.egonet')
    # assert len(ccf239) == len(cf239)
    # for i in ccf239.keys():
    #     for j in ccf239[i].keys():
    #         assert ccf239[i][j] == ccf239[j][i]
    # for i in ccf239.keys():
    #     assert ccf239[i][i] == cf239[i]

    # plt.figure(figsize=(3,5))
    # values = []
    # for v_i, x in ccf239.items():
    #     for v_j, n in x.items():
    #         if int(v_i) > int(v_j):
    #             values.append(int(n))
    # plt.ylim(0,600)
    # plt.hist(values)

    egofile = open(args.eponet_file)
    friend_sets = dict()
    for line in egofile:
        v_i, v = line.split(":")
        friend_sets[v_i] = set(v.split())
    result = dict()
    for v_i in friend_sets.keys():
        result[v_i] = dict()
        for v_j in friend_sets.keys():
            result[v_i][v_j] = len(friend_sets[v_i].intersection(friend_sets[v_j]))
    print(result)
    values = []
    for v_i, x in result.items():
        for v_j, n in x.items():
            if int(v_i) > int(v_j):
                values.append(int(n))
    plt.ylim(0, 600)
    plt.hist(values, bins=30)
    pass

def parse_args():
    parser = ArgumentParser()
    parser.add_argument("--featureList", type=Path,
                        default="./featureList.txt")
    parser.add_argument("--training_path", type=Path,
                        default="./Training")
    parser.add_argument("--egonets_path", type=Path,
                        default="./egonets")
    parser.add_argument("--features", type=Path,
                        default="./features.txt")
    parser.add_argument("--eponet_file", type=Path,
                        default="egonets/0.egonet")
    parser.add_argument("--output_csv", type=Path,
                        default="characterist_profiles.csv")
    parser.add_argument("--userID", type=int,
                        default=239)
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    args = parse_args()
    main(args)


import numpy as np
from utils import assert_right_type
from sklearn.linear_model import LinearRegression
from copy import deepcopy

class Markowitz:
    def __init__(self, data: np.ndarray, rf: np.ndarray = None, logreturns: bool = True, gamma: float = 3.):
        """
        Compute mean-variance optimization with sample covariance matrix for the case of N risky assets.
        The risk-free rate 'rf' is utilized just for the optimization of excess returns.

        :param data: T-by-N price matrix with assets on axis=1 and time series on axis=0.
        :param rf: Risk-free rate. If the purpose is to optimize excess returns, then provide rf.
        :param logreturns: Log-return if True, else simple return.
        """
        assert_right_type(data, np.ndarray)
        self.__data: np.ndarray = data  # (T, N)
        # log_prices: np.ndarray = np.log1p(data)  # (T, N)
        # self.__log_returns: np.ndarray = np.diff(log_prices, axis=0)  # (T, N-1)
        return_type: dict = {True: Markowitz._log_returns, False: Markowitz._simple_returns}

        assert_right_type(gamma, float)
        self.__gamma = gamma
        self.__t: int = data.shape[0]
        self.__n: int = data.shape[1]
        if isinstance(rf, np.ndarray):
            assert_right_type(rf, np.ndarray)
            self.__rf: np.ndarray = rf  # (T-1, 1)
        else:  # rf = None
            self.__rf: np.ndarray = np.zeros((self.__t - 1, 1))  # (T-1, 1)

        # Log or simple returns; in excess of the risk-free rate if passed as argument.
        self.__returns: np.ndarray = return_type[logreturns](self.__data, self.__rf)  # (T-1, N)

        # Expected Returns
        self.__mu: np.ndarray = np.apply_along_axis(func1d=np.mean, axis=0, arr=self.__returns)  # shape -> (N,)

        # Sample covariance matrix
        self._sigma = np.cov(self.__returns.T)  # shape -> (N, N)

    @staticmethod
    def _log_returns(prices: np.ndarray, rf: np.ndarray) -> np.ndarray:
        r = np.diff(np.log1p(prices), axis=0) - rf
        # Check if there is an 'inf' left in the matrix
        assert not np.isinf(r).any(), f"'inf' values left in returns."
        return r  # (T-1, N)

    @staticmethod
    def _simple_returns(prices: np.ndarray, rf: np.ndarray) -> np.ndarray:
        # np.nan prevent 'inf' values to happen most of the time, at the end we convert them to 0
        r = np.diff(prices, axis=0) / prices[:-1, :] - rf
        assert not np.isinf(r).any(), f"'inf' values left in returns."
        return r  # (T-1, N)

    def _sigma_inv(self) -> np.ndarray:
        return np.linalg.inv(self._sigma)  # shape -> (N, N)

    def compute(self) -> np.ndarray:
        w_star = (1 / self.__gamma) * np.matmul(self._sigma_inv(), self.__mu)
        return deepcopy(w_star)  # shape -> (N,)

    def data(self) -> np.ndarray:
        return deepcopy(self.__data)

    def returns(self):
        return deepcopy(self.__returns)

    # def log_returns(self) -> np.ndarray:
    #     return self.__log_returns

    def mu(self) -> np.ndarray:
        return deepcopy(self.__mu)

class MarkowitzShrinkedConstantCorrelation(Markowitz):  # Constant Correlation
    def __constant_correlation(self):
        cor_matrix: np.ndarray = np.corrcoef(self.returns().T)
        mean_cor: np.ndarray = np.mean(cor_matrix)
        rows, columns = cor_matrix.shape
        f: np.ndarray = np.zeros((rows, columns))
        # Far from being optimized
        for i in range(rows):
            for j in range(columns):
                f[i][j] = mean_cor * np.sqrt(cor_matrix[i][i] * cor_matrix[j][j])
        return f

    def _sigma_inv(self) -> np.ndarray:
        lambda_ = .2
        f = self.__constant_correlation()
        shrinked = f * lambda_ + (1 - lambda_) * self._sigma
        return np.linalg.pinv(shrinked)

class MarkowitzShrinkedIdentity(Markowitz):  # Identity Matrix
    def _sigma_inv(self) -> np.ndarray:
        lambda_ = .2
        f = np.identity(self._sigma.shape[0])
        shrinked = f * lambda_ + (1 - lambda_) * self._sigma
        return np.linalg.pinv(shrinked)

class MarkowitzShrinkedSI(Markowitz):  # Single Index
    def __market_portfolio(self):
        data = self.data()
        log = np.log1p(data)
        diff = np.diff(log, axis=0)
        market = np.apply_along_axis(np.sum, 1, diff)
        return market

    def __f(self):
        market = self.__market_portfolio()
        returns = self.returns()
        reg = LinearRegression().fit(returns, market)
        pred = reg.predict(returns)
        residuals = market - pred
        residuals_variance = np.fromiter(
            (np.cov(returns[:, i], residuals)[0, 1] for i in range(returns.shape[1])),
            dtype=float)
        D = np.identity(residuals_variance.shape[0]) * residuals_variance
        s_2 = market.var()
        beta = reg.coef_
        return s_2 * beta * beta.reshape((-1, 1)) + D

    def _sigma_inv(self) -> np.ndarray:
        lambda_ = .2
        shrinked = self.__f() * lambda_ + (1 - lambda_) * self._sigma
        return np.linalg.pinv(shrinked)


import psutil

from tendrl.commons import objects
from tendrl.commons.utils import cmd_utils

class Cpu(objects.BaseObject):
    def __init__(self, architecture=None, cores_per_socket=None,
                 cpu_family=None, cpu_op_mode=None,
                 model=None, model_name=None, vendor_id=None,
                 cpu_util_percent=None,
                 *args, **kwargs):
        super(Cpu, self).__init__(*args, **kwargs)
        cpu = self._getNodeCpu()
        self.architecture = architecture or cpu['Architecture']
        self.cores_per_socket = cores_per_socket or cpu["CoresPerSocket"]
        self.cpu_family = cpu_family or cpu["CPUFamily"]
        self.cpu_op_mode = cpu_op_mode or cpu["CpuOpMode"]
        self.model = model or cpu["Model"]
        self.model_name = model_name or cpu["ModelName"]
        self.vendor_id = vendor_id or cpu["VendorId"]
        self.cpu_util_percent = cpu_util_percent or str(psutil.cpu_percent())
        self.value = 'nodes/{0}/Cpu'

    def _getNodeCpu(self):
        '''returns structure

        {"nodename": [{"Architecture":   "architecture",

                   "CpuOpMode":      "cpuopmode",

                   "CPUs":           "cpus",

                   "VendorId":       "vendorid",

                   "ModelName":      "modelname",

                   "CPUFamily":      "cpufamily",

                   "Model":          "Model",

                   "CoresPerSocket": "corespersocket"}, ...], ...}

        '''
        cmd = cmd_utils.Command("lscpu")
        out, err, rc = cmd.run()
        out = str(out)
        if out:
            info_list = out.split('\n')
            cpuinfo = {
                'Architecture': info_list[0].split(':')[1].strip(),
                'CpuOpMode': info_list[1].split(':')[1].strip(),
                'CPUs': info_list[3].split(':')[1].strip(),
                'VendorId': info_list[9].split(':')[1].strip(),
                'ModelName': info_list[12].split(':')[1].strip(),
                'CPUFamily': info_list[10].split(':')[1].strip(),
                'Model': info_list[11].split(':')[1].strip(),
                'CoresPerSocket': info_list[6].split(':')[1].strip()
            }
        else:
            cpuinfo = {
                'Architecture': '', 'CpuOpMode': '',
                'CPUs': '', 'VendorId': '',
                'ModelName': '', 'CPUFamily': '',
                'Model': '', 'CoresPerSocket': ''
            }

        return cpuinfo

    def render(self):
        self.value = self.value.format(NS.node_context.node_id)
        return super(Cpu, self).render()


from django.contrib.auth import get_user_model
from django.contrib.contenttypes.models import ContentType
from django.test import TestCase
from guardian.forms import BaseObjectPermissionsForm

class BaseObjectPermissionsFormTests(TestCase):

    def setUp(self):
        self.user = get_user_model().objects.create_user(
            'joe', 'joe@example.com', 'joe')
        self.obj = ContentType.objects.create(
            model='bar', app_label='fake-for-guardian-tests')

    def test_not_implemented(self):

        class MyUserObjectPermissionsForm(BaseObjectPermissionsForm):

            def __init__(formself, user, *args, **kwargs):
                self.user = user
                super().__init__(*args, **kwargs)

        form = MyUserObjectPermissionsForm(self.user, self.obj, {})
        self.assertRaises(NotImplementedError, form.save_obj_perms)

        field_name = form.get_obj_perms_field_name()
        self.assertTrue(form.is_valid())
        self.assertEqual(len(form.cleaned_data[field_name]), 0)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# (c) @AlbertEinsteinTG

from pyrogram import filters, Client
from pyrogram.types import InlineKeyboardButton, InlineKeyboardMarkup, CallbackQuery
from bot import Translation, LOGGER # pylint: disable=import-error
from bot.database import Database # pylint: disable=import-error
import asyncio
from pyrogram.errors import FloodWait
from bot.bot import Bot
from bot import ADMINS

db = Database()

@Client.on_message(filters.command(["start"]) & filters.private, group=1)
async def start(bot, update):

    try:
        file_uid = update.command[1]
    except IndexError:
        file_uid = False

    if file_uid:
        file_id, file_name, file_caption, file_type = await db.get_file(file_uid)

        if (file_id or file_type) == None:
            return

        caption = file_caption if file_caption != ("" or None) else ("" + file_name + "")
        try:
            await update.reply_cached_media(
                file_id,
                quote=True,
                caption = caption,
                parse_mode="html",
                reply_markup=InlineKeyboardMarkup(
                    [
                        [
                            InlineKeyboardButton
                                (
                                    'Share 😐', url="https://t.me/share/url?url=https://t.me/Cinema_Haunter"
                                )
                        ]
                    ]
                )
            )
        except Exception as e:
            await update.reply_text(f"Error:\n{e}", True, parse_mode="html")
            LOGGER(__name__).error(e)
        return

    await bot.send_photo(
        chat_id=update.chat.id,
        photo="https://telegra.ph/file/9d589c04d3db03a9ccb99.jpg",
        caption=Translation.START_TEXT.format(
                update.from_user.mention),
        reply_markup=InlineKeyboardMarkup(
            [
                [
                    InlineKeyboardButton("⚙ℍ𝕖𝕝𝕡", callback_data = "help")
                ],
                [
                    InlineKeyboardButton('🏘𝔾𝕣𝕠𝕦𝕡', url="https://t.me/Cinema_Haunter"),
                    InlineKeyboardButton('🎬ℂ𝕙𝕒𝕟𝕟𝕖𝕝', url="https://t.me/CinemaHaunter")
                ],
                [
                    InlineKeyboardButton('🔎𝕌𝕡𝕕𝕒𝕥𝕖𝕤', url="https://t.me/WhatTheCinema"),
                    InlineKeyboardButton('🗃𝕊𝕠𝕦𝕣𝕔𝕖', callback_data = "source")
                ]
            ]
        ),
        parse_mode="html",
        reply_to_message_id=update.message_id
    )

@Client.on_message(filters.command(["help"]) & filters.private, group=1)
async def help(bot, update):
    buttons = [[
        InlineKeyboardButton('𝔸𝕦𝕥𝕠 𝔽𝕚𝕝𝕥𝕖𝕣', callback_data = "autofilter"),
        InlineKeyboardButton('𝕍𝕔 ℙ𝕝𝕒𝕪𝕖𝕣', callback_data = "vcplayer")
    ],[
        InlineKeyboardButton('𝕌𝕟𝕝𝕚𝕞𝕚𝕥𝕖𝕕 𝔽𝕚𝕝𝕥𝕖𝕣', callback_data = "filter"),
        InlineKeyboardButton('𝔽𝕚𝕝𝕖 𝕊𝕥𝕠𝕣𝕖', callback_data = "filestore")
    ],[
        InlineKeyboardButton('𝔸𝕓𝕠𝕦𝕥', callback_data = "about")
    ]]

    reply_markup = InlineKeyboardMarkup(buttons)

    await bot.send_message(
        chat_id=update.chat.id,
        text=Translation.HELP_TEXT,
        reply_markup=reply_markup,
        parse_mode="html",
        reply_to_message_id=update.message_id
    )

@Client.on_message(filters.text & ~ filters.command(["start","help","batch","genlink","cccurrent","userbotjoinchannel","channelplay","play","dplay","splay","player","skip","pause","resume","end","current","playlist","cresume","cplayer","cplaylist","cdplay","unset","csplay","cplay","pmpermit","gcast","userbotleaveall","userbotjoin","admincache","remall","rem","viewfilters","filter","info","set","sets","id","status"]) & filters.private & ~ filters.me)
async def note(bot, update):
    buttons = [[
        InlineKeyboardButton('🏡 𝕄𝕒𝕚𝕟 𝕔𝕙𝕒𝕟𝕟𝕖𝕝', url="https://t.me/CinemaHaunter"),
        InlineKeyboardButton('📽 𝔾𝕣𝕠𝕦𝕡', url="https://t.me/Cinema_Haunter")
    ],[
        InlineKeyboardButton('🤔 ℍ𝕠𝕨 𝕥𝕠 ℝ𝕖𝕢?', url="https://t.me/aska2zmovies/4")
    ],[
        InlineKeyboardButton('𝕊𝕙𝕒𝕣𝕖 𝕋𝕠 𝕐𝕠𝕦𝕣 𝔽𝕣𝕚𝕖𝕟𝕕𝕤 😍', url="https://t.me/share/url?url=https://t.me/Cinema_Haunter")

    ]]

    reply_markup = InlineKeyboardMarkup(buttons)

    if update.from_user.id not in ADMINS:
        await bot.send_message(
            chat_id=update.chat.id,
            text=Translation.REQ_IN_PM,
            reply_markup=reply_markup,
            parse_mode="html",
            reply_to_message_id=update.message_id
        )


from typing import List

class Solution:
    def search(self, nums:List[int], target:int) -> int:
        #set the left and right boundaries
        left = 0
        right = len(nums) - 1

        #under this condition
        while left <= right:

            #Get the middle index and the middle value.
            mid = (left + right) // 2
            #case1: return the middle index.
            if nums[mid] == target:
                return mid
            #case2: discard the smaller half
            elif nums[mid] < target:
                left = mid + 1

             #case3: discard the larger half
            else:
                right = mid -  1

        #If we finish the search without , return -1
        return -1

s = Solution()
n = int(input())
arr = [0]* n
for i in arr:
    arr[i] = int(input())
target = int(input())
print(s.search(arr, target))


"""
Generating word Embeddings:
    Techiniques :
        1. Word2Vec ( Mikolov Google 2013)
        2. GloVe ( Global Vectors for word representation) Stanford 2014
    SemEval 2017 Competition:
        tasks : Semantic comparision (Given two sentences return score or 0 - 5
                                          based on the same meaning)
                Sentiment , humour and truth
                Parsing semantic structures

    Cross entropy: - sum( p(y actuals ) * p(y predicted))
    perplexity : 2 ** cross entropy
    A perfect model has cross entropy of 0 and perplexity of 1
"""

"""
Implementing RNN for character prediction for generating text

"""
from bs4 import BeautifulSoup

import requests

import random

"""
Downloading the data
    1. All articles in the arxiv website related to required categories
    2. Movie conversation from cornel movie data set
"""
base_path = 'http://export.arxiv.org/api/query'

Categories = [
        'Machine Learning',
        'Neural and Evolutionary Computing',
        'Optimization'
        ]

Keywords = [
        'neural',
        'network',
        'deep']

def build_url(amount, offset):
    categories = ' OR '.join('cat:' + x for x in Categories)
    keywords = ' OR '.join('all:' + x for x in Keywords)

    url = base_path
    url += '?search_query=(({}) AND ({}))'.format(categories, keywords)
    url += '&max_results={}&offset={}'.format(amount, offset)

    return url

build_url(0,0)

def get_count():
    url = build_url(0,0)
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'lxml')

    count = int(soup.find('opensearch:totalresults').string)
    print(count, 'papers found')
    return count
num_papers = get_count()
num_papers

page_size = 100

def fetch_page(amount, offset):
    url = build_url(amount,offset)

    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'lxml')

    for entry in soup.findAll('entry'):
        text = entry.find('summary').text
        text = text.strip().replace('\n',' ')

        yield text

def fetch_all():
    for offset in range(0, num_papers, page_size):
        print('Fetch papers {}/{}'.format(offset+ page_size, num_papers ))

        for page in fetch_page(page_size, offset):
            yield page

download_filename = 'arxiv_abstracts.txt'

def download_data():
    if not os.path.exists(download_filename):
        with open(download_filename, 'w') as file:
            for abstract in fetch_all():
                file.write(abstract +'\n')

    with open(download_filename) as file:
        data = file.readlines()

    return data

data = download_data()

len(data)

"""
Optional:
    Adding cornel movie conversational data

"""

with open('movie_lines.txt') as file:
    movie_lines_raw = file.readlines()

len(movie_lines_raw)

lines = [line.split(" +++$+++ ")[-1].replace('\n',"").lower() for line in movie_lines_raw]

"""
Making a paragaph for every 10 conversations:
"""
def converting_lines_paragraph(lines, num_lines_in_paragraph):
    paragraphs = []

    for i in range(int(len(lines)/ int(num_lines_in_paragraph)) -1):
        temp = ''
        for j in range(int(i*20) , int( (i+1)*20 )):
            temp += lines[j]
        paragraphs.append(temp)
    return paragraphs

paragraphs = converting_lines_paragraph(lines, 20)
"""
adding paragraphs to the data
"""

data = data + paragraphs
data[0]

############################################################################
"""
Data preprocessing:
"""

max_sequence_len = 50
batch_size = 100

vocabulary = " $%^&*(){}'""/?.>,<-_+|1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
len(vocabulary)
lookup ={x: i for i, x in enumerate(vocabulary)}

def one_hot(batch, sequence_len = max_sequence_len):
    one_hot_batch = np.zeros( (len(batch), sequence_len,len(vocabulary)))

    for index, line in enumerate(batch):
        line = [x for x in line if x in lookup]
        assert 2 <= len(line) <= max_sequence_len

        for offset, character in enumerate(line):
            code = lookup[character]
            one_hot_batch[index, offset, code] = 1

    return one_hot_batch

def next_batch():

    windows = []

    for line in data:
        for i in range(0, len(line) - max_sequence_len +1, max_sequence_len //2):
            windows.append(line[i: i + max_sequence_len])

    assert all( len(x) == len(windows[0]) for x in windows )

    # print("Number of windows", len(windows))
    # print("length of one windows", len(windows[0]))

    while True:
        random.shuffle(windows)
        for i in range(0 , len(windows), batch_size):
            batch = windows[i: i + batch_size]

            yield one_hot(batch)

test_batch = None
for batch in next_batch():
    test_batch = batch
    print(batch.shape)
    break

tf.reset_default_graph()

sequence = tf.placeholder( tf.float32, [None, max_sequence_len, len(vocabulary)])

x = tf.slice(sequence, (0,0,0), (-1, max_sequence_len -1, -1))

y = tf.slice( sequence, (0,1,0), (-1,-1,-1))

x.shape

y.shape

def get_mask(target):
    mask = tf.reduce_max(tf.abs(target), reduction_indices = 2)
    return mask

def get_sequence_len(target):
    mask = get_mask(target)
    sequence_len = tf.reduce_sum(mask, reduction_indices = 1)

    return sequence_len

num_neurons = 300
cell_layers = 2

num_steps = max_sequence_len  - 1
num_classes = len(vocabulary)

sequence_len = get_sequence_len(y)

sequence_len

###############################################################################

def build_rnn(data, num_steps, sequence_len, initial = None):

    multi_cell = tf.nn.rnn_cell.MultiRNNCell(
            [tf.nn.rnn_cell.GRUCell(num_neurons) for _ in range(cell_layers)])

    output, state = tf.nn.dynamic_rnn(
            inputs = data,
            cell = multi_cell,
            dtype = tf.float32,
            initial_state = initial,
            sequence_length = sequence_len)

    weight = tf.Variable( tf.truncated_normal([num_neurons, num_classes],
                                              stddev = 0.01))
    bias = tf.Variable(tf.constant(0.1, shape = [num_classes]))

    flattened_output = tf.reshape(output, [-1, num_neurons])
    prediction = tf.nn.softmax(tf.matmul(flattened_output, weight) + bias)

    prediction= tf.reshape( prediction, [-1, num_steps, num_classes])

    return prediction , state

prediction, _ = build_rnn( x, num_steps, sequence_len)

mask =  get_mask(y)

prediction = tf.clip_by_value( prediction, 1e-10, 1.0)

cross_entropy = y * tf.log(prediction)
cross_entropy = -tf.reduce_sum(cross_entropy, reduction_indices = 2)
cross_entropy += mask

length = tf.reduce_sum( sequence_len,0)

cross_entropy = tf.reduce_sum( cross_entropy, reduction_indices=1)/length
cross_entropy = tf.reduce_mean(cross_entropy)

logprob = tf.multiply( prediction, y)

logprob = tf.reduce_max(logprob, reduction_indices= 2)

logprob = tf.log(tf.clip_by_value(logprob, 1e-10,1.0 ))/tf.log(2.0)

logprob += mask
length = tf.reduce_sum(sequence_len, 0)
logprob = tf.reduce_sum( logprob, reduction_indices= 1) /length

logprob = tf.reduce_mean(logprob)

optimizer = tf.train.RMSPropOptimizer(0.002)

gradient = optimizer.compute_gradients(cross_entropy)

optimize = optimizer.apply_gradients(gradient)

num_epochs = 10
epoch_size = 50

logprob_evals = []

checkpoint_dir =    'new_sample_checkpoint_output'

"""
Training the model:
"""

with tf.Session() as ses:

    saver = tf.train.Saver()
    ses.run(tf.global_variables_initializer())

    for epoch in range( num_epochs):
        for _ in range(epoch_size):

            batch = next(next_batch())

            logprob_eval, _ = ses.run( (logprob, optimize), {sequence: batch})

            logprob_evals.append( logprob_eval)
        saver.save(ses, os.path.join(checkpoint_dir, 'char_pred'), epoch)

        perplexity = 2 ** -(sum(logprob_evals[-epoch_size:])/  epoch_size )

        print('Epoch {:2d} perplexity {:5.4f}'.format(epoch, perplexity))

    print("training completed!")

###############################################################################
"""
RNN for prediction
Checking the model
"""
seq_len = 2

num_neurons = 300

cell_layers = 2

def build_rnn(data, num_steps, sequence_len, initial = None):

    multi_cell = tf.nn.rnn_cell.MultiRNNCell(
            [tf.nn.rnn_cell.GRUCell(num_neurons) for _ in range(cell_layers)])

    output, state = tf.nn.dynamic_rnn(
            inputs = data,
            cell = multi_cell,
            dtype = tf.float32,
            initial_state = initial,
            sequence_length = sequence_len)

    weight = tf.Variable( tf.truncated_normal([num_neurons, num_classes],
                                              stddev = 0.01))
    bias = tf.Variable(tf.constant(0.1, shape = [num_classes]))

    flattened_output = tf.reshape(output, [-1, num_neurons])
    prediction = tf.nn.softmax(tf.matmul(flattened_output, weight) + bias)

    prediction= tf.reshape( prediction, [-1, num_steps, num_classes])

    return prediction , state

tf.reset_default_graph()

sequence = tf.placeholder( tf.float32, [1, seq_len, len(vocabulary)])

x = tf.slice(sequence, (0,0,0), (-1, seq_len -1, -1))

y = tf.slice( sequence, (0,1,0), (-1,-1,-1))

state1 = tf.placeholder(tf.float32, [1, num_neurons])

state2 = tf.placeholder(tf.float32,[1, num_neurons])

state1

state2

sequence_length = get_sequence_len(y)

prediction, output = build_rnn(x, num_steps = seq_len -1 ,
                               sequence_len = sequence_length,
                               initial = (state1, state2))

ses = tf.Session()

checkpoint = tf.train.get_checkpoint_state(checkpoint_dir)

checkpoint.model_checkpoint_path

if checkpoint and checkpoint.model_checkpoint_path:
    tf.train.Saver().restore(ses, checkpoint.model_checkpoint_path)

gen_seed ="We"
gen_len = 200

curr_state1 = np.zeros( (1, num_neurons))
curr_state2 = np.zeros((1, num_neurons))

gen_text = gen_seed

sampling_temperature = 0.1

def sample(dist):
    dist = np.log(dist)/sampling_temperature
    dist = np.exp(dist) / np.exp(dist).sum()

    choice = np.random.choice(len(dist), p = dist)
    choice = vocabulary[choice]

    return choice

for _ in range(gen_len):
    feed = {
            state1: curr_state1,
            state2: curr_state2,
            sequence: one_hot([gen_text[-1] + '?' ], sequence_len = seq_len)
            }

    gen_prediction_eval, (curr_state1, curr_state2) = ses.run([prediction, output],
                         feed)

    gen_text  += sample(gen_prediction_eval[0, 0])

"""
Output from the model:
"""
gen_text


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('artists', '0005_auto_20160411_0257'),
        ('concerts', '0003_auto_20160515_1655'),
    ]

    operations = [
        migrations.AddField(
            model_name='concert',
            name='artists',
            field=models.ManyToManyField(related_name='concerts', to='artists.Artist'),
        ),
    ]


import os
from flask import Flask, request, jsonify
from werkzeug.utils import secure_filename

UPLOAD_FOLDER = 'C:/uploads'

app = Flask(__name__)
app.secret_key = "secret key"
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024

ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/file-upload', methods=['POST'])
def upload_file():
    if 'file' not in request.files:
        resp = jsonify({'message': 'No file part in the request'})
        resp.status_code = 400
        return resp
    file = request.files['file']
    if file.filename == '':
        resp = jsonify({'message': 'No file selected for uploading'})
        resp.status_code = 400
        return resp
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        resp = jsonify({'message': 'File successfully uploaded'})
        resp.status_code = 201
        return resp
    else:
        resp = jsonify({'message': 'Allowed file types are txt, pdf, png, jpg, jpeg, gif'})
        resp.status_code = 400
        return resp

@app.route('/file-read/<file_name>')
def read_file(file_name):
    if allowed_file(file_name):
        filename=os.path.join(app.config['UPLOAD_FOLDER'],file_name)
        file = open(filename, 'r')
        return file.read()

if __name__ == "__main__":
    app.run()


import pandas as pd
from google.cloud import bigquery
from google.cloud import storage
import db_dtypes

# Definir el esquema de la tabla Auditoria
AUDITORIA_SCHEMA = [
    bigquery.SchemaField('IdAuditoria', 'INTEGER', mode='REQUIRED'),
    bigquery.SchemaField('NroEjecucion', 'INTEGER', mode='REQUIRED'),
    bigquery.SchemaField('Entorno', 'STRING', mode='NULLABLE'),
    bigquery.SchemaField('Proceso', 'STRING', mode='NULLABLE'),
    bigquery.SchemaField('Tabla', 'STRING', mode='NULLABLE'),
    bigquery.SchemaField('RegLeidos', 'INTEGER', mode='NULLABLE'),
    bigquery.SchemaField('RegInsertados', 'INTEGER', mode='NULLABLE'),
    bigquery.SchemaField('RegActualizados', 'INTEGER', mode='NULLABLE'),
    bigquery.SchemaField('RegEliminados', 'INTEGER', mode='NULLABLE'),
]

def obtener_maximo_id_auditoria(bq_client):
    """
    Obtiene el máximo ID de auditoria de la tabla de Auditoria en BigQuery.

    :param bq_client: Cliente de BigQuery.
    :return: El máximo ID de auditoria.
    """
    try:
        query = 'SELECT MAX(IdAuditoria) AS max_id FROM `Modelo_Esperanza_De_Vida.Auditoria`'
        query_job = bq_client.query(query)
        results = query_job.result()
        row = next(results)
        max_id = row['max_id']

        return max_id if max_id is not None else 0

    except Exception as e:
        raise Exception(f"Error al obtener el máximo ID de auditoría: {e}")

def obtener_maximo_nro_ejecucion(bq_client):
    """
    Obtiene el máximo NroEjecucion de la tabla de Auditoria en BigQuery.

    :param bq_client: Cliente de BigQuery.
    :return: El máximo NroEjecucion.
    """
    try:
        query = 'SELECT MAX(NroEjecucion) AS max_nro FROM `possible-willow-403216.Modelo_Esperanza_De_Vida.Auditoria`'
        query_job = bq_client.query(query)
        results = query_job.result()
        row = next(results)
        max_nro = row['max_nro']

        return max_nro

    except Exception as e:
        raise Exception(f"Error al obtener el máximo NroEjecucion: {e}")

def registrar_auditoria_generar_data_ml(bq_client, registros_leidos):
    """
    Registra en la tabla de Auditoria después de generar datos para ML.

    :param bq_client: Cliente de BigQuery.
    :param registros_leidos: Número de registros leídos.
    """
    try:
        # Obtener el máximo IdAuditoria y NroEjecucion
        max_id_auditoria = obtener_maximo_id_auditoria(bq_client)
        max_nro_ejecucion = obtener_maximo_nro_ejecucion(bq_client)

        # Datos para la auditoría
        auditoria_data = {
            'IdAuditoria': max_id_auditoria + 1,
            'NroEjecucion': max_nro_ejecucion,
            'Entorno': 'Dag-Airflow-Composer',
            'Proceso': 'generar_data_ml',
            'Tabla': None,
            'RegLeidos': registros_leidos,
            'RegInsertados': 0,
            'RegActualizados': 0,
            'RegEliminados': 0,
        }

        # Convertir el diccionario a un DataFrame
        df = pd.DataFrame([auditoria_data])

        # Insertar en la tabla de Auditoria
        dataset_ref = bq_client.dataset("Modelo_Esperanza_De_Vida")
        table_ref = dataset_ref.table("Auditoria")

        job_config = bigquery.LoadJobConfig(
            schema=AUDITORIA_SCHEMA,
            write_disposition='WRITE_APPEND',
        )

        job = bq_client.load_table_from_dataframe(df, table_ref, job_config=job_config)
        job.result()

        #Retorno la cantidad de Registros Insertados
        return df.shape[0]

    except Exception as e:
        raise Exception(f"Error al insertar en tabla Auditoria después de generar datos para ML: {e}")

def generar_data_ML(request):
    """
    Genera datos para ML a partir de una consulta en BigQuery y guarda el resultado en un archivo CSV.

    :param request: Datos de la solicitud (no utilizado en esta función).
    :return: Mensaje indicando si la generación fue exitosa.
    """
    try:
        # Configurar el cliente de BigQuery
        bq_client = bigquery.Client()

        # Ejecutar la consulta en BigQuery y convertir el resultado a un DataFrame de pandas
        df_result = bq_client.query('''
            SELECT p.Pais, c.Continente, di.Anio, di.Valor, i.Descripcion as Indicador
            FROM `possible-willow-403216.Modelo_Esperanza_De_Vida.DatosIndicador` as di
            JOIN `possible-willow-403216.Modelo_Esperanza_De_Vida.Pais`           as p
              ON di.IdPais = p.IdPais
            JOIN `possible-willow-403216.Modelo_Esperanza_De_Vida.Continente`     as c
              ON p.IdContinente = c.IdContinente
            JOIN `possible-willow-403216.Modelo_Esperanza_De_Vida.Indicador`      as i
              ON i.IdIndicador = di.IdIndicador
        ''').to_dataframe()

        # Verificar si el DataFrame tiene datos
        if df_result.empty:
            raise Exception("La consulta no devolvió datos.")

        # Exportar el DataFrame a un archivo CSV
        bucket_name = 'pf-henry-esperanza-mlops'
        csv_file_name = 'Data-ML.csv'
        df_result.to_csv(f'/tmp/{csv_file_name}', index=False, encoding='utf-8', sep=';')

        # Subir el archivo CSV al bucket de Google Cloud Storage
        storage_client = storage.Client()
        bucket = storage_client.bucket(bucket_name)
        blob = bucket.blob(csv_file_name)
        blob.upload_from_filename(f'/tmp/{csv_file_name}')

        # Verificar errores y retornar un mensaje de éxito
        if blob.exists():

            # Grabar Registro de Auditoria
            registros_auditoria = registrar_auditoria_generar_data_ml(bq_client, len(df_result))

            # Mostrar estadísticas
            print("*--------------------------------------------*")
            print("*                                            *")
            print("*          Estadísticas de Ejecución         *")
            print("*                                            *")
            print("*--------------------------------------------*")
            print("*")
            print(f' Total de Registros Leídos    : {len(df_result)}')
            print("*")
            print(f" Registros de Auditoria Generados: {registros_auditoria}")
            print("*")
            return 'Archivo Data-ML.csv generado con éxito y guardado en el bucket pf-henry-esperanza-mlops.'
        else:
            return 'Error al subir el archivo CSV al bucket pf-henry-esperanza-mlops.'

    except Exception as e:
        print(f"Error en la generación de datos para ML: {str(e)}")
        return 'Error en la generación de datos para ML.'


#!/usr/bin/python3 -u

from dataclasses import dataclass
from enum import IntEnum
import os
import signal
import sys
from typing import Optional
import warnings

import osc_server
import threading
import time

import jacklib
from jacklib.helpers import c_char_p_p_to_list
from jacklib.enums import JackOptions, JackPortFlags, JackMetadata

# check ALSA LIB
try:
    from pyalsa.alsaseq import SEQ_LIB_VERSION_STR
    ALSA_VERSION_LIST = [int(num) for num in SEQ_LIB_VERSION_STR.split('.')]
    assert ALSA_VERSION_LIST >= [1, 2, 4]
    ALSA_LIB_OK = True
    from alsa_manager import AlsaManager
except:
    ALSA_LIB_OK = False

PORT_TYPE_NULL = 0
PORT_TYPE_AUDIO = 1
PORT_TYPE_MIDI = 2

EXISTENCE_PATH = '/tmp/RaySession/patchbay_daemons/'

# Define a context manager to suppress stdout and stderr.
class suppress_stdout_stderr(object):
    '''
    A context manager for doing a "deep suppression" of stdout and stderr in
    Python, i.e. will suppress all print, even if the print originates in a
    compiled C/Fortran sub-function.
       This will not suppress raised exceptions, since exceptions are printed
    to stderr just before a script exits, and after the context manager has
    exited (at least, I think that is why it lets exceptions through).

    '''
    def __init__(self):
        # Open a pair of null files
        self.null_fds =  [os.open(os.devnull,os.O_RDWR) for x in range(2)]
        # Save the actual stdout (1) and stderr (2) file descriptors.
        self.save_fds = [os.dup(1), os.dup(2)]

    def __enter__(self):
        # Assign the null pointers to stdout and stderr.
        os.dup2(self.null_fds[0],1)
        os.dup2(self.null_fds[1],2)

    def __exit__(self, *_):
        # Re-assign the real stdout/stderr back to (1) and (2)
        os.dup2(self.save_fds[0],1)
        os.dup2(self.save_fds[1],2)
        # Close all file descriptors
        for fd in self.null_fds + self.save_fds:
            os.close(fd)

@dataclass
class TransportPosition:
    frame: int
    rolling: bool
    valid_bbt: bool
    bar: int
    beat: int
    tick: int
    beats_per_minutes: float

class TransportWanted(IntEnum):
    NO = 0         # do not send any transport info
    STATE_ONLY = 1 # send info only when play/pause changed
    FULL = 2       # send all Transport infos

class JackPort:
    id = 0
    name = ''
    type = PORT_TYPE_NULL
    flags = 0
    alias_1 = ''
    alias_2 = ''
    order = None
    uuid = 0

    def __init__(self, port_name:str, jack_client, port_ptr=None):
        # In some cases, port could has just been renamed
        # then, jacklib.port_by_name() fail.
        # that is why, port_ptr can be sent as argument here
        self.name = port_name
        if port_ptr is None:
            port_ptr = jacklib.port_by_name(jack_client, port_name)
        self.flags = jacklib.port_flags(port_ptr)
        self.uuid = jacklib.port_uuid(port_ptr)

        port_type_str = jacklib.port_type(port_ptr)
        if port_type_str == jacklib.JACK_DEFAULT_AUDIO_TYPE:
            self.type = PORT_TYPE_AUDIO
        elif port_type_str == jacklib.JACK_DEFAULT_MIDI_TYPE:
            self.type = PORT_TYPE_MIDI

        order_prop = jacklib.get_property(
            self.uuid, JackMetadata.ORDER)

        ret, alias_1, alias_2 = jacklib.port_get_aliases(port_ptr)
        if ret:
            self.alias_1 = alias_1
            self.alias_2 = alias_2

    def __lt__(self, other: 'JackPort'):
        return self.uuid < other.uuid

class MainObject:
    port_list = list[JackPort]()
    connection_list = list[tuple[str]]()
    metadata_list = list[dict]()
    client_list = list[dict]()
    client_names_queue = list[str]()
    jack_running = False
    osc_server = None
    alsa_mng: Optional['AlsaManager'] = None
    terminate = False
    jack_client = None
    samplerate = 48000
    buffer_size = 1024

    dsp_wanted = True
    transport_wanted = TransportWanted.FULL

    def __init__(self, daemon_port: str, gui_url: str):
        self._daemon_port = daemon_port
        self.ALSA_LIB_OK = ALSA_LIB_OK
        self.last_sent_dsp_load = 0
        self.max_dsp_since_last_sent = 0.00
        self._waiting_jack_client_open = True
        self.last_transport_pos = TransportPosition(0, False, False, 0, 0, 0, 0.0)

        self.osc_server = osc_server.OscJackPatch(self)
        self.osc_server.set_tmp_gui_url(gui_url)
        self.write_existence_file()
        self.start_jack_client()

        if ALSA_LIB_OK:
            self.alsa_mng = AlsaManager(self)
            self.alsa_mng.add_all_ports()

    @staticmethod
    def get_metadata_value_str(prop: jacklib.Property) -> str:
        value = prop.value
        if isinstance(value, bytes):
            return value.decode()
        elif isinstance(value, str):
            return value
        else:
            try:
                value = str(value)
            except:
                return ''
        return value

    def write_existence_file(self):
        if not os.path.isdir(EXISTENCE_PATH):
            os.makedirs(EXISTENCE_PATH)

        try:
            file = open(EXISTENCE_PATH + self._daemon_port, 'w')
        except PermissionError:
            sys.stderr.write(
                'ray-patchbay_to_osc: Error, no permission for existence file\n')
            sys.exit(1)

        contents = 'pid:%i\n' % os.getpid()
        contents += 'port:%i\n' % self.osc_server.port

        file.write(contents)
        file.close()

    def remove_existence_file(self):
        if not os.path.exists(EXISTENCE_PATH + self._daemon_port):
            return

        try:
            os.remove(EXISTENCE_PATH + self._daemon_port)
        except PermissionError:
            sys.stderr.write(
                'ray-patchbay_to_osc: Error, unable to remove %s\n'
                % EXISTENCE_PATH + self._daemon_port)

    @classmethod
    def signal_handler(cls, sig: int, frame):
        if sig in (signal.SIGINT, signal.SIGTERM):
            cls.terminate = True

    def eat_client_names_queue(self):
        while self.client_names_queue:
            client_name = self.client_names_queue.pop(0)
            b_uuid = jacklib.get_uuid_for_client_name(self.jack_client, client_name)

            # convert bytes uuid to int
            uuid = 0
            if isinstance(b_uuid, bytes):
                str_uuid = b_uuid.decode()
                if str_uuid.isdigit():
                    uuid = int(str_uuid)

            if not uuid:
                continue

            for client_dict in self.client_list:
                if client_dict['name'] == client_name:
                    client_dict['uuid'] = uuid
                    break
            else:
                self.client_list.append({'name': client_name, 'uuid': uuid})

    def add_gui(self, gui_url: str):
        self.osc_server.add_gui(gui_url)

    def check_jack_client_responding(self):
        for i in range(100): # JACK has 5s to answer
            time.sleep(0.050)

            if not self._waiting_jack_client_open:
                break
        else:
            # server never answer
            self.osc_server.send_server_lose()
            self.remove_existence_file()

            # JACK is not responding at all
            # probably it is started but totally bugged
            # finally kill this program from system
            os.kill(os.getpid(), signal.SIGKILL)

    def refresh(self):
        if self.jack_running:
            self.get_all_ports_and_connections()
            self.osc_server.server_restarted()

        if self.alsa_mng is not None:
            self.alsa_mng.add_all_ports()

    def remember_dsp_load(self):
        self.max_dsp_since_last_sent = max(
            self.max_dsp_since_last_sent,
            jacklib.cpu_load(self.jack_client))

    def send_dsp_load(self):
        current_dsp = int(self.max_dsp_since_last_sent + 0.5)
        if current_dsp != self.last_sent_dsp_load:
            self.osc_server.send_dsp_load(current_dsp)
            self.last_sent_dsp_load = current_dsp
        self.max_dsp_since_last_sent = 0.00

    def set_transport_wanted(self, transport_wanted: int):
        try:
            self.transport_wanted = TransportWanted(transport_wanted)
        except:
            self.transport_wanted = TransportWanted.FULL

    def _send_transport_pos(self):
        if not self.jack_running:
            return

        pos = jacklib.jack_position_t()
        pos.valid = 0

        state = jacklib.transport_query(self.jack_client, jacklib.pointer(pos))

        if (self.transport_wanted is TransportWanted.STATE_ONLY
                and bool(state) == self.last_transport_pos.rolling):
            return

        transport_position = TransportPosition(
            int(pos.frame),
            bool(state),
            bool(pos.valid & jacklib.JackPositionBits.POSITION_BBT),
            int(pos.bar),
            int(pos.beat),
            int(pos.tick),
            float(pos.beats_per_minute))

        if transport_position == self.last_transport_pos:
            return

        self.last_transport_pos = transport_position
        self.osc_server.send_transport_position(transport_position)

    def connect_ports(self, port_out_name: str, port_in_name: str,
                      disconnect=False):
        if self.alsa_mng is not None and port_out_name.startswith(':ALSA_OUT:'):
            self.alsa_mng.connect_ports(
                port_out_name, port_in_name, disconnect=disconnect)
            return

        if disconnect:
            jacklib.disconnect(self.jack_client, port_out_name, port_in_name)
        else:
            jacklib.connect(self.jack_client, port_out_name, port_in_name)

    def start_loop(self):
        n = 0

        while True:
            self.osc_server.recv(50)

            if self.is_terminate():
                break

            if self.jack_running:
                if n % 4 == 0:
                    self.remember_dsp_load()
                    if self.dsp_wanted and n % 20 == 0:
                        self.send_dsp_load()

                self.eat_client_names_queue()
                if self.transport_wanted is not TransportWanted.NO:
                    self._send_transport_pos()

            else:
                if n % 10 == 0:
                    self.start_jack_client()
            n += 1

            # for faster modulos
            if n == 20:
                n = 0

    def exit(self):
        if self.jack_running:
            jacklib.deactivate(self.jack_client)
            jacklib.client_close(self.jack_client)

        if self.alsa_mng is not None:
            self.alsa_mng.stop_events_loop()
            del self.alsa_mng

        self.remove_existence_file()
        del self.osc_server

    def start_jack_client(self):
        self._waiting_jack_client_open = True

        # Sometimes JACK never registers the client
        # and never answers. This thread will allow to exit
        # if JACK didn't answer 5 seconds after register ask
        jack_waiter_thread = threading.Thread(
            target=self.check_jack_client_responding)
        jack_waiter_thread.start()

        with suppress_stdout_stderr():
            self.jack_client = jacklib.client_open(
                "ray-patch_to_osc",
                JackOptions.NO_START_SERVER | JackOptions.SESSION_ID,
                None)

        self._waiting_jack_client_open = False
        ## Some problems happens to the JACK server sometimes without it
        #time.sleep(0.030)

        jack_waiter_thread.join()

        if self.jack_client:
            self.jack_running = True
            self.set_registrations()
            self.get_all_ports_and_connections()
            self.samplerate = jacklib.get_sample_rate(self.jack_client)
            self.buffer_size = jacklib.get_buffer_size(self.jack_client)
            self.osc_server.server_restarted()
        else:
            self.jack_running = False

    def is_terminate(self)->bool:
        if self.terminate or self.osc_server.is_terminate():
            return True

        return False

    def set_registrations(self):
        if not self.jack_client:
            return

        jacklib.set_client_registration_callback(
            self.jack_client, self.jack_client_registration_callback, None)
        jacklib.set_port_registration_callback(
            self.jack_client, self.jack_port_registration_callback, None)
        jacklib.set_port_connect_callback(
            self.jack_client, self.jack_port_connect_callback, None)
        jacklib.set_port_rename_callback(
            self.jack_client, self.jack_port_rename_callback, None)
        jacklib.set_xrun_callback(
            self.jack_client, self.jack_xrun_callback, None)
        jacklib.set_buffer_size_callback(
            self.jack_client, self.jack_buffer_size_callback, None)
        jacklib.set_sample_rate_callback(
            self.jack_client, self.jack_sample_rate_callback, None)
        jacklib.set_property_change_callback(
            self.jack_client, self.jack_properties_change_callback, None)
        jacklib.on_shutdown(
            self.jack_client, self.jack_shutdown_callback, None)
        jacklib.activate(self.jack_client)

    def get_all_ports_and_connections(self):
        self.port_list.clear()
        self.connection_list.clear()
        self.metadata_list.clear()

        #get all currents Jack ports and connections
        port_name_list = c_char_p_p_to_list(
            jacklib.get_ports(self.jack_client, "", "", 0))

        client_names = []

        for port_name in port_name_list:
            port_ptr = jacklib.port_by_name(self.jack_client, port_name)
            jport = JackPort(port_name, self.jack_client)
            self.port_list.append(jport)

            client_name = port_name.partition(':')[0]
            if not client_name in client_names:
                client_names.append(client_name)

            # get port metadatas
            for key in (JackMetadata.CONNECTED,
                        JackMetadata.ORDER,
                        JackMetadata.PORT_GROUP,
                        JackMetadata.PRETTY_NAME,
                        JackMetadata.SIGNAL_TYPE):
                prop = jacklib.get_property(jport.uuid, key)
                if prop is None:
                    continue

                value = self.get_metadata_value_str(prop)
                self.metadata_list.append(
                    {'uuid': jport.uuid,
                     'key': key,
                     'value': value})

            if jport.flags & JackPortFlags.IS_INPUT:
                continue

            port_ptr = jacklib.port_by_name(self.jack_client, jport.name)

            # this port is output, list its connections
            port_connection_names = tuple(
                jacklib.port_get_all_connections(self.jack_client, port_ptr))

            for port_con_name in port_connection_names:
                self.connection_list.append((jport.name, port_con_name))

        for client_name in client_names:
            uuid = jacklib.get_uuid_for_client_name(self.jack_client, client_name)
            if not uuid:
                continue

            self.client_list.append({'name': client_name, 'uuid': int(uuid)})

            # we only look for icon_name now, but in the future other client
            # metadatas could be enabled
            for key in (JackMetadata.ICON_NAME,):
                prop = jacklib.get_property(int(uuid), JackMetadata.ICON_NAME)
                if prop is None:
                    continue
                value = self.get_metadata_value_str(prop)
                self.metadata_list.append(
                    {'uuid': int(uuid),
                    'key': key,
                    'value': value})

    def jack_shutdown_callback(self, arg=None)->int:
        self.jack_running = False
        self.port_list.clear()
        self.connection_list.clear()
        self.osc_server.server_stopped()
        return 0

    def jack_xrun_callback(self, arg=None)->int:
        self.osc_server.send_one_xrun()
        return 0

    def jack_sample_rate_callback(self, samplerate, arg=None) -> int:
        self.samplerate = samplerate
        self.osc_server.send_samplerate()
        return 0

    def jack_buffer_size_callback(self, buffer_size, arg=None) -> int:
        self.buffer_size = buffer_size
        self.osc_server.send_buffersize()
        return 0

    def jack_client_registration_callback(self, client_name: bytes,
                                          register: int, arg=None) -> int:
        client_name = client_name.decode()
        self.client_names_queue.append(client_name)
        return 0

    def jack_port_registration_callback(self, port_id: int, register: bool,
                                        arg=None) -> int:
        if not self.jack_client:
            return 0

        port_ptr = jacklib.port_by_id(self.jack_client, port_id)
        port_name = jacklib.port_name(port_ptr)

        if register:
            jport = JackPort(port_name, self.jack_client, port_ptr)
            self.port_list.append(jport)
            self.osc_server.port_added(
                jport.name, jport.type, jport.flags, jport.uuid)
        else:
            for jport in self.port_list:
                if jport.name == port_name:
                    self.port_list.remove(jport)
                    self.osc_server.port_removed(jport.name)
                    break
        return 0

    def jack_port_rename_callback(self, port_id: int, old_name: bytes,
                                  new_name: bytes, arg=None)->int:
        for jport in self.port_list:
            if jport.name == str(old_name.decode()):
                ex_name = jport.name
                jport.name = str(new_name.decode())
                self.osc_server.port_renamed(ex_name, jport.name)
                break
        return 0

    def jack_port_connect_callback(self, port_id_A: int, port_id_B: int,
                                   connect_yesno: bool, arg=None)->int:
        port_ptr_A = jacklib.port_by_id(self.jack_client, port_id_A)
        port_ptr_B = jacklib.port_by_id(self.jack_client, port_id_B)

        port_str_A = jacklib.port_name(port_ptr_A)
        port_str_B = jacklib.port_name(port_ptr_B)

        connection = (port_str_A, port_str_B)

        if connect_yesno:
            self.connection_list.append(connection)
            self.osc_server.connection_added(connection)
        elif connection in self.connection_list:
            self.connection_list.remove(connection)
            self.osc_server.connection_removed(connection)

        return 0

    def jack_properties_change_callback(self, uuid: int, name: bytes,
                                        type_: int, arg=None)->int:
        if name is not None:
            name = name.decode()

        value = ''

        if name and type_ != jacklib.JackPropertyChange.DELETED:
            prop = jacklib.get_property(uuid, name)
            if prop is None:
                return 0

            value = self.get_metadata_value_str(prop)

        for metadata in self.metadata_list:
            if metadata['uuid'] == uuid and metadata['key'] == name:
                metadata['value'] = value
                break
        else:
            self.metadata_list.append(
                {'uuid': uuid, 'key': name, 'value': value})

        self.osc_server.metadata_updated(uuid, name, value)

        return 0

    def set_buffer_size(self, buffer_size: int):
        jacklib.set_buffer_size(self.jack_client, buffer_size)

    def set_metadata(self, uuid: int, key: str, value: str):
        jacklib.set_property(uuid, key, value, 'text/plain', jacklib.ENCODING)

    def transport_play(self, play: bool):
        if play:
            jacklib.transport_start(self.jack_client)
        else:
            jacklib.transport_stop(self.jack_client)

    def transport_stop(self):
        jacklib.transport_stop(self.jack_client)
        jacklib.transport_locate(self.jack_client, 0)

    def transport_relocate(self, frame: int):
        jacklib.transport_locate(self.jack_client, frame)

def main_process():
    args = sys.argv.copy()
    daemon_port = ''
    gui_url = ''

    if args:
        this_exec = args.pop(0)

    if args:
        daemon_port = args.pop(0)

    if args:
        gui_url = args[0]

    main_object = MainObject(daemon_port, gui_url)

    for gui_url in args:
        main_object.add_gui(gui_url)

    main_object.start_loop()
    main_object.exit()

if __name__ == '__main__':
    # prevent deprecation warnings python messages
    warnings.filterwarnings("ignore", category=DeprecationWarning)

    signal.signal(signal.SIGINT, MainObject.signal_handler)
    signal.signal(signal.SIGTERM, MainObject.signal_handler)

    main_process()


import dataclasses
from typing import List

from fastapi import APIRouter, HTTPException

from common.dependencies import DependencyWrapper
from common.router_dtos import (
    DownloadDTOIn,
    DownloadDTOOut,
    SuccessResponse,
    DownloadStatusDTO,
)
from downloads import (
    GetDownloadStatus,
    FileDownload,
    FileRetry,
    GetFileStatus,
    GetBulkFileStatus,
    FileRetryAll,
)
from downloads.domain.entities import File, DownloadStatus

router = APIRouter(
    tags=["Files"],
    prefix="/files",
)

@router.get(
    "/count/",
    response_model=DownloadStatus,
    summary="Get the total files downloaded (also works as an health checker)",
)
@DependencyWrapper
def download_status(get_total_files: GetDownloadStatus):
    return get_total_files.query()

@router.post(
    "/",
    response_model=List[DownloadDTOOut],
    summary="Create and schedule a new download",
    status_code=201,
)
@DependencyWrapper
def file_download(files: List[DownloadDTOIn], file_download_uc: FileDownload):
    response = []
    for file in files:
        _result = file_download_uc.execute(
            name=file.name,
            path=file.path,
            url=file.url,
            headers=file.headers,
        )

        if _result:
            response.append(File.to_external(_result))

    return response

@router.get(
    "/{hash}/",
    response_model=DownloadDTOOut,
    responses={404: {"description": "Not found"}},
    summary="Get download status for a single file",
)
@DependencyWrapper
def file_status(hash: str, get_file_status: GetFileStatus):
    _result = get_file_status.query(
        hash=hash,
    )

    if not _result:
        raise HTTPException(404, "Item not found")

    return dataclasses.asdict(_result)

@router.post(
    "/{hash}/retry/",
    response_model=SuccessResponse,
    responses={404: {"description": "Not found"}},
    summary="Retry a single failed download",
    status_code=202,
)
@DependencyWrapper
def file_retry(hash: str, file_retry_uc: FileRetry):
    _result = file_retry_uc.execute(
        hash=hash,
    )

    if not _result:
        raise HTTPException(404, "Item not found")

    return dict(success=True)

@router.post(
    "/retry/",
    response_model=SuccessResponse,
    summary="Retry all failed downloads",
    status_code=202,
)
@DependencyWrapper
def file_retry_all(file_retry_all_uc: FileRetryAll):
    file_retry_all_uc.execute()

    return dict(success=True)

@router.post(
    "/status/",
    response_model=DownloadStatusDTO,
    summary="Get bulk download status for a given list of files",
)
@DependencyWrapper
def file_status_bulk(files: List[str], get_bulk_file_status: GetBulkFileStatus):
    return dict(
        files={
            file.hash: dataclasses.asdict(file)
            for file in get_bulk_file_status.query(
                hash_list=files,
            )
        }
    )


#!/usr/bin/env python
# -*- coding: utf-8 -*-

try:
    from .tools import *
except:
    from tools import *

from itertools import chain

from lxml import html
import requests

SITE_URL = "http://www.narodne-pesmi.si/?lang=&option=slovenska_imena&stran=Zakladi%20Slovenije"

content = requests.get(SITE_URL)
content.encoding="utf-8"
tree = html.fromstring(re.sub("\s\s+", " ", content.text.strip()))

if sex() is "m":
    for name in clean([x.text for x in tree.xpath("//div[contains(@style,'float:left')]//div[@id='leftcontentslimline_right']")]):
        print(name)
else:
    for name in clean([x.text for x in tree.xpath("//div[contains(@style,'float:right')]//div[@id='leftcontentslimline_right']")]):
        print(name)


#!/usr/bin/python
import subprocess as sub
import time
import psutil
from subprocess import Popen, PIPE
import sys
PROCNAME = "tcpdump"
name=sys.argv[1]
timea=sys.argv[2]
user=sys.argv[3]
ch=user+'/'+name
p = sub.Popen(('sudo', 'tcpdump','-i','enp0s8','-n','udp and src host 192.168.3.1','-w','/home/'+ch),stdout=sub.PIPE)
time.sleep(float(timea))
p.stdout.close()
for proc in psutil.process_iter():
    # check whether the process name matches
    if proc.name() == PROCNAME:
        proc.kill()
print("cccc")


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import datetime
import re

from parsers import Message, MediaParser

def can_handle(url):
    return re.match(r"^https?://(www\.)?govoritmoskva\.ru/broadcasts/", url, re.IGNORECASE)

def get_parser(url, args):
    return GovorimoskvaRuParser(url)

RUS_DOW = {
    'Mon': 'Пн',
    'Tue': 'Вт',
    'Wed': 'Ср',
    'Thu': 'Чт',
    'Fri': 'Пт',
    'Sat': 'Сб',
    'Sun': 'Вс',
}

class GovorimoskvaRuParser(MediaParser):
    def get_title(self):
        page = self.parse_page(self.base_url)
        title = page.xpath(r'//div[@class="oneProgramPage"]//h1/text()')[0]
        return title.replace('(16+)', '').strip('«» ')

    def get_broadcast_url(self, month=None, year=None):
        if month is None and year is None:
            return self.base_url
        else:
            return r'%s?month=%d&year=%d' % (self.base_url, month, year)

    def episode_name(self, title, date):
        dow = date.strftime('%a')
        return '%s, %s %d' % (title, RUS_DOW.get(dow, dow), date.day)

    def get_messages(self):
        # Fetch page for current and previous month
        today = datetime.date.today()
        prev = today - datetime.timedelta(days=today.day)

        title = self.get_title()

        messages = []
        for date in (today, prev):
            url = self.get_broadcast_url(date.month, date.year)
            page = self.parse_page(url)
            messages += self.podcasts_for_month(page, date, title)
        return messages

    def podcasts_for_month(self, page, date, title):
        podcasts = []
        descr = page.xpath(r'//div[@class="textDescribe"]/p[last()]/text()')[0]
        for program in page.xpath(r'//ul[@class="programList"]/li'):
            media = program.xpath(r'.//a[@download]')
            if len(media) == 0:
                continue
            assert len(media) == 1, 'Should be only one media, by found: %d' % len(media)
            media_url = media[0].get('href')
            if not media_url.startswith('http'):
                media_url = 'http://govoritmoskva.ru/' + media_url
            size = self.get_file_size(media_url)
            if size is not None:
                day = int(program.xpath(r'.//div[@class="time"]/span/text()')[0].split(' ')[0])
                program_date = date.replace(day=day)
                podcast = Message()
                podcast.title = self.episode_name(title, program_date)
                podcast.url = media_url
                podcast.date = program_date
                podcast.text = descr
                podcast.size = size

                podcasts.append(podcast)
        return podcasts


from rest_framework.routers import DefaultRouter
from django.urls import (
    path,
    include,
)

from blog import views

router = DefaultRouter()
router.register("manage-blog", views.ManageBlogView)

app_name = "blog"

urlpatterns = [
    path("", include(router.urls)),
    path("cafes/<int:cafe_code>/blogs/", views.CafesBlogListView.as_view()),
    path("iran-cafe/", views.IranCafeBlogsView.as_view()),
    path("<str:slug>/", views.BlogDetailView.as_view()),
    path("all/blogs", views.BlogsView.as_view(), name="all_blogs"),
    path("the_latest_blogs", views.LatestBlogsView.as_view(), name="the_latest_blogs"),
]


import streamlit as st
import plotly.graph_objects as go
from connection import OpenAQConnection
from utils import *
import time

st.set_page_config(page_title="OpenAQ Connection", layout="wide")

conn = st.experimental_connection("openaq", type=OpenAQConnection)
readme = load_config("config_readme.toml")

def visualize_variable_on_map(data_dict, variable):
    is_day = is_daytime()
    mapbox_style = "carto-darkmatter" if not is_day else "open-street-map"

    # Initialize lists to store data for multiple locations
    latitudes = []
    longitudes = []
    values = []
    display_names = []
    last_updated = []

    # Loop through the results and extract relevant data for each location
    for result in data_dict.get("results", []):
        measurements = result.get("parameters", [])
        for measurement in measurements:
            if measurement["parameter"] == variable:
                value = measurement["lastValue"]
                display_name = measurement["displayName"]
                latitude = result["coordinates"]["latitude"]
                longitude = result["coordinates"]["longitude"]
                last_updated_value = result["lastUpdated"]

                latitudes.append(latitude)
                longitudes.append(longitude)
                values.append(value)
                display_names.append(display_name)
                last_updated.append(last_updated_value)

    if not latitudes or not longitudes or not values:
        print(f"{variable} data not found.")
        return create_custom_markdown_card(
            f"{variable} data not found for the selected country."
        )

    # Create the visualization
    fig = go.Figure()

    marker = [
        custom_markers["humidity"]
        if variable == "humidity"
        else custom_markers["others"]
    ]

    # Add a single scatter mapbox trace with all locations
    fig.add_trace(
        go.Scattermapbox(
            lat=latitudes,
            lon=longitudes,
            mode="markers+text",
            marker=dict(
                size=20,
                color=values,
                colorscale="Viridis",  # You can choose other color scales as well
                colorbar=dict(title=f"{variable.capitalize()}"),
            ),
            text=[
                f"{marker[0]} {display_name}: {values[i]}<br>Last Updated: {last_updated[i]}"
                for i, display_name in enumerate(display_names)
            ],
            hoverinfo="text",
        )
    )

    # Update map layout
    fig.update_layout(
        mapbox=dict(
            style=mapbox_style,  # Choose the desired map style
            zoom=5,  # Adjust the initial zoom level as needed
            center=dict(
                lat=sum(latitudes) / len(latitudes),
                lon=sum(longitudes) / len(longitudes),
            ),
        ),
        margin=dict(l=0, r=0, t=0, b=0),
    )
    create_custom_markdown_card(information)
    st.plotly_chart(fig, use_container_width=True)

# Info
st.title("Air quality data")
with st.expander("What is this app?", expanded=False):
    st.write(readme["app"]["app_intro"])
    st.write("")
st.write("")
st.sidebar.image(load_image("logo.png"), use_column_width=True)
display_links(readme["links"]["repo"], readme["links"]["other_link"])

with st.spinner("Loading the available countries..."):
    # Countries exist in first 2 pages
    countries = []
    for page in [1, 2]:
        try:
            countries_request = conn.query_countries(page=page)["results"]
            countries = countries + countries_request
        except Exception:
            countries_error = True

    transformed_countries = {
        country["name"]: {
            "code": country["code"],
            "parameters": country["parameters"],
            "locations": country["locations"],
            "lastUpdated": country["lastUpdated"],
        }
        for country in countries
    }

    # Add a global for default when the app is initialised
    transformed_countries["Global"] = {
        "code": None,
        "parameters": general_parameters,
        "locations": None,
        "lastUpdated": None,
    }

# Parameters
st.sidebar.title("Selections")
selected_country = st.sidebar.selectbox(
    "Select the desired country",
    transformed_countries,
    placeholder="Country",
    index=len(transformed_countries) - 1,  # Gets the last one "Global"
    help=readme["tooltips"]["country"],
)

selected_viariable = st.sidebar.selectbox(
    "Select the desired variable",
    transformed_countries[selected_country]["parameters"],
    placeholder="Variable",
    index=1,
    help=readme["tooltips"]["variable"],
)

radius = st.sidebar.slider(
    "Select a radius",
    min_value=100,
    max_value=25000,
    step=100,
    value=1000,
    help=readme["tooltips"]["radius"],
)

total_locations = transformed_countries[selected_country]["locations"]
last_time = transformed_countries[selected_country]["lastUpdated"]
information = f"The selected country is {selected_country}. The total found locations are {total_locations} with last updates at {last_time}."

code = transformed_countries[selected_country]["code"]
locations_response = conn.query(code, radius)
st.title("Map")
visualize_variable_on_map(locations_response, selected_viariable)