from tensorflow.keras.models import load_model
from parser import parser
import cv2
import os
import numpy as np
from camera import transform_img, eon_intrinsics
from model import medmodel_intrinsics
from lanes_image_space import transform_points
import json

BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))

def frame_to_tensor(frame):
    H = (frame.shape[0] * 2) // 3
    W = frame.shape[1]
    in_img1 = np.zeros((6, H // 2, W // 2), dtype=np.uint8)

    in_img1[0] = frame[0:H:2, 0::2]
    in_img1[1] = frame[1:H:2, 0::2]
    in_img1[2] = frame[0:H:2, 1::2]
    in_img1[3] = frame[1:H:2, 1::2]
    in_img1[4] = frame[H:H + H // 4].reshape((-1, H // 2, W // 2))
    in_img1[5] = frame[H + H // 4:H + H // 2].reshape((-1, H // 2, W // 2))
    return in_img1

def get_model_output(loaded_model,raw_frame,prev_frame2model,state=np.zeros((1, 512)),desire=np.zeros((1, 8))):
    parsed = outs = []
    frame_yuv420 = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2YUV_I420)
    imgs_med_model = np.zeros((384, 512), dtype=np.uint8)
    imgs_med_model = transform_img(frame_yuv420, from_intr=eon_intrinsics, to_intr=medmodel_intrinsics, yuv=True,                                            output_size=(512, 256))
    frame_to_model = frame_to_tensor(np.array(imgs_med_model)).astype(np.float32) / 128.0 - 1.0
    if len(prev_frame2model):
        input_img_arr = np.vstack((prev_frame2model,frame_to_model))
        inputs = [input_img_arr[None], desire, state]
        outs = loaded_model.predict(inputs)
        parsed = parser(outs)

    prev_frame2model = frame_to_model;

    return [parsed,outs,prev_frame2model]

def model_thread():
    #pm = messaging.PubMaster(['model'])
    supercombo = load_model(BASEDIR+'/models/test.keras')
    x_left = x_right = x_path = np.linspace(0, 192, 192)
    cap = cv2.VideoCapture(BASEDIR+'/test.hevc')
    state = np.zeros((1, 512))
    desire = np.zeros((1, 8))
    prev_model_frame = []
    frame_counter=0
    #print(supercombo.summary())
    if (cap.isOpened() == False):
        print("Error opening video stream or file")

    while (cap.isOpened()):
        ret, frame = cap.read()
        if ret == True:
            model_output = get_model_output(supercombo, frame, prev_model_frame, state, desire)
            list_holder = [];
            i=0;
            parsed = model_output[0]
            raw_model_output = model_output[1]
            prev_model_frame = model_output[2];
            if len(raw_model_output):

                state = model_output[1][-1];
                for k in raw_model_output:
                    list_holder.insert(i,k.tolist()[0])
                    i=i+1
                print(list_holder)

def main():
 model_thread()

if __name__ == "__main__":
  main()


"""
Non-Markvoian Gaussian Sequence Model (NM-GSM)
implementation.

We assume that the system is described by
    x_t = \phi x_{t-1} + \epsilon_t
    y_t = \sum_{i=0}^t \beta^{t-i} x_i + \eta_t

with x0 = 0.0
"""
import jax
import chex
import distrax
import jax.numpy as jnp
from functools import partial

@chex.dataclass
class ModelParameters:
    """
    Non-Markovian Gaussian Sequence Model
    parameters of the form

    f(xt | xt-1) = N(xt | phi * xt-1, q)
    g(yt | x{1:t}) = N(yt | sum_{k=1}^t beta^{t-k} * xt, r)

    Parameters
    ----------
    phi: mean of latent conditional dependency
    beta: strength of dependence on previous latent variables
    q: variance of latent conditional dependency
    r: variance of observation conditional dependency
    """
    phi: float
    beta: float
    q: float
    r: float

@chex.dataclass
class State:
    """
    State of the NM-GSM

    x: latent variable
    mu: sum of previous latent variables
    """
    x: float
    mu: float

@chex.dataclass
class DataHist:
    """
    Data history
    """
    x: jnp.ndarray
    y: jnp.ndarray

def latent_step(key, x_prev, params):
    """
    Latent step of the Non-Markovian Gaussian Sequence Model
    """
    x = jax.random.normal(key) * jnp.sqrt(params.q) + params.phi * x_prev
    return x

def observation_step(key, x, mu, params):
    """
    Observation step of the Non-Markovian Gaussian Sequence Model
    """
    mu_next =  params.beta * mu + x
    y = jax.random.normal(key) * jnp.sqrt(params.r) + mu_next
    return y, mu_next

def step(state, key, params):
    """
    Step of the Non-Markovian Gaussian Sequence Model
    """
    key_latent, key_observation = jax.random.split(key)
    x = latent_step(key_latent, state.x, params)
    y, mu = observation_step(key_observation, x, state.mu, params)

    state_next = state.replace(x=x, mu=mu)
    memory = DataHist(x=x, y=y)
    return state_next, memory

def simulate(key, params, num_steps):
    """
    Non-Markovian Gaussian Sequence Model
    """
    key_init, keys_steps = jax.random.split(key)
    keys_steps = jax.random.split(keys_steps, num_steps)
    # x_init = jax.random.normal(key_init)
    x_init = 0.0
    state = State(x=x_init, mu=0)

    partial_step = partial(step, params=params)
    _, steps = jax.lax.scan(partial_step, state, keys_steps)

    return steps

def step_target_mean(carry_row, row, params, x_latent):
    """
    Helper function to build means of non-Markovian
    GSM one row at a time.
    """
    carry_row = carry_row * params.beta + row
    mean_val = jnp.einsum("i...,i->...", x_latent, carry_row)
    return carry_row, mean_val

def eval_observation_mean(params, x_latent):
    """
    Evaluate the means of the observation distribution
    given the latent estimates.
    """
    num_steps = x_latent.shape[0]
    init_row = jnp.zeros(num_steps)
    eval_rows = jnp.eye(num_steps)

    partial_target = partial(step_target_mean, params=params, x_latent=x_latent)
    _, means = jax.lax.scan(partial_target, init_row, eval_rows)

    return means

def step_latent_logpdf(x_prev, params):
    mean =  params.phi * x_prev
    logprob = distrax.Normal(loc=mean, scale=jnp.sqrt(params.q)).log_prob(x_prev)
    return logprob

def log_observation(x_latent, y_obs, params):
    """
    Log target density of the Non-Markovian Gaussian Sequence Model
    """
    mean_est = eval_observation_mean(params, x_latent)
    scale_est = jnp.sqrt(params.r)
    log_probs = distrax.Normal(loc=mean_est, scale=scale_est).log_prob(y_obs)
    return log_probs

def log_transition(x_latent, params):
    """
    Log target density of the Non-Markovian Gaussian Sequence Model
    """
    x_cond = jnp.roll(x_latent, 1).at[0].set(0.0) * params.phi
    log_probs = distrax.Normal(loc=x_cond, scale=jnp.sqrt(params.q)).log_prob(x_latent)
    return log_probs

partial(jax.jit, static_argnums=(2,))
def log_joint(x_latent, y_obs, params):
    """
    Log target density of the Non-Markovian Gaussian Sequence Model
    """
    log_probs = log_transition(x_latent, params) + log_observation(x_latent, y_obs, params)
    return log_probs


import random
import numpy as np
import pandas as pd
from sklearn.model_selection import StratifiedKFold, KFold

pd.set_option("display.precision", 15)

def f(x):
    tmp = 2 + x[0] * np.sin(5 * x[0])
    if np.isscalar(x[1]):
        if x[1] > 0.3:
            tmp += 10 * (x[1] - 0.3) ** 2
    else:
        tmp[x[1] > 0.3] += 10 * (x[1][x[1] > 0.3] - 0.3) ** 2

    return tmp

def generate_random_samples(n: int = 100, file_name: str = 'test_sample_{}.csv'):
    x = np.array([[random.random(), random.random()] for _ in range(n)])
    df = pd.DataFrame(np.array([x[:, 0], x[:, 1], f(x.T)]).T, columns=['x', 'y', 'z'], index=None)
    df.to_csv(file_name.format(n))

def generate_train_samples(n: int = 1000):
    X = np.arange(0, 1, 1 / n)
    Y = np.arange(0, 1, 1 / n)
    X, Y = np.meshgrid(X, Y)
    Z = f(np.stack((X.flatten(), Y.flatten())))
    df = pd.DataFrame(np.array([X.flatten(), Y.flatten(), Z]).T, columns=['x', 'y', 'z'], index=None)
    df.to_csv('train_sample_{}.csv'.format(n**2))

# for continuous data
def get_stratified_sample(df, target_col='z', no_of_items=10000):
    df['Fold'] = -1
    bins = 10000
    splits = int(df.shape[0] / no_of_items)

    skf = StratifiedKFold(n_splits=splits, shuffle=True)
    df['grp'] = pd.cut(df[target_col], bins, labels=False)
    target = df.grp

    for fold_no, (t, v) in enumerate(skf.split(target, target)):
        df.loc[v, 'Fold'] = fold_no
    return df[df['Fold'] == 0].iloc[:, 1:4]


import base64
import pickle

from django.shortcuts import render

# Create your views here.
from django_redis import get_redis_connection
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView

from cart import constants
from cart.serializers import CartSerializer, CartSKUSerializer, CartDelSerializer, CartSelectSerializer
from goods.models import SKU

# PUT /cart/selection/
class CartSelectAllView(APIView):
    def perform_authentication(self, request):
        """让当前视图跳转DRF框架默认认证过程"""
        pass
    def put(self,request):
        """
        购物车记录全选和取消全选
        # 1.获取参数selected并进行校验（selected必传）
        # 2.设置用户购物车记录勾选状态
        #     2.1如果用户已登录，设置redis中用户购物车记录勾选状态。
        #     2.2如果用户未登录，设置cookie中用户购物车记录勾选状态。
        # 3.返回应答，设置成功
        """
        # 1.获取参数selected并进行校验（selected必传）
        serializer = CartSelectSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)

        # 获取校验之后的selected
        selected = serializer.validated_data['selected'] # True全选，False全不选

        try:
            # request.user会触发DRF框架认证过程
            user = request.user
        except Exception:
            user = None

        # 2.设置用户购物车记录勾选状态
        if user and user.is_authenticated:
            # 2.1如果用户已登录，设置redis中用户购物车记录勾选状态。
            # 获取redis链接
            redis_conn = get_redis_connection('cart')
            # 从redis hash中获取用户购物车中所有商品的id
            cart_key = 'cart_%s' % user.id
            sku_ids = redis_conn.hkeys(cart_key)

            cart_selected_key = 'cart_selected_%s' % user.id

            if selected:
                # 全选：将用户购物车所有商品的id添加时到redis set中
                redis_conn.sadd(cart_selected_key, *sku_ids)
            else:
                # 全不选：将用户购物车所有商品的id从redis set中移除
                redis_conn.srem(cart_selected_key, *sku_ids)
            return  Response({'message':'OK'})

        else:
            # 2.2如果用户未登录，设置cookie中用户购物车记录勾选状态。
            # 获取cookie的购物车记录
            cookie_cart = request.COOKIES.get('cart')

            if cookie_cart:
                cart_dict = pickle.loads(base64.b64decode(cookie_cart))
            else:
                cart_dict = {}

            # 设置cookie购物车记录勾选状态
            for sku_id, count_selected in cart_dict.items():
                cart_dict[sku_id]['selected'] = selected

            # 3.返回应答，设置成功
            response = Response({'message':'OK'})
            # 设置cookie中的购物车数据
            cart_data = base64.b64encode(pickle.dumps(cart_dict)).decode()
            response.set_cookie('cart', cart_data, max_age=constants.CART_COOKIE_EXPIRES)
            return response

class CartView(APIView):
    def perform_authentication(self, request):
        """让当前视图跳过DRF框架认证过程"""
        pass

    # DELETE /cart/
    def delete(self,request):
        """
        # 购物车记录删除
        1.获取商品sku_id并进行校验（sku_id必传，sku_id对应商品是否存在）
        2.删除用户的浏览记录：
            2.1如果用户已登录，删除redis中对应的购物车记录
            2.2如果用户未登录，删除cookie中对应的购物车记录
        3.返回应答，鬼屋车记录删除成功
        """
        # 1.获取商品sku_id并进行校验（sku_id必传，sku_id对应商品是否存在）
        serializer = CartDelSerializer(data = request.data)
        serializer.is_valid(raise_exception=True)

        # 获取校验之后的数据
        sku_id = serializer.validated_data['sku_id']

        try:
            user = request.user
        except Exception:
            user = None

        # 2.删除用户的浏览记录：
        if user and user.is_authenticated:
            # 2.1如果用户已登录，删除redis中对应的购物车记录
            # 获取redis链接
            redis_conn = get_redis_connection('cart')

            # 从redis hash删除对应商品的id和数量count
            cart_key = 'cart_%s' % user.id
            redis_conn.hdel(cart_key, sku_id)

            # 从redis set中删除对应的商品的id
            cart_selected_key = 'cart_selected_%s' % user.id
            redis_conn.srem(cart_selected_key, sku_id)

            return Response(status=status.HTTP_204_NO_CONTENT)
        else:
            # 2.2如果用户未登录，删除cookie中对应的购物车记录
            response = Response(status=status.HTTP_204_NO_CONTENT)
            # 获取cookie中的购物车记录
            cookie_cart = request.COOKIES.get('cart')

            if cookie_cart is None:
                # 购物车无数据
                return response
            cart_dict = pickle.loads(base64.b64decode(cookie_cart))

            # 删除对应的购物车记录
            if sku_id in cart_dict:
                del cart_dict[sku_id]
                # 重新设置cookie购物车数据
                cart_data = base64.b64encode(pickle.dumps(cart_dict)).decode()
                response.set_cookie('cart', cart_data, max_age=constants.CART_COOKIE_EXPIRES)
            # 3.返回应答，鬼屋车记录删除成功
            return response

    # PUT /cart/
    def put(self,request):
        """
        购物车记录修改：
        1.获取参数并进行校验（参数完整性，sku_id商品是否存在，商品的库存）
        2.修改用户的购物车记录
            2.1如果用户已登录，修改redis中对应的购物车记录
            2.2如果用户未登录，修改cookie中对应的购物车记录
        3.返回应答，购物车记录修改成功
        """
        # 1.获取参数并进行校验（参数完整性，sku_id商品是否存在，商品的库存）
        serializer = CartSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        # 获取校验之后的数据
        sku_id = serializer.validated_data['sku_id']
        count = serializer.validated_data['count']
        selected = serializer.validated_data['selected']

        # 获取User
        try:
            # 调用request.user 会触发DRF框架认证过程
            user = request.user
        except Exception:
            user = None
        # 2.修改用户的购物车记录
        if user and user.is_authenticated:
            # 2.1如果用户已登录，修改redis中对应的购物车记录
            #获取redis链接
            redis_conn = get_redis_connection('cart')

            # 修改redis hash中商品id对应数量count
            cart_key = 'cart_%s' % user.id
            redis_conn.hset(cart_key, sku_id, count)

            # 修改redis set勾选的商品id
            cart_selected_key = 'cart_selected_%s' % user.id

            if selected:
                # 勾选
                redis_conn.sadd(cart_selected_key, sku_id)
            else:
                # 取消勾选
                redis_conn.srem(cart_selected_key, sku_id)
            return Response(serializer.validated_data)
        else:
            # 2.2如果用户未登录，修改cookie中对应的购物车记录
            response = Response(serializer.validated_data)

            # 获取cookie中的购物车数据
            cookie_cart = request.COOKIES.get('cart')

            if cookie_cart is None:
                return response
            # 解析cookie中的购物车数据
            # {
            #     '<sku_id>': {
            #         'count': '<count>',
            #         'selected': '<selected>'
            #     },
            #     ...
            # }
            cart_dict = pickle.loads(base64.b64decode(cookie_cart))

            if not cart_dict:
                # 字典为空
                return response

            # 修改购物车数据
            cart_dict[sku_id] = {
                'count':count,
                'selected':selected
            }
            # 3.返回应答，购物车记录修改成功
            cart_data = base64.b64encode(pickle.dumps(cart_dict)).decode()
            response.set_cookie('cart',cart_data,max_age=constants.CART_COOKIE_EXPIRES)
            return response
    # POST /cart/
    def post(self,request):
        """
        购物车记录添加
        # 1.获取参数并进行校验（参数的完整性，sku_id商品是否存在，count是否大于商品库存
        # 2.保存用户的购物车记录
        #     2.1如果用户已登录，在redis中保存用户的购物车记录
        #     2.2如果用户未登录，在cookie中保存用户的购物车记录
        # 3.返回应答，购物车记录添加成功。
        """
        # 1.获取参数并进行校验（参数的完整性，sku_id商品是否存在，count是否大于商品库存
        serializer = CartSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)

        # 获取校验之后的数据
        sku_id = serializer.validated_data['sku_id']
        count = serializer.validated_data['count']
        selected = serializer.validated_data['selected']

        try:
            # 此代码会触发DRF框架认证过程，但是可以自己进行捕获处理
            user = request.user
        except Exception:
            user = None

        # 2.保存用户的购物车记录
        if user and user.is_authenticated:
        # 2.1如果用户已登录，在redis中保存用户的购物车记录
            # 获取链接对象
            redis_conn = get_redis_connection('cart')

            # hash:保存用户购物车添加的商品id和对应数量count
            cart_key = 'cart_%s' % user.id

            # 如果redis购物车已经添加该商品，数量需要进行累加
            redis_conn.hincrby(cart_key,sku_id,count)

            # set:保存用户购物车中被勾选的商品id
            cart_selected_key = 'cart_selected_%s' % user.id

            if selected:
                redis_conn.sadd(cart_selected_key,sku_id)
            return Response(serializer.validated_data,status=status.HTTP_201_CREATED)
        # 2.2如果用户未登录，在cookie中保存用户的购物车记录
        else:
            # 获取原始cookie购物车数据
            cookie_cart = request.COOKIES.get('cart')# None

            if cookie_cart:
                # {
                #     <sku_id>:{
                #     'count':'<count>',
                #     'selected':'<selected>'
                # },
                # ...
                # }
                cart_dict = pickle.loads(base64.b64decode(cookie_cart))
            else:
                cart_dict = {}
            # 保存购物车数据
            if sku_id in cart_dict:
                # 数量累加
                count += cart_dict[sku_id]['count']

            cart_dict[sku_id] = {
                'count':count,
                'selected':selected
            }

            # 3.返回应答，购物车记录添加成功。
            response = Response(serializer.validated_data,status=status.HTTP_201_CREATED)
            # 设置cookie购物车数据
            cart_data = base64.b64encode(pickle.dumps(cart_dict)).decode() # str
            response.set_cookie('cart',cart_data,max_age=constants.CART_COOKIE_EXPIRES)
            return response
    # GET /cart/
    def get(self,request):
        """
        购物车记录获取
        1.获取用户的购物车记录
            1.1如果用户已登录，从redis中获取用户的购物车记录
            1.2如果用户未登录，从cookie中获取用户的购物车记录
        2.根据用户购物车中商品id获取对应商品的数据
        3.将购物车商品的数据序列化并返回
        """
        try:
            user = request.user
        except Exception:
            user = None
        # 1.获取用户的购物车记录
        if user and user.is_authenticated:
            # 1.1如果用户已登录，从redis中获取用户的购物车记录
            # 获取redis链接
            redis_conn = get_redis_connection('cart')

            # 从redis hash中获取用户购物车中添加的商品id和对应的数量count
            cart_key = 'cart_%s' % user.id
            # {
            #     b'<sku_id>': b'<count>',
            #     ...
            # }
            cart_redis = redis_conn.hgetall(cart_key)

            # 从redis set中获取用户购物车中勾选的商品id
            cart_selected_key = 'cart_selected_%s' % user.id

            # (b'<sku_id>',...)
            cart_selected_redis = redis_conn.smembers(cart_selected_key)

            cart_dict = {}
            for sku_id,count in cart_redis.items():
                cart_dict[int(sku_id)] = {
                    'count':int(count),
                    'selected':sku_id in cart_selected_redis
                }
        else:
            # 1.2如果用户未登录，从cookie中获取用户的购物车记录
            # 获取cookie中的购物车数据
            cookie_cart = request.COOKIES.get('cart')

            if cookie_cart:
                # 解析cookie中购物车数据
                # {
                #     '<sku_id>': {
                #         'count': '<count>',
                #         'selected': '<selected>'
                #     },
                #     ...
                # }
                cart_dict = pickle.loads(base64.b64decode(cookie_cart))
            else:
                cart_dict = {}

        # 2.根据用户购物车中商品id获取对应商品的数据
        sku_ids = cart_dict.keys()
        skus = SKU.objects.filter(id__in = sku_ids)

        for sku in skus:
            # 给sku对象增加属性count和selected，分别保存该商品在购物车中添加的数量和勾选状态
            sku.count = cart_dict[sku.id]['count']
            sku.selected = cart_dict[sku.id]['selected']

        # 3.将购物车商品的数据序列化并返回
        serializer = CartSKUSerializer(skus,many=True)
        return Response(serializer.data)


import matplotlib.pyplot as plt
import pandas
from helpers import clusters

colors = ['r', 'b', 'g', 'k', 'm', 'c', 'y']

all_folders = ['clustering', 'ica', 'pca', 'random_forest', 'randomized_projections']
all_folders_name = ['Original', 'ICA', 'PCA', 'Random Forest', 'Randomized Projections']

nn_items = ['', 'param_ica__n_components', 'param_pca__n_components', 'param_filter__n', 'param_rp__n_components']

nn_items_cluster = ['param_gmm__n_components', 'param_km__n_clusters']

nn_cols = ['mean_fit_time', 'mean_test_score', 'mean_train_score', 'param_NN__alpha', 'param_NN__hidden_layer_sizes', 'params']

def plot_learning_curve(iterations, train_scores, test_scores, title):
#     _, _, test_scores_base = base_curve

    plt.figure()
    plt.title(title)
    plt.ylim((.3, 1.01))

    # if datasetNum == 1:
    #     plt.ylim((.55, 1.01))

    plt.xlabel("# Iterations")
    plt.ylabel("Score")

    plt.grid()

    plt.plot(iterations, train_scores, 'o-', color="r",
             label="Training score")
#     plt.plot(train_sizes, test_scores_base_mean, 'o-', color="b",
#              label="Test Score without CV")
    plt.plot(iterations, test_scores, 'o-', color="g",
             label="Test Score")

    plt.legend(loc="best")
    return plt

def plot_many_curves(iterations, valLabels, valIndex, title, yLabel):
    plt.figure()
    plt.title(title)

    plt.xlabel("# Iterations")
    plt.ylabel(yLabel)

    plt.grid()

    colors = ['r', 'b', 'g', 'k', 'm', 'c', 'y']
    for i in range(len(valLabels)):
        val = valLabels[i][valIndex]
        label = valLabels[i][-1]
        color = colors[i]

        plt.plot(iterations, val, 'o-', color=color,
             label=label)

    plt.legend(loc="best")
    return plt

def plot_timing_curve(iterations, timeDuration, title):
#     _, _, test_scores_base = base_curve

    plt.figure()
    plt.title(title)
    # plt.ylim((.3, 1.01))

    # if datasetNum == 1:
    #     plt.ylim((.55, 1.01))

    plt.xlabel("# Iterations")
    plt.ylabel("Training Time (s)")

    plt.grid()

    plt.plot(iterations, timeDuration, 'o-', color="r",
             label="Duration")

    plt.legend(loc="best")
    return plt

def get_df(path, headerVal=0):
    return pandas.read_csv('../results/' + path, header=headerVal)

def print_latex_row(vals, sig=3):
    sigString = '{:0.' + str(sig) + 'f}'
    vals = ' & '.join(list(map(lambda x: sigString.format(x), vals)))
    print(vals + ' \\\\ \hline')

def save_plot(plot, title):
    print (title.replace(' ', '_').replace('.', 'pt').lower())
    plot.savefig('../analysis/plots/' + title.replace(' ', '_').replace('.', 'pt').lower() + '.jpg')

def plot_item(df, valName, title, yLabel, lineLabel, xLabel = 'Number of Clusters', latexVal = 3):
    clusters = df[df.columns[0]]
    vals = df[valName]
    print_latex_row(vals, latexVal)

    plt.figure()
    plt.title(title)

    plt.xlabel(xLabel)
    plt.ylabel(yLabel)

    plt.grid()

    plt.plot(clusters, vals, 'o-', color="r",
             label=lineLabel)

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_sse(folder, valName, title):
    df = get_df(folder + '/SSE.csv')
    plot_item(df, valName, title, 'Inner Cluster Sum of Squared Errors', "Sum of Squared Errors")

def plot_clustering_accuracy(folderFile, title):
    ami = get_df(folderFile + ' adjMI.csv')
    acc = get_df(folderFile + ' acc.csv')

    clusters = ami.columns[1:].values

    gmm_ami = ami.loc[0].values[1:]
    gmm_acc = acc.loc[0].values[1:]

    k_means_ami = ami.loc[1].values[1:]
    k_means_acc = acc.loc[1].values[1:]

    print_latex_row(k_means_ami)
    print_latex_row(k_means_acc)
    print_latex_row(gmm_ami)
    print_latex_row(gmm_acc)

    newTitle = title + ' Scoring'
    plt.figure()
    plt.title(newTitle)
    plt.xlabel("Number of Clusters")
    plt.ylabel("Score")
    plt.grid()

    plt.plot(clusters, k_means_ami, 'o-', color="g", label="k-means Adjusted Mutual Info Score")
    plt.plot(clusters, k_means_acc, 'o-', color="b", label="k-means Cluster Accuracy")

    plt.plot(clusters, gmm_ami, 'o-', color="r", label="Expectation Maximization Adjusted Mutual Info Score")
    plt.plot(clusters, gmm_acc, 'o-', color="black", label="Expectation Maximization Cluster Accuracy")

    plt.legend(loc="best")

    save_plot(plt, newTitle)
    plt.close()

def plot_log_liklihood(folder, valName, title):
    df = get_df(folder + '/logliklihood.csv')
    plot_item(df, valName, title, 'Log Likelihood', 'Log Likelihood', 'Number of Components', 2)

def plot_eigen(folderFile, title):
    df = get_df(folderFile + ' scree.csv', None)

    plt.figure()
    plt.title(title)

    plt.xlabel('# Components')
    plt.ylabel('Eigenvalues')

    plt.grid()

    plt.plot(df[0].values, df[1].values, 'o-', color="g",
             label='Eigenvalue')
    plt.plot(df[0].values, df[2].values, 'o-', color="b",
             label='Variance')
    plt.plot(df[0].values, df[3].values, 'o-', color="black",
             label='Cumulative Variance')

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_kurt(folderFile, title):
    df = get_df(folderFile + ' scree.csv', None)

    plt.figure()
    plt.title(title)

    plt.xlabel('# Components')
    plt.ylabel('Kurtosis')

    plt.grid()

    plt.plot(df[0].values, df[1].values, 'o-', color="g",
             label='Kurtosis')

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_reconstruction(folderFile, title):
    df = get_df(folderFile + ' scree2.csv')

    plt.figure()
    plt.title(title)

    plt.xlabel('# Components')
    plt.ylabel('Reconstruction Error')

    plt.grid()

    for i in range(len(colors)):
        color = colors[i]

        plt.plot(df[df.columns[0]].values, df[df.columns[i + 1]].values, 'o-', color=color,
                 label='Error for Trial ' + str(i + 1))

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_pairwise_distance_corr(folderFile, title):
    df = get_df(folderFile + ' scree1.csv')

    plt.figure()
    plt.title(title)

    plt.xlabel('# Components')
    plt.ylabel('Parise Distance Correlation')

    plt.grid()

    for i in range(len(colors)):
        color = colors[i]

        plt.plot(df[df.columns[0]].values, df[df.columns[i + 1]].values, 'o-', color=color,
                 label='Correlation for Trial ' + str(i + 1))

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_feature_importances(folderFile, title):
    df = get_df(folderFile + ' scree.csv', None)

    plt.figure()
    plt.title(title)

    plt.xlabel('Feature #')
    plt.ylabel('Importance')

    plt.grid()

    plt.plot(df[0].values, df[1].values, 'o-', color="g",
             label='Importance')

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_all_sse(valName, title):
    title = title + ' SSE Comparison after DR'
    plt.figure()
    plt.title(title)

    plt.xlabel('# of Clusters')
    plt.ylabel('Inner Cluster Sum of Squared Errors')

    plt.grid()

    index = 0
    for folder in all_folders:
        line = '*-'
        if index == 0:
            line = 'o-'
        df = get_df(folder + '/SSE.csv')
        color = colors[index]
        name = all_folders_name[index]
        vals = df[valName]
        plt.plot(clusters, vals, line, color=color,
             label=name + ' SSE')

        index += 1

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_all_log_liklihood(valName, title):
    title = title + ' Log Liklihood Comparison after DR'
    plt.figure()
    plt.title(title)

    plt.xlabel('# of Clusters')
    plt.ylabel('Log Liklihood')

    plt.grid()

    index = 0
    for folder in all_folders:
        line = '*-'
        if index == 0:
            line = 'o-'
        df = get_df(folder + '/logliklihood.csv')
        color = colors[index]
        name = all_folders_name[index]
        vals = df[valName]
        plt.plot(clusters, vals, line, color=color,
             label=name + ' Log Liklihood')

        index += 1

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_all_scoring(fileName, title, kMeans=True):

    title = title + ' Scoring'
    plt.figure()
    plt.title(title)

    plt.xlabel('# of Clusters')
    plt.ylabel('Score')

    plt.grid()

    index = 0
    for folder in all_folders:
        line = '*-'
        if index == 0:
            line = 'o-'

        color = colors[index]
        # color1 = colors[-1 * index - 1]
        name = all_folders_name[index]

        ami = get_df(folder + '/' + fileName + ' adjMI.csv')
        acc = get_df(folder + '/' + fileName + ' acc.csv')

        vals1 = ami.loc[0].values[1:]
        vals2 = acc.loc[0].values[1:]

        if kMeans:
            vals1 = ami.loc[1].values[1:]
            vals2 = acc.loc[1].values[1:]

        plt.plot(clusters, vals1, line, color=color,
             label=name + ' AMI')
        plt.plot(clusters, vals2, line, color=color,
             label=name + ' Accuracy')

        index += 1

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_nn(fileBase, extra, xLabel, title):
    title = title + ' Scoring'
    plt.figure()
    plt.title(title)

    plt.xlabel(xLabel)
    plt.ylabel('Score')

    plt.grid()

    index = 0
    for folder in all_folders:
        fileName = fileBase + extra

        if index == 0:
            folder = 'base'
            fileName = fileBase
            index += 1
            continue

        line = '*-'
        if index == 0:
            line = 'o-'

        color = colors[index]

        name = all_folders_name[index]

        df = get_df(folder + '/' + fileName + '.csv').sort_values('rank_test_score')
        best = df.head(1)
        # print(best)

        best_data = best[nn_cols]

        best_data = map(lambda x: str(x), list(best_data.values[0, :]))
        # print(best_data)
        print(name + ' & ' + ' & '.join(best_data) + ' \\\\ \\hline')

        xVar = nn_items[index]

        nnAlpha = list(best['param_NN__alpha'].values)[0]
        nnLayers = list(best['param_NN__hidden_layer_sizes'].values)[0]

        dfLayers = df['param_NN__hidden_layer_sizes'] == nnLayers
        dfAlpha = df['param_NN__alpha'] == nnAlpha

        best_series = df[dfLayers & dfAlpha].sort_values(xVar)
        # print(best_series)

        xVals = list(set(df[xVar].values))

        vals = best_series['mean_test_score'].values

        # print(xVals)
        # print(vals)

        plt.plot(xVals, vals, line, color=color,
             label=name + ' NN')

        index += 1

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_nn_cluster(fileBase, extra, xLabel, title):
    print('')
    title = title + ' Scoring'
    plt.figure()
    plt.title(title)

    plt.xlabel(xLabel)
    plt.ylabel('Score')

    plt.grid()

    df = get_df('clustering/Housing cluster GMM.csv').sort_values('rank_test_score')
    best = df.head(1)
    # print(best)

    best_data = best[nn_cols]

    best_data = map(lambda x: str(x), list(best_data.values[0, :]))
    # print(best_data)
    print('Expectation Maximization' + ' & ' + ' & '.join(best_data) + ' \\\\ \\hline')

    xVar = nn_items_cluster[0]

    nnAlpha = list(best['param_NN__alpha'].values)[0]
    nnLayers = list(best['param_NN__hidden_layer_sizes'].values)[0]

    dfLayers = df['param_NN__hidden_layer_sizes'] == nnLayers
    dfAlpha = df['param_NN__alpha'] == nnAlpha

    best_series = df[dfLayers & dfAlpha].sort_values(xVar)
    # print(best_series)

    xVals = list(set(df[xVar].values))
    xVals.sort()

    vals = best_series['mean_test_score'].values

    # print(xVals)
    # print(vals)

    plt.plot(xVals, vals, 'o-', color='g',
         label='Expectation Maximization NN')

    df = get_df('clustering/Housing cluster Kmeans.csv').sort_values('rank_test_score')
    best = df.head(1)
    # print(best)

    best_data = best[nn_cols]

    best_data = map(lambda x: str(x), list(best_data.values[0, :]))
    # print(best_data)
    print('k-means' + ' & ' + ' & '.join(best_data) + ' \\\\ \\hline')

    xVar = nn_items_cluster[1]

    nnAlpha = list(best['param_NN__alpha'].values)[0]
    nnLayers = list(best['param_NN__hidden_layer_sizes'].values)[0]

    dfLayers = df['param_NN__hidden_layer_sizes'] == nnLayers
    dfAlpha = df['param_NN__alpha'] == nnAlpha

    best_series = df[dfLayers & dfAlpha].sort_values(xVar)
    # print(best_series)

    xVals = list(set(df[xVar].values))
    xVals.sort()

    vals = best_series['mean_test_score'].values

    # print(xVals)
    # print(vals)

    plt.plot(xVals, vals, 'o-', color='b',
         label='k-Means NN')

    plt.legend(loc="best")

    save_plot(plt, title)
    plt.close()

def plot_clustering():
    plot_sse('clustering', 'perm SSE (left)', 'Default Credit Card Payment SSE')
    plot_sse('clustering', 'housing SSE (left)', 'Wine Quality  SSE')

    plot_clustering_accuracy('clustering/Housing', 'Wine Quality  Clustering')
    plot_clustering_accuracy('clustering/Perm', 'Default Credit Card Payment Clustering')

    plot_log_liklihood('clustering', 'housing log-likelihood', 'Wine Quality Logliklihood')
    plot_log_liklihood('clustering', 'perm log-likelihood', 'Default Credit Card Payment Logliklihood')

def plot_pca():
    plot_eigen('pca/housing', 'Wine Quality  PCA Eigenvalues')
    plot_eigen('pca/perm', 'Default Credit Card Payment PCA Eigenvalues')

    plot_sse('pca', 'perm SSE (left)', 'PCA Default Credit Card Payment SSE')
    plot_sse('pca', 'housing SSE (left)', 'PCA Housing SSE')

    plot_clustering_accuracy('pca/Housing', 'PCA Wine Quality  Clustering')
    plot_clustering_accuracy('pca/Perm', 'PCA Default Credit Card Payment Clustering')

    plot_log_liklihood('pca', 'housing log-likelihood', 'PCA Wine Quality  Logliklihood')
    plot_log_liklihood('pca', 'perm log-likelihood', 'PCA Default Credit Card Payment Logliklihood')

def plot_ica():
    plot_kurt('ica/housing', 'Wine Quality  ICA Kurtosis')
    plot_kurt('ica/perm', 'Default Credit Card Payment ICA Kurtosis')

    plot_sse('ica', 'perm SSE (left)', 'ICA Default Credit Card Payment SSE')
    plot_sse('ica', 'housing SSE (left)', 'ICA Wine Quality  SSE')

    plot_clustering_accuracy('ica/Housing', 'ICA Wine Quality  Clustering')
    plot_clustering_accuracy('ica/Perm', 'ICA Default Credit Card Payment Clustering')

    plot_log_liklihood('ica', 'housing log-likelihood', 'ICA Wine Quality  Logliklihood')
    plot_log_liklihood('ica', 'perm log-likelihood', 'ICA Default Credit Card Paymenta Logliklihood')

def plot_rp():
    plot_reconstruction('randomized_projections/housing', 'Wine Quality  Randomized Projections Reconstruction Error')
    plot_reconstruction('randomized_projections/perm', 'Default Credit Card Payment Randomized Projections Reconstruction Error')

    plot_pairwise_distance_corr('randomized_projections/housing', 'Wine Quality  Randomized Projections Pairwise Dist. Corr.')
    plot_pairwise_distance_corr('randomized_projections/perm', 'Default Credit Card Payment Randomized Projections Pairwise Dist. Corr.')

    plot_sse('randomized_projections', 'perm SSE (left)', 'RP Default Credit Card Payment SSE')
    plot_sse('randomized_projections', 'housing SSE (left)', 'RP Wine Quality  SSE')

    plot_clustering_accuracy('randomized_projections/Housing', 'RP Wine Quality  Clustering')
    plot_clustering_accuracy('randomized_projections/Perm', 'RP Default Credit Card Payment Clustering')

    plot_log_liklihood('randomized_projections', 'housing log-likelihood', 'RP Wine Quality  Logliklihood')
    plot_log_liklihood('randomized_projections', 'perm log-likelihood', 'RP Default Credit Card Payment Logliklihood')

def plot_rf():
    plot_feature_importances('random_forest/housing', 'Wine Quality  Random Forests Feature Importances')
    plot_feature_importances('random_forest/perm', 'Default Credit Card Payment Random Forests Feature Importances')

    plot_sse('random_forest', 'perm SSE (left)', 'RF Default Credit Card Payment SSE')
    plot_sse('random_forest', 'housing SSE (left)', 'RF Wine Quality  SSE')

    plot_clustering_accuracy('random_forest/Housing', 'RF Wine Quality  Clustering')
    plot_clustering_accuracy('random_forest/Perm', 'RF Default Credit Card Payment Clustering')

    plot_log_liklihood('random_forest', 'housing log-likelihood', 'RF Wine Quality  Logliklihood')
    plot_log_liklihood('random_forest', 'perm log-likelihood', 'RF Default Credit Card Payment Logliklihood')

def plot_dr():
    plot_pca()
    plot_ica()
    plot_rp()
    plot_rf()

def plot_comparison():
    plot_all_sse('perm SSE (left)', 'Default Credit Card Payment')
    plot_all_sse('housing SSE (left)', 'Wine Quality ')

    plot_all_log_liklihood('perm log-likelihood', 'Default Credit Card Payment')
    plot_all_log_liklihood('housing log-likelihood', 'Wine Quality ')

    plot_all_scoring('housing', 'Wine Quality  k-Means after DR', True)
    plot_all_scoring('housing', 'Wine Quality  Estimation Maximization after DR', False)

    plot_all_scoring('perm', 'Default Credit Card Payment k-Means after DR', True)
    plot_all_scoring('perm', 'Default Credit Card Payment Estimation Maximization after DR', False)

    plot_nn('housing', ' dim red', '# Components', 'Wine Quality  NN after DR')
    plot_nn_cluster('housing', ' dim red', '# Clusters', 'Wine Quality  NN after Cluster')

plot_clustering()
plot_dr()
plot_comparison()

# plot('NN_OUTPUT/BACKPROP_LOG.txt', 'Backprop NN')

# plot('NN_OUTPUT/RHC_LOG.txt', 'Randomized Hill Climbing NN')

# plot('NN_OUTPUT/SA0.15_LOG.txt', 'Simulated Annealing Cooling .15 NN')
# plot('NN_OUTPUT/SA0.35_LOG.txt', 'Simulated Annealing Cooling .35 NN')
# plot('NN_OUTPUT/SA0.55_LOG.txt', 'Simulated Annealing Cooling .55 NN')
# plot('NN_OUTPUT/SA0.7_LOG.txt', 'Simulated Annealing Cooling .7 NN')
# plot('NN_OUTPUT/SA0.95_LOG.txt', 'Simulated Annealing Cooling .95 NN')

# plot('NN_OUTPUT/GA_50_10_10_LOG.txt', 'Genetic 10 Mate, 10 Mutate NN')
# plot('NN_OUTPUT/GA_50_10_20_LOG.txt', 'Genetic 10 Mate, 20 Mutate NN')
# plot('NN_OUTPUT/GA_50_20_10_LOG.txt', 'Genetic 20 Mate, 10 Mutate NN')
# plot('NN_OUTPUT/GA_50_20_20_LOG.txt', 'Genetic 20 Mate, 20 Mutate NN')

# plot_peaks('Continous Peaks')
# plot_tsp('Traveling Salesman')
# plot_flipflop('Flip Flop')

# iteration,MSE_trg,MSE_val,MSE_tst,acc_trg,acc_val,acc_tst,elapsed
# 0,0.293247932778,0.292920659286,0.300509176221,0.413502881713,0.414196242171,0.398981670061,0.588158795
# 10,0.068612203824,0.0699373695198,0.0645621181263,0.862775592352,0.86012526096,0.870875763747,7.978658689
# 20,0.0686569920033,0.0699354731105,0.0641716006192,0.862684109414,0.86012526096,0.871690427699,15.104390501


from pyspark.sql import SparkSession
from pyspark.sql.functions import col
from pyspark.sql.types import TimestampType, IntegerType, LongType

spark = SparkSession \
        .builder \
        .master('spark://10.10.28.172:7077') \
        .appName('DB_setup') \
        .enableHiveSupport() \
        .getOrCreate()

spark.sql('use twitter_data')

# read raw data
raw_data = spark.read.csv('hdfs://namenode:9000/data/*.csv', header=True)

# check if data got read
if raw_data.count() < 10:
    raise ValueError('There are no csv files in hdfs/data/\nPlease run "load_data_to_hdfs" script')

# casting columns to correct format
data = raw_data.\
withColumn('id', col('id').cast(LongType())).\
withColumn('author_id', col('author_id').cast(LongType())).\
withColumn('created_at', col('created_at').cast(TimestampType()))

# write to hive table - twitter_data.raw_data
spark.sql('drop table if exists raw_data')
data.write.format('hive').mode("overwrite").saveAsTable("raw_data")


import os
from traceback import print_exc

from selenium import webdriver
from selenium.webdriver.common.by import By

from .escribir_txt import append_texto

archivo_ejecutado = os.path.basename(__file__)

def buscar_malaga(driver: webdriver, nombre_archivo, parametro_busqueda):
    try:
        driver = driver
        driver.get("https://www.bopmalaga.es/buscar.php")
        driver.find_elements(by=By.NAME, value="valor")[1].send_keys(parametro_busqueda)
        driver.find_element(by=By.ID, value="boton").click()
        driver.implicitly_wait(10)
        elemento = driver.find_element(by=By.ID, value="divResultado").find_elements(
            by=By.TAG_NAME, value="article"
        )[0]
        texto = elemento.text
        fecha = (
            elemento.find_element(by=By.CLASS_NAME, value="span_enlaces")
            .find_elements(by=By.TAG_NAME, value="a")[0]
            .text
        )
        enlace = (
            elemento.find_element(by=By.CLASS_NAME, value="span_enlaces")
            .find_elements(by=By.TAG_NAME, value="a")[1]
            .get_attribute("href")
        )

        append_texto(
            nombre_archivo=nombre_archivo,
            nombre_boletin_may="BOLETIN DE MALAGA",
            texto=texto,
            enlace=enlace,
            fecha=fecha,
        )

    except Exception as e:
        print(f"Ha habido un error del tipo {e} en el archivo {archivo_ejecutado}")
        print_exc()

if __name__ == "__main__":
    buscar_malaga(webdriver.Chrome(), "prueba_malaga.txt")


#!/usr/bin/env python
from src.lux_cls  import Luxafor
from datetime import datetime as dt, timedelta as td
import time

class Alarm:
    def __init__(self, color, time, blink=True):
        self.color = color
        self.blink = blink
        self.time = time

class Timer:
    def __init__(self,total=60,alarms=[]):
        self.lux = Luxafor()
        self.total=total
        self.alarms = alarms
        if len(alarms) == 0:
            self.alarms.append(Alarm('yellow',10,True))
            self.alarms.append(Alarm('red',5,True))
        self.reset()

    def reset(self):
        self.starttime = dt.now()
        self.endtime = self.starttime + td(seconds=self.total)
        self.state = 0

    def start(self):
        now = dt.now()
        self.lux.set_clr_by_name('green')
        while(now <=  self.endtime ):
            now = dt.now()
            for i in range(0,len(self.alarms)):
                alarm = self.alarms[i]
                if now >= (self.endtime - td(seconds=alarm.time)) and self.state <= i+1:
                    self.state = i+1
                    if alarm.blink:
                        self.lux.flash(alarm.color)
                    else:
                        self.lux.set_clr_by_name(alarm.color)
                        time.sleep(.5)
                elif self.state == 0:
                    time.sleep(.5)
        self.lux.set_clr_by_name('red')

    def stop(self):
        self.lux.set_clr_by_name('no-color')

if __name__ == "__main__":
    alerts=[]
    alerts.append(Alarm('blue', 90, False))
    alerts.append(Alarm('green', 60, False))
    alerts.append(Alarm('yellow', 30, False))
    alerts.append(Alarm('yellow', 15, True))
    alerts.append(Alarm('red', 5, True))
    timer = Timer(90, alerts)
    try:
        action = "A"
        while action is "A" or action is "a":
            timer.reset()
            timer.start()
            action = input("[A]gain or [S]top: ")
    finally:
        timer.stop()


import threading
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import globalz
from DS import *
lastNumOps = 0
lastDS = -1
class LiveGraph(threading.Thread):

  def run(self):

    #initialization code to make graph look pretty
    window = 20
    fig = plt.figure()
    fig.subplots_adjust(hspace=0.6)#more height between subplots
    fig.set_facecolor("#FFFFFF")
    ax1 = fig.add_subplot(2, 1, 1)
    ax2 = fig.add_subplot(2, 1, 2)

    line, = ax1.plot([], [], lw=3, color = "black")
    x = [0,4]
    n, bins, patches = ax2.hist(x, 4, normed=False, facecolor='green', rwidth = 0.95, alpha=0.75)

    ax1.set_ylim(0, 100)
    ax1.set_xlim(0, window)
    ax1.set_xlabel("Time (s)")
    ax1.set_ylabel("#Data Structure Operations")
    ax1.grid()

    ax2.set_ylim(0, 100)
    #ax2.set_xlim(0, window)
    ax2.yaxis.grid()
    labels = ["contains","add","remove","get"]
    ax2.set_xticks(bins)
    ax2.xaxis.set_visible(False);
    ax2.set_ylabel("Frequency of Data Structure Operations");

    # Label operation names below x-axis
    bin_centers = 0.5 * np.diff(bins) + bins[:-1]
    for x, label in zip(bin_centers, labels):
        ax2.annotate(label, xy=(x, 0), xycoords=('data', 'axes fraction'), xytext=(0, -18), textcoords='offset points', va='top', ha='center')

    xdata, ydata = [], []

    # Update graph variables that change over time
    def get_data():
      t = get_data.t
      while 1:
          t += 0.1
          yield t, globalz.num_ops, globalz.which, globalz.contains_ctr, globalz.add_ctr, globalz.remove_ctr, globalz.get_ctr
    get_data.t = 0

    def update(data):
        global lastNumOps
        global lastDS
        # update line graph (ops/sec)
        t,ops, which, contains, add, remove, get = data
        opsPerSec = ops - lastNumOps
        lastNumOps = ops

        xdata.append(t)
        ydata.append(opsPerSec)
        xmin1, xmax1 = ax1.get_xlim()
        ymin1, ymax1 = ax1.get_ylim()

        if t >= xmax1:
            ax1.set_xlim(xmax1, xmax1 + window)
            ax1.figure.canvas.draw()

        if opsPerSec >= ymax1:
            ax1.set_ylim(ymin1, ymax1 * 1.5)
            ax1.figure.canvas.draw()

        line.set_data(xdata, ydata)

        # Draw red circle when data structure changes
        if lastDS != which:
          ax1.scatter(xdata[len(xdata)-1],ydata[len(ydata)-1],s=50,color="red")
          lastDS = which

        plt.title(DS.STRINGS[which], color="#000000")

        # update bar graph (freqency per op)
        ymin2, ymax2 = ax2.get_ylim()
        operations = [contains, add, remove, get]

        for i in range(4):
          patches[i].set_height(operations[i])
          if operations[i] >= ymax2:
            ax2.set_ylim(ymin2, ymax2 * 1.5)
            ax2.figure.canvas.draw()

        return line, patches

    ani = animation.FuncAnimation(fig, update, get_data, blit=False, interval=100,
        repeat=False)

    plt.subplots_adjust(bottom=0.15)
    plt.show()


import sys, os
import argparse

import yaml
import cPickle

import numpy as np

from hexrd import config
from hexrd import matrixutil as mutil

from hexrd.fitgrains import get_instrument_parameters

from hexrd.matrixutil import unitVector

# HARD-CODED DISTORTION!!!
from hexrd.xrd.distortion import GE_41RT

from hexrd.gridutil import cellIndices

from hexrd.xrd import transforms as xf
from hexrd.xrd import transforms_CAPI as xfcapi

from hexrd.xrd import rotations as rot
from hexrd.xrd.symmetry import toFundamentalRegion
from hexrd.xrd.xrdutil import simulateOmeEtaMaps

import matplotlib
matplotlib.use('Qt4Agg')
import matplotlib.pyplot as plt
from matplotlib import cm
plt.ioff()

def check_indexing_plots(cfg_filename, plot_trials=False, plot_from_grains=False):
    cfg = config.open(cfg_filename)[0]    # use first block, like indexing

    working_dir = cfg.working_dir
    analysis_dir = os.path.join(working_dir, cfg.analysis_name)

    #instrument parameters
    icfg = get_instrument_parameters(cfg)
    chi = icfg['oscillation_stage']['chi']

    # load maps that were used
    oem = cPickle.load(
        open(cfg.find_orientations.orientation_maps.file, 'r')
        )
    nmaps = len(oem.dataStore)
    omeEdges = np.degrees(oem.omeEdges); nome = len(omeEdges) - 1
    etaEdges = np.degrees(oem.etaEdges); neta = len(etaEdges) - 1
    delta_ome = abs(omeEdges[1]-omeEdges[0])

    full_ome_range = xf.angularDifference(omeEdges[0], omeEdges[-1]) == 0
    full_eta_range = xf.angularDifference(etaEdges[0], etaEdges[-1]) == 0

    # grab plane data and figure out IDs of map HKLS
    pd = oem.planeData
    gvids = [pd.hklDataList[i]['hklID'] for i in np.where(pd.exclusions == False)[0].tolist()]

    # load orientations
    quats = np.atleast_2d(np.loadtxt(os.path.join(working_dir, 'accepted_orientations.dat')))
    if plot_trials:
        scored_trials = np.load(os.path.join(working_dir, 'scored_orientations.dat'))
        quats = scored_orientations[:4, scored_orientations[-1, :] >= cfg.find_orientations.clustering.completeness]
        pass
    expMaps = np.tile(2. * np.arccos(quats[:, 0]), (3, 1))*unitVector(quats[:, 1:].T)

    ##########################################
    #      SPECIAL CASE FOR FIT GRAINS       #
    ##########################################
    if plot_from_grains:
        distortion = (GE_41RT, icfg['detector']['distortion']['parameters'])
        #
        grain_table = np.atleast_2d(np.loadtxt(os.path.join(analysis_dir, 'grains.out')))
        ngrains = len(grain_table)
        #
        expMaps = grain_table[:, 3:6]
        tVec_c = grain_table[:, 6:9]
        vInv = grain_table[:, 6:12]
        #
        rMat_d = xf.makeDetectorRotMat(icfg['detector']['transform']['tilt_angles'])
        tVec_d = np.vstack(icfg['detector']['transform']['t_vec_d'])
        #
        chi = icfg['oscillation_stage']['chi']
        tVec_s = np.vstack(icfg['oscillation_stage']['t_vec_s'])
        #
        oes = np.zeros(oem.dataStore.shape)
        for i_grn in range(ngrains):
            spots_table = np.loadtxt(os.path.join(analysis_dir, 'spots_%05d.out' %i_grn))
            idx_m = spots_table[:, 0] >= 0
            for i_map in range(nmaps):
                idx_g = spots_table[:, 1] == gvids[i_map]
                idx = np.logical_and(idx_m, idx_g)
                nrefl = sum(idx)

                omes_fit = xf.mapAngle(spots_table[idx, 9], np.radians(cfg.find_orientations.omega.period), units='radians')
                xy_det = spots_table[idx, -3:]
                xy_det[:, 2] = np.zeros(nrefl)

                rMat_s_array = xfcapi.makeOscillRotMatArray(chi, omes_fit)

                # form in-plane vectors for detector points list in DETECTOR FRAME
                P2_d = xy_det.T

                # in LAB FRAME
                P2_l = np.dot(rMat_d, P2_d) + tVec_d # point on detector
                P0_l = np.hstack(
                    [tVec_s + np.dot(rMat_s_array[j], tVec_c[i_grn, :].reshape(3, 1)) for j in range(nrefl)]
                ) # origin of CRYSTAL FRAME

                # diffraction unit vector components in LAB FRAME
                dHat_l = unitVector(P2_l - P0_l)
                P2_l = np.dot(rMat_d, xy_det.T) + tVec_d

                # angles for reference frame
                dHat_ref_l = unitVector(P2_l)

                # append etas and omes
                etas_fit = np.arctan2(dHat_ref_l[1, :], dHat_ref_l[0, :]).flatten()

                # find indices, then truncate or wrap
                i_ome = cellIndices(oem.omeEdges, omes_fit)
                if full_ome_range:
                    i_ome[i_ome < 0] = np.mod(i_ome, nome) + 1
                    i_ome[i_ome >= nome] = np.mod(i_ome, nome)
                else:
                    incl = np.logical_or(i_ome >= 0, i_ome < nome)
                    i_ome = i_ome[incl]
                j_eta = cellIndices(oem.etaEdges, etas_fit)
                if full_eta_range:
                    j_eta[j_eta < 0] = np.mod(j_eta, neta) + 1
                    j_eta[j_eta >= neta] = np.mod(j_eta, neta)
                else:
                    incl = np.logical_or(j_eta >= 0, j_eta < neta)
                    j_eta = j_eta[incl]

                #if np.max(i_ome) >= nome or np.min(i_ome) < 0 or np.max(j_eta) >= neta or np.min(j_eta) < 0:
                #    import pdb; pdb.set_trace()
                # add to map
                oes[i_map][i_ome, j_eta] = 1
            pass
        pass

    # simulate quaternion points
    if not plot_from_grains:
        oes = simulateOmeEtaMaps(omeEdges, etaEdges, pd,
                                 expMaps,
                                 chi=chi,
                                 etaTol=0.01, omeTol=0.01,
                                 etaRanges=None, omeRanges=None,
                                 bVec=xf.bVec_ref, eVec=xf.eta_ref, vInv=xf.vInv_ref)

    # tick labling
    omes = np.degrees(oem.omeEdges)
    etas = np.degrees(oem.etaEdges)
    num_ticks = 7
    xmin = np.amin(etas); xmax = np.amax(etas)
    dx = (xmax - xmin) / (num_ticks - 1.); dx1 = (len(etas) - 1) / (num_ticks - 1.)
    xtlab = ["%.0f" % (xmin + i*dx) for i in range(num_ticks)]
    xtloc = np.array([i*dx1 for i in range(num_ticks)]) - 0.5
    ymin = np.amin(omes); ymax = np.amax(omes)
    dy = (ymax - ymin) / (num_ticks - 1.); dy1 = (len(omes) - 1) / (num_ticks - 1.)
    ytlab = ["%.0f" % (ymin + i*dy) for i in range(num_ticks)]
    ytloc = np.array([i*dy1 for i in range(num_ticks)]) - 0.5

    # Plot the three kernel density estimates
    n_maps = len(oem.iHKLList)

    fig_list =[plt.figure(num=i+1) for i in range(n_maps)]
    ax_list = [fig_list[i].gca() for i in range(n_maps)]
    for i_map in range(n_maps):
        y, x = np.where(oes[i_map] > 0)
        ax_list[i_map].hold(True)
        ax_list[i_map].imshow(oem.dataStore[i_map] > 0.1, cmap=cm.bone)
        ax_list[i_map].set_title(r'Map for $\{%d %d %d\}$' %tuple(pd.hkls[:, i_map]))
        ax_list[i_map].set_xlabel(r'Azimuth channel, $\eta$; $\Delta\eta=%.3f$' %delta_ome)
        ax_list[i_map].set_ylabel(r'Rotation channel, $\omega$; $\Delta\omega=%.3f$' %delta_ome)
        ax_list[i_map].plot(x, y, 'c+')
        ax_list[i_map].xaxis.set_ticks(xtloc)
        ax_list[i_map].xaxis.set_ticklabels(xtlab)
        ax_list[i_map].yaxis.set_ticks(ytloc)
        ax_list[i_map].yaxis.set_ticklabels(ytlab)
        ax_list[i_map].axis('tight')
    plt.show()
    return fig_list, oes

if __name__ == '__main__':
    """
    USAGE : python check_indexing <cfg_file>
    """
    parser = argparse.ArgumentParser(description='Make median dark from cfg file')

    parser.add_argument('cfg', metavar='cfg_filename', type=str, help='a YAML config filename')
    parser.add_argument('-t','--show-trials', help='plot trial orientations', action='store_true', default=False)
    parser.add_argument('-g','--plot-from-grains', help='plot fit orientations', action='store_true', default=False)

    args = vars(parser.parse_args(sys.argv[1:]))

    dark = check_indexing_plots(args['cfg'], plot_trials=args['show_trials'], plot_from_grains=args['plot_from_grains'])


# import the necessary packages
from imutils import face_utils
import numpy as np
import imutils
import dlib
import cv2

# the facial landmark predictor
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("./Hog_face_detector/shape_predictor_68_face_landmarks.dat")

# load the input image, resize it, and convert it to grayscale
image = cv2.imread("Img37.jpg")
image = imutils.resize(image, width=500)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

# detect faces in the grayscale image
rects = detector(gray, 1)
# loop over the face detections
for (i, rect) in enumerate(rects):
    shape = predictor(gray, rect)
    shape = face_utils.shape_to_np(shape)

    # convert dlib's rectangle to a OpenCV-style bounding box
    # [i.e., (x, y, w, h)], then draw the face bounding box
    #(x, y, w, h) = face_utils.rect_to_bb(rect)
    #cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)

    # show the face number
    #cv2.putText(image, "Face #{}".format(i + 1), (x - 10, y - 10),
     #   cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)

    # loop over the (x, y)-coordinates for the facial landmarks
    # and draw them on the image
    for (x, y) in shape:
        #print(x,y)
        cv2.circle(image, (x, y), 5, (0, 0, 255), -1)

    for (name, (i, j)) in face_utils.FACIAL_LANDMARKS_IDXS.items():
        print(name)
        count = 0
        #print(shape[i:j])
        if(name == "jaw"):
            k=0
            while(k<9):
                x1 = shape[i:j][k][0]
                y1 = shape[i:j][k][1]
                nextColumnY = shape[i:j][k+1][1]
                x2 = shape[i:j][16-k][0]
                print(x1,y1,x2,y1,nextColumnY)
                while(y1<nextColumnY):
                    while(x1<x2):
                        count = count+1
                        cv2.circle(image, (x1, y1), 1, (0, 210, 255), -1)
                        x1 = x1+10

                    y1 = y1+10
                    x1 = shape[i:j][k][0]

                k=k+1
    print(count)

# show the output image with the face detections + facial landmarks
cv2.imshow("Output", image)
cv2.waitKey(0)


import numpy as np
from scipy.spatial.distance import cdist
from skimage.feature import peak_local_max
from skimage import filters

def get_idx_2smallest_dist(distances):
    """Given a distance matrix it returns the index of the k
    smallest distances.

    Parameters
    ----------

    distances: 2d-array distance matrix

    Returns
    -------

    d1_ix, d2_ix: Tuple.
        For eact row of the distance matrix it returns the index of the
        smallest and second smallest distance.

    """
    n_rows = distances.shape[0]
    idxs = np.argsort(distances, axis=1)[:, :2]
    d1_ix = [np.arange(n_rows), idxs[:, 0]]
    d2_ix = [np.arange(n_rows), idxs[:, 1]]
    return d1_ix, d2_ix

def match_features(im1_features, im2_features):
    """ Returns the index of the common image features as well as the
    confidence of a correct match.

    Parameters
    ----------

    im1_features: array, features of image 1.

    im2_features: array, features of image 2.

    Returns
    -------

    matches: array
        First column = matches for image 1.
        Second column = matches for image 2.

    confidences: list, confidence for each match identified.
        We define it as:

            confidence = 1 - NNDR

        where:

        NDDR = \frac{d_1}{d_2} = \frac{||D_A - D_B||}{||D_A - D_C||}

        See Equation 4.18 in Section 4.1.3 of Szeliski.
    """
    distances = cdist(im1_features, im2_features, metric="euclidean")
    d1_ix, d2_ix = get_idx_2smallest_dist(distances)
    nddr = distances[d1_ix] / distances[d2_ix]

    matches = np.stack((d1_ix[0], d1_ix[1]), axis=1)
    confidences = 1 - nddr

    return matches, confidences

def get_features(image, x, y, feature_width):
    '''
    Returns a set of feature descriptors for a given set of interest points.

    Parameters
    ----------

    image: a grayscale or color image (your choice depending on your
        implementation)
    x: np array of x coordinates of interest points
    y: np array of y coordinates of interest points
    feature_width: in pixels, is the local feature width. You can assume
        that feature_width will be a multiple of 4 (i.e. every cell of your
        local SIFT-like feature will have an integer width and height).

    Returns
    -------

    features: np array of computed features. It should be of size
            [len(x) * feature dimensionality] (for standard SIFT feature
            dimensionality is 128)

    '''
    if feature_width % 4 != 0:
        raise ValueError("feature_width must be a multiple of 4.")
    x = np.round(x).astype(int).flatten()
    y = np.round(y).astype(int).flatten()
    offset = feature_width // 2
    descriptors = list()
    n_bins = 8

    # compute gradients, magnitudes, angles
    gradients = np.array(np.gradient(image))
    magnitudes = np.linalg.norm(gradients, axis=0)
    angles = np.angle(np.arctan2(gradients[0], gradients[1]), deg=True)

    # Loop through keypoints
    for xi, yi in zip(x, y):
        crop_magnitudes = magnitudes[yi-offset +
                                     1:yi+offset+1, xi-offset+1:xi+offset+1]
        crop_angles = angles[yi-offset+1:yi+offset+1, xi-offset+1:xi+offset+1]
        if crop_magnitudes.shape != (feature_width, feature_width):
            # Crop does not satisfy size constraint, skip keypoint.
            continue

        # Create SIFT descriptor
        patches_magnitudes = np.array(np.hsplit(np.array(
            np.hsplit(crop_magnitudes, 4)).reshape(4, -1), 4)).reshape(-1, feature_width)
        patches_angles = np.array(np.hsplit(np.array(
            np.hsplit(crop_angles, 4)).reshape(4, -1), 4)).reshape(-1, feature_width)
        feature_vector = list()
        for patch_i in range(patches_magnitudes.shape[0]):
            bins = np.digitize(
                patches_angles[patch_i], np.arange(0, 360, 360 // n_bins))
            bin_vector = np.zeros(n_bins)
            for bin_i in range(0, n_bins):
                mask = np.array(bins == bin_i).flatten()
                bin_vector[bin_i] = np.sum(
                    patches_magnitudes[patch_i].flatten()[mask])
            feature_vector.append(bin_vector)
        feature_vector_norm = (np.array(feature_vector).flatten() /
                               np.array(feature_vector).flatten().sum())
        descriptors.append(feature_vector_norm)
    return np.array(descriptors)

def get_interest_points(image, feature_width, sigma=2, k=0.04):
    '''
    Returns a set of interest points for the input image

    Parameters
    ----------

    image: Numpy array.

    feature_width: int.

    k: float.
        Harris detector free parameter.

    Returns
    -------

    xs, ys: Numpy vectors.
        x, y coordinates of interest points.

    Description
    -----------

    1.  Compute the horizontal and vertical derivatives of the image by
        con- volving the original image with derivatives of Gaussians (3.2.3).
    2.  Compute the three images corresponding to the outer products of these
        gradients. (The matrix A is symmetric, so only three entries are
        needed.)
    3.  Convolve each of these images with a larger Gaussian.
    4.  Compute a scalar interest measure using one of the formulas discussed
        above.
    5.  Find local maxima above a certain threshold and report them as
        detected feature point locations.
    '''
    image_blurred = filters.gaussian(image, sigma)
    Iy, Ix = np.gradient(image_blurred)
    Ixx = filters.gaussian(Ix * Ix, sigma)
    Iyy = filters.gaussian(Iy * Iy, sigma)
    Ixy = filters.gaussian(Ix * Iy, sigma)
    R = Ixx * Iyy - Ixy**2 - k * (Ixx + Iyy)**2
    R_norm = (R-np.min(R))/(np.max(R)-np.min(R))
    corners = R_norm
    threshold = np.mean(R_norm)
    mask = [R_norm < threshold]
    corners[mask] = 0
    keypoints = peak_local_max(corners, threshold_rel=0.2, exclude_border=True,
                               num_peaks=2000, min_distance=feature_width//2)
    ys = keypoints[:, 0]
    xs = keypoints[:, 1]
    return xs, ys


from PySide6.QtWidgets import QListWidgetItem

class Category(QListWidgetItem):
    def __init__(self, parent=None, name="", timer_overlay=None, text_overlay=None,
                       timer_font_color="#ffff00", timer_bar_color="#ff0000",
                       text_font_color="#ffff00"):
        super().__init__(name, parent)
        self._parent = parent

        self.name = name
        self.setText(name)

        self.timer_overlay = timer_overlay
        self.text_overlay = text_overlay
        self.timer_font_color = timer_font_color
        self.timer_bar_color = timer_bar_color
        self.text_font_color = text_font_color

    def setName(self, val):
        self.name = val
        self.setText(val)

    def setTimerOverlay(self, overlay):
        self.timer_overlay = overlay

    def setTextOverlay(self, overlay):
        self.text_overlay = overlay

    def serialize(self):
        hash = {
            "name": self.name,
            "timer_overlay": self.timer_overlay,
            "text_overlay": self.text_overlay,
            "timer_font_color": self.timer_font_color,
            "timer_bar_color": self.timer_bar_color,
            "text_font_color": self.text_font_color
        }
        return hash


#!/usr/bin/env python

import os
import json
import cairo

ROOT_DIR = "/Users/nst/Library/Application Support/PulsarRuns"
PULSAR_DIR = "pulsar"
PULSAR_PATH = os.path.sep.join([ROOT_DIR, PULSAR_DIR])

CANVAS_W = 1000
CANVAS_H = 4200

MAX_ACTIVITY_H = 100.0

TOP_MARGIN_H = 50.0

LEFT_MARGIN_W = 50.0
LEFT_FLAT_ZONE_W = 50.0

RIGHT_MARGIN_W = 50.0
RIGHT_FLAT_ZONE_W = 50.0
RIGHT_TEXT_ZONE_W = 460.0

ACTIVITIES_OFFSET_H = 20.0

GRAY_COLOR = (0.5, 0.5, 0.5)
WHITE_COLOR = (1,1,1)
BLACK_COLOR = (0,0,0)
CLEAR_COLOR = (0,0,0,0) # transparent black

def sort_activities(a):
    low = a["lowest_relative_altitude"]
    high = a["highest_relative_altitude"]

    return low if abs(low) > high else high

if __name__ == "__main__":

    # check that relevant directories exist

    for p in [ROOT_DIR, PULSAR_PATH]:
        if not os.path.exists(p):
            print("--" + p + "is missing")
            sys.exit(1)

    # read athlete

    with open(os.path.sep.join([ROOT_DIR, "athlete.json"])) as f:
        athlete = json.load(f)

    # read activities in /pulsar/ directory

    activities_paths = [os.path.sep.join([PULSAR_PATH, f]) for f in os.listdir(PULSAR_PATH) if os.path.splitext(f)[1] == ".json"]

    activities = []

    for p in activities_paths:
        with open(p) as f:
            a = json.load(f)
            activities.append(a)

    max_distance = max([a["distances"][-1] for a in activities])

    max_altitude_delta = max([max(a["relative_altitudes"]) for a in activities])

    activities.sort(key=sort_activities, reverse=True)

    img = cairo.ImageSurface(cairo.FORMAT_ARGB32, CANVAS_W, CANVAS_H)
    c = cairo.Context(img)

    c.set_source_rgb(*BLACK_COLOR)
    c.rectangle(0, 0, CANVAS_W, CANVAS_H)
    c.fill()

    for i,a in enumerate(activities):

        # 1. compute base Y

        base_y = TOP_MARGIN_H + MAX_ACTIVITY_H + ACTIVITIES_OFFSET_H * i

        # 2. draw left gray line

        c.set_source_rgb(*GRAY_COLOR)

        c.move_to(LEFT_MARGIN_W, base_y)
        c.line_to(LEFT_MARGIN_W+LEFT_FLAT_ZONE_W, base_y)
        c.stroke()

        # 3. draw run profile in white

        low = a["lowest_relative_altitude"]
        high = a["highest_relative_altitude"]

        c.move_to(LEFT_MARGIN_W + LEFT_FLAT_ZONE_W, base_y)

        first_altitude = a["relative_altitudes"][0]

        last_x = 0.0

        for j,rel_alt in enumerate(a["relative_altitudes"]):

            y = base_y - MAX_ACTIVITY_H * (rel_alt / max_altitude_delta)

            x_ratio = a["distances"][j] / max_distance

            x = LEFT_MARGIN_W + LEFT_FLAT_ZONE_W + (CANVAS_W - LEFT_MARGIN_W - RIGHT_MARGIN_W - LEFT_FLAT_ZONE_W - RIGHT_FLAT_ZONE_W - RIGHT_TEXT_ZONE_W) * x_ratio
            last_x = x

            c.line_to(x,y)

        profile_is_more_down = abs(low) > high
        fill_color = CLEAR_COLOR if profile_is_more_down else BLACK_COLOR
        c.set_source_rgba(*fill_color)
        c.fill_preserve()

        c.set_source_rgb(*WHITE_COLOR)
        c.stroke()

        # 4. draw right gray line

        c.set_source_rgb(*GRAY_COLOR)

        c.move_to(last_x, base_y)
        c.line_to(CANVAS_W-RIGHT_MARGIN_W-RIGHT_TEXT_ZONE_W, base_y)
        c.stroke()

        # 5. draw run name

        font_size = 16

        c.set_source_rgb(*GRAY_COLOR)
        c.set_font_size(font_size)

        c.select_font_face("Helvetica", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
        c.move_to(1000 - RIGHT_MARGIN_W-RIGHT_TEXT_ZONE_W + 16, base_y)
        c.show_text(a["name"])

    # 6. draw athlete name

    c.move_to(LEFT_MARGIN_W, TOP_MARGIN_H + len(activities) * ACTIVITIES_OFFSET_H + MAX_ACTIVITY_H + 80)

    font_size = 72

    c.select_font_face("Helvetica", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
    c.set_source_rgb(*WHITE_COLOR)
    c.set_font_size(font_size)
    c.show_text(athlete["firstname"] + " " + athlete["lastname"])

    FILE_NAME = "pulsar_runs.png"
    img.write_to_png(FILE_NAME)

    os.system("open -a Safari %s" % FILE_NAME)


#!/usr/bin/env python3
from argparse import ArgumentParser
from html import unescape
from random import choice, shuffle
from time import sleep
import googleapiclient.discovery
import json
import os
import socket
import subprocess as sp
import sys

# HOW TO GET THE API_KEY -> https://developers.google.com/youtube/v3/quickstart/python
# before running this start mpv with:
# mpv --keep-open=yes --idle=yes --ytdl-format="ba*" \
#     --cache=yes --no-video --input-ipc-server=<SOCKET_PATH>

if os.system('pgrep -f "mpv --profile=radio" >/dev/null') != 0:
    sp.Popen(['mpv', '--profile=radio'],
             shell=True, stdout=sp.DEVNULL, stderr=sp.DEVNULL)

TMPDIR = os.getenv('TMPDIR', '/tmp')
PLAYLIST = os.path.join(TMPDIR, 'mpvradio.m3u')
SOCKET_PATH = os.path.join(TMPDIR, 'mpvradio')
HOME = os.getenv('HOME')
HIST = os.path.join(HOME, '.cache/yt_history')
CONF = os.path.join(HOME, '.config/.ytapi')
SAVE = os.path.join(HOME, '.config/ytcli.json')
if not os.path.exists(CONF):
    API_KEY = input('API KEY: ').strip()
    with open(CONF, 'w') as fp:
        fp.write(API_KEY)
else:
    with open(CONF, 'r') as fp:
        API_KEY = fp.readline().strip()

def run(prog: str, args: list, opts: list):
    proc = sp.Popen(
       [prog] + opts,
       stdin=sp.PIPE, stdout=sp.PIPE,
       universal_newlines=True
    )
    out = proc.communicate('\n'.join(args))
    if proc.returncode not in [0, 1]:
        sys.exit(proc.returncode)
    return [i for i in out[0].split('\n') if i]

def parse_arguments():
    parser = ArgumentParser()
    parser.add_argument('--dmenu', action='store_true')
    parser.add_argument('--fzf', action='store_true', default=True)
    parser.add_argument('-r', '--random', action='store_true')
    parser.add_argument('-l', '--history', action='store_true')
    parser.add_argument('-a', '--all', action='store_true')
    parser.add_argument('-s', '--shuffle', action='store_true',
        help='shuffle playlist')
    parser.add_argument('--long', action='store_true',
        help='Only include videos longer than 20 minutes.')
    parser.add_argument('--save', action='store_true',
        help='save playlist')
    parser.add_argument('--load', action='store_true',
        help='load playlist')
    parser.add_argument('argv', nargs='*')
    return parser.parse_args()

def load_history():
    try:
        with open(HIST, 'r') as fp:
            hist = [i.strip() for i in fp.readlines() if i][::-1]
        uniq_hist = list()
        for i in hist:
            if i not in uniq_hist:
                uniq_hist.append(i)
        return uniq_hist
    except FileNotFoundError:
        return list()

def main():
    args = parse_arguments()
    try:
        with open(SAVE, 'r') as fp:
            save = json.load(fp)
    except FileNotFoundError:
        save = dict()

    hist = load_history()
    hist_len = len(hist)
    height = str(hist_len) if hist_len <= 20 else '20'

    if args.argv:
        query = ' '.join(args.argv)
    elif argsload:
        keys = list(save.keys())
        query = run('fzf', keys, [
            '--height', height, '--prompt', 'query: ',
            '--print-query'
        ])[-1]
    elif argsdmenu:
        if argshistory and hist:
            query = run('dmenu', hist, [
                '-i', '-c', '-l', height, '-p', 'YouTube'
            ])[-1]
        else:
            query = run('dmenu', [], ['-c', '-i', '-p', 'YouTube'])[-1]
    elif hist:
        query = run('fzf', hist, [
            '--bind', 'tab:print-query',
            '--height', height, '--prompt', 'YouTube ',
            '--print-query'
        ])[-1]
    else:
        query = input('YouTube ').strip()

    with open(HIST, 'a') as fp:
        fp.write(query + '\n')

    if not argsload:
        youtube = googleapiclient.discovery.build(
            'youtube', 'v3', developerKey=API_KEY)

        # videoCategoryId='10', # Music
        request = youtube.search().list(
            q=query.replace(' ', '-'),
            type='video,playlist',
            part="id,snippet",
            safeSearch='none',
            videoDuration='long' if argslong else 'any',
            maxResults=40
        )
        response = request.execute()

        videos = dict()
        for i in response['items']:
            title = unescape(i['snippet']['title'])
            if 'playlistId' in i['id']:
                _id = i['id']['playlistId']
            else:
                _id = i['id']['videoId']
            c = 0
            _title = title
            while title in videos:
                title = f'{_title} {c}'
                c += 1
            videos[title] = _id
        keys = list(videos.keys())

    if argsload:
        videos = save[query]
        output = list(videos.keys())
    elif argsshuffle:
        shuffle(keys)
        output = keys
    elif argsrandom:
        output = [choice(keys)]
    elif argsall:
        output = keys
    elif argsdmenu:
        output = run('dmenu', keys, ['-c', '-i', '-l', '25'])
    else:
        output = run('fzf',   keys, ['-m', '--height', '25'])

    if argssave:
        save[query] = dict()
        for k in output:
            save[query][k] = videos[k]
        with open(SAVE, 'w') as fp:
            json.dump(save, fp)

    with open(PLAYLIST, "w") as fp:
        fp.write("#EXTM3U\n")
        for k in output:
            url = f'https://www.youtube.com/watch?v={videos[k]}'
            fp.write(f"#EXTINF:0, {k}\n{url}\n")

    mpv = socket.socket(socket.AF_UNIX)
    mpv.connect(SOCKET_PATH)
    cmd = {"command": ["loadlist", PLAYLIST]}
    mpv.send(json.dumps(cmd).encode('utf-8') + b'\n')
    mpv.close()

if __name__ == '__main__':
    main()


"""Module with functions to handle all ipywidget stuff"""
from __future__ import print_function
# Standard library imports
from typing import Optional, Any, Union
import time
import threading

# Third party imports
import ipywidgets
from ipywidgets import HBox
from ipywidgets import VBox
from ipywidgets_toggle_buttons import ToggleButtonsAutoSize

# Local imports
from .layouts import MAIN_VBOX_LAYOUT
from .layouts import HBOX_LAYOUT
from ..class_processes_manager import JupyterProcessesManager

class WidgetProcessesManager(VBox):

    def __init__(self, process_manager_obj : JupyterProcessesManager) -> None:
        """
        Widgets to interact with running processes

        Args:
            process_manager_obj (JupyterProcessesManager): Processes manager
        """
        super().__init__()
        self.process_manager_obj = process_manager_obj
        self.create_widget()

        threading.Thread(
            target=self._start_thread_auto_output_update,
            daemon=True,
        ).start()
        self._is_to_update_output = True

    def init_all_widgets(self) -> None:
        """"""
        self.OUTPUT_PROCESSES_CONDITIONS = ipywidgets.Output()
        self.OUTPUT = ipywidgets.Output()
        self.BUTTONS_CHOOSE_PROCESS = ToggleButtonsAutoSize()
        self.button_stop_all_processes = ipywidgets.Button(
            description='STOP ALL processes',
            button_style='warning',
            layout={"width": "200px"}
        )
        self.button_stop_all_processes.on_click(
            self._on_click_stop_all_processes)
        self.button_stop_process = ipywidgets.Button(
            description='STOP NONE process',
            button_style='warning',
            layout={"width": "200px"}
        )

        self.button_remove_process = ipywidgets.Button(
            description='Remove process',
            button_style='warning',
            layout={"width": "200px"},
            diasbled=True,
        )

        self.togbut_select_what_to_show = ToggleButtonsAutoSize(
            options=["Show process STDOUT", "Show LAST ERROR", "Show ALL ERRORs"])
        self.togbut_select_what_to_show.observe(
            self._update_output, names='value')
        self.button_update = ipywidgets.Button(
            description='FORCE UPDATE',
            button_style='success', # 'success', 'info', 'warning', 'danger' or ''
            layout={"width": "200px"}
        )
        self.button_update.on_click(self._update_output)
        self.HBOX_BUTTONS_CHOOSE_OUTPUT = HBox(layout=HBOX_LAYOUT)
        self.VBOX_CHOOSE_PROCESS = VBox()
        self.VBOX_MAIN_GUI = VBox(layout=MAIN_VBOX_LAYOUT)

    def create_widget(self) -> None:
        """"""
        self.init_all_widgets()
        # Main GUI
        self.create_vbox_main_gui()
        list_hboxes = [self.VBOX_MAIN_GUI]
        # Output to show
        list_hboxes.append(HBox([ipywidgets.HTML("
Output:
")]))
        list_hboxes.append(HBox([self.OUTPUT]))
        list_hboxes.append(HBox([ipywidgets.HTML("
")]))
        self.children = list_hboxes
        self.update_widget()

    def create_vbox_main_gui(self) -> None:
        """"""
        list_hboxes_main = []
        # OUTPUT_PROCESSES_CONDITIONS
        list_hboxes_main.append(
            HBox([self.OUTPUT_PROCESSES_CONDITIONS], layout=HBOX_LAYOUT))
        # Choose which output to show
        self._create_vbox_choose_process()
        list_hboxes_main.append(self.VBOX_CHOOSE_PROCESS)
        # Stop ALL Processes
        list_hboxes_main.append(
            HBox([
                self.button_stop_process,
                self.button_stop_all_processes,
                self.button_remove_process
            ],
            layout=HBOX_LAYOUT))
        list_hboxes_main.append(
            HBox(
                [ipywidgets.HTML("
Select what to show:
")],
                layout=HBOX_LAYOUT)
        )
        list_hboxes_main.append(
            HBox([self.togbut_select_what_to_show],
            layout=HBOX_LAYOUT))

        list_hboxes_main.append(
            HBox([self.button_update],
            layout=HBOX_LAYOUT))

        self.VBOX_MAIN_GUI.children = list_hboxes_main

    def update_widget(self) -> None:
        """"""
        self.update_choose_process()
        self.update_button_process_to_stop()

    def update_choose_process(self, *_) -> None:
        """"""
        self.BUTTONS_CHOOSE_PROCESS.options = list(
            self.process_manager_obj.dict_all_processes_by_id)
        self.BUTTONS_CHOOSE_PROCESS.observe(
            self.update_button_process_to_stop, names='value')
        self.BUTTONS_CHOOSE_PROCESS.observe(
            self._update_output, names='value')

    def update_button_process_to_stop(self, *_) -> None:
        """"""
        self.button_stop_process._click_handlers.callbacks = []
        self.button_remove_process._click_handlers.callbacks = []
        int_chosen_process = self.BUTTONS_CHOOSE_PROCESS.value
        if int_chosen_process is None:
            return None
        process_obj = \
            self.process_manager_obj.dict_all_processes_by_id[int_chosen_process]
        self.button_stop_process.description = \
            f'STOP {int_chosen_process} process'
        self.button_remove_process.description = \
            f'REMOVE {int_chosen_process} process'

        if process_obj.is_alive():
            self.button_stop_process.disabled = False
            self.button_remove_process.disabled = True
            self.button_stop_process.on_click(self._on_click_stop_process)
            self._is_to_update_output = True
        else:
            self.button_stop_process.disabled = True
            self.button_remove_process.disabled = False
            self.button_remove_process.on_click(self._on_click_remove_process)
        return None

    def _create_vbox_choose_process(self) -> None:
        """Create all widgets starting from choose process
        """
        list_hboxes = []
        list_hboxes.append(
            HBox(
                [ipywidgets.HTML("
Select process:
")],
                layout=HBOX_LAYOUT)
        )
        list_hboxes.append(
            HBox([self.BUTTONS_CHOOSE_PROCESS], layout=HBOX_LAYOUT))
        self.VBOX_CHOOSE_PROCESS.children = list_hboxes

    def _update_output(self, *_, clear_output_at_first :bool = True) -> None:
        """"""
        if clear_output_at_first:
            self.OUTPUT.clear_output(wait=True)
        output_type = self.togbut_select_what_to_show.value
        if output_type == "Show process STDOUT":
            self._show_stdout()
        elif output_type == "Show LAST ERROR":
            self._show_last_error()
        elif output_type == "Show ALL ERRORs":
            self._show_all_errors()
        else:
            raise ValueError(f"Wrong value of output type: {output_type}")

    def _on_click_stop_all_processes(self, *_) -> None:
        """"""
        self.OUTPUT.clear_output(wait=True)
        with self.OUTPUT:
            self.process_manager_obj.terminate_all_alive_processes()
            print("=" * 79)
        self._update_output(clear_output_at_first=False)
        self.update_button_process_to_stop()
        self._is_to_update_output = False

    def _on_click_stop_process(self, *_) -> None:
        """"""
        self._is_to_update_output = False
        self.OUTPUT.clear_output(wait=True)
        int_chosen_process = self.BUTTONS_CHOOSE_PROCESS.value

        if int_chosen_process is None:
            return None

        process_obj = \
            self.process_manager_obj.dict_all_processes_by_id[int_chosen_process]

        with self.OUTPUT:
            print("Stopping process: ", int_chosen_process)
            process_obj.terminate()
            print("---> Done. Process %d TERMINATED" % int_chosen_process)
            print("=" * 79)

        self._update_output(clear_output_at_first=False)
        self.update_button_process_to_stop()
        # self._is_to_update_output = False

    def _on_click_remove_process(self, *_) -> None:
        """"""
        self._is_to_update_output = False
        self.OUTPUT.clear_output(wait=True)
        int_chosen_process = self.BUTTONS_CHOOSE_PROCESS.value
        self.process_manager_obj.dict_all_processes_by_id.pop(
            int_chosen_process, None)

    def _show_stdout(self, *_) -> None:
        """"""
        int_chosen_process = self.BUTTONS_CHOOSE_PROCESS.value

        if int_chosen_process is None:
            with self.OUTPUT:
                print("No alive processes to show output")
            return None

        process_obj = \
            self.process_manager_obj.dict_all_processes_by_id[int_chosen_process]
        str_output = process_obj.get_stdout()
        with self.OUTPUT:
            # print("STDOUT:")
            print(str_output.strip())

    def _show_last_error(self, *_) -> None:
        """"""
        int_chosen_process = self.BUTTONS_CHOOSE_PROCESS.value

        if int_chosen_process is None:
            with self.OUTPUT:
                print("No alive processes to show output")
            return None

        process_obj = \
            self.process_manager_obj.dict_all_processes_by_id[int_chosen_process]
        int_errors_happened = len(process_obj.get_list_all_errors())
        with self.OUTPUT:
            print("Last Error:")
            print("ERRORs found: ", int_errors_happened)
            if int_errors_happened:
                str_last_error = process_obj.get_last_error_msg()
                print("Last error message: ")
                print(str_last_error)

    def _show_all_errors(self, *_) -> None:
        """"""
        int_chosen_process = self.BUTTONS_CHOOSE_PROCESS.value
        if int_chosen_process is None:
            with self.OUTPUT:
                print("No alive processes to show output")
            return None
        process_obj = \
            self.process_manager_obj.dict_all_processes_by_id[int_chosen_process]
        all_errors = process_obj.get_list_all_errors()
        with self.OUTPUT:
            print("All Errors:")
            print("ERRORs found: ", len(all_errors))
            for error_num, error_text in enumerate(all_errors):
                print("=" * 79)
                print("ERROR:", error_num)
                print(error_text)

    def _start_thread_auto_output_update(self) -> None:
        """"""
        while True:
            time.sleep(1)
            self.OUTPUT_PROCESSES_CONDITIONS.clear_output(wait=True)
            with self.OUTPUT_PROCESSES_CONDITIONS:
                self.process_manager_obj.print_info_about_running_processes(
                    int_max_processes_to_show=10)
            time.sleep(1)
            if self._is_to_update_output:
                self._update_output(clear_output_at_first=True)


import time
import re
import logging as log
import os

def read_stop_words(file_path):
    with open(file_path, "r", encoding="utf-8-sig") as sw:

        stop_words = set()
        for line in sw:
            stop_words.update(line.strip().split(","))
        return stop_words

def read_datafile(file_path, stop_words):
    word_counts = dict()
    with open(file_path, "r", encoding="utf-8-sig") as file:
        for line in file:
            for word in re.findall(r"\w+", line.lower()):
                if word and word not in stop_words:
                    word_counts[word] = word_counts.get(word,0) + 1
    return word_counts

def main():
    # set the number of top words to find
    k = int(input("Enter the number of top words to find: "))
    filename = log.FILENAME_16GB

    # get the file size
    file_name = "/Users/rushshah/SCU/BigData/" + filename
    file_size = os.path.getsize(file_name)
    file_size_GB = file_size / (1024 * 1024 * 1024)

    # read stop words from file
    stop_words = read_stop_words(log.FILE_STOP_WORDS)
    start_time = time.time()

    # count words from data file
    word_counts = read_datafile(file_name, stop_words)

    current_file = os.path.basename(__file__).split(".")[0]

    # print the top k frequent words
    log.get_top_words_hashmap(word_counts, k)

    # print the performance statistics
    log.print_statistics(current_file, filename, start_time, file_size_GB, None, k)

if __name__ == '__main__':
    main()


from prepare_data_IN import update, prepare
from mrsc_score import score_rsc, score_mrsc, rankDiagnostic
from trend_analysis import trend_analysis
import json
import time
from events import get_events_between
import pandas as pd

"""
Phase 1: 25 March 2020 – 14 April 2020 (21 days)
Phase 2: 15 April 2020 – 3 May 2020 (19 days)
Phase 3: 4 May 2020 – 17 May 2020 (14 days)
Phase 4: 18 May 2020 – 31 May 2020 (14 days)
Phase 5: 1 June 2020 – ongoing (0 days); scheduled to end on 30 June 2020
"""
lockdown_state = [
    {'name':'Maharashtra','start': '5/3/20', 'end':'', 'control':None},
    {'name':'Delhi','start': '5/3/20', 'end':'', 'control':None}
]

def main():
    update()
    prepare()
    metric = ['deaths', 'confirmed']
    file1 = f'../data/processed/in/{metric[0]}.csv'
    file2 = f'../data/processed/in/{metric[1]}.csv'

    #rankDiagnostic(file1, file2)

    d_cf = f'../data/processed/in/daily_confirmed.csv'
    d_df = f'../data/processed/in/daily_deceased.csv'

    daily_confirmed_df = pd.read_csv(d_cf)
    daily_deceased_df = pd.read_csv(d_df)

    metadata = None
    with open(f'../data/processed/in/{metric[0]}.json') as f:
        metadata = json.load(f)

    for state in lockdown_state:
        name = state['name']
        lockdown_date = state['start']
        lockdown_date_index = metadata['dates'].index(lockdown_date)
        end_date_index = len(metadata['dates'])

        events = get_events_between(metadata['dates'][0], metadata['dates'][-1], name)

        trend_analysis(file1, 'in', name, metric[0], metadata['dates'], events=events)
        trend_analysis(file2, 'in', name, metric[1], metadata['dates'], events=events)

        confirmed_daily = daily_confirmed_df[daily_confirmed_df['Province_State']==name]
        confirmed_daily = confirmed_daily.drop(columns=['Province_State'])
        confirmed_daily = confirmed_daily.values.tolist()[0]

        deceased_daily = daily_deceased_df[daily_deceased_df['Province_State'] == name]
        deceased_daily = deceased_daily.drop(columns=['Province_State'])
        deceased_daily = deceased_daily.values.tolist()[0]

        score_mrsc(file1, file2,
                   'Province_State',
                   name, 0,
                   lockdown_date_index,
                   end_date_index,
                   metric[0], 'in',
                   metadata['dates'], lockdown_date,
                   control_group=state['control'],
                   events=events)

        score_mrsc(file2, file1,
                   'Province_State',
                   name, 0,
                   lockdown_date_index,
                   end_date_index,
                   f'{metric[1]} cases', 'in',
                   metadata['dates'], lockdown_date,
                   control_group=state['control'],
                   events=events)

        time.sleep(5)

if __name__ == "__main__":
    main()


import pytest
import unittest
from ddt import ddt, data, unpack
from pages.courses.register_courses_page import RegisterCoursesPage
from pages.home.navigation_page import NavigationPage
from utilities.read_data import get_csv_data
from utilities.teststatus import TestStatus

@pytest.mark.usefixtures('oneTimeSetUp', 'setUp')
@ddt
class RegisterCoursesCSVDataTests(unittest.TestCase):

    @pytest.fixture(autouse=True)
    def objectSetup(self, oneTimeSetUp):
        self.courses = RegisterCoursesPage(self.driver)
        self.ts = TestStatus(self.driver)
        self.nav = NavigationPage(self.driver)

    def setUp(self) -> None:
        self.nav.navigate_to_all_courses()

    @pytest.mark.run(order=1)
    @data(*get_csv_data('testdata/testdata.csv'))
    @unpack
    def test_invalid_enrollment(self, course_name, cc_num, cc_exp, cc_vv, postal_code):
        self.courses.enter_course_name(course_name)
        self.courses.select_course_to_enroll(course_name)
        self.courses.enroll_course(num=cc_num, exp=cc_exp, cvv=cc_vv, postal_code=postal_code)
        result = self.courses.verify_enroll_failed()
        self.ts.mark_final('test_invalid_enrollment', result, 'Enrolment Failed Verification')
        self.driver.find_element_by_xpath("//a[@class='navbar-brand header-logo']").click()