import math
import time
from collections import deque

class Video_flow():
    def __init__(self, finger_list, tuple_list, video_url='', mitm_path='', streamID=-1,
                 state_transition_matrix='') -> None:
        # 指纹列表
        self.finger_list = finger_list
        # 传输使用的五元组列表
        self.tuple_list = tuple_list
        # 视频对应的URL
        self.video_url = video_url
        self.mitm_path = mitm_path
        self.state_transition_matrix = state_transition_matrix
        self.streamID = streamID

class Offline_Process():
    def __init__(self, offline_file, offline_audio_thd=700000) -> None:
        self.offline_file = offline_file
        self.offline_audio_thd = offline_audio_thd
        self.offline_chunk_list = []
        self.get_offline_finger()
        self.small_chunk_clean(self.offline_chunk_list, offline_audio_thd)

    # 获取离线指纹
    def get_offline_finger(self):
        offline_file = open(self.offline_file, mode='r', encoding='utf-8')
        offline_data = offline_file.read()
        offline_datas = offline_data.split('\n')
        streamID = -1
        for lines in offline_datas:
            if lines == '':
                continue
            vals = lines.split(',')
            video_url = vals[3]  # 视频URL
            stream_type = vals[2]  # video or audio
            finger = vals[4].split('/')[1:]  # 指纹
            tuples = vals[5].split('/')[1:]  # 五元组
            mitm_path = vals[0]  # mitm_path路径
            if stream_type == 'video':
                streamID += 1
                video_flow = Video_flow(list(map(int, finger)), tuples, video_url, mitm_path, streamID)
                self.offline_chunk_list.append(video_flow)
            if stream_type == 'audio':
                pass

    # 过滤掉指纹序列中小于阈值的块
    def small_chunk_clean(self, chunk_list, audio_thd):
        for chunk in chunk_list:
            clean_finger = []
            for val in chunk.finger_list:
                if val > audio_thd:
                    clean_finger.append(val)
            chunk.finger_list = clean_finger

class Markov_alg():
    def __init__(self, online_chunk_list, offline_file, offline_audio_thd, high_orders, high_bins_count, high_win_size,
                 low_orders, low_bins_count, low_win_size) -> None:

        self.video_chunk_size_max = 2200000.0
        self.video_chunk_size_min = offline_audio_thd
        # 获取指纹数据
        finger_data = Offline_Process(offline_file, offline_audio_thd)
        self.offline_chunk_list = finger_data.offline_chunk_list
        self.online_chunk_list = online_chunk_list

        self.high_glabal_state_transition_table = self.state_transition_calculate(high_bins_count, high_orders)
        self.high_state_transition_table = self.state_transition_table_generate(high_orders)
        self.low_glabal_state_transition_table = self.state_transition_calculate(low_bins_count, low_orders)
        self.low_state_transition_table = self.state_transition_table_generate(low_orders)

        self.pred_stream = None
        self.zero_prob = 0
        self.online_match(high_orders, high_bins_count, high_win_size, 0)
        self.streamID2video_flow_class()
        self.online_match(low_orders, low_bins_count, low_win_size, 1)
        self.streamID2video_flow_class()

    # 记录指纹库的所有转移概率
    def state_transition_calculate(self, bins_count, orders):
        global_tran_prob_table = {}
        word_count = 0
        orders += 1
        # 计算离线指纹的order阶概率转移矩阵
        index_i = -1
        bin_size = (self.video_chunk_size_max - self.video_chunk_size_min) / bins_count
        for offline_chunk in self.offline_chunk_list:
            index_i += 1
            self.offline_chunk_list[index_i].state_transition_matrix = ''
            # 用一个队列记录n阶的桶关系
            bin_relation_que = deque()
            state_transition_dict = {}
            for chunk in offline_chunk.finger_list:
                # 离线 等分分桶
                # 获得桶的编号
                if chunk >= self.video_chunk_size_max:
                    bin_index_cur = bins_count - 1
                elif chunk <= self.video_chunk_size_min:
                    bin_index_cur = 0
                else:
                    bin_index_cur = int((chunk - self.video_chunk_size_min) / bin_size)
                # 记录转移序列
                bin_relation_que.append(bin_index_cur)
                if len(bin_relation_que) < orders:
                    continue
                relation_key = ''
                for val in bin_relation_que:
                    relation_key += str(val) + '-'
                if relation_key not in state_transition_dict:
                    state_transition_dict[relation_key] = 1
                else:
                    state_transition_dict[relation_key] += 1

                # 全局概率
                if relation_key not in global_tran_prob_table:
                    global_tran_prob_table[relation_key] = 1
                else:
                    global_tran_prob_table[relation_key] += 1
                word_count += 1

                bin_relation_que.popleft()
            self.offline_chunk_list[index_i].state_transition_matrix = state_transition_dict

        for transition, transition_prob in global_tran_prob_table.items():
            prob = math.log(word_count / (transition_prob + 1))
            global_tran_prob_table[transition] = prob
        return global_tran_prob_table

    # 生成指纹库的转移概率表
    def state_transition_table_generate(self, orders):
        state_transition_table = {}
        for offline_chunk in self.offline_chunk_list:
            cur_state_transition_dict = offline_chunk.state_transition_matrix
            off_chunk_len = len(offline_chunk.finger_list)
            if cur_state_transition_dict == '' or len(cur_state_transition_dict) == 0:
                continue
            cur_streamID = offline_chunk.streamID
            for transition, transition_prob in cur_state_transition_dict.items():
                transition_prob = transition_prob / (off_chunk_len + 1 - orders)
                if transition in state_transition_table:
                    state_transition_table[transition].append([cur_streamID, transition_prob])
                else:
                    state_transition_table[transition] = [[cur_streamID, transition_prob]]
        return state_transition_table

    # 在线匹配
    def online_match(self, orders, bins_count, win_size, mutil_order_flag):
        time_sum = 0
        time_count = 0
        orders += 1
        bin_size = (self.video_chunk_size_max - self.video_chunk_size_min) / bins_count
        # 长度太短而不参与匹配的在线指纹数量
        if mutil_order_flag == 0:
            state_transition_table = self.high_state_transition_table
            # global
            global_state_transition_table = self.high_glabal_state_transition_table
        else:
            state_transition_table = self.low_state_transition_table
            # global
            global_state_transition_table = self.low_glabal_state_transition_table

        # 马尔可夫降阶
        if mutil_order_flag == 1:
            if self.zero_prob == 0:
                return 0
        else:
            self.pred_stream = None

        online_chunk = self.online_chunk_list
        online_bin_relation_que = deque()
        online_state_transition_dict = {}
        # 长度太短而不参与匹配,+n则保证至少有n次转移
        if len(online_chunk) < orders + 1:
            return 0
        time_start = time.time()
        ##在线指纹预处理：计算转移过程
        on_chunk_count = 0
        for on_chunk in online_chunk:
            # 动态偏置
            chunk_bias = on_chunk * 0.97214644 - 1145.44281353
            if chunk_bias >= self.video_chunk_size_max:
                bin_index_cur = bins_count - 1
            elif chunk_bias <= self.video_chunk_size_min:
                bin_index_cur = 0
            else:
                bin_index_cur = int((chunk_bias - self.video_chunk_size_min) / bin_size)

            # 记录转移序列
            online_bin_relation_que.append(bin_index_cur)
            if len(online_bin_relation_que) < orders:
                continue
            relation_key = ''
            for val in online_bin_relation_que:
                relation_key += str(val) + '-'
            if relation_key not in online_state_transition_dict:
                online_state_transition_dict[relation_key] = 1
            else:
                online_state_transition_dict[relation_key] += 1
            online_bin_relation_que.popleft()
            # 控制在线匹配指纹的长度
            on_chunk_count += 1
            if on_chunk_count >= win_size:
                break

        # 计算最大概率值对应的streamID
        state_dict = {}
        for on_key, on_val in online_state_transition_dict.items():
            if on_key in state_transition_table:
                for val in state_transition_table[on_key]:
                    if val[0] in state_dict:
                        # state_dict[val[0]] += on_val * val[1]
                        # state_dict[val[0]] +=on_val*val[1]*global_state_transition_table[on_key]
                        state_dict[val[0]] += on_val * val[1] + global_state_transition_table[on_key] * 100
                    else:
                        # state_dict[val[0]] = on_val * val[1]
                        # state_dict[val[0]]=on_val*val[1]*global_state_transition_table[on_key]
                        state_dict[val[0]] = on_val * val[1] + global_state_transition_table[on_key] * 100

        max_prob = 0
        target_id = -1
        if len(state_dict) == 0:
            self.zero_prob = 1
        else:
            for stream_id, prob in state_dict.items():
                if prob > max_prob:
                    max_prob = prob
                    target_id = stream_id
        self.pred_stream = target_id
        time_end = time.time()
        time_sum += time_end - time_start
        time_count += 1

    # 在线匹配过程中streamID与指纹库类的对应关系
    def streamID2video_flow_class(self):
        if self.pred_stream == -1:
            return 0
        for offline_chunk in self.offline_chunk_list:
            if offline_chunk.streamID == self.pred_stream:
                self.pred_stream = offline_chunk


# populate.py
from mongoengine import connect, Document, EmbeddedDocument
from mongoengine.fields import EmbeddedDocumentField, StringField, ListField, IntField, DateTimeField
from datetime import datetime

connect('senderos', host='mongo')

class Comentarios(EmbeddedDocument):
    contenido = StringField(required=True)
    autor = StringField(max_length=120, required=True)
    fecha = DateTimeField(default=datetime.now())

class Excursion(Document):
    nombre = StringField(max_length=120, required=True)
    descripcion = StringField(required=True)
    likes = IntField(default=0)
    visitas = IntField(default=0)
    tags = ListField(StringField(max_length=20))
    duracion = StringField(max_length=20)
    comentarios = ListField(EmbeddedDocumentField(Comentarios))
    fotos = ListField(StringField(required=True))

comentarios = [
    {
        "contenido": 'Primer comentario',
        "autor": 'Yo'
    },
    {
        "contenido": 'Otro comentario',
        "autor": 'Pepito'
    }
]

excursion = Excursion(nombre="Alhama, la ciudad de los tajos", descripcion="Famosa por sus baños árabes explotados desde tiempos romanos,\
 la ciudad de Al-hamman (Alhama), importante plaza del reino nazarí, es un símbolo de la reconquista cristiana del Reino de Granada. \
 Su historia, marcada por la presencia musulmana, se encuentra indefectiblemente enmarcada en el paraje natural de los Tajos, apareciendo,\
  tal como escribiese el poeta Teóphile Gautier, “colgada sobre una enorme roca o pico, como un nido de águila”.", duracion="2,5h",
                      comentarios=comentarios, fotos=["img/alhama1.jpg", "img/alhama2.jpg"])
excursion.save()

excursion = Excursion(nombre="Barranco de La Fonfría y pino de La Señora", descripcion="El Parque Natural de la Sierra de Baza ofrece un \
refrescante recorrido por el arroyo de Baúl que permite contemplar uno de los mayores ejemplares de pino laricio de la provincia de Granada. \
La ruta circular comienza en el Centro de Visitantes de Narváez, ascendiendo en parte por el GR-7 y tomando un desvío en la cabecera del arroyo del Baúl.",
                      comentarios=comentarios, fotos=["img/fonfria1.jpg", "img/fonfria2.jpg", "img/fonfria3.jpg", "img/fonfria4.jpg", "img/fonfria5.jpg"], duracion="7,5h")
excursion.save()

excursion = Excursion(nombre="Acequia del Toril y dólmenes de Alicún", descripcion="Este corta y sencilla ruta permite contemplar los dólmenes \
situados junto al Balneario de Alicún de las Torres y la espectacular y extraña pared mineral que ha formado una acequia de aguas termales a través\
 de los siglos. La presencia de aguas termales a una temperatura de 35 grados, así como su posición estratégica entre dos valles, ha favorecido la\
     creación de asentamientos humanos desde la Prehistoria en este lugar. Prueba de ello es  la presencia de restos del Paleolítico Superior,\
          Neolítico, Edad del Cobre y Cultura del Argar.", comentarios=comentarios,
                      fotos=["img/acequia-alicun1.jpg", "img/acequia-alicun2.jpg", "img/acequia-alicun3.jpg", "img/acequia-alicun4.jpg", "img/acequia-alicun5.jpg", ], duracion="1h")
excursion.save()

excursion = Excursion(nombre="Prueba", descripcion="ruta 1", likes=1,
                      tags=['fácil'], comentarios=comentarios, fotos=["img/foto1.png"])
excursion.save()  # Para escribir en la BD

for excursion in Excursion.objects():
    print(excursion)


from __future__ import annotations

import datetime
from datetime import timedelta
import pendulum
import pandas as pd
import numpy as np

from airflow import DAG
from airflow.decorators import task
from airflow.operators.empty import EmptyOperator
from airflow.operators.trigger_dagrun import TriggerDagRunOperator

from airflow.models import DagModel

from typing import Iterable

import json

#Permite saber la posicion de creación de un operador dado su nombre

# Info del excel
FILE_ID = '1NsPnYiHwqVLBHR8QwOXaXEj1WakJ_kko3MDg2_4U2ds'
SHEET_ID = '559945068'

URL = f'https://docs.google.com/spreadsheets/d/{FILE_ID}/export?format=xlsx&gid={SHEET_ID}'

#Info del ejecuta
info_ejecuta = pd.read_excel(URL,header=None,usecols=[0,1]).replace(np.nan,'').transpose()

header = info_ejecuta.iloc[0]
info_ejecuta.columns= header
info_ejecuta=info_ejecuta[1:]

tags_json= '{'+f'"tags": [{info_ejecuta["Tags"].to_string(index=False)}]'+'}'

tag_dict = json.loads(tags_json)
tag_list = tag_dict['tags']
tag_list.append(info_ejecuta["Grupo"].to_string(index=False))
tag_list.append(info_ejecuta["Unidad De Negocio O Transversales"].to_string(index=False))
tag_list.append(info_ejecuta["Área De Negocio O Transversales"].to_string(index=False))

#Lee el excel
#Cambia NaN a Null, ya que los NaN no son válidos en el typo json de la tabla serialized_dags
ctl_dags = pd.read_excel(URL,usecols=[3,4,5]).replace(np.nan,'')
ctl_dags = ctl_dags[ctl_dags["DAG"]!='']

#Si el dag anterior está vació por default se asigna el operador de inicio
ctl_dags["DAG Anterior"] = ctl_dags["DAG Anterior"].replace('','Inicio')

#Si el dag siguiente está vació por default se asigna el operador de fin
ctl_dags["DAG Siguiente"] = ctl_dags["DAG Siguiente"].replace('','Fin')

#Se guarda la información con el nombre de DAG como índice
params: Iterable[dict] = ctl_dags.set_index('DAG', drop=False).to_dict('records')

if info_ejecuta["Activo"][1]==True:
    with DAG(
            info_ejecuta["Ejecuta"][1],
            schedule=info_ejecuta["Schedule"][1],
            start_date=pendulum.from_format(f'{info_ejecuta["Fecha Inicio"][1]}','YYYY-MM-DD HH:mm:ss', tz="America/Mazatlan"),
            catchup=False,
            dagrun_timeout=datetime.timedelta(minutes=4),
            default_args={'owner':info_ejecuta['Owner'][1],
                            'retries':1,
                            'retry_delay':timedelta(minutes=0)},
            description= "Ejecuta una serie de dags",
            tags=tag_list

    ) as dag:

        Inicio = EmptyOperator(
                        task_id="Inicio",
                        trigger_rule="all_success",
                    )

        Fin = EmptyOperator(
                        task_id="Fin",
                        trigger_rule="all_success",
                    )

        ops_list=[]

        op_info= ("Inicio",0)
        ops_list.append(op_info)

        op_info= ("Fin",1)
        ops_list.append(op_info)

        ops=[]

        ops.append(Inicio)

        ops.append(Fin)

        i=2

        for param in params:
            op_info= (param["DAG"],i)
            ops_list.append(op_info)
            op= TriggerDagRunOperator(
                        task_id=param["DAG"],
                        trigger_dag_id=param["DAG"],
                        trigger_rule="all_success",
                        wait_for_completion=True,
                        poke_interval= 30,
                        dag=dag
                    )
            ops.append(op)
            i=i+1

        df_dags_posiciones =  pd.DataFrame.from_records(data=ops_list,columns=["DAG","Posicion"]).replace(np.nan,'')

        #Se guarda la información con el nombre de DAG como índice
        dags_posiciones: Iterable[dict] = df_dags_posiciones.set_index('DAG', drop=False).to_dict('index')

        for param in params:
            ops[dags_posiciones[param["DAG Anterior"]]["Posicion"]]>>ops[dags_posiciones[param["DAG"]]["Posicion"]]>>ops[dags_posiciones[param["DAG Siguiente"]]["Posicion"]]


from abc import ABC, abstractmethod
from collections import deque, namedtuple
import timeit

import gym
import numpy as np

import jasonpong
from q_learner import QLearner

Transition = namedtuple('Transition', ['state', 'action', 'reward', 'game_over'])

class Agent(ABC):
    def __init__(self, player, env) -> None:
        super().__init__()
        self.player = player
        self.env = env

    @abstractmethod
    def forward(self, state) -> int:
        pass

    def backwards(self, transition: Transition):
        pass

class RandomAgent(Agent):
    def forward(self, state) -> int:
        action = np.random.choice([0, 1, 2])
        return action

class QTableAgent(Agent):
    def __init__(self, player, env) -> None:
        super().__init__(player, env)
        self.gamma = 0.999
        self.learning_rate = 0.001
        self.epsilon = 0.001
        self.num_actions = env.action_space.n
        obs_space = env.observation_space
        self.board_width = int(obs_space.high[2])
        self.board_height = int(obs_space.high[3])
        board_size = self.board_width * self.board_height
        self.q = np.zeros([self.board_width, board_size, self.num_actions], dtype=np.float16)

    def forward(self, state) -> int:
        if np.random.uniform() < self.epsilon:
            action = np.random.randint(0, self.num_actions - 1)
        else:
            paddle_position = int(state[-1, self.player])
            ball_x = int(state[-1, 2])
            ball_y = int(state[-1, 3])
            ball_pos = self.encode_position(ball_x, ball_y)
            q = self.q[paddle_position, ball_pos]

            # randomly break ties
            best_actions = np.argwhere(q == np.amax(q)).flatten()
            action = np.random.choice(best_actions)
        return action

    def backwards(self, transition: Transition):
        paddle_pos = int(transition.state[0, self.player])
        ball_x = int(transition.state[0, 2])
        ball_y = int(transition.state[0, 3])
        ball_pos = self.encode_position(ball_x, ball_y)

        next_paddle_pos = int(transition.state[-1, self.player])
        next_ball_x = int(transition.state[-1, 2])
        next_ball_y = int(transition.state[-1, 3])
        next_ball_pos = self.encode_position(next_ball_x, next_ball_y)

        action = transition.action

        next_q = self.q[next_paddle_pos, next_ball_pos]
        target_q = transition.reward + self.gamma * np.max(next_q)
        prev_q = self.q[paddle_pos, ball_pos, action]
        self.q[paddle_pos, ball_pos, action] += self.learning_rate * (target_q - prev_q)
        foo = 1

    def encode_position(self, x, y) -> int:
        return self.board_width * y + x

class EpisodicQTableAgent(QTableAgent):
    def __init__(self, player, env) -> None:
        super().__init__(player, env)
        # self.epsilon = 0.001
        self.epsilon = 0.0
        self.alpha = 0.5
        self.gamma = 0.99
        self.q = np.zeros([self.board_width, self.board_width, self.board_width, self.board_height, 2, 2, self.num_actions], dtype=np.float16)
        self.visits = np.zeros([self.board_width, self.board_width, self.board_width, self.board_height, 2, 2, 1], dtype=np.uint8)
        self.transitions = []

    def forward(self, state) -> int:
        if np.random.uniform() < self.epsilon:
            action = np.random.randint(0, self.num_actions - 1)
        else:
            paddle_x, paddle_y, ball_x, ball_y, vel_x, vel_y = state[-1].astype(int)
            if vel_x < 0: vel_x = 0
            if vel_y < 0: vel_y = 0
            q = self.q[paddle_x, paddle_y, ball_x, ball_y, vel_x, vel_y]

            # randomly break ties
            best_actions = np.argwhere(q == np.amax(q)).flatten()
            action = np.random.choice(best_actions)
        return action

    def backwards(self, transition: Transition):
        self.transitions.append(transition)
        if transition.game_over:
            prev_reward = transition.reward
            for i in range(len(self.transitions) - 1, -1, -1):
                state = self.transitions[i].state[-1].astype(int)
                paddle0, paddle1, ball_x, ball_y, vel_x, vel_y = state
                if vel_x < 0: vel_x = 0
                if vel_y < 0: vel_y = 0
                action = self.transitions[i].action
                prev_q = self.q[paddle0, paddle1, ball_x, ball_y, vel_x, vel_y, action]
                visits = self.visits[paddle0, paddle1, ball_x, ball_y, vel_x, vel_y, 0]
                if i == len(self.transitions) - 1:
                    new_q = prev_reward + visits * prev_q
                    visits += 1
                    new_q /= visits
                else:
                    # visits = 1
                    new_q = (1 - self.alpha) * prev_q + self.alpha * (self.gamma * prev_reward)
                self.q[paddle0, paddle1, ball_x, ball_y, vel_x, vel_y, action] = new_q
                self.visits[paddle0, paddle1, ball_x, ball_y, vel_x, vel_y, 0] = visits
                prev_reward = self.q[paddle0, paddle1, ball_x, ball_y, vel_x, vel_y, action]
            self.transitions.clear()

class QLearnerAgent(Agent):
    def __init__(self, player, env) -> None:
        super().__init__(player, env)
        self.learning_mode = True
        # exploration_rate = 0.001
        exploration_rate = 0.0
        # Use Jason's refined backprop q learning code
        self.ql = QLearner(exploration_rate, 3)
        self.state_seq = []
        self.action_seq = []

    def forward(self, state) -> int:
        # Internally the q learner agent uses a dictionary which requires a format
        # that can be converted to a tuple as a key
        # Thus we convert the numpy array to a list -> tuple
        new_state = state.tolist()

        # For faster convergence you can have the new state just refined to the bat and ball position
        action = self.ql.select(new_state, self.learning_mode)

        # Push state and action data for training
        if self.learning_mode:
            self.state_seq.append(new_state)
            self.action_seq.append(action)

        return action

    def backwards(self, transition: Transition):
        # If its game over push the state action data into the q learner bot
        if self.state_seq and transition.reward:
            self.ql.update(self.state_seq, self.action_seq, transition.reward)
            self.reset()

    def reset(self):
        # Start each game with no information
        self.state_seq.clear()
        self.action_seq.clear()

    # Change between learning and testing modes
    def set_learn_mode(self, learning_mode):
        self.learning_mode = learning_mode

RENDER_STEP = False
RENDER_EPISODE = True

def play_game():
    # env = gym.make('JasonPong-v1')
    env = gym.make('VisualPong-v1')
    # env = gym.make('JasonPong2d-v1')
    # agents = [QTableAgent(i, env) for i in range(2)]
    # agents = [RandomAgent(i, env) for i in range(2)]
    # agents = [QTableAgent(0, env), RandomAgent(1, env)]
    # agents = [RandomAgent(0, env), QTableAgent(1, env)]
    # agents = [QLearnerAgent(i, env) for i in range(2)]
    agents = [EpisodicQTableAgent(i, env) for i in range(2)]
    obs_buffer = deque(maxlen=2)
    results = {
        'winners': [],
        'steps': []
    }
    start_time = timeit.default_timer()
    update_time = timeit.default_timer()
    update_times = deque(maxlen=100)

    for episode in range(int(1e3)):
        step = 0
        obs = env.reset()
        obs_buffer.extend([obs, obs])
        if RENDER_STEP:
            print(f'Episode {episode:,}')
            env.render()
        is_game_over = False
        while not is_game_over:
            state = np.array(obs_buffer)
            transitions = [None, None]
            for player, agent in enumerate(agents):
                action = agent.forward(state)
                next_obs, reward, is_game_over, _ = env.step(action)
                transitions[player] = Transition(state, action, reward, is_game_over)

            obs_buffer.append(next_obs)

            if is_game_over:
                transitions[0] = Transition(transitions[0].state, transitions[0].action, transitions[1].reward * -1,
                                            is_game_over)

            for player, agent in enumerate(agents):
                agent.backwards(transitions[player])

            step += 1
            if RENDER_STEP:
                env.render()
            if is_game_over:
                # calculate metrics
                results['winners'].append(env.winner)
                results['steps'].append(step)
                update_times.append(step / (timeit.default_timer() - update_time))
                update_time = timeit.default_timer()
                if RENDER_EPISODE:
                    print(f'Episode: {episode:>4,}   winner: {env.winner}   steps: {step:<5,} ',
                          f'steps/second: {np.mean(update_times):,.0f}')
                # if env.winner == 2:
                #     raise ValueError('Solved')

        if RENDER_STEP:
            print()

    steps = np.array(results['steps'])
    winners = np.array(results['winners'])
    print()
    print('run time:', timeit.default_timer() - start_time)
    print('steps mean:', np.mean(steps))
    print('steps max:', np.max(steps))
    print('winners:', np.bincount(winners))

if __name__ == '__main__':
    play_game()


import json

import google.protobuf.json_format
from fds.protobuf.stach.v2.RowOrganized_pb2 import RowOrganizedPackage

from fds.protobuf.stach.extensions.v2.RowOrganizedStachExtension import RowOrganizedStachExtension

class RowStachExtensionBuilder:

    def set_package(self, pkg):
        """
        Sets the package to the stach extension builder. The input package can be instance of Package, or
        string or object representation of the Package.

        :param pkg: Package object in string or object form.
        :return: builder instance
        """
        if (type(pkg) is RowOrganizedPackage):
            self.package = pkg
        elif (type(pkg) is str):
            package = RowOrganizedPackage()
            google.protobuf.json_format.Parse(pkg, package, ignore_unknown_fields=True, descriptor_pool=None)
            self.package = package
        else:
            pkg = json.dumps(pkg)
            package = RowOrganizedPackage()
            google.protobuf.json_format.Parse(pkg, package, ignore_unknown_fields=True, descriptor_pool=None)
            self.package = package

        return self

    def add_table(self, id, table):
        """
        Adds the table to the builder for conversion.

        :param id: id of the table.
        :param table: the inuput table object to be converted.
        :return: builder instance
        """
        if (type(table) is RowOrganizedPackage.Table):
            # we are checking if self.package is available, so that we can add the given table to the existing package.
            # if self.package is not available, we do create a package and add table to it.
            if (not hasattr(self, "package")):
                self.package = RowOrganizedPackage()
            empty_table = self.package.tables.get_or_create(id)
            empty_table.CopyFrom(table)
            return self
        elif (type(table) is str):
            pass
        else:
            table = json.dumps(table)

        tableInstance = RowOrganizedPackage.Table()
        google.protobuf.json_format.Parse(table, tableInstance, ignore_unknown_fields=True, descriptor_pool=None)

        # we are checking if self.package is available, so that we can add the given table to the existing package.
        # if self.package is not available, we do create a package and add table to it.
        if (not hasattr(self, "package")):
            self.package = RowOrganizedPackage()
        empty_table = self.package.tables.get_or_create(id)
        empty_table.CopyFrom(tableInstance)

        return self

    def get_package(self):
        """
        Returns the RowOrganizedPackage object set for the builder

        :return: RowOrganizedPackage instance
        """
        if (not hasattr(self, "package")):
            return None
        return self.package

    def build(self):
        """
        Builds and returns the instance of ColumnOrganizedStachExtension class

        """
        return RowOrganizedStachExtension(self.package)


import argparse
import os
import random
import time
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
import einops as ein

from FID.InceptionNet import model as inception_model
from FID.FID import calculate_fretchet

# Needed to download datasets on sites without ssl/https
import ssl
ssl._create_default_https_context = ssl._create_unverified_context

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset', required=True, help='cifar10 | lsun | mnist |imagenet | folder | lfw | fake')
    parser.add_argument('--dataroot', required=False, help='path to dataset')
    parser.add_argument('--workers', type=int, help='number of data loading workers', default=2)
    parser.add_argument('--batchSize', type=int, default=64, help='input batch size')
    parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network')
    parser.add_argument('--ngf', type=int, default=64)
    parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for')
    parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002')
    parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5')
    parser.add_argument('--cuda', action='store_true', help='enables cuda')
    parser.add_argument('--dry-run', action='store_true', help='check a single training cycle works')
    parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use')
    parser.add_argument('--netG', default='', help="path to netG (to continue training)")
    parser.add_argument('--outf', default='.', help='folder to output images and model checkpoints')
    parser.add_argument('--manualSeed', type=int, help='manual seed')
    parser.add_argument('--classes', default='bedroom', help='comma separated list of classes for the lsun data set')

    opt = parser.parse_args()
    print(opt)

    try:
        os.makedirs(opt.outf)
    except OSError:
        pass

    if opt.manualSeed is None:
        opt.manualSeed = random.randint(1, 10000)
    print("Random Seed: ", opt.manualSeed)
    random.seed(opt.manualSeed)
    torch.manual_seed(opt.manualSeed)

    cudnn.benchmark = True

    if torch.cuda.is_available() and not opt.cuda:
        print("WARNING: You have a CUDA device, so you should probably run with --cuda")

    if opt.dataroot is None and str(opt.dataset).lower() != 'fake':
        raise ValueError("`dataroot` parameter is required for dataset \"%s\"" % opt.dataset)

    if opt.dataset in ['imagenet', 'folder', 'lfw']:
        # folder dataset
        dataset = dset.ImageFolder(root=opt.dataroot,
                                transform=transforms.Compose([
                                    transforms.Resize(opt.imageSize),
                                    transforms.CenterCrop(opt.imageSize),
                                    transforms.ToTensor(),
                                    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                                ]))
        nc=3
    elif opt.dataset == 'lsun':
        classes = [ c + '_train' for c in opt.classes.split(',')]
        dataset = dset.LSUN(root=opt.dataroot, classes=classes,
                            transform=transforms.Compose([
                                transforms.Resize(opt.imageSize),
                                transforms.CenterCrop(opt.imageSize),
                                transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                            ]))
        nc=3
    elif opt.dataset == 'cifar10':
        dataset = dset.CIFAR10(root=opt.dataroot, download=True,
                            transform=transforms.Compose([
                                transforms.Resize(opt.imageSize),
                                transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                            ]))
        nc=3

    elif opt.dataset == 'mnist':
            dataset = dset.MNIST(root=opt.dataroot, download=True,
                            transform=transforms.Compose([
                                transforms.Resize(opt.imageSize),
                                transforms.RandomHorizontalFlip(),
                                transforms.RandomRotation(30, interpolation=transforms.InterpolationMode.BILINEAR),
                                transforms.ToTensor(),
                                transforms.Normalize((0.5,), (0.5,)),
                            ]))
            nc=1

    elif opt.dataset == 'fake':
        dataset = dset.FakeData(image_size=(3, opt.imageSize, opt.imageSize),
                                transform=transforms.ToTensor())
        nc=3

    assert dataset
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize,
                                            shuffle=True, num_workers=int(opt.workers))

    device = torch.device("cuda:0" if opt.cuda else "cpu")
    ngpu = int(opt.ngpu)
    ngf = int(opt.ngf)
    z_res = 4

    # custom weights initialization called on netG and netD
    def weights_init(m):
        classname = m.__class__.__name__
        if classname.find('Conv') != -1:
            torch.nn.init.normal_(m.weight, 0.0, 0.02)
        elif classname.find('BatchNorm') != -1:
            torch.nn.init.normal_(m.weight, 1.0, 0.02)
            torch.nn.init.zeros_(m.bias)

    class Generator(nn.Module):
        def __init__(self):
            super(Generator, self).__init__()
            self.main = nn.Sequential(
                # nn.Linear(z_res**2, z_res**2),
                # nn.Linear(z_res**2, z_res**2),

                nn.Unflatten(1, (z_res**2, 1, 1)),

                # input is Z, going into a convolution
                nn.ConvTranspose2d(     z_res**2, ngf * 8, 4, 1, 0, bias=False),
                nn.BatchNorm2d(ngf * 8),
                nn.ReLU(True),
                # state size. (ngf*8) x 4 x 4
                nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),
                nn.BatchNorm2d(ngf * 4),
                nn.ReLU(True),
                # state size. (ngf*4) x 8 x 8
                nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False),
                nn.BatchNorm2d(ngf * 2),
                nn.ReLU(True),
                # state size. (ngf*2) x 16 x 16
                nn.ConvTranspose2d(ngf * 2,     ngf, 4, 2, 1, bias=False),
                nn.BatchNorm2d(ngf),
                nn.ReLU(True),
                # state size. (ngf) x 32 x 32
                nn.ConvTranspose2d(    ngf,      nc, 4, 2, 1, bias=False),
                nn.Tanh()
                # state size. (nc) x 64 x 64
            )

        def forward(self, input):
            return self.main(input)

    netG = Generator().to(device)
    netG.apply(weights_init)
    if opt.netG != '':
        netG.load_state_dict(torch.load(opt.netG))
    # print(netG)
    print("# of parameters in G:", sum(p.numel() for p in netG.parameters() if p.requires_grad))

    inception_model = inception_model.to(device)

    criterion = nn.L1Loss()

    fixed_noise = torch.randn(opt.batchSize, z_res**2, device=device)
    real_label = 1
    fake_label = 0

    # setup optimizer
    optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))

    if opt.dry_run:
        opt.niter = 1

    r = torch.randperm(z_res**2)
    r2 = torch.randperm((z_res**2)**2)
    rand = torch.rand([z_res**2])

    for epoch in range(opt.niter):
        for i, data in enumerate(dataloader, 0):
            input = data[0].to(device)
            # input = ein.reduce(input, "b c (h i) (w j) -> b (h w)", "mean", i=opt.imageSize//z_res, j=opt.imageSize//z_res) # Avg pool to 8x8 then to z

            # # NOTE: Maybe shuffle here?
            # input = input[:,r]

            netG.zero_grad()
            # output = netG(input)

            with torch.no_grad():
                t = input.flatten(1)[:, r2]
                t = ein.reduce(t, "b (n i) -> b n", "mean", i=z_res**2)
                t = t.argsort(1)
                t = rand[t]

            output = netG(t)
            errG = criterion(output, input)
            errG.backward()
            optimizerG.step()

            if i % 100 == 0:
                print('[%d/%d][%d/%d] Loss_G: %.4f '
                    % (epoch, opt.niter, i, len(dataloader), errG.item()))
                vutils.save_image(data[0],
                        '%s/real_samples.png' % opt.outf,
                        normalize=True)
                with torch.no_grad():
                    fake = netG(fixed_noise)
                vutils.save_image(fake.detach(),
                        '%s/fake_samples_epoch_%03d_%03d.png' % (opt.outf, epoch, i),
                        normalize=True)

            if opt.dry_run:
                break
        # # Save FID
        # with torch.no_grad():
        #     if real_cpu.shape[1] == 1:
        #         real_cpu = real_cpu.expand(-1, 3, -1, -1)
        #         fake = fake.expand(-1, 3, -1, -1)
        #     fid = calculate_fretchet(real_cpu, fake, inception_model)
        # with open(f"{opt.outf}/fid.txt", "a+") as file:
        #     file.write(f"{epoch}, {i}, {round(time.time())}, {fid}\n")

        # do checkpointing
        torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (opt.outf, epoch))


from sklearn.feature_extraction.text import TfidfVectorizer

doc1 = 'She likes python';

doc2 = 'She hates python';

tfv = TfidfVectorizer();
result = tfv.fit_transform([doc1, doc2]).toarray(); # 단어를 벡터형태로

print(tfv.vocabulary_)
print(result)


import streamlit as st
import glob
from nltk.sentiment import SentimentIntensityAnalyzer
from pathlib import Path
import plotly.express as px

analyzer = SentimentIntensityAnalyzer()
filepaths = glob.glob("diary/*.txt")
filenames = []
positve_scores = []
negative_scores = []

for filepath in filepaths:
    filenames.append(Path(filepath).stem)
    with open(filepath, 'r') as file:
        text = file.read()

    score = analyzer.polarity_scores(text)
    positve_scores.append(score['pos'])
    negative_scores.append(score['neg'])

st.header("Diary Tone")
st.subheader("Positivity")
fig = px.line(x=filenames, y=positve_scores, labels={"x": "Dates", "y": "Positivity"})
st.plotly_chart(fig)
st.subheader("Negativity")
fig = px.line(x=filenames, y=negative_scores, labels={"x": "Dates", "y": "Negativity"})
st.plotly_chart(fig)


#!/usr/bin/python3
"""
Fabric script methods:
    do_pack(): packs web_static/ files into .tgz archive
    do_deploy(archive_path): deploys archive to webservers
    deploy(): do_packs && do_deploys
    do_clean(n=0): removes old versions and keeps n (or 1) newest versions only
Usage:
    fab -f 3-deploy_web_static.py do_clean:n=2 -i my_ssh_private_key -u ubuntu
"""
from fabric.api import local, env, put, run
from time import strftime
import os.path
env.hosts = ['35.229.54.225', '35.231.225.251']

def do_pack():
    """generate .tgz archive of web_static/ folder"""
    timenow = strftime("%Y%M%d%H%M%S")
    try:
        local("mkdir -p versions")
        filename = "versions/web_static_{}.tgz".format(timenow)
        local("tar -cvzf {} web_static/".format(filename))
        return filename
    except:
        return None

def do_deploy(archive_path):
    """
    Deploy archive to web server
    """
    if os.path.isfile(archive_path) is False:
        return False
    try:
        filename = archive_path.split("/")[-1]
        no_ext = filename.split(".")[0]
        path_no_ext = "/data/web_static/releases/{}/".format(no_ext)
        symlink = "/data/web_static/current"
        put(archive_path, "/tmp/")
        run("mkdir -p {}".format(path_no_ext))
        run("tar -xzf /tmp/{} -C {}".format(filename, path_no_ext))
        run("rm /tmp/{}".format(filename))
        run("mv {}web_static/* {}".format(path_no_ext, path_no_ext))
        run("rm -rf {}web_static".format(path_no_ext))
        run("rm -rf {}".format(symlink))
        run("ln -s {} {}".format(path_no_ext, symlink))
        return True
    except:
        return False

def deploy():
    archive_path = do_pack()
    if archive_path is None:
        return False
    success = do_deploy(archive_path)
    return success

def do_clean(number=0):
    if number == 0:
        number = 1
    with cd.local('./versions'):
            local("ls -lt | tail -n +{} | rev | cut -f1 -d" " | rev | \
            xargs -d '\n' rm".format(1 + number))
    with cd('/data/web_static/releases/'):
            run("ls -lt | tail -n +{} | rev | cut -f1 -d" " | rev | \
            xargs -d '\n' rm".format(1 + number))


from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import unittest

from mock import patch

from api import resource_name_converters as rnc
from api.v3 import projects_servicer
from api.v3 import converters
from api.v3.api_proto import projects_pb2
from api.v3.api_proto import project_objects_pb2
from api.v3.api_proto import issue_objects_pb2
from framework import exceptions
from framework import monorailcontext
from framework import permissions
from testing import fake
from services import service_manager

class ProjectsServicerTest(unittest.TestCase):

  def setUp(self):
    self.cnxn = fake.MonorailConnection()
    self.services = service_manager.Services(
        features=fake.FeaturesService(),
        issue=fake.IssueService(),
        project=fake.ProjectService(),
        config=fake.ConfigService(),
        user=fake.UserService(),
        template=fake.TemplateService(),
        usergroup=fake.UserGroupService())
    self.projects_svcr = projects_servicer.ProjectsServicer(
        self.services, make_rate_limiter=False)

    self.user_1 = self.services.user.TestAddUser('user_111@example.com', 111)

    self.project_1 = self.services.project.TestAddProject(
        'proj', project_id=789)
    self.template_1 = self.services.template.TestAddIssueTemplateDef(
        123, 789, 'template_1_name', content='foo bar', summary='foo')
    self.project_1_resource_name = 'projects/proj'
    self.converter = None

  def CallWrapped(self, wrapped_handler, mc, *args, **kwargs):
    self.converter = converters.Converter(mc, self.services)
    self.projects_svcr.converter = self.converter
    return wrapped_handler.wrapped(self.projects_svcr, mc, *args, **kwargs)

  def testListIssueTemplates(self):
    """We can list a project's IssueTemplates."""
    request = projects_pb2.ListIssueTemplatesRequest(
        parent=self.project_1_resource_name)
    mc = monorailcontext.MonorailContext(
        self.services, cnxn=self.cnxn, requester=self.user_1.email)
    response = self.CallWrapped(
        self.projects_svcr.ListIssueTemplates, mc, request)

    expected_issue = issue_objects_pb2.Issue(
        summary=self.template_1.summary,
        state=issue_objects_pb2.IssueContentState.Value('ACTIVE'),
        status=issue_objects_pb2.Issue.StatusValue(
            status=self.template_1.status,
            derivation=issue_objects_pb2.Derivation.Value('EXPLICIT')))
    expected_template = project_objects_pb2.IssueTemplate(
        name='projects/{}/templates/{}'.format(
            self.project_1.project_name, self.template_1.name),
        display_name=self.template_1.name,
        issue=expected_issue,
        summary_must_be_edited=False,
        template_privacy=project_objects_pb2.IssueTemplate.TemplatePrivacy
        .Value('PUBLIC'),
        default_owner=project_objects_pb2.IssueTemplate.DefaultOwner.Value(
            'DEFAULT_OWNER_UNSPECIFIED'),
        component_required=False)

    self.assertEqual(
        response,
        projects_pb2.ListIssueTemplatesResponse(templates=[expected_template]))

  @patch('project.project_helpers.GetThumbnailUrl')
  def testListProjects(self, mock_GetThumbnailUrl):
    """We can list Projects."""
    mock_GetThumbnailUrl.return_value = 'xyz'

    request = projects_pb2.ListProjectsRequest()

    mc = monorailcontext.MonorailContext(
        self.services, cnxn=self.cnxn, requester=self.user_1.email)
    response = self.CallWrapped(self.projects_svcr.ListProjects, mc, request)

    expected_project = project_objects_pb2.Project(
        name=self.project_1_resource_name,
        display_name=self.project_1.project_name,
        summary=self.project_1.summary,
        thumbnail_url='xyz')

    self.assertEqual(
        response,
        projects_pb2.ListProjectsResponse(projects=[expected_project]))


"""Common functions for chirp and depending modules."""

import itertools
import logging
import os
import subprocess

import six

LG = logging.getLogger("c4irp")

# Pyhton2 doesn't have ProcessLookupError, we just use some not to generic
# exception instead
if six.PY2:
    MyProcessLookupError = subprocess.CalledProcessError
else:
    MyProcessLookupError = ProcessLookupError  # noqa

class TransparentStrRepr(str):
    """A string implementation with a transparent repr."""

    def __repr__(self):
        """Just return the string: make repr transparent."""
        return self

def collect_processes(procs):
    """Terminate and collect output of subprocesses."""
    TR = TransparentStrRepr
    for proc in procs:
        assert isinstance(proc, subprocess.Popen)
        if proc.returncode is None:
            try:
                proc.kill()
            except MyProcessLookupError:  # pragma: no cover
                pass
        LG.debug(
            "Proc %s returncode: %s",
            proc.args,
            proc.returncode,
        )
        LG.debug(
            "Proc %s stdout: %s",
            proc.args,
            TR(proc.stdout.read().decode("UTF-8")),
        )
        LG.debug(
            "Proc %s stderr: %s",
            proc.args,
            TR(proc.stderr.read().decode("UTF-8")),
        )

def complete_config(config, base):
    """Complete a given config with values from a object.

    like: :class:`.const.Config`.

    :param config: The supplied config as either a object or a dictionary.
    :type  config: :py:class:`object` or :py:class:`dict`
    :param   base: The base object which contains the defaults.
    :type    base: object
    :rtype: :class:`.const.Config`
    """
    if config is None:
        return base
    if isinstance(config, dict):
        config_dict = config
    else:
        config_dict = {
            key: value for (key, value) in itertools.chain(
                vars(config.__class__).items(),
                vars(config).items(),
            ) if not key.startswith("_")
        }
    for (key, value) in config_dict.items():
        setattr(base, key, value)

    return base

def get_crypto_files():
    """Get default cert.pem and dh.pem.

    :rtype: list
    """
    folder = os.path.abspath(__file__).split(
        os.path.sep
    )[:-1]
    cert   = list(folder)
    dh     = list(folder)
    cert.append("cert.pem")
    dh.append("dh.pem")
    return (
        os.path.sep.join(cert),
        os.path.sep.join(dh),
    )


# -*- coding: utf-8 -*-

from django.conf import settings
from django.urls import reverse
from django.utils.six.moves.urllib.parse import urlparse
from django.utils.translation import ugettext as _

import pytest

from ..factories import AdminFactory, UserFactory

@pytest.mark.django_db
def test_login_required(django_app):
    url = reverse('mobetta:icu_file_list', kwargs={'lang_code': 'nl'})
    response = django_app.get(url, status=302)
    redirect = urlparse(response.location)
    assert redirect.path == settings.LOGIN_URL
    assert redirect.query == 'next={}'.format(url)

@pytest.mark.django_db
def test_no_permission(django_app):
    url = reverse('mobetta:icu_file_list', kwargs={'lang_code': 'nl'})
    django_app.get(url, user=UserFactory.create(), status=403)

@pytest.mark.django_db
def test_file_stats(django_app, real_icu_file):
    user = AdminFactory.create()
    url = reverse('mobetta:icu_file_list', kwargs={'lang_code': 'nl'})
    response = django_app.get(url, user=user, status=200)
    file_row = response.html.find('tr', id='file_detail_{}'.format(real_icu_file.pk))

    col_titles = ['name', 'total_messages', 'filename',
                  'created', 'edit_history', 'download']
    stats_cells = file_row.find_all('td', recursive=False)
    stats_results = dict(zip(col_titles, [cell.text for cell in list(stats_cells)]))

    assert stats_results['name'] == real_icu_file.name
    assert stats_results['total_messages'] == '2'
    assert stats_results['filename'] == real_icu_file.filepath
    assert stats_results['edit_history'] == _('View')
    assert stats_results['download'] == _('Download')

@pytest.mark.django_db
def test_file_download_link(django_app, real_icu_file):
    """
    Assert that the download link works correctly.
    """
    user = AdminFactory.create()
    url = reverse('mobetta:icu_file_list', kwargs={'lang_code': 'nl'})
    response = django_app.get(url, user=user, status=200)
    download = response.click(_('Download'))  # there should only be one entry
    assert download.status_code == 200
    assert download.content_type == 'application/json'
    assert download.content_disposition == 'attachment; filename="{}_nl.json"'.format(
        real_icu_file.name.lower()
    )
    with open(real_icu_file.filepath, 'rb') as json_file:
        expected_content = json_file.read()
    assert download.content == expected_content


import numpy as np
from numpy import linalg as LA
from scipy import optimize

if __name__ == '__main__':

    def rhs(X, A, alpha):
        return (1 - alpha) * X + alpha / np.sum(X) * np.dot(A, X)

    m = 10  # number of agents
    # A = np.identity(m)  # grade matrix, shape m x m
    A = np.full((m, m), 0.8)
    alpha = 0.1  # convergence parameter

    X0 = np.dot(A, np.full(m, 1/m))
    X = optimize.fixed_point(rhs, X0, args=(A, alpha))
    print(X)

    w, v = LA.eig(A)
    # A.X = np.sum(X)*X


from rest_framework.test import APITestCase, APIClient
from rest_framework import status
from django.urls import reverse

client = APIClient()

class GetAllPostsTest(APITestCase):

    def test_post_creation(self):

        url = reverse('get-posts')
        data = {
            'id': 7,
            'url': "/posts/7",
            'title': "New substitution rules",
            'body': "balloteelli is a foolxd",
            'created': "2020-05-15",
            'modified': "2020-05-17",
            'tags': [
                "New",
                "substitution",
                "rules"
            ],
            'userId': 8
        }
        response = client.post(url, data, format='json')
        self.assertEqual(response.status_code, 200)


from typing import List
import heapq

class Solution:
    # Does not work
    # O(N^2) time | O(1) space
    def minMeetingRooms(self, intervals: List[List[int]]) -> int:
        intervals.sort(lambda x: x[0])

        used_rooms = 0

        for i in range(len(intervals)):
            for j in range(len(intervals)):
                if intervals[j][1] <= intervals[i][0]:
                    used_rooms += 1
                    break

        return used_rooms

class Solution2:
    # O(Nlog(N)) Time | O(N) Space
    def minMeetingRooms(self, intervals: List[List[int]]) -> int:

        start_times = []
        end_times = []

        for start, end in intervals:
            start_times.append(start)
            end_times.append(end)

        start_times.sort()
        end_times.sort()

        start_ptr = 0
        end_ptr = 0
        used_rooms = 0

        while start_ptr < len(intervals):
            start_room = start_times[start_ptr]
            end_room = end_times[end_ptr]

            if start_room < end_room:
                used_rooms += 1
            else:
                end_ptr += 1

            start_ptr += 1

        return used_rooms

class Solution3:
    # O(NLog(N)) time | O(N) space
    def minMeetingRooms(self, intervals: List[List[int]]) -> int:

        # If there is no meeting to schedule then no room needs to be allocated.
        if not intervals:
            return 0

        # The heap initialization
        free_rooms = []

        # Sort the meetings in increasing order of their start time.
        intervals.sort(key=lambda x: x[0])

        # Add the first meeting. We have to give a new room to the first meeting.
        heapq.heappush(free_rooms, intervals[0][1])

        # For all the remaining meeting rooms
        for i in intervals[1:]:

            # If the room due to free up the earliest is free, assign that room to this meeting.
            if free_rooms[0] <= i[0]:
                heapq.heappop(free_rooms)

            # If a new room is to be assigned, then also we add to the heap,
            # If an old room is allocated, then also we have to add to the heap with updated end time.
            heapq.heappush(free_rooms, i[1])

        # The size of the heap tells us the minimum rooms required for all the meetings.
        return len(free_rooms)

class Solution4:
    def minMeetingRooms(self, intervals: List[List[int]]) -> int:
        sorted_intervals = sorted(intervals)
        meeting_rooms = [sorted_intervals[0]]

        for i in range(1, len(sorted_intervals)):
            conference = sorted_intervals[i]
            meeting_room_is_availble = False
            j = 0
            idx = 0
            while j < len(meeting_rooms) and not meeting_room_is_availble:
                meeting_room = meeting_rooms[j]
                if conference[0] >= meeting_room[1]:
                    idx = j
                    meeting_room_is_availble = True
                j += 1

            if meeting_room_is_availble:
                meeting_rooms[idx] = conference
            else:
                meeting_rooms.append(conference)

        return len(meeting_rooms)

if __name__ == '__main__':
    intervals = [[0, 30], [5, 10], [15, 20]]

    sol = Solution2()

    print(sol.minMeetingRooms(intervals))

    rooms = [3, 4, 1, 2, 5]

    rooms = [-1 * x for x in rooms]

    print(rooms)

    rooms.sort()
    heapq.heapify(rooms)
    print(rooms)

    heapq.heappush(rooms, 7)

    print(rooms)

    print(heapq.heappop(rooms) * -1)

    print(rooms)

    heapq.heappop(rooms)

    print(rooms)

Bloomberg MFA Questions/253. Meeting Rooms II/meeting-rooms-ii.py


from random import randint
from readchar import readkey
from readchar.key import CTRL_C

def clear():
    print("\x1bc")

clear()
print("Welcome to Hangman!")
print("Press enter to play")
readkey()

clear()

words=[
    "ready",
    "apple",
    "death",
    "five",
    "app",
    "seattle",
    "salty",
    "nord",
    "coffee"
]

word = words[randint(0, len(words) - 1)]
print("Guess the word\nReady?")

guess = 7
letters = {}

fail = False
while guess > 0:
    print ("You have", guess , "chances left")
    fail = False
    for c in word:
        if c in letters:
            print(c, end="")
        else:
            print("_", end="")
            fail = True

    print("")
    if not fail:
        clear()
        print("Congrates you win the game! The word was", word)
        print("Hit enter to exit hangman")
        readkey()
        exit()

    data = readkey()
    if data == CTRL_C:
        exit(0)

    if data in letters:
        print("Already guessed that letter.")
        guess = guess - 1

    letters[data] = True

    if data not in word:
        guess = guess - 1

if fail:
    clear()
    print("You've failed :(")
    print("The word was", word + ".")


from PIL import Image
import os
import glob

def imgcrop(input_p, xPieces, yPieces):
    filename, file_extension = os.path.splitext(input_p)
    im = Image.open(input_p)
    aoi = input_p.split('/')[-1].split('.')[0]
    imgwidth, imgheight = im.size
    height = imgheight // yPieces
    width = imgwidth // xPieces
    count = 0
    count2 = 0
    count3 = 0
    save_path = os.path.split(os.path.split(input_p)[0])[0] + "/cropped"
    if not os.path.exists(save_path+ "/"+ aoi):
        os.makedirs(save_path+ "/"+ aoi)

    for i in range(0, yPieces):
        for j in range(0, xPieces):
            box = (j * width, i * height, (j + 1) * width, (i + 1) * height)
            a = im.crop(box)
            try:
                print(save_path+"/"+ aoi + "/"+aoi+"_blob_" + str(count2) + file_extension)
                a.save(save_path+"/"+ aoi + "/"+aoi+"_blob_" + str(count2) + file_extension)
                count +=1
                count2 += 5
                if count == 7:
                    print('changing')
                    count = 0
                    print("count : ",count)
                    count3 += 1
                    count2 = 0
                    count2 += count3
                    print("count2 : ",count2)
            except:
                pass

lst = glob.glob("/home/adminspin/Music/sarvesh/validation/*/grid_*/BI_bg/*.jpeg")

for sar in lst:
    imgcrop(sar,7,5)
    print("**"*50)
    print(f"AOI :{sar.split('/')[-1].split('.')[0]} Saved successfully ")
    print("**"*50)


import pandas as pd
import csv
import os
import matplotlib.image as mpimg
from matplotlib import gridspec
from skimage import color
import numpy as np
from skimage.util import dtype

def _prepare_rgba_array(arr):
    """Check the shape of the array to be RGBA and convert it to
    floating point representation.
    """
    arr = np.asanyarray(arr)

    if arr.ndim not in [3, 4] or arr.shape[-1] != 4:
        msg = ("the input array must have a shape == (.., ..,[ ..,] 4)), "
               "got {0}".format(arr.shape))
        raise ValueError(msg)

    return dtype.img_as_float(arr)

def rgba2rgb(rgba, background=(1, 1, 1)):
    """RGBA to RGB conversion.
    Parameters
    ----------
    rgba : array_like
        The image in RGBA format, in a 3-D array of shape ``(.., .., 4)``.
    background : array_like
        The color of the background to blend the image with. A tuple
        containing 3 floats between 0 to 1 - the RGB value of the background.
    Returns
    -------
    out : ndarray
        The image in RGB format, in a 3-D array of shape ``(.., .., 3)``.
    Raises
    ------
    ValueError
        If `rgba` is not a 3-D array of shape ``(.., .., 4)``.
    References
    ----------
    .. [1] https://en.wikipedia.org/wiki/Alpha_compositing#Alpha_blending
    Examples
    --------
    >>> from skimage import color
    >>> from skimage import data
    >>> img_rgba = data.logo()
    >>> img_rgb = color.rgba2rgb(img_rgba)
    """
    arr = _prepare_rgba_array(rgba)
    if isinstance(background, tuple) and len(background) != 3:
        raise ValueError('the background must be a tuple with 3 items - the '
                         'RGB color of the background. Got {0} items.'
                         .format(len(background)))

    alpha = arr[..., -1]
    channels = arr[..., :-1]
    out = np.empty_like(channels)

    for ichan in range(channels.shape[-1]):
        out[..., ichan] = np.clip(
            (1 - alpha) * background[ichan] + alpha * channels[..., ichan],
            a_min=0, a_max=1)
    return out

def load_type_dict():
    data_path = "./data"
    types_file = "types.csv"
    types_path = os.path.join(data_path,types_file)
    type_dict = {}
    with open(types_path,"r") as f:
        reader = csv.reader(f)
        next(reader,None) #Skip the header
        for row in reader:
            type_dict[int(row[0])] = {
                "label": row[1],
                "color" : row[4]
            }
    return type_dict

def load_pokemon_dict():
    """Loads the Pokemon Dictionary into memory. This dictionary contains
    information of Pokemon Typing, stored as:

    ID: {type_01: value, type_02: value}

    """
    data_path = "./data"
    pokemon_file = "pokemon_types.csv"
    pokemon_path = os.path.join(data_path,pokemon_file)
    pokemon_dict = {}
    with open(pokemon_path,"r") as f:
        reader = csv.reader(f)
        next(reader,None) #Skip the header
        for row in reader:
            pkm_id = int(row[0])
            type_slot = int(row[2])
            pkm_type = int(row[1])
            if pkm_id not in pokemon_dict:
                pokemon_dict[pkm_id] = {"type_01": None, "type_02": None}
            if type_slot == 1:
                pokemon_dict[pkm_id]["type_01"] = pkm_type
            elif type_slot == 2:
                pokemon_dict[pkm_id]["type_02"] = pkm_type
            else:
                raise ValueError("Unexpected type slot value")
    return pokemon_dict

def load_dataframe(gen_folders):
    pkm_dict = load_pokemon_dict()
    df_dict = {"id" : [], "gen": [], "type_01": [], "type_02": [], "sprite" : []}
    sprites_folder = "./sprites/pokemon/centered-sprites/"
    for gen, max_pkm in gen_folders.items():
        gen_folder = os.path.join(sprites_folder,gen)
        for pkm_id in range(1,max_pkm+1):
            image_file = "{id}.png".format(id=pkm_id)
            image_path = os.path.join(gen_folder,image_file)
            image = mpimg.imread(image_path)
            if image.shape[2] == 4:
                image = rgba2rgb(image)
            image = color.rgb2hsv(image)
            df_dict["id"].append(pkm_id)
            df_dict["type_01"].append(pkm_dict[pkm_id]["type_01"])
            df_dict["type_02"].append(pkm_dict[pkm_id]["type_02"])
            df_dict["sprite"].append(image)
            df_dict["gen"].append(gen)
    return pd.DataFrame.from_dict(df_dict)


# -*- coding:utf-8 -*-

from beem import Steem
from beem.comment import Comment
from beem.exceptions import ContentDoesNotExistsException

from steem.settings import STEEM_HOST
from steem.markdown import SteemMarkdown
from utils.logging.logger import logger

class SteemComment:

    def __init__(self, comment=None, author_perm=None, url=None):
        self.comment = comment
        self.author_perm = author_perm
        self.url = url

    def get_author_perm(self):
        if self.author_perm is None:
            self.author_perm = "/".join(self.url.split("/")[-2:])
        return self.author_perm

    def get_comment(self):
        if self.comment is None:
            self.comment = Comment(self.get_author_perm())
        return self.comment

    def get_url(self):
        if self.url is None:
            if self.author_perm:
                self.url = u"{}/{}".format(STEEM_HOST, self.author_perm)
            else:
                c = self.get_comment()
                if c.authorperm:
                    self.url = u"{}/{}".format(STEEM_HOST, c.authorperm)
                else:
                    self.url = u"{}/@{}/{}".format(STEEM_HOST, c.author, c.permlink)

        return self.url

    def get_pic_url(self, regex=False):
        body = self.get_comment().body
        return SteemMarkdown(body).get_top_image(regex)

    def get_text_body(self):
        body = self.get_comment().body
        return SteemMarkdown(body).get_rendered_text()

    def get_compatible_markdown(self):
        body = self.get_comment().body
        return SteemMarkdown(body).get_steem_markdown()

    def get_tags(self):
        c = self.get_comment()
        if c.json_metadata and 'tags' in c.json_metadata:
            return c.json_metadata['tags']
        else:
            self.refresh()
            c = self.get_comment()
            if c.json_metadata and 'tags' in c.json_metadata:
                return c.json_metadata['tags']
        return []

    def has_tag(self, tag):
        return tag in self.get_tags()

    def has_tags(self, tags):
        if not tags or len(tags) == 0:
            return False
        for tag in tags:
            if self.has_tag(tag):
                return True
        return False

    def refresh(self):
        c = self.get_comment()
        try:
            c.refresh()
        except ContentDoesNotExistsException:
            logger.info("Failed when refresh {} with title [{}], which is probably deleted.".format(self.get_url(), c.title))
            return None

        return c

    def log(self):
        c = self.get_comment()
        logger.info("@%s | %s | %s | %s" % (c.author, c.title, self.get_url(), c['created']))

    def is_commented_by(self, account):
        commented_by = [reply['author'] for reply in self.get_comment().get_replies()]
        return account in commented_by

    def is_upvoted_by(self, account):
        voters = self.get_comment().get_curation_rewards()['active_votes'].keys()
        return account in voters

    def is_downvoted_by(self, account):
        has_downvoted = False
        for vote in self.get_comment().get_votes():
            if vote.voter == account and vote.percent < 0:
                has_downvoted = True
        return has_downvoted

    def is_comment(self):
        c = self.get_comment()
        return c["parent_author"] != ''


#!/usr/bin/env python
#-*- coding: utf-8 -*-
################################################################################
#                     - EMULATIONSTATION GAMELIST PATCH -                      #
#                                - MERGE XML -                                 #
#------------------------------------------------------------------------------#
# NORDIC POWER amiga15@outlook.fr                 0.2.0  15/05/2017-02/07/2017 #
#------------------------------------------------------------------------------#
#0.2.0 - 02/07/2017 - Support des folders                                      #
################################################################################

#IMPORT STD---------------------------------------------------------------------
import os.path
import sys
import time
import shutil
import traceback
import argparse
from os.path import basename, splitext

#IMPORT NORDICPOWER--------------------------------------------------------------
from GameListXml import *
from GameListDir import *
from GameListTools import *
from GameListPatch import *

#CONSTANTS-----------------------------------------------------------------------
VERSION='0.2.0 BETA'

#CONSTANTS ARGUMENTS LANCEMENT---------------------------------------------------
ARG_FILE_SRC='file_src'
ARG_FILE_DEST='file_dest'

#MSG LOCALISATION----------------------------------------------------------------
config=Config()

msg_local={
('MSG_ERROR_MX_EXCEPTION','FR'):u'{} : {}',
('MSG_ERROR_MX_EXCEPTION','EN'):u'{} : {}',
('MSG_ERROR_MX_GAMEXML_SRC_NOT_FOUND','FR'):u'Fichier {} non trouv\u00E9 !',
('MSG_ERROR_MX_GAMEXML_SRC_NOT_FOUND','EN'):u'file {} not found !',
('MSG_INFO_MX_GAMEXML_DEST_NOT_FOUND','FR'):u'Fichier dest {} non trouv\u00E9, copie du fichier source',
('MSG_INFO_MX_GAMEXML_DEST_NOT_FOUND','EN'):u'destination file {} not found, make a simple copy',
('MSG_ERROR_MX_GAMEXML_DEST_COPY','FR'):u'Erreur dans la copie',
('MSG_ERROR_MX_GAMEXML_DEST_COPY','EN'):u'error during the copy',
('MSG_ERROR_MX_LOADING_GAMEXML','FR'):u'Erreur dans le fichier xml {}',
('MSG_ERROR_MX_LOADING_GAMEXML','EN'):u'xml error in {}',
('MSG_INFO_MX_PRG_END','FR'):u'Fusion termin\u00E9',
('MSG_INFO_MX_PRG_END','EN'):u'Merge end'
}

#---------------------------------------------------------------------------------------------------
def get_args():
	parser = argparse.ArgumentParser(description='fusion de fichiers gamelist.xml',epilog='(C) NORDIC POWER')
	parser.add_argument(ARG_FILE_SRC)
	parser.add_argument(ARG_FILE_DEST)
	return parser.parse_args()

#---------------------------------------------------------------------------------------------------
#---------------------------------------- MAIN -----------------------------------------------------
#---------------------------------------------------------------------------------------------------
def main():

	#Test Arg
	args=get_args()

	#Init Log
	logger=Logger.logr
	Logger.setLevel(logging.DEBUG)
	Logger.add_handler_console()

	logger.info('mergexml - NordicPower - Version '+VERSION)
	#Lecture Configuration
	config.load_from_file()

	if not os.path.isfile(args.file_src):
		logger.error(msg_local.get(('MSG_ERROR_MX_GAMEXML_SRC_NOT_FOUND',config.language)).format(args.file_src))
		sys.exit(1)

	#Chargement du fichier source
	gamesList_src  = GamesList()
	try:
		gamesList_src.import_xml_file(args.file_src)
	except MyError:
		logger.error(msg_local.get(('MSG_ERROR_MX_LOADING_GAMEXML',config.language)).format(args.file_src))
		sys.exit(1)

	#Pas de fichier destination, réalisation d'une simple copie de fichiers
	if not os.path.isfile(args.file_dest):
		try:
			shutil.copyfile(args.file_src, args.file_dest)
			logger.error(msg_local.get(('MSG_INFO_MX_GAMEXML_DEST_NOT_FOUND',config.language)).format(args.file_dest))
		except:
			pass
		sys.exit(0)

	#Chargement du fichier destination
	gamesList_dest = GamesList()
	try:
		gamesList_dest.import_xml_file(args.file_dest)
	except MyError:
		logger.error(msg_local.get(('MSG_ERROR_MX_LOADING_GAMEXML',config.language)).format(args.file_dest))
		sys.exit(1)

	#Fusion des XML
	for game_XML_src in gamesList_src.get_games():

		try:
			game_XML_dest = gamesList_dest.search_game_by_path(game_XML_src.path)

			#Game trouvé, la source écrase les infos de la destination
			game_XML_dest.path = game_XML_src.path
			if game_XML_src.name!='':game_XML_dest.name = game_XML_src.name
			if game_XML_src.desc!='':game_XML_dest.desc = game_XML_src.desc
			if game_XML_src.image!='':game_XML_dest.image = game_XML_src.image
			if game_XML_src.rating!='':game_XML_dest.rating = game_XML_src.rating
			if game_XML_src.releasedate!='':game_XML_dest.releasedate = game_XML_src.releasedate
			if game_XML_src.developer!='':game_XML_dest.developer = game_XML_src.developer
			if game_XML_src.publisher!='':game_XML_dest.publisher = game_XML_src.publisher
			if game_XML_src.genre!='':game_XML_dest.genre = game_XML_src.genre
			if game_XML_src.players!='':game_XML_dest.players = game_XML_src.players
			if game_XML_src.ref_id!='':game_XML_dest.ref_id = game_XML_src.ref_id
			if game_XML_src.source!='':game_XML_dest.source = game_XML_src.source
			if game_XML_src.playcount!='':game_XML_dest.playcount = game_XML_src.playcount
			if game_XML_src.lastplayed!='':game_XML_dest.lastplayed = game_XML_src.lastplayed

			gamesList_dest.update_game(game_XML_dest)

		except MyError:
			#Game non trouvé, simple recopie
			game_XML_dest = Game()
			game_XML_dest.path = game_XML_src.path
			game_XML_dest.copy_from_game(game_XML_src)

			gamesList_dest.add_game(game_XML_dest)

	#Fusion des XML
	for folder_XML_src in gamesList_src.get_folders():

		try:
			folder_XML_dest = gamesList_dest.search_folder_by_path(folder_XML_src.path)

			#folder trouvé, la source écrase les infos de la destination
			folder_XML_dest.path = folder_XML_src.path
			if folder_XML_src.name!='':folder_XML_dest.name = folder_XML_src.name
			if folder_XML_src.desc!='':folder_XML_dest.desc = folder_XML_src.desc
			if folder_XML_src.image!='':folder_XML_dest.image = folder_XML_src.image
			if folder_XML_src.hidden!='':folder_XML_dest.hidden = folder_XML_src.hidden

			gamesList_dest.update_folder(folder_XML_dest)

		except MyError:
			#folder non trouvé, simple recopie
			folder_XML_dest = Folder()
			folder_XML_dest.path = folder_XML_src.path
			folder_XML_dest.name = folder_XML_src.name
			folder_XML_dest.desc = folder_XML_src.desc
			folder_XML_dest.image = folder_XML_src.image
			folder_XML_dest.hidden = folder_XML_src.hidden
			gamesList_dest.add_folder(folder_XML_dest)

	#STEP END - Sauvegarde si intervention sur le fichier--------------------------------------------
	if gamesList_dest.modified:

		if len(gamesList_dest.get_games())>SEUIL_SAUVEGARDE_BIGROM:
			#logger.debug(msg_local.get(('MSG_INFO_GLP_PRG_SAVE_BIGROM',config.language)).format(args.file_dest))
			gamesList_dest.save_xml_file(args.file_dest,False)
		else:
			#logger.debug(msg_local.get(('MSG_INFO_GLP_PRG_SAVE',config.language)).format(args.file_dest))
			gamesList_dest.save_xml_file(args.file_dest)

	logger.info(msg_local.get(('MSG_INFO_MX_PRG_END',config.language)))

	#Fin
	sys.exit(0)

#---------------------------------------------------------------------------------------------------
if __name__ == '__main__':
	main()


"""郵便番号から住所を取得する
http://zipcloud.ibsnet.co.jp/
"""

import json
import urllib.request

def get_address(zipcode):
    with urllib.request.urlopen('http://zipcloud.ibsnet.co.jp/api/search?zipcode=' + zipcode) as response:
        d = json.loads(response.read().decode('utf-8'))
    return '{address1} {address2} {address3}'.format(**d['results'][0])

#print(get_address('8540001'))

def test_get_address():
    actual = get_address('7830060')
    expected = '高知県 南国市 蛍が丘'
    assert actual == expected

#test_get_address()


from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults

class LoadFactOperator(BaseOperator):

    ui_color = '#F98866'

    @apply_defaults
    def __init__(self,
                 postgres_conn_id='',
                 sql='',
                 table='',
                 truncate=False,
                 *args, **kwargs):

        super(LoadFactOperator, self).__init__(*args, **kwargs)
        self.postgres_conn_id = postgres_conn_id
        self.sql = sql
        self.table = table
        self.truncate = truncate

    def execute(self, context):
        postgres = PostgresHook(postgres_conn_id=self.postgres_conn_id)

        if self.truncate:
            self.log.info(f'Truncate table {self.table}')
            postgres.run(f'TRUNCATE {self.table}')

        self.log.info(f'Load fact table {self.table}')
        postgres.run(f'INSERT INTO {self.table} {self.sql}')


import sys
from collections import deque

def bfs():
    q = deque()
    q.append([A, ''])
    while q:
        n, ans = q.popleft()
        if n == B:
            return ans
        dn = [(2 * n) % 10000, (n - 1) % 10000, ((n * 10) + (n // 1000)) % 10000, (n // 10) + ((n % 10) * 1000)]
        dans = ['D', 'S', 'L', 'R']
        for i in range(4):
            nn = dn[i]
            if nn not in visited:
                q.append([nn, ans + dans[i]])
                visited.append(nn)

T = int(sys.stdin.readline())

for _ in range(T):
    A, B = map(int, sys.stdin.readline().split())
    visited = [A]

    print(bfs())


from flask import current_app
from functools import wraps

from client import Client, ClientDisconnectedException
from state import ModuleState

def connected(f):
    """
    Decorator for functions that require a connected client.  This will
    create the client and module state and ensure that the client is
    connected.
    """
    @wraps(f)
    def decorated(*args, **kwargs):

        if current_app.client is None:

            print("* Initializing app's client and state")
            current_app.client = Client(address=current_app.config["DISTRIBUTED_SERVER_ADDRESS"],
                                        verbose=True)
            current_app.state = ModuleState(current_app.client)

        if not current_app.client.is_connected():
            # The client isn't connected, attempt to connect now
            if not current_app.client.connect():
                raise ClientDisconnectedException("Could not connect to client")

            # Since the client was just connected (possible re-connected) reset
            # the state object
            current_app.state.reset()

        return f(*args, **kwargs)
    return decorated


# -*- coding: utf-8 -*-
import scrapy
import json
from ..items import TencentItem

class TencentSpider(scrapy.Spider):
    name = 'tencent'
    allowed_domains = ['careers.tencent.com']
    def start_requests(self):
        url = 'https://careers.tencent.com/tencentcareer/api/post/Query?' \
              'pageIndex={}&pageSize=10&language=zh-cn&area=tw'
        for page in range(1, 31):
            yield scrapy.Request(
                url=url.format(page),
                callback=self.parse
            )

    def parse(self, response):
        html_dict = json.loads(response.text)
        for position in html_dict['Data']['Posts']:
            item = TencentItem()
            item['title'] = position['RecruitPostName']
            item['category'] = position['CategoryName']
            post_id = position['PostId']
            two_url = 'https://careers.tencent.com/tencentcareer/api/post/ByPostId?' \
                      'postId={}&language=zh-cn'
            yield scrapy.Request(
                url=two_url.format(post_id),
                meta={
                    'item': item,
                },
                callback=self.parse_two_html
            )

    def parse_two_html(self, response):
         html_dict = json.loads(response.text)
         item = response.meta['item']
         item['duty'] = html_dict['Data']['Responsibility']
         item['required'] = html_dict['Data']['Requirement']
         yield item


import argparse
import copy
import os
import torch
from torch.utils.data import SubsetRandomSampler
from torchvision import models
from pytorch_grad_cam import GradCAM
from pytorch_grad_cam.utils.image import show_cam_on_image
from shape_generator import classes, classes_to_string
from train import get_noedges_loader, get_nocorners_loader
from PIL import Image

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

def main(args):

    os.makedirs(args.out_dir, exist_ok=True)

    if args.model == "vgg16":
        model = models.vgg16()
        model.fc = torch.nn.Linear(512, len(classes))
        model.load_state_dict(torch.load(args.pretrained_path)["state_dict"])

    elif args.model == "resnet18":
        model = models.resnet18()
        model.fc = torch.nn.Linear(512, len(classes))
        model.load_state_dict(torch.load(args.pretrained_path)["state_dict"])

    else:
        raise Exception(f"{args.model} is not a supported model")

    target_layers = [model.layer4[-1]]
    cam = GradCAM(model=model, target_layers=target_layers, use_cuda=args.use_cuda)

    num_indices = len(get_noedges_loader(args, sampler=None)) * args.batch_size
    sampler_a = list(SubsetRandomSampler(list(range(num_indices))))
    sampler_b = copy.deepcopy(sampler_a)
    noedges_loader = get_noedges_loader(args, sampler=sampler_a)
    nocorners_loader = get_nocorners_loader(args, sampler=sampler_b)

    for images, labels in noedges_loader:
        grayscale_noedges_cam = cam(input_tensor=images, targets=None)
        for i in range(len(labels)):
            label = labels[i]
            noedges_cam = grayscale_noedges_cam[i, :]
            image = images[i].type(torch.float32)
            image = torch.permute(image, (1, 2, 0))
            image = image.detach().numpy()

            viz = show_cam_on_image(image, noedges_cam, use_rgb=True)
            im = Image.fromarray(viz)
            shape_label = classes_to_string[label.detach().item()]
            im.save(f"{args.out_dir}/{i}_noedges_label_{shape_label}.png")

        break

    for images, labels in nocorners_loader:
        grayscale_nocorners_cam = cam(input_tensor=images, targets=None)
        for i in range(len(labels)):
            label = labels[i]
            nocorners_cam = grayscale_nocorners_cam[i, :]
            image = images[i].type(torch.float32)
            image = torch.permute(image, (1, 2, 0))
            image = image.detach().numpy()

            viz = show_cam_on_image(image, nocorners_cam, use_rgb=True)
            im = Image.fromarray(viz)
            shape_label = classes_to_string[label.detach().item()]
            im.save(f"{args.out_dir}/{i}_nocorners_label_{shape_label}.png")

        break

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("--data", "-d", type=str, required=True, default="./images/shapes")
    parser.add_argument("--out-dir", "-o", type=str, required=True, default="./viz/shapes")
    parser.add_argument("--model", "-m", type=str, default="resnet18")
    parser.add_argument("--pretrained-path", "-pp", type=str, required=True)
    parser.add_argument("--num-workers", "-w", type=int, default=4)
    parser.add_argument("--batch-size", "-b", type=int, default=10)
    parser.add_argument("--use-cuda", "-c", action="store_false")
    args = parser.parse_args()

    print("*** Arguments: ***")
    print("\n".join(f'{k}={v}' for k, v in vars(args).items()))

    main(args)


import torch.nn.functional as F
import torch.nn as nn
import random
import torch
import math

def transform_batch(batch, data_queue, device):
    """
    Slow but easily understandable non-vectorized transformation for *numerical* data only.
    Inspired from SMOTE.
    :param batch: torch.tensor : The batch data to transform.
    :param data_queue: torch.tensor: The labeled data stored in the lab_memory_module object.
    :param device: torch.device : The device.
    :return: torch.Tensor: The transformed data.
    """
    full_data = torch.cat([batch, data_queue])
    transformed_batch = torch.tensor([], device=device, dtype=torch.float32)

    for i in range(len(batch)):
        # Ignore duplicates and current point
        full_data_mask = [j for j in range(len(full_data))]  # if not torch.equal(full_data[j], full_data[i])
        full_data_mask.remove(i)

        # similarities = euclidean_distances(X=batch[i].reshape(1, -1), Y=full_data[full_data_mask])[0]
        similarities = nn.CosineSimilarity()(batch[i].repeat(len(full_data_mask), 1), full_data[full_data_mask])

        closest_point_index = full_data_mask[similarities.argmax()]

        diff_vect = (full_data[closest_point_index] - batch[i]) * random.uniform(0, 1)

        transformed_batch = torch.cat([transformed_batch, (batch[i] + diff_vect).view(1, -1)])

    return transformed_batch

def transform_batch_vectorized(batch, data_queue, device, batch_size=100):
    """
    Faster but harder to understand vectorized transformation for *numerical* data only.
    Inspired from SMOTE.
    See 'transform_batch' to really understand the logic.
    :param batch: torch.tensor: The batch data to transform.
    :param data_queue: torch.tensor: The labeled data stored in the lab_memory_module object.
    :param device: torch.device: The device.
    :param batch_size: int: During computation, the batch is cut in blocs of size batch_size. If you have memory errors, reduce it.
    :return: torch.Tensor: The transformed data.
    """
    full_data = torch.cat([batch, data_queue])

    full_similarities_matrix = torch.tensor([], device=device, dtype=torch.long)
    n_batchs = math.ceil((full_data.shape[0]) / batch_size)
    batch_start_index, batch_end_index = 0, min(batch_size, len(full_data))
    for batch_index in range(n_batchs):
        batch_similarities = F.cosine_similarity(batch.unsqueeze(1), full_data[batch_start_index:batch_end_index], dim=-1)
        full_similarities_matrix = torch.cat([full_similarities_matrix, batch_similarities], dim=1)

        batch_start_index += batch_size
        batch_end_index = min((batch_end_index + batch_size), full_data.shape[0])

    full_similarities_matrix -= torch.eye(len(batch), len(full_data), device=device)  # This way, itself wont be in the most similar instances

    most_similar_indexes = full_similarities_matrix.argmax(dim=1)

    batch_diff_vect = (full_data[most_similar_indexes] - batch) * torch.rand(len(batch), device=device).view(-1, 1)

    return batch + batch_diff_vect

def smotenc_transform_batch(batch, cat_features_mask, data_queue, device, k_neighbors=5, dist='cosine'):
    """
    Slow but easily understandable non-vectorized transformation for mixed numerical and categorical data.
    Inspired from SMOTE-NC.
    :param batch: torch.tensor: The batch data to transform.
    :param cat_features_mask: Array-like object of length (x.shape[1]). Where each element is False if this column is numerical and True if categorical.
    :param data_queue: torch.tensor: The unlabeled data stored in the unlab_memory_module object.
    :param device: torch.device: The device.
    :param k_neighbors: int: The number of neighbors to consider during the transformation.
    :param dist: The distance metric to use. Choices: ['cosine', 'euclidean'].
    :return: torch.tensor: The transformed data.
    """
    full_data = torch.cat([batch, data_queue])
    transformed_batch = torch.tensor([], device=device, dtype=torch.float32)

    for i in range(len(batch)):
        full_data_mask = torch.arange(len(full_data))
        full_data_mask = full_data_mask[full_data_mask != i]

        if dist == 'cosine':
            similarities = nn.CosineSimilarity()(batch[i].repeat(len(full_data_mask), 1), full_data[full_data_mask])
            topk_res = similarities.topk(k=k_neighbors)
            topk_similar_indexes = topk_res.indices
        elif dist == 'euclidean':
            similarities = torch.cdist(batch[i].view(1, -1), full_data[full_data_mask])
            topk_res = similarities.topk(k=k_neighbors, largest=False)
            topk_similar_indexes = topk_res.indices

        closest_points_indexes = full_data_mask[topk_similar_indexes]  # indexes in masked data

        # Select a random point between the k closest
        closest_point_index = closest_points_indexes[random.randrange(k_neighbors)]

        diff_vect = (full_data[closest_point_index] - batch[i]) * torch.rand(1, device=device)
        augmented_vect = batch[i] + diff_vect  # At this point, the categorical values are wrong

        cat_columns_indexes = torch.arange(len(cat_features_mask))[cat_features_mask]
        augmented_vect[cat_columns_indexes] = full_data[closest_points_indexes][:, cat_columns_indexes].mode(0).values

        transformed_batch = torch.cat([transformed_batch, augmented_vect.view(1, -1)])

    return transformed_batch

def smotenc_transform_batch_2(batch, cat_columns_indexes, data_queue, device, k_neighbors=5, dist='cosine', batch_size=100):
    """
    Faster but harder to understand vectorized transformation for mixed numerical and categorical data.
    See 'smotenc_transform_batch' to really understand the logic.
    Inspired from SMOTE-NC.
    :param batch: torch.tensor: The batch data to transform.
    :param cat_columns_indexes: Array-like object of the indexes of the categorical columns. Only useful when transform_method='new_2'.
    :param data_queue: The unlabeled data stored in the unlab_memory_module object.
    :param device: torch.device: The device.
    :param k_neighbors: int: The number of neighbors to consider during the transformation.
    :param dist: The distance metric to use. Choices: ['cosine', 'euclidean'].
    :param batch_size: int: During computation, the batch is cut in blocs of size batch_size. If you have memory errors, reduce it.
    :return: torch.tensor: The transformed data.
    """
    full_data = torch.cat([batch, data_queue])

    full_similarities_matrix = torch.tensor([], device=device, dtype=torch.long)

    n_batchs = math.ceil((full_data.shape[0]) / batch_size)
    batch_start_index, batch_end_index = 0, min(batch_size, len(full_data))
    for batch_index in range(n_batchs):
        if dist == 'cosine':
            similarities = F.cosine_similarity(batch.unsqueeze(1), full_data[batch_start_index:batch_end_index], dim=-1)
        elif dist == 'euclidean':
            # ToDo (below is the non-vectorized code)
            # similarities = torch.cdist(batch[i].view(1, -1), full_data)
            # similarities[i] += torch.inf  # This way, itself wont be in the most similar instances
            # topk_similar_indexes = similarities.topk(k=k_neighbors, largest=False).indices
            pass

        full_similarities_matrix = torch.cat([full_similarities_matrix, similarities], dim=1)

        batch_start_index += batch_size
        batch_end_index = min((batch_end_index + batch_size), full_data.shape[0])

    full_similarities_matrix -= torch.eye(len(batch), len(full_data), device=device)  # This way, itself wont be in the most similar instances

    batch_topk_similar_indexes = full_similarities_matrix.topk(k=k_neighbors, dim=1).indices

    # Select a random point between the k closest
    batch_closest_point_index = torch.gather(batch_topk_similar_indexes, 1, torch.randint(low=0, high=k_neighbors, size=(len(batch),), device=device).view(-1, 1))
    batch_closest_point_index = batch_closest_point_index.flatten()

    batch_closest_point = full_data[batch_closest_point_index]

    batch_diff_vect = (batch_closest_point - batch) * torch.rand(len(batch), device=device).view(-1, 1)

    augmented_batch = batch + batch_diff_vect  # At this point, the categorical values are wrong, next line fixes that

    augmented_batch[:, cat_columns_indexes] = full_data[:, cat_columns_indexes.flatten()][batch_topk_similar_indexes].mode(1).values

    return augmented_batch