from __future__ import division
#from pig_util import outputSchema

from datetime import datetime, timedelta

statuscodes = {
    "failed": 'x',
    "defined": 'd',
    "holding": 'h',
    "merging": 'm',
    "pending": 'p',
    "running": 'r',
    "activated": 'a',
    "cancelled": 'c',
    "transferring": 't',
    "sent": 'e',
    "closed": 'o',
    "assigned": 'i',
                "finished": 'f',
                "starting": 's',
                "waiting": 'w',
}

def total_seconds(td):
    return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6

def toDatetime(timestr):
    split_time = timestr.split('.')
    date = datetime.strptime(split_time[0], "%Y-%m-%d %H:%M:%S")
    if len(split_time) > 1:
        date += timedelta(seconds=float('.' + split_time[1]))

    return date

# def toDatetime(timestr):
#     try:
#         return datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S.%f")
#     except ValueError:
#         return datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S")

@outputSchema('stats:tuple(PANDAID:long, jobstatus_start:chararray, jobstatus_end:chararray, path:chararray, time_start:chararray, time_end:chararray, failed:float, defined:float, holding:float, merging:float, pending:float, running:float, activated:float, cancelled:float, transferring:float, sent:float, closed:float, assigned:float, finished:float, starting:float, waiting:float)')
def make_times_path(arg):
    res = {
        "failed": 0.,
        "defined": 0.,
        "holding": 0.,
        "merging": 0.,
        "pending": 0.,
        "running": 0.,
        "activated": 0.,
        "cancelled": 0.,
        "transferring": 0.,
        "sent": 0.,
        "closed": 0.,
        "assigned": 0.,
        "finished": 0.,
        "starting": 0.,
        "waiting": 0.,
    }

    pandaid, modificationtime_last, jobstatus_last = arg[0]
    jobstatus_start = jobstatus_last
    time_start = modificationtime_last
    time_end = modificationtime_last

    modificationtime_last = toDatetime(modificationtime_last)
    for event in arg[1:]:
        pandaid, modificationtime, jobstatus = event
        time_end = modificationtime
        modificationtime = toDatetime(modificationtime)
        elapsed_time = total_seconds(modificationtime - modificationtime_last)
        res[jobstatus_last] += elapsed_time
        modificationtime_last, jobstatus_last = modificationtime, jobstatus

    path = ''.join(statuscodes[i[2]] for i in arg)

    if not '.' in time_start:
        time_start += '.000000'
    if not '.' in time_end:
        time_end += '.000000'

    return (
        pandaid,
        jobstatus_start,
        jobstatus_last,
        path,
        time_start,
        time_end,
        res["failed"] or None,
        res["defined"] or None,
        res["holding"] or None,
        res["merging"] or None,
        res["pending"] or None,
        res["running"] or None,
        res["activated"] or None,
        res["cancelled"] or None,
        res["transferring"] or None,
        res["sent"] or None,
        res["closed"] or None,
        res["assigned"] or None,
        res["finished"] or None,
        res["starting"] or None,
        res["waiting"] or None,

    )

@outputSchema('stats:tuple(PANDAID:long, jobstatus_start:chararray, jobstatus_end:chararray, path:chararray, time_start:chararray, time_end:chararray, failed:float, defined:float, holding:float, merging:float, pending:float, running:float, activated:float, cancelled:float, transferring:float, sent:float, closed:float, assigned:float, finished:float, starting:float, waiting:float)')
def mergeSummaries(arg):
    if len(arg) > 2:
        raise Exception("can only merge two records" + repr(arg))
    if len(arg) == 1:
        return arg

    start_time0, start_time1 = toDatetime(arg[0][4]), toDatetime(arg[1][4])
    if start_time0 > start_time1:
        first, second = list(arg[1]), list(arg[0])
    else:
        first, second = list(arg[0]), list(arg[1])

    for x in [first, second]:
        for i, v in enumerate(x):
            x[i] = v or 0.

    pandaid = first[0]
    jobstatus_start = first[1]
    jobstatus_last = second[2]
    path = first[3] + second[3][1:]
    time_start = first[4]
    time_end = second[5]
    failed = first[6] + second[6]
    defined = first[7] + second[7]
    holding = first[8] + second[8]
    merging = first[9] + second[9]
    pending = first[10] + second[10]
    running = first[11] + second[11]
    activated = first[12] + second[12]
    cancelled = first[13] + second[13]
    transferring = first[14] + second[14]
    sent = first[15] + second[15]
    closed = first[16] + second[16]
    assigned = first[17] + second[17]
    finished = first[18] + second[18]
    starting = first[19] + second[19]
    waiting = first[20] + second[20]

    return (
        pandaid,
        jobstatus_start,
        jobstatus_last,
        path,
        time_start,
        time_end,
        failed or None,
        defined or None,
        holding or None,
        merging or None,
        pending or None,
        running or None,
        activated or None,
        cancelled or None,
        transferring or None,
        sent or None,
        closed or None,
        assigned or None,
        finished or None,
        starting or None,
        waiting or None,
    )


W, H = list(map(int, input().split()))
if W>H:
  W,H=H,W

mod=int(1e9+7)
# calc (W+H-2)C(W-1)   1...W-1  H...H+W-2
def prodmod(a,b):
  return a*b%mod
def divmod(a):
  return powermod(a,mod-2)
def powermod(a,n):
  if n==0:
    return 1
  if n==1:
    return a%mod

  if n%2:
    return powermod(a,n//2)**2*a%mod
  else:
    return powermod(a,n//2)**2%mod

from functools import reduce
print(prodmod(reduce(prodmod, list(range(H,H+W-1))),
              divmod(reduce(prodmod, list(range(1,W)))))
     )


import sys
from argparse import ArgumentParser

import currencies

from generic import *
from parsers import *

def process_goods(retailer_price_file, market_data_file, site_data_file, site_price_file, log_file):
    """

    Parameters
    ----------
    retailer_price_file     : str or unicode
        Retailer prices file name
    market_data_file        : str or unicode
        Market prices file name
    site_data_file          : str or unicode
        File with product mapping between site and market
    site_price_file         : str or unicode
        File with prices from site
    log_file                : str or unicode
        File for logging all types of unusual situations

    Returns
    -------
        list of dicts

    """
    current_course = currencies.get_usd_course()
    retailer_data = get_retailer_data(retailer_price_file)
    market_data = get_market_prices(market_data_file)
    site_prices = get_site_prices(site_price_file)
    site_product_mapping = get_product_mapping(site_data_file)
    log = open(log_file, "w")
    print(datetime.datetime.now(), file=log)
    log.close()
    log = open(log_file, "a")
    final_list = []

    for site_product in site_prices:
        for mapping in site_product_mapping:
            if site_product.get("site_product_name") == mapping.get("site_product_name"):
                site_product["retailer_name"] = mapping.get("retailer_product_name", "")
                site_product["market_id"] = mapping.get("market_id", "")

    retailer_product_list = [item.get("product_name") for item in retailer_data]
    site_product_list = [item.get("retailer_name") for item in site_prices]

    for site_product in site_prices:
        for market_product in market_data:
            if site_product.get("site_id") == market_product.get("site_id"):
                site_product["lowest_price"] = market_product.get("lowest_price") if \
                    market_product.get("lowest_price") else 0
                site_product["second_price"] = market_product.get("second_price") if \
                    market_product.get("second_price") else 0

    for retailer_product in retailer_data:
        for site_product in site_prices:
            if retailer_product.get("product_name") == site_product.get("retailer_name"):
                final_price = get_suggested_price(site_product,
                                                  retailer_product.get("retailer_price"),
                                                  current_course, log)
                if final_price:
                    site_product["final_price"] = final_price
                    site_product["status"] = "Active"
                    site_product["availability"] = "1"
                    final_list.append(site_product)
                else:
                    print("Не удалось определить цену для {device}, необходима ручная проверка".
                          format(device=retailer_product.get("product_name")), file=log)

            for product in site_product_list:
                if product not in retailer_product_list:
                    # site_product["final_price"] = site_product["site_price"]
                    # site_product["status"] = "Inactive"
                    # site_product["availability"] = "0"
                    # final_list.append(site_product)
                    pass
            for product in retailer_product_list:
                if product not in site_product_list:
                    print("У нас не добавлен этот девайс: {device}".
                          format(device=product), file=log)
    return final_list

def main():
    arg_parser = ArgumentParser(prog="accessories_parser")
    arg_parser.add_argument("-r", "--retailer-prices",
                            help="Excel file with prices from retailer (XLSX)",
                            type=str)
    arg_parser.add_argument("-m", "--market-prices",
                            help="Excel file with market prices (CSV)",
                            type=str)
    arg_parser.add_argument("-s", "--site-prices",
                            help="File with prices from site",
                            type=str)
    arg_parser.add_argument("-M", "--mapping-data",
                            help="File with mapping site IDs and names to "
                                 "Market IDs and retailer names (CSV)",
                            type=str)
    arg_parser.add_argument("-v", "--verbose",
                            help="Sets verbosity of output",
                            action="store_true")
    arguments = arg_parser.parse_args()
    log = init_logger("accessories_parser", arguments.verbose)
    if arguments.accessories and arguments.general_price:
        log.error("Cannot work in both accessories and general price mode "
                  "at once")
        return 1

    log.info("Loading Excel file for parsing")
    return 0

if __name__ == "__main__":
    sys.exit(main())


from numpy import array
from keras.models import Sequential
from keras.layers import Dense, LSTM

# 5 * 3, 시계열 데이터 : x = 1,2,3 / y = 4
x = array([[1,2,3],[2,3,4],[3,4,5],[4,5,6],[5,6,7]])
# 5,
y = array([4,5,6,7,8])
# x shaep 5*3 > 5*3*1
x = x.reshape(x.shape[0],x.shape[1],1)

model = Sequential()
# (3, 1)의 '1'은 1개씩 잘라서 계산한다는 뜻
# [1,2,3]의 데이터를  1> 2> 3> 으로 잘라준다.
model.add(LSTM(30,activation='relu', input_shape=(3,1)))
model.add(Dense(50))
model.add(Dense(30))
model.add(Dense(10))
model.add(Dense(5))
model.add(Dense(1))

#3.훈련
model.compile(loss ='mse', optimizer ='adam', metrics=['mse'])
model.fit(x,y,epochs=100,batch_size=1)
loss, mae = model.evaluate(x,y,batch_size=1)
print(loss,mae)

x_input = array([6,7,8])
x_input = x_input.reshape(1,3,1)

y_predict = model.predict(x_input)
print(y_predict)


from marshmallow import fields

from core.models.model import ModelSchema

class Location:
    """Handles all functionality related to locations.

    A location is the primary entity for I'm Feeling Hungry. It's typically a restaurant but could
    also be another type of business or destination.
    """

    def __init__(self, id, name, categories, url, image_url, rating, reviews, price):
        self.id = id
        self.name = name
        self.categories = categories
        self.url = url
        self.image_url = image_url
        self.rating = rating
        self.reviews = reviews
        self.price = price

class LocationSchema(ModelSchema):
    """Defines the schema structure for locations.

    A location has several core attributes including its name, review rating and price tier.
    """

    id = fields.Str(
        description="The location's unique identifier.",
        example="8lQdlnHokoGzTCpR8c1PPw",
        required=True,
    )
    name = fields.Str(
        description="The location's name.", example="Salt & Straw", required=True
    )
    categories = fields.List(
        fields.Str(),
        description="The categories the location belongs to.",
        example=["Desserts"],
        required=True,
    )
    url = fields.Url(
        description="The location's search index url.",
        example="https://www.yelp.com/biz/salt-and-straw-los-angeles-4",
        required=True,
    )
    image_url = fields.Url(
        description="The location's primary image url.",
        example="https://s3-media3.fl.yelpcdn.com/bphoto/Pzjk-12kqUgMPMffP08Ekw/o.jpg",
        required=True,
    )
    rating = fields.Float(
        description="The location's overall customer rating.",
        example=4.5,
        required=True,
    )
    reviews = fields.Int(
        description="The location's total number of customer reviews.",
        example=1134,
        required=True,
    )
    price = fields.Str(
        description="The location's price tier.", example="$$", required=True
    )


import json

default_config = {
    'increment.deadlift': 10,
    'lift.light_squat': False,
    'lift.chinup': False,
    'lift.clean': False,
    'weights': [45, 25, 10, 5, 2.5]
}

class WorkoutConfig:
    def __init__(self, override_configs={}):
        self.config = {**default_config, **override_configs}

    def weight_increment(self, lift_type):
        key = f"increment.{lift_type}"
        if key not in self.config:
            return 5

        return self.config[key]

    def backoff(self, lift_name):
        key = f"backoff.{lift_name}"
        if key not in self.config:
            return False

        return self.config[key]

    def target_reps(self, lift_name):
        key = f"reps.{lift_name}"
        if key not in self.config:
            return 5

        return self.config[key]

    @property
    def min_weight_unit(self):
        weights = self.config.get('weights', None)
        if not weights:
            return 5
        return min(weights)

    def lift_enabled(self, lift_name):
        key = f"lift.{lift_name}"
        if key not in self.config:
            raise Exception(f"lift_name {lift_name} is not valid")

        return self.config[key]

    def __repr__(self):
        return f"WorkoutConfig({json.dumps(self.config)})"


# -*- coding: utf-8 -*-
"""
Created on Mon Jan 20 21:49:12 2020

@author: Guru Prasad Muppana

Slot machine learning algo.
There are three slot machines.

This algo explain the concepts of explore vs exploit.

We give epsilon,a small number to represent the perentage of exploration.

while exploring if we get a probability less than espilon, we will randomly select
a slot machines from three of the slot machines

"""
import numpy as np
import matplotlib.pyplot as plt

class SlotMachine:
    def __init__(self, m_id,init_mean):
        self.m_id = m_id # machine id
        self.runningmean = init_mean
        self.N = 0 # the number of data points for a given SlotMachine

    def pull(self):
        #return a gaussian normal value from the slot machine for this machine.
        # machine id is added to separate the points between the points for a given machines
        return np.random.randn() + self.m_id

    def update(self,x): # Update the machines mean value (a kind of score)
        self.N += 1 # increment the counter for slot machine
        self.runningmean = (1-1/self.N)*self.runningmean + (1/self.N)*x

def run_experiment(machine1, machine2, machine3, N, init_mean=10):

    # Create slot machines:
    machines = [SlotMachine(machine1,init_mean),SlotMachine(machine2,init_mean),SlotMachine(machine3,init_mean)]

    data = np.empty(N) # create N element array adn initilize with empty values. not zero values.
    x_axis = np.empty(N)

    for i in range(N): # take one sample at a time and process it.
        #Greedy
        # get the best slot machine whose mean values is hightest among three slot machines.
        j = np.argmax([m.runningmean for m in machines])

        x = machines[j].pull()
        machines[j].update(x)

        x_axis[i] = j + 1 # +1 to move the points from zero.

        data[i] = x # adding the outcome from the slot machine i

        cumulative_average = np.cumsum(data)/ (np.arange(N)+1)
        # (arange(N)+1) : an array of numbers : 1,2,3,4,.... N
        # np.cumsum(data) is partially populated array with N items. the values gets filled from 0,1,2...
        # It is running average

#    plt.scatter(x_axis,data)
#    plt.show()

    # plot the moviing average CTR (Click trough rate)
    plt.plot(cumulative_average)
    plt.plot(np.ones(N)*machine1)
    plt.plot(np.ones(N)*machine2)
    plt.plot(np.ones(N)*machine3)
    plt.xscale("log") # log scale
    plt.show()

    for machine in machines:
        print("machine id", machine.m_id)
        print("machine Running Mean", machine.runningmean)
        print("Number of trails (x)", machine.N)

    return cumulative_average

# try with 100 points, 10 points etc
if __name__ == "__main__" :
#    ca_10 = run_experiment(1,2,3,0.1,10000) # 10 %
#    ca_5 = run_experiment(1,2,3,0.05,10000) # 5%
#    ca_1 = run_experiment(1,2,3,0.01,10000) # 1%

    N=10000
    ca_10 = run_experiment(1,2,3,N,10)
    ca_5 = run_experiment(1,2,3,N,10)
    ca_1 = run_experiment(1,2,3,N,10)

    plt.plot(ca_10,label="eps = 10%")
    plt.plot(ca_5,label="eps = 5%")
    plt.plot(ca_1,label="eps = 1%")
    plt.xscale("log") # log scale
    plt.title("Log daigram")
    plt.legend()
    plt.grid(True)
    plt.show()

    plt.plot(ca_10,label="eps = 10%")
    plt.plot(ca_5,label="eps = 5%")
    plt.plot(ca_1,label="eps = 1%")
    plt.title("Normal daigram")
    plt.legend()
    plt.grid(True)
    plt.show()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
from collections import OrderedDict
import copy

# In[2]:

mnist = np.loadtxt('./mnist.csv', delimiter=',')

# In[3]:

def train_test_split(csv_dataset): # i는 0~100 중의 하나의 수로 train_set의 비율을 나타낸다. ex) 70 => train_set 70% test_set 30%
# numpy matrix는 A[start:end:step] 형태로 표현이 가능하며, 다차원 배열의 원소 중 일부분만 접근하고 싶은 경우 파이썬 슬라이싱(slicing)과 comma(,)로 표현한다.
#먼저 label과 data(픽셀)부분을 나눈다. label은 첫번째 열의 리스트이고, 나머지 열은 data(픽셀)에 해당하므로 위의 기준으로 slicing한다. 이 때, data는 정규화를 위해 256으로 나눈다.

#그 후, 트레이닝 부분과 test부분으로 나눈다.  mnist.csv는 총 10000개의 행을 가지고 있고 0~7999는 training, 8000~9999는 test용으로 사용한다. 이 둘을 나누기 위해 위에서 구한 data와 label matrix를 8000번째 행을 기준으로 나눠 각각 저장한다.

    label = csv_dataset[:, :1]
    data = csv_dataset[:, 1:] /256

    train_X = data[:8000, :]
    train_T = label[:8000, :]
    test_X = data[8000: , ]
    test_T = label[8000:, ]

    return train_X, train_T, test_X, test_T

# In[4]:

def one_hot_encoding(T): # T is data의 label
# T는 2차원 배열로, 행은 input data의 개수이고 열은 label인 (8000,1)의 형태를 가진다.
# one hot label은 data 개수만큼의 행을 가지고 class개수만큼의 열을 가진 2차원 array 형태로 표현한다. 행의 개수는 T.shape[0], 열의 개수는 10인 matrix이고 처음엔 모든 원소 값을 0으로 초기화한다.
# 그 후 i번째 행에서 label 열만 0을 1로 바꾼다. label 열은  int(T[i])의 형태로 T의 i번째 행의값을 정수로 변환하여 접근한다.
    data = np.zeros((T.shape[0],10))
    for i in range (T.shape[0]):
        data[i][int(T[i])]=1
        i += 1
    return data #one_hot_label

# In[5]:

def Softmax(ScoreMatrix): # 제공.

    if ScoreMatrix.ndim == 2:
        temp = ScoreMatrix
        temp = temp - np.max(temp, axis=1, keepdims=True)
        y_predict = np.exp(temp) / np.sum(np.exp(temp), axis=1, keepdims=True)
        return y_predict
    temp = ScoreMatrix - np.max(ScoreMatrix, axis=0)
    expX = np.exp(temp)
    y_predict = expX / np.sum(expX)
    return y_predict

# In[6]:

def setParam_He(neuronlist):

    np.random.seed(1) # seed값 고정을 통해 input이 같으면 언제나 같은 Weight와 bias를 출력하기 위한 함수
# neuronlist의 입력은 각 레이어의 뉴런 개수,[input Layer neuron, hidden layer1 neuron, hidden layer2 neuron, output layer neuron]으로 들어온다.
# W1, b1은 input layer를 받아 forward함수를 사용할 때 쓰이는 weight parameter와 bias 값이며, W2, b2는 hidden layer1을 받아 forward 함수를 사용할 때 쓰이는 weight parameter와 bias값, 마지막으로 W3,b3은 hidden layer2를 받을 때 사용하는 값이다.
# He 방식의 초기화는  W = np.random.randn(fan_in,fan_out) / np.sqrt(fan_in/2) 이므로 W의 fan_in에는 각각의 input, fan_out에는 각각의 output을 대입한다.
#bias의 형태는 X*W와 같은 형태이어야 한다. 즉, b1은 X*W1와 같은 형태이며 이는 neuronlist[1]과 같다. 이와 마찬가지로 b2, b3의 형태를 구하면 된다.
    W1 = np.random.randn(neuronlist[0], neuronlist[1]) / np.sqrt(neuronlist[0]/2)
    W2 = np.random.randn(neuronlist[1], neuronlist[2]) / np.sqrt(neuronlist[1]/2)
    W3 = np.random.randn(neuronlist[2], neuronlist[3]) / np.sqrt(neuronlist[2]/2)
    b1 = np.zeros(neuronlist[1])
    b2 = np.zeros(neuronlist[2])
    b3 = np.zeros(neuronlist[3])

    return W1, W2, W3, b1, b2, b3

# In[7]:

class linearLayer:
    def __init__(self, W, b):
        #backward에 필요한 X, W, b 값 저장 + dW, db값 받아오기

        self.X = None
        self.W = W
        self.b = b
        self.dW = None
        self.db = None

    def forward(self, x):
        self.X = x
        #내적연산을 통한 Z값 계산
        Z = np.dot(self.X, self.W) + self.b
        return Z
    #linear score function f(X,W) = X*W + b 이다. np.dot함수를 이용하여 X와 W행렬을 곱하고 마지막으로 bias값을 더하여 score값을 구하였다.

    def backward(self, dZ):
        #백워드 함수
        #backward 함수를 이용하여 최종적으로는 input인 x의 변화량에 따른 output Z의 값 변화량인 dx( dZ/dx)를 구하려 한다. parameter로 dZ가 주어지므로 dx = dZ*W.T 이고 dx와 더불어 dW값과 db값을 update한다.
        #참고로, dW와 db는 각각 dZ/dW, dZ/db를 의미하는 것이다.
        dx = np.dot(dZ,self.W.T)
        self.dW = np.dot(self.X.T,dZ )
        self.db = np.sum(dZ,axis = 0 )

        return dx

# In[8]:

class SiLU:
    def __init__(self):
        self.Z = None # 백워드 시 사용할 로컬 변수
        self.sig = None #백워드 시 사용할 로컬 변수

    def forward(self, Z):
        #수식에 따른 forward 함수 작성
        # linear function의 결과값인 Z에 activation function인 SiLU함수를 사용한다. SiLU함수는 x∗sigmoid(x)이다. 즉, 리턴값인 Activation은 앞에서 언급한 식의 x에 Z를 대입한 값이다.
        sig = 1 / (1 + np.exp(-Z))
        Activation = Z * sig
        self.Z = Z
        self.sig = sig
        return Activation

    def backward(self, dActivation):
        #input이 Z이고, activation 함수에 의해 만들어진 결과값이 out이라 하면, backward의 최종 결과값은 input의 변화량에 따른 output의 변화량이다. 즉, dx = dout*(activation함수의 미분값)이 리턴 값이다.
        #SILU함수를 f(x)라 하고, x에 대한 sigmoid함수의 결과값을 sig라 하면 f(x) = sig *x, f'(x) = sig + x*(1-sig)*sig이다. 여기에 dActivate(Activate가 forward에서의 out이므로 의미상으로 dout에 해당한다.)를 곱한 값이 backward에서 최종적으로 리턴시키는 값인 input에 대한 미분값 dZ이다.
        dZ = dActivation*(self.sig + self.Z*(1-self.sig)*self.sig)
        return dZ

# In[9]:

class SoftmaxWithLoss(): # 제공

    def __init__(self):
        self.loss = None
        self.softmaxScore = None
        self.label = None

    def forward(self, score, one_hot_label):

        batch_size = one_hot_label.shape[0]
        self.label = one_hot_label
        self.softmaxScore = Softmax(score)
        self.loss = -np.sum(self.label * np.log(self.softmaxScore + 1e-20)) / batch_size

        return self.loss

    def backward(self, dout=1):
        batch_size = self.label.shape[0]
        dx = (self.softmaxScore - self.label) / batch_size

        return dx

# In[10]:

class dropOut :
#만약 train_flg가 TRUE인 경우, dropOu의 forward를 진행하며  x와 같은 형상으로 배열을 random하게 생성하고, 그 값이 drop_outㅁratio보다 큰 원소를 TRUE로 설정하여 self.mask에 저장한다. 해당 mask를 기존의 input인 x에 곱한다. 그 결과 dropout_ratio만큼의 뉴런만 존재하게 된다. 만약 drop_out을 실행하고 싶지 않거나 drop_out실행 후 다시 원래의 상태로 돌아가려 한다면 train_flg를 FALSE로 실행하여 1 - self.dropout_ratio(삭제한 비율)를 x에 곱하여 리턴한다.
#backward는 parameter로 out을 미분한 dout에 이미 저장되어있는 self.mask만크의 비율을 곱하여 리턴한다.
#마지막으로 killRate함수를 이용하여 외부에서 ㄴ주어진 dropout_ratio를 self.dropout_ratio에 삽입한다.
    def __init__(self, dropout_ratio = 0) :
        self.dropout_ratio = dropout_ratio
        self.mask = None

    def forward(self, x, train_flg = True):
        if train_flg:
            self.mask = np.random.rand(*x.shape) > self.dropout_ratio
            return x * self.mask
        else:
            return x * (1.0 - self.dropout_ratio)

    def backward(self, dout):
        return dout * self.mask
    def killRate(self,dropout_ratio) :
        self.dropout_ratio = dropout_ratio

# In[11]:

class ThreeLayerNet :
 #dropout을 수행할 경우를 위해 L1과 SiLU함수 사이, L2와 SiLU함수 사이에 각각 droOut 클래스 객체를 선언하였다. 또한 dropout을 사용하였을 떄의 foward를 진행하여 최종적으로 Loss를 구하는 함수인 forwardWithDropout함수를 선언하였다.

    def __init__(self, paramlist):

        W1, W2, W3, b1, b2, b3 = setParam_He(paramlist)
        self.flag = 0
        self.params = {}
        self.params['W1'] = W1
        self.params['W2'] = W2
        self.params['W3'] = W3
        self.params['b1'] = b1
        self.params['b2'] = b2
        self.params['b3'] = b3

        self.layers = OrderedDict()
        self.layers['L1'] = linearLayer(self.params['W1'], self.params['b1'])
        self.dropOut1 = dropOut()
        self.layers['SiLU1'] = SiLU()
        self.layers['L2'] = linearLayer(self.params['W2'], self.params['b2'])
        self.dropOut2 = dropOut()
        self.layers['SiLU2'] = SiLU()
        self.layers['L3'] = linearLayer(self.params['W3'], self.params['b3'])
        self.lastLayer = SoftmaxWithLoss()

    def scoreFunction(self, x):
        for layer in self.layers.values():
            # 한 줄이 best
            #과제2 document를 참고하면 각 layer에서 forward함수를 호출하여 마지막 레이어 전까지의 값을 구할 수 있었다.
            #즉, linear layer와 SiLU 레이어 둘 다 forward함수를 호출할 수 있으므로 self.layer.values를 이용해 dictionary의 값만큼  x값에 지속적으로 layer.forward를 실행하면 activation function 직전까지의 score값을 알 수 있다.
            x = layer.forward(x)
        score = x
        return score

    def forwardWithDropout(self, x, label):
    #dropout용 forward를 구하여 최종 loss값을 리턴하는 함수이다. forward 함수를 이용하여 loss를 구해가는 방향인 것은 맞지만, 히든 레이어에서 일정한 비율로 뉴런을 삭제 후 그 뉴런을 바탕으로 forward를 진행하는 forwardWithDropout함수를 기존의 forward대신 써야 하는 때가 있기 때문에 , 그 부분을 if문으로 처리하였다.

        for layer in self.layers:
            x = self.layers[layer].forward(x)
            if layer == 'L1': #layer가 L1인 경우,  다음 for문에서는 SiLU1의 forward가 호출될 것이다.그 전에 dropout1의 forward를 진행해야 한다.
                x = self.dropOut1.forward(x)
            if layer == 'L2': #layer가 L2인 경우, 다음 for문에서는 SiLU2의 forward가 호출될 것이다. 그 전에 dropout2의 forward를 진행한다.
                x = self.dropOut2.forward(x)

        return self.lastLayer.forward(x, label)

    def killRate(self, kill_n_h1, kill_n_h2):
        self.flag = 1
        self.dropOut1.killRate(kill_n_h1)
        self.dropOut2.killRate(kill_n_h2)

    def forward(self, x, label):
        if(self.flag == 1):
            return self.forwardWithDropout(x,label)
        else:
            score = self.scoreFunction(x)
            return self.lastLayer.forward(score, label)

    def accuracy(self, x, label):

        score = self.scoreFunction(x)
        score_argmax = np.argmax(score, axis=1)

        if label.ndim != 1 : #label이 one_hot_encoding 된 데이터면 if문을
            label_argmax = np.argmax(label, axis = 1)

        accuracy  = np.sum(score_argmax==label_argmax) / int(x.shape[0])

        return accuracy

    def backpropagation(self, x, label):

        #백워드 함수 작성 스코어펑션을 참고하세요 lastlayer는 ordered dictionary가 아니기 때문에 가장 먼저 호출 후 그 값을 변수로 저장한다.
        #그 이후 orderd diction의 파이썬 문법인 reversed함수를 구현하여 forward 순서의 반대 순서로 접근이 가능하다. scorefunction와 같은 방식으로 forward 대신 backward를 실행하면 그대로 backward 연산하면 backpropagation을 구현할 수있다.
        #backward 함수를 실행하면 dx값 뿐만 아니라 dW와 db의 값도 얻을 수 있으므로 해당 값들을 최종적으로 grad배열의 dW,db에 update시킨다.
        backx = self.lastLayer.backward()
        for layer in reversed(self.layers.values()):
            backx = layer.backward(backx)

        grads = {}
        grads['W1'] = self.layers['L1'].dW
        grads['b1'] = self.layers['L1'].db
        grads['W2'] = self.layers['L2'].dW
        grads['b2'] = self.layers['L2'].db
        grads['W3'] = self.layers['L3'].dW
        grads['b3'] = self.layers['L3'].db

        return grads

    def gradientdescent(self, grads, learning_rate):

        self.params['W1'] -= learning_rate*grads['W1']
        self.params['W2'] -= learning_rate*grads['W2']
        self.params['W3'] -= learning_rate*grads['W3']
        self.params['b1'] -= learning_rate*grads['b1']
        self.params['b2'] -= learning_rate*grads['b2']
        self.params['b3'] -= learning_rate*grads['b3']

# In[12]:

def batchOptimization(dataset, ThreeLayerNet, learning_rate, epoch=1000):
  #epoch가 10번째일때마다 accuracy를 체크한다.위 함수에서는 아직 train_acc_list와 test_acc_list,Loss_list가 선언이 되어있지 않았으므로 선언을 해야 한다. 위의 세 리스트를 선언 후 accuracy에 저장한다. 빈 리스트의 선언은 '리스트명' = []로 선언하고 원소 추가는 append를 사용한다.
    train_acc_list = []
    test_acc_list = []
    Loss_list = []

    for i in range(epoch+1):

#ThreeLayerNet의 forward함수는  마지막 layer 전까지 score를 구한 후 activation function을 통해 입력으로 들어온 data에 대한 Loss값을 구한다. 이를 이용하여 data_train과 data_test의 Loss를 구한다.

#이제 loss값을 backpropergation을 하여 loss값이 줄어드는 방향으로 W와 b값을 최적화 해야 한다. backpropagation함수 역시threeLayerNet에 구현되어있으므로 해당 함수를 불러올 수 있다. backpropagation함수의 결과로 세 레이어 각각의 dW와 db값을 저장한다. 그런데 이 값을 그대로 update하는 것이 아니라 learning rate만큼 곱한 정도만큼을 W와 b에 update해야 하므로 gradientdescent 함수를 불러 learning rate를 곱한 정도만큼 값을 update한다. update된 값들은 self.params에 저장이 되며 이 과정까지가 한 epoch당 optimization하는 것이 된다. 이를 epoch만큼 실행하며, 10번마다 train_acc_list, test_acc_list,Loss_list에 저장 후 출력한다. 최종 리턴값은 언급한 list에 Threelayernet을 포함한다.

#한편, dictionary 객체는 순서가 없기 때문에 key값으로 접근할 수 있으며,  key값 접근은 ' ' 형식이므로 dataset의 train_data와 one_hot_train(label을 one-hot-encoding형식으로 바꾼 것)를 dataset['train_data'], dataset['one_hot_train']형식으로 접근한다. 다른 data들 역시 위의 방식으로 접근한다. 따라서 함수를 구현하면 다음과 같은 방식으로 구현이 가능하다.
        Loss = ThreeLayerNet.forward(dataset['train_X'], dataset['one_hot_train'])
        back = ThreeLayerNet.backpropagation(dataset['train_X'], dataset['one_hot_train'])
        ThreeLayerNet.gradientdescent(back, learning_rate)

        if (i % 10 == 0) and ((i//10) < 3 or ( i//10) >97):
            train_acc = ThreeLayerNet.accuracy(dataset['train_X'], dataset['one_hot_train'])
            test_acc = ThreeLayerNet.accuracy(dataset['test_X'], dataset['one_hot_test'])
            print(i, '\t번째 Loss = ', Loss)
            print(i, '\t번째 Train_Accuracy : ', train_acc)
            print(i, '\t번째 Test_Accuracy : ', test_acc)
            train_acc_list.append(train_acc)
            test_acc_list.append(test_acc)
            Loss_list.append(Loss)

    return ThreeLayerNet, train_acc_list, test_acc_list, Loss_list

# In[13]:

def minibatch_Optimization(dataset, ThreeLayerNet, learning_rate, epoch=100, batch_size=100):

    np.random.seed(5)
    for i in range(epoch+1):
        # 코드 작성
        #train_X와 one_hot_train을 np.random.shuffle을 사용해서 섞는다. 대신 한 쌍 단위로 섞어 한 쌍의 단위처럼 셔플이 되어야 한다.
        #마찬가지로 train_acc_list, test_acc_list,Loss_list가 선언되어있지 않으므로 선언을 한다.
        train_X = dataset['train_X']
        train_T = dataset['one_hot_train']
       # train_acc_list = []
        #test_acc_list = []
        #Loss_list = []
       # set = list(zip(train_X,train_T))
        #np.random.shuffle(set)
        #train_X,train_T = zip(*set)

        #shuffle된 data를 batch단위로 나눠서 forward와 backpropagation, gradientdescent를 한다.
        #numpy.arange(Start,stop,step)함수를 사용하여 data를 batch_size만큼의 사이즈로 슬라이싱할 것이다. train_X의 행의 개수가 전체 data를 의미 하므로, 0부터 train_X.shape[0]을 이용한다. 자르는 단위는 batch_size(100)만큼 나눈다. 이 결과로 miniSize에는 minibatch로 나누어질 수 있는 배열의 개수가 나온다.
        #for문을 minibatch만큼 돌려서 batch_size만큼 train_X와 train_T 배열을 슬라이싱 한 후 각각 forward와 backward, gradient descent를 시킨다.
#int형 랜덤변수 seed값을 받아서 해당 seed값으로 data와 label에 동일하게 셔플을 진행시킨다. 즉 setseed라는 값으로 seed값을 변수로 저장시키고 그 값으로 shuffle을 진행해야 train과 label의 순서가 바뀌지 않는다.
        miniSize = np.arange(0,train_X.shape[0], batch_size)
        setseed = np.random.randint(0,10000)
        np.random.seed(setseed)
        train_X = np.random.shuffle(train_X)
        np.random.seed(setseed)
        train_T = np.random.shuffle(train_T)

        for j in miniSize:
            mini_X = dataset['train_X'][j: j + batch_size,:]
            mini_T = dataset['one_hot_train'][j: j + batch_size,:]

            Loss = ThreeLayerNet.forward(mini_X, mini_T)
            back = ThreeLayerNet.backpropagation(mini_X, mini_T)
            ThreeLayerNet.gradientdescent(back, learning_rate)

        if (i % 10 == 0) and ((i//10) < 3 or ( i//10) >8):
            train_acc = ThreeLayerNet.accuracy(dataset['train_X'], dataset['one_hot_train'])
            test_acc = ThreeLayerNet.accuracy(dataset['test_X'], dataset['one_hot_test'])
            print(i, '\t번째 Loss = ', Loss)
            print(i, '\t번째 Train_Accuracy : ', train_acc)
            print(i, '\t번째 Test_Accuracy : ', test_acc)
            #train_acc_list.append(train_acc)
            #test_acc_list.append(test_acc)
            #Loss_list.append(Loss)

    return ThreeLayerNet #, train_acc_list, test_acc_list, Loss_list

# In[14]:

def dropout_use_Optimizer(dataset, ThreeLayerNet, learning_rate, epoch, kill_n_h1 = 0.25, kill_n_h2 = 0.15):
#ThreeLayerNet.killRate함수를 이용하면 해당 클래스 내의 self.flag 값이 1로 setting이 된다. 이는 ThreeLayerNet에서 forward연산 진행 시 flag=1이므로 dropout을 사용하는 if문의 함수가 실행되어 dropout 방식을 사용하게 된다.
#그렇기 때문에 현재 함수에서는 killRate를 설정하는 것 외에는 기존의 batch optimizer방식과 동일하다.

    ThreeLayerNet.killRate(kill_n_h1, kill_n_h2) #threeLayerNet에 kill rate를 세팅한다.
    train_acc_list = []
    test_acc_list = []
    Loss_list = []
    for i in range(epoch+1):
        #코드 작성
        Loss = ThreeLayerNet.forward(dataset['train_X'], dataset['one_hot_train'])
        back = ThreeLayerNet.backpropagation(dataset['train_X'], dataset['one_hot_train'])
        ThreeLayerNet.gradientdescent(back, learning_rate)

        if (i % 10 == 0) and ((i//10) < 3 or ( i//10) >97):
            train_acc = ThreeLayerNet.accuracy(dataset['train_X'], dataset['one_hot_train'])
            test_acc = ThreeLayerNet.accuracy(dataset['test_X'], dataset['one_hot_test'])
            print(i, '\t번째 Loss = ', Loss)
            print(i, '\t번째 Train_Accuracy : ', train_acc)
            print(i, '\t번째 Test_Accuracy : ', test_acc)
            train_acc_list.append(train_acc)
            test_acc_list.append(test_acc)
            Loss_list.append(Loss)
    return ThreeLayerNet, train_acc_list, test_acc_list, Loss_list

# In[15]:

#과제 채점을 위한 세팅
train_X, train_label, test_X, test_label = train_test_split(mnist)

one_hot_train = one_hot_encoding(train_label)
one_hot_test = one_hot_encoding(test_label)

dataset = {}
dataset['train_X'] = train_X
dataset['test_X'] = test_X
dataset['one_hot_train'] = one_hot_train
dataset['one_hot_test'] = one_hot_test

neournlist = [784, 60, 30, 10]

TNN_batchOptimizer = ThreeLayerNet(neournlist)
TNN_minibatchOptimizer = copy.deepcopy(TNN_batchOptimizer)
TNN_dropout = copy.deepcopy(TNN_minibatchOptimizer)

# In[16]:

#채점은 이 것의 결과값으로 할 예정입니다.

trained_batch, tb_train_acc_list, tb_test_acc_list, tb_loss_list =  batchOptimization(dataset, TNN_batchOptimizer, 0.1, 1000)
trained_minibatch = minibatch_Optimization(dataset, TNN_minibatchOptimizer, 0.1, epoch=100, batch_size=100)
trained_dropout, td_train_acc_list, td_test_acc_list, td_loss_list = dropout_use_Optimizer(dataset, TNN_dropout, 0.1, 1000, 0.25, 0.15)

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:


from social_django.models import UserSocialAuth
from django.shortcuts import render
import vk
import random
import time
# Create your views here.
def LogMainView(request):
    if request.user.is_authenticated:
        access_token = UserSocialAuth.objects.filter(user_id__username=request.user.username) \
                .last().extra_data['access_token']
        session = vk.Session(access_token=access_token)
        api = vk.API(session, v='5.4', lang='ru', timeout=10)
        name = api.users.get(user_ids=request.user.username.replace('id', ""))[0]['first_name'] + ' ' + \
                api.users.get(user_ids=request.user.username.replace('id', ""))[0]['last_name']
        all_frends = api.friends.get(user_ids=request.user.username.replace('id', ""))['items']

        frends = []
        for i in range(5):
            fr = random.choice(all_frends)
            frn = api.users.get(user_ids=fr)[0]['first_name']
            time.sleep(0.8)
            frn = frn + ' '+ api.users.get(user_ids=fr)[0]['last_name']
            frends.append(frn)
        return render(request, 'tvc/login.html', {'name': name, 'frends':frends})
    return render(request, 'tvc/login.html')


#############################################################
#
#      graph embedding using path levels
#
#############################################################

import networkx as nx
import pandas as pd
import numpy as np
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegressionCV
from tensorflow import keras
from networkx.linalg.graphmatrix import adjacency_matrix
from scipy.special import softmax
import networkx.generators.classic as Graphs
from scipy.sparse import csr_matrix
from scipy.sparse import identity
import time
from tqdm import tqdm
import random as rd

def normalise_weights(G,max_weight=None,convert=False):
    '''
    using softmax function

    '''
    n = len(G.nodes)

    #l = set(list(range(1,n+1)))

    #print(l)

    M = adjacency_matrix(G,nodelist=G.nodes())#,nodelist=l)

    #M = nx.convert_matrix.to_numpy_matrix(G)
    rows = M.nonzero()[0]
    cols = M.nonzero()[1]

    index = list(zip(rows,cols))
    #print(tuple(index))
    print("matrix shape :  " + str(M.shape[1]) +" X "  +  str(M.shape[0]) )
    count  = 0
    if convert:
        for i in index:
                M[i] = max_weight - M[i]
                #print(str(i)+'      ' +str(M[0,i]))
                count = count +1
                if count > 100:
                    break

    ## Softmax
    L = M.tolil()
    L = L.asfptype()
    #print(L.todense())
    #ind =M.indices
    #for i in ind:
     #   print(M[i])
    for i in range(len(L.rows)):
        X = []
        #print(L.rows[i])
        for j in L.rows[i]:
            X.append(L[i,j])

        if len(X) > 1:
            X = softmax( np.array(X))
        else:
            continue
        c = 0
        for j in L.rows[i]:
            #print(X[c])
            L[i,j] = X[c]
            #print(L[i,j])
            c=c+1
        #print (X)
        del(X)
    #print(L.todense())

    return L

def create_transaction_M(M):
    '''
    takes as input the normalized adjacency matrix of graph
    '''
    for i in range(len(M.rows)):
        M[i,i] = 0

    return M

def create_levels(G,M,T,level,alpha=.99):
    Multi_level = []
    matrix = M
    #G = nx.to_directed(G)
    N = M.shape[0]
    #Multi_level.append(identity(N).tolil())
    Multi_level.append(N)
    Multi_level.append(M)
    #edges = G.edges()
    level_edges = G.edges()
    Path = {}
    Path[1] = level_edges
    node_map = {}
    t = []
    leng = []
    leng.append(len(level_edges ))
    c = 0
    #with open('_file.txt', 'w',encoding='utf8') as f:
    for i in list(G.nodes.items()):
        node_map[i[0]] = c
        c += 1

    del(c)
    #src = []
    #dst = []
    start =time.time()
    for i in range(2,level+1):
        temp = csr_matrix((N, N), dtype=np.float64).tolil()
        next_level = []
        #prev = Multi_level[i-1]

        print(f" {len(level_edges)}   edges in current level ' {i} '")
        for edge in tqdm(level_edges):
            last_elt = list(edge)[-1]
            node_nbr = G.edges(last_elt)
            for nbr in node_nbr:
                elt = list(edge)[0]
                next_elt = list(nbr)[-1]
                if next_elt not in list(edge):
                    a = list(edge)
                    a.append(next_elt)#.append(list(nbr)[-1])
                    next_level.append(a)
                    temp[node_map[elt],node_map[next_elt]] +=  alpha*Multi_level[i-1][node_map[elt],node_map[last_elt]]*T[node_map[last_elt],node_map[next_elt]]
                    #print(f"list : {edge} ,{next_elt}  ")
            #print(G.nodes())#.index(edge[0]))
            #print(G.get_edge_data(edge[1],edge[1]))
            #print(T[node_map[edge[0]],node_map[edge[1]]])
        #print(temp.todense())
        level_edges = next_level
        leng.append(len(level_edges))
        Path[i] = level_edges

        Multi_level.append(temp)
        t.append(time.time() - start)
        del(next_level)
        del(temp)
        #del()
        print(f"level {i} built  in {time.time() - start}")
    return Multi_level , t,leng

def embedding(nodes,Multi_level):

    level = len(Multi_level)
    E = {}
    #print(type(nodes))
    c = 1
    for l in Multi_level[c:]:
        rep = {}
        #print(l)
        for v in range(len(nodes)):

            coor = 0
            #print(type(l))
            for val in l.rows[v]:
                #print(l[v,val])
                coor = coor +  l[v,val]

            rep[nodes[v]] = coor
        E[c] = rep
        c+=1
    '''
    for v in range(len(nodes)):

        rep = []

        for l in Multi_level[1:]:

            coor = 0
            #print(type(l))
            for val in l.rows[v]:
                #print(l[v,val])
                coor = coor +  l[v,val]

            rep.append(coor)
        E[l][nodes[v]] = rep
        #print(rep)
    #for elt in E:
     #   print(elt)
    '''
    return E

def embedding_g(nodes,Multi_level):

    level = len(Multi_level)
    E = []
    #print(type(nodes))
    c = 0
    for l in Multi_level[c:]:

        #print(type(l))#.toarray())
        C = sum(l.toarray())/len(nodes)
        #print(sum(C))
        E.append(sum(C))

    return E

def nodes_embedding(G,level=2,alpha=.99):

    #sorted_list = sorted(G.edges(data=True),key= lambda x: x[2]['weight'])

    #max_weight = sorted_list[-1][2]['weight']

    #print(type(max_weight))
    level = level
    alpha = alpha

    L = adjacency_matrix(G,nodelist=G.nodes()).tolil().asfptype()

    start = time.time()
    #############
    M = normalise_weights(G)#,max_weight)
    print("   matrix is normalized         ")
    T = create_transaction_M(M)
    print("   matrix of transaction built       ")
    level_M = create_levels(G,L,T,level,alpha)
    print("   begin embedding         ")
    Embed = embedding(list(G.nodes),level_M[0])
    #print(M.todense())
    print(f"Embeding done in  {time.time()-start}")
    return Embed

def graph_embedding(G,level=2,alpha=.99):

    #sorted_list = sorted(G.edges(data=True),key= lambda x: x[2]['weight'])

    #max_weight = sorted_list[-1][2]['weight']

    #print(type(max_weight))
    level = level
    alpha = alpha

    L = adjacency_matrix(G,nodelist=G.nodes()).tolil().asfptype()

    start = time.time()
    #############

    M = normalise_weights(G)#,max_weight)
    print("   matrix is normalized         ")
    T = create_transaction_M(M)
    print("   matrix of transaction built       ")
    level_M, roll ,leng = create_levels(G,L,T,level,alpha)

    print("   begin embedding         ")
    Embed = embedding_g(list(G.nodes),level_M)
    print(f"Embeding done in  {time.time()-start}")

    return Embed , roll,leng


import logging
import requests
import subprocess

logger = logging.getLogger()

def delete_request(url: str, headers: dict = None) -> None:
    """Send a DELETE request to an HTTP API endpoint

    Args:
        url (str): The URL to send the request to
        headers (dict, optional): A dictionary of any headers to send with the
                                  request. Defaults to None.
    """
    resp = requests.delete(url, headers=headers)

    if not resp:
        logger.error(resp.text)
        raise RuntimeError(resp.text)

def get_request(
    url: str,
    headers: dict = None,
    params: dict = None,
    json: bool = False,
    text: bool = False,
):
    """Send a GET request to an HTTP API endpoint

    Args:
        url (str): The URL to send the request to
        headers (dict): A dictionary of headers to send with the request
        params (dict): A dictionary of parameters to send with the request
        json (bool, optional): Returns the json payload. Defaults to False.
        text (bool, optional): Returns the text payload. Defaults to False.
    """
    if json and text:
        raise ValueError("json and text kwargs cannot both be true")

    resp = requests.get(url, headers=headers, params=params)

    if not resp:
        logger.error(resp.text)
        raise RuntimeError(resp.text)

    if json:
        return resp.json()
    elif text:
        return resp.text
    else:
        return resp

def post_request(
    url: str, headers: dict = None, json: dict = None, return_json: bool = True
) -> None:
    """Send a POST request to an HTTP API endpoint

    Args:
        url (str): The URL to send the request to
        headers (dict, optional): A dictionary of any headers to send with the
                                  request. Defaults to None.
        json (dict, optional): A dictionary containing JSON payload to send with
                               the request. Defaults to None.
        return_json (bool, optional): Return the JSON payload response.
                                      Defaults to False.
    """
    resp = requests.post(url, headers=headers, json=json)

    if not resp:
        logger.error(resp.text)
        raise RuntimeError(resp.text)

    if return_json:
        return resp.json()

def run_cmd(cmd: list) -> dict:
    """Use Popen to run a bash command in a sub-shell

    Args:
        cmd (list): The bash command to run

    Returns:
        dict: The output of the command, including status code and error
              messages
    """
    proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )

    msgs = proc.communicate()

    result = {
        "returncode": proc.returncode,
        "output": msgs[0].decode(encoding=("utf-8")).strip("\n"),
        "err_msg": msgs[1].decode(encoding=("utf-8")).strip("\n"),
    }

    return result


# This file is based on the ghp-import package released under
# the Tumbolia Public License.

#                            Tumbolia Public License

# Copyright 2013, Paul Davis <paul.joseph.davis@gmail.com>

# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice and
# this notice are preserved.

# TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION

#   0. opan saurce LOL

from __future__ import unicode_literals

import os
import subprocess as sp
import sys
import time
import unicodedata

from six import binary_type, text_type

def git_path(path):
    path = os.path.normpath(path)
    # Fix unicode pathnames on OS X; see
    # <http://stackoverflow.com/a/5582439/44289>.
    if sys.platform == 'darwin':
        if isinstance(path, binary_type):
            path = path.decode('utf-8')
        path = unicodedata.normalize('NFKC', path)
    return '/'.join(path.split(os.path.sep))

def make_when(timestamp=None):
    if timestamp is None:
        timestamp = int(time.time())
    timezone = '{:+05d}'.format(-1 * time.timezone / 3600 * 100)
    return '{} {}'.format(timestamp, timezone)

def get_config(key):
    cmd = ['git', 'config', key]
    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True)
    stdout, stderr = p.communicate()
    if p.wait() != 0:
        raise ValueError('Error getting config: {}'.format(stderr))
    return stdout.strip()

def get_latest_commit(branch):
    cmd = ['git', 'rev-parse', branch]
    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True)
    stdout, stderr = p.communicate()
    if p.wait() != 0:
        raise ValueError('Error getting latest commit: {}'.format(stderr))
    return stdout.strip()

def update_branch(remote, branch):
    try:
        rev = get_latest_commit(branch)
        cmd = ['git', 'update-ref', 'refs/heads/{}'.format(branch), rev]
        if sp.call(cmd) != 0:
            raise ValueError('Failed to update branch')
    except ValueError:
        # Couldn't get any commits, so there's probably no branch (which is
        # fine).
        pass

class FileInfo(object):
    def __init__(self, path, data, mode='100644'):
        self.path = path
        self.mode = mode
        self.data = data

class Commit(object):
    def __init__(self, branch, message):
        cmd = ['git', 'fast-import', '--date-format=raw', '--quiet']
        self._pipe = sp.Popen(cmd, stdin=sp.PIPE, universal_newlines=False)
        self._start_commit(branch, message)

    def _write(self, data):
        if isinstance(data, text_type):
            data = data.encode('utf-8')
        return self._pipe.stdin.write(data)

    def _start_commit(self, branch, message):
        name = get_config('user.name')
        email = get_config('user.email')
        self._write('commit refs/heads/{}\n'.format(branch))
        self._write('committer {name} <{email}> {time}\n'.format(
            name=name, email=email, time=make_when()
        ))
        self._write('data {length}\n{message}\n'.format(
            length=len(message), message=message
        ))
        try:
            head = get_latest_commit(branch)
            self._write('from {}\n'.format(head))
        except ValueError:
            pass

    def delete_files(self, files):
        if files == '*':
            self._write('deleteall\n')
        elif files:
            for f in files:
                self._write('D {}\n'.format(f))

    def add_file_data(self, file_info):
        self._write('M {mode} inline {path}\n'.format(
            path=git_path(file_info.path), mode=file_info.mode
        ))
        self._write('data {}\n'.format(len(file_info.data)))
        self._write(file_info.data)
        self._write('\n')

    def finish(self):
        self._write('\n')
        self._pipe.stdin.close()
        if self._pipe.wait() != 0:
            raise ValueError('Failed to process commit')

def push_branch(remote, branch, force=False):
    cmd = ['git', 'push'] + (['--force'] if force else []) + [remote, branch]
    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True)
    stdout, stderr = p.communicate()
    if p.wait() != 0:
        raise ValueError('Failed to push branch: {}'.format(stderr))

def walk_files(srcdir, dstdirs=['']):
    for path, _, filenames in os.walk(srcdir):
        for f in filenames:
            inpath = os.path.join(path, f)
            with open(inpath, 'rb') as fd:
                mode = '100755' if os.access(inpath, os.X_OK) else '100644'
                data = fd.read()
            for d in dstdirs:
                outpath = os.path.join(
                    d, os.path.relpath(inpath, start=srcdir)
                )
                yield FileInfo(outpath, data, mode)

def read_file(branch, filename):
    cmd = ['git', 'show', '{branch}:{filename}'.format(
        branch=branch, filename=filename
    )]
    p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
    stdout, stderr = p.communicate()
    if p.wait() != 0:
        raise ValueError('Unable to read file: {}'.format(stderr))
    return stdout.decode('utf-8')


import argparse
import os

import torch
import torch.distributed.autograd as dist_autograd
import torch.multiprocessing as mp
import torch.distributed.rpc as rpc
import torch.nn as nn
import torch.optim as optim
from torch.distributed.optim import DistributedOptimizer
from torchvision import datasets, transforms
from tqdm import tqdm

from cuda_rpc_forward_tensor import DistributedCUDARPCSequential, WorkerModule, layer_on_device

IS_SLURM = os.getenv('SLURM_LOCALID')
USE_TQDM = os.getenv('USE_TQDM', True if not IS_SLURM else False)

def run_main():
    batch_size = 100

    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ])

    train_data = datasets.MNIST('./data', train=True, download=True, transform=transform)
    valid_data = datasets.MNIST('./data', train=False, transform=transform)

    train_dataloader = torch.utils.data.DataLoader(train_data, batch_size=batch_size)
    valid_dataloader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size)

    model = DistributedCUDARPCSequential(
        WorkerModule("worker1", layer_on_device("cuda"), nn.Flatten),
        WorkerModule("worker2", layer_on_device("cuda"), nn.Linear, 28 * 28, 128),
        WorkerModule("worker3", layer_on_device("cuda"), nn.ReLU),
        WorkerModule("worker4", layer_on_device("cuda"), nn.Linear, 128, 64),
        WorkerModule("worker5", layer_on_device("cuda"), nn.ReLU),
        WorkerModule("worker6", layer_on_device("cuda"), nn.Linear, 64, 10),
    )

    criterion = nn.CrossEntropyLoss()
    optimizer = DistributedOptimizer(
        optim.Adam,
        model.parameter_rrefs(),
    )

    loaders = {"train": train_dataloader, "valid": valid_dataloader}

    max_epochs = 10
    for epoch in range(max_epochs):
        print(f"Epoch: {epoch + 1}")
        epoch_correct = 0
        epoch_all = 0
        for k, dataloader in loaders.items():
            for i, (x_batch, y_batch) in enumerate(tqdm(dataloader) if USE_TQDM else dataloader):
                x_batch = x_batch.to(0)
                y_batch = y_batch.to(0)
                if k == "train":
                    model.train()
                    with dist_autograd.context() as context_id:
                        outp = model(x_batch)
                        preds = outp.argmax(-1)
                        correct = (preds == y_batch).sum()
                        all = len(y_batch)
                        epoch_correct += correct.item()
                        epoch_all += all
                        loss = criterion(outp, y_batch)
                        dist_autograd.backward(context_id, [loss])
                        optimizer.step(context_id)
                else:
                    model.eval()
                    with torch.no_grad():
                        outp = model(x_batch)
                        preds = outp.argmax(-1)
                        correct = (preds == y_batch).sum()
                        all = len(y_batch)
                        epoch_correct += correct.item()
                        epoch_all += all

            print(f"Loader: {k}. Accuracy: {epoch_correct / epoch_all}")

def run_worker(rank, world_size, args):
    os.environ['MASTER_ADDR'] = args.master_addr
    os.environ['MASTER_PORT'] = args.master_port
    options = rpc.TensorPipeRpcBackendOptions(num_worker_threads=256)

    if rank == 0:
        options.set_device_map("worker1", {0: 0})
        options.set_device_map("worker2", {0: 0})
        options.set_device_map("worker3", {0: 0})
        options.set_device_map("worker4", {0: 0})
        options.set_device_map("worker5", {0: 0})
        options.set_device_map("worker6", {0: 0})

        rpc.init_rpc(
            "master",
            rank=rank,
            world_size=world_size,
            rpc_backend_options=options
        )
        run_main()
    else:
        if not IS_SLURM:
            os.environ['CUDA_VISIBLE_DEVICES'] = str(rank - 1)
        if rank == 1:
            options.set_device_map("worker2", {0: 0})
        elif rank == 2:
            options.set_device_map("worker3", {0: 0})
        elif rank == 3:
            options.set_device_map("worker4", {0: 0})
        elif rank == 4:
            options.set_device_map("worker5", {0: 0})
        elif rank == 5:
            options.set_device_map("worker6", {0: 0})

        rpc.init_rpc(
            f"worker{rank}",
            rank=rank,
            world_size=world_size,
            rpc_backend_options=options
        )

    rpc.shutdown()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Pipeline experiments')

    parser.add_argument('--world_size', type=int, default=7,
                        help='the world size')
    parser.add_argument('--rank', type=int, default=None,
                        help="Global rank of this process. Pass in 0 for master.")
    parser.add_argument('--master_addr', type=str, default='localhost',
                        help="""Address of master, will default to localhost if not provided. Master must be able to accept network traffic on the address + port.""")
    parser.add_argument('--master_port', type=str, default='29500',
                        help="""Port that master is listening on, will default to 29500 if not provided. Master must be able to accept network traffic on the host and port.""")

    args = parser.parse_args()
    if args.rank is None:
        mp.spawn(run_worker, args=(args.world_size, args,), nprocs=args.world_size, join=True)
    else:
        run_worker(args.rank, args.world_size, args)


from django.shortcuts import render
from django.http import HttpResponse
from django.template import loader
from django.shortcuts import get_object_or_404
from .models import Member

# Create your views here.
def memberView(request):
    return HttpResponse("teammember")

def index(request):
    member_list = Member.objects.all()
    context = {
        'member_list' : member_list,
    }
    return render(request, 'team/member.html', context)

def team_view(request):
    from team.models import Group, Member
    template_name = 'team.html'
    group_list = Group.objects.all().order_by("ordering")
    for group in group_list:
        group.member_list = group.member_set.all().order_by("id")

    context = {
        'group_list':group_list,
    }

    return render(request, template_name, context)

def team_year(request, year):
    from team.models import Group, Member, Year
    template_name = 'team.html'
    year = get_object_or_404(Year, year=year)
    group_list = year.group_set.all().order_by("ordering")
    for group in group_list:
        group.member_list = group.member_set.all().order_by("id")

    context = {
        'group_list':group_list,
    }

    return render(request, template_name, context)


#!/usr/bin/python3
""" titles of all hot articles """
import requests

def recurse(subreddit, hot_list=[], after=None):
    """titles of all hot articles"""
    headers = {'user-agent': 'X-Modhash'}
    limit = {"limit": 100}
    if after is not None:
        limit['after'] = after
    url = 'https://www.reddit.com/r/'
    status = requests.get(url + '{}/hot.json'.format(subreddit),
                          headers=headers, params=limit)
    subs = status.json()
    if status.status_code is 200:
        for i in range(len(subs['data']['children'])):
            hot_list.append(subs['data']['children'][i]['data']['title'])
        if subs['data']['after'] is not None:
            recurse(subreddit, hot_list, subs['data']['after'])
        return hot_list
    else:
        return None


# -*- coding: utf-8 -*-
""" Properties of iCalendar objects as descriptors

In this module we use this naming convention:
A *component* is one of the primary elements of a calendar like
VCALENDAR, VEVENT or VALARM.
A *property* describes the component; depending on the type of
the component different properties are defined.
A property can have additional *parameters*.

The properties are implemented as descriptors for the components.
The values of the properties are stored in a dict on the
component.

TODO: handle multiple values
"""
import six
from datetime import datetime, date, timedelta
from collections import namedtuple

from .utils import (datetime_to_iso, parse_duration,
                    timedelta_to_duration,
                    get_date_or_datetime, parse_cal_date, parse_cal_datetime)
from .parse import ContentLine

ICAL_PROPERTIES = {
    'VCALENDAR': {
        'properties': {
            'PRODID':   {'required': True, },
            'VERSION':  {'required': True, },
            'CALSCALE': {},
            'METHOD':   {},
        },
        'components': [
            'VEVENT',
            'VTODO',
            'VJOURNAL',
            'VFREEBUSY',
            'VTIMEZONE',
        ]
    },
    'VEVENT': {
        'properties': {
            'DTSTAMP':     {'required': True, },
            'UID':         {'required': True, },
            'DTSTART':     {},
            'CLASS':       {},
            'CREATED':     {},
            'DESCRIPTION': {},
            'GEO':         {},
            'LAST-MOD':    {},
            'LOCATION':    {},
            'ORGANIZER':   {},
            'PRIORITY':    {},
            'SEQ':         {},
            'STATUS':      {},
            'SUMMARY':     {},
            'TRANSP':      {},
            'URL':         {},
            'RECURID':     {},
            'DTEND':       {},
            'DURATION':    {},
            'RRULE':       {'multiple': True, },
            'ATTACH':      {'multiple': True, },
            'ATTENDEE':    {'multiple': True, },
            'CATEGORIES':  {'multiple': True, },
            'COMMENT':     {'multiple': True, },
            'CONTACT':     {'multiple': True, },
            'EXDATE':      {'multiple': True, },
            'RSTATUS':     {'multiple': True, },
            'RELATED':     {'multiple': True, },
            'RESOURCES':   {'multiple': True, },
            'RDATE':       {'multiple': True, },
        },
        'components': [
            'VALARM',
        ],
        'rules': [
            "There has to be a 'dtstart' in the event or a 'method' in the calendar",
            "Either 'dtend' or 'duration' may appear in a event",
            "RRULE  SHOULD NOT occur more than once",
        ]
    },
    'VTODO': {
        'properties': {
            "DTSTAMP":     {'required': True, },
            "UID":         {'required': True, },
            "CLASS":       {},
            "COMPLETED":   {},
            "CREATED":     {},
            "DESCRIPTION": {},
            "DTSTART":     {},
            "GEO":         {},
            "LAST-MOD":    {},
            "LOCATION":    {},
            "ORGANIZER":   {},
            "PERCENT":     {},
            "PRIORITY":    {},
            "RECURID":     {},
            "SEQ":         {},
            "STATUS":      {},
            "SUMMARY":     {},
            "URL":         {},
            "DUE":         {},
            "DURATION":    {},
            "RRULE":       {'multiple': True, },
            "ATTACH":      {'multiple': True, },
            "ATTENDEE":    {'multiple': True, },
            "CATEGORIES":  {'multiple': True, },
            "COMMENT":     {'multiple': True, },
            "CONTACT":     {'multiple': True, },
            "EXDATE":      {'multiple': True, },
            "RSTATUS":     {'multiple': True, },
            "RELATED":     {'multiple': True, },
            "RESOURCES":   {'multiple': True, },
            "RDATE":       {'multiple': True, },
        },
        'components': [
            "VALARM",
        ],
        'rules': [
            "RRULE  SHOULD NOT occur more than once",
            "Either 'due' or 'duration' MAY appear in a 'todoprop', but 'due' and 'duration' MUST NOT occur in the same 'todoprop'.",
            "If 'duration' appear in a 'todoprop', then 'dtstart' MUST also appear in the same 'todoprop'.",
        ],
    },
    'VJOURNAL': {
        'properties': {
            "DTSTAMP":     {'required': True, },
            "UID":         {'required': True, },
            "CLASS":       {},
            "CREATED":     {},
            "DTSTART":     {},
            "LAST-MOD":    {},
            "ORGANIZER":   {},
            "RECURID":     {},
            "SEQ":         {},
            "STATUS":      {},
            "SUMMARY":     {},
            "URL":         {},
            "RRULE":       {'multiple': True, },
            "ATTACH":      {'multiple': True, },
            "ATTENDEE":    {'multiple': True, },
            "CATEGORIES":  {'multiple': True, },
            "COMMENT":     {'multiple': True, },
            "CONTACT":     {'multiple': True, },
            "DESCRIPTION": {'multiple': True, },
            "EXDATE":      {'multiple': True, },
            "RELATED":     {'multiple': True, },
            "RDATE":       {'multiple': True, },
            "RSTATUS":     {'multiple': True, },
        },
        'components': [
        ],
        'rules': [
            "RRULE  SHOULD NOT occur more than once",
        ],
    },
    'VFREEBUSY': {
        'properties': {
            "DTSTAMP":   {'required': True, },
            "UID":       {'required': True, },
            "CONTACT":   {},
            "DTSTART":   {},
            "DTEND":     {},
            "ORGANIZER": {},
            "URL":       {},
            "ATTENDEE":  {'multiple': True, },
            "COMMENT":   {'multiple': True, },
            "FREEBUSY":  {'multiple': True, },
            "RSTATUS":   {'multiple': True, },
        },
        'components': [
        ],
        'rules': [
        ],
    },
    'VTIMEZONE': {
        'properties': {
            "TZID":     {'required': True, },
            "LAST-MOD": {},
            "TZURL":    {},
        },
        'components': [
            'STANDARD',
            'DAYLIGHT',
        ],
        'rules': [
            "One of 'STANDARD' or 'DAYLIGHT' MUST occur and each MAY occur more than once.",
        ],
    },
    'STANDARD': {
        'properties': {
            "DTSTART":      {'required': True, },
            "TZOFFSETTO":   {'required': True, },
            "TZOFFSETFROM": {'required': True, },
            "RRULE":        {'multiple': True, },
            "COMMENT":      {'multiple': True, },
            "RDATE":        {'multiple': True, },
            "TZNAME":       {'multiple': True, },
        },
        'rules': [
            "RRULE  SHOULD NOT occur more than once",
        ],
    },
    'DAYLIGHT': {
        'properties': {
            "DTSTART":      {'required': True, },
            "TZOFFSETTO":   {'required': True, },
            "TZOFFSETFROM": {'required': True, },
            "RRULE":        {'multiple': True, },
            "COMMENT":      {'multiple': True, },
            "RDATE":        {'multiple': True, },
            "TZNAME":       {'multiple': True, },
        },
        'rules': [
            "RRULE  SHOULD NOT occur more than once",
        ],
    },
    'VALARM': {
        'properties': {
            "ACTION":      {'required': True, },
            "TRIGGER":     {'required': True, },
            "DURATION":    {},
            "REPEAT":      {},
            "DESCRIPTION": {},
            "SUMMARY":     {},
            "ATTENDEE":    {'multiple': True, },
            "ATTACH":      {'multiple': True, },
        },
        'components': [
        ],
        'rules': [
            "Depending on 'action', 'description', 'summary', 'attendee' and 'attach' may be required",
            "'duration' and 'repeat' are both OPTIONAL, and MUST NOT occur more than once each; but if one occurs, so MUST the other."
        ],
    },
}

PropertyDesc = namedtuple(
    'PropertyDesc',
    ['type', 'required', 'multiple']
)

# If MEMOIZE_DATA is True,
# setting properties memoizes their python value.
# Is False for development/debugging purposes.
MEMOIZE_DATA = False

def property_description(component_type, line):
    """ Return a description of the property of this component_type

    Defaults to PropertyDesc(type=ICalProperty, required=False,
        multiple=False)
    For unknown components or properties 'multiple' is True
    """
    comp_def = ICAL_PROPERTIES.get(component_type, {'properties': {}})
    definition = comp_def['properties'].get(line, {'multiple': True})
    return PropertyDesc(
        type=definition.get('type', ICalProperty),
        required=definition.get('required', False),
        multiple=definition.get('multiple', False))

def get_required_properties(component_type):
    comp_def = ICAL_PROPERTIES.get(component_type, {'properties': {}})
    properties = comp_def.get('properties', {})
    return [prop for prop, features in properties.items()
            if features.get('required', False)]

class ICalProperty(object):
    """ Property for an ical object implemented as descriptor
    """
    def __init__(self, ical_name,
                 validation=None,
                 default=None,
                 create_on_access=False):
        """ Initialize the property

        Args:
            ical_name (string): the name of this property in the rfc 5545
            validation (string): name of a method of the instance used to
                validate (and process) the input.
                The method must return the validated value.
            default (value of appropriate type
                     or method name of the instance
                     or function not expecting parameters):
                to be used when the property is accessed but not defined.
                The expected interface of the method is like this:
                `def default(self, property_name):`
                where `property_name` is the ical_name of the property.
                It should return a python value of the appropriate type.
                The method or function is called for every access to the
                property as long as it is not set.
            create_on_access: if there is no value for the property
                when it is accessed, set the result of `default`

        `required` and `multiple` are ignored when `parent_type` is given
        """
        self.ical_name = ical_name
        self.validation = validation
        self.default = default
        self.create_on_access = create_on_access
        self.memoized = False

    def __get__(self, instance, owner):
        if MEMOIZE_DATA and self.memoized:
            return self.memo
        if self.ical_name in instance._properties:
            line = instance._properties[self.ical_name]
            return self._memoize(self.to_python(line, instance))
        default = self._get_default(instance)
        if self.create_on_access:
            self.__set__(instance, default)
        return default

    def _get_default(self, instance):
        if self.default:
            if callable(self.default):
                return self.default()
            if (isinstance(self.default, six.string_types) and
                    callable(getattr(instance, self.default, None))):
                return getattr(instance, self.default)(self.ical_name)
            return self.default
        return None

    def __set__(self, instance, value):
        """ Set the value of the property

        Before the value is set, the validation method of the instance is
        called. This method can change the value and should raise an
        exception, if the value doesn't fit.

        Args:
            value: the new value of the property.
            This can be a string or a tuple in the form of (value, params).
        """
        if self.validation:
            value = getattr(instance, self.validation)(value)
        if value is None:
            if self.ical_name in instance._properties:
                # Setting a property to None is deleting it
                self.__delete__(instance)
        else:
            self._memoize(value)
            p_value = self.from_python(value)
            instance._properties[self.ical_name] = p_value

    def __delete__(self, instance):
        if self.required(instance):
            raise TypeError("{} is required for {}".format(
                            self.ical_name, instance._TYPE))
        del instance._properties[self.ical_name]
        self._memoize(None)

    def from_python(self, value, params={}):
        """ Translate a python data type to the format of iCalendar
        """
        return ContentLine(self.ical_name, value=value, params=params)

    def to_python(self, line, instance):
        """ Translate the format of an iCalendar property to a python value
        """
        return line.value

    def required(self, instance):
        """ Return True if this property is required for this instance
        """
        return property_description(instance._TYPE, self.ical_name).required

    def multiple(self, instance):
        """ Return True if this property can occur multiple times in this instance
        """
        return property_description(instance._TYPE, self.ical_name).multiple

    def _memoize(self, value):
        if MEMOIZE_DATA:
            self.memoized = True
            self.memo = value
        return value

class TextProperty(ICalProperty):
    """ ICalProperty with 'TEXT' value type
    """
    def from_python(self, value, params={}):
        value = six.text_type(value)
        if value:
            value = value.replace("\\", "\\\\")
            value = value.replace(";", "\\;")
            value = value.replace(",", "\\,")
            value = value.replace("\n", "\\n")
        return ContentLine(self.ical_name, value=value, params=params)

    def to_python(self, line, instance):
        if not line:
            return None
        value = line.value
        if value:
            value = value.replace("\\;", ";")
            value = value.replace("\\,", ",")
            value = value.replace("\\n", "\n")
            value = value.replace("\\N", "\n")
            value = value.replace("\\\\", "\\")
        return value

class DateTimeProperty(ICalProperty):
    """ ICalProperty with 'DATE-TIME' value type
    """
    def from_python(self, value, params={}):
        return ContentLine(self.ical_name, value=datetime_to_iso(value),
                           params=params)

    def to_python(self, line, instance):
        return parse_cal_datetime(line, instance.get_timezones())

class DateTimeOrDateProperty(ICalProperty):
    """ ICalProperty with 'DATE-TIME' or 'DATE' value type
    """
    def __set__(self, instance, value):
        value = get_date_or_datetime(value)
        if self.validation:
            value = getattr(instance, self.validation)(value)
        if value is None:
            if self.ical_name in instance._properties:
                # Setting a property to None means deleting it
                self.__delete__(instance)
        else:
            p_value = self.from_python(value)
            instance._properties[self.ical_name] = p_value

    def from_python(self, value, params={}):
        params = params or {}
        if isinstance(value, date) and not isinstance(value, datetime):
            params['VALUE'] = 'DATE'
            s_value = value.strftime('%Y%m%d')
        else:
            s_value = value.strftime('%Y%m%dT%H%M%S')
            if value.tzinfo:
                # handle timezone
                if value.tzinfo.utcoffset(value):
                    params['TZID'] = [value.tzinfo.tzname(value)]
                else:
                    # UTC
                    s_value += 'Z'
        return ContentLine(self.ical_name, value=s_value, params=params)

    def to_python(self, line, instance):
        value = line.value
        params = line.params or {}
        if params.get('VALUE') == 'DATE':
            return parse_cal_date(value)
        return parse_cal_datetime(line, instance.get_timezones())

class DurationProperty(ICalProperty):
    """ ICalProperty with 'DURATION' value type

    |  Will return a timedelta object.
    |  May be set to anything that timedelta() understands.
    |  May be set with a dict ({"days":2, "hours":6}).
    """

    def from_python(self, value, params={}):
        if isinstance(value, dict):
            value = timedelta(**value)
        elif value is not None and not isinstance(value, timedelta):
            value = timedelta(value)
        return ContentLine(self.ical_name,
                           value=timedelta_to_duration(value),
                           params=params)

    def to_python(self, line, instance):
        return parse_duration(line.value)

class VersionProperty(ICalProperty):
    """ ICalProperty for calendar 'VERSION'

    |  If there is a minversion and a maxversion,
    |  it will return the maxversion
    """

    def to_python(self, line, instance):
        if ';' in line.value:
            return line.value.split(';', 1)[1]
        return line.value


from __future__ import annotations

import uuid
import weakref
from collections import defaultdict
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any

import dask.config

from distributed.metrics import time

if TYPE_CHECKING:
    from distributed import Scheduler, Worker
    from distributed.scheduler import TaskGroup, TaskState, TaskStateState, WorkerState

@contextmanager
def span(*tags: str) -> Iterator[str]:
    """Tag group of tasks to be part of a certain group, called a span.

    This context manager can be nested, thus creating sub-spans. If you close and
    re-open a span context manager with the same tag, you'll end up with two separate
    spans.

    Every cluster defines a global "default" span when no span has been defined by the
    client; the default span is automatically closed and reopened when all tasks
    associated to it have been completed; in other words the cluster is idle save for
    tasks that are explicitly annotated by a span. Note that, in some edge cases, you
    may end up with overlapping default spans, e.g. if a worker crashes and all unique
    tasks that were in memory on it need to be recomputed.

    Examples
    --------
    >>> import dask.array as da
    >>> import distributed
    >>> client = distributed.Client()
    >>> with span("my workflow"):
    ...     with span("phase 1"):
    ...         a = da.random.random(10)
    ...         b = a + 1
    ...     with span("phase 2"):
    ...         c = b * 2
    ... d = c.sum()
    >>> d.compute()

    In the above example,
    - Tasks of collections a and b are annotated to belong to span
      ``('my workflow', 'phase 1')``, 'ids': (<id0>, <id1>)}``;
    - Tasks of collection c (that aren't already part of a or b) are annotated to belong
      to span ``('my workflow', 'phase 2')``;
    - Tasks of collection d (that aren't already part of a, b, or c) are *not*
      annotated but will nonetheless be attached to span ``('default', )``.

    You may also set more than one tag at once; e.g.
    >>> with span("workflow1", "version1"):
    ...     ...

    Finally, you may capture the ID of a span on the client to match it with the
    :class:`Span` objects the scheduler:
    >>> cluster = distributed.LocalCluster()
    >>> client = distributed.Client(cluster)
    >>> with span("my workflow") as span_id:
    ...     client.submit(lambda: "Hello world!").result()
    >>> span = client.cluster.scheduler.extensions["spans"].spans[span_id]

    Notes
    -----
    Spans are based on annotations, and just like annotations they can be lost during
    optimization. Set config ``optimization.fuse.active: false`` to prevent this issue.
    """
    if not tags:
        raise ValueError("Must specify at least one span tag")

    prev_tags = dask.config.get("annotations.span.name", ())
    # You must specify the full history of IDs, not just the parent, because
    # otherwise you would not be able to uniquely identify grandparents when
    # they have no tasks of their own.
    prev_ids = dask.config.get("annotations.span.ids", ())
    ids = tuple(str(uuid.uuid4()) for _ in tags)
    with dask.annotate(span={"name": prev_tags + tags, "ids": prev_ids + ids}):
        yield ids[-1]

class Span:
    #: (<tag>, <tag>, ...)
    #: Matches ``TaskState.annotations["span"]["name"]``, both on the scheduler and the
    #: worker.
    name: tuple[str, ...]

    #: <uuid>
    #: Taken from ``TaskState.annotations["span"]["id"][-1]``.
    #: Matches ``distributed.scheduler.TaskState.group.span_id``
    #: and ``distributed.worker_state_machine.TaskState.span_id``.
    id: str

    _parent: weakref.ref[Span] | None

    #: Direct children of this span, sorted by creation time
    children: list[Span]

    #: Task groups *directly* belonging to this span.
    #:
    #: See also
    #: --------
    #  traverse_groups
    #:
    #: Notes
    #: -----
    #: TaskGroups are forgotten when the last task is forgotten. If a user calls
    #: compute() twice on the same collection, you'll have more than one group with the
    #: same tg.name in this set! For the same reason, while the same TaskGroup object is
    #: guaranteed to be attached to exactly one Span, you may have different TaskGroups
    #: with the same key attached to different Spans.
    groups: set[TaskGroup]

    #: Time when the span first appeared on the scheduler.
    #: The same property on parent spans is always less than or equal to this.
    #:
    #: See also
    #: --------
    #: start
    #: stop
    enqueued: float

    _cumulative_worker_metrics: defaultdict[tuple[str, ...], float]

    # Support for weakrefs to a class with __slots__
    __weakref__: Any

    __slots__ = tuple(__annotations__)

    def __init__(self, name: tuple[str, ...], id_: str, parent: Span | None):
        self.name = name
        self.id = id_
        self._parent = weakref.ref(parent) if parent is not None else None
        self.enqueued = time()
        self.children = []
        self.groups = set()
        self._cumulative_worker_metrics = defaultdict(float)

    def __repr__(self) -> str:
        return f"Span<name={self.name}, id={self.id}>"

    @property
    def parent(self) -> Span | None:
        if self._parent:
            out = self._parent()
            assert out
            return out
        return None

    def traverse_spans(self) -> Iterator[Span]:
        """Top-down recursion of all spans belonging to this branch off span tree,
        including self
        """
        yield self
        for child in self.children:
            yield from child.traverse_spans()

    def traverse_groups(self) -> Iterator[TaskGroup]:
        """All TaskGroups belonging to this branch of span tree"""
        for span in self.traverse_spans():
            yield from span.groups

    @property
    def start(self) -> float:
        """Earliest time when a task belonging to this span tree started computing;
        0 if no task has *finished* computing yet.

        Note
        ----
        This is not updated until at least one task has *finished* computing.
        It could move backwards as tasks complete.

        See also
        --------
        enqueued
        stop
        distributed.scheduler.TaskGroup.start
        """
        return min(
            (tg.start for tg in self.traverse_groups() if tg.start != 0.0),
            default=0.0,
        )

    @property
    def stop(self) -> float:
        """Latest time when a task belonging to this span tree finished computing;
        0 if no task has finished computing yet.

        See also
        --------
        enqueued
        start
        distributed.scheduler.TaskGroup.stop
        """
        return max(tg.stop for tg in self.traverse_groups())

    @property
    def states(self) -> defaultdict[TaskStateState, int]:
        """The number of tasks currently in each state in this span tree;
        e.g. ``{"memory": 10, "processing": 3, "released": 4, ...}``.

        See also
        --------
        distributed.scheduler.TaskGroup.states
        """
        out: defaultdict[TaskStateState, int] = defaultdict(int)
        for tg in self.traverse_groups():
            for state, count in tg.states.items():
                out[state] += count
        return out

    @property
    def done(self) -> bool:
        """Return True if all tasks in this span tree are completed; False otherwise.

        Notes
        -----
        This property may transition from True to False, e.g. when a new sub-span is
        added or when a worker that contained the only replica of a task in memory
        crashes and the task need to be recomputed.

        See also
        --------
        distributed.scheduler.TaskGroup.done
        """
        return all(tg.done for tg in self.traverse_groups())

    @property
    def all_durations(self) -> defaultdict[str, float]:
        """Cumulative duration of all completed actions in this span tree, by action

        See also
        --------
        duration
        distributed.scheduler.TaskGroup.all_durations
        """
        out: defaultdict[str, float] = defaultdict(float)
        for tg in self.traverse_groups():
            for action, nsec in tg.all_durations.items():
                out[action] += nsec
        return out

    @property
    def duration(self) -> float:
        """The total amount of time spent on all tasks in this span tree

        See also
        --------
        all_durations
        distributed.scheduler.TaskGroup.duration
        """
        return sum(tg.duration for tg in self.traverse_groups())

    @property
    def nbytes_total(self) -> int:
        """The total number of bytes that this span tree has produced

        See also
        --------
        distributed.scheduler.TaskGroup.nbytes_total
        """
        return sum(tg.nbytes_total for tg in self.traverse_groups())

    @property
    def cumulative_worker_metrics(self) -> defaultdict[tuple[str, ...], float]:
        """Replica of Worker.digests_total and Scheduler.cumulative_worker_metrics, but
        only for the metrics that can be attributed to the current span tree.
        The span_id has been removed from the key.

        At the moment of writing, all keys are
        ``("execute", <task prefix>, <activity>, <unit>)``
        but more may be added in the future with a different format; please test for
        ``k[0] == "execute"``.
        """
        out: defaultdict[tuple[str, ...], float] = defaultdict(float)
        for child in self.traverse_spans():
            for k, v in child._cumulative_worker_metrics.items():
                out[k] += v
        return out

    @staticmethod
    def merge(*items: Span) -> Span:
        """Merge multiple spans into a synthetic one.
        The input spans must not be related with each other.
        """
        out = Span(name=("(merged)",), id_="(merged)", parent=None)
        out.children.extend(items)
        out.enqueued = min(child.enqueued for child in items)
        return out

class SpansSchedulerExtension:
    """Scheduler extension for spans support"""

    #: All Span objects by id
    spans: dict[str, Span]

    #: Only the spans that don't have any parents, sorted by creation time.
    #: This is a convenience helper structure to speed up searches.
    root_spans: list[Span]

    #: All spans, keyed by their full name and sorted by creation time.
    #: This is a convenience helper structure to speed up searches.
    spans_search_by_name: defaultdict[tuple[str, ...], list[Span]]

    #: All spans, keyed by the individual tags that make up their name and sorted by
    #: creation time.
    #: This is a convenience helper structure to speed up searches.
    #:
    #: See Also
    #: --------
    #: find_by_tags
    #: merge_by_tags
    spans_search_by_tag: defaultdict[str, list[Span]]

    def __init__(self, scheduler: Scheduler):
        self.spans = {}
        self.root_spans = []
        self.spans_search_by_name = defaultdict(list)
        self.spans_search_by_tag = defaultdict(list)

    def observe_tasks(self, tss: Iterable[TaskState]) -> None:
        """Acknowledge the existence of runnable tasks on the scheduler. These may
        either be new tasks, tasks that were previously unrunnable, or tasks that were
        already fed into this method already.

        Attach newly observed tasks to either the desired span or to ("default", ).
        Update TaskGroup.span_id and wipe TaskState.annotations["span"].
        """
        default_span = None

        for ts in tss:
            # You may have different tasks belonging to the same TaskGroup but to
            # different spans. If that happens, arbitrarily force everything onto the
            # span of the earliest encountered TaskGroup.
            tg = ts.group
            if not tg.span_id:
                ann = ts.annotations.get("span")
                if ann:
                    span = self._ensure_span(ann["name"], ann["ids"])
                else:
                    if not default_span:
                        default_span = self._ensure_default_span()
                    span = default_span

                tg.span_id = span.id
                span.groups.add(tg)

            # The span may be completely different from the one referenced by the
            # annotation, due to the TaskGroup collision issue explained above.
            # Remove the annotation to avoid confusion, and instead rely on
            # distributed.scheduler.TaskState.group.span_id and
            # distributed.worker_state_machine.TaskState.span_id.
            ts.annotations.pop("span", None)

    def _ensure_default_span(self) -> Span:
        """Return the currently active default span, or create one if the previous one
        terminated. In other words, do not reuse the previous default span if all tasks
        that were not explicitly annotated with :func:`spans` on the client side are
        finished.
        """
        defaults = self.spans_search_by_name["default",]
        if defaults and not defaults[-1].done:
            return defaults[-1]
        return self._ensure_span(("default",), (str(uuid.uuid4()),))

    def _ensure_span(self, name: tuple[str, ...], ids: tuple[str, ...]) -> Span:
        """Create Span if it doesn't exist and return it"""
        try:
            return self.spans[ids[-1]]
        except KeyError:
            pass

        assert len(name) == len(ids)
        assert len(name) > 0

        parent = None
        for i in range(1, len(name)):
            parent = self._ensure_span(name[:i], ids[:i])

        span = Span(name=name, id_=ids[-1], parent=parent)
        self.spans[span.id] = span
        self.spans_search_by_name[name].append(span)
        for tag in name:
            self.spans_search_by_tag[tag].append(span)
        if parent:
            parent.children.append(span)
        else:
            self.root_spans.append(span)

        return span

    def find_by_tags(self, *tags: str) -> Iterator[Span]:
        """Yield all spans that contain any of the given tags.
        When a tag is shared both by a span and its (grand)children, only return the
        parent.
        """
        by_level = defaultdict(list)
        for tag in tags:
            for sp in self.spans_search_by_tag[tag]:
                by_level[len(sp.name)].append(sp)

        seen = set()
        for _, level in sorted(by_level.items()):
            seen.update(level)
            for sp in level:
                if sp.parent not in seen:
                    yield sp

    def merge_by_tags(self, *tags: str) -> Span:
        """Return a synthetic Span which is the sum of all spans containing the given
        tags
        """
        return Span.merge(*self.find_by_tags(*tags))

    def heartbeat(
        self, ws: WorkerState, data: dict[str, dict[tuple[str, ...], float]]
    ) -> None:
        """Triggered by SpansWorkerExtension.heartbeat().

        Populate :meth:`Span.cumulative_worker_metrics` with data from the worker.

        See also
        --------
        SpansWorkerExtension.heartbeat
        Span.cumulative_worker_metrics
        """
        for span_id, metrics in data.items():
            span = self.spans[span_id]
            for k, v in metrics.items():
                span._cumulative_worker_metrics[k] += v

class SpansWorkerExtension:
    """Worker extension for spans support"""

    worker: Worker

    def __init__(self, worker: Worker):
        self.worker = worker

    def heartbeat(self) -> dict[str, dict[tuple[str, ...], float]]:
        """Apportion the metrics that do have a span to the Spans on the scheduler

        Returns
        -------
        ``{span_id: {("execute", prefix, activity, unit): value}}``

        See also
        --------
        SpansSchedulerExtension.heartbeat
        Span.cumulative_worker_metrics
        """
        out: defaultdict[str, dict[tuple[str, ...], float]] = defaultdict(dict)
        for k, v in self.worker.digests_total_since_heartbeat.items():
            if isinstance(k, tuple) and k[0] == "execute":
                _, span_id, prefix, activity, unit = k
                assert span_id is not None
                out[span_id]["execute", prefix, activity, unit] = v
        return dict(out)


import cv2
import os
import numpy as np
import sys

data_path = os.getcwd()+'./Face_Mask_Dataset/Test'
if os.path.exists(data_path):
  dir_list = os.listdir(data_path)
  print("Lista directorios y archivos:", dir_list)
  labels = []
  facesData = []
  label = 0
  for name_dir in dir_list:
    print (name_dir)
    sub_dir = data_path +"/" + name_dir;
    print ("Sub dir, ",{sub_dir})
    for file_name in os.listdir(sub_dir):
      imagen_path= sub_dir + "/" + file_name
      print ("Image File name, ",{imagen_path})
      imagen_redimencionada = cv2.resize(imagen_path, (72,72))
      imagen = cv2.imread(imagen_redimencionada)
      image = cv2.imread(imagen_path)
      cv2.imshow("Imagenes", image)
      cv2.waitKey(10)
      facesData.append(image)
      labels.append(label)
    label += 1

  print("Etiqueta 0: ", np.count_nonzero(np.array(labels) == 0))
  print("Etiqueta 1: ", np.count_nonzero(np.array(labels) == 1))
  # LBPH FaceRecognizer
  face_mask = cv2.face.LBPHFaceRecognizer_create()

  # Entrenamiento
  print("No molestar estoy entrenando ahora ... 😆💪")
  face_mask.train(facesData, np.array(labels))

  # Almacenar modelo
  # face_mask.write("mascarillas_model.xml")
  print("El modelo ya esta listo y etrenado 😎")

else:
  print("👎 NO ES POSIBLE CARGAR LOS DATOS PARA EL ENTRENAMIENTO, {}".format(data_path))
  sys.exit(1)


import librosa
import numpy as np

class Song(object):

    def __init__(self, track, part_size=25):
        self.stft = librosa.stft(librosa.to_mono(track))
        padding = np.zeros((self.stft.shape[0], part_size // 2))
        self.stftPadded = np.abs(
            np.concatenate((padding, librosa.stft(librosa.to_mono(track)), padding, padding), axis=1))
        self.size = part_size

    def __iter__(self):
        return Song.SongIterator(self.stftPadded, self.size)

    class SongIterator:

        def __init__(self, stft, part_size):
            self.stft = stft
            self.index = 0
            self.step = part_size
            self.end_index = stft.shape[1]

        def __iter__(self):
            return self

        def __next__(self):
            if self.index + self.step < self.end_index:
                start_index = self.index
                end_index = self.index + self.step
                self.index += 1
                return self.stft[:, start_index: end_index]
            else:
                raise StopIteration