# -*- coding: utf-8 -*-
"""
Created on Fri Aug 13 17:05:15 2021

@author: liuhaolin
"""

import os,time
import requests
import re
from selenium import webdriver
from selenium.common.exceptions import UnexpectedAlertPresentException
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains  #鼠标动作
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import selenium.common.exceptions  #支持浏览器插件
import pandas as pd
from datetime import date,datetime, timedelta
import random
from tqdm import tqdm

#浏览器相关
def driver_options():
    global driver
    global wait
    # 设置get直接返回，不再等待界面加载完成
    desired_capabilities = DesiredCapabilities.CHROME
    desired_capabilities["pageLoadStrategy"] = "none"
    options = webdriver.ChromeOptions()

    # 附带本地插件
    # extension_path = r'D:\Reports\Tools\SellerSprite2.7.0_0.crx'
    # options.add_extension(extension_path)

    # 启用本地带插件的浏览器 chrome://version/  个人资料路径
    options.add_argument("--user-data-dir="+r"D:/Program Files/GoogleCache/Chrome/UserData/")
    # options.add_argument("--user-data-dir="+r"C:/Users/liuhaolin/AppData/Local/Google/Chrome/User Data/")

    # 无窗口模式
    # options.add_argument('--headless')

    # 禁止硬件加速，避免严重占用cpu
    options.add_argument('--disable-gpu')
    # 关闭安全策略
    options.add_argument("disable-web-security")
    # 禁止图片加载
    # options.add_experimental_option('prefs', {'profile.managed_default_content_settings.images': 2})
    # 隐藏Chrome正在受到自动软件的控制
    options.add_argument('disable-infobars')
    # 设置开发者模式启动，该模式下webdriver属性为正常值
    options.add_experimental_option('excludeSwitches', ['enable-automation'])
    #指定浏览器分辨率
    options.add_argument('window-size=1920x1080')

    # 模拟移动设备
    # options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"')

    #需要指定Google驱动的文件位置,chromedriver_Home     下载地址：http://npm.taobao.org/mirrors/chromedriver/
    driver = webdriver.Chrome(chrome_options=options,executable_path="D:/Reports/Tools/chromedriver")
    # 返回驱动等待的变量
    # wait = WebDriverWait(driver, 15)
    driver.maximize_window()

if __name__ == '__main__':

    # SITE = 'https://www.sif.com/search'
    # SITE = 'https://app.isellerpal.com/data/asinreverse'
    SITE = 'https://members.helium10.com/cerebro?'

    #ASIN列表Excel地址以及一次多少条
    path = r"C:\Users\liuhaolin\Desktop\666.xlsx"
    LoadBunch = 10

    driver_options()

    Now_Time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

    i = 0

    df = pd.read_excel(path, sheet_name = 0)
    #ASIN列转列表
    AsinList = df['ASIN'].values.tolist()[:200]
    StepByLoadBunch = [AsinList[i:i+LoadBunch] for i in range(0,len(AsinList),LoadBunch)]

    for y,x in enumerate(tqdm(StepByLoadBunch)):

        time.sleep(random.randint(5,10))

        Try_Count = 0
        while Try_Count <=3:
            try:
                driver.get(SITE)

                #打乱下ASIN顺序以免某些狗掉的ASIN导致无法搜索
                random.shuffle(x)

                ASINs = " ".join(x) + ' '

                # #SIF版本
                # time.sleep(random.randint(5,10))
                # AsinSearch = driver.find_element_by_xpath('//*[@placeholder="输入ASIN，多个请使用逗号分隔"]')
                # AsinSearch.clear()
                # AsinSearch.send_keys(ASINs)

                # driver.find_element_by_xpath("//*[contains(text(),'搜 索')]").click()

                # time.sleep(3)
                # driver.find_element_by_xpath("//*[@class='batch']/*[contains(text(),'批量下载流量词')]").click()

                # time.sleep(3)
                # driver.find_element_by_xpath("//*[@class='el-checkbox__input']").click()

                # time.sleep(3)
                # driver.find_element_by_xpath("//*[@class='batch']/*[contains(text(),'下载流量词')]").click()

                # #isellerpal版本
                # time.sleep(random.randint(5,10))
                # driver.find_element_by_xpath("//*[@class='el-popover__reference-wrapper']//*[contains(text(),'热门搜索词')]").click()
                # time.sleep(0.5)
                # driver.find_element_by_xpath("//*[@class='el-dropdown-menu__item']//*[contains(text(),'全部关键词')]").click()
                # time.sleep(0.5)
                # AsinSearch = driver.find_element_by_xpath('//*[@placeholder="请输入ASIN"]')
                # AsinSearch.send_keys(ASINs)
                # time.sleep(1)
                # driver.find_element_by_xpath("//*[@type='button']//*[@class='el-icon-search']").click()

                # time.sleep(random.randint(5,10))
                # driver.find_element_by_xpath("//*[@type='button']/*[contains(text(),'下载')]").click()

                # #Helium10版本
                WebDriverWait(driver, 72).until(ec.visibility_of_element_located((By.XPATH,"//*[@class='cerebro-search-form']//*[contains(text(),'Get Keywords')]")))

                time.sleep(1)

                AsinSearch = driver.find_element_by_xpath("//*[@class='multi-asin-search-container']//input")
                AsinSearch.send_keys(ASINs)

                time.sleep(2)
                driver.find_element_by_xpath("//*[@class='cerebro-search-form']//*[contains(text(),'Get Keywords')]").click()

                time.sleep(2)
                try:
                    driver.find_element_by_xpath("//*[@class='sa-button-container']//*[contains(text(),'New Search')]").click()

                finally:
                    WebDriverWait(driver, 72).until(ec.visibility_of_element_located((By.XPATH,"//*[@class='dropdown float-right']//button")))

                    time.sleep(1)

                    driver.find_element_by_xpath("//*[@class='dropdown float-right']//button").click()
                    time.sleep(1)
                    driver.find_element_by_xpath("//*[@class='dropdown float-right show']//*[@data-format='xlsx']").click()

                    WebDriverWait(driver, 72, 0.5).until(ec.visibility_of_element_located((By.XPATH,"//*[@class='tooltipster-box']//*[contains(text(),'Exported!')]")))
                    time.sleep(random.randint(1,3))
                    i += 1
                    break

            except:
                Try_Count += 1
                print("获取失败,第%s次尝试刷新"%Try_Count)
                driver.refresh()
                time.sleep(5)
                continue

driver.quit()

print('搞完了，最后一个ASIN：%s'%x[:-1])

# End_Time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

# # 发送到钉钉群
# json_data = {
#           "msgtype": "text",
#           "text": {
#               "content": "AMZ(SearchTerm)\n启动时间：{0}\n结束时间：{1}\n数据量：{2}\最后ASIN：{3}".format(Now_Time,End_Time,i*10,ASINs[:-1])  # 发送内容
#           },
#           "at": {
#               "atMobiles": [
#               ],
#               "isAtAll": False  # 是否要@某位用户
#           }
#       }

# ding_url = 'https://oapi.dingtalk.com/robot/send?access_token=' \
#                     '66571ff131625d3a0f9f8a053333939a571ff902ee1f9a8a5af8a8dc5de04b9f'
#                     # 公司钉钉群 '314ba5ccba18c377ec9d688543a6b5afcb2baf6319023e29878278453f3ab96c'
#                     # GOGO哒     '4d7674403085618ece3003a8166f5f7bce07f160851e496f78e2abc4c5026e4a'
#                     # 执行记录    '66571ff131625d3a0f9f8a053333939a571ff902ee1f9a8a5af8a8dc5de04b9f'

# requests.post(url=ding_url, json=json_data)
# print('信息发送成功。')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 20 15:51:23 2020

@author: emmadevin
"""

import os
import sys
import pandas as pd
import numpy as np
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter

#os.environ["CARTOPY_USER_BACKGROUNDS"] = "/home/opt/anaconda3/lib/python3.7/site-packages/cartopy/data/raster/natural_earth"

bias = pd.read_csv('/Users/emmadevin/Work/USGS Work/USGS PLUM/Data/LME_SoCal/bias.csv')
siteTerms = pd.read_csv('/Users/emmadevin/Work/USGS Work/USGS PLUM/Data/LME_SoCal/site_terms.csv')
eventTerms = pd.read_csv('/Users/emmadevin/Work/USGS Work/USGS PLUM/Data/LME_SoCal/event_terms.csv')

dataset = pd.read_csv('/Users/emmadevin/Work/USGS Work/USGS PLUM/Data/SoCal_withMMIpred.csv')

locations = dataset[['StationID','StationLatitude','StationLongitude']]
locations =  locations.drop_duplicates(subset = 'StationID')
locations = locations.reset_index()

df = locations.merge(siteTerms, how='inner',indicator=False)

low = df['TotalResidual']>-1
high = df['TotalResidual']<1

df1 = df[low]
df1 = df1[high]

lon = df1['StationLongitude']
lat = df1['StationLatitude']
site = df1['TotalResidual']

data_crs = ccrs.PlateCarree()

# s = site
# S = np.array(site)
# S1 = S.reshape(len(lon), len(lat))
# S2 = np.transpose(S1)
SiteTermsCoordinates = pd.DataFrame()

SiteTermsCoordinates['lat']=lat
SiteTermsCoordinates['lon']=lon
SiteTermsCoordinates['site']=site

SiteTermsCoordinates.to_csv('/Users/emmadevin/Work/USGS Work/USGS PLUM/Data/SiteTermsCoord.csv')

ax = plt.axes(projection=ccrs.PlateCarree())
#ax.background_img(name='BM', resolution='low')
#ax.stock_img()
ax.set_global()
ax.coastlines()
ax.set_xticks([-125,-120,-119,-118,-117,-116,-115,-110], crs=ccrs.PlateCarree())
ax.set_yticks([-90, -60, -30, 0, 30,33,34,35,40,45, 60, 90], crs=ccrs.PlateCarree())
ax.set_extent([-119, -117.5, 33.5, 34.5], crs=ccrs.PlateCarree())
lon_formatter = LongitudeFormatter(zero_direction_label=True)
lat_formatter = LatitudeFormatter()
ax.xaxis.set_major_formatter(lon_formatter)
ax.yaxis.set_major_formatter(lat_formatter)
plt.scatter(lon, lat, s = 4, transform=data_crs, c=site, cmap='seismic')
#plt.contour(lon, lat, S2, cmap='seismic')
cbar = plt.colorbar()
cbar.set_label('Site Term')
#cbar.set_clim(-1.0, 1.0)
plt.title('Map of LA with Site Terms')
plt.scatter(-118.2437, 33.9522, s = 6000, facecolors = 'none', edgecolors = 'k', label = 'LA')


from abc import ABC
from abc import abstractmethod
import time

import torch

class TimerBase(ABC):

    def __init__(self, name):
        self.name = name

    @abstractmethod
    def start(self, barrier=False):
        pass

    @abstractmethod
    def stop(self, barrier=False):
        pass

    @abstractmethod
    def reset(self):
        pass

    @abstractmethod
    def elapsed(self, reset=True, barrier=False):
        pass

class DummyTimer(TimerBase):

    def __init__(self):
        super().__init__('dummy timer')

    def start(self, barrier=False):
        return

    def stop(self, barrier=False):
        return

    def reset(self):
        return

    def elapsed(self, reset=True, barrier=False):
        raise Exception('dummy timer should not be used to '
                        'calculate elapsed time')

class Timer(TimerBase):
    """
    Comment on using `barrier`: If this flag is passed, then all
    the caller processes will wait till all reach the timing routine.
    It is up to the user to make sure all the ranks in `barrier_group`
    call it otherwise, it will result in a hang.
    Comment on `barrier_group`: By default it is set to None which
    in torch distributed land, it will result in the global communicator.
    """

    def __init__(self, name):
        super().__init__(name)
        self._elapsed = 0.0
        self._started = False
        # Note that None will default to the global process group
        self._barrier_group = None
        self._start_time = time.time()

    def set_barrier_group(self, barrier_group):
        self._barrier_group = barrier_group

    def start(self, barrier=False):
        """Start the timer."""
        assert not self._started, 'timer has already been started'
        if barrier:
            torch.distributed.barrier(group=self._barrier_group)
        torch.cuda.synchronize()
        self._start_time = time.time()
        self._started = True

    def stop(self, barrier=False):
        """Stop the timer."""
        assert self._started, 'timer is not started'
        if barrier:
            torch.distributed.barrier(group=self._barrier_group)
        torch.cuda.synchronize()
        self._elapsed += (time.time() - self._start_time)
        self._started = False

    def reset(self):
        """Reset timer."""
        self._elapsed = 0.0
        self._started = False

    def elapsed(self, reset=True, barrier=False):
        """Calculate the elapsed time."""
        _started = self._started
        # If the timing in progress, end it first.
        if self._started:
            self.stop(barrier=barrier)
        # Get the elapsed time.
        _elapsed = self._elapsed
        # Reset the elapsed time
        if reset:
            self.reset()
        # If timing was in progress, set it back.
        if _started:
            self.start(barrier=barrier)
        return _elapsed

class Timers:
    """Group of timers."""

    def __init__(self, log_level, log_option):
        self._log_level = log_level
        self._log_option = log_option
        self._timers = {}
        self._log_levels = {}
        self._dummy_timer = DummyTimer()
        self._max_log_level = 2

    def __call__(self, name, log_level=None):
        # If the timer has already been set, then check if the log-level
        # is provided, it matches the one that the timer was created with.
        if name in self._timers:
            if log_level is not None:
                assert log_level == self._log_levels[name], \
                    'input log level {} does not match already existing '\
                    'log level {} for {} timer'.format(
                        log_level, self._log_levels[name], name)
            return self._timers[name]
        # If timer does not exist and no log level is provided,
        # set it to the max log level which is 2.
        if log_level is None:
            log_level = self._max_log_level
        assert log_level <= self._max_log_level, \
            'log level {} is larger than max supported log level {}'.format(
                log_level, self._max_log_level)
        # Now if the input log level is larger than the one set for
        # the timers class, just ignore it and return a dummy timer.
        if log_level > self._log_level:
            return self._dummy_timer
        # Otherwise, initalize the timer and set the level.
        self._timers[name] = Timer(name)
        self._log_levels[name] = log_level
        return self._timers[name]

    def _get_elapsed_time_all_ranks(self, names, reset, barrier):
        """
        Assumptions:
            - All the ranks call this function.
            - `names` are identical on all ranks.
        If the above assumptions are not met, calling this function will
        result in hang.
        Arguments:
            - names: list of timer names
            - reset: reset the timer after recording the elapsed time
            - barrier: if set, do a global barrier before time measurments
        """

        # First make sure all the callers are in sync.
        if barrier:
            torch.distributed.barrier()

        world_size = torch.distributed.get_world_size()
        rank = torch.distributed.get_rank()

        # Here we can use gather on the rank we want to print the
        # timing, however, there is no gather_base support in
        # pytorch yet. It is simpler to deal with a single tensor
        # and since we are only gathering a small amount of data,
        # it should be ok to use all-gather instead of gather.
        rank_name_to_time = torch.zeros((world_size, len(names)),
                                        dtype=torch.float,
                                        device=torch.cuda.current_device())
        for i, name in enumerate(names):
            if name in self._timers:
                # Here we don't need to pass the barrier flag as all
                # the processes are already in sync. This avoids the
                # issue of different timers having different barrier
                # groups inside their class.
                rank_name_to_time[rank, i] = self._timers[name].elapsed(
                    reset=reset)

        # See the note above for why we are not using gather.
        torch.distributed._all_gather_base(rank_name_to_time.view(-1),
                                           rank_name_to_time[rank, :].view(-1))

        return rank_name_to_time

    def _get_global_min_max_time(self, names, reset, barrier, normalizer):
        """Report only min and max times across all ranks."""

        rank_name_to_time = self._get_elapsed_time_all_ranks(names, reset,
                                                             barrier)
        name_to_min_max_time = {}
        for i, name in enumerate(names):
            rank_to_time = rank_name_to_time[:, i]
            # filter out the ones we did not have any timings for
            rank_to_time = rank_to_time[rank_to_time > 0.0]
            # If the timer exists:
            if rank_to_time.numel() > 0:
                name_to_min_max_time[name] = (
                    rank_to_time.min().item() / normalizer,
                    rank_to_time.max().item() / normalizer)
        return name_to_min_max_time

    def _get_global_min_max_time_string(self, names, reset, barrier,
                                        normalizer, max_only):
        name_to_min_max_time = self._get_global_min_max_time(
            names, reset, barrier, normalizer)
        if not name_to_min_max_time:
            return None
        output_string = '(min, max) time across ranks (ms):'
        for name in name_to_min_max_time:
            min_time, max_time = name_to_min_max_time[name]
            if max_only:
                output_string += '\n    {}: {:.2f}'.format(
                    (name+' ').ljust(48, '.'), max_time)
            else:
                output_string += '\n    {}: ({:.2f}, {:.2f})'.format(
                    (name+' ').ljust(48, '.'), min_time, max_time)
        return output_string

    def _get_all_ranks_time_string(self, names, reset, barrier, normalizer):
        """Report times across all ranks."""
        rank_name_to_time = self._get_elapsed_time_all_ranks(names, reset,
                                                             barrier)

        output_string = 'times across ranks (ms):'
        no_reported_timing = True
        for i, name in enumerate(names):
            not_yet_found = True
            for rank in range(torch.distributed.get_world_size()):
                if rank_name_to_time[rank, i] > 0:
                    no_reported_timing = False
                    if not_yet_found:
                        not_yet_found = False
                        output_string += '\n  {}:'.format(name)
                    output_string += '\n     rank {:2d}: {:.2f}'.format(
                        rank, rank_name_to_time[rank, i] / normalizer)
        if no_reported_timing:
            return None
        return output_string

    def log(self, names, rank=None, normalizer=1.0, reset=True, barrier=False):
        """Log a group of timers."""

        # Print.
        assert normalizer > 0.0
        if self._log_option in ['max', 'minmax']:
            max_only = False
            if self._log_option == 'max':
                max_only = True
            output_string = self._get_global_min_max_time_string(
                names, reset, barrier, normalizer/1000.0, max_only)
        elif self._log_option == 'all':
            output_string = self._get_all_ranks_time_string(names,
                                                            reset, barrier,
                                                            normalizer/1000.0)
        else:
            raise Exception('unknown timing log option {}'.format(
                self._log_option))

        # If no input rank is provided, log on last rank.
        if rank is None:
            rank = torch.distributed.get_world_size() - 1
        if rank == torch.distributed.get_rank() and output_string is not None:
            print(output_string, flush=True)

    def write(self, names, writer, iteration, normalizer=1.0,
              reset=False, barrier=False):
        """Write timers to a tensorboard writer
        Note that we only report maximum time across ranks to tensorboard.
        """
        # currently when using add_scalars,
        # torch.utils.add_scalars makes each timer its own run, which
        # polutes the runs list, so we just add each as a scalar
        assert normalizer > 0.0
        name_to_min_max_time = self._get_global_min_max_time(
            names, reset, barrier, normalizer)
        if writer is not None:
            for name in name_to_min_max_time:
                _, max_time = name_to_min_max_time[name]
                writer.add_scalar(name + '-time', max_time, iteration)


import os, pickle, logging, numpy as np
from tqdm import tqdm

from .. import utils as U
from .transformer import pre_normalization

class NTU_Reader():
    def __init__(self, args, root_folder, transform, ntu60_path, ntu120_path, **kwargs):
        self.max_channel = 3
        self.max_frame = 300
        self.max_joint = 25
        self.max_person = 4
        self.select_person_num = 2
        self.dataset = args.dataset
        self.split = args.dataset_args['split']
        self.progress_bar = not args.no_progress_bar
        self.transform = transform
        self.mutual_only = ('mutual' in self.dataset)
        self.mutual_action_map = {50: 1, 51: 2, 52: 3, 53: 4, 54: 5, 55: 6, 56: 7, 57: 8, 58: 9, 59: 10, 60: 11, 106: 12, 107: 13, 108: 14, 109: 15, 110: 16, 111: 17, 112: 18, 113: 19, 114: 20, 115: 21, 116: 22, 117: 23, 118: 24, 119: 25, 120: 26}

        # Set paths
        ntu_ignored = '{}/ntu_ignore.txt'.format(os.path.dirname(os.path.realpath(__file__)))
        if self.transform:
            self.out_path = '{}/transformed/{}'.format(root_folder, self.dataset+'-'+self.split)
        else:
            self.out_path = '{}/{}'.format(root_folder, self.dataset+'-'+self.split)
        U.create_folder(self.out_path)

        # Divide train and eval samples
        training_samples = dict()

        training_samples['xview'] = [2, 3]
        training_samples['xsub'] = [
            1, 2, 4, 5, 8, 9, 13, 14, 15, 16, 17, 18, 19, 25, 27, 28, 31, 34, 35,
            38, 45, 46, 47, 49, 50, 52, 53, 54, 55, 56, 57, 58, 59, 70, 74, 78,
            80, 81, 82, 83, 84, 85, 86, 89, 91, 92, 93, 94, 95, 97, 98, 100, 103
        ]
        training_samples['xset'] = set(range(2, 33, 2))
        self.training_sample = training_samples[self.split]

        # Get ignore samples
        try:
            with open(ntu_ignored, 'r') as f:
                self.ignored_samples = [line.strip() + '.skeleton' for line in f.readlines()]
        except:
            logging.info('')
            logging.error('Error: Wrong in loading ignored sample file {}'.format(ntu_ignored))
            raise ValueError()

        # Get skeleton file list
        self.file_list = []
        for folder in [ntu60_path, ntu120_path]:
            for filename in os.listdir(folder):
                self.file_list.append((folder, filename))
            if '120' not in self.dataset:  # for NTU 60, only one folder
                break

    def read_file(self, file_path):
        skeleton = np.zeros((self.max_person, self.max_frame, self.max_joint, self.max_channel), dtype=np.float32)
        with open(file_path, 'r') as fr:
            frame_num = int(fr.readline())
            for frame in range(frame_num):
                person_num = int(fr.readline())
                for person in range(person_num):
                    person_info = fr.readline().strip().split()
                    joint_num = int(fr.readline())
                    for joint in range(joint_num):
                        joint_info = fr.readline().strip().split()
                        skeleton[person,frame,joint,:] = np.array(joint_info[:self.max_channel], dtype=np.float32)
        return skeleton[:,:frame_num,:,:], frame_num

    def get_nonzero_std(self, s):  # (T,V,C)
        index = s.sum(-1).sum(-1) != 0  # select valid frames
        s = s[index]
        if len(s) != 0:
            s = s[:, :, 0].std() + s[:, :, 1].std() + s[:, :, 2].std()  # three channels
        else:
            s = 0
        return s

    def gendata(self, phase):
        sample_data = []
        sample_label = []
        sample_path = []
        sample_length = []
        iterizer = tqdm(sorted(self.file_list), dynamic_ncols=True) if self.progress_bar else sorted(self.file_list)
        for folder, filename in iterizer:
            if filename in self.ignored_samples:
                continue

            # Get sample information
            file_path = os.path.join(folder, filename)
            setup_loc = filename.find('S')
            camera_loc = filename.find('C')
            subject_loc = filename.find('P')
            action_loc = filename.find('A')
            setup_id = int(filename[(setup_loc+1):(setup_loc+4)])
            camera_id = int(filename[(camera_loc+1):(camera_loc+4)])
            subject_id = int(filename[(subject_loc+1):(subject_loc+4)])
            action_class = int(filename[(action_loc+1):(action_loc+4)])

            # filter out single-person samples
            if self.mutual_only and action_class not in self.mutual_action_map:
                continue

            # Distinguish train or eval sample
            if self.split == 'xview':
                is_training_sample = (camera_id in self.training_sample)
            elif self.split == 'xsub':
                is_training_sample = (subject_id in self.training_sample)
            elif self.split == 'xset':
                is_training_sample = (setup_id in self.training_sample)
            else:
                logging.info('')
                logging.error('Error: Do NOT exist this dataset {}'.format(self.dataset))
                raise ValueError()
            if (phase == 'train' and not is_training_sample) or (phase == 'eval' and is_training_sample):
                continue

            # Read one sample
            data = np.zeros((self.max_channel, self.max_frame, self.max_joint, self.select_person_num), dtype=np.float32)
            skeleton, frame_num = self.read_file(file_path)

            # Select person by max energy
            energy = np.array([self.get_nonzero_std(skeleton[m]) for m in range(self.max_person)])
            index = energy.argsort()[::-1][:self.select_person_num]
            skeleton = skeleton[index]
            data[:,:frame_num,:,:] = skeleton.transpose(3, 1, 2, 0)

            sample_data.append(data)
            sample_path.append(file_path)
            if self.mutual_only:
                action_class = self.mutual_action_map[action_class]
            sample_label.append(action_class - 1)  # to 0-indexed
            sample_length.append(frame_num)

        # Save label
        with open('{}/{}_label.pkl'.format(self.out_path, phase), 'wb') as f:
            pickle.dump((sample_path, list(sample_label), list(sample_length)), f)

        # Transform data
        sample_data = np.array(sample_data)
        if self.transform:
            sample_data = pre_normalization(sample_data, progress_bar=self.progress_bar)

        # Save data
        np.save('{}/{}_data.npy'.format(self.out_path, phase), sample_data)

    def start(self):
        for phase in ['train', 'eval']:
            logging.info('Phase: {}'.format(phase))
            self.gendata(phase)


# (c) adarsh-goel
import time
import math
import logging
import secrets
import mimetypes
from ..vars import Var
from aiohttp import web
from ..bot import StreamBot
from Adarsh import StartTime
from ..utils.custom_dl import TGCustomYield, chunk_size, offset_fix
from Adarsh.utils.render_template import render_page
from ..utils.time_format import get_readable_time
routes = web.RouteTableDef()
from urllib.parse import quote_plus
kg18="ago"

@routes.get("/", allow_head=True)
async def root_route_handler(request):
    return web.json_response({"status": "running",
                              "maintained_by": "Adarsh Goel",
                              "uptime": get_readable_time(time.time() - StartTime),
                              "Bot was last updated": get_readable_time(time.time()),
                              "ago":"",
                              "telegram_bot": '@'+(await StreamBot.get_me()).username,
                              "Bot Version":"3.0.1"})

@routes.get("/watch/{message_id}")
async def stream_handler(request):
    try:
        message_id = int(request.match_info['message_id'])
        return web.Response(text=await render_page(message_id), content_type='text/html')
    except ValueError as e:
        logging.error(e)
        raise web.HTTPNotFound

@routes.get("/download/{message_id}")
@routes.get("/{message_id}")
async def old_stream_handler(request):
    try:
        message_id = int(request.match_info['message_id'])
        return await media_streamer(request, message_id)
    except ValueError as e:
        logging.error(e)
        raise web.HTTPNotFound

async def media_streamer(request, message_id: int):
    range_header = request.headers.get('Range', 0)
    media_msg = await StreamBot.get_messages(Var.BIN_CHANNEL, message_id)
    file_properties = await TGCustomYield().generate_file_properties(media_msg)
    file_size = file_properties.file_size

    if range_header:
        from_bytes, until_bytes = range_header.replace('bytes=', '').split('-')
        from_bytes = int(from_bytes)
        until_bytes = int(until_bytes) if until_bytes else file_size - 1
    else:
        from_bytes = request.http_range.start or 0
        until_bytes = request.http_range.stop or file_size - 1

    req_length = until_bytes - from_bytes

    new_chunk_size = await chunk_size(req_length)
    offset = await offset_fix(from_bytes, new_chunk_size)
    first_part_cut = from_bytes - offset
    last_part_cut = (until_bytes % new_chunk_size) + 1
    part_count = math.ceil(req_length / new_chunk_size)
    body = TGCustomYield().yield_file(media_msg, offset, first_part_cut, last_part_cut, part_count,
                                      new_chunk_size)

    file_name = file_properties.file_name if file_properties.file_name \
        else f"{secrets.token_hex(2)}.jpeg"
    mime_type = file_properties.mime_type if file_properties.mime_type \
        else f"{mimetypes.guess_type(file_name)}"

    return_resp = web.Response(
        status=206 if range_header else 200,
        body=body,
        headers={
            "Content-Type": mime_type,
            "Content-Range": f"bytes {from_bytes}-{until_bytes}/{file_size}",
            "Content-Disposition": f'attachment; filename="{file_name}"',
            "Accept-Ranges": "bytes",
        }
    )

    if return_resp.status == 200:
        return_resp.headers.add("Content-Length", str(file_size))

    return return_resp


# -*- coding: utf-8 -*-

import os
from hashlib import sha224

import pymongo
import redis
from scrapy.exceptions import DropItem

from .tools.bloomfilter import BloomFilter

class BloomFilterPipeline(object):
    def __init__(self, host, port, db, capacity, error_rate):
        self.redis = redis.StrictRedis(host=host, port=port, db=db)
        self.bloomfilter_articles = BloomFilter(self.redis, capacity=capacity, error_rate=error_rate,
                                                redis_key='JOVI_ARTICLES')
        self.bloomfilter_urls = BloomFilter(self.redis, capacity=capacity, error_rate=error_rate, redis_key='JOVI_URLS')

    @classmethod
    def from_crawler(cls, crawler):
        return cls(host=crawler.settings.get('REDIS_HOST'),
                   port=crawler.settings.get('REDIS_PORT'),
                   db=crawler.settings.get('REDIS_DB'),
                   capacity=crawler.settings.get('BLOOM_CAPACITY'),
                   error_rate=crawler.settings.get('BLOOM_ERROR_RATE'),
                   )

    def process_item(self, item, spider):
        self.bloomfilter_urls.add(item['article_url'])
        if self.bloomfilter_articles.contains(item['article_title']):
            raise DropItem('文章已经存在')
        else:
            self.bloomfilter_articles.add(item['article_title'])
            return item

class Duppipline(object):
    def process_item(self, item, spider):
        if item['article_title']:
            if len(item['article_content']) < 200:
                print('文章短----%s----%s' % (item['article_title'], item['article_url']))
                raise DropItem
            else:
                return item
        else:
            print('没有标题----%s' % item['article_url'])
            raise DropItem

class Redispipline(object):
    def __init__(self, redis_host, redis_port, redis_db):
        self.redis_host = redis_host
        self.redis_port = redis_port
        self.redis_db = redis_db
        self.r = redis.Redis(host=self.redis_host, port=self.redis_port, db=self.redis_db)

    @classmethod
    def from_crawler(cls, crawler):
        return cls(redis_host=crawler.settings.get('REDIS_HOST'),
                   redis_port=crawler.settings.get('REDIS_PORT'),
                   redis_db=crawler.settings.get('REDIS_DB')
                   )

    def process_item(self, item, spider):
        name = spider.__class__.__name__
        # name='test'
        self.r.sadd(name, item['article_url'])
        # 生成fingerprint
        s1 = sha224(item['article_title'].encode('utf-8'))
        fp = s1.hexdigest()
        if self.r.sismember('article_title_fp_new', fp):
            print('文章重复----%s-----%s' % (item['article_title'], item['article_url']))
            raise DropItem
        else:
            self.r.sadd('article_title_fp_new', fp)
            return item

class Mongopipline(object):
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DB')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        collection_name = spider.__class__.__name__
        self.db[collection_name].insert_one(dict(item))

        # self.db['test'].update({'article_url': item['article_url']}, {'$set': dict(item)}, True)
        return item

# 对于有三级标签的spider,使用这个pipeline
class To_csv(object):
    def __init__(self, START_DIR, stats):
        self.stats = stats
        self.START_DIR = START_DIR
        self.stats.set_value('*总数', 0)
        if os.path.exists(self.START_DIR):
            os.chdir(self.START_DIR)
        else:
            os.mkdir(self.START_DIR)
            os.chdir(self.START_DIR)

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            START_DIR=crawler.settings.get('START_DIR'),
            stats=crawler.stats
        )

    def process_item(self, item, spider):
        first_tag = item['first_tag']
        second_tag = item['second_tag']
        third_tag = item['third_tag']
        if os.path.exists('%s\%s' % (self.START_DIR, first_tag)):
            os.chdir('%s\%s' % (self.START_DIR, first_tag))
        else:
            os.mkdir('%s\%s' % (self.START_DIR, first_tag))
            self.stats.set_value('*' + first_tag, 0)
            os.chdir('%s\%s' % (self.START_DIR, first_tag))
        if os.path.exists('%s' % second_tag):
            os.chdir('.\%s' % second_tag)
        else:
            os.mkdir('%s' % second_tag)
            self.stats.set_value('*' + first_tag + '_' + second_tag, 0)
            os.chdir('.\%s' % second_tag)
        with open(third_tag + ".txt", 'a', encoding='utf-8') as file:
            file.write((item['article_title'] + "," + item['article_content'] + '\n'))
            print(item['article_title'] + "," + item['article_content'][:150] + '\n')
            # print('成功保存----%s' % self.stats.get_value('item_scraped_count'))
        self.stats.inc_value('*' + first_tag)
        self.stats.inc_value('*' + first_tag + '_' + second_tag)
        self.stats.inc_value('*总数')
        return item

    # def close_spider(self, spider): #统计函数
    #     os.chdir('e:\\统计')
    #     name = spider.__class__.__name__
    #     a = self.stats.get_stats()
    #     date = self.stats.get_value('start_time').strftime('%m%d')
    #     self.stats.set_value('start_time', 0)
    #     self.stats.set_value('finish_time', 0)
    #     with open('e:\\统计\\%s%s.txt' % (name, date), 'w', encoding='utf-8') as f:
    #         for i in a.keys():
    #             if i.startswith('*'):
    #                 lines = '%s:%d\n' % (i, a[i])
    #                 f.write(lines)

# 对于只有二级标签的spider,使用这个pipeline
class To_csv1(object):
    def __init__(self, stats, START_DIR):
        self.stats = stats
        self.START_DIR = START_DIR
        self.stats.set_value('*总数', 0)
        if os.path.exists(self.START_DIR):
            os.chdir(self.START_DIR)
        else:
            os.mkdir(self.START_DIR)
            os.chdir(self.START_DIR)

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            START_DIR=crawler.settings.get('START_DIR'),
            stats=crawler.stats
        )

    def process_item(self, item, spider):
        first_tag = item['first_tag']
        second_tag = item['second_tag']
        if os.path.exists('%s\%s' % (self.START_DIR, first_tag)):
            os.chdir('%s\%s' % (self.START_DIR, first_tag))
        else:
            self.stats.set_value('*' + second_tag, 0)
            os.mkdir('%s\%s' % (self.START_DIR, first_tag))
            os.chdir('%s\%s' % (self.START_DIR, first_tag))
        with open(second_tag + ".txt", 'a', encoding='utf-8') as file:
            file.write((item['article_title'] + "," + item['article_content'] + '\n'))
            print(item['article_title'] + "," + item['article_content'][:150] + '\n')
            self.stats.inc_value('*' + second_tag)
            self.stats.inc_value('*总数')
        return item

    def close_spider(self, spider):
        pass
        # os.chdir('e:\\统计')
        # name = spider.__class__.__name__
        # a = self.stats.get_stats()
        # date = self.stats.get_value('start_time').strftime('%m%d')
        # self.stats.set_value('start_time', 0)
        # self.stats.set_value('finish_time', 0)
        # with open('e:\\统计\\%s%s.txt' % (name, date), 'w', encoding='utf-8') as f:
        #     record = list()
        #     for k,v in a.items():
        #         if k.startswith('*'):
        #             record.append('%s:%s'%(k,v))
        #     record.sort()
        #     f.write(str(record))
        # os.chdir('e:\\统计')
        # name = spider.__class__.__name__
        # a = self.stats.get_stats()
        # date = self.stats.get_value('start_time').strftime('%m%d')
        # self.stats.set_value('start_time', 0)
        # self.stats.set_value('finish_time', 0)
        # with open('e:\\统计\\%s%s.txt' % (name, date), 'w', encoding='utf-8') as f:
        #     for i in a.keys():
        #         if i.startswith('*'):
        #             lines = '%s:%d\n' % (i, a[i])
        #             f.write(lines)


import os
import re

import wget
from qiniu import Auth, put_file, etag

import config

# 加载配置
access_key = config.access_key
secret_key = config.secret_key
bucket_name = config.bucket_name
ROOT_DIR_PATH = config.ROOT_DIR_PATH
OLD_IMG_URL_PREFIX = config.OLD_IMG_URL_PREFIX
OLD_IMG_URL_REPLACE_PREFIX = config.OLD_IMG_URL_REPLACE_PREFIX
NEW_IMG_URL_PREFIX = config.NEW_IMG_URL_PREFIX

def upload(local_path, upload_path):
    """
    上传方法
    :param local_path:
    :param upload_path:
    :return:
    """
    print("正在上传 {} ，上传到 {}".format(local_path, upload_path))
    # 构建鉴权对象
    q = Auth(access_key, secret_key)
    key = upload_path
    # 生成上传 Token，可以指定过期时间等
    token = q.upload_token(bucket_name, key, 3600)

    # 要上传文件的本地路径
    localfile = local_path
    ret, info = put_file(token, key, localfile, version='v2')
    print(info)

    assert ret['key'] == key
    assert ret['hash'] == etag(localfile)

def subdir_list(dirname):
    """获取目录下所有子目录名
    @param dirname: str 目录的完整路径
    @return: list(str) 所有子目录完整路径组成的列表
    """
    return list(filter(os.path.isdir, map(lambda filename: os.path.join(dirname, filename), os.listdir(dirname))))

def file_list(dirname, ext='.csv'):
    """获取目录下所有特定后缀的文件
    @param dirname: str 目录的完整路径
    @param ext: str 后缀名, 以点号开头
    @return: list(str) 所有子文件名(不包含路径)组成的列表
    """
    return list(filter(lambda filename: os.path.splitext(filename)[1] == ext, os.listdir(dirname)))

def list_file_path(root_dirname, ext='.md'):
    """
    获取文件路径列表
    :param root_dirname:
    :param ext:
    :return:
    """
    file_path_list = []
    dirname_list = subdir_list(root_dirname)
    for dirname in dirname_list:
        filename_list = file_list(dirname=dirname, ext='.md')
        for filename in filename_list:
            file_path_list.append(r'{}\{}'.format(dirname, filename))
    return file_path_list

def replace_file_img_url(file_path):
    """
    替换文件中的图床地址，通过下载图片再上传到七牛云实现
    :param file_path:
    :return:
    """
    # 存储读取的文件内容，用于更新文件，从而实现修改的效果
    file_data = ""
    i = 0
    with open(file_path, "r", encoding="utf-8") as f:
        for line in f:
            # 正则匹配当前行
            # 读内容
            res = re.search(OLD_IMG_URL_PREFIX + r'.+\.(png|jpg|gif|jpeg|webp)', line)
            if res:
                # 提取url
                img_url = line[res.span()[0]: res.span()[1]]

                # 替换成备用的url
                img_url = img_url.replace(OLD_IMG_URL_PREFIX, OLD_IMG_URL_REPLACE_PREFIX)
                print(os.path.dirname(__file__) + os.path.basename(img_url))

                # 通过备用的url下载到本地
                img_path = wget.download(img_url, "temp/" + os.path.basename(img_url))

                # 上传到七牛云
                upload(img_path, img_url[len(OLD_IMG_URL_REPLACE_PREFIX) + 1:])

                # 删除本地图片
                os.remove(img_path)

                # 更新当前行的图床地址
                line = line.replace(OLD_IMG_URL_PREFIX, NEW_IMG_URL_PREFIX)
            file_data += line

    # 写出文件
    with open(file_path, "w", encoding="utf-8") as f:
        f.write(file_data)

if __name__ == '__main__':
    for file_path in list_file_path(ROOT_DIR_PATH):
        print("正在处理：" + file_path)
        replace_file_img_url(file_path)


from fastapi import Depends, FastAPI, exceptions
from loguru import logger

from app.routers import media_route, tweet_route, user_route
from app.utils.database import engine
from app.utils.exceptions import (
    logging_dependency,
    my_exception_handler,
    my_validation_exception_handler,
)
from app.utils.logconfig import init_logger

init_logger()
app = FastAPI(debug=True)
app.add_exception_handler(
    exceptions.RequestValidationError, handler=my_validation_exception_handler
)
app.add_exception_handler(exceptions.HTTPException, handler=my_exception_handler)
app.include_router(
    user_route.router,
    prefix="/api",
    tags=["users"],
    dependencies=[Depends(logging_dependency)],
)
app.include_router(
    media_route.router,
    prefix="/api",
    tags=["medias"],
    dependencies=[Depends(logging_dependency)],
)
app.include_router(
    tweet_route.router,
    prefix="/api",
    tags=["tweets"],
    dependencies=[Depends(logging_dependency)],
)

@app.on_event("startup")
async def startup():
    logger.info("Server started")

@app.on_event("shutdown")
async def shutdown():
    await engine.dispose()


import os
import re
import textract

from pyspark import SparkContext, SQLContext
from pyspark.rdd import RDD
from pyspark.sql import SparkSession, DataFrame, Row, functions as F
from pyspark.sql.types import StringType, StructField, StructType

START_DIR = "/home/jeff/git/BigData/026-ProfilesPySpark/profiles/"
STOP_WORDS_FILENAME: str = "terrier-stop.txt"

FULL_WEIGHTED_OUTPUT_FILENAME:str  = "FullWeighted.out"
JOINED_OUTPUT_FILENAME:str = "Joined.out"

WORD_COLUMN_NAME:  str    = "word"
COUNT_COLUMN_NAME: str    = "count"
WEIGHTED_COLUMN_NAME: str = "weight"
FREQUENCY_COUNT_FN: str   = "freq_count"

# FOUR PROFILES TO GATHER DATA ON
THE_CHOSEN_FOUR:list = [ "JeffreyReiherProfile.doc",\
                         "VicenteGuevaraProfile.doc",\
                         "KislayJhaProfile.docx",\
                         "ChrisPeng-Profile.docx"\
                       ]

TOOL_LIST: list =      [ "kafka",  "spark",   "emr",     "mr",        "mapreduce",
                         "flume",  "aws",     "hadoop",  "spark",     "data bricks",
                         "hive",   "presto",  "airflow", "ec2",       "amazon",
                         "python", "pyspark", "redshift" "snowflake", "googleads"
                       ]
SC: SparkContext = SparkContext()
SPARK: SparkSession = SparkSession(SC)

total_counts_for_the_four: list = []

def read_all_profiles() -> str:
    """
    Open each profile in the directory and extract the text
    :return: a string of all the profiles concatenated
    """
    text = ""

    # READ EACH OF THE PROFILE DOCUMENTS AND ADD TO TEXT VAR
    for dir, subdirs, files in os.walk(START_DIR):
        for fname in files:
            full_path = START_DIR + fname
            try:
                text += str(textract.process(full_path).decode("utf-8") ).lower()
            except:
                print("Exception on " + fname)
    return text.lower()

def read_one_profile( fname: str ) -> str:
    # READ EACH OF THE PROFILE DOCUMENTS AND ADD TO TEXT VAR
    full_path = START_DIR + fname
    return str(textract.process(full_path).decode("utf-8") ).lower()

#def into_lines( text:str ) -> list:
def into_unique_lines(text: str) -> RDD:
    """
    Split a string into a list of strings by separating the supplied string at each carriage return
    :param text: The text to split
    :return: a list of strings - one for each line in the text
    """
    result = []
    split_lines: list = re.split("\n", text )
    for i in split_lines:
        result.append( i.strip() )

    return SC.parallelize(result).distinct()

def into_words( rdd_in: RDD ) -> RDD:
    words: RDD = rdd_in.flatMap( lambda x: x.split(' ') ).filter( lambda x: len(x) > 1 )
    return words.map( lambda x: re.sub( ',()',"", x ))

def remove_stop_words( rdd_in: RDD ) -> RDD:
    file = open(STOP_WORDS_FILENAME, "r")
    sw_list: list = file.read().split("\n")
    return rdd_in.filter( lambda x: x not in sw_list )

def compute_sum_of_values( rdd_in: RDD ) ->  int:
    """
    Given an RDD, compute sum of all the counts (the 2nd item in the tuple)
    :param rdd_in: the rdd to process
    :return: the total number of words counted in this rdd
    """
    sum: int = 0
    for (i,j) in rdd_in:
        sum+=j

    return sum

def write_weighted_rdd( weighted_rdd: RDD, file_name: str ):
    sql_ctx: SQLContext = SQLContext(SC)
    df:DataFrame = sql_ctx.createDataFrame(weighted_rdd, [ WORD_COLUMN_NAME,COUNT_COLUMN_NAME,
                                                           WEIGHTED_COLUMN_NAME
                                                         ]).orderBy( F.col(COUNT_COLUMN_NAME).desc() )

    df.coalesce(1).write.format("com.databricks.spark.csv")\
                .options(header="true")\
                .save("file:///home/jeff/git/BigData/026-ProfilesPySpark/results/" + file_name )

def clean_row( r: Row ) -> Row:
    for i in [1,3,5,7]:
        if r[i] == "":
            r[i] = 0
    r[9] = r[1] + r[3] + r[5] + r[7]
    return r

def write_joined_df( df: DataFrame, file_name: str ) -> DataFrame:
    #df=df.fillna(0, subset= [ "count_0",  "count_1",  "count_2",  "count_3"  ] )
    #df=df.fillna(0, subset= [ "weight_0", "weight_1", "weight_2", "weight_3" ] )
    df=df.fillna( 0 )
    df=df.withColumn("total", df[1] + df[3] + df[5] + df[7])

    sum_df: DataFrame = df.groupBy().sum()
    total_words:int = sum_df.collect()[0][4]

    #freq_db_fn = F.udf(lambda cnt: freq_db(cnt, total_words) )
    df=df.withColumn("total-weight", F.round(df[9]/total_words*100,3) )

    df.coalesce(1).write.format("com.databricks.spark.csv")\
                .options(header="true")\
                .save("file:///home/jeff/git/BigData/026-ProfilesPySpark/results/" + file_name )
    return df

def calculate_weights( rdd_in: RDD ) -> RDD:
    rdd_in = rdd_in.collect()
    total: int = compute_sum_of_values( rdd_in )
    total_counts_for_the_four.append( total )

    return SC.parallelize(rdd_in).map(lambda x: (x[0], x[1], round(100*x[1]/total)))

def total_weight_of_list( words: list, joined_weights: DataFrame ) -> list:
    # WORD LIST TO DF
    words_df = SPARK.createDataFrame(words, StringType()).toDF( WORD_COLUMN_NAME )
    #joined_weights = joined_weights.fillna(0)

    # INNER JOIN DFs
    words_df = words_df.join( joined_weights, on=WORD_COLUMN_NAME, how="inner")

    # COMPUTE TOTAL COUNTS AND WEIGHTS FOR EACH "PROFILE"
    words_df = words_df.agg(F.sum("count_0"), F.sum("count_1"), F.sum("count_2"), F.sum("count_3")).collect()

    return [ 100*words_df[0][i]/total_counts_for_the_four[i] for i in range(4) ]

def convert_to_words( lines: list ) -> RDD:
    words: RDD = into_words( u_lines )
    return remove_stop_words( words )

if __name__ == '__main__':
    # READ TEXT FROM DOC FILES AND CONVERT TO TEXT
    text: str    = read_all_profiles()

    # CONVERT TEXT INTO A LIST OF UNIQUE LINES
    u_lines: RDD = into_unique_lines( text )

    '''
    with open( "profile-lines.txt", "w" ) as file:
        for i in u_lines.collect():
            file.write(i)
            file.write('\n')
    quit()
    '''

    # CONVERT TEXT INTO A LIST OF WORDS (WITH STOP WORDS REMOVED)
    word_rdd: RDD = convert_to_words( u_lines )

    # MAP AND REDUCE THE DATA
    reduced: RDD = word_rdd.map(lambda x: (x, 1)).reduceByKey(lambda i, j: i + j)

    # CALCULATE THE WEIGHT
    weighted_rdd: RDD = calculate_weights( reduced )

    # WRITE CSV TO DISK
    write_weighted_rdd( weighted_rdd, FULL_WEIGHTED_OUTPUT_FILENAME )

    join_df: DataFrame = None
    filenum:int = 0
    for f in THE_CHOSEN_FOUR:
        # READ TEXT FROM DOC FILES AND CONVERT TO TEXT
        text = read_one_profile( f )

        # CONVERT TEXT INTO A LIST OF LINES
        u_lines = into_unique_lines( text )

        # CONVERT TEXT INTO A LIST OF WORDS (WITH STOP WORDS REMOVED)
        word_rdd = convert_to_words( u_lines )

        # MAP AND REDUCE THE DATA
        reduced = word_rdd.map(lambda x: (x, 1)).reduceByKey(lambda i, j: i + j)

        # CALCULATE THE WEIGHT
        weighted_rdd = calculate_weights( reduced )

        # WRITE CSV TO DISK
        write_weighted_rdd( weighted_rdd, f )

        # sql_ctx: SQLContext = SQLContext(SC)
        sql_ctx: SQLContext = SQLContext(SC)
        new_df: DataFrame\
            = sql_ctx.createDataFrame( weighted_rdd,
                                       [ WORD_COLUMN_NAME,
                                         COUNT_COLUMN_NAME + "_" + str(filenum),
                                         WEIGHTED_COLUMN_NAME + "_" + str(filenum)
                                       ]
                                      )

        if join_df == None:
            join_df = new_df
        else:
            join_df = join_df.join( new_df, on=WORD_COLUMN_NAME, how="outer" )

        filenum += 1

    join_df = write_joined_df( join_df, JOINED_OUTPUT_FILENAME )

    for i in total_weight_of_list(TOOL_LIST, join_df ):
        print(i)


import tensorflow as tf
import numpy as np
import pickle
from tools import accuracy

##########parameters of data###########
image_size = 48
num_labels = 7
num_channels = 1
#######################################

###############load data###############
pickle_file = './data/fer2013.pickle'
f = open(file=pickle_file, mode='rb')
data = pickle.load(file=f)

train_data, train_labels = data['train_data'], data['train_labels']
valid_data, valid_labels = data['valid_data'], data['valid_labels']
test_data, test_labels = data['test_data'], data['test_labels']
del data

def reformat(dataset, labels): # we have to reshapebecause of conv layers
    dataset = dataset.reshape((-1, image_size, image_size, num_channels)).astype(np.float32)
    return dataset, labels
train_data, train_labels = reformat(train_data, train_labels)
valid_data, valid_labels = reformat(valid_data, valid_labels)
test_data, test_labels = reformat(test_data, test_labels)

print('Loaded from pickle.')
print('Training set:', train_data.shape, ', labels:', train_labels.shape)
print('Validation set:', valid_data.shape, ', labels:', valid_labels.shape)
print('Test set:', test_data.shape, ', labels:', test_labels.shape)
print('#'*40)
#######################################

###########hyper parameters############
batch_size = 64

reg_const = 0.05 # l2 reg constant

# learning rate: exponential decay
initial_learning_rate = 0.04
decay_steps = 1000
decay_rate = 0.65

filter1 = 5 # patch_size
filter2 = 2
depth1 = 20
depth2 = 50

hidden = 500 # num of neurons in fully connected hidden layer

num_steps = 64*20
#######################################

graph = tf.Graph()
with graph.as_default():

    # input data
    tf_train_data = tf.placeholder(shape=(batch_size, image_size, image_size, num_channels),
                                   dtype=tf.float32)
    tf_train_labels = tf.placeholder(shape=(batch_size, num_labels),
                                     dtype=tf.float32)
    tf_valid_data = tf.constant(value=valid_data)
    tf_test_data = tf.constant(value=test_data)

    # variables
    global_step = tf.Variable(0) # counts number of steps taken

    l1_conv = tf.Variable(tf.truncated_normal([filter1, filter1, num_channels, depth1], stddev=0.1))
    b1_conv = tf.Variable(tf.zeros([depth1]))
    l2_conv = tf.Variable(tf.truncated_normal([filter2, filter2, depth1, depth2], stddev=0.1))
    b2_conv = tf.Variable(tf.zeros([depth2]))

    l3 = tf.Variable(tf.truncated_normal([(image_size//4)*(image_size//4)*depth2, hidden], stddev=0.1))
    b3 = tf.Variable(tf.zeros([hidden]))
    l4 = tf.Variable(tf.truncated_normal([hidden, num_labels], stddev=0.1))
    b4 = tf.Variable(tf.zeros([num_labels]))

    weights_and_biases = [l1_conv, b1_conv, l2_conv, b2_conv, l3, b3, l4, b4] # list of all weights and biases

    # computation
    def model(X):
        cl1 = tf.nn.conv2d(input=X,
                           filter=l1_conv,
                           strides=[1,1,1,1],
                           padding='SAME')
        cl1_relu = tf.nn.relu(cl1 + b1_conv)
        cl1_maxpool = tf.nn.max_pool(value=cl1_relu,
                                     ksize=[1,2,2,1], # velikost v jednotlivyh dimenzich: 1 v batch, 2 v height a width, 1 v channels
                                     strides=[1,2,2,1], # posun v jednotlivych dimenzich
                                     padding='SAME')

        cl2 = tf.nn.conv2d(input=cl1_maxpool,
                           filter=l2_conv,
                           strides=[1,1,1,1],
                           padding='SAME')
        cl2_relu = tf.nn.relu(cl2 + b2_conv)
        cl2_maxpool = tf.nn.max_pool(value=cl2_relu,
                                     ksize=[1,2,2,1],
                                     strides=[1,2,2,1],
                                     padding='SAME')

        shape = cl2_maxpool.get_shape().as_list()
        cl2_reshape = tf.reshape(tensor=cl2_maxpool, shape=[shape[0], shape[1]*shape[2]*shape[3]]) # for example from [128, 7, 7, 16] to [16, 784] = [128, 7*7*16]
        cl3 = tf.nn.relu(tf.matmul(cl2_reshape, l3) + b3)
        output = tf.matmul(cl3, l4) + b4
        return output

    logits = model(tf_train_data) # logits for training

    penalty = 0.5*reg_const*sum([tf.nn.l2_loss(x) for x in weights_and_biases]) # penalties for weights and biases
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits,tf_train_labels)) + penalty

    # optimizer
    learning_rate = tf.train.exponential_decay(learning_rate=initial_learning_rate,
                                               global_step=global_step,
                                               decay_steps=decay_steps,
                                               decay_rate=decay_rate)
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss=loss, global_step=global_step)

    # predictions
    train_prediction = tf.nn.softmax(logits)
    valid_prediction = tf.nn.softmax(model(X=valid_data))
    test_prediction = tf.nn.softmax(model(X=test_data))

with tf.Session(graph=graph) as session:
    tf.initialize_all_variables().run()
    print('Initialized\n{}'.format('-'*40))
    for step in range(num_steps):
        offset = (step*batch_size) % (train_labels.shape[0] - batch_size) # we slide through all data
        batch_data = train_data[offset:(offset+batch_size), :]
        batch_labels = train_labels[offset:(offset+batch_size), :]
        feed_dict = {tf_train_data: batch_data, # feed by this batch data
                     tf_train_labels: batch_labels}
        _, l, predictions = session.run(fetches=[optimizer, loss, train_prediction],
                                        feed_dict=feed_dict)

        if (step % 500 == 0):
            print('Minibatch loss at step {}: {:.3f}'.format(step, l))
            print('Learning rate: {:.3f}'.format(learning_rate.eval()))
            print('Minibatch accuracy: {:.1f}%'.format(accuracy(predictions, batch_labels)))
            print('Validation accuracy: {:.1f}%'.format(accuracy(valid_prediction.eval(), valid_labels)))
            print('-'*40)
    print('Test accuracy: {:.1f}%'.format(accuracy(test_prediction.eval(), test_labels)))


from telegram.ext import ConversationHandler
from handlers.food_handlers.food_handlers import (
    get_day_ration_handler,
    get_one_ration_handler,
    select_food_handler
)
from handlers.workout_handlers.test_workout_handlers import (
    back_2_main_menu_handler
)
from handlers.general_handlers import (
    encorrect_msg_main_handler
)

from utils.navigation import (
    CHOICE_FOOD_KEY,
    BACK_KEY,
    MAIN_MENU_KEY
)

# Выбор рациона питания
conv_food = ConversationHandler(
    entry_points=[select_food_handler],
    states={
        # ADD_USER_TRAIN: [conv_add_user_train], Вписать все в choice_train_key
        CHOICE_FOOD_KEY: [
            get_day_ration_handler,
            get_one_ration_handler,
            back_2_main_menu_handler,
            encorrect_msg_main_handler
        ],
    },
    fallbacks=[],
    map_to_parent={
        BACK_KEY: MAIN_MENU_KEY,
        MAIN_MENU_KEY: MAIN_MENU_KEY
    },
    name="food_conversation",
    persistent = True,
)


#!/usr/bin/python3

from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='Postcodes',

    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
    version='1.0.0',

    description='library that supports validating and formatting post codes for UK.',
    long_description='Write a library that supports validating and formatting post codes for UK.\
      The details of which post codes are valid and which are the parts they consist of can be found at \
      https://en.wikipedia.org/wiki/Postcodes_in_the_United_Kingdom#Formatting.',

    # The project's main homepage.
    url='https://github.com/garyb1/PostCodes',

    # Author details
    author='Gary Byrne',
    author_email='garyb1@live.ie',

    # Choose your license
    license='MIT',
    py_modules=['postcode/postcode'],
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        'Development Status :: 3 - Alpha',

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Build Tools',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: MIT License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ],

    # What does your project relate to?
    keywords='UK Postcodes',
    install_requires=['geopy==1.11.0', 'requests==2.20.0'],
)


import nilearn
from dfsio import readdfs, writedfs
from surfproc import view_patch_vtk, patch_color_labels
import numpy as np
from scipy.spatial import cKDTree

def interpolate_labels(fromsurf=[], tosurf=[]):
    ''' interpolate labels from surface to to surface'''
    tree = cKDTree(fromsurf.vertices)
    d, inds = tree.query(tosurf.vertices, k=1, p=2)
    tosurf.labels = fromsurf.labels[inds]
    return tosurf

hemi = 'right'

outbase = 'data/BCI'
brannetome_base = '/ImagePTE1/ajoshi/code_farm/svreg/USCBrainMulti/Brainnetome/BCI-Brainnetome'
uscbrain_base = '/ImagePTE1/ajoshi/code_farm/svreg/USCBrainMulti/USCBrain'
hemi_surf_uscbrain_file = uscbrain_base + '.'+hemi+'.mid.cortex.dfs'
hemi_surf_brainnetome_file = brannetome_base + '.'+hemi+'.mid.cortex.dfs'
hemi_surf_out_file = outbase + '.'+hemi+'.mid.cortex.dfs'
hemi_surf_smooth_out_file = outbase + '.'+hemi+'.smooth.mid.cortex.dfs'

uscbrain_ids = [[502, 503, 504, 505]]
brainnnetome_ids = [[163, 164, 165, 166,
                     167, 168, 169, 170, 171, 172, 173, 174]]
uscbrain_new_ids = [[563, 564, 565, 566,
                     567, 568, 569, 570, 571, 572, 573, 574]]

uscbrain_hemi = readdfs(hemi_surf_uscbrain_file)
uscbrain_hemi_labs = uscbrain_hemi.labels.copy()
brainnetome_hemi = readdfs(hemi_surf_brainnetome_file)
brainnetome_hemi_vertices = brainnetome_hemi.vertices
brainnetome_hemi.vertices = uscbrain_hemi.vertices
uscbrain_hemi = patch_color_labels(uscbrain_hemi)
brainnetome_hemi = patch_color_labels(brainnetome_hemi)

view_patch_vtk(uscbrain_hemi)
view_patch_vtk(brainnetome_hemi)

class btmlCl:
    pass

btml = btmlCl()

class ubCl:
    pass

ub = ubCl()

for i in range(len(uscbrain_ids)):
    uid = uscbrain_ids[i]
    bid = brainnnetome_ids[i]
    uid_new = uscbrain_new_ids[i]

    u_ind = np.in1d(uscbrain_hemi.labels, uid).nonzero()[0]
    ub.labels = uscbrain_hemi.labels[u_ind]
    ub.vertices = uscbrain_hemi.vertices[u_ind, ]

    b_ind = np.in1d(brainnetome_hemi.labels, bid).nonzero()[0]
    btml.labels = brainnetome_hemi.labels[b_ind]

    btml.vertices = brainnetome_hemi.vertices[b_ind, ]

    print('Replacing', uid)
    print('With', uid_new)

    interpolate_labels(fromsurf=btml, tosurf=ub)

    for ind, r in enumerate(bid):
        ub.labels[ub.labels == r] = uid_new[ind]

    uscbrain_hemi.labels[u_ind] = ub.labels

uscbrain_hemi = patch_color_labels(uscbrain_hemi)
brainnetome_hemi = patch_color_labels(brainnetome_hemi)

view_patch_vtk(uscbrain_hemi)

#writedfs(hemi_surf_out_file, uscbrain_hemi)

uscbrain_hemi.vertices = brainnetome_hemi_vertices

writedfs(hemi_surf_out_file, uscbrain_hemi)

uscbrain_hemi.labels = uscbrain_hemi_labs

writedfs(hemi_surf_smooth_out_file, uscbrain_hemi)


import torch
import torch.nn.functional as F
from torch import nn

'''LeNet_3_32_32 in PyTorch.'''

# codes are import from https://github.com/xternalz/WideResNet-pytorch/blob/master/wideresnet.py
# original author: xternalz

import math

class Expression(nn.Module):
    def __init__(self, func):
        super(Expression, self).__init__()
        self.func = func

    def forward(self, input):
        return self.func(input)

class Model(nn.Module):
    def __init__(self, i_c=1, n_c=10):
        super(Model, self).__init__()

        self.conv1 = nn.Conv2d(i_c, 32, 5, stride=1, padding=2, bias=True)
        self.pool1 = nn.MaxPool2d((2, 2), stride=(2, 2), padding=0)

        self.conv2 = nn.Conv2d(32, 64, 5, stride=1, padding=2, bias=True)
        self.pool2 = nn.MaxPool2d((2, 2), stride=(2, 2), padding=0)

        self.flatten = Expression(lambda tensor: tensor.view(tensor.shape[0], -1))
        self.fc1 = nn.Linear(7 * 7 * 64, 1024, bias=True)
        self.fc2 = nn.Linear(1024, n_c)

    def forward(self, x_i, _eval=False):

        if _eval:
            # switch to eval mode
            self.eval()
        else:
            self.train()

        x_o = self.conv1(x_i)
        x_o = torch.relu(x_o)
        x_o = self.pool1(x_o)

        x_o = self.conv2(x_o)
        x_o = torch.relu(x_o)
        x_o = self.pool2(x_o)

        x_o = self.flatten(x_o)

        x_o = torch.relu(self.fc1(x_o))

        self.train()

        return self.fc2(x_o)

class BasicBlock(nn.Module):
    def __init__(self, in_planes, out_planes, stride, dropRate=0.0):
        super(BasicBlock, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.relu1 = nn.LeakyReLU(0.1, inplace=True)
        self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                               padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_planes)
        self.relu2 = nn.LeakyReLU(0.1, inplace=True)
        self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1,
                               padding=1, bias=False)
        self.droprate = dropRate
        self.equalInOut = (in_planes == out_planes)
        self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
                                                                padding=0, bias=False) or None

    def forward(self, x):
        if not self.equalInOut:
            x = self.relu1(self.bn1(x))
        else:
            out = self.relu1(self.bn1(x))
        out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x)))
        if self.droprate > 0:
            out = F.dropout(out, p=self.droprate, training=self.training)
        out = self.conv2(out)
        return torch.add(x if self.equalInOut else self.convShortcut(x), out)

class NetworkBlock(nn.Module):
    def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0):
        super(NetworkBlock, self).__init__()
        self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate)

    def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate):
        layers = []
        for i in range(int(nb_layers)):
            layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate))
        return nn.Sequential(*layers)

    def forward(self, x):
        return self.layer(x)

class WideResNet(nn.Module):
    def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0):
        super(WideResNet, self).__init__()
        nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor]
        assert ((depth - 4) % 6 == 0)
        n = (depth - 4) / 6
        block = BasicBlock
        # 1st conv before any network block
        self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1,
                               padding=1, bias=False)
        # 1st block
        self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate)
        # 2nd block
        self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate)
        # 3rd block
        self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate)
        # global average pooling and classifier
        self.bn1 = nn.BatchNorm2d(nChannels[3])
        self.relu = nn.LeakyReLU(0.1, inplace=True)
        self.fc = nn.Linear(nChannels[3], num_classes)
        self.nChannels = nChannels[3]

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.bias.data.zero_()

    def forward(self, x, _eval=False):
        if _eval:
            # switch to eval mode
            self.eval()
        else:
            self.train()

        out = self.conv1(x)
        out = self.block1(out)
        out = self.block2(out)
        out = self.block3(out)
        out = self.relu(self.bn1(out))
        out = F.avg_pool2d(out, 8)
        out = out.view(-1, self.nChannels)

        self.train()

        return self.fc(out)

class RestNetBasicBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride):
        super(RestNetBasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=stride, padding=1)
        self.bn2 = nn.BatchNorm2d(out_channels)

    def forward(self, x):
        output = self.conv1(x)
        output = F.relu(self.bn1(output))
        output = self.conv2(output)
        output = self.bn2(output)
        return F.relu(x + output)

class RestNetDownBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride):
        super(RestNetDownBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride[0], padding=1)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=stride[1], padding=1)
        self.bn2 = nn.BatchNorm2d(out_channels)
        self.extra = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride[0], padding=0),
            nn.BatchNorm2d(out_channels)
        )

    def forward(self, x):
        extra_x = self.extra(x)
        output = self.conv1(x)
        out = F.relu(self.bn1(output))

        out = self.conv2(out)
        out = self.bn2(out)
        return F.relu(extra_x + out)

class RestNet18(nn.Module):
    def __init__(self):
        super(RestNet18, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)
        self.bn1 = nn.BatchNorm2d(64)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

        self.layer1 = nn.Sequential(RestNetBasicBlock(64, 64, 1),
                                    RestNetBasicBlock(64, 64, 1))

        self.layer2 = nn.Sequential(RestNetDownBlock(64, 128, [2, 1]),
                                    RestNetBasicBlock(128, 128, 1))

        self.layer3 = nn.Sequential(RestNetDownBlock(128, 256, [2, 1]),
                                    RestNetBasicBlock(256, 256, 1))

        self.layer4 = nn.Sequential(RestNetDownBlock(256, 512, [2, 1]),
                                    RestNetBasicBlock(512, 512, 1))

        self.avgpool = nn.AdaptiveAvgPool2d(output_size=(1, 1))

        self.fc = nn.Linear(512, 10)

    def forward(self, x):
        out = self.conv1(x)
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = self.avgpool(out)
        out = out.reshape(x.shape[0], -1)
        out = self.fc(out)
        return out

class CNN_1_cifar10(nn.Module):
    def __init__(self):
        super(CNN_1_cifar10, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class Alex_1_cifar10(nn.Module):
    def __init__(self):
        super(Alex_1_cifar10, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(x.size()[0], -1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class net_cifar10(nn.Module):
    def __init__(self):
        # the input size = Batch *3*32*32
        super(net_cifar10, self).__init__()
        self.seq = torch.nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=(3, 3)),
            nn.ReLU(),
            nn.Conv2d(64, 64, (3, 3)),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2)),
            nn.Conv2d(64, 128, (3, 3)),
            nn.ReLU(),
            nn.Conv2d(128, 128, (3, 3)),
            nn.ReLU(),
            nn.MaxPool2d((2, 2)),
        )
        self.seq2 = nn.Sequential(
            nn.Linear(in_features=128 * 5 * 5, out_features=256),
            nn.ReLU(),
            nn.Linear(256, 256),
            nn.ReLU(),
            nn.Linear(256, 10)
        )

        self.modules_to_hook = ('seq.1',
                                'seq.3',
                                'seq.6',
                                'seq.8',
                                'seq2.4')

    def forward(self, x):
        r1 = self.seq(x)
        r1 = r1.view(r1.size(0), -1)
        r2 = self.seq2(r1)
        return r2

class LeNet_cifar10(nn.Module):
    def __init__(self):
        super(LeNet_cifar10, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        out = F.relu(self.conv1(x))
        out = F.max_pool2d(out, 2)
        out = F.relu(self.conv2(out))
        out = F.max_pool2d(out, 2)
        out = out.view(out.size(0), -1)
        out = F.relu(self.fc1(out))
        out = F.relu(self.fc2(out))
        out = self.fc3(out)
        return out

# x1 = torch.rand(1, 3, 32, 32)
# x2 = torch.rand(1, 1, 28, 28)
#
# net1 = net_cifar10()
# net2 = Net_mnist()
# print(net1(x1).size())
# print(net2(x2).size())

def function(data=[]):
    data.append(1)
    return data
function()
function()
function()
print(function())


from fasttext import load_model
import streamlit as st
import os
from gensim.utils import simple_preprocess
import wget

@st.cache(allow_output_mutation=True)
def download_model():
    # Download model
    model_name = "model.bin"
    model_url = "https://drive.google.com/file/d/1-7ZNbPdwBifcKFScNjb1GEop4gTyVEkc/view?usp=sharing"
    dir_path = os.path.dirname(os.path.realpath(__file__))
    file_model = wget.download(model_url, out=model_name)

    # Import model to FastText
    path = dir_path + "/" + model_name
    model = load_model(model_name)

    # Decrypto prediction to category name
    category_file_name = "/path_to_end_category.txt"
    path_to_end_category = {}
    with open(dir_path + category_file_name, encoding='utf8') as file:
        for line in file:
            key, value = line.split("_/_")
            path_to_end_category[int(key)] = value

download_model()

def make_prediction(corpus):
    corpus = simple_preprocess(corpus)
    res = model.predict(corpus)
    if res in path_to_end_category.keys():
        ans = path_to_end_category[res[0]]
    else:
        ans = "Категория для товара отстутвует"

    return ans

# _______________________Web________________________________________ #
# _______________________Title_____________________________________ #
st.title("Предскажем категорию товара по его названию и описанию")
st.write(
    """
    **Автор** Фомин Анатолий Витальевич
    **Контактные данные** +7(926)4395526, kmno4.172@gmail.com
    **Цель** Помочь продавцам сориентироваться среди тысяч категорий
    ***
    """)

# _______________________Page_Layout______________________________ #
col2, col3 = st.columns((10, 1))

st.text_input(
    label="Введи название и описание товара",
    value=""

)

# Предсказать КИН по данным:
pred = predict_1(dist=federal_distr, pvt=pvt, carbonate=is_carbonate, por=por, perm=perm, visc=visc)
# print("Предсказанный КИН:", pred)

st.write(
    f""" ## Предсказанный КИН на основании месторождений аналогов: {pred:.2f}
    """
)


import pyinotify
import logging
import asyncio
import os
import shlex
import xdg.DesktopEntry
import xdg.BaseDirectory
import xdg.Menu
from pathlib import PosixPath, Path
from typing import Optional, List, Union, Dict, Callable
import qubesadmin
import qubesadmin.vm
import qubesadmin.events

from . import constants

logger = logging.getLogger('qubes-appmenu')

def exec_parse(desktop_entry: xdg.DesktopEntry.DesktopEntry):
    """
    Parse Exec field according to specification and return an already-split
    exec command, ready to be used in subprocess.Popen et al.
    """
    split_str = shlex.split(desktop_entry.getExec())
    result = []
    for s in split_str:
        if s in ['%f', '%F', '%u', '%U', '%d', '%D', '%n', '%N', '%v',
                 '%m', '%k']:
            continue
        if s == '%i' and desktop_entry.getIcon():
            result.extend(['--icon', desktop_entry.getIcon()])
            continue
        if s == '%c':
            result.append(desktop_entry.getName())
            continue
        result.append(s)
    return result

class ApplicationInfo:
    """
    Class representing data within a single .desktop file.
    """
    def __init__(self, qapp, file_path):
        self.qapp: qubesadmin.Qubes = qapp
        self.file_path: PosixPath = file_path
        self.app_icon: Optional[str] = None
        self.vm_icon: Optional[str] = None
        self.app_name: Optional[str] = None
        self.sort_name: Optional[str] = None
        self.vm: Optional[qubesadmin.vm.QubesVM] = None
        self.entry_name: Optional[str] = None
        self.exec: List[str] = []
        self.disposable: bool = False
        self.categories = []
        self.entries: List = []

    def load_data(self, entry):
        """Fill own data with information from xdg.DesktopEntry provided."""
        vm_name = entry.get('X-Qubes-VmName') or None
        try:
            self.vm = self.qapp.domains[vm_name]
        except KeyError:
            self.vm = None

        self.app_name = entry.getName() or ''
        if self.vm:
            self.app_name = self.app_name.split(": ", 1)[-1]
        self.sort_name = str(self.app_name).lower()
        self.vm_icon = self.vm.icon if self.vm else None
        self.app_icon = entry.getIcon()
        self.disposable = bool(entry.get('X-Qubes-NonDispvmExec'))
        self.entry_name = entry.get('X-Qubes-AppName') or self.file_path.name
        if self.disposable:
            self.entry_name = constants.DISPOSABLE_PREFIX + self.entry_name
        self.exec = exec_parse(entry)

        self.categories = entry.getCategories()

        for menu_entry in self.entries:
            menu_entry.update_contents()

    def get_command_for_vm(self, vm=None):
        """Get execution command for a specified VM. We're not using contents
        of an Exec field directly because freshly-minted DispVMs don't have
         their own .desktop files."""
        command = self.exec
        if vm and not self.vm:
            logger.warning('Unexpected command: cannot run local'
                           ' application for a non-local VM: %s', vm)
            return command
        if vm and str(self.vm) != str(vm):
            # replace name of the old VM - used for opening apps from DVM
            # template in their child dispvm
            if len(command) < 6 or command[5] != str(self.vm):
                logger.error(
                    'Unexpected command for a disposable VM: %s', command)
                return []
            return command[:5] + [str(vm)] + command[6:]
        return command

    def is_qubes_specific(self):
        """Check if the current file represents a qubes-generated app."""
        return 'X-Qubes-VM' in self.categories

class DesktopFileManager:
    """
    Class that loads, caches and observes changes in .desktop files.
    """
    desktop_dirs = [
        Path(xdg.BaseDirectory.xdg_data_home) / 'applications',
        Path('/usr/share/applications')]

    # pylint: disable=invalid-name
    class EventProcessor(pyinotify.ProcessEvent):
        """pyinotify helper class"""
        def __init__(self, parent):
            self.parent = parent
            super().__init__()

        def process_IN_CREATE(self, event):
            """On file create, attempt to load it. This can lead to spurious
            warnings due to 0-byte files being loaded, but in some cases
            is necessary to correctly process files."""
            try:
                self.parent.load_file(event.pathname)
            except FileNotFoundError:
                self.parent.remove_file(event.pathname)

        def process_IN_DELETE(self, event):
            """
            On file delete, remove the tile and all its children menu entries
            """
            self.parent.remove_file(event.pathname)

        def process_IN_MODIFY(self, event):
            """On modify, simply attempt to laod the file again."""
            try:
                self.parent.load_file(event.pathname)
            except FileNotFoundError:
                self.parent.remove_file(event.pathname)

        def process_IN_MOVED_FROM(self, event):
            """On move from, act like delete happened."""
            self.process_IN_DELETE(event)

        def process_IN_MOVED_TO(self, event):
            """On move to, act like create happened."""
            self.process_IN_CREATE(event)

    def __init__(self, qapp):
        self.qapp = qapp
        self.watch_manager = None
        self.notifier = None
        self.watches = []
        self._callbacks: List[Callable] = []

        # directories used by Qubes menu tools, not necessarily all possible
        # XDG directories
        self.current_environments = \
            os.environ.get('XDG_CURRENT_DESKTOP', '').split(':')

        self.app_entries: Dict[Path, ApplicationInfo] = {}

        for directory in self.desktop_dirs:
            if not os.path.exists(directory):
                try:
                    os.makedirs(directory, exist_ok=True)
                except OSError:
                    # situation is strange, just ignore this directory
                    continue
            for file in os.listdir(directory):
                self.load_file(directory / file)
        self._initialize_watchers()

    def register_callback(self, func):
        """
        Register callbacks to be executed on newly loaded files.
        Will only be executed on correctly loaded ApplicationInfos. The callback
        should also add created widget (if any) to ApplicationInfo's entries
        field.
        """
        self._callbacks.append(func)
        for info in self.app_entries.values():
            func(info)

    def get_app_infos(self):
        """Get all available ApplicationInfos. Needed for initial loading
        of favorites."""
        for info in self.app_entries.values():
            yield info

    def remove_file(self, path: Union[str, Path]):
        """Remove a file provided by path from local cache. Also removes
        all child menu entries."""
        if isinstance(path, str):
            path = Path(path)
        app_info = self.app_entries.get(path)

        if app_info:
            for child in app_info.entries:
                parent = child.get_parent()
                parent.remove(child)
                parent.invalidate_filter()
            del self.app_entries[path]

    def load_file(self, path: Union[str, Path]):
        """
        Load a file. If file was already known, its ApplicationInfo is
        refreshed, otherwise a new ApplicationInfo object will be created
        and all callbacks registered will be executed."""
        if isinstance(path, str):
            path = Path(path)
            if not path.exists() or path.stat().st_size == 0:
                # event received while file was being deleted or created
                self.remove_file(path)
                return

        if not path.name.endswith('.desktop'):
            return

        try:
            entry = xdg.DesktopEntry.DesktopEntry(path)
        except Exception as ex:  # pylint: disable=broad-except
            logger.warning(
                'Cannot load desktop entry file %s: %s', path, str(ex))
            self.remove_file(path)
            return

        if not self._eligibility_check(entry):
            self.remove_file(path)
            return

        app_info = self.app_entries.get(path, None)
        if not app_info:
            new_entry = True
            app_info = ApplicationInfo(self.qapp, path)
            self.app_entries[path] = app_info
        else:
            new_entry = False
        app_info.load_data(entry)

        if new_entry:
            for func in self._callbacks:
                func(app_info)

    def _eligibility_check(self, entry: xdg.DesktopEntry.DesktopEntry):
        """Check if the loaded entry should be shown in the menu at all,
        based on current environment."""
        if entry.getHidden():
            return False
        if entry.getOnlyShowIn():
            if not set(entry.getOnlyShowIn()).intersection(
                    self.current_environments):
                return False
        if entry.getNotShowIn():
            if set(entry.getNotShowIn()).intersection(
                    self.current_environments):
                return False
        if entry.get('X-AppStream-Ignore'):
            return False
        return True

    def _initialize_watchers(self):
        """
        Initialize all watcher entities.
        """
        self.watch_manager = pyinotify.WatchManager()

        # pylint: disable=no-member
        mask = pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
               pyinotify.IN_MODIFY | pyinotify.IN_MOVED_FROM | \
               pyinotify.IN_MOVED_TO

        loop = asyncio.get_event_loop()

        self.notifier = pyinotify.AsyncioNotifier(
            self.watch_manager, loop,
            default_proc_fun=DesktopFileManager.EventProcessor(self))

        for path in self.desktop_dirs:
            self.watches.append(
                self.watch_manager.add_watch(
                    str(path), mask, rec=True, auto_add=True))