# -*- coding: UTF-8 -*-
import logging
import config_default
import time
from utils import crawler_utils
from utils import log_utils
from utils import search_weixinhao_utils
from get_article_list import get_wechat_name

def search_article(weixinhaos):
    """
    微信号爬取
    :param weixinhaos:微信号列表
    :param page_nos:待搜索的页面列表
    1
    2
    3
    :return:
    """
    for weixinhao, account_name in weixinhaos:
        logging.info('weixinhao：{},account_name:{} 开始'.format(weixinhao, account_name))
        weixinhao_url, account_name = search_weixinhao_utils.get_weixinhao_url(weixinhao)
        if weixinhao_url:
            search_weixinhao_utils.save_wechat_article(weixinhao_url, weixinhao, account_name)
        else:
            logging.error('weixinhao：{}失效'.format(weixinhao))
        crawler_utils.wait_sleep('两个微信号爬取间隔', '')

if __name__ == '__main__':
    is_first = True
    while True:
        # 初始化日志配置
        file_handler, stream_handler = log_utils.log_init()
        config = config_default.configs
        weixinhao_list = config['weixinhao']
        logging.info(u'爬取任务开始：')
        # 搜索公众号
        weixinhao_list = get_wechat_name()
        search_article(weixinhao_list)
        logging.info(u'爬取任务结束！')
        next_turn_seconds = config['next_turn_seconds']
        logging.info(u'休息{}秒后进行下一轮爬取'.format(next_turn_seconds))
        # time.sleep(next_turn_seconds)
        # 日志handler清除
        logging.getLogger().removeHandler(file_handler)
        logging.getLogger().removeHandler(stream_handler)

        # break, 用定时任务重启
        break


import sqlite3

conn = sqlite3.connect('database.db')

c = conn.cursor()
# c.execute("""CREATE TABLE mobie(
#     use_name varchar(32),
#     call_duration int
# )""")
c.execute("delete  FROM mobie ")

conn.commit()
conn.close()


import re
import asyncio
import subprocess
import telepot
import telepot.aio
from telepot import glance
from telepot.namedtuple import InlineKeyboardMarkup, InlineKeyboardButton

from config import telegram_bot, torrents, auth
from models.message import Command

class TorrentBot(telepot.aio.Bot):
    def __init__(self, token=None):
        super().__init__(token)

        self.torrent_commands = {
            'keyboard': [
                ['torrents list ▤', 'start torrent ✓', 'stop torrent ✕'],
            ],
            'one_time_keyboard': True,
            'resize_keyboard': True
        }

    async def on_chat_message(self, msg):
        content_type, chat_type, chat_id = telepot.glance(msg)

        if not self.auth(msg['from']):
            await self.sendMessage(chat_id, "You're not authorized to work with this bot ¯\_(ツ)_/¯")
            return

        if 'entities' in msg:
            message = msg['text'].strip()
            magnet_link = Command.create('magnet_link')
            if magnet_link.check_magnet_link(message) is not None:
                subprocess.call(["transmission-remote", "-n",
                                 f"{torrents['username']}:{torrents['password']}", "-a", message])
                await self.sendMessage(chat_id, "Link has been added to queue")
                return

        if content_type == 'document':
            torrent_file = Command.create('torrent_file')
            file_name = msg['document']['file_name']
            file_id = await self.getFile(msg['document']['file_id'])

            url = f"{telegram_bot['tg_endpoint']}/" \
                   f"file/" \
                   f"bot{telegram_bot['token']}/" \
                   f"{file_id['file_path']}"

            file_path = f"{torrents['files_dir']}/{file_name}"
            torrent_file.download(url=url, path=file_path)
            subprocess.call(["transmission-remote", "-n",
                             f"{torrents['username']}:{torrents['password']}", "-a", file_path])

            await self.sendMessage(chat_id, f"Added to queue: {file_name}")
            return

        if msg['text'] == '/list' or 'list' in msg['text']:
            await self.send_torrents_list(chat_id=chat_id)

        if msg['text'] == '/start':
            await self.sendMessage(chat_id, "I'm waiting for a command...", reply_markup=self.torrent_commands)
            return

        if msg['text'] == '/start_torrent' or 'start' in msg['text']:
            keyboard = self.get_torrents_for_select(action='start')
            await self.sendMessage(chat_id, 'Select a torrent to start', reply_markup=keyboard)
            return

        if msg['text'] == '/stop_torrent' or 'stop' in msg['text']:
            keyboard = self.get_torrents_for_select(action='stop')
            await self.sendMessage(chat_id, 'Select a torrent to stop', reply_markup=keyboard)
            return

        await self.sendMessage(chat_id, "I'm waiting for a command...", reply_markup=self.torrent_commands)

    @staticmethod
    def get_torrents_for_select(action='stop'):
        fetched_torrents_list = subprocess.check_output([torrents['fetch_ids']])
        list_ids = re.split('\n', fetched_torrents_list.decode('utf-8'))
        torrents_ids = list_ids[1:-2]

        torrents_out = {}
        for id in torrents_ids:
            torrent_name = subprocess.check_output([torrents['fetch_names'], str(id)])
            torrents_out[id] = torrent_name.decode('utf-8').strip().strip('Name: ')

        keyboard_ids = list([InlineKeyboardButton(text=torrents_out[id],
                                                  callback_data=f"{action}_{id}") for id in torrents_out])
        keyboard = InlineKeyboardMarkup(inline_keyboard=[keyboard_ids])

        return keyboard

    async def on_callback_query(self, msg):
        query_id, from_id, query_data = glance(msg, flavor='callback_query')

        if 'start' in query_data:
            await self.answerCallbackQuery(query_id, text='Torrent is being started...')
            subprocess.call([torrents['start_torrent'], query_data.strip('start_')])
            await self.sendMessage(from_id, 'Torrent started')
            await self.sendMessage(from_id, "I'm waiting for a command...", reply_markup=self.torrent_commands)

        if 'stop' in query_data:
            await self.answerCallbackQuery(query_id, text='Torrent is being stopped...')
            subprocess.call([torrents['stop_torrent'], query_data.strip('stop_')])
            await self.sendMessage(from_id, 'Torrent stopped')
            await self.sendMessage(from_id, "I'm waiting for a command...", reply_markup=self.torrent_commands)

    async def send_torrents_list(self, chat_id=None, message=''):
        if chat_id is None:
            return
        torrents_list = self.get_torrents_list()
        await self.sendMessage(chat_id, f"{message}*ID\tDone\tStatus\tName*\n{torrents_list}", parse_mode='markdown')

    @staticmethod
    def get_torrents_list():
        fetched_torrents_list = subprocess.check_output([torrents['fetch_list']])
        torrents_list = re.split('\n', fetched_torrents_list.decode('utf-8'))
        without_extra_fields = torrents_list[1:-2]

        status_torrents_list = '\n'.join(without_extra_fields)
        return status_torrents_list

    @staticmethod
    def auth(user=None):
        if not user:
            return False
        credentials = zip([user['id'], user['username'], user['first_name'], user['last_name']],
                          [auth['id'], auth['username'], auth['first_name'], auth['last_name']])
        pairs_check = [True if pair[0] == pair[1] else None for pair in credentials]
        if all(pairs_check):
            return True
        return False

pi_torrent_bot = TorrentBot(token=telegram_bot['token'])

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.create_task(pi_torrent_bot.message_loop())
    loop.run_forever()


from copy import deepcopy
import typing as tp

from . import operations as ops
from .external_sort import ExternalSort

class Graph:
    """Computational graph implementation"""

    def __init__(self) -> None:
        self.__operations: list["ops.Operation"] = []
        self.__graphs_for_join: list["Graph"] = []

    @staticmethod
    def graph_from_iter(name: str) -> "Graph":
        """Construct new graph which reads data from row iterator (in form of sequence of Rows
        from "kwargs" passed to "run" method) into graph data-flow
        Use ops.ReadIterFactory
        :param name: name of kwarg to use as data source
        """
        graph = Graph()
        graph.__operations = [deepcopy(ops.ReadIterFactory(name))]
        return graph

    @staticmethod
    def graph_from_file(filename: str, parser: tp.Callable[[str], ops.TRow]) -> "Graph":
        """Construct new graph extended with operation for reading rows from file
        Use ops.Read
        :param filename: filename to read from
        :param parser: parser from string to Row
        """
        graph = Graph()
        graph.__operations = [deepcopy(ops.Read(filename, parser))]
        return graph

    def map(self, mapper: ops.Mapper) -> "Graph":
        """Construct new graph extended with map operation with particular mapper
        :param mapper: mapper to use
        """
        self.__operations.append(ops.Map(mapper=mapper))
        return self

    def reduce(self, reducer: ops.Reducer, keys: tp.Sequence[str]) -> "Graph":
        """Construct new graph extended with reduce operation with particular reducer
        :param reducer: reducer to use
        :param keys: keys for grouping
        """
        self.__operations.append(ops.Reduce(keys=keys, reducer=reducer))
        return self

    def sort(self, keys: tp.Sequence[str], reverse: bool = False) -> "Graph":
        """Construct new graph extended with sort operation
        :param keys: sorting keys (typical is tuple of strings)
        :param reverse: reversed sort
        """
        self.__operations.append(ExternalSort(keys=keys, reverse=reverse))
        return self

    def join(self, joiner: ops.Joiner, join_graph: "Graph", keys: tp.Sequence[str]) -> "Graph":
        """Construct new graph extended with join operation with another graph
        :param joiner: join strategy to use
        :param join_graph: other graph to join with
        :param keys: keys for grouping
        """
        self.__operations.append(ops.Join(joiner=joiner, keys=keys))
        self.__graphs_for_join.append(join_graph)
        return self

    def run(self, **kwargs: tp.Any) -> ops.TRowsIterable:
        """Single method to start execution; data sources passed as kwargs"""
        operations = iter(self.__operations)
        graph = next(operations)(**kwargs)
        i = 0
        for operation in operations:
            if type(operation) == ops.Join:
                if len(self.__graphs_for_join):
                    table = self.__graphs_for_join[i].run(**kwargs)
                    graph = operation(graph, table)
                    i += 1
            else:
                graph = operation(graph)

        return graph


from chatterbot import ChatBot #import the chatbot
from chatterbot.trainers import ChatterBotCorpusTrainer
import os

import logging
logger = logging.getLogger()
logger.setLevel(logging.ERROR)

bot = ChatBot("Bot", preprocessors=["chatterbot.preprocessors.clean_whitespace"]) #"mypreprocessors.fix_typos_in_statement"
trainer = ChatterBotCorpusTrainer(bot)

corpus_path='F:/BackUpMyData/MySVNRepo/MyData/Tech_TargetLearns/Machine_Learning/Workspaces/MyBot/data/english/'

while True:
	message = input('You:')
	if message.strip() == 'Bye':
		print('ChatBot: Bye')
		break
	else:
		reply = bot.get_response(message)
		print('ChatBot:', reply)


import fitz  # PyMuPDF
import re
import json
from unidecode import unidecode

#Codigo responsável pela leitura de um arquivo com os dados de orientadores e suas áreas
#Do PDF responsável pelo enic daquele ano e cria um arquivo de dados contendo os resumos de cada orientador junto as suas áreas e subáreas

# A pagina inicial deve ser a pagina do pdf que começam os resumos -1
quantidadeResumos =  103 # Variável que irá ser responsável por percorrer todas as páginas que possuem resumo
resumos: list = []  # Lista onde ficará armazenado os dicionários sobre cada resumo

nome_pdf = "../DadosEnic/enic17.pdf"
resumo_nome = "../resumoOrientadores/resumoOrientadores17.json"
caminho_projetos = "../CriadorDadosModelo/projetos2017.json"
planos_nao_achados = "../resumoOrientadores/nao_achados17.txt"

# Função responsável por retirar o email do nome
# No Enic 17 foi necessário essa mudança, pois o algoritmo atual armazenava o email junto a string do nome
def retira_email(nome_e_email):

    frase: str= nome_e_email

    palavras: list = frase.split()

    palavra_final: str = ""

    for palavra in palavras:
        palavra_final += (palavra if '@' not in palavra else '') + ' '

    return palavra_final

#Abre o arquivo json de projetos
def abre_projetos(projetos_arquivo):

    projetos = list()
    with open(projetos_arquivo) as file:
        projetos = json.load(file)
    file.close()
    return projetos

def clean_text(text):
    # Substituir múltiplos espaços por um único espaço
    text = re.sub(r'\s+', ' ', text)

    # Remover caracteres especiais e não-ASCII
    text = re.sub(r'[^\x00-\x7F]+', ' ', text)

    # Remover quebras de linha e tabulações
    text = text.replace('\n', ' ').replace('\t', ' ')

    # Remover espaços em branco extras no início e no final
    text = text.strip()

    return text

#Função responsável por receber uma string e retornar apenas em formato asc
def asc_string(string):

    text = ''
    for letter in string:
        if ord(letter) <= 122 and ord(letter) >= 97:
            text += letter

    return text

#Estamos abrindo um arquivo para leitura binária, nomeado de resumo
with open(nome_pdf, 'rb') as resumo_pdf:
    # Fazendo a leitura do documento e armazenando em um objeto chamado resumoRead
    resumoRead = fitz.open(stream=resumo_pdf.read(), filetype="pdf")

    # Pega-se o número de páginas e armazenar em um objeto chamado resumoPages
    resumoPages = resumoRead.page_count

    # Nome do resumo a ser criado
    projetos = abre_projetos(caminho_projetos)

    # Cria lista de projetos que foram achados ou não, inicaliza todas as variáveis como False
    achados = 0
    projetos_contidos = list()

    for i in range(len(projetos)):
        projetos_contidos.append(False)

    enic_resumos = list()
    enic_resumos_sem_espaco = list()
    while quantidadeResumos < resumoPages:
        print(quantidadeResumos)

        # Vamos ler a página dada como parâmetro e armazenar em um objeto chamado page

        page = resumoRead.load_page(quantidadeResumos)

        # Extraimos o texto daquele objeto extraido na linha anterior

        pageConteudo = page.get_text("text")
        pageConteudo = re.sub('\n', '', pageConteudo)

        # Lista para retirar os resumos
        enic_resumos.append(pageConteudo)

        # Tratamento para localizar o plano
        # Pois algumas instancias do ENIC bugava com letras não ascii
        page_sem_espaco = (re.sub(' ', '', pageConteudo)).lower()
        page_sem_espaco = asc_string(page_sem_espaco)
        enic_resumos_sem_espaco.append(page_sem_espaco)

        quantidadeResumos += 1

    print("Lido todo o PDF")

    for i, projeto in enumerate(projetos):
        # Realiza o tratamento do texto para o nome do plano
        projeto_plano = re.sub('\n', '', projeto['Plano:'])
        projeto_plano = re.sub(' ', '', projeto_plano)
        text_treated = asc_string(projeto_plano.lower())

        projeto_achado = False
        indice_projeto_achado = 0

        for j, page in enumerate(enic_resumos):

            if text_treated in enic_resumos_sem_espaco[j]:
                projeto_achado = True
                achados += 1
                projetos_contidos[i] = True
                indice_projeto_achado = i

            if(projeto_achado) :
                posicaoResumoInicial: int = 0
                #Vamos pegar a posição do início do resumo daquela página
                if page.find("Resumo:") != - 1:
                    posicaoResumoInicial = page.find("Resumo:")
                else:
                    posicaoResumoInicial = page.find("RESUMO") - 1

                #Posicao Final do resumo é quando achamos a string "Palavras-Chave:"

                posicaoResumoFinal: int = 0
                if pageConteudo.find("Palavras-Chave:") != -1:
                    posicaoResumoFinal = page.find("Palavras-Chave:")
                elif pageConteudo.find("Palav ras-Chave") != -1:
                    posicaoResumoFinal = page.find("Palav ras-Chave")
                elif pageConteudo.find("Palavras Chave") != -1:
                    posicaoResumoFinal = page.find("Palavras Chave")
                else:
                    posicaoResumoFinal = page.find("Palavras -Chave")

                #Recebe-se o dicionário
                resumo_dict = projetos[indice_projeto_achado]
                resumo_dict['texto:'] = page[posicaoResumoInicial + 8:posicaoResumoFinal].lstrip()
                resumo_dict['texto:'] = resumo_dict["texto:"].rstrip()

                incluso = False
                for resumo in resumos:
                    if resumo["id"] == resumo_dict["id"]:
                        incluso = True

                if not incluso:
                    #Adiciona-se o dicionario na lista de resumos
                    resumos.append(resumo_dict.copy())
                break

    # Responsável por criar o json do resumo dos orientadores
    with open(resumo_nome, 'w') as resumo_js:
        json.dump(resumos, resumo_js, indent = 4, ensure_ascii=False)
    resumo_js.close()

    # Cria um arquivo txt responsável por dizer quais foram os planos não achados
    arq_nao_achados = open(planos_nao_achados, "w+")

    for i in range(len(projetos_contidos)):
        if projetos_contidos[i] == False:
            arq_nao_achados.write(str(i) + ' ')
            arq_nao_achados.write('\n')

    arq_nao_achados.close()

    print(f'Foram achados {len(resumos)} resumos')

    resumo_pdf.close()


from django.core.management.base import BaseCommand
import json

# import sys
# sys.path.append('/users')
from users.models import Nodes

# List of JSON files to import
json_files = [
    '/src/json-files/park_locations.json',
     "/src/json-files/park_node_locations",
    "/src/json-files/worship_locations",
    "/src/json-files/museum_art_locations",
    "/src/json-files/library_locations",
    "/src/json-files/walking_node_locations",
    "/src/json-files/community_locations"
]

# file_path = r'C:\Users\corma\COMP47360\ucdSummerProject\src\json-files\museum_art_locations.json'
class Command(BaseCommand):
    help = 'Import JSON file into PostgreSQL'

    def add_arguments(self, parser):
        parser.add_argument('file_path', type=str, help='Path to the JSON file')

    def handle(self, *args, **options):

        file_path = options['file_path']

        # for file_path in json_files:
        with open(file_path) as f:
            data = json.load(f)

        for item in data['data']:
            node = Nodes(
                id_str=item['id'],
                name=item['name'],
                type=item['type'],
                address=item['address'],
                internet_access=item['internet_access'],
                wheelchair_accessible=item['wheelchair_accessible'],
                opening_hours=item['opening_hours'],
                grid_id=item['grid-id'],
                taxi_zone=item['taxi-zone'],
                precinct=item['precinct'],
                b_score=item['b-score'],
                c_score=item['c-score'],
                rating=item['rating'],
                latitude=item['location']['latitude'],
                longitude=item['location']['longitude'],
            )
            node.save()


import torch
import torchvision.transforms as transforms
from torch.utils.data import Dataset

class PPNetDataset(Dataset):
    def __init__(self, ppg, sbp, dbp, hr):
        self.transform = transforms.Compose([transforms.ToTensor()])
        self.ppg = ppg
        self.sbp = sbp
        self.dbp = dbp
        self.hr = hr

    def __getitem__(self, index):
        if torch.is_tensor(index):
            index = index.tolist()

        ppg = torch.tensor(self.ppg[index], dtype=torch.float32)
        sbp = torch.tensor(self.sbp[index], dtype=torch.float32)
        dbp = torch.tensor(self.dbp[index], dtype=torch.float32)
        hr = torch.tensor(self.hr[index], dtype=torch.float32)

        inputs = ppg.reshape(1,-1)
        targets = torch.stack([sbp,dbp,hr],dim=0)

        if torch.cuda.is_available():
            inputs = inputs.to('cuda')
            targets = targets.to('cuda')

        return inputs, targets

    def __len__(self):
        return len(self.ppg)


# one vs rest model

# logistic regression for multi-class classification using built-in one-vs-rest
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
# define dataset
X, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, n_classes=3, random_state=1)
# define model
model = LogisticRegression(multi_class='ovr')
# fit model
model.fit(X, y)
# make predictions
yhat = model.predict(X)

from sklearn.multiclass import OneVsRestClassifier
# define dataset
X, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, n_classes=3, random_state=1)
# define model
model = LogisticRegression()
# define the ovr strategy
ovr = OneVsRestClassifier(model)
# fit model
ovr.fit(X, y)
# make predictions
yhat = ovr.predict(X)

# one vs one model

from sklearn.svm import SVC
# define dataset
X, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, n_classes=3, random_state=1)
# define model
model = SVC(decision_function_shape='ovo')
# fit model
model.fit(X, y)
# make predictions
yhat = model.predict(X)

from sklearn.svm import SVC
from sklearn.multiclass import OneVsOneClassifier
# define dataset
X, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, n_classes=3, random_state=1)
# define model
model = SVC()
# define ovo strategy
ovo = OneVsOneClassifier(model)
# fit model
ovo.fit(X, y)
# make predictions
yhat = ovo.predict(X)


from collections import deque

"""
    Use DFS to determine whether an undirected graph has a cycle, if so find such a cycle
"""
class Cycle:
    def __init__(self, G):
        if self.has_parallel_edges(G): return

        self.marked = [False] * G.V
        self.edge_to = [0] * G.V
        self.cycle = None
        for v in range(G.V):
            if not self.marked[v]:
                self._dfs(G, v)

    def _dfs(self, G, v):
        self.marked[v] = True
        for w in G.adj[v]:
            if self.cycle: return

            elif not self.marked[w]:
                self.edge_to[w] = v
                self._dfs(G, w)
            elif w != self.edge_to[v]:    # ignore reverse edge leading to v
                self.cycle = deque()
                self.cycle.appendleft(w)
                x = v
                while x != w:
                    self.cycle.appendleft(x)
                    x = self.edge_to[x]
                self.cycle.appendleft(w)
                return

    def has_parallel_edges(self, G):
        self.marked = [False] * G.V
        for v in range(G.V):
            for w in G.adj[v]:
                if self.marked[w]:
                    self.cycle = deque()
                    self.cycle.append(v)
                    self.cycle.append(w)
                    self.cycle.append(v)
                    return True
                self.marked[w] = True
            for w in G.adj[v]:
                self.marked[w] = False
        return False

    def hascycle(self):
        return self.cycle != None

    def getcycle(self):
        return self.cycle

if __name__ == '__main__':
    import sys
    from graph import Graph
    G = Graph.create(sys.stdin)
    c = Cycle(G)
    if c.hascycle():
        print(f"Graph has cycle:")
        print('-'.join(str(v) for v in c.getcycle()))
    else:
        print(f"Graph is acyclic")


from sfi.server.models import Education, Users
import json
import random

def test_education(client, app, auth):
    auth.login_researcher()
    degree = ["Associate's degree", "Bachelor's degree", "Master's degree", "Doctoral degree"]
    field_of_study = ["Computer Science", "Biology", "Chemistry", "Physics", "Engineering", "Mathematics"]
    institution = ["UCC", "UCD", "TCD", "DCU", "UL", "NUIG", "NUIM"]
    location = ["Cork", "Dublin", "Limerick", "Galway", "Maynooth"]
    year_degree_award = ["2014-1-1", "2010-1-1", "2007-1-1", "2012-1-1"]

    response = client.post(
        '/profile/education',
        data=json.dumps({
            "degree": random.choice(degree),
            "field_of_study": random.choice(field_of_study),
            "institution": random.choice(institution),
            "location": random.choice(location),
            "year_degree_award": random.choice(year_degree_award)
        }),
        content_type='application/json'
    )

    print(f'resp: {response}')
    with app.app_context():
        query = Education.query.first()
    assert query is not None


import json
import os
from argparse import ArgumentParser
from tabnanny import check

import numpy as np
import torch
import torch.nn as nn
import torch.cuda.amp as amp
import utils
from functools import partial
from data_generators.vl_data_generator import ArenaRefDataset
from engine import train, validate
from torch.optim.lr_scheduler import MultiStepLR
import shutil
import resource
from models import build_predictor
from utils import worker_init_fn
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (8000, rlimit[1]))

def main(args):
    utils.init_distributed_mode(args)
    if args.mode in {"evaluate", "train_evaluate"}:
        raise NotImplementedError("Evaluation mode not implemented. "
                                 "Please run end-to-end mission level evaluation after training.")
    training = True if (args.mode in {"train", "train_evaluate"}) else False
    # use our dataset and defined transformations
    if training:
        dataset = ArenaRefDataset(
            metadata_file=args.train_metadata_file,
            images_root=args.images_root,
            split="train",
            args=args)
        init_fn = partial(worker_init_fn,
                          num_workers=args.num_workers,
                          rank=args.rank,
                          seed=0)
        train_sampler = torch.utils.data.distributed.DistributedSampler(dataset, shuffle=True)
        data_loader = torch.utils.data.DataLoader(
            dataset,
            sampler=train_sampler,
            batch_size=args.batch_size,
            shuffle=False,
            num_workers=args.num_workers,
            worker_init_fn=init_fn,
            collate_fn=utils.collate_fn,
            drop_last=True,
            pin_memory=True
            )
    if args.mode in {"evaluate", "train_evaluate"}:
        dataset_test = ArenaRefDataset(
            metadata_file=args.test_metadata_file,
            images_root=args.images_root,
            split="validation",
            args=args)
        test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False)
        data_loader_test = torch.utils.data.DataLoader(
            dataset_test,
            batch_size=16,
            shuffle=False,
            sampler=test_sampler,
            num_workers=args.num_workers,
            drop_last=False,
            pin_memory=True)

    # get the model using our helper function
    model, param_list = build_predictor(args)
    if args.sync_bn:
        model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
    if args.distributed:
        model = nn.parallel.DistributedDataParallel(model.cuda(),
                                                    device_ids=[args.gpu],
                                                    find_unused_parameters=True)
        model_without_ddp = model.module
    print("Model loaded!")
    device = "cuda"
    model.to(device)
    params = filter(lambda p: p.requires_grad, model.parameters())
    num_params = sum([np.prod(p.size()) for p in params])
    print("Number of model params: ", num_params)
    optimizer = torch.optim.Adam(param_list,
                                 lr=args.base_lr,
                                 weight_decay=args.weight_decay)
    scheduler = MultiStepLR(optimizer,
                           milestones=args.milestones,
                           gamma=args.lr_decay)
    scaler = amp.GradScaler()
    exp_save_log_folder = os.path.join(args.save_log_folder, f"exp{args.exp_num}")
    if utils.is_main_process():
        if not os.path.exists(exp_save_log_folder):
            os.makedirs(exp_save_log_folder)
            os.makedirs(os.path.join(exp_save_log_folder, "model_checkpoints"))
        elif training and not args.resume:
            raise ValueError(f"Exp folder already exists at {exp_save_log_folder},"
                            " Please delete / change exp num / ensure there is no unwanted overwriting of information.")
        if not args.resume and args.mode != "evaluate":
            src_save_folder = os.path.join(exp_save_log_folder, "src")
            if not os.path.exists(src_save_folder):
                os.makedirs(src_save_folder)
                shutil.make_archive(
                    f"{src_save_folder}/src", 'zip',
                    f"{args.project_root}/")
            print(f"Saving source code to {src_save_folder}/src.zip")
            with open(os.path.join(exp_save_log_folder, f"parameters_{args.mode}.json"), 'w') as f:
                json.dump(args.__dict__, f, indent=2)
        stats_save_path = os.path.join(exp_save_log_folder, "stats_logs")
        if not os.path.exists(stats_save_path):
            os.makedirs(stats_save_path)
    best_iou = 0.0
    if training:
        num_epochs = args.num_epochs
        start_epoch = 0
        if args.resume:
            resume_checkpoint = utils.get_last_saved_checkpoint(exp_save_log_folder)
            print(f"Resuming from checkpoint {resume_checkpoint}")
            start_epoch = resume_checkpoint + 1
            path_to_model_checkpoint = os.path.join(exp_save_log_folder, f"model_checkpoints/{resume_checkpoint}.pth")
            checkpoint = torch.load(path_to_model_checkpoint,  map_location="cpu")
            state_dict = checkpoint['model']
            best_iou = checkpoint['best_iou']
            try:
                model_without_ddp.load_state_dict(state_dict)
            except:
                state_dict = {k.partition('module.')[2]: state_dict[k] for k in state_dict.keys()}
                model_without_ddp.load_state_dict(state_dict)
            optimizer.load_state_dict(checkpoint["optim"])
            scheduler.load_state_dict(checkpoint['scheduler'])

        print(f"Launching training experiment {args.exp_num}!")
        for epoch in range(start_epoch, num_epochs):
            train_sampler.set_epoch(epoch + 1)
            optimizer.zero_grad()
            print(f"\nEpoch {epoch}")
            # train for one epoch, printing every print-freq iterations
            loss_meter, iou_meter, pr_meter = train(data_loader, model, optimizer, scheduler, scaler, epoch + 1,
                  args)
            iou = 0
            prec_dict = {}
            if (args.mode == "train_evaluate") and ((epoch + 1) % args.eval_freq == 0):
                iou, prec_dict = validate(data_loader_test, model, epoch + 1, args)

            if utils.is_main_process():
                fsave = os.path.join(exp_save_log_folder, "model_checkpoints", '{}.pth'.format(epoch))
                torch.save({
                    'iou': iou,
                    'best_iou': best_iou,
                    'prec': prec_dict,
                    'model': model_without_ddp.state_dict(),
                    'optim': optimizer.state_dict(),
                    'scheduler': scheduler.state_dict(),
                    'epoch': epoch,
                    'args': args
                    }, fsave)
                if iou >= best_iou:
                    best_iou = iou
                fbest = os.path.join(stats_save_path, f"{epoch}.json")
                stats = {
                    'running_loss_avg': loss_meter.avg,
                    'running_iou_avg': iou_meter.avg,
                    'running_prec_at_50_avg': pr_meter.avg,
                }
                with open(fbest, 'wt') as f:
                    json.dump(stats, f, indent=2)
            scheduler.step()
            torch.cuda.empty_cache()
        print("Training complete.")

if __name__ == "__main__":
    parser = ArgumentParser()

    # Running mode
    parser.add_argument("--mode", dest="mode", type=str,
                        default="train",
                        help="Modes: {train, evaluate, train_evaluate}")
    parser.add_argument("--eval-freq", dest="eval_freq", type=int,
                        default=10,
                        help="Frequency to evaluate with if mode = train_evaluate")
    parser.add_argument("--print-freq", dest="print_freq", type=int,
                        default=10,
                        help="Frequency to print logs in steps")

    # Training devicea
    parser.add_argument("--rank-0-gpu", dest="rank_0_gpu", type=int, default=0,
                        help="Which GPU to consider rank 0 for single-node, multi-worker, separate jobs."
                        "(rank 0 + world-size) gpu ids are used for the run.")
    parser.add_argument("--data-parallel", dest="data_parallel", action='store_true',
                        help="Whether to train in data parallel mode")
    parser.add_argument("--device-ids", dest="device_ids", type=list, default=[0],
                        help="gpu device id for training/evaluation")
    parser.add_argument("--num-workers", dest="num_workers", type=int, default=4,
                        help="Number of workers for data loading")
    parser.add_argument("--distributed", dest="distributed", type=bool, default=False,
                        help="Whether to perform data distributed training")
    parser.add_argument('--world-size', dest="world_size", default=1, type=int,
                        help='number of distributed processes')
    parser.add_argument('--dist-url', default='tcp://127.0.0.1:23456', help='url used to set up distributed training')

    # Experiment
    parser.add_argument("--exp-num", dest="exp_num", type=int, default=114,
                        help="exp id for tracking purposes")
    parser.add_argument("--resume", dest="resume", type=bool, default=False,
                        help="Whether to resume training an experiment")

    # Data
    repo_root = os.path.dirname(os.path.abspath("../"))
    data_root = "{}/data/trajectory-data/".format(repo_root)
    project_root = os.path.dirname(os.path.abspath(__file__))
    parser.add_argument("--project-root", dest="project_root", type=str, default=project_root,
                        help="Project root for running training and evaluation of vision models")
    parser.add_argument("--images-root", dest="images_root", type=str,
                        default=f"{data_root}/mission_images",
                        help="path to image and metadata files root dir")
    parser.add_argument("--train-metadata-file", dest="train_metadata_file", type=str,
                        default=f"{data_root}/train.json",
                        help="path to text file with all metadata files")
    parser.add_argument("--test-metadata-file", dest="test_metadata_file", type=str,
                        default=f"{data_root}/valid.json",
                        help="path to text file with all metadata files")
    parser.add_argument("--class-to-idx-file", dest="class_to_idx_file", type=str,
                        default=f"{repo_root}/data/vision-data/class_to_idx.json",
                        help="Path to precomputed json file that maps classes to indices")
    parser.add_argument("--obj-id-to-class-file", dest="obj_id_to_class_file", type=str,
                        default=f"{repo_root}/data/vision-data/obj_id_to_class_customized.json",
                        help="Path to precomputed json file that maps classes to object id")

    # Model
    parser.add_argument("--clip-pretrain", dest="clip_pretrain", type=str,
                        default=os.path.join(project_root, "pretrained/RN50.pt"),
                        help="Path to pretrained clip model")
    parser.add_argument("--word-len", dest="word_len", type=int, default=22,
                        help="Maximum number of words in language prompt")
    parser.add_argument("--word-dim", dest="word_dim", type=int, default=1024,
                        help="Word dimension")
    parser.add_argument("--fpn-in", dest="fpn_in", type=list, default= [512, 1024, 1024],
                        help="Fpn in")
    parser.add_argument("--fpn-out", dest="fpn_out", type=list, default=[256, 512, 1024],
                        help="Fpn out")
    parser.add_argument("--num-layers", dest="num_layers", type=int, default=3,
                        help="Number of layers")
    parser.add_argument("--vis-dim", dest="vis_dim", type=int, default=512,
                        help="Visual feature dims")
    parser.add_argument("--num-head", dest="num_head", type=int, default=8,
                        help="Number of heads for trans")
    parser.add_argument("--dim-ffn", dest="dim_ffn", type=int, default=2048,
                        help="Feed forward network dim")
    parser.add_argument("--dropout", dest="dropout", type=float, default=0.1,
                        help="Dropout prob")
    parser.add_argument("--intermediate", dest="intermediate", type=bool, default=False,
                        help="Intermediate")
    parser.add_argument("--sync-bn", dest="sync_bn", type=bool, default=True,
                        help="Batchnormalization sync between processes")

    parser.add_argument("--hidden-layer-size", dest="hidden_layer_size", type=int,
                        default=512,
                        help="Hidden layer size for mask predictor branch")
    # Hyperparameters
    parser.add_argument("--batch-size", dest="batch_size", type=int,
                        default=8,
                        help="Batch size for training and testing data loaders. "
                             "For distributed mode, this is the batchsize per gpu")
    parser.add_argument("--lr", dest="lr", type=float,
                        default=0.00125,
                        help="Learning rate")
    parser.add_argument("--lr-multi", dest="lr_multi", type=float,
                        default=0.1,
                        help="Learning rate multiplier for clip backbone")
    parser.add_argument("--lr-seq", dest="lr_seq", type=float,
                        default=0.0001,
                        help="Learning rate for sequential act obj pred")
    parser.add_argument("--base-lr", dest="base_lr", type=float,
                        default=0.00001,
                        help="Learning rate head")
    parser.add_argument("--decay-epoch", dest="decay_epoch", type=int,
                        default=25,
                        help="learning rate is decayed every this many epochs")
    parser.add_argument("--lr-decay", dest="lr_decay", type=float,
                        default=0.1,
                        help="learning rate is decayed every decay_epochs by this factor")
    parser.add_argument("--milestones", dest="milestones", type=list,
                        default=[35],
                        help="Learning rate schedule milestones to decay lr")
    parser.add_argument("--momentum", dest="momentum", type=float,
                        default=0.9,
                        help="Momentum for optimizer")
    parser.add_argument("--weight-decay", dest="weight_decay", type=float,
                        default=0.00001,
                        help="Weight decay for optimizer")
    parser.add_argument("--num-epochs", dest="num_epochs", type=int,
                        default=75,
                        help="Number of epochs to train.")
    parser.add_argument("--optimizer", dest="optimizer", type=str,
                        default="SGD",
                        help="Optimization algo.")
    parser.add_argument("--max-norm", dest="max_norm", type=float,
                        default=0.0,
                        help="Grad clip norm")
    # Logs
    parser.add_argument("--save-log-folder", dest="save_log_folder", type=str,
                        default="/home/ubuntu/training_logs/",
                        help="Folder to save model checkpoints and training logs")

    args = parser.parse_args()
    main(args)


import math

from flask import Blueprint, g
from lin import (
    DocResponse,
    Forbidden,
    GroupLevelEnum,
    Logger,
    NotFound,
    ParameterError,
    Success,
    admin_required,
    db,
    manager,
    permission_meta,
)
from sqlalchemy import func

from app.api import AuthorizationBearerSecurity, api
from app.api.cms.schema import ResetPasswordSchema
from app.api.cms.schema.admin import (
    AdminGroupListSchema,
    AdminGroupPermissionSchema,
    AdminUserPageSchema,
    CreateGroupSchema,
    GroupBaseSchema,
    GroupIdWithPermissionIdListSchema,
    GroupQuerySearchSchema,
    UpdateUserInfoSchema,
)
from app.util.page import get_page_from_query

admin_api = Blueprint("admin", __name__)

@admin_api.route("/permission")
@permission_meta(name="查询所有可分配的权限", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
)
def permissions():
    """
    查询所有可分配的权限
    """
    return manager.get_ep_infos()

@admin_api.route("/users")
@permission_meta(name="查询所有用户", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(r=AdminUserPageSchema),
    before=GroupQuerySearchSchema.offset_handler,
)
def get_admin_users(query: GroupQuerySearchSchema):
    """
    查询所有用户
    """
    # 根据筛选条件和分页，获取 用户id 与 用户组id 的一对多 数据
    # 过滤root 分组
    query_root_group_id = db.session.query(manager.group_model.id).filter(
        manager.group_model.level == GroupLevelEnum.ROOT.value
    )
    _user_groups_list = db.session.query(
        manager.user_group_model.user_id.label("user_id"),
        func.group_concat(manager.user_group_model.group_id).label("group_ids"),
    ).filter(~manager.user_group_model.group_id.in_(query_root_group_id))
    if g.group_id:
        _user_groups_list = _user_groups_list.filter(
            manager.user_group_model.group_id == g.group_id
        )
    user_groups_list = (
        _user_groups_list.group_by(manager.user_group_model.user_id)
        .offset(g.offset)
        .limit(g.count)
        .all()
    )

    # 获取 符合条件的 用户id 数量
    # 过滤root 分组
    _total = db.session.query(
        func.count(func.distinct(manager.user_group_model.user_id))
    ).filter(~manager.user_group_model.group_id.in_(query_root_group_id))
    if g.group_id:
        _total = _total.filter(manager.user_group_model.group_id == g.group_id)
    total = _total.scalar()

    # 获取本次需要返回的用户的数据
    user_ids = [x.user_id for x in user_groups_list]
    users = manager.user_model.query.filter(manager.user_model.id.in_(user_ids)).all()
    user_dict = dict()
    for user in users:
        user.groups = list()
        user._fields.append("groups")
        user_dict[user.id] = user

    # 使用用户组来过滤，则不需要补全用户组信息
    if not g.group_id:
        # 拿到本次请求返回用户的所有 用户组 id List
        group_ids = [
            int(i)
            for i in set().union(*(x.group_ids.split(",") for x in user_groups_list))
        ]
        # 获取本次需要返回的用户组的数据
        groups = manager.group_model.query.filter(
            manager.group_model.id.in_(group_ids)
        ).all()
        group_dict = dict()
        for group in groups:
            group_dict[group.id] = group
        items = []

        # 根据 用户与用户组 一对多的关系， 补全用户的用户组详细信息
        for user_id, group_ids in user_groups_list:
            group_id_list = [int(gid) for gid in group_ids.split(",")]
            groups = [group_dict[group_id] for group_id in group_id_list]
            user_dict[user_id].groups = groups
            items.append(user_dict[user_id])

    total_page = math.ceil(total / g.count)
    page = get_page_from_query()
    return AdminUserPageSchema(
        count=g.count, total=total, total_page=total_page, page=page, items=users
    )

@admin_api.route("/user/<int:uid>/password", methods=["PUT"])
@permission_meta(name="修改用户密码", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    resp=DocResponse(NotFound("用户不存在"), Success("密码修改成功")),
    security=[AuthorizationBearerSecurity],
)
def change_user_password(uid: int, json: ResetPasswordSchema):
    """
    修改用户密码
    """

    user = manager.find_user(id=uid)
    if not user:
        raise NotFound("用户不存在")

    with db.auto_commit():
        user.reset_password(g.new_password)

    return Success("密码修改成功")

@admin_api.route("/user/<int:uid>", methods=["DELETE"])
@permission_meta(name="删除用户", module="管理员", mount=False)
@Logger(template="管理员删除了一个用户")
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(NotFound("用户不存在"), Success("删除成功"), Forbidden("用户不能删除")),
)
def delete_user(uid):
    """
    删除用户
    """
    user = manager.user_model.get(id=uid)
    if user is None:
        raise NotFound("用户不存在")
    groups = manager.group_model.select_by_user_id(uid)
    # 超级管理员分组的用户仅有一个分组
    if groups[0].level == GroupLevelEnum.ROOT.value:
        raise Forbidden("无法删除此用户")
    with db.auto_commit():
        manager.user_group_model.query.filter_by(user_id=uid).delete(
            synchronize_session=False
        )
        user.hard_delete()
    return Success("操作成功")

@admin_api.route("/user/<int:uid>", methods=["PUT"])
@permission_meta(name="管理员更新用户信息", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(
        NotFound("用户不存在"),
        Success("更新成功"),
        ParameterError("邮箱已被注册，请重新输入邮箱"),
    ),
)
def update_user(uid: int, json: UpdateUserInfoSchema):
    """
    更新用户信息
    """
    user = manager.user_model.get(id=uid)
    if user is None:
        raise NotFound("用户不存在")
    if user.email != g.email:
        exists = manager.user_model.get(email=g.email)
        if exists:
            raise ParameterError("邮箱已被注册，请重新输入邮箱")
    with db.auto_commit():
        user.email = g.email
        group_ids = g.group_ids
        # 清空原来的所有关联关系
        manager.user_group_model.query.filter_by(user_id=user.id).delete(
            synchronize_session=False
        )
        # 根据传入分组ids 新增关联记录
        user_group_list = list()
        # 如果没传分组数据，则将其设定为 guest 分组
        if not group_ids:
            group_ids = [manager.group_model.get(level=GroupLevelEnum.GUEST.value).id]
        for group_id in group_ids:
            user_group = manager.user_group_model()
            user_group.user_id = user.id
            user_group.group_id = group_id
            user_group_list.append(user_group)
        db.session.add_all(user_group_list)
    return Success("操作成功")

@admin_api.route("/group/all")
@permission_meta(name="查询所有分组", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(NotFound("不存在任何分组"), r=AdminGroupListSchema),
)
def get_all_group():
    """
    获取所有分组
    """
    groups = manager.group_model.query.filter(
        manager.group_model.is_deleted == False,
        manager.group_model.level != GroupLevelEnum.ROOT.value,
    ).all()
    if groups is None:
        raise NotFound("不存在任何分组")
    return groups

@admin_api.route("/group/<int:gid>")
@permission_meta(name="查询一个分组及其权限", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(NotFound("分组不存在"), r=AdminGroupPermissionSchema),
)
def get_group(gid):
    """
    获取一个分组及其权限
    """
    group = manager.group_model.get(id=gid, one=True, soft=False)
    if group is None:
        raise NotFound("分组不存在")
    permissions = manager.permission_model.select_by_group_id(gid)
    setattr(group, "permissions", permissions)
    group._fields.append("permissions")
    return group

@admin_api.route("/group", methods=["POST"])
@permission_meta(name="新建分组", module="管理员", mount=False)
@Logger(template="管理员新建了一个分组")  # 记录日志
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(
        ParameterError("用户组已存在, 不可创建同名分组"),
        Success("新建用户组成功"),
    ),
)
def create_group(json: CreateGroupSchema):
    """
    新建分组
    """
    exists = manager.group_model.get(name=g.name)
    if exists:
        raise Forbidden("分组已存在，不可创建同名分组")
    with db.auto_commit():
        group = manager.group_model.create(
            name=g.name,
            info=g.info,
        )
        db.session.flush()
        group_permission_list = list()
        for permission_id in g.permission_ids:
            gp = manager.group_permission_model()
            gp.group_id = group.id
            gp.permission_id = permission_id
            group_permission_list.append(gp)
        db.session.add_all(group_permission_list)
    return Success("新建分组成功")

@admin_api.route("/group/<int:gid>", methods=["PUT"])
@permission_meta(name="更新一个分组", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(
        ParameterError("分组不存在,更新失败"),
        Success("更新分组成功"),
    ),
)
def update_group(gid, json: GroupBaseSchema):
    """
    更新一个分组基本信息
    """
    exists = manager.group_model.get(id=gid)
    if not exists:
        raise NotFound("分组不存在，更新失败")
    exists.update(name=g.name, info=g.info, commit=True)
    return Success("更新成功")

@admin_api.route("/group/<int:gid>", methods=["DELETE"])
@permission_meta(name="删除一个分组", module="管理员", mount=False)
@Logger(template="管理员删除一个分组")  # 记录日志
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(NotFound("分组不存在，删除失败"), Forbidden("分组不可删除"), Success("删除分组成功")),
)
def delete_group(gid):
    """
    删除一个分组
    """
    exist = manager.group_model.get(id=gid)
    if not exist:
        raise NotFound("分组不存在，删除失败")
    guest_group = manager.group_model.get(level=GroupLevelEnum.GUEST.value)
    root_group = manager.group_model.get(level=GroupLevelEnum.ROOT.value)
    if gid in (guest_group.id, root_group.id):
        raise Forbidden("不可删除此分组")
    if manager.user_model.select_page_by_group_id(gid, root_group.id):
        raise Forbidden("分组下存在用户，不可删除")
    with db.auto_commit():
        # 删除group id 对应的关联记录
        manager.group_permission_model.query.filter_by(group_id=gid).delete(
            synchronize_session=False
        )
        # 删除group
        exist.delete()
    return Success("删除分组成功")

@admin_api.route("/permission/dispatch/batch", methods=["POST"])
@permission_meta(name="分配多个权限", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    security=[AuthorizationBearerSecurity],
    resp=DocResponse(
        Success("添加权限成功"),
    ),
)
def dispatch_auths(json: GroupIdWithPermissionIdListSchema):
    """
    分配多个权限
    """
    with db.auto_commit():
        for permission_id in g.permission_ids:
            one = manager.group_permission_model.get(
                group_id=g.group_id, permission_id=permission_id
            )
            if not one:
                manager.group_permission_model.create(
                    group_id=g.group_id,
                    permission_id=permission_id,
                )
    return Success("添加权限成功")

@admin_api.route("/permission/remove", methods=["POST"])
@permission_meta(name="删除多个权限", module="管理员", mount=False)
@admin_required
@api.validate(
    tags=["管理员"],
    resp=DocResponse(Success("删除权限成功")),
    security=[AuthorizationBearerSecurity],
)
def remove_auths(json: GroupIdWithPermissionIdListSchema):
    """
    删除多个权限
    """

    with db.auto_commit():
        db.session.query(manager.group_permission_model).filter(
            manager.group_permission_model.permission_id.in_(g.permission_ids),
            manager.group_permission_model.group_id == g.group_id,
        ).delete(synchronize_session=False)

    return Success("删除权限成功")


import matplotlib.pyplot
import numpy
from gensim.models import word2vec
import numpy as np

'''生成长序列串的子序列集，k为kmer长度'''
def k_mergenerator(WaiTingKmer, k):
    result = open("result.txt", mode="w")
    DNA = []
    for i in WaiTingKmer:
        line = []
        for j in range(len(i)-k+1):
            line.append(i[j:j+k])
            result.write(i[j:j+k])
            result.write(" ")
        DNA.append(line)
        result.write("\n")
    return DNA

'''生成整条基因数据的向量，生成方式为将每个kmer向量相加'''
def DNA_to_array(DNA, module):
    DNA_array = []
    sum = 0
    for i in DNA:
        for j in i:
            sum = sum + module.wv[j]
        DNA_array.append(sum)
    return DNA_array

if __name__ == '__main__':
    splice = open("splice.data")
    lable = []
    DNA = []
    for i in splice:
        lable.append(i.split(",")[0])
        DNA.append(i.split(",")[2].replace(" ", "").replace("\n", ""))
    splice.close()
    # 获取Kmer
    DNAkmer = k_mergenerator(DNA, 3)
    # 使用word2vec出基因数据
    genes = word2vec.Text8Corpus("result.txt")
    model = word2vec.Word2Vec(genes, sg=1, min_count=1, vector_size=100)
    print(model)

    model.wv.save_word2vec_format("VectorForGene", binary=False)

    """转化成其他模块能用向量数据"""
    # word_list = model.wv.index_to_key
    # # print(word_list)
    # embeddings = np.array([model.wv[word] for word in word_list])
    # print(embeddings)
    """把DNA文件转化为向量（还是列表）"""
    lable = np.array(lable)
    DNAarray = DNA_to_array(DNAkmer, model)

    '''试试看能不能用kmeans'''
    # # print(npemd)
    # from sklearn.cluster import k_means
    # fuck = k_means(DNAarray,n_clusters=3)
    # result = numpy.array(fuck[1])
    # print(result)
    # fu = []
    # for cluster in range(3):
    #     fu.append(list(lable[np.where(fuck[1] == cluster)]))
    # print("第一类 EI",fu[0].count("EI")," IE ",fu[0].count("IE")," N ",fu[0].count("N"),)
    # print("第二类 EI",fu[1].count("EI")," IE ",fu[1].count("IE")," N ",fu[1].count("N"),)
    # print("第三类 EI",fu[2].count("EI")," IE ",fu[2].count("IE")," N ",fu[2].count("N"),)
    # """kmeans 结果大概78左右"""

    '''试试看能不能svm'''
    DNAarray = np.array(DNAarray)
    print(DNAarray.shape)

    from sklearn.model_selection import train_test_split
    X_train, X_test, y_train, y_test = train_test_split(np.array(DNAarray),np.array(lable),test_size=0.2)
    from sklearn.svm import SVC
    svmclass = SVC(kernel="rbf")
    print(len(X_train),len(y_train))
    svmclass.fit(X_train,y_train)
    res = svmclass.predict(X_test)
    import sklearn.metrics as sm
    bg = sm.classification_report(y_test, res)
    SB = sm.confusion_matrix(y_test,res)
    print(SB)
    sm.ConfusionMatrixDisplay
    # for i in range(len(y_test)):
    #     print(y_test[i],res[i])
    print('分类报告：', bg, sep='\n')


from fastsklearnfeature.transformations.Transformation import Transformation
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
import itertools
import numpy_indexed as npi
from typing import List, Dict, Set
from fastsklearnfeature.candidates.CandidateFeature import CandidateFeature
import sympy
import copy
import pandas as pd

class FastGroupByThenTransformation(BaseEstimator, TransformerMixin, Transformation):
    def __init__(self, method, sympy_method):
        self.method = method
        self.sympy_method = sympy_method
        Transformation.__init__(self, 'GroupByThen' + self.method.__name__,
                 number_parent_features=2,
                 output_dimensions=1,
                 parent_feature_order_matters=True,
                 parent_feature_repetition_is_allowed=False)

    #0th feature will be aggregated, 1th-nth = key attributes
    def fit(self, X, y=None):
        self.final_mapping = dict(npi.group_by(keys=X[:, 1], values=X[:, 0].astype(np.float64), reduction=self.method))
        return self

    def transform(self, X):
        #initialize result
        n = None
        if self.method == np.nanstd:
            n = np.zeros((X.shape[0], 1))
        elif self.method == len:
            n = np.ones((X.shape[0], 1))
        else:
            n = copy.deepcopy(X[:, 0]).reshape(-1, 1).astype(np.float64)

        #apply mapping
        for k in self.final_mapping:
            n[X[:, 1] == k] = self.final_mapping[k]
        return n

    def is_applicable(self, feature_combination: List[CandidateFeature]):
        #we handle conditional idempotence via sympy

        if 'missing_values' in feature_combination[0].properties and feature_combination[0].properties['missing_values']:
            return False

        if 'missing_values' in feature_combination[1].properties and feature_combination[1].properties['missing_values']:
            return False

        #the aggregated column has to be numeric
        if 'float' in str(feature_combination[0].properties['type']) \
            or 'int' in str(feature_combination[0].properties['type']) \
            or 'bool' in str(feature_combination[0].properties['type']):

            return True

        return False

    def get_combinations(self, features):
        #self.parent_feature_order_matters and not self.parent_feature_repetition_is_allowed:

        iterable_collection = []
        for i in range(2, self.number_parent_features+1):
            iterable_collection.append(itertools.permutations(features, r=i))

        return itertools.chain(*iterable_collection)

    def get_name(self, candidate_feature_names):
        return "(" + self.method.__name__ + "(" + str(candidate_feature_names[0]) + ") GroupyBy " + str(candidate_feature_names[1]) + ")"

    def get_sympy_representation(self, input_attributes):
        return self.sympy_method(input_attributes[0], input_attributes[1])

    def derive_properties(self, training_data, parents: List[CandidateFeature]):
        properties = {}
        # type properties
        properties['type'] = training_data.dtype

        try:
            # missing values properties
            properties['missing_values'] = pd.isnull(training_data).any()

            # range properties
            if (parents[0].properties['min'] == 0.0 and self.method == np.nanmin) or \
               (parents[0].properties['max'] == 0.0 and self.method == np.nanmax):
                properties['has_zero'] = True
            else:
                properties['has_zero'] = 0 in training_data

            if self.method == np.nanmin:
                properties['min'] = parents[0].properties['min']
            else:
                properties['min'] = np.nanmin(training_data)

            if self.method == np.nanmax:
                properties['max'] = parents[0].properties['max']
            else:
                properties['max'] = np.nanmax(training_data)
        except:
            # was nonnumeric data
            pass
        properties['number_distinct_values'] = parents[1].properties['number_distinct_values']
        return properties


from torch import nn
import torch

class SmallEncoder(nn.Module):

    def __init__(self):
        super(SmallEncoder, self).__init__()

        self.conv1 = nn.Conv2d(3, 8, kernel_size=4, stride=2)
        self.conv2 = nn.Conv2d(8, 16, kernel_size=4, stride=2)
        self.conv3 = nn.Conv2d(16, 32, kernel_size=4, stride=2)
        self.conv4 = nn.Conv2d(32, 64, kernel_size=4, stride=2)

        self.l_mu = nn.Linear(256, 256)
        self.l_logvar = nn.Linear(256, 256)

    def forward(self, x):

        out = x

        out = self.conv1(out)
        out = nn.functional.relu(out)
        out = self.conv2(out)
        out = nn.functional.relu(out)
        out = self.conv3(out)
        out = nn.functional.relu(out)
        out = self.conv4(out)
        out = nn.functional.relu(out)

        out = out.reshape(-1, 256)

        logvar = self.l_logvar(out)
        mu = self.l_mu(out)

        std = torch.exp(0.5*logvar)
        eps = torch.randn_like(std)

        return mu + eps*std

class SmallDecoder(nn.Module):

    def __init__(self):
        super(SmallDecoder, self).__init__()

        self.deconv1 = nn.ConvTranspose2d(256, 32, kernel_size=5, stride=2)
        self.deconv2 = nn.ConvTranspose2d(32, 16, kernel_size=5, stride=2)
        self.deconv3 = nn.ConvTranspose2d(16, 8, kernel_size=6, stride=2)
        self.deconv4 = nn.ConvTranspose2d(8, 3, kernel_size=6, stride=2)

    def forward(self, x):

        out = x.reshape(-1, 256, 1, 1)

        out = self.deconv1(out)
        out = nn.functional.relu(out)
        out = self.deconv2(out)
        out = nn.functional.relu(out)
        out = self.deconv3(out)
        out = nn.functional.relu(out)
        out = self.deconv4(out)
        out = torch.sigmoid(out)

        return out

if __name__ == "__main__":

    x = torch.ones(1, 3, 64, 64)

    print(x.shape)

    encoder = SmallEncoder()
    decoder = SmallDecoder()

    y = encoder.forward(x)

    y = decoder.forward(y)

    print(y.shape)


# -*- coding = utf-8 -*-
# @Time : 2023/08/09 18:57
# @Autor : Fucloud
# @FIle : tts_voice.py
# @Software : PyCharm

from enum import Enum

class EnumVoice(Enum):
    # 在这里添加其他音频源
    XIAO_XIAO = {
        "name": "晓晓",
        "voiceName": "zh-CN-XiaoxiaoNeural",
        "styleList": {
            "angry": "angry",
            "lyrical": "lyrical",
            "calm": "gentle",
            "assistant": "affectionate",
            "cheerful": "cheerful"
        }
    }
    YUN_XI = {
        "name": "云溪",
        "voiceName": "zh-CN-YunxiNeural",
        "styleList": {
            "angry": "angry",
            "lyrical": "disgruntled",
            "calm": "calm",
            "assistant": "assistant",
            "cheerful": "cheerful"
        }
    }
    XIAO_YI = {
        "name": "晓依",
        "voiceName": "zh-CN-XiaoyiNeural",
        "styleList": {
            "angry": "angry",
            "lyrical": "disgruntled",
            "calm": "calm",
            "assistant": "assistant",
            "cheerful": "cheerful"
        }
    }
    YUN_FENG = {
        "name": "云枫",
        "voiceName": "zh-CN-YunfengNeural",
        "styleList": {
            "angry": "angry",
            "lyrical": "disgruntled",
            "calm": "calm",
            "assistant": "assistant",
            "cheerful": "cheerful"
        }
    }

def get_voice_list():
    return [EnumVoice.YUN_XI, EnumVoice.XIAO_XIAO, EnumVoice.XIAO_YI, EnumVoice.YUN_FENG]

def get_voice_of(name):
    for voice in get_voice_list():
        if voice.name == name:
            return voice
    return None


import model_functions
import pandas as pd
from sklearn.ensemble import IsolationForest

input_file_path = 'master_products_5per_error.csv' #path to users file

# we may want to pass read_csv settings to user
# get delimiter? escape character?
raw_df = pd.read_csv(input_file_path)
print('Read file of shape: {0}'.format(raw_df.shape))

# only using the hierarchy columns for evaluation
hier_cols = [col for col in raw_df.columns if 'train_level' in col]

input_df = raw_df[hier_cols].copy(deep=True)

# summary_df contains one record per column in input
# SZ_ values summarize number of records associated w/ each unique value
summary_df = model_functions.generate_col_summary(input_df)

# copy the input dataframe and append columns we will use as features
# more feature-generating functions could be added here
feature_df = input_df.copy(deep=True)
feature_df = model_functions.generate_tfidf_features(summary_df, feature_df)
feature_df = model_functions.generate_size_features(summary_df, feature_df)
feature_df = model_functions.generate_cumulative_size(summary_df, feature_df)

# isolation forest model applied
get_est_contamination = .05 #get from user input
n_estimators = 100 #adjust based on input size. more is better

iForest = IsolationForest(n_estimators=n_estimators, \
                          bootstrap=True, contamination=get_est_contamination)

# training variables are any columns to the right of the original input
train_variables = feature_df.iloc[:,input_df.shape[1]:].fillna(0)
iForest.fit(train_variables)
# write the model prediction and score to columns in the dataframe
feature_df['i_PREDICT'] = iForest.predict(train_variables)
feature_df['i_DECS'] = iForest.decision_function(train_variables)

# we may want to remove the features we generated before dropping to csv
to_file_path = 'output.csv'
feature_df.to_csv(to_file_path, index=False)

blend_df = raw_df.merge(feature_df, how='inner', left_index=True, right_index=True)
blend_df['train_result'] = blend_df.train_status == 'Correct Record'

prec = blend_df[(blend_df.train_result == False) & (blend_df.i_PREDICT == -1)].shape[0] / blend_df[(blend_df.i_PREDICT == -1)].shape[0]
print('Precision: {0}%'.format(round(100*prec)))

recall = blend_df[(blend_df.train_result == False) & (blend_df.i_PREDICT == -1)].shape[0] / blend_df[(blend_df.train_result == False)].shape[0]
print('Recall {0}%'.format(round(100*recall)))

f_score = 2 * ((prec * recall) / (prec + recall))
print('F-Score: {0}'.format(round(100*f_score)))


'''This module defines the Player class. To get a brief description of the class,
use the following syntax: <module name as imported>.Player.__doc__'''

import pygame, random, math
import LineParams as line
import MoveFunctionsUpdated as move
import NonplayerClasses as np

class Player(np.Game):
    '''This class is a child class of Game, which is defined in the module
    NonplayerClass. It's also the parent class of Goalkeeper and Outfielder.
    Player has the following methods:
        __init__, load, setFootStartPos, adjustFootStartPos, blitFeet,
        blitBody, getStartPos, getCenter, setCenterPos, getCenterPos,
        getCorners, getRotation, getBodyAngle, getFootAngle,
        setMovingRotation, getMovingRotation, getDistMoved,
        getDistToBall, getEnding, feetToFront, move, chooseKickingFoot,
        kickBall, checkBallTouch, updateFeet, and updateAll.
    To get a brief description of each method, use the following syntax:
        <module name as imported>.Player.<method name>.__doc__'''
    def __init__(self, screenSize):
        '''This function initializes the class and sets its core attributes.'''
        np.Game.__init__(self, screenSize)   # initialize the parent class
        self.lFoot, self.rFoot, self.body = [None] * 3
        self.footSize = 20, 25
        self.bodySize = 64, 40
        self.strongFoot = random.choice(['lFoot', 'rFoot'])
        self.touchedBall = False

    def load(self, imageName1, imageName2):
        '''This function loads images of the player's foot and body into PyGame
        and rotates them 90 degrees counterclockwise.'''
        self.lFoot = self.loadImage(imageName1,self.footSize[1],self.footSize[0])
        self.body = self.loadImage(imageName2, self.bodySize[1],self.bodySize[0])
        # rotate 90 degrees clockwise
        self.lFoot = pygame.transform.rotate(self.lFoot, 90)
        self.rFoot = self.lFoot
        self.body = pygame.transform.rotate(self.body, 90)
        # holder variables for values at the start of the program (i.e.,
        # before any change/movement has been made)
        self.footStart, self.bodyStart = self.lFoot, self.body
        self.footStartCenter = self.footStart.get_rect().center
        self.bodyStartCenter = self.bodyStart.get_rect().center

    def setFootStartPos(self):
        '''This function sets the positions where the feet will be
        drawn, assuming that the player will be facing upward at the
        start of the program.'''
        # coordinates of the body center at the start of the program (i.e.,
        # before any change/movement has been made)
        self.bodyCenterPos = (self.bodyStartPos[0]+self.bodyStartCenter[0],
                              self.bodyStartPos[1]+self.bodyStartCenter[1])
        # positions of the feet
        self.betweenFeet = 8
        self.lFootStartPos = (
            self.bodyCenterPos[0]-self.getCenter()[0][0]*2-self.betweenFeet/2,
            self.bodyStartPos[1]+2)
        self.rFootStartPos = (self.bodyCenterPos[0]+self.betweenFeet/2,
                              self.lFootStartPos[1])
        # coordinates of the foot centers at the start of the program
        self.lFootCenterPos = (self.lFootStartPos[0]+self.footStartCenter[0],
                               self.lFootStartPos[1]+self.footStartCenter[1])
        self.rFootCenterPos = (self.rFootStartPos[0]+self.footStartCenter[0],
                               self.rFootStartPos[1]+self.footStartCenter[1])

    def adjustFootStartPos(self):
        '''This function adjusts the positions where the feet will be drawn at
        the start of the program. To do that, both the coordinates of the body
        center and those of the foot centers are adjusted.'''
        newBodyCenterPos = (self.bodyStartPos[0]+self.getCenter()[1][0],
                            self.bodyStartPos[1]+self.getCenter()[1][1])
        # distance from the old coordinates
        moveX, moveY = self.getDistMoved(newBodyCenterPos)
        self.setCenterPos('body', newBodyCenterPos[0], newBodyCenterPos[1])
        self.setCenterPos('lFoot', self.getCenterPos()[0][0]+moveX,
                          self.getCenterPos()[0][1]+moveY)
        self.setCenterPos('rFoot', self.getCenterPos()[1][0]+moveX,
                          self.getCenterPos()[1][1]+moveY)
        self.setMovingRotation(self.startRot)
        self.feetToFront()
        self.lFootStartPos = (
            self.getCenterPos()[0][0]-self.getCenter()[0][0],
            self.getCenterPos()[0][1]-self.getCenter()[0][1])
        self.rFootStartPos = (
            self.getCenterPos()[1][0]-self.getCenter()[0][0],
            self.getCenterPos()[1][1]-self.getCenter()[0][1])

    def blitFeet(self):
        '''This function rotates both the feet and the body and then draws only
        the feet onto the screen.
        rotate is the angle (measured in degrees) by which the feet will be
        rotated from the original upward rotation.'''
        self.setFootStartPos()   # where the feet will be drawn
        # rotate both feet and body
        self.lFoot = pygame.transform.rotate(self.lFoot, self.startRot)
        self.rFoot = pygame.transform.rotate(self.rFoot, self.startRot)
        self.body = pygame.transform.rotate(self.body, self.startRot)
        self.adjustFootStartPos()
        # draw feet
        self.screen.blit(self.lFoot, self.lFootStartPos)
        self.screen.blit(self.rFoot, self.rFootStartPos)
        self.things = [self.lFoot, self.rFoot]

    def blitBody(self):
        '''This function draws the body onto the screen.
        rotate is the angle (measured in degrees) by which the image will be
        rotated from the original upward rotation.'''
        self.screen.blit(self.body, self.bodyStartPos)
        self.things += [self.body]

    def getStartPos(self):
        '''This function returns the positions of the feet and the body at the
        start of the program.'''
        return [self.lFootStartPos, self.rFootStartPos, self.bodyStartPos]

    def getCenter(self):
        '''This function returns the centers of the feet and the body.'''
        footCenter = self.lFoot.get_rect().center
        bodyCenter = self.body.get_rect().center
        return footCenter, bodyCenter

    def setCenterPos(self, bodyPart, X, Y):
        '''This function sets a new position for the center of one of the feet
        or the center of the body.'''
        if bodyPart == 'lFoot':
            self.lFootCenterPos = X, Y
        elif bodyPart == 'rFoot':
            self.rFootCenterPos = X, Y
        else:
            self.bodyCenterPos = X, Y

    def getCenterPos(self):
        '''This function returns the coordinates of the centers of the feet
        and of the body.'''
        return self.lFootCenterPos, self.rFootCenterPos, self.bodyCenterPos

    def getCorners(self):
        '''This function returns a list containing the coordinates of the
        four corners of the player's body. The corners are listed in the
        following order: front left, front right, back left, and back right.
        Note that the size of the original image (before resizing) was
        (31, 22), and and that the image of the body has extra pixels along
        both the front and the back to account for the player's head.'''
        # number of extra pixels along each side (front/back)
        extraPix = 3 / 22 * self.bodyStart.get_rect().height
        # coordinates of the body center
        bodyCenterPos = self.getCenterPos()[2]
        # coordinates of the corners if the player were/is facing upward
        corner1 = (bodyCenterPos[0]-self.bodyStartCenter[0],
                   bodyCenterPos[1]-self.bodyStartCenter[1]+extraPix)
        corner2 = corner1[0]+self.bodySize[0], corner1[1]
        corner3 = corner1[0], corner1[1]+(self.bodyStartCenter[1]-extraPix)*2
        corner4 = corner2[0], corner3[1]
        corners = [corner1, corner2, corner3, corner4]
        # account for player's current rotation
        for i in range(len(corners)):
            corners[i] = move.rotate('right', corners[i], bodyCenterPos,
                                     step=self.getRotation())[0]
        return corners

    def getRotation(self):
        '''This function returns the player's current rotation, i.e., the
        direction the player is currently facing. It is an angle (measured in
        degrees) formed by two lines: (1) the line connecting the body center
        and the midpoint between the two foot centers, and (2) the y-axis
        pointing downward from the midpoint.'''
        # coordinates of the midpoint between the two foot centers
        midpoint = ((self.getCenterPos()[0][0]+self.getCenterPos()[1][0])/2,
                    (self.getCenterPos()[0][1]+self.getCenterPos()[1][1])/2)
        # current rotation
        currentRot = line.getParams(midpoint, self.getCenterPos()[2])[3]
        return currentRot

    def getBodyAngle(self, ball):
        '''This function returns the angle (measured in degrees) of the body
        with respect to the y-axis pointing downward from the ball.'''
        return line.getParams(ball.getCenterPos(), self.getCenterPos()[2])[3]

    def getFootAngle(self, ball):
        '''This function returns the angles (measured in degrees) of the feet
        with respect to the y-axis pointing downward from the ball.'''
        left = line.getParams(ball.getCenterPos(), self.getCenterPos()[0])[3]
        right = line.getParams(ball.getCenterPos(), self.getCenterPos()[1])[3]
        return left, right

    def setMovingRotation(self, rotate):
        '''This function sets the angle (measured in degrees) by which the
        rotation of the player has changed, after a movement, from that at the
        start of the program.'''
        self.rotate = rotate

    def getMovingRotation(self):
        '''This function returns the angle (measured in degrees) by which the
        rotation of the player has changed, after a movement, from that at the
        start of the program.'''
        return self.rotate

    def getDistMoved(self, newCenterPos):
        '''This function returns the change in the position of
        the body after a movement.
        newCenterPos is the new coordinates of the body center.'''
        return line.getParams(self.getCenterPos()[2], newCenterPos)[:2]

    def getDistToBall(self, ball):
        '''This function returns the distance from the ball to each foot and
        to the body.'''
        lFoot = line.getParams(ball.getCenterPos(), self.getCenterPos()[0])[2]
        rFoot = line.getParams(ball.getCenterPos(), self.getCenterPos()[1])[2]
        body = line.getParams(ball.getCenterPos(), self.getCenterPos()[2])[2]
        return lFoot, rFoot, body

    def getEnding(self, bodyPart, ball, finalDist):
        '''This function returns the point to which the body/foot will move
        when it's headed for the ball, along with the direction the body/foot
        will face when it reaches the endpoint (angle measured in degrees).
        finalDist is the distance between the center of the body/foot and
        the ball center when the body/foot is at the target point.'''
        if 'Foot' in bodyPart:
            if bodyPart == 'lFoot':
                angle = self.getFootAngle(ball)[0]
            else:
                angle = self.getFootAngle(ball)[1]
        else:
            angle = self.getBodyAngle(ball)
        # point to which the body will move
        endPointX = ball.getCenterPos()[0] + finalDist * np.getTrig(angle)[0]
        endPointY = ball.getCenterPos()[1] + finalDist * np.getTrig(angle)[1]
        endPoint = endPointX, endPointY
        # final rotation, i.e., direction the body will face when it reaches
        # the endpoint (measured in degrees)
        endAngle = line.getParams(ball.getCenterPos(), endPoint)[3]
        return endPoint, endAngle

    def feetToFront(self):
        '''This function moves the feet to the front of the body.
        target is either the coordinates of the point around which the player
        rotates or the coordinates of the point to which the player moves.'''
        currentRot = self.getRotation()   # measured in degrees
        # move feet
        self.lFootCenterPos = move.rotate(
            'right', self.getCenterPos()[0], self.getCenterPos()[2],
            step=self.getMovingRotation()-currentRot)[0]
        self.rFootCenterPos = move.rotate(
            'right', self.getCenterPos()[1], self.getCenterPos()[2],
            step=self.getMovingRotation()-currentRot)[0]

    def move(self, moveType, direction, ball):
        '''This function moves the player in one of three ways: in a straight
        line, straight toward the ball, or in a circle around the ball.'''
        # move body
        if moveType == 'straight':
            newCenterPos, rotate = move.straight(
                direction,self.getCenterPos()[2],objCenter=self.getCenter()[1],
                screenSize=(self.screenWidth,self.screenHeight))
        elif moveType == 'to ball':
            finalDist = self.bodyStartCenter[1] + ball.center[1]
            endPoint, endAngle = self.getEnding('body', ball, finalDist)
            newCenterPos, rotate = move.toPoint(self.getCenterPos()[2],
                                                endPoint, endAngle, 3)
        elif moveType == 'circle':
            bodyAngle = self.getBodyAngle(ball)
            newCenterPos, rotate = move.rotate(
                direction, self.getCenterPos()[2], ball.getCenterPos(),
                screenSize=(self.screenWidth,self.screenHeight),
                objCenter=self.getCenter()[1])
        else:
            newCenterPos = self.getCenterPos()[2]
        if newCenterPos != self.getCenterPos()[2]:
            self.moved = True
            moveX, moveY = self.getDistMoved(newCenterPos)
            self.setCenterPos('body', newCenterPos[0], newCenterPos[1])
            self.setMovingRotation(rotate)
            # move feet at the same distance as body
            self.setCenterPos('lFoot', self.getCenterPos()[0][0]+moveX,
                              self.getCenterPos()[0][1]+moveY)
            self.setCenterPos('rFoot', self.getCenterPos()[1][0]+moveX,
                              self.getCenterPos()[1][1]+moveY)
            if self.getRotation() != self.getMovingRotation():
                # player facing the wrong direction
                self.feetToFront()

    def chooseKickingFoot(self, ball):
        '''This function chooses the foot that will kick the ball and returns:
        (1) a string indicating the foot,
        (2) the coordinates of the foot's center point, and
        (3) the distance between the ball and that foot.'''
        lFootToBall, rFootToBall = self.getDistToBall(ball)[:2]
        if lFootToBall < rFootToBall:
            return 'lFoot', self.getCenterPos()[0], lFootToBall
        elif lFootToBall > rFootToBall:
            return 'rFoot', self.getCenterPos()[1], rFootToBall
        else:   # ball is equidistant from both feet
            if self.strongFoot == 'lFoot':
                return self.strongFoot, self.getCenterPos()[0], lFootToBall
            else:
                return self.strongFoot, self.getCenterPos()[1], rFootToBall

    def kickBall(self, ball, gk=False):
        '''This function moves one of the feet to kick the ball.
        gk denotes whether or not the player is the goalkeeper.
        If he is, he only kicks the ball when it's nearby. If not, he
        only kicks the ball when it's not in the goalkeeper's possession.'''
        currentRot = self.getRotation()   # measured in degrees
        kickingFoot, kFootCenterPos, distToBall = self.chooseKickingFoot(ball)
        if kickingFoot == 'lFoot':
            endAngle = self.getFootAngle(ball)[0]
        else:
            endAngle = self.getFootAngle(ball)[1]
        minDist = self.footStartCenter[1] + ball.center[1]
        if distToBall <= minDist:
            endPoint = kFootCenterPos
        else:
            endPoint = self.getEnding(kickingFoot, ball, minDist)[0]
        # move foot
        newKFootCenterPos, kFootRotate = move.toPoint(kFootCenterPos,
                                                      endPoint, endAngle)
        if newKFootCenterPos == endPoint:
            self.touchedBall = True
        else:
            self.touchedBall = False
        if newKFootCenterPos != kFootCenterPos:
            if (gk and self.touchedBall) or \
               ((not gk) and (not ball.gkCaught) and (distToBall <= minDist*2)):
                if kickingFoot == 'lFoot':
                    newCenterPos = newKFootCenterPos, self.getCenterPos()[1]
                    rotate = kFootRotate, currentRot
                else:
                    newCenterPos = self.getCenterPos()[0], newKFootCenterPos
                    rotate = currentRot, kFootRotate
                self.updateFeet(kickingFoot, newCenterPos, rotate)
                self.updateDisplay()
                pygame.time.wait(100)   # pause program for 100 ms
                # bring foot back to its position and rotation before the kick
                if kickingFoot == 'lFoot':
                    newCenterPos = kFootCenterPos, self.getCenterPos()[1]
                else:
                    newCenterPos = self.getCenterPos()[0], kFootCenterPos
                self.updateFeet(kickingFoot, newCenterPos, (currentRot,)*2)

    def updateFeet(self, foot, newCenterPos, rotate):
        '''This function updates the positions and rotations of only the feet.
        newCenterPos is a tuple/list of the new coordinates of the
        center of each foot.
        rotate is an array of the angles (measured in degrees) by which
        the rotations of the feet have changed from those at the start of
        the program.'''
        moved = self.lFoot, self.rFoot, self.footStart, self.footStart
        lFootIndex = self.allThings.index(self.lFoot)
        move.update(self.screen, self.allThings, self.allPos, moved,
                    newCenterPos, rotate)
        self.lFoot, self.rFoot = self.allThings[lFootIndex:lFootIndex+2]

    def updateAll(self):
        '''This function updates the positions and rotations of both the body
        and the feet.'''
        moved = (self.lFoot, self.rFoot, self.body,
                 self.footStart, self.footStart, self.bodyStart)
        newCenterPos = self.getCenterPos()
        lFootIndex = self.allThings.index(self.lFoot)
        bodyIndex = self.allThings.index(self.body)
        move.update(self.screen, self.allThings, self.allPos, moved,
                    newCenterPos, rotate=(self.getMovingRotation(),)*3)
        # new values
        self.lFoot, self.rFoot = self.allThings[lFootIndex:lFootIndex+2]
        self.body = self.allThings[bodyIndex]

class Goalkeeper(Player):
    '''This class is a child class of Player and has the following methods:
    __init__ and moveAcross.
    To get a brief description of each method, use the following syntax:
        <module name as imported>.Goalkeeper.<method name>.__doc__'''
    def __init__(self, screenSize):
        '''This function initializes the class and sets its core attributes.'''
        Player.__init__(self, screenSize)   # initialize the parent class
        self.bodyStartPos = (self.screenWidth-76)/2, 80
        self.startRot = 180   # measured in degrees
        self.direction = random.choice([1, -1])   # right or left
        self.speed = 5   # number of pixels per movement

    def moveAcross(self, goal):
        '''This function makes the goalkeeper move between the goal posts,
        only stopping when he catches the ball (i.e., when the ball hits
        the front of his body).'''
        currentRot = self.getRotation()   # measured in degrees
        # points to which the player will move
        lEndPoint = goal.getPosts()[0] + self.getCenter()[1][0]
        rEndPoint = goal.getPosts()[1] - self.getCenter()[1][0]
        if self.direction == 1:   # playing moving to the right
            if self.getCenterPos()[2][0] >= rEndPoint:   # reached right post
                self.direction = -1
        else:   # player moving left
            if self.getCenterPos()[2][0] <= lEndPoint:   # reached left post
                self.direction = 1
        # move player
        self.setCenterPos('body',
                          self.getCenterPos()[2][0]+self.speed*self.direction,
                          self.getCenterPos()[2][1])
        self.setCenterPos('lFoot',
                          self.getCenterPos()[0][0]+self.speed*self.direction,
                          self.getCenterPos()[0][1])
        self.setCenterPos('rFoot',
                          self.getCenterPos()[1][0]+self.speed*self.direction,
                          self.getCenterPos()[1][1])
        self.updateAll()

class Outfielder(Player):
    '''This class is a child class of Player and has only one mthod: __init__.
    To get a brief description of each method, use the following syntax:
        <module name as imported>.Outfielder.<method name>.__doc__'''
    def __init__(self, screenSize):
        '''This function initializes the class and sets its core attributes.'''
        Player.__init__(self, screenSize)   # initialize the parent class
        bodyStartPosX = random.uniform(5, self.screenWidth-5-self.bodySize[0])
        bodyStartPosY = random.uniform(100,self.screenHeight-5-self.bodySize[1])
        self.bodyStartPos = bodyStartPosX, bodyStartPosY
        self.startRot = 0
        self.moved = False


"""
We define a function, F, on a string, P, as follows:
    F(P)=(length(P)^(distinct(P)))%(10^9 +7)
where:
    length(P) denotes the number of characters in string P.
    distinct(P) denotes the number of distinct characters in string P.
Consuela loves creating string challenges and she needs your help testing her newest one! Given
a string, S, consisting of N lowercase letters, compute the summation of function F (provided
above) over all possible distinct substrings of S. As the result is quite large, print it
modulo 10^9 +7.
Input Format
    The first line contains a single integer, T, denoting the number of test cases.
    Each of the T subsequent lines contains a string, S.
Constraints
    1<=T<=100
    1<=N<=10^5
    The sum of N over all test cases does not exceed 10^5.
Scoring
    N<=100 for 20% of test data.
    M<=1000 for 40% of test data.
    N<=10^5 for 100% of test data.
Output Format
    For each test case, print the answer modulo 10^9 +7.
Sample Input
    3
    aa
    aba
    abc
Sample Output
    3
    19
    38
Explanation
    Test 0:
        "a" and "aa" are the only distinct substrings.
        * F("a")=(1^1)%10^9 +7 =1
        * F("aa")=(2^1)%1000000007=2
        ans=(1+2)%1000000007=3
    Test 1:
        "a", "b", "ab", "aba", and "ba" are the only distinct substrings.
        * F("a")=(1^1)%1000000007=1
        * F("ab")=(2^2)%1000000007=4
        * F("aba")=(3^2)%1000000007=9
        * F("b")=(1^1)%1000000007=1
        * F("ba")=(2^2)%1000000007=4
        ans=(1+4+9+1+4)%1000000007=19
"""
from string import ascii_lowercase
from bisect import bisect_left, bisect_right
from itertools import zip_longest, islice

MOD = 7 + 10 ** 9

N = 10 ** 5

sum_pow = [[0] * (N + 1) for k in range(27)]
for k in range(1, 27):
    for n in range(1, N + 1):
        sum_pow[k][n] = (sum_pow[k][n - 1] + pow(n, k, MOD)) % MOD

def get_sp(left, right, k):
    if left > right or right <= 0:
        return 0
    if left <= 0:
        return sum_pow[k][right]
    return (sum_pow[k][right] + MOD - sum_pow[k][left - 1]) % MOD

def all_occ(s):
    n = len(s)
    occ = [[] for _ in range(26)]
    for i in range(n):
        occ[ord(s[i]) - ord('a')].append(i)
    return occ

def occ_from_i(occ, i):
    occ_i = []
    for j in range(26):
        if len(occ[j]) == 0 or i > occ[j][-1]:
            continue
        first = bisect_left(occ[j], i)
        occ_i.append(occ[j][first])
    return sorted(occ_i)

def sorted_idx(items):
    unique = sorted(set(items))
    idx = dict(zip(unique, range(len(unique))))
    return [idx[v] for v in items]

def suffix_array(s):
    n = len(s)
    k = 1
    sa = sorted_idx(s)
    while max(sa) < n - 1:
        sa = sorted_idx([a * (n + 1) + b + 1 for
                         (a, b) in zip_longest(sa,
                                               islice(sa, k, None),
                                               fillvalue=-1)])
        k <<= 1
    return sa

def lcp_sa(s):
    inv_sa = suffix_array(s)
    n = len(inv_sa)
    sa = [0] * n
    for i in range(n):
        sa[inv_sa[i]] = i
    lcp = [0] * n
    k = 0
    for i in range(n):
        if inv_sa[i] == n - 1:
            k = 0
            continue
        j = sa[inv_sa[i]+1]
        while i + k < n and j + k < n and s[i + k] == s[j + k]:
            k += 1
        lcp[inv_sa[i] + 1] = k
        if k > 0:
            k -= 1
    return sa, lcp

def solve(s):
    n = len(s)
    occ = all_occ(s)
    sa, lcp = lcp_sa(s)
    ans = 0
    #print(sa)
    #print(lcp)
    for i in range(n):
        o = occ_from_i(occ, sa[i])
        t = sa[i] + lcp[i]
        if t >= o[-1]:
            ans = (ans + get_sp(lcp[i] + 1, n - sa[i], len(o))) % MOD
            continue
        k = bisect_right(o, t)
        ans = (ans + get_sp(lcp[i] + 1, o[k] - sa[i], k)) % MOD
        for j in range(k + 1, len(o)):
            ans = (ans + get_sp(o[j - 1] - sa[i] + 1, o[j] - sa[i], j)) % MOD

        ans = (ans + get_sp(o[-1] - sa[i] + 1, n - sa[i], len(o))) % MOD

    return ans

def sum_p_bf(left, right, k):
    ans = 0
    for n in range(max(left, 1), right + 1):
        ans = (ans + pow(n, k, MOD)) % MOD
    return ans

q = int(input().strip())
for _ in range(q):
    string = input().strip()
    print(solve(string))