from fragview.models import User, Library, UserProject, PendingProject
from django.test import TestCase
from tests.utils import ProjectTestCase
from tests.project_setup import Project

class TestUserProject(TestCase):
    ERR_MSG = "dummy error msg"

    def setUp(self):
        self.project = UserProject.create_new("MID2", "20201122")
        self.assertTrue(self.project.is_pending())

    def test_set_ready(self):
        """
        test setting pending project into 'ready' state
        """
        self.project.set_ready()
        self.assertFalse(self.project.is_pending())

    def test_set_failed(self):
        """
        test setting error on a pending project
        """

        # set error on the project
        self.project.set_failed(self.ERR_MSG)

        # should still be pending
        self.assertTrue(self.project.is_pending())

        # check that error message was set in the database
        pend_proj = PendingProject.objects.get(project=self.project.id)
        self.assertIsNotNone(pend_proj)
        self.assertEqual(pend_proj.error_message, self.ERR_MSG)

class TestLibrary(ProjectTestCase):
    PROJECTS = [
        Project(
            protein="Nsp5",
            proposal="20180453",
            datasets=[],
            crystals=[],
            results=[],
        ),
        Project(
            protein="MID2",
            proposal="20180453",
            datasets=[],
            crystals=[],
            results=[],
        ),
    ]

    def setUp(self):
        super().setUp()

        lib = Library(name="public")
        lib.save()

        lib = Library(name="private", project_id=self.project.id)
        lib.save()

    def _assert_libs(self, libs, expected_names: set[str]):
        got_names = {lib.name for lib in libs}
        self.assertSetEqual(got_names, expected_names)

    def test_get_all(self):
        # for first project, we should get all libraries
        libs = Library.get_all(self.project)
        self._assert_libs(libs, {"public", "private"})

        # for second project, we should only get 'public' library
        libs = Library.get_all(self.projects[1])
        self._assert_libs(libs, {"public"})

    def test_get_by_name(self):
        #
        # for first project we should be able to get both libs
        #
        lib = Library.get_by_name(self.project, "public")
        self.assertEqual(lib.name, "public")

        lib = Library.get_by_name(self.project, "private")
        self.assertEqual(lib.name, "private")

        #
        # for second project, we should only be able to get
        # the 'public' library
        #
        lib = Library.get_by_name(self.projects[1], "public")
        self.assertEqual(lib.name, "public")

        with self.assertRaises(Library.DoesNotExist):
            Library.get_by_name(self.projects[1], "private")

class TestUser(ProjectTestCase):
    USERNAME = "muser"

    def setUp(self):
        super().setUp()

        self.user_project = UserProject.get(self.projects[0].id)
        self.user = User(username=self.USERNAME)
        self.user.save()

    def test_set_current_project(self):
        """
        test User.set_current_project
        """
        # new user should not have a current_project set initially
        self.assertIsNone(self.user.current_project)

        # set the current project for the user
        self.user.set_current_project(self.user_project)

        #
        # reload user from the database and check it's current project
        #
        rl_usr = User.objects.get(username=self.USERNAME)
        self.assertEqual(rl_usr.current_project.id, self.user_project.id)

    def test_get_current_project_none(self):
        """
        no current project set for the user, and user does not have
        access to any other projects
        """
        self.assertIsNone(self.user.get_current_project(["20160101"]))

    def test_get_current_project_explicit(self):
        """
        get explicitly set current project
        :return:
        """
        self.user.set_current_project(self.user_project)

        proj = self.user.get_current_project(self.proposals)
        self.assertEqual(proj.id, self.user_project.id)

    def test_get_current_project_implicitly(self):
        """
        user have not set current_project, but have access to one of the projects
        """

        # make sure no explicit current project is set
        self.assertIsNone(self.user.current_project)

        proj = self.user.get_current_project(self.proposals)
        self.assertEqual(proj.id, self.user_project.id)


import argparse
import os
import sys
sys.path.append("src/python/")

import numpy as np
import pandas as pd
import anndata as ann
import scanpy as sc

from utils import diffexp, dge_top_n

def main(h5ad_loc, save_loc, dataset_name, rep):
    # Load h5ad file
    adata = sc.read_h5ad(h5ad_loc)

    # Extract summary statistics from h5ad file
    num_batches_ds = adata.uns["downsampling_stats"]["num_batches"]
    num_celltypes_ds = adata.uns["downsampling_stats"]["num_celltypes_downsampled"]
    prop_ds = adata.uns["downsampling_stats"]["proportion_downsampled"]

    # Subset adatas based on method for integration and store lognorm counts in raw
    # attribute for diffexp testing
    methods = ["harmony", "scvi", "scanorama", "seurat", "liger", "bbknn"]
    method_adatas = []
    for method in methods:
        adata_copy = adata.copy()
        adata_subset = adata_copy[adata_copy.obs["integration_method"] == method]
        adata_subset.X = adata_subset.layers["raw"] # Unlogged, unnorm counts
        sc.pp.normalize_total(
            adata_subset,
            target_sum = 1e4
        )
        sc.pp.log1p(adata_subset)
        adata_subset.raw = adata_subset # Freeze for DGE test
        method_adatas.append(adata_subset)

    # Extract list of all DGEs for all leiden clusters in each method
    method_dge_dfs = []
    for adata_method_subset in method_adatas:
        adata_method_subset = diffexp(
            adata_method_subset,
            groupby = "leiden",
            use_raw = True,
            method = "wilcoxon"
        )
        dge_results = dge_top_n(
            adata_method_subset,
            n = len(adata.var),
            obs_group = "leiden"
        )
        method_dge_dfs.append(dge_results)

    # For each method, compute the ranking metrics for all genes in the dataset
    all_genes = np.sort(adata.var.index.values)
    all_genes_tiled = np.tile(all_genes, (len(methods)))
    methods_repeat = np.repeat(methods, len(all_genes))
    gene_max_imp_per_method = []
    gene_min_imp_per_method = []
    for method_dge_df in method_dge_dfs:
        method_clusters = np.unique(method_dge_df["Cluster"].__array__())
        cluster_ranks = []
        for cluster in method_clusters:
            cluster_sub = method_dge_df[method_dge_df["Cluster"] == cluster]
            gene_ranks_sorted = np.argsort(cluster_sub.iloc[:, 1].__array__())
            cluster_ranks.append(gene_ranks_sorted)
        cluster_ranks_stack = np.stack(cluster_ranks, axis = 0)
        # Min for max because lowest number for ranking corresponds to highest importance
        gene_max_imp_per_method.append(
            np.min(cluster_ranks_stack, axis = 0)
        )
        gene_min_imp_per_method.append(
            np.max(cluster_ranks_stack, axis = 0)
        )

    # Concatenate max and min ranks for each method
    gene_max_imp_per_method_concat = np.concatenate(gene_max_imp_per_method)
    gene_min_imp_per_method_concat = np.concatenate(gene_min_imp_per_method)

    # Create summary df of all genes and their ranking metrics
    dge_ranking_summary_df = pd.DataFrame({
        "Dataset": dataset_name,
        "Number of batches downsampled": num_batches_ds,
        "Number of celltypes downsampled": num_celltypes_ds,
        "Proportion downsampled": prop_ds,
        "Replicate": rep,
        "Method": methods_repeat,
        "Gene": all_genes_tiled,
        "Max rank": gene_max_imp_per_method_concat,
        "Min rank": gene_min_imp_per_method_concat
    })
    dge_ranking_summary_df.to_csv(save_loc, sep = "\t", index = False)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description = "Input and output files for dge ranking summary"
    )
    parser.add_argument(
        "--infile",
        type = str,
        help = "Path of integrated h5ad file"
    )
    parser.add_argument(
        "--outfile",
        type = str,
        help = "Filepath for saving dge ranking statistics of integrated h5ad file"
    )
    parser.add_argument(
        "--dataset",
        type = str,
        help = "Name of dataset"
    )
    parser.add_argument(
        "--rep",
        type = int,
        help = "Repetition number"
    )
    args = parser.parse_args()
    main(
        h5ad_loc = args.infile,
        save_loc = args.outfile,
        dataset_name = args.dataset,
        rep = args.rep
    )


import jwt
import datetime
from app import app
from models.Article import ArticleModel

class jwtclass:

 def encode_auth_token(self,user_id):
     try:
        payload = {
            'exp': datetime.datetime.utcnow() + datetime.timedelta(days=0, seconds=5),
            'iat': datetime.datetime.utcnow(),
            'sub': user_id
        }
        return jwt.encode(
            payload,
            app.config.get('SECRET_KEY'),
            algorithm='HS256'
        )
     except Exception as e:
        return e

 def decode_auth_token(self,auth_token):
    """
    Decodes the auth token
    :param auth_token:
    :return: integer|string
    """

    try:
        payload = jwt.decode(auth_token, app.config.get('SECRET_KEY'))
        return payload['sub']
    except jwt.ExpiredSignatureError:
        return 'Signature expired. Please log in again.'
    except jwt.InvalidTokenError:
        return 'Invalid token. Please log in again.'


import sqlite3

def setup():
    # Connect to or create the database
    conn = sqlite3.connect('example.db')

    # Create a cursor
    cursor = conn.cursor()

    # Create a tablex
    cursor.execute('''
        CREATE TABLE IF NOT EXISTS replay (
            id INTEGER PRIMARY KEY,
            nodeTraversal TEXT,
            ksuid CHAR(100) NOT NULL
        )
    ''')
    return conn

def insert(nodeTraversal, ksu_id):
    # Insert values into the table
    conn = setup()
    cursor = conn.cursor()
    # rows = fetchallRows()
    rows = select(ksu_id);
    if(len(rows) > 0):
        query = 'Update replay set nodeTraversal =?  where ksuid = ?'
        cursor.execute(query, (nodeTraversal, ksu_id))
    else:
        cursor.execute('INSERT INTO replay (nodeTraversal, ksuid) VALUES (?, ?)', (nodeTraversal, ksu_id))
    conn.commit()

    # Close the connection
    conn.close()

def select(ksu_id):
    conn = setup()
    cursor = conn.cursor()
    query = 'SELECT nodeTraversal FROM replay where ksuid =?'
    cursor.execute(query, [ksu_id])
    rows = cursor.fetchall()

    # Close the connection
    conn.close()
    return rows

def fetchallRows():
    conn = setup()
    cursor = conn.cursor()
    query = 'SELECT ksuid, nodeTraversal FROM replay'
    cursor.execute(query)
    rows = cursor.fetchall()
    print(rows)
    # Close the connection
    conn.close()
    return rows


import requests
import simplejson as js
from datetime import datetime
import pystrava
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

print('Start!')

# Load settings --------------------------------------------------------------------------------------------------------
with open('setup.ini') as fid:
    initial_settings = js.load(fid)
saving_path = initial_settings['saving_path']

# Connect to strava ----------------------------------------------------------------------------------------------------
stcon = pystrava.strava_user(base_url = 'https://www.strava.com/api/v3',client_id=initial_settings['client_id'],
                                      client_secret=initial_settings['client_secret'],
                                      code=initial_settings['code'])
# Get token ------------------------------------------------------------------------------------------------------------
stcon.get_token()

# Download all activity into a JSON ------------------------------------------------------------------------------------
download_all_enabled = False
if download_all_enabled:
    act_list = stcon.get_activity_list()
    with open(saving_path + '/activity_list.json','w+') as fid:
        js.dump(act_list,fp=fid)
else:
    with open(saving_path + '/activity_list.json','r') as fid:
        act_list = js.load(fp=fid)

# Create a DataFrame from the JSON
act_data = pd.DataFrame(act_list)
act_data['start_date_utc'] = act_data.start_date.apply(lambda x: datetime.strptime(x, '%Y-%m-%dT%H:%M:%SZ').timestamp())

# Get activity details -------------------------------------------------------------------------------------------------
act_ids = [2041949158,2006283792,1990887172]
act_streams_dict = stcon.get_activity_streams(act_ids)

# Create dict of DF from activity details
act_streams_dict_df = dict()
for k,v in act_streams_dict.items():
    temp_dict = dict()
    for df_k, df_v in v.items():
        temp_dict[df_k] = df_v['data']

    act_streams_dict_df[k] = pd.DataFrame(temp_dict)

id_sel = str(2041949158)
ax = act_streams_dict_df[id_sel].plot(x='time',y='heartrate',color='C1')
ax.legend(['heartrate'], loc=2)
ax2 = ax.twinx()
act_streams_dict_df[id_sel].plot(x='time',y='altitude',ax=ax2,color='C2')
ax2.legend(['altitude'], loc=1)
ax.grid(True,which='major')
plt.show(block=False)

# Get all preferred segments -------------------------------------------------------------------------------------------
seg_list = stcon.get_starred_segments()
with open(saving_path + '/star_seg_list.json', 'w+') as fid:
    js.dump(seg_list, fp=fid)

print('Finish!')


import requests

class YaUploader:

    def __init__(self, token):
        self.token = token

    def get_headers(self):
        return {
            'Content-Type': 'application/json',
            'Authorization': 'OAuth {}'.format(self.token)
        }

    def create_folder(self, path):
        url = "https://cloud-api.yandex.net/v1/disk/resources"
        headers = self.get_headers()
        res = requests.put(f'{url}?path={path}', headers=headers)
        print(f"Папка \"{path}\" на яндекс диске создана")
        return res


#!/usr/bin/env python3

# Version 1.1
# Author Alexis Blanchet-Cohen
# Date: 13/04/2014

import argparse
import configparser
import glob
import os
import os.path
import subprocess
import sys
import util

parser = argparse.ArgumentParser(description="Separates BAM files by strand.")
parser.add_argument("-s", "--scriptsDirectory", help="Scripts directory. DEFAULT=separatebamsbystrand", default="separatebamsbystrand")
parser.add_argument("-i", "--inputDirectory", help="Input directory with BAM files. DEFAULT=../results/star", default="../results/star")
parser.add_argument("-o", "--outputDirectory", help="Output directory with bedgraph files. DEFAULT=../results/star", default="../results/star")
parser.add_argument("-e", "--extension", help="Extension to identify BAM files to sort. DEFAULT=.bam", default=".bam")
parser.add_argument("-q", "--submitJobsToQueue", help="Submit jobs to queue immediately.", choices=["yes", "no", "y", "n"], default="no")
args = parser.parse_args()

# If not in the main scripts directory, cd to the main scripts directory, if it exists.
util.cdMainScriptsDirectory()

# Read samples files
samples = util.getSamples()

# Process command line arguments.
scriptsDirectory = os.path.abspath(args.scriptsDirectory)
inputDirectory = os.path.abspath(args.inputDirectory)
outputDirectory = os.path.abspath(args.outputDirectory)
extension = args.extension

# Check if the inputDirectory exists, and is a directory.
util.checkInputDirectory(inputDirectory)

# Create script and output directories, if they do not exist yet.
util.makeDirectory(outputDirectory)
util.makeDirectory(scriptsDirectory)

# Read configuration files
config = util.readConfigurationFiles()

header = config.getboolean("server", "PBS_header")

# cd to scripts directory
os.chdir(scriptsDirectory)

# Function to check BAM file names.
# Create a symbolic link to accepted_hits.bam named sampleName.bam. Useful for TopHat output, which is named accepted_hits.bam, by default
#def createSymbolicLinks():
#    for sample in samples:
#        command = "ln -fs " + os.path.join(inputDirectory, sample, "accepted_hits.bam") + " " + os.path.join(inputDirectory, sample, sample + ".bam")
#        print(command)
#        subprocess.call(command, shell=True)

#if (args.symbolicLinks.lower() == "yes") | (args.symbolicLinks.lower() == "y"):
#    createSymbolicLinks()

strands = ["positive", "negative"]

# Write script for each sample and each strand
for sample in samples:
    for strand in strands:
        # Create script file.
        scriptName = "separatebamsbystrand_" + sample + "_" + strand + ".sh"
        script = open(scriptName, "w")
        if header:
            util.writeHeader(script, config, "separatebambystrand")
        # BAM to bedgraph
        script.write("separatebambystrand.py" + " \\\n")
        script.write("--input_bam " + os.path.relpath(os.path.join(inputDirectory, sample, sample + ".bam")) + " \\\n")
        script.write("--strand " + strand + " \\\n")
        script.write("&> " + scriptName + ".log")

if (args.submitJobsToQueue.lower() == "yes") | (args.submitJobsToQueue.lower() == "y"):
    subprocess.call("submitJobs.py", shell=True)


import sys
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_validate
from sklearn.ensemble import ExtraTreesRegressor
from  RegressionMetrics  import RegressorMetrics

def ExtraTreesReg(data, labels, nof ,file , filename):
    params = {
        'criterion' : ['mse', 'mae'] ,
        'max_features' : ['auto', 'sqrt', 'log2' ],
        'n_estimators' : [10,20,50,100,200],
        'max_depth' : [3,5,10,20,50,100,None]
    }
    clf = GridSearchCV(ExtraTreesRegressor(), params, n_jobs=-1, cv=nof)
    clf.fit(data, labels)
    best_parameters = clf.best_params_
    mf = best_parameters['max_features']
    ne = best_parameters['n_estimators']
    cr = best_parameters['criterion']
    md = best_parameters['max_depth']

    model = ExtraTreesRegressor( n_estimators= ne , max_features=mf , criterion=cr , max_depth=md)
    r = RegressorMetrics()
    r.set_predictorName("ExtraTreesRegressor")
    r.set_contin(False)
    r.set_DatasetName(filename)
    bestparameters = 'max_features= ' +str(mf) + ' n_estimators= '+str(ne) + ' criterion= '+str(cr) + ' max_depth= '+str(md)
    r.set_best(bestparameters)
    r.PrintbestParameters()

    cv_results = cross_validate(model, data, labels, cv=nof, scoring=r.scorer, return_train_score = True)
    r.Print("ExtraTreesRegressor")
    r.Print("test")
    r.PrintResults("MAE" , cv_results['test_MAE'])
    r.PrintResults("RMAE" , cv_results['test_RMAE'])
    r.PrintResults("MSE" , cv_results['test_MSE'])
    r.PrintResults("RMSE" , cv_results['test_RMSE'])
    r.PrintResults("RS" , cv_results['test_RS'])
    r.Print("train")
    r.PrintResults("MAE" , cv_results['train_MAE'])
    r.PrintResults("RMAE" , cv_results['train_RMAE'])
    r.PrintResults("MSE" , cv_results['train_MSE'])
    r.PrintResults("RMSE" , cv_results['train_RMSE'])
    r.PrintResults("RS" , cv_results['train_RS'])

    # file.write(str(cv_results['test_MAE']))
    # file.write(str(cv_results['test_RMAE']))
    # file.write(str(cv_results['test_MSE']))
    # file.write(str(cv_results['test_RMSE']))
    # file.write(str(cv_results['test_RS']))


import pygame

import math
from random import randint, choice

class Asteroid(pygame.sprite.Sprite):
    def __init__(self):
        super().__init__()

        self.speed = 5

        self.bounced_off = False

        self.x = randint(-200, 1200)
        if 0 <= self.x <= 1000:
            self. y = choice([randint(-200, -50), randint(850, 1000)])
        else:
            self.y = randint(-200, 1000)

        middle_x = 500
        middle_y = 400

        self.radians = math.atan2(self.y - middle_y, self.x - middle_x)
        self.distance = int(math.hypot(middle_x - self.x, middle_y - self.y) / self.speed)
        self.dist_x = math.cos(self.radians) * self.speed
        self.dist_y = math.sin(self.radians) * self.speed

        image_choice = choice(['asteroid_1', 'asteroid_2'])
        self.image = pygame.image.load(f'assets/graphics/asteroids/{image_choice}.png')
        self.rect = self.image.get_rect(center=(self.x, self.y))

    def movement(self):
        self.x -= self.dist_x
        self.y -= self.dist_y
        self.rect.center = (self.x, self.y)

        if self.bounced_off and ((10 > self.x or self.x > 1010) or (self.y < 10 or self.y > 810)):
            self.kill()

    def bounce_off(self):
        if not self.bounced_off:
            self.bounced_off = True
            self.dist_x = -self.dist_x
            self.dist_y = -self.dist_y

    def update(self):
        self.movement()


from faker import Faker
from app.models import Admin, Category, Post, Comment
from app.extensions import db

fake = Faker()

def fake_admin():
    admin = Admin(
        username = 'nblog',
        blog_title = 'nblog',
        blog_sub_title = 'blog blog',
        name = 'Li Ning',
        about = '个人主页'
    )
    admin.set_password('20201111')
    db.session.add(admin)
    db.session.commit()

def faker_categories(count=10):
    category = Category(name='Default')
    db.session.add(category)

    for i in range(count):
        category = Category(name=fake_word())
        db.session.add(category)
        try:
            db.session.commit()
        except IntegrityError:
            db.session.rollback()

def fake_posts(count=50):
    for i in range(count):
        post = Post(
            title = fake.sentense(),
            body = fake.text(2000),
            timestamp = fake.date_time_this_year(),
            category = Category.query.get(random.randint(1, Category.query.count()))
        )
        db.session.add(post)
    db.session.commit()

def fake_comments(count=500):
    for i in range(count):
        comment = Comment(
            author = fake.name(),
            email = fake.email(),
            site = fake.url(),
            body = fake.sentense(),
            timestamp = fake.date_time_this_year(),
            reviewed = True,
            post = Post.query.get(random.randint(1, Post.query.count()))
        )
        db.session.add(comment)

    salt = int(count*0.1)
    for i in range(salt):
        comment = Comment(
            author = fake.name(),
            email = fake.email(),
            site = fake.url(),
            body = fake.sentense(),
            timestamp = fake.date_time_this_year(),
            reviewed = False,
            post = Post.query.get(random.randint(1, Post.query.count()))
        )
        db.session.add(comment)

        comment = Comment(
            author = 'nonm',
            email = 'nonmcw@126.com',
            site = 'localhost',
            body = fake.sentense(),
            timestamp = fake.date_time_this_year(),
            from_admin = True,
            reviewed = True,
            post = Post.query.get(random.randint(1, Post.query.count()))
        )
        db.session.add(comment)
    db.session.comment()

    for i in range(salt):
        comment = Comment(
            author = fake.name(),
            email = fake.email(),
            site = fake.url(),
            body = fake.sentense(),
            timestamp = fake.date_time_this_year(),
            reviewed = False,
            replied = Comment.query.get(random.randint(1, Comment.query.count())),
            post = Post.query.get(random.randint(1, Post.query.count()))
        )
        db.session.add(comment)
    db.session.commit()


# conda install -n py36 python=3.6 openpyxl pandas selenium requests pyinstaller
# pyinstaller -F credamo_download_gui20230224.py --hidden-import "openpyxl.cell._writer"

import time
import openpyxl
import pandas as pd
import tkinter as tk
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import requests
import json

def open_credamo(browser):
    browser.get("https://www.credamo.com")
    # 检测登录
    login = False
    for i in range(5):
        try:
            # show_execute_info("请登录Credamo账号。")
            browser.find_element_by_class_name("login").click()
            time.sleep(1)
            for cookie in browser.get_cookies():
                if cookie['name']=='credamo-dms-auth':
                    login=True
                    break
            # print("未检测到登录，请登录Credamo账号。")
        except:
            time.sleep(1)
            pass
            # print("已登陆。")
    if login==True:
        show_execute_info("已登陆，请在浏览器中手动打开您需要拒绝被试的项目。")
    else:
        show_execute_info("请登录账号，然后打开一个项目。")
    show_execute_info("准备好excel后点击自动拒绝被试")

def get_surveyId(browser):
    url = browser.current_url
    try:
        # 获得surveyId
        startloc = url.index("surveyId")+len("surveyId=")
        endloc = url.index("#")
        surveyId = url[url.index("surveyId")+len("surveyId="):url.index("#")]
    except:
        # print("还没有打开项目,读取不到surveyId,请打开一个项目")
        show_execute_info("还没有打开项目,读取不到surveyId,请打开一个项目")
        return 0

    #检测是否打开数据清理页面，如果没有则自动点击清理
    try:
        browser.find_element_by_class_name("iconfont icon-shaixuan")
        # 如果找到，表明已经打开数据清理页面
        # print("已打开数据清理页面")
        show_execute_info("已打开数据清理页面")
    except:
    # 如果报错，没有打开数据清理页面，自动点击“数据清理”
        try:
            browser.find_element_by_class_name("iconfont icon-shaixuan")
        except:
            browser.find_element_by_link_text("数据清理").click()
            time.sleep(3)

    return int(surveyId)

def set_page_size(browser,page_size):
    page_sizes = [5,10,30,50,100]
    try:
        size_choise = page_sizes.index(page_size)
    except:
        # print("没有该条数/页的选项，请检查")
        show_execute_info("没有该条数/页的选项，请检查")
        return -2
    # 设置100页
    set_size = False
    retry_time = 0
    while (not set_size) and retry_time <= 5:
        try:
            browser.find_element_by_class_name('el-input__inner').click()
            time.sleep(1)
            pageSizes = browser.find_elements_by_class_name('el-select-dropdown__item')
            pageSizes[size_choise].click()
            page_size = page_sizes[size_choise]
            set_size = True
            return page_size
        except:
            # print("设置条数/页失败，正在重试")
            show_execute_info("设置条数/页失败，正在重试")
            retry_time += 1
    # print("设置页码失败，请检查网页。")
    show_execute_info("设置页码失败，请检查网页。")

def go_to_page(browser,page):
    try:
        loc = browser.find_element_by_class_name('el-pagination__jump').find_element_by_class_name('el-input__inner')
        loc.send_keys(Keys.CONTROL+'a')     #全选
        loc.send_keys(Keys.DELETE)		# 删除，清空
        loc.send_keys(page)	# 写入新的值
        loc.send_keys(Keys.ENTER)
        return page
    except:
        # print("无法跳转页面")
        show_execute_info("无法跳转页面")
        return 0

def set_encoding(browser):
    time.sleep(1)
    browser.find_element_by_link_text("数据清理").click()
    time.sleep(1)
    browser.find_element_by_xpath("/html/body/div[@id='survey_bg']/div[4]/div[1]/div[@class='route']/div[@class='button2']").click()
    time.sleep(1)
    browser.find_element_by_xpath("/html/body/div[@id='survey_bg']/div[4]/div[@class='resultView']/div[@class='search-box']/div[@class='popover'][4]").click()
    time.sleep(1)
    browser.find_element_by_xpath("/html/body/div[@id='survey_bg']/div[4]/div[1]/div[@class='route']/div[@class='button1']").click()
    time.sleep(1)

def get_page_df(browser,surveyId,page_size,page_num,download=False):
    try:
        time.sleep(1)
        cookies = browser.get_cookies()
        cookie_str = ""
        for cookie in cookies:
            cookie_str = "{}={}; ".format(cookie['name'],cookie['value']) + cookie_str
        cookie_str = cookie_str[:-2]

        headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
                    "referer": "https://www.credamo.com/survey.html?surveyId=%d}" % surveyId,
                    "cookie":cookie_str}
        url = "https://www.credamo.com/v1/cleanVar/qstOverviewBySurId/%d?currPageSize=%d&currPageIndex=%d"  % (surveyId,page_size,page_num)
    except:
        # print("获取cookie失败，请重试。")
        show_execute_info("获取cookie失败，请重试。")
        return []
    try:
        resp = requests.get(url=url,headers=headers)
        if resp.status_code == 200:
            output = json.loads(resp.content.decode('utf8'))
            data = output['data']
            ques_headers = data['header']
            total_cnt = output['total']
            ques_headers_kv = {}
            for header in ques_headers:
                ques_headers_kv[header["id"]] = header['qNum']+" "+header['questionName']
            df = pd.DataFrame(output['data']['rowList'])
            #如果下载，则不替换列名称，最后再替换
            if not download:
                df.columns = [ques_headers_kv[id] if id not in ['status', 'answerId', 'userId'] else id for id in df.columns]
                return df,total_cnt
            else:
                return df,total_cnt,ques_headers_kv
        else:
            # print("无法登录到问卷页面")
            show_execute_info("无法登录到问卷页面")
            return pd.DataFrame([]),0
    except:
        # print("无法get问卷页面")
        show_execute_info("无法get问卷页面")
        return pd.DataFrame([]),0

def download_data(browser,download_name):
    surveyId = get_surveyId(browser)
    set_encoding(browser)
    page_size = set_page_size(browser,page_size=100)
    page_num = 1
    page_num = go_to_page(browser,page_num)
    page_df,total_cnt,ques_headers_kv = get_page_df(browser,surveyId,page_size,page_num,download=True)
    total_page_num = total_cnt//page_size+1
    page_dfs = pd.DataFrame([])
    for page_num in range(total_page_num):
        page_num += 1
        # print(page_num)
        # go_to_page(browser,page_num)
        # time.sleep(5)
        page_df,total_cnt,ques_headers_kv = get_page_df(browser,surveyId,page_size,page_num,download=True)
        page_df.set_index('userId')
        if str(page_dfs.shape)=="(0, 0)":
            page_dfs = page_df
        else:
            page_dfs = pd.concat([page_dfs,page_df],ignore_index=True)
    page_dfs.columns = [ques_headers_kv[id] if id not in ['status', 'answerId', 'userId'] else id for id in page_df.columns]
    page_dfs = page_dfs[(page_dfs=="*").sum(axis=1)==0] #剔除拒绝的
    download_name_str = download_name.get('0.0','end').strip()+"_"+time.strftime('%Y%m%d_%H%M')+".xlsx"
    page_dfs.to_excel(download_name_str)
    show_execute_info("{}下载完成".format(download_name_str))

def get_userId_loc(excel_name,page_df,page_num,autobatchReject):
    # excel_name_str = excel_name.get("1.0","end").strip()
    # file = pd.read_excel(excel_name_str)

    # print(excel_name_str)

    # try:
        # file = pd.read_excel(excel_name_str)
        # print(file)
    # except:
    #     # print('没有excel文件,请检查exe所在文件夹下是否有"待拒绝被试.xlsx"文件。\n\n')
    #     show_execute_info('没有excel文件,请检查exe所在文件夹下是否有"{}"文件。\n\n'.format(excel_name_str))
    #     return []
    # chosen_ids = file['用户ID'].tolist()

    chosen_ids = excel_name.get("1.0","end").strip().split('\n')
    print(chosen_ids)

    page_userIds = page_df['userId'].tolist()
    page_chosen_ids = list(set(chosen_ids) & set(page_userIds))
    chosen_rows = []
    for chosen_id in page_chosen_ids:
        chosen_row = page_userIds.index(chosen_id)+1
        chosen_rows.append(chosen_row)
        browser.find_element_by_xpath("/html/body/div[1]/div[4]/div[2]/div[3]/div/div[3]/div/div[2]/table/tbody/tr[%d]" % chosen_row)\
            .find_element_by_class_name('vxe-cell--checkbox').click()
    # 批量拒绝
    browser.find_element_by_class_name("el-dropdown").click()
    time.sleep(1)
    if autobatchReject == True:
        browser.find_elements_by_class_name("el-dropdown-menu__item")[1].click()
        time.sleep(1)
        msgbox = browser.find_element_by_class_name("el-message-box")
        msgbox.click()
        # print("请根据网页提示进行操作")
        time.sleep(3)
        msgbox.find_element_by_class_name("el-button").click()
        show_execute_info("第%d页已拒绝%d个被试" % (page_num,len(chosen_rows)))
        time.sleep(3)
    return chosen_rows

def batchReject(browser,excel_name,page_num_assign_text,autoReject=True):
    surveyId = get_surveyId(browser)
    page_size = set_page_size(browser,page_size=100)
    page_num = 1
    page_num = go_to_page(browser,page_num)
    page_df,total_cnt = get_page_df(browser,surveyId,page_size,page_num)
    total_page_num = total_cnt//page_size+1
    if autoReject:
        for page_num in range(total_page_num):
            page_num += 1
            # print(page_num)
            go_to_page(browser,page_num)
            time.sleep(5)
            page_df,total_cnt = get_page_df(browser,surveyId,page_size,page_num)
            # print(total_cnt)
            userId_loc = get_userId_loc(excel_name,page_df,page_num,autoReject)
            # print(userId_loc)
        show_execute_info('所有页面已拒绝。\n你可以打开另一个项目，清空原用户ID后粘贴新项目待拒绝用户ID，重新点击拒绝被试。\n或者关闭浏览器，关闭本程序')
    else:
        #手动拒绝
        try:
            page_num_assign = int(page_num_assign_text.get('0.0','end').strip())
        except:
            show_execute_info("无法读取页面数，请在上方小方框中输入正确的页面数字")
            return 0
        if page_num_assign <= 0 or page_num_assign > total_cnt//page_size+1:
            show_execute_info("输入的页面数字超过答卷范围，请检查页面数")
        go_to_page(browser,page_num_assign)
        time.sleep(5)
        page_df,total_cnt = get_page_df(browser,surveyId,page_size,page_num_assign)
        # print(total_cnt)
        userId_loc = get_userId_loc(excel_name,page_df,page_num_assign,autoReject)
        # print(userId_loc)
        show_execute_info('页面%d已选中待拒绝被试，可以手动在浏览器中点击拒绝。\n'%page_num_assign)

def show_execute_info(var,cmd=False):
    if cmd:
        print(var)
    else:
        execute_info.insert("0.0", '\n' + time.strftime('%Y-%m-%d %H:%M:%S ') + var + '\n')

window = tk.Tk()
window.title("批量拒绝被试")
window.geometry("600x450")

login_Frame = tk.Frame(window)
login_Frame.pack()

open_button = tk.Button(login_Frame, text='打开Credamo网站', width=15, height=1,
                        command=lambda:open_credamo(browser))
open_button.pack(side="left")

blank1 = tk.Label(login_Frame, text="  ",
                                  width=3)
blank1.pack(side="left")

download_button = tk.Button(login_Frame, text='下载数据', width=15, height=1,
                        command=lambda:download_data(browser,download_name))
download_button.pack(side="left")

download_name = tk.Text(login_Frame, height=1, width=20)
download_name.insert("0.0", 'credamo_data')
download_name.pack(side="left")

download_format = tk.Label(login_Frame, text="_YYYYmmdd_HHMMSS.xlsx",
                                  width=23)
download_format.pack(side="left")

#拒绝被试list
excel_instruc = tk.Label(window, text="请在下面粘贴待拒绝被试的用户ID",
                                width=30, height=2)
excel_instruc.pack()

excel_name = tk.Text(window, height=10)
excel_name.pack()

reject_Frame = tk.Frame(window)
reject_Frame.pack()

reject_button = tk.Button(reject_Frame, text='自动拒绝被试', width=10, height=1,
                        command=lambda:batchReject(browser,excel_name,page_num_assign_text,True))
reject_button.pack(side="left")

blank1 = tk.Label(reject_Frame, text="  ",
                                  width=3)
blank1.pack(side="left")

choose_button = tk.Button(reject_Frame, text='手动拒绝被试', width=10, height=1,
                            command=lambda:batchReject(browser,excel_name,page_num_assign_text,False))
choose_button.pack(side="left")

page_instruc1 = tk.Label(reject_Frame, text="第",
                                width=2)
page_instruc1.pack(side="left")

page_num_assign_text = tk.Text(reject_Frame, height=1, width=4)
page_num_assign_text.insert("0.0", '1')
page_num_assign_text.pack(side="left")

page_instruc2 = tk.Label(reject_Frame, text="页",
                                width=2)
page_instruc2.pack(side="left")

blank_instruc = tk.Label(window, text="----下面是提示----",
                                # bg="green", font=('Arial',12),
                                width=30, height=2)
blank_instruc.pack()

execute_info = tk.Text(window, height=10)
first_info = '一共三个步骤，登录Credamo后打开一个项目，粘贴待拒绝用户ID，最后拒绝被试。\n请在点击第一个按钮登录Credamo，打开您的项目'
show_execute_info(first_info)
execute_info.pack()

browser = webdriver.Chrome()

window.mainloop()


import pika
import os
import sys
import uuid
import json
import logging
import flask
import time
import requests
from functools import partial
from multiprocessing import Value

from lithops.version import __version__
from lithops.utils import setup_lithops_logger, b64str_to_dict
from lithops.worker import function_handler
from lithops.worker.utils import get_runtime_metadata
from lithops.constants import JOBS_PREFIX
from lithops.storage.storage import InternalStorage

from lithops.serverless.backends.k8s import config

logger = logging.getLogger('lithops.worker')

proxy = flask.Flask(__name__)

JOB_INDEXES = {}

@proxy.route('/get-range/<jobkey>/<total_calls>/<chunksize>', methods=['GET'])
def get_range(jobkey, total_calls, chunksize):
    global JOB_INDEXES

    range_start = 0 if jobkey not in JOB_INDEXES else JOB_INDEXES[jobkey]
    range_end = min(range_start + int(chunksize), int(total_calls))
    JOB_INDEXES[jobkey] = range_end

    range = "-1" if range_start == int(total_calls) else f'{range_start}-{range_end}'
    remote_host = flask.request.remote_addr
    proxy.logger.info(f'Sending range "{range}" to Host {remote_host}')

    return range

def run_master_server():
    # Start Redis Server in the background
    # logger.info("Starting redis server in Master Pod")
    # os.system("redis-server --bind 0.0.0.0 --daemonize yes")
    # logger.info("Redis server started")

    proxy.logger.setLevel(logging.DEBUG)
    proxy.run(debug=True, host='0.0.0.0', port=config.MASTER_PORT, use_reloader=False)

def extract_runtime_meta(payload):
    logger.info(f"Lithops v{__version__} - Generating metadata")

    runtime_meta = get_runtime_metadata()

    internal_storage = InternalStorage(payload)
    status_key = '/'.join([JOBS_PREFIX, payload['runtime_name'] + '.meta'])
    logger.info(f"Runtime metadata key {status_key}")
    dmpd_response_status = json.dumps(runtime_meta)
    internal_storage.put_data(status_key, dmpd_response_status)

def run_job_k8s(payload):
    logger.info(f"Lithops v{__version__} - Starting kubernetes execution")

    os.environ['__LITHOPS_ACTIVATION_ID'] = str(uuid.uuid4()).replace('-', '')[:12]
    os.environ['__LITHOPS_BACKEND'] = 'k8s'

    total_calls = payload['total_calls']
    job_key = payload['job_key']
    chunksize = payload['chunksize']

    call_ids = payload['call_ids']
    data_byte_ranges = payload['data_byte_ranges']

    master_ip = os.environ['MASTER_POD_IP']

    job_finished = False
    while not job_finished:
        call_ids_range = None

        while call_ids_range is None:
            try:
                server = f'http://{master_ip}:{config.MASTER_PORT}'
                url = f'{server}/get-range/{job_key}/{total_calls}/{chunksize}'
                res = requests.get(url, timeout=0.1)
                call_ids_range = res.text  # for example: 0-5
            except Exception:
                time.sleep(0.1)

        logger.info(f"Received range: {call_ids_range}")
        if call_ids_range == "-1":
            job_finished = True
            continue

        start, end = map(int, call_ids_range.split('-'))
        dbr = [data_byte_ranges[int(call_id)] for call_id in call_ids[start:end]]
        payload['call_ids'] = call_ids[start:end]
        payload['data_byte_ranges'] = dbr
        function_handler(payload)

    logger.info("Finishing kubernetes execution")

def run_job_k8s_rabbitmq(payload, running_jobs):
    logger.info(f"Lithops v{__version__} - Starting kubernetes execution")

    act_id = str(uuid.uuid4()).replace('-', '')[:12]
    os.environ['__LITHOPS_ACTIVATION_ID'] = act_id
    os.environ['__LITHOPS_BACKEND'] = 'k8s_rabbitmq'

    function_handler(payload)
    running_jobs.value += len(payload['call_ids'])

    logger.info("Finishing kubernetes execution")

# Callback to receive the payload and run the jobs
def callback_work_queue(ch, method, properties, body):
    global cpus_pod

    logger.info(f"Call from lithops received.")

    message = json.loads(body)
    tasks = message['total_calls']

    running_jobs = Value('i', cpus_pod)  # Shared variable to track completed jobs

    # If there are more tasks than cpus in the pod, we need to send a new message
    if tasks <= running_jobs.value:
        processes_to_start = tasks
    else:
        processes_to_start = running_jobs.value

        message_to_send = message.copy()
        message_to_send['total_calls'] = tasks - running_jobs.value
        message_to_send['call_ids']  = message_to_send['call_ids'][running_jobs.value:]
        message_to_send['data_byte_ranges'] = message_to_send['data_byte_ranges'][running_jobs.value:]

        message['total_calls'] = running_jobs.value
        message['call_ids'] = message['call_ids'][:running_jobs.value]
        message['data_byte_ranges'] = message['data_byte_ranges'][:running_jobs.value]

        ch.basic_publish(
            exchange='',
            routing_key='task_queue',
            body=json.dumps(message_to_send),
            properties=pika.BasicProperties(
                delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE
            ))

    logger.info(f"Starting {processes_to_start} processes")

    message['worker_processes'] = running_jobs.value
    running_jobs.value -= processes_to_start
    run_job_k8s_rabbitmq(message, running_jobs)

    logger.info(f"All processes completed")
    ch.basic_ack(delivery_tag=method.delivery_tag)

def start_rabbitmq_listening(payload):
    global cpus_pod

    # Connect to rabbitmq
    params = pika.URLParameters(payload['amqp_url'])
    connection = pika.BlockingConnection(params)
    channel = connection.channel()
    channel.queue_declare(queue='task_queue', durable=True)
    channel.basic_qos(prefetch_count=1)

    # Get the number of cpus of the pod
    cpus_pod = payload['cpus_pod']

    # Start listening to the new job
    channel.basic_consume(queue='task_queue', on_message_callback=callback_work_queue)

    logger.info(f"Listening to rabbitmq...")
    channel.start_consuming()

if __name__ == '__main__':
    action = sys.argv[1]
    encoded_payload = sys.argv[2]

    payload = b64str_to_dict(encoded_payload)
    setup_lithops_logger(payload.get('log_level', 'INFO'))

    switcher = {
        'get_metadata': partial(extract_runtime_meta, payload),
        'run_job': partial(run_job_k8s, payload),
        'run_master': run_master_server,
        'start_rabbitmq': partial(start_rabbitmq_listening, payload)
    }

    func = switcher.get(action, lambda: "Invalid command")
    func()


"""
NLP system for extracting key words/phrases from text. The system tokenizes
strings, classifies each token and finds the keyphrases based on the
classification tags.
"""
import nltk

import itertools
import re

# Tokenization regexp from the NLTK Book
pattern = r"""(?x)
              (?:[A-Z]\.)+           # abbreviations, e.g. U.S.A.
              |\d+(?:\.\d+)?%?       # numbers, incl. currency and percentages
              |\w+(?:[-']\w+)*       # words w/ optional internal hyphens/apostrophe
              |(?:[+/\-@&*])         # special characters with meanings
"""

pos_pattern = r"""
    KP: {(<JJ>* <NN.*>+ <IN>)? <JJ>* <NN.*>+}
"""

def get_stopwords(filename):
    """
    Retrieves a set of stopwords for word cloud filtering.
    """
    words = set()
    with open(filename, 'r') as f:
        words.update( f.read().splitlines() )
    f.close()
    return words

stopwords = get_stopwords("stopwords.txt")
tokenizer = nltk.tokenize.RegexpTokenizer(pattern)
wnl = nltk.WordNetLemmatizer()

def normalise(word):
    word = word.lower()
    word = wnl.lemmatize(word)
    return word

def good_phrase(phrase):
    if re.sub('[^A-Za-z0-9]+', '', phrase) in stopwords:
        return False
    if "/" in phrase or "*" in phrase:
        return False
    return True

def extract_chunks(text):
    """
    Extract possible keyphrases from a text string.
    Returns a list of candidate keyphrases and the count of tokens created overall.
    """
    chunker = nltk.RegexpParser(pos_pattern)

    tokens = tokenizer.tokenize(text)
    count = len(tokens)
    pos_tokens = nltk.pos_tag(tokens)

    all_chunks = chunker.parse(pos_tokens)

    # Get key phrases from all chunks
    kp_chunks = (subtree.leaves() for subtree in
                 all_chunks.subtrees(filter = lambda t: t.label() == "KP"))

    # Join the words of key phrases, normalising in the process
    kp_candidates = (" ".join(normalise(word) for word, tag in chunk)
                     for chunk in kp_chunks)

    candidates = [phrase for phrase in kp_candidates if good_phrase(phrase)]

    return candidates, count

def rank_keyphrases(texts, top_n=50):
    """
    Rank keyphrases with a simple frequency distribution.

    This approach favours unigrams, which is fine since unigrams are better suited
    for a word cloud.
    Args:
        texts - an iterable of strings
        top_n - number of keyphrases returned. If the amount of text is insufficient,
                less than top_n keyphrases might be returned.
    Returns:
    The keyphrases and a total count of words tokenized.
    """
    kp_lists = [extract_chunks(text) for text in texts]
    words = list(itertools.chain.from_iterable(l for l,_ in kp_lists))
    fd = nltk.FreqDist(words)

    # Count the total number of tokens created using keyphrase extraction
    word_count = sum(c for _,c in kp_lists)

    # If there are few keyphrases, take top 20% instead of the specified top_n
    n = min(top_n, int(fd.B() / 5))

    keyphrases = fd.most_common(n)
    # Return the rank as a frequency among keyphrase candidates
    return keyphrases, word_count


import itertools

from django import forms
from django.utils.translation import gettext

from .conf import get_available_languages, get_default_language
from .fields import get_instance_field_value
from .translator import get_i18n_field
from .utils import build_localized_fieldname, get_language

LANGUAGE_OPTIONS = ["browser", "fallback"]

class TranslationModelFormOptions(forms.models.ModelFormOptions):
    """Add the translation form options to the Meta options."""

    def __init__(self, options=None):
        super().__init__(options)
        self.languages = getattr(options, "languages", ["browser", "fallback"])
        self.fallback_language = getattr(options, "fallback_language", None)

class TranslationModelFormMetaClass(forms.models.ModelFormMetaclass):
    def __new__(mcs, name, bases, attrs):
        """
        Include all translation fields for translatable fields declared in the form.

        We use the standard ModelForm field declaration procedure of the ModelFormMetaClass,
        so that all fields are declared during creation of the new object.

        Different actions are required for different Meta class options:
        1) fields - if fields are declared explicitly with the fields option, the related i18n fields will not be
        included, and these have to be added to base_fields.
        2) exclude - if fields are declared implicitly with the exclude option, the i18n translation fields, as well as
        the i18n field itself, will be included. We will remove the i18n field.

        Here all languages are included, so that in the form we can curate which fields are available according to
        settings of the form instance (e.g. kwarg overrides, or model instance fallbacks).
        """
        new_class = super().__new__(mcs, name, bases, attrs)

        # from ModelFormMetaClass, needed for calling fields for model
        base_formfield_callback = None
        for b in bases:
            if hasattr(b, "Meta") and hasattr(b.Meta, "formfield_callback"):
                base_formfield_callback = b.Meta.formfield_callback
                break
        formfield_callback = attrs.pop("formfield_callback", base_formfield_callback)

        # get options, with extra translation form options
        opts = new_class._meta = TranslationModelFormOptions(getattr(new_class, "Meta", None))
        model_class = opts.model

        languages = get_available_languages()

        # Note that base_fields generated by ModelForm do not yet include translation fields if generated with "fields"
        # Meta option, but includes all translation fields if generated with "excludes" Meta option.
        base_fields = list(new_class.base_fields.keys())

        # convert to list in case it is a tuple
        opts_fields = list(opts.fields) if opts.fields else None
        opts_exclude = list(opts.exclude) if opts.exclude else None

        if model_class:
            i18n_field = get_i18n_field(model_class)
            if i18n_field:
                for original_field_name in i18n_field.fields:  # for all translated fields
                    # for all possible system languages
                    for language in languages:
                        field_name = build_localized_fieldname(
                            original_field_name, language, ignore_default=True
                        )

                        # add i18n field if an explicitly chosen field
                        if (
                            opts.fields
                            and original_field_name in base_fields
                            and field_name not in base_fields
                        ):
                            base_fields.append(field_name)
                            opts_fields.append(field_name)

                        # remove field if an explicitly excluded field
                        if (
                            opts.exclude
                            and original_field_name in opts.exclude
                            and field_name in base_fields
                        ):
                            base_fields.remove(field_name)
                            opts_exclude.append(field_name)

                    # Remove the i18n field if present (e.g. because of using the exclude option)
                    name = f"{original_field_name}_i18n"
                    if name in base_fields:
                        base_fields.remove(name)
                    if opts.exclude:
                        opts_exclude.append(name)

                    # Remove the i18n field for the system default language, because that already exists as the default
                    name = f"{original_field_name}_{get_default_language()}"
                    if name in base_fields:
                        base_fields.remove(name)
                    if opts.fields and name in opts.fields:
                        opts_fields.remove(name)
                    if opts.exclude and name not in opts.exclude:
                        opts_exclude.append(name)

                opts.fields = opts_fields
                opts.exclude = opts_exclude

                base_fields = forms.fields_for_model(
                    opts.model,
                    opts.fields,
                    opts.exclude,
                    opts.widgets,
                    formfield_callback,
                    opts.localized_fields,
                    opts.labels,
                    opts.help_texts,
                    opts.error_messages,
                    opts.field_classes,
                    # limit_choices_to will be applied during ModelForm.__init__().
                    apply_limit_choices_to=False,
                )

                # Override default model fields with any custom declared ones
                # (plus, include all the other declared fields).
                base_fields.update(new_class.declared_fields)
            else:
                base_fields = new_class.declared_fields

            # Override base_fields with properly determined set of all translation fields based on ModelForm
            new_class.base_fields = base_fields

        return new_class

class TranslationModelForm(forms.ModelForm, metaclass=TranslationModelFormMetaClass):
    """
    Model form that only adds translation fields for specified languages.

    Meta options and form parameters include:
    - languages: a list defining languages for which translation fields are added.
        - Options are:
            - "browser": current browser language
            - "fallback": the current fallback language, which is the system fallback,
                          or a customized fallback of the translation field.
            - "fr": a language code
        - Overlap is removed, e.g. ["browser", "fr", "fallback"], becomes ["fr"] if all are equal.
        - The list order determines the order of fields in the form.
        - included languages can also be passed via form kwargs to customize on the fly, useful for
          generating translation forms for a specific language.
    - fallback_language: for adapting the fallback language specification on the fly and override the Meta option and/or
    model translation field custom fallback.

    Form parameters take priority of Meta options.

    For the fallback language the following priority holds:
    1) fallback_language passed as form parameter: Form(fallback_language="fr")
    2) the Meta option "fallback_language":
        e.g. Meta:
                fallback_language = "fr"
    3) a custom fallback of a model instance set via "fallback_language_field":
        e.g. i18n = TranslationField(fields=("title", "header"), fallback_language_field="language_code")
    4) The default language of the system. If not Meta option is given fallback reverts to get_default_language()

    Code example:
        class ChallengeModel(models.Model):
            language_code = CharField()
            title = CharField()
            description = CharField()
            i18n = TranslationField(fields=("title", "description"), fallback_language_field="language_code")

        class ChallengeModelForm(RefactoredModelTransForm):

            class Meta:
                fields = ["title", "description"] or exclude = ["language_code"]
                languages = ["browser", "es", "fallback"]

        class ChallengeCreateUpdateView(GenericCreateUpdateView):
            '''An update view with a browser language, spanish and browser field for translations.'''
            model = Challenge
            form_class = ChallengeModelForm()

        class ChallengeTranslationView(GenericCreateUpdateView):
            '''
            A translation view with a specific translation language field and non-editable fallback field as reference.
            '''
            model = Challenge
            form_class = ChallengeModelForm(
                languages=[translation_language_code, "fallback"],
            )
    """

    def __init__(self, *args, languages=None, fallback_language=None, **kwargs):
        """Prune the translation fields based on included languages and fallback_language."""

        self.model_i18n_field = get_i18n_field(self._meta.model)
        self.languages = languages or self._meta.languages
        self.i18n_fields = [
            field for field in self.model_i18n_field.fields if field in self.base_fields.keys()
        ]

        # Given that we set opt.fields and opts.exclude in Meta,
        # the setting of initial values that occurs in ModelForm includes
        # any additional translation fields that have been added in Meta.
        super().__init__(*args, **kwargs)

        # the following require the instance generated in the super call
        self.fallback_language = self.get_fallback_language(fallback_language)
        self.language_codes = self.get_language_codes()
        self.included_fields = self.get_included_fields()

        self.remove_excess_fields()
        self.set_translation_field_attributes()
        self.order_translation_fields()

    def get_included_fields(self):
        """
        Return a dictionary mapping original field names to a list of included field names,
        which may or may not include the original field name.
        """

        fields = {}
        for original_field in self.i18n_fields:
            fields[original_field] = [
                build_localized_fieldname(original_field, language_code, ignore_default=True)
                for language_code in self.language_codes
            ]
        fields["__all__"] = list(itertools.chain.from_iterable(fields.values()))
        return fields

    def set_translation_field_attributes(self):
        """Apply settings of all original field to relevant translation fields."""

        for original_field_name in self.i18n_fields:
            original_field = self.base_fields[original_field_name]
            for field_name in self.included_fields[original_field_name]:
                language = get_default_language()
                if field_name != original_field_name:
                    language = field_name.replace(f"{original_field_name}_", "")
                is_translation = language != self.fallback_language
                label_text = (
                    gettext("translation language")
                    if is_translation
                    else gettext("default language")
                )
                label = f"{original_field.label} ({language.upper()}, {label_text})"
                self.fields[field_name].label = label
                self.fields[field_name].required = (
                    False if is_translation else original_field.required
                )
                self.fields[field_name].widget = original_field.widget

    def order_translation_fields(self):
        """
        Set the order of the fields, ideally replacing the original field with the set of fields in included fields.

        For the Meta.excludes use a cruder ordering where translated fields types are grouped,
        but per type the languages are in order of: browser, other, fallback
        """

        # Form parameter field_order takes priority, otherwise adopt order of fields in Meta fields option, if available
        field_order = self.field_order or None
        if not field_order and self._meta.fields and self._meta.fields != "all":
            field_order = self._meta.fields

        # in case of an explicit field order replace original field with set of included fields
        if field_order:
            new_field_order = list(field_order)
            for original_field in self.i18n_fields:
                if original_field in new_field_order:
                    index = new_field_order.index(original_field) + 1
                    new_field_order[index:index] = self.included_fields[original_field]
                    new_field_order.pop(index - 1)
            field_order = new_field_order
        else:
            # if no explicit field order adopt the order of i18n form fields used to generate the included fields
            field_order = self.included_fields["__all__"]

        self.order_fields(field_order)

    def remove_excess_fields(self):
        """Remove translations fields that are not included in languages."""
        fields = self.fields.copy()
        for original_field_name in self.i18n_fields:
            for field in fields:
                if original_field_name in field and field not in self.included_fields["__all__"]:
                    self.fields.pop(field)

    def get_fallback_language(self, fallback_language=None):
        """
        Get the fallback language.

        Priority is defined as:
        1) key word argument
        2) form Meta option
        3) model instance custom fallback
        4) system default fallback
        """
        if fallback_language:
            return fallback_language

        if self._meta.fallback_language:
            return self._meta.fallback_language

        if self.model_i18n_field.fallback_language_field and self.instance.pk:
            return get_instance_field_value(
                self.instance, self.model_i18n_field.fallback_language_field
            )

        return get_default_language()

    def get_language_codes(self):
        """Get the languages based on options included in include languages, in the order they are submitted."""

        valid_languages = get_available_languages()
        languages = []
        if self.languages:
            for value in self.languages:
                if value == "browser":
                    value = get_language()
                elif value == "fallback":
                    value = self.fallback_language
                if value not in valid_languages:
                    raise ValueError(f"languages: value {value} is not permitted")
                languages.append(value)

        if not languages:
            raise ValueError("languages: No languages have been defined.")

        # remove duplicates while preserving the order
        no_repeats = set()
        return [x for x in languages if x not in no_repeats and not no_repeats.add(x)]


import subprocess
import os
import platform

import sys
import re

from typing import Union, Generator

PTN_QUOTED = r"'(.*?)(?<!\\)'|\"(.*?)(?<!\\)\""
PTN_WORD = rf"([-\w\.=:]+({PTN_QUOTED})*)"

unquote = lambda w: re.sub(PTN_QUOTED, r"\1" or r"\2", w)

#git command class
class Git:
    PATH_GITIGNORE  = ".gitignore"

    # Status
    STATUS_UNMODIFIED   = 0b00000
    STATUS_IGNORED      = 0b00001
    STATUS_UNTRACKED    = 0b00010
    STATUS_UNMERGED     = 0b00100
    STATUS_NOTSTAGED    = 0b01000
    STATUS_STAGED       = 0b10000

    def __init__(self, context):
        prefs = context.preferences.addons[__package__].preferences
        path = prefs.git_execpath

        #try to use user supplied path first
        if path:
            self.git_execpath = path
        else: #try to autodetect
            if platform.architecture()[1] == "WindowsPE":
                #assume default git path for windows here, since we dont have winreg access via integrated python
                if platform.architecture()[0] == "64bit":
                    self.git_execpath = "C:/Program Files/Git/bin/git.exe"
                elif platform.architecture()[0] == "32bit":
                    self.git_execpath = "C:/Program Files (x86)/Git/bin/git.exe"
            else: #Linux, Mac
                self.git_execpath = "/usr/bin/git"

        self.operative = os.path.isfile(self.git_execpath)

        gcon = context.window_manager.git_context
        self.chdir(gcon.rootdir)

    # Update workdir file with specific version

    # def backup(self, filename, dirpath, commit_hash):
    def backup(self, blobnr, filepath):
        if blobnr!=None:
            p = subprocess.Popen(
                [self.git_execpath, "cat-file", "blob", blobnr],
                stdout = subprocess.PIPE,
                stderr = subprocess.STDOUT
                )
            blob = p.stdout.read()

            with open(filepath, "wb+") as tmp:
                tmp.write(blob)
        return

    def get_blobs(self, commit_hash):
        blobs = {}
        results = self.command(["ls-tree", "-r", commit_hash])
        for l in results:
            tab = l.split("\t")
            name = tab[1]
            _,blob,blobnr = tab[0].split(" ")

            if blob=="blob":
                blobs[name] = blobnr
        return blobs

    # Miscellaneous

    def write_ignore(self, statement: str):
        with open(self.PATH_GITIGNORE, "a+") as f:
            f.write(statement+"\n")

    def clean_ignore(self):
        ignore_dict = {}
        with open(self.PATH_GITIGNORE, "r") as f:
            # fromkeys : remove overwrapping
            # reversed : statement priority is last written
            lines = list(dict.fromkeys([l.strip() for l in reversed(f.readlines())]))
            # reversed : restore order
            for line in reversed(lines):
                ignore_dict[line.lstrip("!")] = "!" if line.startswith("!") else ""

        with open(self.PATH_GITIGNORE, "w+") as f:
            f.writelines(
                [pattern+file+"\n" for file, pattern in ignore_dict.items()]
                )

    def chdir(self, dirpath) -> 'bool: Is path exist':
        if os.path.isdir(dirpath):
            os.chdir(dirpath)
            return True
        return False

    def open_dir(self, dirpath) -> 'exist: bool':
        if os.path.exists(dirpath):
            if platform.system() == 'Windows':
                subprocess.Popen([
                    'explorer',
                    dirpath.replace("/", "\\")
                    ])
            # elif platform.system() == 'Linux':
            #     print("%s already exists" % dirpath)
            # elif platform.system() == 'Mac':
            #     subprocess.Popen([
            #         'finder',
            #         dirpath.replace("/", "\\")
            #         ])

                # print(dirpath.replace("/", "\\"))
            return True
        else:
            return False

    # generic Git command with [textual feedback]
    # => for interactive, get result as generator

    def command(self, cmd: Union[str, list, tuple]) -> Generator[str, None, bytes]:

        if not self.operative:
            return None

        args = [self.git_execpath]

        if type(cmd) is str:
            # (entire-matched, *other_groups) = findall
            args += [m[0] for m in re.findall(PTN_WORD, cmd)]
        elif type(cmd) in [list, tuple]:
            args += cmd
        else:
            print(f"TypeError: invalid argument type for 'cmd: Union[list, str]': {type(cmd)}", file=sys.stderr)
            return None

        # remove quote-character overwrapping
        args = list(map(unquote, args))
        # print(args)

        p = subprocess.Popen(
            args,
            stdout = subprocess.PIPE,
            stderr = subprocess.STDOUT
            )

        while True:
            line = p.stdout.readline()
            if line:
                try:
                    out = line.decode('utf-8').rstrip('\n')
                    # print(out)
                    yield out
                except UnicodeDecodeError as e:
                    print(e)
                    out = line
                    yield out

            if not line and p.poll() is not None:
                break


from django.urls import path
from .views import *

urlpatterns = [
    path('change-status/',change_leaderboard,name="change"),
    path('competitions/', CompetionList.as_view(),name="competitions"),
    path('competitions/crate/',CreateCompetition.as_view(),name="createcomp"),
    path('competitions/<int:pk>/', SubmitContest.as_view(),name='comp_detail'),
    path('competitions/<int:pk>/results/',get_result_csv,name="get_result"),
    path('uploadsubm/',UploadSubmission.as_view(),name='ajax_upload'),
    path('accounts/signup/',SignUp.as_view(),name='signup'),
    path('accounts/signout/',SignOut.as_view(),name='signout'),
    path('my/settings/',AccountView.as_view(),name='account'),
]


from pprint import pprint
from fastapi import FastAPI, Request
import redis

app = FastAPI()
r = redis.Redis(host='localhost', port=6379, db=0)

@app.get("/")
async def root():
    return {"message": "Hello World"}

@app.post("/")
async def listen(request: Request):
    body = await request.body()
    r.rpush("nglue", body)
    return True


import config

import zmq
import zmq.asyncio

import logging
import json
import asyncio
import traceback

import utils.concurrent
import utils.meta

from typing import Optional

logger = logging.getLogger('SERVER_SERVICE')

PARSE_MESSAGE_OR_SAVE_LOOP_INTERRUPTED = False
SERVER_SOCKET_RECV_LOOP_INTERRUPTED = False

@utils.meta.JSONClass
class ServerMessage:
    def __init__(self, method: str, payload: object = None, request_id: int = None):
        self.method = method
        self.payload = payload
        self.request_id = request_id

class ServerService(utils.concurrent.AsyncZmqActor):

    def __init__(self):
        super(ServerService, self).__init__()
        self.__aiter_inited = False
        self.__request_next_id = 1
        self.__responses = dict()
        self.start()

    async def send_message_to_server(self, message: ServerMessage):
        # Following message will be sent to actor's self._on_message()
        # We do it cuz we created self.__server_socket in self._run() method,
        # which runs in the actor's thread, not the thread we created ServerService

        # in theory, we can try to use zmq.proxy:
        # zmq.proxy(self.__actor_socket, self.__server_socket)
        # and do here something like:
        # self.__actor_socket.send_string(json.dumps(message.to_json()))
        await self._put_message_to_thread(json.dumps(message.to_json()))

    async def send_request_to_server(self, message: ServerMessage) -> object:
        if message.request_id is not None:
            raise ValueError('Message can`t have request_id before it is scheduled')
        request_id = message.request_id = self.__request_next_id
        self.request_next_id = self.__request_next_id + 1
        asyncio.ensure_future(self.send_message_to_server(message))
        # you should await self.__responses[request_id] which should be a task,
        # which you resolve somewhere else
        while request_id not in self.__responses:
            await asyncio.sleep(1)
        response = self.__responses[request_id]
        del self.__responses[request_id]
        return response

    def __aiter__(self):
        if self.__aiter_inited:
            raise RuntimeError('Can`t iterate twice')
        __aiter_inited = True
        return self

    async def __anext__(self) -> ServerMessage:
        while not PARSE_MESSAGE_OR_SAVE_LOOP_INTERRUPTED:
            thread_message = await self._recv_message_from_thread()
            server_message = self.__parse_message_or_save(thread_message)
            if server_message is None:
                continue
            else:
                return server_message

    async def _run_thread(self):
        logger.info("Binding to %s ..." % config.ZMQ_CONNECTION_STRING)
        self.__server_socket = self._zmq_context.socket(zmq.PAIR)
        self.__server_socket.bind(config.ZMQ_CONNECTION_STRING)
        await self.__server_socket_recv_loop()

    async def _on_message_to_thread(self, message: str):
        await self.__server_socket.send_string(message)

    async def __server_socket_recv_loop(self):
        while not SERVER_SOCKET_RECV_LOOP_INTERRUPTED:
            received_string = await self.__server_socket.recv_string()
            if received_string == 'PING':
                asyncio.ensure_future(self.__handle_ping())
            else:
                asyncio.ensure_future(self._send_message_from_thread(received_string))

    async def __handle_ping(self):
        await self.__server_socket.send_string('PONG')

    def __parse_message_or_save(self, text: str) -> Optional[ServerMessage]:
        try:
            message_object = json.loads(text)
            message = ServerMessage.from_json(message_object)
            if message.request_id is not None:
                self.__responses[message_object['requestId']] = message.payload
                return None
            return message
        except Exception:
            error_text = traceback.format_exc()
            logger.error("__handle_message Exception: '%s'" % error_text)


import numpy as np
import pytest

from Metrics.src.image_quality_estimation import IQS
from NLM.src.denoise import (
    nlm,
    nlm_CEST,
    pad_image,
    get_comparison_neighborhood,
    get_small_neighborhood,
)
from test_support_function.src.CEST import generate_Z_3D

def test_pad_image():
    image = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    padded_image = pad_image(image, 2, 1)
    expected_padded_image = np.array(
        [
            [0, 0, 0, 0, 0],
            [0, 1, 2, 3, 0],
            [0, 4, 5, 6, 0],
            [0, 7, 8, 9, 0],
            [0, 0, 0, 0, 0],
        ]
    )
    np.testing.assert_equal(padded_image, expected_padded_image)

def test_get_comparison_neighborhood():
    padded_image = np.array(
        [
            [0, 1, 2, 3, 4],
            [5, 6, 7, 8, 9],
            [9, 10, 11, 12, 13],
            [14, 15, 16, 17, 18],
            [19, 20, 21, 22, 23],
        ]
    )
    comparison_neighborhood = get_comparison_neighborhood(padded_image, 1, 1, 1)
    expected_comparison_neighborhood = np.array([[0, 1, 2], [5, 6, 7], [9, 10, 11]])
    np.testing.assert_equal(comparison_neighborhood, expected_comparison_neighborhood)

def test_get_small_neighborhood():
    padded_image = np.array(
        [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]
    )
    small_neighborhood = get_small_neighborhood(padded_image, 0, 0, 3)
    expected_small_neighborhood = np.array([[1, 2, 3], [5, 6, 7], [9, 10, 11]])
    np.testing.assert_equal(small_neighborhood, expected_small_neighborhood)

def test_nlm():
    image = np.ones((42, 42)) * 125
    noise = image + np.random.randint(0, 5, (42, 42))
    denoise = nlm(noise.copy(), 22, 6)
    iqs = IQS(pixel_max=255, ref_image=image)
    assert iqs.psnr(denoise) > iqs.psnr(noise)

def test_nlm_CEST():
    iqs = IQS(pixel_max=1)
    Z = generate_Z_3D(img_size=(42, 42), dyn=5, ppm=3, a=1, b=-0.5, c=3)
    Z_noisy = Z + 0.2 * np.random.random(Z.shape)

    Z_denoise = nlm_CEST(Z, 22, 6)

    assert Z.shape == Z_denoise.shape
    assert iqs.psnr(Z_denoise[:, :, 0], Z[:, :, 0]) > iqs.psnr(
        Z_noisy[:, :, 0], Z[:, :, 0]
    )

def test_nlm_CEST_ml():
    iqs = IQS(pixel_max=1)
    Z = generate_Z_3D(img_size=(42, 42), dyn=5, ppm=3, a=1, b=-0.5, c=3)
    Z_noisy = Z + 0.2 * np.random.random(Z.shape)

    Z_denoise = nlm_CEST(Z, 22, 6, multi_processing=True)

    assert Z.shape == Z_denoise.shape
    assert iqs.psnr(Z_denoise[:, :, 0], Z[:, :, 0]) > iqs.psnr(
        Z_noisy[:, :, 0], Z[:, :, 0]
    )

if __name__ == "__main__":
    pytest.main()


from collections import deque

n = int(input())
list = []
for i in range(n):
    word = deque(input())
    for j in range(len(word)):
        tmp = ''.join(word)
        if tmp in list:
            break
        word.rotate()
    else:
        list.append(''.join(word))
print(len(list))


import asyncio
import serial
from serial.tools.list_ports import comports
from serial.serialutil import SerialException
from signal import SIGINT, SIGTERM

def initialize_port():
    """
    Initialize the serial port.
    :return:
    """
    plist = comports()
    selected = []
    print("Checking for serial ports....")
    print("Found:")
    for i, p in enumerate(plist):
        print("{}. {}".format(i, p))
        if 'usbserial' in p.device:
            selected.append(p)
    print(selected)
    if len(selected) == 1:
        print("Selecting {}".format(selected[0]))
        return serial.Serial(selected[0].device, 500000)
    else:
        print("Could not find appropriate port.")
        return None

async def serial_task():
    """
    Async task for serial port monitoring.
    :return:
    """
    port = initialize_port()
    max_reconnects = 10
    reconnect_count = 0

    if port:
        while True:
            try:
                sdata = port.read(1024)
                print("Got {} chars of data".format(len(sdata)))
                await asyncio.sleep(0.01)
            except asyncio.CancelledError:
                port.close()
                break
            except:
                print("Serial port disconnected.")
                print("Trying to reconnect...")
                port = initialize_port()

                if port is None:
                    print("Failed to read.")
    else:
        pass

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    stask = loop.create_task(serial_task())
    try:
        loop.run_until_complete(stask)
    except KeyboardInterrupt as e:
        print("Caught keyboard interrupt. Canceling tasks...")
        stask.cancel()
    finally:
        loop.close()


"""
    Note: when testing this do:
    $ pip install --editable .

    But if you make a change:
    $ pip uninstall azul
    $ pip install --editable .

    Also note, if you are in the top azul directory and type "azul" with no other options the shell will take you
    the azul directory instead of running the command line.
"""

import click
import azul
import logbook

@click.group()
def cli():
    """azul is a command line tool for downloading historical price data that can be used in a zipline bundle."""

    # install a logbook handler before performing any other operations
    logbook.StderrHandler(level='NOTICE').push_application()
    global log
    log = logbook.Logger('AzulCli')

@click.command()
@click.option(
    '--symbol-source',
    type=click.STRING,
    metavar="['sp500', 'polygon_cs', 'iex', 'faang']",
    default='faang',
    help='The source to get symbols from.'
)
@click.option(
    '--data-source',
    type=click.STRING,
    metavar="['polygon', 'iex']",
    default='iex',
    help='The source to get data from.'
)
@click.option(
    '-o',
    '--output-dir',
    type=click.Path(file_okay=False),
    default=None,
    metavar='[~/.azul/<data-source>]',
    show_default=False,
    help="The directory write the data too."
)
@click.option(
    '-s',
    '--start',
    type=click.DateTime(formats=['%Y-%m-%d']),
    default=None,
    help='The date to start downloading data from.',
)
@click.option(
    '-e',
    '--end',
    type=click.DateTime(formats=['%Y-%m-%d']),
    default=None,
    help='The date to end downloading data from.',
)
def download(symbol_source, data_source, output_dir, start, end):
    """Download historical price data."""
    try:
        azul.get_price_data(symbol_source, data_source, output_dir, start, end)
    except Exception as e:
        log.error(e)
        raise

@click.command()
def update():
    """Update symbols with any new data."""
    pass

cli.add_command(download)
cli.add_command(update)


import sys
import json
from os import listdir
from os.path import isfile, join
import re
import argparse
import pickle
def run_process(fname,fpath):
    sentences = []
    with open(join(fpath,fname),'r') as fin:
        doc = fin.read()
    for line in doc.split('\n')[:-1]:
        # print(line)
        segs = line.split('\t')
        # print(segs)
        sentence = re.sub(r'(.|,|!) *((I )|(me )|(Me )|(We)|(we))',segs[1],segs[2])
        sentences.append(sentence)
    fout = join('CLEF/test/',f.rsplit('.',1)[0] + '_fp.pkl')
    with open(fout,'wb') as fout:
        pickle.dump(sentences,fout)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Input file/fphath, at least one must be provided.')
    # parser.add_argument('--folder',
    #                help='given folder or not')
    # parser.add_argument('file', type=str, default=None,
    #                     help='path to the file that needs to be processed')
    parser.add_argument('folder', type=str,default=None,
                        help='path to the folder that all the files need to be processed')

    args = parser.parse_args()

    # if args.file != None:
    #     run_process(args.file)

    if args.folder != None:
        files = [f for f in listdir(args.folder) if isfile(join(args.folder, f))]
        for f in files:
            run_process(f,args.folder)


import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from sklearn.model_selection import train_test_split
from torch.optim.lr_scheduler import StepLR
from torch.utils.data import Dataset, DataLoader

device = "cuda" if torch.cuda.is_available() else "cpu"
import multiprocessing as mp

NUM_PROCS = mp.cpu_count()

class SimpleGRUClassifier(nn.Module):
    """GRU without lightning."""

    def __init__(
        self,
        n_classes: int = 2,
        input_size: int = 1000,
        hidden_layer_size: int = 500,
        n_layers: int = 1,
        hidden_size: int = 1,
        batch_size: int = 1,
    ):
        """Instantiate the instance."""
        super().__init__()
        # self.n_classes = len(label_encoder.classes_)
        self.n_classes = n_classes
        self.n_layers = n_layers
        self.batch_size = batch_size
        self.hidden_size = hidden_size
        self.hidden_layer_size = hidden_layer_size
        self.gru = nn.GRU(
            input_size, hidden_layer_size, n_layers, batch_first=True
        )
        self.fc1 = nn.Linear(hidden_layer_size, hidden_layer_size)
        self.fc = nn.Linear(hidden_layer_size, self.n_classes)
        self.relu = nn.ReLU()
        self.criterion = nn.CrossEntropyLoss()

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """Forward computing."""
        hidden = self._init_hidden(self.hidden_layer_size)
        output, hidden = self.gru(x, hidden.view(self.n_layers, -1))
        fc_output = self.relu(self.fc1(output))
        fc_output = self.fc(fc_output)
        return fc_output

    def _init_hidden(self, batch_size):
        """Return initial condition of hidden state tensor."""
        hidden = torch.zeros(self.n_layers, batch_size, self.hidden_size)
        return hidden.clone().detach().to(device)

def train(
    model: SimpleGRUClassifier,
    device: str,
    criterion: nn.CrossEntropyLoss,
    train_loader: DataLoader,
    optimizer: torch.optim.Adam,
    epoch: int,
):
    """Train the neural network without Lightning."""
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % 20 == 0:
            print(
                f"Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.dataset)} ({100.0 * batch_idx / len(train_loader):.0f}%)]\tLoss: {loss.item():.6f}"
            )

def test(
    model: SimpleGRUClassifier,
    device: str,
    criterion: nn.CrossEntropyLoss,
    test_loader: DataLoader,
):
    """Test the neural network without lightning."""
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += criterion(output, target).item()  # sum up batch loss
            pred = output.argmax(
                dim=1, keepdim=True
            )  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print(
        f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: {correct}/{len(test_loader.dataset)} ({100.0 * correct / len(test_loader.dataset):.0f}%)\n"
    )

class MushroomDataSet(Dataset):
    """Dataset."""

    def __init__(
        self,
        X: np.ndarray,
        Y: np.ndarray
    ):
        """Initialize the instance."""
        self.X = X
        self.Y = Y
        self.len = len(X)

    def __getitem__(self, index):
        """Get an item from given index."""
        return torch.Tensor(
            self.X[index]
        ), int(self.Y[index])

    def __len__(self):
        """Return length of dataset."""
        return self.len

def main() -> None:
    """Run the code."""
    mushroomsfile = 'mushrooms.csv'
    data_array = np.genfromtxt(
        mushroomsfile, delimiter=',', dtype=str, skip_header=1)
    for col in range(data_array.shape[1]):
        data_array[:, col] = np.unique(data_array[:, col], return_inverse=True)[1]

    X = data_array[:, 1:].astype(np.float32)
    Y = data_array[:, 0].astype(np.int32)[:, None]
    df = pd.DataFrame(X)
    df['label'] = Y
    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.33, random_state=112)

    train_dataset = MushroomDataSet(
        X_train,
        y_train
    )
    test_dataset = MushroomDataSet(
        X_test,
        y_test
    )
    train_loader = torch.utils.data.DataLoader(
        train_dataset,
        shuffle=True,
        batch_size=1,
        num_workers=NUM_PROCS,
    )
    test_loader = torch.utils.data.DataLoader(
        test_dataset,
        shuffle=False,
        batch_size=1,
        num_workers=NUM_PROCS,
    )
    model = SimpleGRUClassifier(n_classes=2, input_size=22)
    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
    criterion = nn.CrossEntropyLoss()
    scheduler = StepLR(optimizer, step_size=1, gamma=0.7)

    for epoch in range(1, 1 + 1):
        train(model, device, criterion, train_loader, optimizer, epoch)
        test(model, device, criterion, test_loader)
        scheduler.step()

if __name__ == "__main__":
    main()


from lib import coord_conversion
from lib import route_conversion
from lib import stop_conversion
from lib import arrival_conversion
from lib import message_conversion
import app_configuration as ac
import sys

local_coords = sys.argv[1]
gps_coords = sys.argv[2]
arrivals = sys.argv[3]
messages = sys.argv[4]
scenario = sys.argv[5]

print("Making trips list...")
trips = coord_conversion.make_trips(local_coords, gps_coords)

print("Making routes list...")
routes = route_conversion.make_routes(gps_coords)

print("Making stops list...")
stops = stop_conversion.make_stops(gps_coords, scenario)

print("Making arrivals list...")
arrivals = arrival_conversion.make_arrivals(arrivals, gps_coords)

print("Making messages list...")
messages = message_conversion.message_json(messages, gps_coords)

with open("toolkit/visualization/app/app.js", 'w') as template:
    ac.write_lines(routes, messages, stops, arrivals, trips, template, scenario)


import os
import pandas as pd
import features_extraction

# pre-processing
from scipy import signal
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler

# Constant to declare for the parameters for the sliding window
WINDOW_SIZE = 512
STEP = 128
order = 3
f_sample = 100
f_cutoff = 30
nyquist_rate = f_cutoff / (f_sample / 2)

# Data path should arrive in the format of <phone>/<vibrated>/<frequency>/<.csv> file
def create_windows(data_path):
    columns = ["x", "y", "z"]
    df = pd.read_csv(data_path, usecols=[1, 2, 3], header=None)
    # Assigning columns to the data set
    df.columns = columns
    data_readings = df.values
    # # `noise reduction
    for axis in range(0, 3):
        # order 3 lowpass butterworth filter
        b, a = signal.butter(order, nyquist_rate)
        data_readings[:, axis] = signal.filtfilt(b, a, data_readings[:, axis])

    ###############################################################################################
    # Trying different scaling functions
    ###############################################################################################
    # Min Max Scaler
    # min_max_scaler = preprocessing.MinMaxScaler()
    # data_readings = min_max_scaler.fit_transform(data_readings)

    # Standard Scaler
    scaler = StandardScaler()
    data_readings = scaler.fit_transform(data_readings)
    num_rows = len(data_readings)
    all_window = []

    for start in range(0, num_rows - WINDOW_SIZE, STEP):
        window = data_readings[start:start + WINDOW_SIZE]
        all_window.append(window)

    return all_window

# Getting labels for phone data
phone_list = []
data_dir = os.path.abspath("./100hz_data")
for phone in os.listdir(data_dir):
    phone_list.append(phone)

columns = ['min_x', 'min_y', 'min_z', 'min_rss', 'max_x', 'max_y', 'max_z', 'max_rss', 'std_x', 'std_y', 'std_z',
           'std_rss', 'mean_x', 'mean_y', 'mean_z', 'mean_rss', 'variance_x', 'variance_y', 'variance_z',
           'variance_rss', 'kurtosis_x', 'kurtosis_y', 'kurtosis_z', 'kurtosis_rss',
           'skew_x', 'skew_y', 'skew_z', 'skew_rss', 'phone']

main_df = pd.DataFrame(columns=columns)

# Setting up to traverse through CSV files to extract features
for folder in os.listdir(data_dir):
    data_segments = []
    vibrated_folder = os.path.abspath(os.path.join(data_dir, folder))
    list_of_csv = os.listdir(vibrated_folder)
    for csv in list_of_csv:
        csv_path = os.path.join(vibrated_folder, csv)

        # Making the entire CSV file return a list of list of segments
        data_segments = create_windows(csv_path)

        # For each segment, i extract the features
        for i in data_segments:
            features_csv = []
            features_csv = features_extraction.extract_features(i)
            label = str(folder)
            features_csv.append(label)
            main_df = main_df.append(pd.Series(features_csv, index=main_df.columns), ignore_index=True)

# Export to CSV for machine learning
main_df.to_csv("./features.csv")

# ##################################################################
# # For graph plotting and visualization purposes
# ##################################################################
#
# data = "D:\\Y4S1\\CS4276\\device-fingerprint\\data\\black_huawei\\gyro_100hz_14102019_143558.csv"
# # data = "D:\\Y4S1\\CS4276\\device-fingerprint\\data\\htc_u11\\gyro_100hz_16102019_205643.csv"
# # # data = "D:\\Y4S1\\CS4276\\device-fingerprint\\data\\mi_max\\gyro_100hz_14102019_212145.csv"
# data1 = "D:\\Y4S1\\CS4276\\device-fingerprint\\data\\black_huawei\\gyro_100hz_14102019_143651.csv"
# #
# name1 ="black_huawei"
# name2 ="black_huawei"
# #
# #
# # # For Black huawei
# data_segments = create_segments(data)
# # data_windows = create_window(data)
# #
# # # for blue huawei
# data1_segments = create_segments(data1)
# # data1_windows = create_window(data1)
# #
# # for black huawei
# features_segment = []
# for i in data_segments:
#     features_segment.append(features_extraction.extract_features(i))
#
# # for blue huawei
# features_segment1 = []
# for i in data1_segments:
#     features_segment1.append(features_extraction.extract_features(i))
#
#
# for j in range(len(features_segment[0])):
#     list1 = []
#     list2 = []
#     for i in features_segment:
#         list1.append(i[j])
#
#     for i in features_segment1:
#         list2.append(i[j])
#
#     plt.plot(list1, label=name1)
#     plt.plot(list2, label=name2)
#     plt.xlabel("segments")
#     plt.ylabel("readings")
#     legend = plt.legend(loc='upper center', shadow=True, fontsize='large')
#     if not os.path.exists("./" + name1 + "_" + name2):
#         os.mkdir("./" + name1 + "_" + name2)
#     plt.savefig("./" + name1 + "_" + name2 + "/" + str(j) + name1 + "_" + name2 + '.png')
#     plt.clf()
#


from csv_data import CsvData
from node import Node
from test_case import TestCase
from decision_tree import DecisionTree
from anytree import Node as anyNode, RenderTree
import uuid

# set data first time
percent = input("Split percent(%) : ")

dataset = CsvData("quality")
dataset.setPath("data.csv")
data_test = dataset.splitData(percent)
dataset.initialize()
dataset.calculate()

root = Node(None, None, None, dataset)
root.uuid = str(uuid.uuid4().fields[-1])[:5]
root.an = anyNode(root.csv_data.max_attribute + ' >= ' + str(root.csv_data.result_calculate[root.csv_data.max_attribute].median) + " (" + str(len(root.csv_data.rows)) +")")

generate = DecisionTree(root, "quality")
generate.generate(root)
generate.copyGenerataGraph()
generate.printGraph()

root = generate.index

test_case = TestCase(root, data_test)
test_case.generateConfusedMatrix()

# print("lenght data test ",len(data_test))

# print("tp ", test_case.tp)
# print("tn ", test_case.tn)
# print("fp ", test_case.fp)
# print("fn ", test_case.fn)

total = test_case.tp + test_case.tn + test_case.fp + test_case.fn

# print("correctly ", test_case.tp+test_case.fn)
# print("incorrectly ", test_case.tn+test_case.fp)

# print("empty false ", test_case.empty_false)
# print("empty true ", test_case.empty_true)
print("Split Test (%) : ", percent)
print("\n====== Accuracy ======")
print("correcly percent (%) ", (test_case.tp+test_case.fn) / total * 100)
print("incorrectly percent (%) ", (test_case.tn+test_case.fp) / total * 100)

print("\n==== Confusion Matrix ====")
print("a : b")
print(test_case.fn, " : ", test_case.tn, " | a = no")
print(test_case.fp, " : ", test_case.tp, " | b = yes")

# print accuracy
yesDict = dict()
noDict = dict()

yesDict["tpRate"] = test_case.fp / (test_case.fp + test_case.tn)
yesDict["fpRate"] = test_case.tp / (test_case.tp + test_case.fn)

noDict["tpRate"] = test_case.fn / (test_case.fn + test_case.tp)
noDict["fpRate"] = test_case.tn / (test_case.tn + test_case.fp)

yesDict["recall"] = yesDict["tpRate"]
noDict["recall"] = noDict["tpRate"]

yesDict["precision"] = test_case.tp / (test_case.tp + test_case.fp)
noDict["precision"] = test_case.fn / (test_case.fn + test_case.tn)

yesDict["f-measure"] = 2 * (yesDict["precision"] * yesDict["recall"] / (yesDict["precision"] + yesDict["recall"]))
noDict["f-measure"] = 2 * (noDict["precision"] * noDict["recall"] / (noDict["precision"] + noDict["recall"]))

print("\n====== Detail accuracy by class =====")
print("Tp Rate : Fp Rate : Precision : Recall : F-Measure")
print(noDict["tpRate"], " : ", noDict["fpRate"], " : ", noDict["precision"], " : ", noDict["tpRate"], " : ", noDict["f-measure"], " -no")
print(yesDict["tpRate"], " : ", yesDict["fpRate"], " : ", yesDict["precision"], " : ", yesDict["tpRate"], " : ", yesDict["f-measure"], " -yes")
# print(dataset.attribute_types, " ", dataset.rows)


from PIL import Image
import cv2 as cv
import numpy as np
import mediapipe as mp
from syncronizer import Capture
from tqdm import tqdm

class Video:
    def __init__(self, name, reference, path):
        self.name = name
        self.reference = reference
        self.path = path
        self.Captures = []
        self.All_Normalized_Landmarks = []  # Format: [[[x,y,z],[x,y,z], ... , [x,y,z]], ...] # Dimensions: length x 33 x 3
        self.All_World_Landmarks = []  # Format: [[[x,y,z],[x,y,z], ... , [x,y,z]], ...] # Dimensions: length x 33 x 3
        self.reference_length_candidates = None
        self.position_candidates = None
        self.reference_lengths = []  # [0] -> Normalized reference length, [1] -> World reference length
        self.reference_positions = []  # [0] -> Normalized reference position, [1] -> World reference position
        self.normalization_factors = []  # [0] -> Normalized normalization factor, [1] -> World normalization factor
        self.translation_factor = []  # [0] -> Normalized translation factor, [1] -> World translation factor
        self.mean_errors = []
        self.fps = 30

    def deconstruct(self, shape):
        VIDEO_FILE = cv.VideoCapture(self.path)
        self.fps = int(VIDEO_FILE.get(cv.CAP_PROP_FPS) + 0.5)

        time = 0

        # Deconstruction into folders
        while True:
            # Extract images
            ret, frame = VIDEO_FILE.read()
            if not ret:
                break
            cv.resize(frame, shape, interpolation=cv.INTER_NEAREST)
            self.Captures.append(Capture(time, np.array(frame), self))
            time += 1

    def process(self, detector):
        counter = 0
        for capture in tqdm(self.Captures):
            image = mp.Image(image_format=mp.ImageFormat.SRGB, data=capture.frame)
            capture.detection_result = detector.detect(image)

            _ = []
            try:
                for landmark in capture.detection_result.pose_landmarks[0]:
                    _.append([landmark.x, landmark.y, landmark.z])
                self.All_Normalized_Landmarks.append(_)

                _ = []
                for landmark in capture.detection_result.pose_world_landmarks[0]:
                    _.append([landmark.x, landmark.y, landmark.z])
                self.All_World_Landmarks.append(_)
                counter += 1

            except:
                self.Captures[counter].detection_result = self.Captures[counter - 1].detection_result
                self.All_Normalized_Landmarks.append(self.All_Normalized_Landmarks[counter - 1])
                self.All_World_Landmarks.append(self.All_World_Landmarks[counter - 1])
                print(f"At {counter} the model had to approximate")
                counter += 1

            if len(self.All_Normalized_Landmarks) != counter:
                print(f"At {counter} normalized landmark is missing")

            if len(self.All_World_Landmarks) != counter:
                print(f"At {counter} world landmark is missing")

    def set_normalization_factors(self, ref):
        if self is not ref:
            normalized_normalization_factor = ref.reference_lengths[0] / self.reference_lengths[0]
            world_normalization_factor = ref.reference_lengths[1] / self.reference_lengths[1]

            self.normalization_factors = [normalized_normalization_factor, world_normalization_factor]
            # Set translation factors
            self.translation_factor.append(self.reference_positions[0]*self.normalization_factors[0] - ref.reference_positions[0])
            self.translation_factor.append(self.reference_positions[1]*self.normalization_factors[0] - ref.reference_positions[1])
        else:
            self.normalization_factors = [1, 1]
            self.translation_factor = [[0, 0, 0], [0, 0, 0]]

def set_average_length(objects):
    for obj in objects:
        obj.reference_lengths = [max(list(obj.reference_length_candidates[0].keys())),
                                 max(list(obj.reference_length_candidates[1].keys()))]

        # reference-point = 27
        obj.reference_positions = [obj.position_candidates[0][0][27],
                                   obj.position_candidates[1][0][27]]

def find_min(objects):
    lengths = []

    for obj in objects:
        lengths.append(obj.reference_lengths[0])

    return objects[lengths.index(min(lengths))]