from django.conf.urls import url
from . import views

app_name = 'project'

urlpatterns = [
    url(r'^$', views.index, name='index'),

    # project urls
    url(r'^project-list/$', views.project_list, name='project-list'),
    url(r'^(?P<project_id>[0-9]+)/project-detail/$', views.project_view, name='project-detail'),
    url(r'^project-new/', views.project_create, name='project-create'),
    url(r'^project-update/$', views.project_update, name='project-update'),
    url(r'^(?P<project_id>[0-9]+)/project-delete/$', views.project_delete, name='project-delete'),

    # employee urls
    url(r'^employee-list/$', views.employee_list, name='employee-list'),
    url(r'^(?P<employee_id>[0-9]+)/employee-detail/$', views.employee_view, name='employee-detail'),
    url(r'^employee-new/', views.employee_create, name='employee-create'),
    url(r'^(?P<employee_id>[0-9]+)/employee-update/$', views.employee_update, name='employee-update'),
    url(r'^(?P<employee_id>[0-9]+)/employee-delete/$', views.employee_delete, name='employee-delete'),

    # income urls
    url(r'^income-list/$', views.income_list, name='income-list'),
    url(r'^income-new/$', views.income_create, name='income-create'),
    url(r'^income-update/$', views.income_update, name='income-update'),
    url(r'^(?P<income_id>[0-9]+)/income-delete/$', views.income_delete, name='income-delete'),
    url(r'^(?P<income_id>[0-9]+)/income-detail/$', views.income_view, name='income-detail'),

    # transaction urls
    url(r'^transaction-list/$', views.transaction_list, name='transaction-list'),
    url(r'^transaction-new/$', views.transaction_create, name='transaction-create'),
    # url(r'^income-update/$', views.income_update, name='income-update'),
    # url(r'^(?P<income_id>[0-9]+)/income-delete/$', views.income_delete, name='income-delete'),
    # url(r'^(?P<income_id>[0-9]+)/income-detail/$', views.income_view, name='income-detail'),

    # tasks urls
    url(r'^task-list/$', views.task_list, name='task-list'),
    url(r'^task-new/$', views.task_create, name='task-create'),
    url(r'^task-update/$', views.task_update, name='task-update'),
    url(r'^(?P<task_id>[0-9]+)/task-delete/$', views.task_delete, name='task-delete'),

    # salary urls
    url(r'^salary-list/$', views.salary_list, name='salary-list'),
    url(r'^salary-new/$', views.salary_create, name='salary-create'),
    url(r'^salary-update/$', views.salary_update, name='salary-update'),
    url(r'^(?P<salary_id>[0-9]+)/salary-delete/$', views.salary_delete, name='salary-delete'),

    # debt urls
    url(r'^debt-list/$', views.debt_list, name='debt-list'),
    url(r'^debt-new/$', views.debt_create, name='debt-create'),
    url(r'^debt-update/$', views.debt_update, name='debt-update'),
    url(r'^(?P<debt_id>[0-9]+)/debt-delete/$', views.debt_delete, name='debt-delete'),

    # CompanyEquipment
    url(r'^companyEquipment-list/$', views.company_equipment_list, name='companyEquipment-list'),
    url(r'^companyEquipment-new/$', views.company_equipment_create, name='companyEquipment-create'),
    url(r'^companyEquipment-update/$', views.company_equipment_update, name='companyEquipment-update'),
    url(r'^(?P<companyEquipment_id>[0-9]+)/companyEquipment-delete/$', views.company_equipment_delete,
        name='companyEquipment-delete'),

    # employee work days urls
    url(r'^workDay-list/$', views.workDay_list, name='workDay-list'),
    url(r'^workDay-new/$', views.workDay_create, name='workDay-create'),
    url(r'^workDay-update/$', views.workDay_update, name='workDay-update'),
    url(r'^(?P<workDay_id>[0-9]+)/workDay-delete/$', views.workDay_delete, name='workDay-delete'),

    # financial urls
    url(r'^financial/$', views.financial, name='financial'),

    # 505 urls
    url(r'^505/$', views.pageNotExist, name='pageNotExist'),
]


import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint
from lmfit import minimize, Parameters, Parameter, report_fit
from decimal import Decimal

N = 1000

# Defining system of equations
def f(y,t,params):
  S = y[0]
  I = y[1]

  try:
    k0 = params['k0'].value
  except keyError:
    k0 = params

  f1 = (.0109 * N) - (.007 * S) - (k0 * S * I)/N
  f2 = (k0 * S * I)/N - (.007 + .0819)*I - (.9181 * S)
  return [f1,f2]

# Solving ODE with initial condition
def g(t, x0, param):
  x = odeint(f,x0,t, args=(params,))
  return x

# Computing residual between actual and fitted
def residual(params, t, data):
  init = params['S0'].value, params['I0'].value
  model = g(t,init,params)
  I_model = model[:,1]
  return (I_model - data).ravel()

def estimate(S0,I0,N,k0):
# initial conditions
  y0 = [S0,I0]

# insert data
  time = np.linspace(0, 9, 10)
  I_measured = np.array([1,9,15,29,50,85,120,140,152,160])

# insert parameters
  params = Parameters()
  params.add('S0', value=S0, vary=False)
  params.add('I0',value=I0, vary=False)
  params.add('k0',value=.4, min=.00001, max=10.)

# fit model
  result = minimize(residual, params, args=(time, I_measured), method='leastsq')  # leastsq nelder
# check results of the fit
  data_fitted = g(time, y0, result.params)
  estimated_infected = (timepoint, y0, result.params)
  print("There are " + estimated_infected[1,1] + " estimated infected")
  print("The infection rate is approximately " + k0 + " infections per infected")
  print("Statistical analysis of the model wll also be provided")
  # display fitted statistics
  return report_fit(result)

ninput = input("Input initial total population: ")
try:
  N = int(ninput)
  print("Accepted")
except ValueError:
  print("Please try again with a numeral input")

s0input = input("Input initial infected population: ")
try:
  I0 = int(s0input)
except ValueError:
  print("Please try again with a numeral input")

S0 = N - I0

k0input = input("What is your first guess for infection rate? (Usually a decimal from 0 to 10) ")
try:
  k0 = Decimal(k0input)
  print("Accepted")
except ValueError:
  print("Please try again with a numeral input")

tinput = input("Input a timepoint at which you would like to predict the total infected population: ")
try:
  timepoint = int(tinput)
  print("Accepted. Calculating total infected population and adjusted infection rate... ")
except ValueError:
  print("Please try again with a numeral input")


import PySimpleGUI as psg
import string
# from output_g_code import conversion_van

psg.theme("default1")

layout = [
    [
        psg.Text("Enter Text: ", size=(10, 1)),
        psg.InputText(do_not_clear=False)
    ],
    [
        psg.Text("Select Options: "),
        psg.Radio("H = 2", "group1"),
        psg.Radio("H = 3", "group1"),
        psg.Radio("H = 4", "group1"),
        psg.Radio("H = 5", "group1"),
        psg.Radio("H = 6", "group1"),
    ],
    [
        psg.Button("Convert"),
        psg.Button("Exit")
    ]
]

window = psg.Window("Form", layout)

a_var = []
e_var = []
h_var = ""
low_string = list(string.ascii_lowercase)

while True:
    event, values = window.read()
    if event == "Exit" or event == psg.WIN_CLOSED:
        break

    elif event == "Convert":
        in_string = values[0]
        for index in in_string:
            a_var.append(index)
        for i in in_string:
            e_var.append(low_string.index(i) + 1)

        var_list = [values[1], values[2], values[3],
                    values[4], values[5],
                    ]
        for index in var_list:
            # print(index) debugging
            if index is True:
                h_var = str(var_list.index(index) + 2)

        # output finished code
        for index in values[0]:
            print("G65 P8200 H{0}. E{1} F100. D100. R3. Z-2.5 ({2})"
                  .format(h_var, e_var, a_var))

        # debugging
        print(a_var)
        print(e_var)
        print("H" + h_var + ".")
        print(values)

window.close()


import numpy as np
import gpflow

import matplotlib.pyplot as plt
from  matplotlib import cm
#import matplotlib2tikz as m2t
plt.close('all')
plt.ion()

colors500 = [ cm.coolwarm(x) for x in np.linspace(0, 1, 500) ]
x = np.linspace(-1, 1, 100)[:, np.newaxis]

##############################
## GP samples
num_samples = 200

k = gpflow.kernels.RBF(1, lengthscales=0.3)
K = k.compute_K(x, x)
Z = np.random.multivariate_normal(np.zeros(x.shape[0]), K, num_samples+10000).T

colors = [ cm.coolwarm_r(x) for x in np.linspace(0, 1, num_samples) ]
#colors = [ cm.bone_r(x) for x in np.linspace(0, 1, num_samples) ]

plt.figure(figsize=(9, 5))
for i in range(num_samples):
    plt.plot(x, Z[:,i], color=colors[i], linewidth=.8, alpha=.8)
plt.xlim((-1, 1))
plt.ylim((-4, 4))
plt.xticks([-1,1], fontsize=14)
plt.yticks([-4, 0, 4], fontsize=14)
plt.xlabel('$x$', fontsize=18)
plt.ylabel('$Y(x)$', fontsize=18)

plt.tight_layout()
plt.savefig('gp_samples.pdf', transparent=True)

##############################
## GP definition
n = 10
idx = np.random.choice(np.arange(x.shape[0]), n)

for id in idx:
    plt.axvline(x[id], 0, 1, color='k', linestyle='--', linewidth=.5)

plt.savefig('gp_definition_a.pdf', transparent=True)

A = np.random.uniform(-2, 2, (1, n))

Y = A @ Z[idx, :]
Y = Y.flatten()

plt.figure(figsize=(5,5))
plt.hist(Y, bins=21, density=True)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.xlabel('$\\alpha^T Y(X)$', fontsize=18)
plt.ylabel('', fontsize=18)
plt.tight_layout()
plt.savefig('gp_definition_b.pdf', transparent=True)

##############################
## GP distribution
x = np.linspace(-1, 1, 200)[:, np.newaxis]

num_samples = 20
colors = [ cm.coolwarm_r(x) for x in np.linspace(0, 1, num_samples) ]

means = [3*x, 0*x]
kerns = [gpflow.kernels.RBF, gpflow.kernels.Matern12]

for j in range(2):
    mean = means[j]
    ki = kerns[j](1, lengthscales=0.3)

    kx = ki.compute_K(x, np.zeros((1,1)))

    K = ki.compute_K(x, x)
    Z = mean + np.random.multivariate_normal(np.zeros(x.shape[0]), K, num_samples).T

    z_range = (np.min(Z), np.max(Z))
    plt.figure(figsize=(11, 3.5))

    plt.subplot(1, 3, 1)
    plt.plot(x, mean)
    plt.xticks([-1, 1], fontsize=14)
    plt.yticks(fontsize=14)
    plt.title('$m(x)$', fontsize=18)
    plt.ylim(z_range)

    plt.subplot(1, 3, 2)
    plt.plot(x, kx)
    plt.xticks([-1, 0, .5, 1], fontsize=14)
    plt.yticks(fontsize=14)
    plt.title('$k(0, x)$', fontsize=18)
    plt.axvline(0, 0, 1, color='k', linestyle='--', linewidth=.5)
    plt.axvline(0.5, 0, 1, color='k', linestyle='--', linewidth=.5)

    plt.subplot(1, 3, 3)
    for i in range(num_samples):
        plt.plot(x, Z[:,i], color=colors[i], linewidth=.8, alpha=.8)
    plt.xlim((-1, 1))
    plt.ylim((-4, 4))
    plt.xticks([-1, 0, .5, 1], fontsize=14)
    plt.yticks(fontsize=14)
    plt.title('$Y(x)$', fontsize=18)
    plt.ylim(z_range)
    plt.axvline(0, 0, 1, color='k', linestyle='--', linewidth=.5)
    plt.axvline(0.5, 0, 1, color='k', linestyle='--', linewidth=.5)

    plt.tight_layout()
    plt.savefig('gp_distribution_{}.pdf'.format(j), transparent=True)

##############################
## observations
def f(x):
    y = 2*x + 2*np.sin(2*np.pi*x)
    return y

n = 7
X = np.linspace(-.7, .7, n)[:, np.newaxis]
Y = f(X)

plt.figure(figsize=(9, 5))
plt.scatter(X, Y, marker='x', color='k', s=25, linewidth=2)
plt.xlim((-1, 1))
plt.ylim((-5, 5))
plt.xticks([-1,1], fontsize=14)
plt.yticks([-5, 0, 4], fontsize=14)
plt.xlabel('$x$', fontsize=18)
plt.ylabel('$f$', fontsize=18)

plt.tight_layout()
plt.savefig('gp_obs.pdf', transparent=True)

##############################
## GP prior samples
num_samples = 200

k = gpflow.kernels.RBF(1, variance=2., lengthscales=0.15)
K = k.compute_K(x, x)
Z = np.random.multivariate_normal(np.zeros(x.shape[0]), K, num_samples+10000).T

colors = [ cm.coolwarm_r(x) for x in np.linspace(0, 1, num_samples) ]
#colors = [ cm.bone_r(x) for x in np.linspace(0, 1, num_samples) ]

plt.figure(figsize=(9, 5))
for i in range(num_samples):
    plt.plot(x, Z[:,i], color=colors[i], linewidth=.8, alpha=.8)
plt.xlim((-1, 1))
plt.ylim((-5, 5))
plt.xticks([-1,1], fontsize=14)
plt.yticks([-4, 0, 4], fontsize=14)
plt.xlabel('$x$', fontsize=18)
plt.ylabel('$Y(x)$', fontsize=18)

plt.tight_layout()
plt.savefig('gp_samples2.pdf', transparent=True)

#############################
## conditional samples

m = gpflow.models.GPR(X, Y, kern=k)
m.likelihood.variance = 0.001

num_samples = 50

Z = m.predict_f_samples(x, num_samples)[:,:,0].T

colors = [ cm.coolwarm_r(x) for x in np.linspace(0, 1, num_samples) ]
#colors = [ cm.bone_r(x) for x in np.linspace(0, 1, num_samples) ]

plt.figure(figsize=(9, 5))
for i in range(num_samples):
    plt.plot(x, Z[:,i], color=colors[i], linewidth=.8, alpha=.8)
plt.xlim((-1, 1))
plt.ylim((-5, 5))
plt.xticks([-1,1], fontsize=14)
plt.yticks([-4, 0, 4], fontsize=14)
plt.xlabel('$x$', fontsize=18)
plt.ylabel('$Y(x) | Y(X)=F$', fontsize=18)
plt.plot(X, Y, 'kx', mew=2)

plt.tight_layout()
plt.savefig('gp_cond_samples.pdf', transparent=True)

def plot(x, m):
    mean, var = m.predict_f(x)
    plt.fill_between(x[:,0],
                     mean[:,0] - 2*np.sqrt(var[:,0]),
                     mean[:,0] + 2*np.sqrt(var[:,0]),
                     color='C0', alpha=0.2)
    plt.plot(x, mean, 'C0', lw=2)
    plt.plot(x, mean[:,0]-2*np.sqrt(var[:,0]), 'C0', lw=.5)
    plt.plot(x, mean[:,0]+2*np.sqrt(var[:,0]), 'C0', lw=.5)
    plt.xlim((-1, 1))
    plt.ylim((-5, 5))
    plt.xticks([-1,1], fontsize=14)
    plt.yticks([-4, 0, 4], fontsize=14)
    plt.xlabel('$x$', fontsize=18)
    plt.ylabel('$Y(x) | Y(X)=F$', fontsize=18)
    plt.plot(X, Y, 'kx', mew=2)

plt.figure(figsize=(9, 5))
plot(x, m)
plt.tight_layout()
plt.savefig('gp_model.pdf', transparent=True)


# -*- coding: utf-8 -*-
"""
Created on Mon Nov 23 09:30:53 2015

@author: voxxys
"""
from pylsl import StreamInlet, resolve_stream

import time
import numpy as np
from scipy import io
from scipy.signal import butter, lfilter
from scipy.linalg import eig
import matplotlib.pyplot as plt
import theano
import os
import scipy.signal as spsig

import pickle

os.chdir('D:/BCI_MINI/recordings')

class params:
    def __init__(self):
        self.numch = 0 #TOFIX to fix elsewhere

        self.srate = 0
        self.win_av = 0

        self.without_emp_mask = 0

        self.pre_fl = 0
        self.pre_fh = 0

        self.filt_order = 0

        self.M_eog = 0
        self.all_CSPs = 0
        self.freqs = 0

        self.W1_saved = 0
        self.b1_saved = 0
        self.W2_saved = 0
        self.b2_saved = 0

with open('params_0.pkl', 'rb') as input:
        params_1 = pickle.load(input)

all_CSPs = params_1.all_CSPs
filtorder = params_1.filt_order
win = params_1.win_av
freq = params_1.freqs
srate = params_1.srate
M_eog = params_1.M_eog

fmin_pre = params_1.pre_fh
fmax_pre = params_1.pre_fl

without_emp_mask = params_1.without_emp_mask

##NN

# ____________________Set NN____________________

X = theano.tensor.fmatrix('X')
Y = theano.tensor.imatrix('Y')

# initialize weights and biases
W1_init = params_1.W1_saved
b1_init = params_1.b1_saved
W2_init = params_1.W2_saved
b2_init = params_1.b2_saved

W1 = theano.shared(W1_init, name = 'W1')
b1 = theano.shared(b1_init, name = 'b1')
W2 = theano.shared(W2_init, name = 'W2')
b2 = theano.shared(b2_init, name = 'b2')

dot_1 = theano.tensor.dot(X, W1) + b1
activ_1 = theano.tensor.nnet.sigmoid(dot_1)
dot_2 = theano.tensor.dot(activ_1, W2) + b2
activ_final = theano.tensor.nnet.softmax(dot_2)

pred_y = theano.tensor.argmax(activ_final)

# Function to get classes' probabilities for a given input
pred_proba = theano.function([X], activ_final, allow_input_downcast = True)

# Function to predict class for a given input
predict_val = theano.function([X], pred_y, allow_input_downcast = True)

# Function to check accuracy on a dataset (proportion of correct)
def accuracy_for_dataset(inputs, labels):
    return sum([predict_val(inputs[i,:].reshape(1,inputs.shape[1])) == labels[i] for i in range(inputs.shape[0])])/float(inputs.shape[0])

######

numf = len(params_1.freqs)
numc = 0

num_all = len(all_CSPs)
for i in range(num_all):
    if(all_CSPs[i].shape[0] > 0):
        numc = numc + 1

numch = 24 #TOFIX

# preprocessing high and lowpass filters

a_pre_high = np.zeros((1,filtorder + 1))
b_pre_high = np.zeros((1,filtorder + 1))
a_pre_low = np.zeros((1,filtorder + 1))
b_pre_low = np.zeros((1,filtorder + 1))

Zlast_pre_high = np.zeros((1,numch,filtorder))
Zlast_pre_low = np.zeros((1,numch,filtorder))

[b_pre_high[0], a_pre_high[0]] = spsig.butter(filtorder, float(fmin_pre)/(srate/2), 'high')
[b_pre_low[0], a_pre_low[0]] = spsig.butter(filtorder, float(fmax_pre)/(srate/2), 'low')

# create bandpass filters

a_high = np.zeros((numf,filtorder + 1))
b_high = np.zeros((numf,filtorder + 1))
a_low = np.zeros((numf,filtorder + 1))
b_low = np.zeros((numf,filtorder + 1))

numch_without_emp = np.sum(without_emp_mask)

Zlast_high = np.zeros((numf,numch_without_emp,filtorder))
Zlast_low = np.zeros((numf,numch_without_emp,filtorder))

for fr in range(numf):

    fmin = freq[fr][0]
    fmax = freq[fr][1]

    [b_high[fr], a_high[fr]] = spsig.butter(filtorder, float(fmin)/(srate/2), 'high')
    [b_low[fr], a_low[fr]] = spsig.butter(filtorder, float(fmax)/(srate/2), 'low')

# create moving average filter
Zlast_ma = np.zeros((numc,win-1))
a_ma = 1
b_ma = np.ones((1,win))

###

exptime = 900
dt_rate = 0.1

# allocating buffers
received_data_buf = np.zeros((numch, exptime*srate*1.2))
states_predicted_buf = np.zeros((1, exptime*srate*1.2))
pos = 0
pos_pred = 0

# first resolve an EEG stream on the lab network
print("looking for an EEG stream...")
streams = resolve_stream('type', 'Data')

# create a new inlet to read from the stream
inlet = StreamInlet(streams[0])

globalstart = time.time()

while (time.time() - globalstart < exptime):

    startwhile = time.time()

    chunk, timestamp = inlet.pull_chunk()
    np_ar_chunk = np.asarray(chunk)
    chunk_size = np_ar_chunk.shape[0]

    if chunk_size > 0:

        data_chunk_test = np_ar_chunk.T

        received_data_buf[:,pos:(pos+chunk_size)] = data_chunk_test
        pos = pos + chunk_size + 1

        [data_chunk_test,Zlast_pre_high[0,:,:]] = spsig.lfilter(b_pre_high[0], a_pre_high[0], data_chunk_test, 1, Zlast_pre_high[0,:,:])
        [data_chunk_test,Zlast_pre_low[0,:,:]] = spsig.lfilter(b_pre_low[0], a_pre_low[0], data_chunk_test, 1, Zlast_pre_low[0,:,:])

        data_chunk_test = data_chunk_test[without_emp_mask,:]
        #chan_names_test_used = chan_names_test[:,without_emp_mask]
        data_chunk_test = np.dot(M_eog,data_chunk_test)

        ###filt_apply_CSPs(data, sr, freq_range, all_CSPs, how_to_filt, win, order=5, normalize=False):

        N_csp_per_freq = len(all_CSPs)/len(freq)
        all_CSPs_copy = list(all_CSPs)
        transformed_data_chunk = np.zeros((0, data_chunk_test.shape[1]))

        for fr_ind in range(len(freq)):

            [data_chunk_test,Zlast_high[fr_ind,:,:]] = spsig.lfilter(b_high[fr_ind], a_high[fr_ind], data_chunk_test, 1, Zlast_high[fr_ind,:,:])
            [data_chunk_test,Zlast_low[fr_ind,:,:]] = spsig.lfilter(b_low[fr_ind], a_low[fr_ind], data_chunk_test, 1, Zlast_low[fr_ind,:,:])

            for csp_ind in range(N_csp_per_freq):
                transformed_data_chunk = np.vstack((transformed_data_chunk, np.dot(all_CSPs_copy.pop(0), data_chunk_test)))

        final_data_chunk = transformed_data_chunk[1:,:]**2
        a_ma = 1
        b_ma = np.ones(win)/float(win)

        [final_data_chunk,Zlast_ma] = spsig.lfilter(b_ma, a_ma, transformed_data_chunk, 1, Zlast_ma)

        inputs = final_data_chunk

        chunk_res = np.array([predict_val(inputs[:,i].reshape(1,inputs.shape[0])) for i in range(inputs.shape[1])])

        res_chunk_size = chunk_res.shape[0]
        states_predicted_buf[:,pos_pred:(pos_pred+res_chunk_size)] = chunk_res
        pos_pred = pos_pred+res_chunk_size

        print(chunk_res[-1])

    dif = dt_rate - time.time() + startwhile
    if(dif > 0):
        time.sleep(dif)


#!/usr/bin/env python3

__author__    = "Po-E (Paul) Li, Bioscience Division, Los Alamos National Laboratory"
__credits__   = ["Po-E Li", "Jason Gans", "Tracey Freites", "Patrick Chain"]
__version__   = "2.1 BETA"
__date__      = "2016/05/31"
__copyright__ = """
Copyright (2014). Los Alamos National Security, LLC. This material was produced
under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National Laboratory
(LANL), which is operated by Los Alamos National Security, LLC for the U.S.
Department of Energy. The U.S. Government has rights to use, reproduce, and
distribute this software.  NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY,
LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE
USE OF THIS SOFTWARE.  If software is modified to produce derivative works, such
modified software should be clearly marked, so as not to confuse it with the
version available from LANL.

Additionally, this program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option) any
later version. Accordingly, this program is distributed in the hope that it will
be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
"""

import argparse as ap, textwrap as tw
import sys, os, time, subprocess
import gottcha_taxonomy as gt
from re import search,findall
from multiprocessing import Pool
from itertools import chain

def parse_params( ver ):
	p = ap.ArgumentParser( prog='gottcha.py', description="""Genomic Origin Through Taxonomic CHAllenge (GOTTCHA) is an
			annotation-independent and signature-based metagenomic taxonomic profiling tool
			that has significantly smaller FDR than other profiling tools. This program
			is a wrapper to map input reads to pre-computed signature databases using BWA-MEM
			and/or to profile mapped reads in SAM format. (VERSION: %s)""" % ver)

	eg = p.add_mutually_exclusive_group( required=True )

	eg.add_argument( '-i','--input', metavar='[FASTQ]', nargs='+', type=str,
	  				help="Input one or multiple FASTQ file(s). Use space to separate multiple input files.")

	eg.add_argument( '-s','--sam', metavar='[SAMFILE]', nargs=1, type=ap.FileType('r'),
					help="Specify the input SAM file. Use '-' for standard input.")

	p.add_argument( '-d','--database', metavar='[BWA_INDEX]', type=str, default=None,
	                help="The path of signature database. The database can be in FASTA format or BWA index (5 files).")

	p.add_argument( '-l','--dbLevel', metavar='[LEVEL]', type=str, default='',
	                choices=['superkingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species', 'strain'],
	                help="""Specify the taxonomic level of the input database. You can choose one rank from "superkingdom", "phylum", "class", "order", "family", "genus", "species" and "strain". The value will be auto-detected if the input database ended with levels (e.g. GOTTCHA_db.species).""")

	p.add_argument( '-ti','--taxInfo', metavar='[FILE]', type=str, default='',
	                help="""Specify the path of taxonomy information file (taxonomy.tsv). GOTTCHA2 will try to locate this file when user doesn't specify a path. If '--database' option is used, the program will try to find this file in the directory of specified database. If not, the 'database' directory under the location of gottcha.py will be used as default.""")

	p.add_argument( '-pm','--mismatch', metavar='<INT>', type=int, default=5,
					help="Mismatch penalty for BWA-MEM (pass to option -B while BWA-MEM is running). You can use 99 for not allowing mismatch in alignments (except for extreme cases). [default: 5]")

	p.add_argument( '-m','--mode', type=str, default='summary',
	                choices=['summary', 'full', 'tree', 'class', 'extract', 'lineage'],
					help="""You can specify one of the following output modes:
                            "summary" : report a summary of profiling result;
                            "full"    : other than a summary result, this mode will report unfiltered profiling results with more detail;
                            "tree"    : report results with lineage of taxonomy;
                            "class"   : output results of classified reads;
                            "extract" : extract mapped reads;
                            "lineage" : output abundance and lineage in a line;
						  Note that only results/reads belongs to descendants of TAXID will be reported/extracted if option [--taxonomy TAXID] is specified. [default: summary]""" )

	p.add_argument( '-x','--taxonomy', metavar='[TAXID]', type=str,
	        		help="Specify a NCBI taxonomy ID. The program  will only report/extract the taxonomy you specified.")

	p.add_argument( '-r','--relAbu', metavar='[FIELD]', type=str, default='LINEAR_DOC',
					choices=['LINEAR_LENGTH','TOTAL_BP_MAPPED','READ_COUNT','LINEAR_DOC'],
					help='The field will be used to calculate relative abundance. You can specify one of the following fields: "LINEAR_LENGTH", "TOTAL_BP_MAPPED", "READ_COUNT" and "LINEAR_DOC". [default: LINEAR_DOC]')

	p.add_argument( '-t','--threads', metavar='<INT>', type=int, default=1,
					help="Number of threads [default: 1]")

	p.add_argument( '-o','--outdir', metavar='[DIR]', type=str, default='.',
					help="Output directory [default: .]")

	p.add_argument( '-p','--prefix', metavar='<STR>', type=str, required=False,
					help="Prefix of the output file [default: <INPUT_FILE_PREFIX>]")

	p.add_argument( '-mc','--minCov', metavar='<FLOAT>', type=float, default=0.005,
					help="Minimum linear coverage to be considered valid in abundance calculation [default: 0.005]")

	p.add_argument( '-mr','--minReads', metavar='<INT>', type=int, default=3,
					help="Minimum number of reads to be considered valid in abundance calculation [default: 3]")

	p.add_argument( '-ml','--minLen', metavar='<INT>', type=int, default=60,
					help="Minimum unique length to be considered valid in abundance calculation [default: 60]")

	p.add_argument( '-nc','--noCutoff', action="store_true",
					help="Remove all cutoffs. This option is equivalent to use [-mc 0 -mr 0 -ml 0].")

	p.add_argument( '-c','--stdout', action="store_true",
					help="Write on standard output.")

	p.add_argument( '--silent', action="store_true",
	                help="Disable all messages.")

	args_parsed = p.parse_args()

	"""
	Checking options
	"""
	if args_parsed.input and not args_parsed.database:
		p.error( '--database option is missing.' )

	if args_parsed.input and args_parsed.sam:
		p.error( '--input and --same are incompatible options.' )

	if args_parsed.database:
		#assign default path for database name
		if "/" not in args_parsed.database and not os.path.isfile( args_parsed.database + ".amb" ):
			bin_dir = os.path.dirname(os.path.realpath(__file__))
			args_parsed.database = bin_dir + "/database/" + args_parsed.database

	if args_parsed.database:
		if not os.path.isfile( args_parsed.database + ".amb" ):
			p.error( 'Incorrect BWA index: missing %s.amb.' % args_parsed.database )
		if not os.path.isfile( args_parsed.database + ".ann" ):
			p.error( 'Incorrect BWA index: missing %s.ann.' % args_parsed.database )
		if not os.path.isfile( args_parsed.database + ".pac" ):
			p.error( 'Incorrect BWA index: missing %s.pac.' % args_parsed.database )
		if not os.path.isfile( args_parsed.database + ".bwt" ):
			p.error( 'Incorrect BWA index: missing %s.bwt.' % args_parsed.database )
		if not os.path.isfile( args_parsed.database + ".sa" ):
			p.error( 'Incorrect BWA index: missing %s.sa.' % args_parsed.database )

	if not args_parsed.taxInfo:
		if args_parsed.database:
			db_dir = search( '^(.*?)[^\/]+$', args_parsed.database )
			args_parsed.taxInfo = db_dir.group(1) + "taxonomy.tsv"
		else:
			bin_dir = os.path.dirname(os.path.realpath(__file__))
			args_parsed.taxInfo = bin_dir + "/database/taxonomy.tsv"

	if not os.path.isfile( args_parsed.taxInfo ):
		p.error( 'Taxonomy information (taxonomy.tsv) not found.' )

	if not args_parsed.prefix:
		if args_parsed.input:
			name = search('([^\/\.]+)\..*$', args_parsed.input[0] )
			args_parsed.prefix = name.group(1)
		elif args_parsed.sam:
			name = search('([^\/]+).\w+.\w+$', args_parsed.sam[0].name )
			args_parsed.prefix = name.group(1)
		else:
			args_parsed.prefix = "GOTTCHA_"

	if not args_parsed.dbLevel:
		if args_parsed.database:
			name = search('\.(\w+).\w+$', args_parsed.database )
			args_parsed.dbLevel = name.group(1)
		elif args_parsed.sam:
			name = search('\.gottcha_(\w+).sam$', args_parsed.sam[0].name )
			args_parsed.dbLevel = name.group(1)
		else:
			p.error( '--dbLevel is missing and cannot be auto-detected.' )

	if args_parsed.noCutoff:
		#p.error( 'conflict options: cutoff(s) are specified with --noCutoff option.' )
		args_parsed.minCov = 0
		args_parsed.minReads = 0
		args_parsed.minLen = 0

	return args_parsed

def dependency_check(cmd):
	proc = subprocess.Popen("which " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	outs, errs = proc.communicate()
	return outs.decode().rstrip() if proc.returncode == 0 else False

def join_ranges(data, offset=0):
	"""
	Merge location regions
	"""
	flatten = chain.from_iterable
	LEFT, RIGHT = 1, -1
	data = sorted(flatten(((start, LEFT), (stop + offset, RIGHT)) for start, stop in data))
	c = 0
	for value, label in data:
		if c == 0:
			x = value
		c += label
		if c == 0:
			yield [x, value - offset]

def worker(lines):
	"""Make a dict out of the parsed, supplied lines"""
	res = gt._autoVivification()

	for line in lines:
		k, r, n, rd, rs, rq, flag, cigr = parse(line)
		if k in res:
			res[k]["ML"] = list( join_ranges( res[k]["ML"] + [r] ) )
			res[k]["MB"] += r[1] - r[0] + 1
			res[k]["MR"] += 1
			res[k]["NM"] += n
		else:
			res[k]["ML"] = [r]
			res[k]["MB"] = r[1] - r[0] + 1
			res[k]["MR"] = 1
			res[k]["NM"] = n

	return res

def parse(line):
	"""
	Parse SAM format
	read1   0   test    11  0   5S10M3S *   0   0   GGGGGCCCCCCCCCCGGG  HHHHHHHHHHHHHHHHHH  NM:i:0  MD:Z:10 AS:i:10 XS:i:0
	read2   16  test    11  0   3S10M5S *   0   0   GGGCCCCCCCCCCGGGGG  HHHHHHHHHHHHHHHHHH  NM:i:0  MD:Z:10 AS:i:10 XS:i:0
	"""
	temp = line.split('\t')
	name = temp[0]
	match_len    = search('(\d+)M', temp[5])
	mismatch_len = search('NM:i:(\d+)', temp[11])
	start = int(temp[3])
	end   = start + int(match_len.group(1)) - 1;

	ref = temp[2].rstrip('|')
	ref = ref[: ref.find(".0") if ref.find(".0")>0 else None ]

	return ref, [start, end], int(mismatch_len.group(1)), name, temp[9], temp[10], temp[1], temp[5]

def timeSpend( start ):
	done = time.time()
	elapsed = done - start
	return time.strftime( "%H:%M:%S", time.gmtime(elapsed) )

def processSAMfile( sam_fp, numthreads, numlines ):
	result = gt._autoVivification()
	mapped_reads = 0

	lines = sam_fp.readlines();
	if len(lines) > numlines and numthreads > 2:
		pool = Pool(processes=numthreads)
		result_list = pool.map(worker,
		    (lines[line:line+numlines] for line in range(0,len(lines),numlines) ) )

		for res in result_list:
			for k in res:
				if k in result:
					result[k]["ML"] = list( join_ranges( result[k]["ML"] + res[k]["ML"] ) )
					result[k]["MB"] += res[k]["MB"]
					result[k]["MR"] += res[k]["MR"]
					result[k]["NM"] += res[k]["NM"]
				else:
					result[k].update(res[k])

	else:
		result = worker(lines)

	# convert mapped regions to linear length
	for k in result:
		result[k]["LL"] = 0
		mapped_reads += result[k]["MR"]
		for cr in result[k]["ML"]:
			result[k]["LL"] += cr[1]-cr[0]

	return result, mapped_reads

def isDescendant( taxid, taxid_ant ):
	fullLineage = gt.taxid2fullLineage( taxid )
	if "|%s|" % taxid_ant in fullLineage:
		return True
	else:
		return False

def processSAMfileReadClass( f, o, tg_rank, taxid_fi ):
	for line in f:
		ref, region, nm, rname, rseq, rq, flag, cigr = parse(line)
		acc, start, stop, tid = ref.split('|')

		if taxid_fi:
			if not isDescendant( tid, taxid_fi ):
				continue

		o.write( "%s\t%s\t%s\t%s\t%s\n" % (
		    rname,
		    gt.taxid2taxidOnRank(tid, tg_rank),
			"%s:%s..%s" % (ref, region[0], region[1]),
			cigr,
		    gt.taxid2nameOnRank( tid, tg_rank )
		))

def processSAMfileReadExtract( f, o, taxid ):
	for line in f:
		ref, region, nm, rname, rseq, rq, flag, cigr = parse(line)
		acc, start, stop, t = ref.split('|')
		fullLineage = gt.taxid2fullLineage(t)

		if int(flag) & 16:
			g = findall("\d+\w", cigr)
			cigr = "".join(list(reversed(g)))
			rseq = seqReverseComplement(rseq)
			rq = rq[::-1]

		if isDescendant( t, taxid ):
			o.write( "@%s %s:%s..%s %s\n%s\n+\n%s\n" % (rname, ref, region[0], region[1], cigr, rseq, rq) )

def seqReverseComplement( seq ):
	for base in seq:
		if base not in 'ACGTURYSWKMBDHVNacgturyswkmbdhvn':
			print("[ERROR] NOT a DNA sequence")
			return None
	seq1 = 'ACGTURYSWKMBDHVNTGCAAYRSWMKVHDBNacgturyswkmbdhvntgcaayrswmkvhdbn'
	seq_dict = { seq1[i]:seq1[i+16] for i in range(64) if i < 16 or 32<=i<48 }
	return "".join([seq_dict[base] for base in reversed(seq)])

def taxonomyRollUp( r ):
	"""
	Take parsed SAM output and rollup to superkingdoms
	"""
	res_rollup = gt._autoVivification()
	res_tree = gt._autoVivification()

	for ref in r:
		(acc, start, stop, t) = ref.split('|')

		tree = gt.taxid2fullLinkDict( t )

		for pid, tid in tree.items():
			res_tree[pid][tid] = 1
			if tid in res_rollup:
				res_rollup[tid]["ML"] += ";%s:%s" %  ( ref, ",".join("..".join(map(str,l)) for l in r[ref]["ML"]) )
				res_rollup[tid]["MB"] += r[ref]["MB"]
				res_rollup[tid]["MR"] += r[ref]["MR"]
				res_rollup[tid]["NM"] += r[ref]["NM"]
				res_rollup[tid]["LL"] += r[ref]["LL"]
				res_rollup[tid]["SL"] += int(stop) - int(start) + 1
			else:
				res_rollup[tid]["ML"] = "%s:%s" %  ( ref, ",".join("..".join(map(str,l)) for l in r[ref]["ML"]) )
				res_rollup[tid]["MB"] = r[ref]["MB"]
				res_rollup[tid]["MR"] = r[ref]["MR"]
				res_rollup[tid]["NM"] = r[ref]["NM"]
				res_rollup[tid]["LL"] = r[ref]["LL"]
				res_rollup[tid]["SL"] = int(stop) - int(start) + 1

	return res_rollup, res_tree

def outputResultsAsTree( tid, res_tree, res_rollup, indent, dbLevel, lvlFlag, taxid_fi, o, mc, mr, ml ):
	"""
	iterate taxonomy tree and print results recursively
	"""
	if int(tid) == 1:
		o.write( "%s%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % ( "", "NAME", "LEVEL", "READ_COUNT", "TOTAL_BP_MAPPED", "TOTAL_BP_MISMATCH", "LINEAR_LENGTH", "LINEAR_COV", "LINEAR_DOC" ) )
	else:
		if gt.taxid2rank(tid) == dbLevel: lvlFlag = 1
		if mc <= int(res_rollup[tid]["MB"])/int(res_rollup[tid]["LL"]) and mr <= int(res_rollup[tid]["MR"]) and ml <= res_rollup[tid]["LL"]:
			o.write( "%s%s\t%s\t%s\t%s\t%s\t%s\t%.4f\t%.4f\n" % (
				indent,
				gt.taxid2name(tid),
				gt.taxid2rank(tid),
				res_rollup[tid]["MR"],
				res_rollup[tid]["MB"],
				res_rollup[tid]["NM"],
				res_rollup[tid]["LL"],
				int(res_rollup[tid]["LL"])/int(res_rollup[tid]["SL"]),
				int(res_rollup[tid]["MB"])/int(res_rollup[tid]["LL"])
				)
			)

	if len( res_tree[tid] ):
		indent += "    "
		for cid in res_tree[tid]:
			if taxid_fi:
				if not isDescendant( tid, taxid_fi ):
					continue

			if lvlFlag and gt.taxid2rank(cid) != dbLevel:
				continue

			if res_rollup[tid] and ( mc > int(res_rollup[tid]["LL"])/int(res_rollup[tid]["SL"]) or mr > int(res_rollup[tid]["MR"]) or ml > int(res_rollup[tid]["LL"]) ):
				continue

			outputResultsAsTree( cid, res_tree, res_rollup, indent, dbLevel, lvlFlag, taxid_fi, o, mc, mr, ml )

def outputResultsAsRanks( res_rollup, o, tg_rank, relAbu, mode, mc, mr, ml ):
	output = gt._autoVivification()
	major_ranks = {"superkingdom":1,"phylum":2,"class":3,"order":4,"family":5,"genus":6,"species":7,"strain":8}

	for tid in res_rollup:
		rank = gt.taxid2rank(tid)

		if mode == "summary" and ( mc > res_rollup[tid]["LL"]/res_rollup[tid]["SL"] or mr > int(res_rollup[tid]["MR"]) or ml > int(res_rollup[tid]["LL"]) ):
			continue

		if rank in major_ranks and major_ranks[rank] <= major_ranks[tg_rank]:
			if relAbu == "LINEAR_LENGTH":
				abundance = int(res_rollup[tid]["LL"])
			elif relAbu == "TOTAL_BP_MAPPED":
				abundance = int(res_rollup[tid]["MB"])
			elif relAbu == "READ_COUNT":
				abundance = int(res_rollup[tid]["MR"])
			else:
				abundance = int(res_rollup[tid]["MB"])/int(res_rollup[tid]["LL"])

			output[rank]["RES"][tid] = abundance

			if "TOT_ABU" in output[rank]:
				output[rank]["TOT_ABU"] += abundance
			else:
				output[rank]["TOT_ABU"] = abundance

	add_field = "\tLINEAR_COV\tNOTE\tMAPPED_REGION" if mode == "full" else ""

	o.write( "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%s\n" %
			( "LEVEL", "NAME", "TAXID", "READ_COUNT", "TOTAL_BP_MAPPED",
			  "TOTAL_BP_MISMATCH", "LINEAR_LENGTH", "LINEAR_DOC", "REL_ABUNDANCE", add_field ) )

	for rank in sorted( major_ranks, key=major_ranks.__getitem__ ):
		if major_ranks[rank] > major_ranks[tg_rank]: break
		taxas = output[rank]["RES"]
		for tid in sorted( taxas, key=taxas.__getitem__, reverse=True):
			mc_note = "Filtered out for minCov. "   if mc > res_rollup[tid]["LL"]/res_rollup[tid]["SL"] else ""
			mr_note = "Filtered out for minReads. " if mr > int(res_rollup[tid]["MR"]) else ""
			ml_note = "Filtered out for minLen. "   if ml > int(res_rollup[tid]["LL"]) else ""

			add_field = "\t%.4f\t%s%s%s\t%s" % (
			    (res_rollup[tid]["LL"]/res_rollup[tid]["SL"]),
				mc_note,
				mr_note,
				ml_note,
				res_rollup[tid]["ML"]
			) if mode == "full" else ""

			if mc_note and mr_note and ml_note and mode=="summary":
				continue

			o.write( "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%.4f\t%.4f%s\n" %
				( rank,
				  gt.taxid2name(tid),
				  tid,
				  res_rollup[tid]["MR"],
				  res_rollup[tid]["MB"],
				  res_rollup[tid]["NM"],
				  res_rollup[tid]["LL"],
				  int(res_rollup[tid]["MB"])/int(res_rollup[tid]["LL"]),
				  output[rank]["RES"][tid]/output[rank]["TOT_ABU"],
				  add_field
				)
			)

def outputResultsAsLineage( res_rollup, o, tg_rank, relAbu, mode, mc, mr, ml ):
	for tid in res_rollup:
		rank = gt.taxid2rank(tid)

		if rank != tg_rank or ( mc > res_rollup[tid]["LL"]/res_rollup[tid]["SL"] or mr > int(res_rollup[tid]["MR"]) or ml > int(res_rollup[tid]["LL"]) ):
			continue

		if relAbu == "LINEAR_LENGTH":
			abundance = int(res_rollup[tid]["LL"])
		elif relAbu == "TOTAL_BP_MAPPED":
			abundance = int(res_rollup[tid]["MB"])
		elif relAbu == "READ_COUNT":
			abundance = int(res_rollup[tid]["MR"])
		else:
			abundance = int(res_rollup[tid]["MB"])/int(res_rollup[tid]["LL"])

		o.write( "%s\t%s\n" %
			( abundance,
			  '\t'.join( gt.taxid2lineage(tid).split('|') )
			)
		)

def readMapping( reads, db, threads, mm_penalty, samfile, logfile ):
	"""
	mapping reads to database
	"""

	input_file = " ".join(reads)

	bash_cmd   = "set -o pipefail; set -x;"
	bwa_cmd    = "bwa mem -k30 -T30 -A1 -B%s -O99 -E99 -L0 -t%s %s %s" % (mm_penalty, threads, db, input_file )
	filter_cmd = "awk -F\\\\t '!/^@/ && !and($2,4) && !and($2,2048){print $_}'"
	cmd        = "%s %s 2> %s | %s > %s" % ( bash_cmd, bwa_cmd, logfile, filter_cmd, samfile )

	proc = subprocess.Popen( cmd, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
	outs, errs = proc.communicate()
	exitcode = proc.poll()

	return exitcode, bwa_cmd, errs

def print_message( msg, silent, start):
	if not silent:
		sys.stderr.write( "[%s] %s\n" % (timeSpend(start), msg) )

if __name__ == '__main__':
	argvs    = parse_params( __version__ )
	begin_t  = time.time()
	numlines = 10000
	sam_fp   = argvs.sam[0] if argvs.sam else ""
	samfile  = "%s/%s.gottcha_%s.sam" % ( argvs.outdir, argvs.prefix, argvs.dbLevel ) if not argvs.sam else sam_fp.name
	logfile  = "%s/%s.gottcha_%s.log" % ( argvs.outdir, argvs.prefix, argvs.dbLevel )

	print_message( "Starting GOTTCHA (v%s)" % __version__, argvs.silent, begin_t )

	#dependency check
	if sys.version_info < (3,0):
		sys.exit("[ERROR] Python 3.0 or above is required.")

	if not dependency_check("bwa"):
		sys.exit("[ERROR] Executable bwa not found.")

	#prepare output object
	out_fp = sys.stdout
	outfile = "STDOUT"
	if not argvs.stdout:
		#create output directory if not exists
		if not os.path.exists(argvs.outdir):
			os.makedirs(argvs.outdir)
		ext = "fastq" if argvs.mode == "extract" else "tsv"
		tg_taxid = argvs.taxonomy if argvs.taxonomy else ""
		outfile = "%s/%s.%s%s.%s" % ( argvs.outdir, argvs.prefix, argvs.mode, tg_taxid, ext)
		out_fp = open( outfile, 'w')

	print_message( "Arguments and dependencies checked:", argvs.silent, begin_t )
	print_message( "    Input reads      : %s" % argvs.input,     argvs.silent, begin_t )
	print_message( "    Input SAM file   : %s" % samfile,         argvs.silent, begin_t )
	print_message( "    Database         : %s" % argvs.database,  argvs.silent, begin_t )
	print_message( "    Database level   : %s" % argvs.dbLevel,   argvs.silent, begin_t )
	print_message( "    Mismatch penalty : %s" % argvs.mismatch,  argvs.silent, begin_t )
	print_message( "    Abundance        : %s" % argvs.relAbu,    argvs.silent, begin_t )
	print_message( "    Output path      : %s" % argvs.outdir,    argvs.silent, begin_t )
	print_message( "    Prefix           : %s" % argvs.prefix,    argvs.silent, begin_t )
	print_message( "    Mode             : %s" % argvs.mode,      argvs.silent, begin_t )
	print_message( "    Specific taxid   : %s" % argvs.taxonomy,  argvs.silent, begin_t )
	print_message( "    Threads          : %d" % argvs.threads,   argvs.silent, begin_t )
	print_message( "    Minimal L_DOC    : %s" % argvs.minCov,    argvs.silent, begin_t )
	print_message( "    Minimal L_LEN    : %s" % argvs.minLen,    argvs.silent, begin_t )
	print_message( "    Minimal reads    : %s" % argvs.minReads,  argvs.silent, begin_t )
	print_message( "    BWA path         : %s" % dependency_check("bwa"),       argvs.silent, begin_t )
	#print_message( "    SAMTOOLS path    : %s" % dependency_check("samtools"),  argvs.silent, begin_t )

	#load taxonomy
	print_message( "Loading taxonomy information...", argvs.silent, begin_t )
	gt.loadTaxonomy( argvs.taxInfo )
	print_message( "Done.", argvs.silent, begin_t )

	#load user-defined strain name
	#if os.path.isfile( argvs.database + ".list" ):
	#	print_message( "Loading non-standard groups/strains information...",  argvs.silent, begin_t )
	#	gt.loadStrainName( argvs.database + ".list" )
	#	print_message( "Done",  argvs.silent, begin_t )

	if argvs.input:
		print_message( "Running read-mapping...", argvs.silent, begin_t )
		exitcode, cmd, msg = readMapping( argvs.input, argvs.database, argvs.threads, argvs.mismatch, samfile, logfile )
		print_message( "Logfile saved to %s." % logfile, argvs.silent, begin_t )
		#print_message( "COMMAND: %s" % cmd, argvs.silent, begin_t )

		if exitcode != 0:
			sys.exit( "[%s] ERROR: error occurred while running read mapping (exit: %s, message: %s).\n" % (timeSpend(begin_t), exitcode, msg) )
		else:
			print_message( "Done mapping reads to %s signature database." % argvs.dbLevel, argvs.silent, begin_t )
			print_message( "Mapped SAM file saved to %s." % samfile, argvs.silent, begin_t )
			sam_fp = open( samfile, "r" )

	if argvs.mode == 'class':
		processSAMfileReadClass( sam_fp, out_fp, argvs.dbLevel, argvs.taxonomy )
		print_message( "Done classifying reads. Results printed to %s." % outfile, argvs.silent, begin_t )

	elif argvs.mode == 'extract':
		processSAMfileReadExtract( sam_fp, out_fp, argvs.taxonomy )
		print_message( "Done extracting reads to %s." % outfile, argvs.silent, begin_t )

	else:
		(res, mapped_r_cnt) = processSAMfile( sam_fp, argvs.threads, numlines)
		print_message( "Done processing SAM file. %s reads mapped." % mapped_r_cnt, argvs.silent, begin_t )

		(res_rollup, res_tree) = taxonomyRollUp( res )
		print_message( "Done taxonomy rolling up.", argvs.silent, begin_t )

		if argvs.mode == 'summary' or argvs.mode == 'full':
			outputResultsAsRanks( res_rollup, out_fp, argvs.dbLevel, argvs.relAbu, argvs.mode, argvs.minCov, argvs.minReads, argvs.minLen )
		elif argvs.mode == 'tree':
			outputResultsAsTree( "1", res_tree, res_rollup, "", argvs.dbLevel, 0, argvs.taxonomy, out_fp, argvs.minCov, argvs.minReads, argvs.minLen )
		elif argvs.mode == 'lineage':
			outputResultsAsLineage( res_rollup, out_fp, argvs.dbLevel, argvs.relAbu, argvs.mode, argvs.minCov, argvs.minReads, argvs.minLen )

		print_message( "Done taxonomy profiling; %s results printed to %s." % (argvs.mode, outfile), argvs.silent, begin_t )


import sys

from awx.main.utils.pglock import advisory_lock
from awx.main.models import Instance, InstanceGroup

from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

class InstanceNotFound(Exception):
    def __init__(self, message, changed, *args, **kwargs):
        self.message = message
        self.changed = changed
        super(InstanceNotFound, self).__init__(*args, **kwargs)

class RegisterQueue:
    def __init__(
        self, queuename, instance_percent, inst_min, hostname_list, is_container_group=None, pod_spec_override=None, max_forks=None, max_concurrent_jobs=None
    ):
        self.instance_not_found_err = None
        self.queuename = queuename
        self.instance_percent = instance_percent
        self.instance_min = inst_min
        self.hostname_list = hostname_list
        self.is_container_group = is_container_group
        self.pod_spec_override = pod_spec_override
        self.max_forks = max_forks
        self.max_concurrent_jobs = max_concurrent_jobs

    def get_create_update_instance_group(self):
        created = False
        changed = False
        (ig, created) = InstanceGroup.objects.get_or_create(name=self.queuename)
        if ig.policy_instance_percentage != self.instance_percent:
            ig.policy_instance_percentage = self.instance_percent
            changed = True
        if ig.policy_instance_minimum != self.instance_min:
            ig.policy_instance_minimum = self.instance_min
            changed = True

        if self.is_container_group and (ig.is_container_group != self.is_container_group):
            ig.is_container_group = self.is_container_group
            changed = True

        if self.pod_spec_override and (ig.pod_spec_override != self.pod_spec_override):
            ig.pod_spec_override = self.pod_spec_override
            changed = True

        if self.max_forks and (ig.max_forks != self.max_forks):
            ig.max_forks = self.max_forks
            changed = True

        if self.max_concurrent_jobs and (ig.max_concurrent_jobs != self.max_concurrent_jobs):
            ig.max_concurrent_jobs = self.max_concurrent_jobs
            changed = True

        if changed:
            ig.save()

        return (ig, created, changed)

    def add_instances_to_group(self, ig):
        changed = False

        instance_list_unique = {x for x in (x.strip() for x in self.hostname_list) if x}
        instances = []
        for inst_name in instance_list_unique:
            instance = Instance.objects.filter(hostname=inst_name).exclude(node_type='hop')
            if instance.exists():
                instances.append(instance[0])
            else:
                raise InstanceNotFound("Instance does not exist or cannot run jobs: {}".format(inst_name), changed)

        ig.instances.add(*instances)

        instance_list_before = ig.policy_instance_list
        instance_list_after = instance_list_unique
        new_instances = set(instance_list_after) - set(instance_list_before)
        if new_instances:
            changed = True
            ig.policy_instance_list = ig.policy_instance_list + list(new_instances)
            ig.save()

        return (instances, changed)

    def register(self):
        with advisory_lock('cluster_policy_lock'):
            with transaction.atomic():
                changed2 = False
                (ig, created, changed1) = self.get_create_update_instance_group()
                if created:
                    print("Creating instance group {}".format(ig.name))
                elif not created:
                    print("Instance Group already registered {}".format(ig.name))

                try:
                    (instances, changed2) = self.add_instances_to_group(ig)
                    for i in instances:
                        print("Added instance {} to {}".format(i.hostname, ig.name))
                except InstanceNotFound as e:
                    self.instance_not_found_err = e

        if changed1 or changed2:
            print('(changed: True)')

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument('--queuename', dest='queuename', type=str, help='Queue to create/update')
        parser.add_argument(
            '--hostnames', dest='hostnames', type=str, help='Comma-Delimited Hosts to add to the Queue (will not remove already assigned instances)'
        )
        parser.add_argument(
            '--instance_percent', dest='instance_percent', type=int, default=0, help='The percentage of active instances that will be assigned to this group'
        ),
        parser.add_argument(
            '--instance_minimum',
            dest='instance_minimum',
            type=int,
            default=0,
            help='The minimum number of instance that will be retained for this group from available instances',
        )

    def handle(self, **options):
        queuename = options.get('queuename')
        if not queuename:
            raise CommandError("Specify `--queuename` to use this command.")
        inst_per = options.get('instance_percent')
        instance_min = options.get('instance_minimum')
        hostname_list = []
        if options.get('hostnames'):
            hostname_list = options.get('hostnames').split(",")

        rq = RegisterQueue(queuename, inst_per, instance_min, hostname_list)
        rq.register()
        if rq.instance_not_found_err:
            print(rq.instance_not_found_err.message)
            sys.exit(1)


from django.shortcuts import render
from django.views import View
from apps.core import models as core_models
from rest_framework.views import exception_handler as drf_exception_handler
from rest_framework.response import Response
from rest_framework.renderers import TemplateHTMLRenderer, JSONRenderer
from rest_framework.views import APIView
from rest_framework import permissions,status
from apps.core.utils import validate_data, validate_response ,convert_price_to_string , convert_sex ,convert_tthn, convert_hinhthuc, convert_trinhdo, convert_hocvan, convert_ngoaingu, unique_order_id_generator
from . import models as seeker_models
from . import serializers as seeker_serializers
from apps.seeker.tasks import gui_hoso_mana
from apps.users.models import User
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.auth import logout
from apps.company.models import TinTuyenDung
from django.db.models import Q
from apps.company import models as company_models
from django.core.paginator import Paginator

# Create your views here.

class ListJobViewFull(View):
    def get(self, request):
        nghe_nghiep=core_models.NgheNghiep.objects.all().order_by("name_job")
        thanh_pho=core_models.ThanhPho.objects.all().order_by("name")
        list_full=[]
        if TinTuyenDung.objects.filter(Q(status_tin=1,hinh_thuc_lam_viec=1) | Q(status_tin=1,hinh_thuc_lam_viec=2)).order_by("create_at").exists():
            job_full=TinTuyenDung.objects.filter(Q(status_tin=1,hinh_thuc_lam_viec=1) | Q(status_tin=1,hinh_thuc_lam_viec=2)).order_by("create_at")
            for it in job_full:
                sub_item = {}
                dia_diem=core_models.ThanhPho.objects.get(code=it.dia_diem_lam_viec)
                nghe_nghiep_tin=core_models.NgheNghiep.objects.get(code_job=it.nganh_nghe)
                sub_item['nghe_nghiep'] = nghe_nghiep_tin.name_job
                sub_item['congty'] = it.congty.ten_cong_ty
                sub_item['dia_diem'] = dia_diem.name
                sub_item['muc_luong'] = convert_price_to_string(it.muc_luong)
                sub_item['caption_tin_tuc'] = it.caption_tin_tuc
                sub_item['slug'] = it.slug
                sub_item['ngay_het_han'] = it.ngay_het_han
                list_full.append(sub_item)
        else:
            job_full=None

        if len(list_full) < 10:
            list_full = list_full
        else:
            paginator = Paginator(list_full, 9)
            page_number = request.GET.get('page')
            list_full = paginator.get_page(page_number)

        context={
            'list_nghe_nghiep':nghe_nghiep,
            'list_thanh_pho':thanh_pho,
            'type_page':2,
            'job_full':job_full,
            'list_full':list_full
        }
        return render(request, 'job_detail/job_full.html',context)

class ListJobViewMana(View):
    def get(self, request):
        nghe_nghiep=core_models.NgheNghiep.objects.all().order_by("name_job")
        thanh_pho=core_models.ThanhPho.objects.all().order_by("name")
        list_mana=[]
        if TinTuyenDung.objects.filter(status_tin=1).order_by("create_at").exists():
            job_mana=TinTuyenDung.objects.filter(status_tin=1).order_by("create_at")
            for it in job_mana:
                sub_item = {}
                dia_diem=core_models.ThanhPho.objects.get(code=it.dia_diem_lam_viec)
                nghe_nghiep_tin=core_models.NgheNghiep.objects.get(code_job=it.nganh_nghe)
                sub_item['nghe_nghiep'] = nghe_nghiep_tin.name_job
                sub_item['congty'] = it.congty.ten_cong_ty
                sub_item['dia_diem'] = dia_diem.name
                sub_item['muc_luong'] = convert_price_to_string(it.muc_luong)
                sub_item['caption_tin_tuc'] = it.caption_tin_tuc
                sub_item['slug'] = it.slug
                sub_item['ngay_het_han'] = it.ngay_het_han
                list_mana.append(sub_item)
        else:
            job_mana=None

        if len(list_mana) < 10:
            list_mana = list_mana
        else:
            paginator = Paginator(list_mana, 9)
            page_number = request.GET.get('page')
            list_mana = paginator.get_page(page_number)

        context={
            'list_nghe_nghiep':nghe_nghiep,
            'list_thanh_pho':thanh_pho,
            'type_page':1,
            'job_mana':job_mana,
            'list_mana':list_mana
        }

        return render(request, 'job_detail/job_mana.html',context)

class ListJobViewFree(View):
    def get(self, request):
        nghe_nghiep=core_models.NgheNghiep.objects.all().order_by("name_job")
        thanh_pho=core_models.ThanhPho.objects.all().order_by("name")
        list_free=[]
        if TinTuyenDung.objects.filter(Q(status_tin=1,hinh_thuc_lam_viec=6)).order_by("create_at").exists():
            job_free=TinTuyenDung.objects.filter(Q(status_tin=1,hinh_thuc_lam_viec=6)).order_by("create_at")
            for it in job_free:
                sub_item = {}
                dia_diem=core_models.ThanhPho.objects.get(code=it.dia_diem_lam_viec)
                nghe_nghiep_tin=core_models.NgheNghiep.objects.get(code_job=it.nganh_nghe)
                sub_item['nghe_nghiep'] = nghe_nghiep_tin.name_job
                sub_item['congty'] = it.congty.ten_cong_ty
                sub_item['dia_diem'] = dia_diem.name
                sub_item['muc_luong'] = convert_price_to_string(it.muc_luong)
                sub_item['caption_tin_tuc'] = it.caption_tin_tuc
                sub_item['slug'] = it.slug
                sub_item['ngay_het_han'] = it.ngay_het_han
                list_free.append(sub_item)
        else:
            job_free=None

        if len(list_free) < 10:
            list_free = list_free
        else:
            paginator = Paginator(list_free, 9)
            page_number = request.GET.get('page')
            list_free = paginator.get_page(page_number)

        context={
            'list_nghe_nghiep':nghe_nghiep,
            'list_thanh_pho':thanh_pho,
            'type_page':3,
            'job_free':job_free,
            'list_free':list_free
        }

        return render(request, 'job_detail/job_free.html',context)

class ListJobViewPart(View):
    def get(self, request):
        nghe_nghiep=core_models.NgheNghiep.objects.all().order_by("name_job")
        thanh_pho=core_models.ThanhPho.objects.all().order_by("name")
        list_part=[]
        if TinTuyenDung.objects.filter(Q(status_tin=1,hinh_thuc_lam_viec=3) | Q(status_tin=1,hinh_thuc_lam_viec=4) ).order_by("create_at").exists():
            job_part=TinTuyenDung.objects.filter(Q(status_tin=1,hinh_thuc_lam_viec=3) | Q(status_tin=1,hinh_thuc_lam_viec=4) ).order_by("create_at")
            for it in job_part:
                sub_item = {}
                dia_diem=core_models.ThanhPho.objects.get(code=it.dia_diem_lam_viec)
                nghe_nghiep_tin=core_models.NgheNghiep.objects.get(code_job=it.nganh_nghe)
                sub_item['nghe_nghiep'] = nghe_nghiep_tin.name_job
                sub_item['congty'] = it.congty.ten_cong_ty
                sub_item['dia_diem'] = dia_diem.name
                sub_item['muc_luong'] = convert_price_to_string(it.muc_luong)
                sub_item['caption_tin_tuc'] = it.caption_tin_tuc
                sub_item['slug'] = it.slug
                sub_item['ngay_het_han'] = it.ngay_het_han
                list_part.append(sub_item)
        else:
            job_part=None

        if len(list_part) < 10:
            list_part = list_part
        else:
            paginator = Paginator(list_part, 9)
            page_number = request.GET.get('page')
            list_part = paginator.get_page(page_number)

        context={
            'list_nghe_nghiep':nghe_nghiep,
            'list_thanh_pho':thanh_pho,
            'type_page':4,
            'job_part':job_part,
            'list_part':list_part
        }

        return render(request, 'job_detail/job_part.html',context)

class HoSoView(LoginRequiredMixin,View):
    login_url = '/dang-nhap/'
    def get(self, request):
        nghe_nghiep=core_models.NgheNghiep.objects.all().order_by("name_job")
        thanh_pho=core_models.ThanhPho.objects.all().order_by("name")
        user=User.objects.get(id=request.user.id)
        if user.loai_user == 2:
            logout(request)
            context={'list_nghe_nghiep':nghe_nghiep,'list_thanh_pho':thanh_pho,'user':user}
            return render(request, 'job_detail/yeu_cau_login.html',context)
        else:
            if seeker_models.HoSoUngTuyen.objects.filter(user=request.user).exists():
                hoso=seeker_models.HoSoUngTuyen.objects.get(user=user)
                context={
                    'list_nghe_nghiep':nghe_nghiep,
                    'list_thanh_pho':thanh_pho,
                    'hoso':hoso,
                    'user':user,
                    'gt':convert_sex(user.gender),
                    'tthn':convert_tthn(hoso.hon_nhan),
                    'hoc_van':convert_hocvan(hoso.trinh_do_hoc_van),
                    'hinh_thuc':convert_hinhthuc(hoso.hinh_thuc_lam_viec),
                    'td_nn':convert_trinhdo(hoso.trinh_do_ngoai_ngu),
                    'nn':convert_ngoaingu(hoso.ngoai_ngu)
                }
            else:
                context={
                    'list_nghe_nghiep':nghe_nghiep,
                    'list_thanh_pho':thanh_pho,
                    'user':user,
                }
            return render(request, 'job_detail/hoso.html',context)

class SearchJobView(View):
    def get(self, request):
        nghe_nghiep=core_models.NgheNghiep.objects.all().order_by("name_job")
        thanh_pho=core_models.ThanhPho.objects.all().order_by("name")
        context={
            'list_nghe_nghiep':nghe_nghiep,
            'list_thanh_pho':thanh_pho,
        }
        return render(request, 'job_detail/search.html',context)

class SubmitInfoAccAPI(APIView):
    permission_classes = (permissions.AllowAny, )
    renderer_classes = (JSONRenderer, TemplateHTMLRenderer, )
    def post(self, request, format=None):
        valid_data = validate_data(seeker_serializers.SubmitInfoAccAPISer, request.data)
        ten_day_du = valid_data.get('ten_day_du')
        dien_thoai_lien_he = valid_data.get('dien_thoai_lien_he')
        ngay_sinh = valid_data.get('ngay_sinh')
        dia_chi_lien_he = valid_data.get('dia_chi_lien_he')
        thanh_pho_song = int(valid_data.get('thanh_pho_song'))
        gioi_tinh = int(valid_data.get('gioi_tinh'))
        tinh_trang_hn = int(valid_data.get('tinh_trang_hn'))
        if ten_day_du == "" or dien_thoai_lien_he == "" or ngay_sinh == "" or dia_chi_lien_he == "" or thanh_pho_song == 0:
            return Response(data={"err":2}, status=status.HTTP_200_OK)
        else:
            user = User.objects.get(id=request.user.id)
            if user.loai_user == 1:
                user.fullname=ten_day_du
                user.gender=gioi_tinh
                user.birth_day=ngay_sinh
                user.dia_chi_tinh_thanh_pho=thanh_pho_song
                user.dia_chi_chi_tiet=dia_chi_lien_he
                user.sdt=dien_thoai_lien_he
                user.save()
                if seeker_models.HoSoUngTuyen.objects.filter(user=request.user).exists():
                    hoso=seeker_models.HoSoUngTuyen.objects.get(user=user)
                else:
                    hoso=seeker_models.HoSoUngTuyen.objects.create(user=user)
                hoso.hon_nhan=tinh_trang_hn
                hoso.save()
            return Response(data={"err":1,'gt':convert_sex(gioi_tinh) , 'tthn':convert_tthn(tinh_trang_hn)}, status=status.HTTP_200_OK)

class SubmitInfoNormalAPI(APIView):
    permission_classes = (permissions.AllowAny, )
    renderer_classes = (JSONRenderer, TemplateHTMLRenderer, )
    def post(self, request, format=None):
        valid_data = validate_data(seeker_serializers.SubmitInfoNormalAPISer, request.data)
        vi_tri = valid_data.get('vi_tri')
        nganh_nghe = valid_data.get('nganh_nghe')
        thanh_pho_lv = valid_data.get('thanh_pho_lv')
        trinh_do = valid_data.get('trinh_do')
        nam_kn = valid_data.get('nam_kn')
        hinhthuc_lv = valid_data.get('hinhthuc_lv')
        luong_sal = valid_data.get('luong_sal')

        if vi_tri == "" or nganh_nghe == 0 or thanh_pho_lv == 0 or trinh_do == 0 or nam_kn == 0 or hinhthuc_lv == 0 or luong_sal == "":
            return Response(data={"err":2}, status=status.HTTP_200_OK)
        else:
            user = User.objects.get(id=request.user.id)
            if seeker_models.HoSoUngTuyen.objects.filter(user=request.user).exists():
                hoso=seeker_models.HoSoUngTuyen.objects.get(user=user)
            else:
                hoso=seeker_models.HoSoUngTuyen.objects.create(user=user)
            hoso.nganh_nghe=nganh_nghe
            hoso.vi_tri_mong_muon=vi_tri
            hoso.dia_diem_lam_viec=thanh_pho_lv
            hoso.trinh_do_hoc_van=trinh_do
            hoso.so_nam_kinh_nghiem=nam_kn
            hoso.hinh_thuc_lam_viec=hinhthuc_lv
            hoso.muc_luong_toi_thieu=luong_sal
            hoso.save()
            data={
                "err":1,
                'hoc_van':convert_hocvan(hoso.trinh_do_hoc_van),
                'hinh_thuc':convert_hinhthuc(hoso.hinh_thuc_lam_viec),
                'td_nn':convert_trinhdo(hoso.trinh_do_ngoai_ngu)
            }
            return Response(data=data, status=status.HTTP_200_OK)

class SubmitInfoExpAPI(APIView):
    permission_classes = (permissions.AllowAny, )
    renderer_classes = (JSONRenderer, TemplateHTMLRenderer, )
    def post(self, request, format=None):
        valid_data = validate_data(seeker_serializers.SubmitInfoExpAPISer, request.data)
        cty_lv = valid_data.get('cty_lv')
        bang_cap = valid_data.get('bang_cap')
        dao_tao = valid_data.get('dao_tao')
        chuyen_nganh = valid_data.get('chuyen_nganh')
        bat_dau = valid_data.get('bat_dau')
        ket_thuc = valid_data.get('ket_thuc')
        loai_tot_nghiep = valid_data.get('loai_tot_nghiep')

        if cty_lv == "" or bang_cap == "" or dao_tao == "" or chuyen_nganh == "" or bat_dau == "0" or ket_thuc == "0" or ket_thuc == 0:
            return Response(data={"err":2}, status=status.HTTP_200_OK)
        else:
            user = User.objects.get(id=request.user.id)
            if seeker_models.HoSoUngTuyen.objects.filter(user=request.user).exists():
                hoso=seeker_models.HoSoUngTuyen.objects.get(user=user)
            else:
                hoso=seeker_models.HoSoUngTuyen.objects.create(user=user)
            hoso.ten_cty_tung_lam=cty_lv
            hoso.ten_bang_cap=bang_cap
            hoso.don_vi_dao_tao=dao_tao
            hoso.chuyen_nganh=chuyen_nganh
            hoso.thoi_gian_bat_dau=bat_dau
            hoso.thoi_gian_ket_thuc=ket_thuc
            hoso.loai_tot_nghiep=loai_tot_nghiep
            hoso.save()
            return Response(data={"err":1}, status=status.HTTP_200_OK)

class SubmitInfoNgoaiNguAPI(APIView):
    permission_classes = (permissions.AllowAny, )
    renderer_classes = (JSONRenderer, TemplateHTMLRenderer, )
    def post(self, request, format=None):
        valid_data = validate_data(seeker_serializers.SubmitInfoNgoaiNguAPISer, request.data)
        ngoai_ngu = valid_data.get('ngoai_ngu')
        trinh_do_nn = valid_data.get('trinh_do_nn')
        ky_nang = valid_data.get('ky_nang')
        so_thich = valid_data.get('so_thich')

        if ngoai_ngu == "" or trinh_do_nn == 0 or ky_nang == "" or so_thich == "":
            return Response(data={"err":2}, status=status.HTTP_200_OK)
        else:
            user = User.objects.get(id=request.user.id)
            if seeker_models.HoSoUngTuyen.objects.filter(user=request.user).exists():
                hoso=seeker_models.HoSoUngTuyen.objects.get(user=user)
            else:
                hoso=seeker_models.HoSoUngTuyen.objects.create(user=user)
            hoso.ngoai_ngu=ngoai_ngu
            hoso.trinh_do_ngoai_ngu=trinh_do_nn
            hoso.ky_nang=ky_nang
            hoso.so_thich=so_thich
            hoso.save()
            return Response(data={"err":1,'tdnn':convert_trinhdo(hoso.trinh_do_ngoai_ngu),'nn_lv':convert_ngoaingu(hoso.ngoai_ngu)}, status=status.HTTP_200_OK)

class SubmitDuyetHoSoAPI(APIView):
    permission_classes = (permissions.AllowAny, )
    renderer_classes = (JSONRenderer, TemplateHTMLRenderer, )
    def post(self, request, format=None):
        valid_data = validate_data(seeker_serializers.SubmitDuyetHoSoAPISer, request.data)
        mana = valid_data.get('mana')

        user = User.objects.get(id=request.user.id)
        if seeker_models.HoSoUngTuyen.objects.filter(user=request.user).exists():
            hoso=seeker_models.HoSoUngTuyen.objects.get(user=user)
            # tao ma ho so
            hoso.code_hoso = unique_order_id_generator(request.user.username,1)
            hoso.save()
            gui_hoso_mana.delay(hoso.id)
            return Response(1, status=status.HTTP_200_OK)
        else:
            return Response(1, status=status.HTTP_404_NOT_FOUND)

class DetailJobView(View):
    def get(self, request, slug):
        nghe_nghiep=core_models.NgheNghiep.objects.all().order_by("name_job")
        thanh_pho=core_models.ThanhPho.objects.all().order_by("name")
        user=User.objects.get(id=request.user.id)

        tin_td=company_models.TinTuyenDung.objects.get(slug=slug)
        list_it=[]
        sub_item = {}
        dia_diem=core_models.ThanhPho.objects.get(code=tin_td.dia_diem_lam_viec)
        nghe_nghiep_tin=core_models.NgheNghiep.objects.get(code_job=tin_td.nganh_nghe)
        sub_item['caption_tin_tuc'] = tin_td.caption_tin_tuc
        sub_item['code_tin'] = tin_td.code_tin
        sub_item['congty'] =  tin_td.congty.ten_cong_ty
        sub_item['ngay_het_han'] =  tin_td.ngay_het_han
        sub_item['muc_luong'] = convert_price_to_string(tin_td.muc_luong)
        sub_item['so_nam_kinh_nghiem'] =  tin_td.so_nam_kinh_nghiem
        sub_item['trinh_do_hoc_van'] = convert_hocvan(tin_td.trinh_do_hoc_van)
        sub_item['so_luong_tuyen'] =  tin_td.so_luong_tuyen
        sub_item['nghe_nghiep'] = nghe_nghiep_tin.name_job
        sub_item['dia_diem_lam_viec'] =  dia_diem.name
        sub_item['hinh_thuc_lam_viec'] = convert_hinhthuc(tin_td.hinh_thuc_lam_viec)
        sub_item['ngoai_ngu'] = convert_ngoaingu(tin_td.ngoai_ngu)
        sub_item['trinh_do_ngoai_ngu'] = convert_trinhdo(tin_td.trinh_do_ngoai_ngu)
        sub_item['mota_congviec'] = tin_td.mota_congviec
        sub_item['quyenloi_congviec'] = tin_td.quyenloi_congviec
        sub_item['yeucau_congviec'] = tin_td.yeucau_congviec

        sub_item['dia_chi'] = tin_td.congty.dia_chi
        sub_item['sdt_lienhe'] = tin_td.congty.sdt_lienhe
        sub_item['mail_lien_he'] = tin_td.congty.mail_lien_he
        sub_item['name_lien_he'] = tin_td.congty.name_lien_he

        list_it.append(sub_item)
        context={
            'list_provincial':thanh_pho,
            'list_nghe_nghiep':nghe_nghiep,
            'tin_tuyendung':list_it
        }

        return render(request, 'job_detail/job_detail.html',context)

class FeedbackHoSoMana(APIView):
    permission_classes = (permissions.AllowAny, )
    renderer_classes = (JSONRenderer, TemplateHTMLRenderer, )
    def post(self, request, format=None):
        valid_data = validate_data(seeker_serializers.FeedbackHoSoManaSer, request.data)
        ten_day_du = valid_data.get('ten_day_du')

        return Response(1, status=status.HTTP_200_OK)


from pathlib import Path
from clip.clip import load
from CLIP import CLIP, build_transform
from clip_tokenizer import SimpleTokenizer
import PIL
from PIL import Image
import torch
from torchvision import transforms
import numpy as np
import matplotlib.pyplot as plt
import cv2
import pickle
import types
from dawnet.data.image import show_images

def get_attention_maps(model, visual=True):
    if visual:
        component = model.visual.transformer
    else:   # for text
        component = model.transformer

    attention_layers = []
    for name, module in component.named_modules():
        if hasattr(module, 'attention_masks_out'):
            attention_layers.append(module.attention_masks_out)

    return attention_layers

intermediate = {}
def debug_hook(module, input, output):
    if isinstance(output, PIL.Image.Image):
        return output
    name = str(type(module))
    idx = len(intermediate)
    if idx == 3:
        import pdb; pdb.set_trace()
        print(input[0].sum())
    if isinstance(output, tuple):
        to_log = tuple([each.cpu().data.numpy() for each in output])
    else:
        to_log = output.cpu().data.numpy()
    log = {
        'name': name,
        'output': to_log
    }
    intermediate[idx] = log
    return output

# torch.nn.modules.module.register_module_forward_hook(debug_hook)

if __name__ == '__main__':

    MODEL_PATH = 'clip.pth'
    VOCAB_PATH = 'bpe_simple_vocab_16e6.txt.gz'

    # model = CLIP(attention_probs_dropout_prob=0, hidden_dropout_prob=0)
    # model.load_state_dict(state_dict = torch.load(MODEL_PATH))
    model, transform = load('ViT-B/32', jit=False)

    tokenizer = SimpleTokenizer(
            bpe_path=VOCAB_PATH,
            context_length=model.context_length)
    # transform = build_transform(model.input_resolution.item())
    view_transform = transforms.Compose([
        transforms.Resize(224, interpolation=Image.BICUBIC),
        transforms.CenterCrop(224),
        lambda image: image.convert('RGB')])
    is_fp16 = False

    # device = "cuda" if torch.cuda.is_available() else "cpu"
    device = 'cpu'
    if is_fp16:
        model.to(device=device).eval().half()
    else:
        model.to(device=device).eval().float()

    model.eval()
    # model.register_module_forward_hook(debug_hook)
    with torch.no_grad():
        query = ["a horse", "a human", "an apple", "a tiger", "a cat", "a human and a tiger"]
        # query = ["a photo of a tinca tinca", "a photo of a wombat", "a photo of a restaurant"]
        text = tokenizer.encode(query).to(device)
        text_features = model.encode_text(text)  # N_queries x 512

        # image_path = "/home/john/datasets/imagenet/object_localization/val/n01440764/ILSVRC2012_val_00002138.JPEG"
        image_path = 'images/tiger1.jpg'
        image_name = Path(image_path).stem
        image_vis = np.asarray(view_transform(Image.open(image_path)))
        image = transform(Image.open(image_path)).unsqueeze(0).to(device)
        image_features = model.encode_image(image) # 1 x 512

        visual_attention = get_attention_maps(model, visual=True) #[<n_heads, t, t>]

        for layer_n, each_attention_layer in enumerate(visual_attention):
            layer_images = []
            for idx in range(each_attention_layer.size(0)):
                vis = each_attention_layer[idx, 0, 1:].reshape(7,7).detach().numpy()
                vis -= vis.min()
                vis /= vis.max()
                vis = cv2.resize(vis, (224, 224))[...,np.newaxis]
                result = (vis * image_vis).astype(np.uint8)
                layer_images.append(result)

            output_file = Path(f'logs/{image_name}/layer_{layer_n:02d}.png')
            output_file.parent.mkdir(parents=True, exist_ok=True)
            show_images(layer_images, max_columns=4, show=False, output=str(output_file))

        logits_per_image, logits_per_text = model(image, text)
        probs = logits_per_image.softmax(dim=-1).cpu().numpy()

    print(query)
    print("Label probs:", probs)


from datetime import datetime

from test_wework.api.department import Department

class TestDepartment:
    department = Department()

    def setup_class(self):
        pass

    def test_list(self):
        r = self.department.list("")
        assert r["errcode"] == 0
        assert r["department"][0]["name"] == "海鑫金融"

    def test_create(self):
        name = "影音子部门8" + str(datetime.now().second)

        r = self.department.create(name, 2, 97, 48)
        assert r["errcode"] == 0
        assert r["id"] != None

        # 写法一：
        # exist = False
        # for depart in self.department.list("")["department"]:
        #     if depart["id"] == r["id"]:
        #         exist = True
        # assert exist == True

        #写法二：
        self.department.list("")
        print("&*&*&*&*&*&")
        print("$.department[?(@.id==%s)]" % r["id"])
        assert self.department.jsonpath("$.department[?(@.id==%s)]" % r["id"]) == 46
        assert self.message.jsonpath("$.errcode") == 0
        # print(self.department.jsonpath("$.department"))
        # assert self.department.jsonpath("$.department[?(@.id==%s)]" % r["id"])[0]["name"] == name

    def test_delete(self):
        cre_r = self.department.create("影音子部门5", 2, 96, 25)
        if cre_r["errcode"] == 0:
            del_id = cre_r["id"]
            del_r = self.department.delete(del_id)
        assert del_r["errcode"] == 0

    def test_update(self):
        cre_r = self.department.create("影音子部门7", 2, 95, 27)
        if cre_r["errcode"] == 0:
            upd_id = cre_r["id"]
            update_r = self.department.update(upd_id, "修改后的子部门", 2, 98)
        assert update_r["errcode"] == 0

    def test_update_1(self):
        depart = self.department.list("")['department'][1]
        id = depart['id']
        name = depart['name']
        assert self.department.update_1(id, name=name + '1')['errcode'] == 0
        assert self.department.list(id)['department'][0]['name'] == name + '1'


"""
Code used for the ACROBAT 2023 Grand Challenge
"""

import pathlib
import os
import shutil
from time import time
import argparse
import numpy as np
import pandas as pd
import pyvips

from valis import registration, warp_tools, preprocessing, slide_io
from valis.micro_rigid_registrar import MicroRigidRegistrar

ANON_COL = "anon_id"
PT_COL = "point_id"
SRC_COL = "wsi_source"
TARGET_COL = "wsi_target"
LANDMARK_COL = "landmarks_csv"
DST_DIR_COL = "output_dir_name"

SRC_MMP = "mpp_source"
SRC_X_COL = "x_source"
SRC_Y_COL = "y_source"

TARGET_X_COL = "x_target"
TARGET_Y_COL = "y_target"
TARGET_MMP = "mpp_target"

TARGET_IMG_NAME = "registered_source_image.ome.tiff"
TARGET_LANDMARK_NAME = "registered_landmarks.csv"

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Register images for the 2023 ACROBAT Grand Challenge')
    parser.add_argument('-PATH_TABLE_CSV', type=str, help='csv containing image pairs and landmarks')
    parser.add_argument('-PATH_IMAGES_DIR', type=str, help='csv containing image pairs and landmarks')
    parser.add_argument('-PATH_ANNOT_DIR', type=str, help='csv containing image pairs and landmarks')
    parser.add_argument('-PATH_OUTPUT_DIR', type=str, help='csv containing image pairs and landmarks')
    parser.add_argument('-row_idx', type=int, help='name/row of sample')
    parser.add_argument('-micro_reg_fraction', type=str, help='Size of images used for 2nd registation')
    parser.add_argument('-use_masks', type=str, help='Whether or not to use masks for registration')

    args = vars(parser.parse_args())

    micro_reg_fraction = args["micro_reg_fraction"]
    row_idx = args["row_idx"]
    img_src_dir = args["PATH_IMAGES_DIR"]
    landmark_src_dir = args["PATH_ANNOT_DIR"]
    parent_dst_dir = args["PATH_OUTPUT_DIR"]
    input_table = args["PATH_TABLE_CSV"]
    use_masks = eval(args["use_masks"])

    micro_reg_fraction = eval(micro_reg_fraction)
    input_df = pd.read_csv(input_table)
    sample_row = input_df.iloc[row_idx]

    src_img_f = sample_row[SRC_COL]
    target_img_f = sample_row[TARGET_COL]
    landmarks_f = os.path.join(landmark_src_dir, sample_row[LANDMARK_COL])
    sample_id = str(sample_row[DST_DIR_COL])
    dst_dir = os.path.join(parent_dst_dir, sample_id)

    print(sample_id)
    # Do initial registration, setting the reference image to be the H&E image
    img_list = [os.path.join(img_src_dir, d) for d in [target_img_f, src_img_f]]

    start = time()
    registrar = registration.Valis("./",  os.path.split(dst_dir)[0],
                                   crop="reference",
                                   reference_img_f=target_img_f,
                                   align_to_reference=True,
                                   img_list=img_list,
                                   create_masks=use_masks,
                                   max_processed_image_dim_px=500,
                                   max_non_rigid_registration_dim_px=2000,
                                   micro_rigid_registrar_cls=MicroRigidRegistrar,
                                   name=sample_id)

    rigid_registrar, non_rigid_registrar, error_df = registrar.register(brightfield_processing_cls=preprocessing.StainFlattener,
                                                                        brightfield_processing_kwargs={"adaptive_eq":True})

    # Determine how large of an image to use for micro registration
    img_dims = np.array([slide_obj.slide_dimensions_wh[0] for slide_obj in registrar.slide_dict.values()])
    min_max_size = np.min([np.max(d) for d in img_dims])
    img_areas = [np.multiply(*d) for d in img_dims]
    max_img_w, max_img_h = tuple(img_dims[np.argmax(img_areas)])

    if micro_reg_fraction == 1.0:
        # Full size image
        micro_reg_size = None
    if isinstance(micro_reg_fraction, float):
        micro_reg_size = np.floor(min_max_size*micro_reg_fraction).astype(int)
    else:
        micro_reg_size = micro_reg_fraction
        micro_reg_fraction = micro_reg_fraction/min_max_size

    micro_reg, micro_error = registrar.register_micro(max_non_rigid_registration_dim_px=micro_reg_size,
                                                      reference_img_f=target_img_f,
                                                      align_to_reference=True,
                                                      brightfield_processing_cls=preprocessing.StainFlattener)
    stop = time()
    elapsed = stop - start
    elapsed_min = np.round(elapsed/60, 6)

    # Create benchmarking results
    pathlib.Path(dst_dir).mkdir(exist_ok=True, parents=True)

    non_rigid_error = np.nanmean(micro_error["non_rigid_D"].values)
    rigid_error = np.nanmean(micro_error["rigid_D"].values)
    apply_non_rigid = non_rigid_error < rigid_error
    if not apply_non_rigid:
        print(f"Non-rigid registration didn't improve alignments for {registrar.name}")

    # Read in points
    pt_df = pd.read_csv(landmarks_f)
    src_mpp = pt_df[SRC_MMP].values[0]
    src_xy_um = pt_df[[SRC_X_COL, SRC_Y_COL]].values
    src_xy_px = src_xy_um/src_mpp

    # Warp points
    src_slide = registrar.get_slide(src_img_f)
    ref_slide = registrar.get_slide(target_img_f)
    warped_xy_px = src_slide.warp_xy_from_to(src_xy_px, ref_slide, non_rigid=apply_non_rigid)

    # Clip any points that fell outside of image
    ref_slide_shape_wh = ref_slide.slide_dimensions_wh[0]
    warped_xy_px[:, 0] = np.clip(warped_xy_px[:, 0], 0, ref_slide_shape_wh[0])
    warped_xy_px[:, 1] = np.clip(warped_xy_px[:, 1], 0, ref_slide_shape_wh[1])

    dst_mpp = pt_df[TARGET_MMP].values[0]
    warped_xy_um = warped_xy_px*dst_mpp

    registered_df = pd.DataFrame({ANON_COL: pt_df[ANON_COL].values,
                                  PT_COL: pt_df[PT_COL].values,
                                  TARGET_X_COL: warped_xy_um[:, 0],
                                  TARGET_Y_COL: warped_xy_um[:, 1]})

    pt_f_out = os.path.join(dst_dir, TARGET_LANDMARK_NAME)
    registered_df.to_csv(pt_f_out, index=False)

    # Save registered source image
    s = 1/8
    source_slide = registrar.get_slide(src_img_f)
    warped_slide = source_slide.warp_slide(level=0, non_rigid=apply_non_rigid)
    warped_thumb = warp_tools.rescale_img(warped_slide, s)
    thumb_xyzct = slide_io.get_shape_xyzct((warped_thumb.width, warped_thumb.height), warped_thumb.bands)
    bf_dtype = slide_io.vips2bf_dtype(warped_thumb.format)
    thumb_px_phys_size = (dst_mpp/s, dst_mpp/s, 'µm')

    ome_xml_obj = slide_io.update_xml_for_new_img(source_slide.reader.metadata.original_xml,
                                                  new_xyzct=thumb_xyzct,
                                                  bf_dtype=bf_dtype,
                                                  is_rgb=source_slide.reader.metadata.is_rgb,
                                                  series=source_slide.reader.series,
                                                  pixel_physical_size_xyu=thumb_px_phys_size,
                                                  channel_names=source_slide.reader.metadata.channel_names
                                                  )

    ome_xml_obj.creator = f"pyvips version {pyvips.__version__}"
    ome_xml_str = ome_xml_obj.to_xml()

    dst_f = os.path.join(dst_dir, TARGET_IMG_NAME)
    slide_io.save_ome_tiff(warped_thumb, dst_f, ome_xml_str)

    # Delete output not needed by acrobat
    for f in os.listdir(dst_dir):
        if not f in [TARGET_LANDMARK_NAME, TARGET_IMG_NAME]:
            full_f = os.path.join(dst_dir, f)
            if os.path.isdir(full_f):
                # remove directories
                shutil.rmtree(full_f)
            else:
                # remove files
                os.remove(full_f)

    registration.kill_jvm()


import collections
import numpy as np

class ModelSelector:

    def __init__(self):
        self.history = collections.defaultdict(list)

    def log(self, the_dict):
        for key, value in the_dict.items():
            self.history[key].append(value)
        best_info = {}
        for key in self.history:
            # valid/ms_fv
            # => best-valid/valid_ms_fv
            best_info["best-" + key] = max(self.history[key])
        return best_info

    def is_best_model(self, indicator):
        assert indicator in self.history
        arr = self.history[indicator]
        return np.argmax(arr) == (len(arr) - 1)

    def should_stop(self, indicator, early_stop=10):
        arr = self.history[indicator]
        if len(arr) - 1 - np.argmax(arr) >= early_stop:
            return True
        return False

    def get_best_step_info(self, indictor, print_it=True):
        index = np.argmax(self.history[indictor])
        ans = {}
        if print_it:
            for key in self.history:
                v = self.history[key][index]
                print("%s\t%.4f" % (key, v))
                ans[key] = v
        return ans


import asyncio

from telethon.errors.rpcerrorlist import YouBlockedUserError

from userbot import ALIVE_NAME
from userbot import CMD_HANDLER as cmd
from userbot import CMD_HELP
from userbot.utils import skyzu_cmd

@skyzu_cmd(pattern="logo(?: |$)(.*)")
async def _(event):
    if event.fwd_from:
        return
    aing = await event.client.get_me()
    text = event.pattern_match.group(1)
    if not text:
        await event.edit("`Give a name too!`")
    else:
        await event.edit("`Processing`")
    chat = "@tdtapibot"
    async with event.client.conversation(chat) as conv:
        try:
            msg = await conv.send_message(f"/logo {text}")
            response = await conv.get_response()
            logo = await conv.get_response()
            """ - don't spam notif - """
            await event.client.send_read_acknowledge(conv.chat_id)
        except YouBlockedUserError:
            await event.edit(
                "**Error: Mohon Buka Blokir** @Nastymusiicbot **Dan Coba Lagi!**"
            )
            return
        await asyncio.sleep(0.5)
        await event.client.send_file(
            event.chat_id,
            logo,
            caption=f"Logo by [{ALIVE_NAME}](tg://user?id={aing.id})",
        )
        await event.client.delete_messages(conv.chat_id, [msg.id, response.id, logo.id])
        await event.delete()

CMD_HELP.update(
    {
        "logo": f"**Plugin : **`logo`\
        \n\n  •  **Syntax :** `{cmd}logo` <text>\
        \n  •  **Function : **Membuat logo dari Teks yang diberikan\
    "
    }
)


from flask import Flask, jsonify, abort, request, make_response, url_for
import time
import sys
sys.path.append("Service")
import twitterService
import collectRadius
from flask_httpauth import HTTPBasicAuth
import uuid
import threading

app = Flask(__name__, static_url_path = "")

# Error Handling
@app.errorhandler(400)
def bad_request(error):
    return make_response(jsonify( { 'error': 'Bad request' } ), 400)

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify( { 'error': 'Not found' } ), 404)

# API
'''
### Name: startCollectTweet
### API: /stream/startCollectTweet
### Description:
'''
@app.route('/stream/startCollectTweet', methods = ['POST'])
def startCollectTweet():
    if not request.json or not 'keyWords' in request.json or not 'databaseName' in request.json or not 'ConsumerKey' in request.json or not 'ConsumerSecret' in request.json or not 'AccessToken' in request.json or not 'TokenSecret' in request.json:
        abort(400)
    databaseName = request.json['databaseName']
    key_words = request.json['keyWords']
    consumer_key = request.json['ConsumerKey']
    consumer_secret = request.json['ConsumerSecret']
    access_token = request.json['AccessToken']
    access_token_secret = request.json['TokenSecret']

    return_info = twitterService.startStreamTweet(databaseName,key_words,consumer_key,consumer_secret,access_token,access_token_secret)
    return jsonify( return_info ), 200

'''
### Name: stopCollectTweet
### API: /stream/stopCollectTweet
### Description:
'''
@app.route('/stream/stopCollectTweet', methods = ['POST'])
def stopCollectTweet():
    if not request.json or not 'stream_id' in request.json:
        abort(400)
    stream_id = request.json['stream_id']
    return_info = twitterService.stopStreamTweet(stream_id)
    return jsonify(return_info), 200

'''
### Name: queryStreamList
### API: '/stream/queryStreamList'
### Description: ''
'''
@app.route('/stream/queryStreamList', methods = ['GET'])
def queryStreamList():
    return_info = twitterService.queryStreamList()
    return jsonify(return_info), 200

'''''''''''''''''''''''''''''''''''''''''''''

'''''''''''''''''''''''''''''''''''''''''''''

'''
### Name: startCollectByRadius
### API: '/search/startCollectByRadius'
### Description: ''
'''
australia = '-29.1425,133.1389,2081km'
melbourne = '-37.7867,144.9082,100km'
sydney = '-33.8813,151.2128,100km'
brisbane = '-27.5394,153.1024,100km'
adelaide = '-34.9328,138.6444,100km'
perth = '-32.0379,115.8808,100km'

collecting_thread_information = []
collecting_thread_list = []
@app.route('/search/startCollectByRadius', methods = ['POST'])
def startCollectByRadius():
    if not request.json:
        abort(400)
    provinces = ""
    geocodes = []
    query = ""
    since = ""
    until = ""
    db_name = None
    if(request.json['db_name'] !=None):
        db_name = request.json['db_name']

    if(request.json['provinces'] !=None):
        provinces = request.json['provinces']

    if(request.json['geocodes'] !=None):
        geocodes = request.json['geocodes']

    if(request.json['since'] !=None):
        since = request.json['since']

    if(request.json['until']!=None):
        until = request.json['until']

    if(request.json['query'] !=None):
        query = request.json['query']

    #thread = threading.Thread(args=(query, provinces, geocodes, since, until))
    thread_id = str(uuid.uuid1())
    thread = collectRadius.StoppableThreadForCollectTweet(args=(query, provinces, geocodes, since, until,db_name,thread_id))
    thread_entity = {
        "thread_id":thread_id,
        "thread":thread,
        "information":{
            "provinces":provinces,
            "geocodes":geocodes,
            "since":since,
            "until":until,
            "query":query
        }
    }
    collecting_thread_list.append(thread_entity)

    thread_information = {
        "thread_id":thread_id,
        "information":{
            "provinces":provinces,
            "geocodes":geocodes,
            "since":since,
            "until":until,
            "query":query
        }
    }
    collecting_thread_information.append(thread_information)

    try:
        thread.start()
    except Exception as e:
        return jsonify( {"errorMessage":e, 'isSuccess': False } ), 200

    return jsonify( {"threadEntity":thread_information, 'isSuccess': True } ), 200

'''
### Name: stopCollectByRadius
### API: '/search/stopCollectByRadius'
### Description: ''
'''
@app.route('/search/stopCollectByRadius', methods = ['POST'])
def stopCollectByRadius():
    thread_ids = request.json['ids']
    for thread_id in thread_ids:
        thread_entity = None
        thread_information = None
        for i in range(len(collecting_thread_list)):
            if(collecting_thread_list[i]["thread_id"] == thread_id):
                thread_entity = collecting_thread_list.pop(i)
                thread_information = collecting_thread_information.pop(i)
                break

        if(thread_entity == None):
            return jsonify( {"errorMessage":"Thread not found", 'isSuccess': False } ), 200
        else:
            thread = thread_entity["thread"]
            try:
                thread.stop()
                thread.join()
                return jsonify( {"threadEntity":thread_information, 'isSuccess': False } ), 200
            except Exception as e:
                print(str(e))
                return jsonify( {"threadEntity":thread_information, 'isSuccess': True } ), 200
'''
### Name: collectByRadiusList
### API: '/search/queryCollectingThreadList'
### Description: ''
'''
@app.route('/search/queryCollectingThreadList', methods = ['GET'])
def collectByRadiusList():
    return jsonify( {"threadList":collecting_thread_information, 'isSuccess': True } ), 200

if __name__ == '__main__':
    app.run(port=5001,debug=True,threaded=True,host='0.0.0.0')


import time
from socket import *

import cv2
import numpy

def recv_all(sock, count):
    buf = b''
    while count:
        newbuf = sock.recv(count)
        if not newbuf:
            return None
        buf += newbuf
        count -= len(newbuf)
    return buf

def test():
    host = socket()
    addr = ("localhost", 123)
    print("连接", addr)
    host.connect(addr)

    while True:
        host.send(bytes("i want a video", "utf-8"))
        try:
            tempdata_len = recv_all(host, 16)
            if tempdata_len is None:
                print("结束")
                break
            if len(tempdata_len) == 16:
                print("test", tempdata_len.decode())
                stringData = recv_all(host, int(tempdata_len.decode()))
                data = numpy.frombuffer(stringData, dtype="uint8")
                tmp = cv2.imdecode(data, 1)
                img = cv2.resize(tmp, (320, 240))
                cv2.imshow('Video', img)
                if cv2.waitKey(1) == 27:  # 按下ESC键
                    print("按下 Esc")
                    break
        except KeyboardInterrupt:
            print("Ctrl-C")
            break
        except Exception as e:
            print(e, "传输异常")
            break

    print("断开连接")
    host.close()
    print("关闭窗口")
    cv2.destroyAllWindows()

if __name__ == "__main__":
    test()
    pass


import argparse
import csv
from collections import defaultdict
import re
from typing import Dict

from mappings import GLOBAL_LABEL_MAPPING
from to_label_desc_format import strip_numbering

def compute_perc(counts: defaultdict[str, int], del_none: bool = False) -> Dict[str, float]:
    perc = {}
    if del_none:
        del counts['none']
    total_num = sum(counts.values())
    for label, count in counts.items():
        perc[label] = count / total_num
    return perc

def create_label_str(counts: defaultdict[str, int], perc: Dict[str, float], label: str):
    proc_label = re.sub(r'&', r'and', label)
    return f"{proc_label} & {counts[label]} & {100*perc[label]:.1f}"

def create_label_list(counts: defaultdict[str, int], perc: Dict[str, float], label: str):
    return [label, counts[label], 100*perc[label]]

def main(args: argparse.Namespace) -> None:
    label_sexist_counts = defaultdict(int)
    label_category_counts = defaultdict(int)
    label_vector_counts = defaultdict(int)
    with open(args.input) as fin:
        reader = csv.DictReader(fin)
        for row in reader:
            label_sexist_counts[row['label_sexist']] += 1
            label_category_counts[row['label_category']] += 1
            label_vector_counts[row['label_vector']] += 1
    label_sexist_perc = compute_perc(label_sexist_counts)
    label_category_perc = compute_perc(label_category_counts, del_none=True)
    label_vector_perc = compute_perc(label_vector_counts, del_none=True)

    label_lists = {}
    for label in label_sexist_counts:
        label_lists[label] = create_label_list(label_sexist_counts, label_sexist_perc, label)
    for label in label_category_counts:
        label_lists[label] = create_label_list(label_category_counts, label_category_perc, label)
    for label in label_vector_counts:
        label_lists[label] = create_label_list(label_vector_counts, label_vector_perc, label)
    # convert to latex table
    # header = r"label A & \# & \% & label B & \# & \% & label C & \# & \% \\"
    # empty_label_str = '& & &'
    # hline = r'\hline'
    # hdashline = r'\hdashline'
    # rows = [
    #     f"{hline} \\\\",
    #     f"{label_str['sexist']} & {label_str['1. threats, plans to harm and incitement']} & {label_str['1.1 threats of harm']} \\\\",
    #     f"{empty_label_str} {empty_label_str} {label_str['1.2 incitement and encouragement of harm']} \\\\",
    #     f"{hdashline}",
    #     f"{empty_label_str} {label_str['2. derogation']} & {label_str['2.1 descriptive attacks']} \\\\",
    #     f"{empty_label_str} {empty_label_str} {label_str['2.2 aggressive and emotive attacks']} \\\\",
    #     f"{empty_label_str} {empty_label_str} {label_str['2.3 dehumanising attacks & overt sexual objectification']} \\\\",
    #     f"{hdashline}",
    # ]
    # print(header)
    # for row in rows:
    #     print(row)
    rows = [
        ['label A', '#', '%', 'label B', '#', '%', 'label C', '#', '%'],
        label_lists['sexist'] + label_lists['1. threats, plans to harm and incitement'] + label_lists['1.1 threats of harm'],
        6*['-'] + label_lists['1.2 incitement and encouragement of harm'],
        3*['-'] + label_lists['2. derogation'] + label_lists['2.1 descriptive attacks'],
        6*['-'] + label_lists['2.2 aggressive and emotive attacks'],
        6*['-'] + label_lists['2.3 dehumanising attacks & overt sexual objectification'],
        3*['-'] + label_lists['3. animosity'] + label_lists['3.1 casual use of gendered slurs, profanities, and insults'],
        6*['-'] + label_lists['3.2 immutable gender differences and gender stereotypes'],
        6*['-'] + label_lists['3.3 backhanded gendered compliments'],
        6*['-'] + label_lists['3.4 condescending explanations or unwelcome advice'],
        3*['-'] + label_lists['4. prejudiced discussions'] + label_lists['4.1 supporting mistreatment of individual women'],
        6*['-'] + label_lists['4.2 supporting systemic discrimination against women as a group'],
        label_lists['not sexist'] + 6*['-'],
    ]
    with open(args.output, 'w') as fout:
        writer = csv.writer(fout)
        for row in rows:
            writer.writerow(row)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', help='Path to input file in json format as produced by "compute_corpus_stats.py."')
    parser.add_argument('-o', '--output', help='Path to output csv-file.')
    # parser.add_argument('-c', '--corpus', help='Corpus name.')
    cmd_args = parser.parse_args()
    main(cmd_args)