text:
# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np
import os
from PIL import Image
import imghdr
import base64
from io import BytesIO

import dash
from dash.dependencies import Input,Output,State
import dash_table
import dash_core_components as dcc
import dash_html_components as html
from dash.exceptions import PreventUpdate

# ~ print('data/')
# ~ print('rank.npy')

file_list = []
label_list = []
character_list = []
current_label=None
font_type_dict={}
font_type_list=[]
font_type_list_count_dict={}
only_font_name_list=[]
single_font_name_list=[]

#filename,label,character--->list
for (dir,subs,files) in os.walk('data/'):
	for file in files:
		target=os.path.join(dir,file)
		if os.path.isfile(target):
			if imghdr.what(target)!=None:
				file_list.append(target)
				if os.path.basename(os.path.dirname(target))!=current_label:
					current_label=os.path.basename(os.path.dirname(target))
					label_list.append(os.path.basename(os.path.dirname(target)))
				if len(label_list)<2:
					character_list.append(os.path.splitext(os.path.basename(target))[0].split('_')[1])

file_list.sort()
label_list.sort()
character_list.sort()

#fontname-weight--->font_type_list,fontname--->single_font_name_list
for i in range(len(label_list)):
	if current_label!=label_list[i].split('-')[0]:
		c=0
		current_label=label_list[i].split('-')[0]
		font_type_list_count_dict[current_label]=0
	font_type_list_count_dict[current_label]+=1
	if len(label_list[i].split('-'))>1:
		if c==0:
			font_type_dict[current_label]=[label_list[i].split('-')[1]]
			c=1
		else:
			font_type_dict[current_label].append(label_list[i].split('-')[1])
		if not label_list[i].split('-')[1] in font_type_list:
			font_type_list.append(label_list[i].split('-')[1])
	else:
		single_font_name_list.append(label_list[i])

only_font_name_list=list(font_type_list_count_dict.keys())

# ~ print(len(only_font_name_list))
# ~ print(len(single_font_name_list))

# ~ print('single check')

#display single font name
for i in range(len(single_font_name_list)):
	if not single_font_name_list[i] in only_font_name_list:
		print(i)
		print(single_font_name_list[i])
# ~ print('check over')

current_result_nd=np.load('rank.npy')
# ~ print(current_result_nd)
current_result_df=pd.DataFrame(current_result_nd).rank()
# ~ print(current_result_df)

#image display function
def numpy_to_b64(array):
	im_pil = Image.fromarray(array)
	buff = BytesIO()
	im_pil.save(buff, format="png")
	im_b64 = base64.b64encode(buff.getvalue()).decode("utf-8")
	return im_b64

#Dash app
app = dash.Dash(__name__)

server=app.server

app.layout = html.Div([
	html.A(html.Div('What fonts are similar?',style={'fontSize':30}),href='https://fontcomparisonsystem.herokuapp.com/'),
	html.Br(),
	html.Div('Select Font',style={'fontSize':20}),
	dcc.Dropdown(
		id='labels',
		options=[
			{ 'label': x,'value': x} for x in label_list
		],multi=True,value=[]
	),
	html.Div(id='output-labels',children='labels'),
	html.Br(),
	html.Div('Input Character 0〜9',style={'fontSize':20}),
	html.Div(dcc.Input(id='input-box',type='text')),
	html.Br(),
	html.Div('Display Number',style={'fontSize':20}),
	html.Div(dcc.Input(id='display-box',type='number',value='10')),
	html.Button('Generate Images',id='images-button'),
	html.Div(id='output-container-images'),
])

#labels_number
@app.callback(
	Output('output-labels', 'children'),
	[Input('labels', 'value')])
def update_output_methods(value):
	now_labels=value
	labels_number=len(now_labels)
	return f'Labels:{labels_number}'

#image_generate
generate_count=0
@app.callback(
	Output('output-container-images', 'children'),
	[Input('labels','value'),
	Input('input-box', 'value'),
	Input('display-box','value'),
	Input('images-button', 'n_clicks')])
def path_to_images(compare_labels,input_text,display_number,n_clicks):
	global generate_count
	global current_result_df
	global label_list
	global character_list
	if n_clicks==None:
		raise PreventUpdate
	if generate_count<n_clicks:
		generate_count+=1
		# ~ print(compare_labels)
		# ~ print(input_text)
		# ~ print(display_number)
		# ~ print(n_clicks)
		# ~ print(generate_count)
		temp_df=[]
		ret_list=[]
		flag=''
		display_number=int(display_number)
		if input_text==None:
			return html.Div('Input text, please.',style={'fontSize':20}),
		elif len(list(input_text))==0:
			return html.Div('Input text, please.',style={'fontSize':20}),
		else:
			input_char=list(input_text)
			# ~ print(input_char)
			for h in range(len(input_char)):
				if not input_char[h] in character_list:
					flag='ex_char'
			if flag=='ex_char':
				return html.Div('Input 0〜9, please.',style={'fontSize':20}),
			if len(compare_labels)==0:
				return html.Div('Select font, please.',style={'fontSize':20}),
			else:
				for i in range(len(compare_labels)):
					for j in range(len(input_char)):
						image_temp=Image.open('data/'+f'{compare_labels[i]}/{compare_labels[i]}_{input_char[j]}.png')
						if image_temp.mode!="L":
							image=image_temp.convert("L")
						else:
							image=image_temp
						img=np.array(image)
						if j==0:
							con_img=img
						else:
							con_img=np.hstack((con_img,img))
					ret_list.append(html.Div(f'target{i+1}\'s name :',style={'fontSize':20}),)
					font_name_char_list=list(compare_labels[i].split('-')[0])
					for k in range(len(font_name_char_list)):
						if k==0:
							temp_name=font_name_char_list[k]
							continue
						else:
							if font_name_char_list[k].isupper():
								if not font_name_char_list[k-1].isupper():
									temp_name+='+'
							temp_name+=font_name_char_list[k]
					ret_list.append(html.A(html.Div(f'{compare_labels[i]}',style={'fontSize':20}),href=f'https://fonts.google.com/specimen/{temp_name}'),)
					# ~ print(f'target{i+1}\'s name : {compare_labels[i]} ok')
					ret_list.append(html.Img(src='data:image/png;base64, ' + numpy_to_b64(con_img),style={'height': 'auto','display': 'block','margin': 'auto'}),)
					if i==0:
						temp_df=pd.DataFrame(current_result_df.iloc[:,label_list.index(compare_labels[i])])
					else:
						temp_df+=pd.DataFrame(current_result_df.iloc[:,label_list.index(compare_labels[i])]).values
				temp_df/=len(compare_labels)
				temp_df=temp_df.rank()
				# ~ print(temp_df)
				temp_df.sort_values(temp_df.columns.values[0],inplace=True)
				# ~ print(temp_df)
				font_index_list=temp_df.index.values
				# ~ print(font_index_list)

				count=0
				font_count=0
				for i in range(display_number):
					for j in range(len(input_char)):
						image_temp=Image.open('data/'+f'{label_list[font_index_list[i]]}/{label_list[font_index_list[i]]}_{input_char[j]}.png')
						if image_temp.mode!="L":
							image=image_temp.convert("L")
						else:
							image=image_temp
						img=np.array(image)
						if j==0:
							con_img=img
						else:
							con_img=np.hstack((con_img,img))
					ret_list.append(html.Div('Font\'s name :',style={'fontSize':20}),)
					font_name_char_list=list(label_list[font_index_list[i]].split('-')[0])
					for k in range(len(font_name_char_list)):
						if k==0:
							temp_name=font_name_char_list[k]
							continue
						else:
							if font_name_char_list[k].isupper():
								if not font_name_char_list[k-1].isupper():
									temp_name+='+'
							temp_name+=font_name_char_list[k]
					ret_list.append(html.A(html.Div(f'{label_list[font_index_list[i]]}',style={'fontSize':20}),href=f'https://fonts.google.com/specimen/{temp_name}'),)
					# ~ print(f'Font\'s name : {label_list[font_index_list[i]]} ok')
					ret_list.append(html.Img(src='data:image/png;base64, ' + numpy_to_b64(con_img),style={'height': 'auto','display': 'block','margin': 'auto'}),)
				ret_div=html.Div(ret_list),
				return ret_div
	elif n_clicks==1:
		generate_count=n_clicks-1

if __name__ == '__main__':
	app.run_server(debug=True)

ext: py

max_stars_repo_name: sshhoo/fcs

max_issues_repo_name: sshhoo/fcs

max_forks_repo_name: sshhoo/fcs

reason: import numpy


text:
# Self Attention: Batched Implementation

# Steps
# 1. Create Query, Key, and Value using input vectors.
# 2. Compute attention scores using Query and Key (transpose).
# 3. Convert attention scores to probability distribution using SoftMax.
# 4. Compute weighted values by multiplying attention scores to corresponding values.
# [Q₁ x K₁] * V₁, [Q₁ x K₂] * V₂ … [Q₁ x Kₙ] * Vₙ
# [Q₂ x K₁] * V₁, [Q₂ x K₂] * V₂ … [Q₂ x Kₙ] * Vₙ
# …
# [Qₙ x K₁] * V₁, [Qₙ x K₂] * V₂ … [Qₙ x Kₙ] * Vₙ
#
# Where "x" is the dot product and "*" is the point-wise matrix multiplication.
# Also, Qₙ is defined as-
# Q = [
# [0, 1, 1], # Q₁
# [4, 6, 0], # Q₂
# [2, 3, 1], # Q₃
# ]
# Similarly, Vₙ is a row of Value matrix, and Kₙ is the column of Key Matrix.

# 5. Add-up the weighted values, computed using the scores of a particular query.
# [Q₁ x K₁] * V₁+ [Q₁ x K₂] * V₂ … + [Q₁ x Kₙ] * Vₙ (R₁)
# [Q₂ x K₁] * V₁+ [Q₂ x K₂] * V₂ … + [Q₂ x Kₙ] * Vₙ (R₂)
# …
# [Qₙ x K₁]* V₁+ [Qₙ x K₂]* V₂… + [Qₙ x Kₙ]* Vₙ (Rₙ)

import numpy
import torch
from torch import nn, matmul
from torch.nn.functional import softmax

w_query = torch.tensor(
    [
        [0, 0, 1],
        [1, 1, 0],
        [0, 1, 0],
        [1, 1, 0],
    ], dtype=torch.float32)

w_key = torch.tensor(
    [
        [1, 0, 1],
        [1, 0, 0],
        [0, 1, 0],
        [1, 0, 1],
    ]
    , dtype=torch.float32)

w_value = torch.tensor(
    [
        [1, 0, 1],
        [1, 1, 0],
        [0, 1, 1],
        [0, 0, 1],
    ], dtype=torch.float32)

class SelfAttention(nn.Module):
    def __init__(self, embeddings, model_dim):
        super(SelfAttention, self).__init__()
        self.to_query = nn.Linear(embeddings, model_dim, bias=False)
        self.to_query.weight = nn.Parameter(w_query.t())

        self.to_key = nn.Linear(embeddings, model_dim, bias=False)
        self.to_key.weight = nn.Parameter(w_key.t())

        self.to_value = nn.Linear(embeddings, model_dim, bias=False)
        self.to_value.weight = nn.Parameter(w_value.t())

    def forward(self, inputs):
        # Create Q, K, and V using input vectors
        q = self.to_query(inputs)
        k = self.to_key(inputs)
        v = self.to_value(inputs)
        # Compute Attention scores
        attn_scores = matmul(q, k.transpose(-1, -2))
        # Convert attention scores into probability distributions
        softmax_attn_scores = softmax(attn_scores, dim=-1)

        # Format the values
        # print(v)
        # v_formatted = v[:, None]
        # print(v_formatted)

        # Format the attention scores
        # print(softmax_attn_scores)
        # softmax_attn_scores_transpose = softmax_attn_scores.t()
        # attn_scores_formatted = softmax_attn_scores_transpose[:, :, None]
        # print(attn_scores_formatted)

        #  Compute the final output
        # v_weighted = attn_scores_formatted * v_formatted
        # print(v_weighted)
        # output = v_weighted.sum(dim=0)
        # print(output)
        print(matmul(softmax_attn_scores, v))
        # print(softmax_attn_scores)

x = torch.tensor([[
    [1, 0, 1, 0],  # input 1
    [0, 2, 2, 2],  # input 2
    [1, 1, 1, 1],  # input 3
],
[
    [1, 0, 1, 0],  # input 1
    [0, 2, 2, 2],  # input 2
    [1, 1, 1, 1],  # input 3
]],
dtype=torch.float32)

attn = SelfAttention(4, 3)
attn(x)

# tensor([[1.9100, 3.2405, 3.5752],
#         [2.0000, 3.9999, 4.0000],
#         [2.0000, 3.9865, 3.9932]], grad_fn=<MmBackward>)

max_stars_repo_name: patelrajnath/makeesy-deep-learning

max_issues_repo_name: patelrajnath/makeesy-deep-learning

max_forks_repo_name: patelrajnath/makeesy-deep-learning


text:
#!/usr/bin/env python3
import dash
from dash.dependencies import Input, Output, State
import dash_core_components as dcc
import dash_html_components as html
import plotly.graph_objects as go

import pandas as pd
import numpy as np
import scipy.optimize as spopt
import json

app = dash.Dash(__name__)

server = app.server

world_df = pd.read_csv('data/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv')

def get_label(row):
    label = row.loc['Country/Region']
    state = row.loc['Province/State']
    if pd.notna(state):
        label += f" ({state})"
    return label

with open('data/los_angeles.json') as f:
    data = json.load(f)

new_case_data = data['New Cases'][1:]
indices = data['Date'][1:]
total_dat = []
total = 0
for (i, val) in enumerate(new_case_data):
    total += val
    total_dat.append(total)

indices.extend(['Province/State', 'Country/Region'])
total_dat.extend(['Los Angeles', 'US'])
la_df = pd.DataFrame(data=[total_dat], columns=indices)
world_df = world_df.append(la_df)
world_df.reset_index(inplace=True)

dropdown = []
for (i, row) in world_df.iterrows():
    label = get_label(row)
    dropdown.append({'label': label, 'value': i})

total_fig = go.Figure()
total_fig.update_layout(title='Total Cases', yaxis_title="Cases")

def resid(x, N, t):
    return N - 2**((t - x[1])/x[0])

def get_jac(x, N, t):
    t_sub_t0 = t - x[1]
    exp_part = 2**(t_sub_t0/x[0])
    res = np.array(
            [np.log(2)*t_sub_t0*exp_part/x[0]**2, exp_part*np.log(2)/x[0]])
    return res.transpose()

def get_rate(series):
    t = series.index - pd.to_datetime('1/1/2020')
    t = t.values / pd.Timedelta('1d')
    result = spopt.least_squares(
            resid, [3, t[0]], jac=get_jac,
            kwargs={'N': series, 't': t})
    return 1/result.x[0]

rate_fig = go.Figure()
rate_fig.update_layout(
        title='Exponential Growth: 3 Day Fit',
        yaxis_title=r'Doubling Growth Factor',
        showlegend=True)

rate_fig_7 = go.Figure()
rate_fig_7.update_layout(
        title='7 Day Fit',
        yaxis_title=r'Exponential Growth: Doubling Growth Factor',
        showlegend=True)

app.layout = html.Div([
    html.H2('Coronalyzer'),
    html.Div([
        dcc.Dropdown(
            id='dropdown',
            options=dropdown,
            value=[226, 137, 62, 238],
            multi=True)
        ]),
    dcc.Graph(
        id='total-cases-1',
        figure=total_fig
    ),
    dcc.Graph(
        id='rate',
        figure=rate_fig
    ),
    dcc.Graph(
        id='rate7',
        figure=rate_fig_7
    ),
    html.Div(id='output')
])

def row_to_series(row_df):
    series = row_df[5:]
    series.index = pd.DatetimeIndex(series.index)
    series = series[series > 0]
    return series

def create_fig_data(row):
    row_df = world_df.loc[row]
    series = row_to_series(row_df)
    return dict(x=series.index, y=series,
                type='scatter', name=get_label(row_df))

def create_rate_data(row, window):
    row_df = world_df.loc[row]
    series = row_to_series(row_df)
    series = series.rolling(window).apply(get_rate)
    return dict(x=series.index, y=series,
                type='scatter', name=get_label(row_df))

@app.callback(
        Output('total-cases-1', 'figure'),
        [Input('dropdown', 'value')],
        [State('total-cases-1', 'figure')])
def update_output(values, fig):
    data = []
    for value in values:
        data.append(create_fig_data(value))

    fig['data'] = data
    return fig

@app.callback(
        Output('rate', 'figure'),
        [Input('dropdown', 'value')],
        [State('rate', 'figure')])
def update_rate(values, fig):
    data = []
    for value in values:
        data.append(create_rate_data(value, '3d'))
    fig['data'] = data
    return fig

@app.callback(
        Output('rate7', 'figure'),
        [Input('dropdown', 'value')],
        [State('rate7', 'figure')])
def update_rate7(values, fig):
    data = []
    for value in values:
        data.append(create_rate_data(value, '7d'))

    fig['data'] = data
    return fig

if __name__ == '__main__':
    app.run_server(debug=True)

max_stars_repo_name: joelfrederico/coronalyzer

max_issues_repo_name: joelfrederico/coronalyzer

max_forks_repo_name: joelfrederico/coronalyzer

reason: import numpy,import scipy


text:
# import the necessary packages
from imutils import paths
import numpy as np
import argparse
import imutils
import pickle
import cv2
import os
from collections import Counter
from sklearn.preprocessing import LabelEncoder
from sklearn.svm import SVC
from imutils.video import VideoStream
from imutils.video import FPS
import time
from tkinter import Label, Frame, Button, Entry, Tk, X
from tkinter import messagebox
import pandas as pd

ARIAL = ("arial", 10, "bold")

class FacialUi:
    def __init__(self, root):
        self.root = root
        self.header = Label(self.root,
                            text="Facial Recognition",
                            bg="#0019fc",
                            fg="white",
                            font=("arial", 20, "bold"))
        self.header.pack(fill=X)
        self.frame = Frame(self.root, bg="#0019fc", width=800, height=500)
        root.geometry("800x500")
        self.button1 = Button(self.frame,
                              text=("Facial Recognition."
                                    "Click to begin transactions"),
                              bg="#50A8B0",
                              fg="white",
                              font=ARIAL,
                              command=self.begin_page)
        self.q = Button(self.frame,
                        text="Quit",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.root.destroy)
        self.q.place(x=275, y=340, width=200, height=40)
        self.button1.place(x=155, y=230, width=500, height=30)
        self.frame.pack()
        self.countter = 2

    def begin_page(self):
        self.frame.destroy()
        self.frame = Frame(self.root, bg="#0019fc", width=800, height=500)
        root.geometry("800x500")
        self.enroll = Button(self.frame,
                             text="Enroll",
                             bg="#50A8B0",
                             fg="white",
                             font=ARIAL,
                             command=self.enroll_user)
        self.verify = Button(self.frame,
                             text="Verify Page",
                             bg="#50A8B0",
                             fg="white",
                             font=ARIAL,
                             command=self.verify_page)
        self.q = Button(self.frame,
                        text="Quit",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.root.destroy)
        self.enroll.place(x=0, y=315, width=200, height=50)
        self.verify.place(x=600, y=315, width=200, height=50)
        self.q.place(x=340, y=340, width=120, height=20)
        self.frame.pack()

    def verify_page(self):
        self.frame.destroy()
        self.frame = Frame(self.root, bg="#0019fc", width=800, height=500)
        # Login Page Form Components
        self.label1 = Label(self.frame,
                            text="Note:",
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label2 = Label(self.frame,
                            text=("1.By clicking on the 'Verify Face Id' "
                                  " button, we proceed to perform facial "
                                  "recognition."),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label3 = Label(self.frame,
                            text=("2.Each capture will take 15 seconds, "
                                  "you are required to move your face in "
                                  "different directions while being "
                                  "captured."),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label4 = Label(self.frame,
                            text=("6.To begin, click the"
                                  "'Verify Face Id' button below"),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.button = Button(self.frame,
                             text="Verify Face Id",
                             bg="#50A8B0",
                             fg="white",
                             font=ARIAL,
                             command=self.video_check)
        self.q = Button(self.frame,
                        text="Quit",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.root.destroy)
        self.b = Button(self.frame,
                        text="Back",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.begin_page)
        self.label1.place(x=100, y=100, width=600, height=20)
        self.label2.place(x=100, y=120, width=600, height=20)
        self.label3.place(x=100, y=140, width=600, height=20)
        self.label4.place(x=100, y=160, width=600, height=20)
        self.button.place(x=100, y=250, width=600, height=30)
        self.q.place(x=480, y=360, width=120, height=20)
        self.b.place(x=280, y=360, width=120, height=20)
        self.frame.pack()

    def enroll_user(self):
        self.frame.destroy()
        self.frame = Frame(self.root,
                           bg="#0019fc",
                           width=800,
                           height=500)
        # Login Page Form Components
        self.userlabel = Label(self.frame,
                               text="Full Name",
                               bg="#0019fc",
                               fg="white",
                               font=ARIAL)
        self.uentry = Entry(self.frame,
                            bg="honeydew",
                            highlightcolor="#50A8B0",
                            highlightthickness=2,
                            highlightbackground="white")
        self.button1 = Button(self.frame,
                              text="Click to enroll",
                              bg="#50A8B0",
                              fg="white",
                              font=ARIAL,
                              command=self.write_to_csv)
        self.button2 = Button(self.frame,
                              text=("Click to go to video capture"
                                    "after enrolling"),
                              bg="#50A8B0",
                              fg="white",
                              font=ARIAL,
                              command=self.video_capture_page)
        self.q = Button(self.frame,
                        text="Quit",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.root.destroy)
        self.b = Button(self.frame,
                        text="Back",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.begin_page)
        self.userlabel.place(x=125, y=100, width=120, height=20)
        self.uentry.place(x=153, y=130, width=200, height=20)
        self.button1.place(x=155, y=230, width=180, height=30)
        self.button2.place(x=355, y=230, width=350, height=30)
        self.q.place(x=480, y=360, width=120, height=20)
        self.b.place(x=280, y=360, width=120, height=20)
        self.frame.pack()

    def write_to_csv(self):
        import csv
        from random import randint
        n = 10; range_start = 10**(n-1); range_end = (10**n)-1
        n = 5; range_start = 10**(n-1); range_end = (10**n)-1
        unique_id = randint(range_start, range_end)
        name = self.uentry.get()
        with open(r'user_details.csv', 'a', newline='\n') as f:
            writer = csv.writer(f)
            writer.writerow([unique_id, name])
        messagebox._show("Enrollment Info!", "Successfully Enrolled!")

    def video_capture_page(self):
        self.frame.destroy()
        self.frame = Frame(self.root,
                           bg="#0019fc",
                           width=800,
                           height=500)
        # Login Page Form Components
        self.label1 = Label(self.frame,
                            text="Note:",
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label2 = Label(self.frame,
                            text=("1.By clicking on the 'Capture' "
                                  "button below, your image gets captured"),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label3 = Label(self.frame,
                            text=("2.You will be required to capture "
                                  "5 images for full registration"),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label4 = Label(self.frame,
                            text=("3.To capture each image click the "
                                  "space bar on your keyboard when the "
                                  "camera turn on:"),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label5 = Label(self.frame,
                            text=("4. Please wait till you are notified that "
                                  "your capture was successful before "
                                  "leaving the page"),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.label6 = Label(self.frame,
                            text=("5.To begin, click the 'Capture' button "
                                  "below and click the space bar to capture "
                                  "a new image"),
                            bg="#0019fc",
                            fg="white",
                            font=ARIAL)
        self.button = Button(self.frame,
                             text="Capture",
                             bg="#50A8B0",
                             fg="white",
                             font=ARIAL,
                             command=self.captureuser)
        self.q = Button(self.frame,
                        text="Quit",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.root.destroy)
        self.b = Button(self.frame,
                        text="Back",
                        bg="#50A8B0",
                        fg="white",
                        font=ARIAL,
                        command=self.enroll_user)
        self.label1.place(x=100, y=100, width=600, height=20)
        self.label2.place(x=100, y=120, width=600, height=20)
        self.label3.place(x=100, y=140, width=600, height=20)
        self.label4.place(x=100, y=160, width=600, height=20)
        self.label5.place(x=100, y=180, width=600, height=20)
        self.label6.place(x=100, y=200, width=600, height=20)
        self.button.place(x=100, y=230, width=600, height=30)
        self.q.place(x=480, y=360, width=120, height=20)
        self.b.place(x=280, y=360, width=120, height=20)
        self.frame.pack()

    # Hit space bar to capture
    def captureuser(self):
        data = pd.read_csv('user_details.csv')
        name = data.loc[:, 'unique_id'].values[-1]
        cam = cv2.VideoCapture(0)

        cv2.namedWindow("capture")

        img_counter = 0

        dirname = f'dataset/{name}'
        os.mkdir(dirname)

        while True:
            ret, frame = cam.read()
            cv2.imshow("capture", frame)

            if img_counter == 5:
                cv2.destroyWindow("capture")
                break
            if not ret:
                break
            k = cv2.waitKey(1)

            if k % 256 == 27:
                # ESC pressed
                print("Escape hit, closing...")
                break
            elif k % 256 == 32:
                # SPACE pressed
                path = f'dataset/{name}'
                img_name = "{}.jpg".format(img_counter)
                cv2.imwrite(os.path.join(path, img_name), frame)
                print("{} written!".format(img_name))
                img_counter += 1

        cam.release()

        cv2.destroyAllWindows()

        self.get_embeddings()
        self.train_model()
        messagebox._show("Registration Info!",
                         "Face Id Successfully Registered!")

    def get_embeddings(self):
        # summary:
        # construct the argument parser and parse the arguments
        ap = argparse.ArgumentParser()
        ap.add_argument("-i", "--dataset",
                        required=True,
                        help="path to input directory of faces + images")
        ap.add_argument("-e", "--embeddings", required=True,
                        help=("path to output serialized db of facial "
                              "embeddings"))
        ap.add_argument("-d", "--detector", required=True,
                        help="path to OpenCV's deep learning face detector")
        ap.add_argument("-m", "--embedding-model", required=True,
                        help=("path to OpenCV's deep learning "
                              "face embedding model"))
        ap.add_argument("-c", "--confidence", type=float, default=0.5,
                        help="minimum probability to filter weak detections")
        # args = vars(ap.parse_args())

        # load our serialized face detector from disk
        print("[INFO] loading face detector...")

        detector = (cv2
                    .dnn
                    .readNetFromCaffe('face_detection_model/deploy.prototxt',
                                      ('face_detection_model/res10_300x300_ss'
                                       'd_iter_140000.caffemodel')))
        # load our serialized face embedding model from disk
        embedder = cv2.dnn.readNetFromTorch('openface_nn4.small2.v1.t7')

        # grab the paths to the input images in our dataset
        print("[INFO] quantifying faces...")
        imagePaths = list(paths.list_images('dataset'))
        # initialize our lists of extracted facial embeddings and
        # corresponding people names
        knownEmbeddings = []
        knownNames = []
        # initialize the total number of faces processed
        total = 0
        # loop over the image paths
        for (i, imagePath) in enumerate(imagePaths):
            # extract the person name from the image path
            print("[INFO] processing image {}/{}".format(i + 1,
                                                         len(imagePaths)))
            name = imagePath.split(os.path.sep)[-2]

            # load the image, resize it to have a width of 600 pixels (while
            # maintaining the aspect ratio), and then grab the image
            # dimensions
            image = cv2.imread(imagePath)
            image = imutils.resize(image, width=600)
            (h, w) = image.shape[:2]
            # construct a blob from the image
            imageBlob = cv2.dnn.blobFromImage(
                cv2.resize(image, (300, 300)), 1.0, (300, 300),
                (104.0, 177.0, 123.0), swapRB=False, crop=False)

            # apply OpenCV's deep learning-based face detector to localize
            # faces in the input image
            detector.setInput(imageBlob)
            detections = detector.forward()

            # ensure at least one face was found
            if len(detections) > 0:
                # we're making the assumption that each image has only ONE
                # face, so find the bounding box with the largest probability
                i = np.argmax(detections[0, 0, :, 2])
                confidence = detections[0, 0, i, 2]

                # ensure that the detection with the largest probability also
                # means our minimum probability test (thus helping filter out
                # weak detections)
                if confidence > 0.5:
                    # compute the (x, y)-coordinates of the bounding box for
                    # the face
                    box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
                    (startX, startY, endX, endY) = box.astype("int")

                    # extract the face ROI and grab the ROI dimensions
                    face = image[startY:endY, startX:endX]
                    (fH, fW) = face.shape[:2]

                    # ensure the face width and height are sufficiently large
                    if fW < 20 or fH < 20:
                        continue

                    # construct a blob for the face ROI, then pass the blob
                    # through our face embedding model to obtain the 128-d
                    # quantification of the face
                    faceBlob = cv2.dnn.blobFromImage(face,
                                                     1.0 / 255,
                                                     (96, 96),
                                                     (0, 0, 0),
                                                     swapRB=True,
                                                     crop=False)
                    embedder.setInput(faceBlob)
                    vec = embedder.forward()

                    # add the name of the person + corresponding face
                    # embedding to their respective lists
                    knownNames.append(name)
                    knownEmbeddings.append(vec.flatten())
                    total += 1
        # dump the facial embeddings + names to disk
        print("[INFO] serializing {} encodings...".format(total))
        data = {"embeddings": knownEmbeddings, "names": knownNames}
        f = open('output/embeddings.pickle', "wb")
        f.write(pickle.dumps(data))
        f.close()

    def train_model(self):
        # summary
        print("[INFO] loading face embeddings...")
        data = pickle.loads(open('output/embeddings.pickle', "rb").read())
        le = LabelEncoder()
        labels = le.fit_transform(data["names"])
        # train the model used to accept the 128-d embeddings of the face and
        # then produce the actual face recognition
        print("[INFO] training model...")
        recognizer = SVC(C=1.0, kernel="linear", probability=True)
        recognizer.fit(data["embeddings"], labels)
        # write the actual face recognition model to disk
        f = open('output/recognizer.pickle', "wb")
        f.write(pickle.dumps(recognizer))
        f.close()

        # write the label encoder to disk
        f = open('output/le.pickle', "wb")
        f.write(pickle.dumps(le))
        f.close()
        print("[INFO] done training model.")

    def video_check(self):
        detector = cv2.dnn.readNetFromCaffe('face_detection_model/'
                                            'deploy.prototxt',
                                            ('face_detection_model/res10_300x3'
                                             '00_ssd_iter_140000.caffemodel'))
        # summary
        # load our serialized face embedding model from disk
        print("[INFO] loading face recognizer...")
        embedder = cv2.dnn.readNetFromTorch('openface_nn4.small2.v1.t7')

        # load the actual face recognition model along with the label encoder
        recognizer = (pickle.loads(open('output/recognizer.pickle', "rb")
                                   .read()))
        le = pickle.loads(open('output/le.pickle', "rb").read())

        # initialize the video stream, then allow the camera sensor to warm up
        print("[INFO] starting video stream...")
        vs = VideoStream(src=0).start()
        time.sleep(2.0)

        # run check for only 15seconds and then stop
        timeout = time.time() + 15

        # start the FPS throughput estimator
        fps = FPS().start()

        # loop over frames from the video file stream
        real_user_list = []
        while True:
            # run check for only 15seconds and then stop
            if time.time() > timeout:
                cv2.destroyWindow("Frame")
                break

            # grab the frame from the threaded video stream
            frame = vs.read()

            # resize the frame to have a width of 600 pixels (while
            # maintaining the aspect ratio), and then grab the image
            # dimensions
            frame = imutils.resize(frame, width=600)
            (h, w) = frame.shape[:2]

            # construct a blob from the image
            imageBlob = cv2.dnn.blobFromImage(
                cv2.resize(frame, (300, 300)), 1.0, (300, 300),
                (104.0, 177.0, 123.0), swapRB=False, crop=False)

            # apply OpenCV's deep learning-based face detector to localize
            # faces in the input image
            detector.setInput(imageBlob)
            detections = detector.forward()

            # TODO: if 2 faces are detected alert the user of a warning
            # loop over the detections
            for i in range(0, detections.shape[2]):
                # extract the confidence (i.e., probability) associated with
                # the prediction
                confidence = detections[0, 0, i, 2]

                # filter out weak detections
                if confidence > 0.5:
                    # compute the (x, y)-coordinates of the bounding box for
                    # the face
                    box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
                    (startX, startY, endX, endY) = box.astype("int")

                    # extract the face ROI
                    face = frame[startY:endY, startX:endX]
                    (fH, fW) = face.shape[:2]

                    # ensure the face width and height are sufficiently large
                    if fW < 20 or fH < 20:
                        continue

                    # construct a blob for the face ROI, then pass the blob
                    # through our face embedding model to obtain the 128-d
                    # quantification of the face
                    faceBlob = cv2.dnn.blobFromImage(face,
                                                     1.0 / 255,
                                                     (96, 96),
                                                     (0, 0, 0),
                                                     swapRB=True,
                                                     crop=False)
                    embedder.setInput(faceBlob)
                    vec = embedder.forward()

                    # perform classification to recognize the face
                    preds = recognizer.predict_proba(vec)[0]
                    j = np.argmax(preds)
                    proba = preds[j]
                    name = le.classes_[j]

                    # draw the bounding box of the face along with the
                    # associated probability
                    text = "{}: {:.2f}%".format(name, proba * 100)
                    y = startY - 10 if startY - 10 > 10 else startY + 10
                    cv2.rectangle(frame, (startX, startY), (endX, endY),
                                  (0, 0, 255), 2)
                    cv2.putText(frame, text,
                                (startX, y),
                                cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2)
                    # TODO: Handle if 2 faces are given.
                    # Decision boundary
                    if (name == 'unknown') or (proba * 100) < 20:
                        real_user_list.append('0000')
                        print("New Member")
                    else:
                        real_user_list.append(name)
                        cv2.destroyWindow("Frame")
                        break

            # update the FPS counter
            fps.update()

            # show the output frame
            cv2.imshow("Frame", frame)
            key = cv2.waitKey(1) & 0xFF

            # if the `q` key was pressed, break from the loop
            if key == ord("q"):
                break

        # stop the timer and display FPS information
        fps.stop()
        print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
        print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

        # do a bit of cleanup
        vs.stop()
        cv2.destroyAllWindows()
        try:
            Counter(real_user_list).most_common(1)[0][0] == 'unknown'
        except IndexError:
            if self.countter != 0:
                messagebox._show("Verification Info!",
                                 "Face Id match failed! You "
                                 f"have {self.countter} trials left"
                                 )
                self.countter = self.countter - 1
                self.video_check()
            else:
                messagebox._show("Verification Info!", "Could Not View User")
                self.begin_page()
                self.countter = 2

        else:
            if Counter(real_user_list).most_common(1)[0][0] == '0000':
                if self.countter != 0:
                    messagebox._show("Verification Info!",
                                     "Face Id match failed! You "
                                     f"have {self.countter} trials left")
                    self.countter = self.countter - 1
                    self.video_check()
                else:
                    messagebox._show("Verification Info!", "New Member")
                    self.begin_page()
                    self.countter = 2

            else:
                self.real_user = int(Counter(real_user_list)
                                     .most_common(1)[0][0])
                data = pd.read_csv('user_details.csv')
                self.username = (data[data.loc[:, 'unique_id']
                                 ==
                                 self.real_user].loc[:, 'name'].values[0])
                messagebox._show("Verification Info!",
                                 f"Hello {self.realuser}")

root = Tk()
root.title("Facial Recognition")
root.geometry("600x420")
root.configure(bg="red")
obj = FacialUi(root)
root.mainloop()

max_stars_repo_name: Andyphied/CV-Facial-Recongnition

max_issues_repo_name: Andyphied/CV-Facial-Recongnition

max_forks_repo_name: Andyphied/CV-Facial-Recongnition


text:
from typing import Any, List, Optional
import json
import ntpath
import os
import shutil

import joblib
import numpy as np
import pandas as pd
from tinytag import TinyTag
from tqdm import tqdm

from pyutils.core.utils import get_timetaken_fstring
from pyutils.data_wrangler.transform import rank_and_sort

def get_extension(filepath: str) -> str:
    return os.path.splitext(filepath)[-1][1:]

def get_basename_from_filepath(filepath: str) -> str:
    """Returns base-name of the file/folder from the given `filepath` (along with the extension, if any)"""
    head, tail = ntpath.split(p=filepath)
    return tail or ntpath.basename(head)

def get_absolute_filepath(filepath: str) -> str:
    """Returns absolute filepath of the file/folder from the given `filepath` (along with the extension, if any)"""
    absolute_filepath = os.path.realpath(path=filepath)
    return absolute_filepath

def get_line_count(filepath: str) -> int:
    """Returns count of number of lines in the given file"""
    num_lines = sum(1 for _ in open(file=filepath, encoding="utf8"))
    return num_lines

def filter_filepaths_by_extensions(
        filepaths: List[str],
        extensions: List[str],
    ) -> List[str]:
    """
    Filters given filepaths by the desired extensions.

    >>> filter_filepaths_by_extensions(
        filepaths=['one.js', 'two.py', 'three.css', 'four.go', 'five.html', 'six.py', 'seven.js'],
        extensions=['css', 'js'],
    )
    """
    extensions = list(
        map(lambda extension: extension.strip().lower(), extensions)
    )
    filepaths_needed = list(
        filter(lambda filepath: get_extension(filepath=filepath).strip().lower() in extensions, filepaths)
    )
    return filepaths_needed

def get_unique_extensions(filepaths: List[str]) -> List[str]:
    """Returns all unique extensions available in the list of filepaths given"""
    all_extensions = list(map(get_extension, filepaths))
    unique_extensions = sorted(list(set(all_extensions)))
    return unique_extensions

def __get_filepaths_at_first_level(src_dir: str) -> List[str]:
    folders_and_files_in_directory = os.listdir(src_dir)
    folders_and_files_in_directory = list(
        map(lambda folder_or_file: os.path.join(src_dir, folder_or_file), folders_and_files_in_directory)
    )
    files_in_directory = list(
        filter(lambda folder_or_file: os.path.isfile(folder_or_file), folders_and_files_in_directory)
    )
    return files_in_directory

def __get_filepaths_at_all_levels(src_dir: str) -> List[str]:
    filepaths = []
    for path, _, filenames in os.walk(src_dir):
        for filename in filenames:
            filepath = os.path.join(path, filename)
            filepaths.append(filepath)
    return filepaths

def get_filepaths(
        src_dir: str,
        depth: str,
        extensions: Optional[List[str]] = None,
    ) -> List[str]:
    """
    Gets list of all filepaths (of files) from source directory.

    Parameters:
        - src_dir (str): Filepath to the source directory. Can be absolute or relative.
        - depth (str): Options: ['all_levels', 'first_level'].
        Set to 'all_levels' if you want to get filepaths from all sub-directories (if any) in the given source directory.
        Set to 'first_level' if you want to get filepaths only from the first directory given.
        - extensions (list): List of extensions to filter the filepaths by (optional).

    >>> get_filepaths(
            src_dir="SOME_SOURCE_DIR",
            depth='all_levels',
            extensions=['csv', 'xlsx'],
        )
    """
    depth_options = ['all_levels', 'first_level']
    if depth not in depth_options:
        raise ValueError(f"Expected `depth` to be in {depth_options}, but got '{depth}'")

    if depth == 'all_levels':
        filepaths = __get_filepaths_at_all_levels(src_dir=src_dir)
    elif depth == 'first_level':
        filepaths = __get_filepaths_at_first_level(src_dir=src_dir)
    if extensions is not None:
        filepaths = filter_filepaths_by_extensions(filepaths=filepaths, extensions=extensions)
    return filepaths

def create_archive_file(
        src_dir: str,
        archive_format: str,
    ) -> None:
    """
    Creates archive file of the given source directory.
    Options for `archive_format` are: ['zip', 'tar', 'gztar', 'bztar', 'xztar'].
    """
    absolute_path_to_src_dir = get_absolute_filepath(filepath=src_dir)
    basename_of_src_dir = get_basename_from_filepath(filepath=src_dir)
    shutil.make_archive(
        base_name=basename_of_src_dir,
        format=archive_format,
        root_dir=absolute_path_to_src_dir,
    )
    return None

def get_line_count_info(
        filepaths: List[str],
        rank_by_line_count: Optional[bool] = False,
    ) -> pd.DataFrame:
    """
    Gets line-count of all filepaths given.
    Returns DataFrame having columns: ['Filepath', 'LineCount', 'Basename', 'Extension'].
    If `rank_by_line_count` is set to True, then there will be an additional column called: 'RankByLineCount'.

    >>> get_line_count_info(
            filepaths=['file1.txt', 'file2.txt', 'file3.txt'],
            rank_by_line_count=True,
        )
    """
    df = pd.DataFrame()
    df['Filepath'] = filepaths
    df['LineCount'] = df['Filepath'].apply(get_line_count)
    df['Basename'] = df['Filepath'].apply(get_basename_from_filepath)
    df['Extension'] = df['Filepath'].apply(get_extension)
    if rank_by_line_count:
        df = rank_and_sort(
            data=df,
            rank_column_name='RankByLineCount',
            rank_by=['LineCount'],
            ascending=[False],
            how='dense_rank',
        )
    return df

def get_media_metadata(filepaths: List[str]) -> pd.DataFrame:
    """
    Gets metadata of all media-file filepaths given i.e; files with extensions mp3, mp4, etc.
    Returns DataFrame having columns: ['Filepath', 'Basename', 'Extension', 'BitRate',
    'DurationInSeconds', 'Duration', 'SizeInMb', 'SampleRate'].

    >>> get_media_metadata(filepaths=['audio.mp3', 'audio.m4a', 'video.mp4'])
    """
    bytes_per_mb = 1048576
    df = pd.DataFrame()
    df['TinyTagObject'] = [TinyTag.get(filepath) for filepath in tqdm(filepaths)]
    df['Filepath'] = filepaths
    df['Basename'] = df['Filepath'].apply(get_basename_from_filepath)
    df['Extension'] = df['Filepath'].apply(get_extension)
    df['BitRate'] = df['TinyTagObject'].apply(lambda obj: obj.bitrate)
    df['DurationInSeconds'] = df['TinyTagObject'].apply(
        lambda obj: None if obj.duration is None else int(np.ceil(obj.duration))
    )
    df['Duration'] = df['TinyTagObject'].apply(
        lambda obj: None if obj.duration is None else get_timetaken_fstring(
            num_seconds=int(np.ceil(obj.duration))
        )
    )
    df['SizeInMb'] = df['TinyTagObject'].apply(
        lambda obj: None if obj.filesize is None else round(obj.filesize / bytes_per_mb, 2)
    )
    df['SampleRate'] = df['TinyTagObject'].apply(lambda obj: obj.samplerate)
    df.drop(labels=['TinyTagObject'], axis=1, inplace=True)
    return df

def pickle_load(filepath: str) -> Any:
    """Loads data from pickle file, via joblib module"""
    python_obj = joblib.load(filename=filepath)
    return python_obj

def pickle_save(data_obj: Any, filepath: str) -> None:
    """Saves data as pickle file, via joblib module"""
    joblib.dump(value=data_obj, filename=filepath)
    return None

def save_object_as_json(obj: Any, filepath: str) -> None:
    """Saves Python object as JSON file"""
    with open(file=filepath, mode='w') as fp:
        json.dump(obj=obj, fp=fp, indent=4)
    return None

def read_to_dataframe(filepath: str) -> pd.DataFrame:
    """
    Reads DataFrame from filepaths having any of the following extensions: ['csv', 'xlsx'].
    >>> df = read_to_dataframe(filepath="some_data.csv")
    """
    extension = get_extension(filepath=filepath).lower()
    if extension == 'csv':
        data = pd.read_csv(filepath)
    elif extension == 'xlsx':
        data = pd.read_excel(filepath)
    else:
        raise ValueError(f"Expected filepath's extension to be in ['csv', 'xlsx'], but got '{extension}'")
    return data

def save_dataframe(
        data: pd.DataFrame,
        filepath: str,
    ) -> None:
    """
    Saves DataFrame to one of the following file types: ['csv', 'xlsx', 'json'].
    >>> save_dataframe(
            data=pd.DataFrame(data={'Column1': [1, 2, 3, 4]}), # Some DataFrame
            filepath="some_data.csv",
        )
    """
    extension = get_extension(filepath=filepath).lower()
    if extension == 'csv':
        data.to_csv(filepath, index=False)
    elif extension == 'xlsx':
        data.to_excel(filepath, index=False)
    elif extension == 'json':
        data.to_json(filepath, orient='records', indent=4)
    else:
        raise ValueError(f"Expected filepath's extension to be in ['csv', 'xlsx', 'json'], but got '{extension}'")
    return None


text:
# -*- coding: utf-8 -*-
# @Time    : 2021/4/12 7:12 下午
# @Author  : shaoguowen
# @Email   : shaoguowen@tencent.com
# @FileName: run.py
# @Software: PyCharm

import os
import argparse
import json
import datetime
import numpy as np
import tensorflow as tf
import cv2
import utils

parser = argparse.ArgumentParser(description='censeo-algorithm inference')
parser.add_argument('--saved_model', required=True, help='saved model path')
parser.add_argument('--dst_file_path', required=True, help='dst file path')
parser.add_argument('--ref_file_path', help='ref file path')
parser.add_argument('--save_name', help='save name of json for predicted results')
args = parser.parse_args()

# config
config_path = os.path.join(args.saved_model, "config.json")
with open(config_path, "r") as fr:
    config = json.load(fr)
    config = utils.Dict2Obj(config)

# load model
config_tf = tf.ConfigProto()
config_tf.gpu_options.allow_growth = True
sess = tf.Session(graph=tf.Graph(), config=config_tf)
tf.saved_model.loader.load(sess, ['serve'], args.saved_model)

def predict_fr(dst_path, ref_path):
    """
    用于FR的预测函数
    :param dst_path:
    :param ref_path:
    :return:
    """
    assert ref_path is not None
    if utils.is_img(dst_path):
        img_dst = cv2.imread(dst_path)
        img_dst = utils.transform(img_dst, config.input_process)[np.newaxis, ...]
        img_ref = cv2.imread(ref_path)
        img_ref = utils.transform(img_ref, config.input_process)[np.newaxis, ...]
        img_input = np.concatenate([img_dst, img_ref], 1)
        mos_pred = sess.run('test_output:0', feed_dict={'test_input:0': img_input})
        if config.output_process.use_sigmoid:
            mos_pred = utils.sigmoid(mos_pred)
        mos_pred = mos_pred * config.output_process.scale
        return mos_pred.reshape(-1).tolist()
    elif utils.is_video(dst_path):
        vcap_dst = utils.read_video_gen(dst_path, 0, -1, 10, 4)
        vcap_ref = utils.read_video_gen(ref_path, 0, -1, 10, 4)
        mos_preds = []
        while True:
            imgs_dst = next(vcap_dst, None)
            imgs_ref = next(vcap_ref, None)
            if imgs_dst is None or len(imgs_dst) == 0 or imgs_ref is None or len(imgs_ref) == 0:
                break
            imgs_dst = [utils.transform(img, config.input_process) for img in imgs_dst]
            imgs_dst = np.stack(imgs_dst)
            imgs_ref = [utils.transform(img, config.input_process) for img in imgs_ref]
            imgs_ref = np.stack(imgs_ref)
            imgs_input = np.concatenate([imgs_dst, imgs_ref], 1)
            mos_pred = sess.run('test_output:0', feed_dict={'test_input:0': imgs_input})
            if config.output_process.use_sigmoid:
                mos_pred = utils.sigmoid(mos_pred)
            mos_pred = mos_pred.reshape(-1) * config.output_process.scale
            mos_preds.append(mos_pred)
        return np.concatenate(mos_preds).tolist() if mos_preds else []

def predict_nr(dst_path):
    """
    用于nr模式的预测
    :param dst_path:
    :return:
    """
    if utils.is_img(dst_path):
        img_dst = cv2.imread(dst_path)
        img_dst = utils.transform(img_dst, config.input_process)[np.newaxis, ...]
        img_input = img_dst
        mos_pred = sess.run('test_output:0', feed_dict={'test_input:0': img_input})
        if config.output_process.use_sigmoid:
            mos_pred = utils.sigmoid(mos_pred)
        mos_pred = mos_pred * config.output_process.scale
        return mos_pred.reshape(-1).tolist()
    elif utils.is_video(dst_path):
        vcap_dst = utils.read_video_gen(dst_path, 0, -1, 10, 4)
        mos_preds = []
        while True:
            imgs_dst = next(vcap_dst, None)
            if imgs_dst is None or len(imgs_dst) == 0:
                break
            imgs_dst = [utils.transform(img, config.input_process) for img in imgs_dst]
            imgs_dst = np.stack(imgs_dst)
            imgs_input = imgs_dst
            mos_pred = sess.run('test_output:0', feed_dict={'test_input:0': imgs_input})
            if config.output_process.use_sigmoid:
                mos_pred = utils.sigmoid(mos_pred)
            mos_pred = mos_pred.reshape(-1) * config.output_process.scale
            mos_preds.append(mos_pred)
        return np.concatenate(mos_preds).tolist() if mos_preds else []

def predict(dst_path, ref_path=None):
    """
    模型预测
    :param dst_path:
    :param ref_path:
    :return:
    """
    if config.model.is_fr:
        return predict_fr(dst_path, ref_path)
    else:
        return predict_nr(dst_path)

if __name__ == '__main__':
    # result dir
    RESULT_DIR = "results"
    os.makedirs(RESULT_DIR, exist_ok=True)

    # model inference
    results = {}
    if os.path.isdir(args.dst_file_path):
        for i, vname in enumerate(os.listdir(args.dst_file_path)):
            print(i, vname)
            vpath_dst = os.path.join(args.dst_file_path, vname)
            if config.model.is_fr:
                results[vname] = predict(vpath_dst, os.path.join(args.ref_file_path, vname))
            else:
                results[vname] = predict(vpath_dst)
            print(results[vname])
    else:
        vname = os.path.basename(args.dst_file_path)
        print(vname)
        if config.model.is_fr:
            results[os.path.basename(args.dst_file_path)] = predict(args.dst_file_path, args.ref_file_path)
        else:
            results[os.path.basename(args.dst_file_path)] = predict(args.dst_file_path)
        print(results[vname])

    if args.save_name is None:
        date_str = datetime.datetime.now().strftime("%m-%d-%H-%M-%S")
        save_name = os.path.basename(args.dst_file_path) + "-" + date_str + ".json"
    else:
        save_name = args.save_name + ".json"
    with open(os.path.join("results", save_name), "w") as fw:
        print("saving result at :{}".format(os.path.join("results", save_name)))
        json.dump(results, fw)
    sess.close()

max_stars_repo_name: usutdzxych/CenseoQoE


text:
"""
ThorAxe pipeline and script functions.
"""

import argparse
import os
from ast import literal_eval
from Bio.Alphabet import generic_protein
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Align import MultipleSeqAlignment
import numpy as np

from thoraxe import subexons
from thoraxe import utils
from thoraxe import transcript_info
from thoraxe.transcript_info import phases
from .version import __version__

def parse_command_line():
    """
    Parse command line.

    It uses argparse to parse thoraxe' command line arguments and returns
    the argparse parser.
    """
    parser = argparse.ArgumentParser(
        prog="thoraxe",
        description="""
        thoraxe is a tool to identify orthologous exonic regions (s-exons) in a
        set of transcripts from orthologous genes in a set of species.
        """,
        epilog="""
        It has been developed at LCQB (Laboratory of Computational and
        Quantitative Biology), UMR 7238 CNRS, Sorbonne Université.
        """,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '-i',
        '--inputdir',
        help='Input directory. The input folder should have an Ensembl '
        'subfolder as the generated by transcript_query.',
        type=str,
        default='.')
    parser.add_argument(
        '-o',
        '--outputdir',
        help='Output directory, the indicated input directory is used '
        'by default.',
        type=str,
        default='')
    parser.add_argument('-a',
                        '--aligner',
                        help='Path to ProGraphMSA.',
                        type=str,
                        default='ProGraphMSA')
    parser.add_argument('-s',
                        '--maxtsl',
                        help='Maximum Transcript Support Level (TSL) to use '
                        'when TSL is available for a transcript.',
                        type=int,
                        default=3)
    parser.add_argument('-m',
                        '--minlen',
                        help='Minimum exon length.',
                        type=int,
                        default=4)
    parser.add_argument('-g',
                        '--mingenes',
                        help='Minimum number of genes to consider a path in '
                        'the splice graph.',
                        type=int,
                        default=1)
    parser.add_argument('-t',
                        '--mintranscripts',
                        help='Minimum number of transcripts to consider a '
                        'path in the splice graph.',
                        type=int,
                        default=2)
    parser.add_argument(
        '-c',
        '--coverage',
        help='Minimum alignment coverage of the shorter exon to include both '
        'exons in the same cluster.',
        type=float,
        default=80.0)
    parser.add_argument(
        '-p',
        '--identity',
        help='Minimum percent identity to include exons in the same cluster.',
        type=float,
        default=30.0)
    parser.add_argument('--gapopen',
                        help='Penalty for a gap opening.',
                        type=int,
                        default=-10)
    parser.add_argument('--gapextend',
                        help='Penalty for gap extensions.',
                        type=int,
                        default=-1)
    parser.add_argument('-r',
                        '--rescue_unaligned_subexons',
                        help='The sub-exons that do not align against any '
                        'other are deleted from their cluster, and they '
                        'could be reassigned during the sub-exon rescue '
                        'phase. By default, they are kept into their '
                        'original exon cluster.',
                        action='store_true')
    parser.add_argument(
        '--padding',
        help='Length of padding, Xs, in the chimeric alignment.',
        type=int,
        default=10)
    parser.add_argument(
        '-y',
        '--phylosofs',
        help='Save inputs to run PhyloSofS in the phylosofs folder.',
        action='store_true')
    parser.add_argument('--no_movements',
                        help='Do not move one/two residue sub-exon blocks.',
                        action='store_true')
    parser.add_argument('--no_disintegration',
                        help='Do not disintegrate one-residue-length s-exons.',
                        action='store_true')
    parser.add_argument(
        '--plot_chimerics',
        help='Save plotly/html plot for the chimeric alignments in the '
        '_intermediate folder.',
        action='store_true')
    parser.add_argument(
        '-l',
        '--specieslist',
        help='It could be a list of more than one species separated by commas '
        'and without spaces, e.g. homo_sapiens,mus_musculus, or a single file '
        'with the species list (one species per line). If nothing is '
        'indicated, all the available species are used.',
        default='')
    parser.add_argument(
        '--canonical_criteria',
        help='List of column names of the `path_table` separated by commas '
        'used to sort the row. If nothing is indicated, the following list is '
        'used: MinimumConservation,MinimumTranscriptWeightedConservation,'
        'MeanTranscriptWeightedConservation,TranscriptLength,TSL',
        default='')
    parser.add_argument('--version',
                        action='version',
                        version='ThorAxe version {}'.format(__version__))
    return parser

def get_transcripts(input_folder, max_tsl_level=3.0, species_list=None):
    """Return a DataFrame with the transcript information."""
    ensembl_info = os.path.join(input_folder, 'Ensembl')
    return transcript_info.read_transcript_info(
        os.path.join(ensembl_info, 'tsl.csv'),
        os.path.join(ensembl_info, 'exonstable.tsv'),
        os.path.join(ensembl_info, 'sequences.fasta'),
        max_tsl_level=max_tsl_level,
        remove_na=False,
        species_list=species_list)

def get_subexons(  # pylint: disable=too-many-arguments
        transcript_table,
        minimum_len=4,
        coverage_cutoff=80.0,
        percent_identity_cutoff=30.0,
        gap_open_penalty=10,
        gap_extend_penalty=1):
    """
    Return a DataFrame with subexon information and clustered exons.

    Exons are clustered according to their protein sequence using the blosum50
    matrix and the following arguments: minimum_len, coverage_cutoff,
    percent_identity_cutoff, gap_open_penalty and gap_extend_penalty.
    """
    clustered = transcript_info.exon_clustering(
        transcript_table,
        minimum_len=minimum_len,
        coverage_cutoff=coverage_cutoff,
        percent_identity_cutoff=percent_identity_cutoff,
        gap_open_penalty=gap_open_penalty,
        gap_extend_penalty=gap_extend_penalty)
    subexon_table = subexons.create_subexon_table(clustered)
    return merge_clusters(subexon_table)

def _merge_clusters_in_list(cluster_lists):
    """Return a list of sets, each set has the clusters to merge."""
    output_list = []
    for input_list in cluster_lists:
        input_set = set(input_list)
        added = False
        for output_set in output_list:
            if output_set.intersection(input_set):
                output_set.update(input_set)
                added = True
        if not added:
            output_list.append(input_set)
    return output_list

def merge_clusters(subexon_table):
    """Merge 'Cluster's that share subexons."""
    clusters_df = subexon_table.groupby('SubexonIDCluster').agg(
        {'Cluster': lambda col: set(val for val in col if val != 0)})
    cluster_lists = clusters_df.loc[map(lambda x: len(
        x) > 1, clusters_df['Cluster']), 'Cluster'].apply(
            repr).drop_duplicates().apply(lambda x: sorted(literal_eval(x)))
    cluster_sets = _merge_clusters_in_list(cluster_lists)
    for cluster_set in cluster_sets:
        clusters = sorted(cluster_set)
        cluster_id = clusters[0]
        for cluster in clusters[1:]:
            cluster_index = subexon_table.index[subexon_table['Cluster'] ==
                                                cluster]
            for i in cluster_index:
                subexon_table.at[i, 'Cluster'] = cluster_id
    return subexon_table

def _outfile(output_folder, prefix, name, ext):
    """
    Return the output file name.

    >>> _outfile("", "gene_ids_", 10, ".txt")
    'gene_ids_10.txt'
    """
    return os.path.join(output_folder, prefix + str(name) + ext)

def _create_chimeric_msa(  # pylint: disable=too-many-arguments
        output_folder,
        cluster,
        subexon_df,
        gene2speciesname,
        connected_subexons,
        aligner='ProGraphMSA',
        padding='XXXXXXXXXX',
        species_list=None):
    """Return a modified subexon_df, the dict of chimerics and the msa."""
    subexon_df, subexon_matrix = subexons.alignment.create_subexon_matrix(
        subexon_df)
    chimerics = subexons.alignment.create_chimeric_sequences(
        subexon_df, subexon_matrix, connected_subexons, padding=padding)
    msa_file = _outfile(output_folder, "chimeric_alignment_", cluster,
                        ".fasta")
    if chimerics:
        chimerics = subexons.alignment.sort_species(chimerics,
                                                    gene2speciesname,
                                                    species_list)
        subexons.alignment.run_aligner(chimerics,
                                       aligner=aligner,
                                       output_path=msa_file)
        msa = subexons.alignment.read_msa_fasta(msa_file)
    else:
        if os.path.isfile(msa_file):
            os.remove(msa_file)
        msa = None
    return subexon_df, chimerics, msa

def _create_and_test_chimeric_msa(  # noqa pylint: disable=too-many-arguments,too-many-locals
        cluster2data,
        subexon_table,
        cluster_index,
        output_folder,
        cluster,
        gene2speciesname,
        connected_subexons,
        cutoff=30.0,
        min_col_number=4,
        aligner='ProGraphMSA',
        padding='XXXXXXXXXX',
        species_list=None,
        keep_single_subexons=False):
    """
    Update cluster2data and return a set of exons to delete.

    For each cluster, it adds the subexon dataframe, the chimeric sequences
    and the msa to cluster2data.
    """
    subexon_df = subexon_table.loc[cluster_index, :]
    subexon_df, chimerics, msa = _create_chimeric_msa(
        output_folder,
        cluster,
        subexon_df,
        gene2speciesname,
        connected_subexons,
        aligner=aligner,
        padding=padding,
        species_list=species_list)
    cluster2data[cluster] = (subexon_df, chimerics, msa)
    if msa is None:
        return {}
    return subexons.alignment.delete_subexons(
        subexon_df,
        chimerics,
        msa,
        cutoff=cutoff,
        min_col_number=min_col_number,
        keep_single_subexons=keep_single_subexons)

def create_chimeric_msa(  # pylint: disable=too-many-arguments,too-many-locals
        output_folder,
        subexon_table,
        gene2speciesname,
        connected_subexons,
        clusters=None,
        cutoff=30.0,
        min_col_number=4,
        aligner='ProGraphMSA',
        padding='XXXXXXXXXX',
        species_list=None,
        keep_single_subexons=False):
    """
    Return a dict from cluster to cluster data.

    For each cluster, there is a tuple with the subexon dataframe, the
    chimeric sequences and the msa.

    This function can take a clusters argument with the list of 'Cluster'
    identifiers to use. If that list is not given, all the positive 'Cluster'
    identifiers from subexon_table are used.
    """
    if clusters is None:
        clusters = {
            cluster
            for cluster in subexon_table['Cluster'] if cluster > 0
        }

    cluster2data = {}
    for cluster in clusters:
        cluster_index = subexon_table.index[subexon_table['Cluster'] ==
                                            cluster]
        to_delete = _create_and_test_chimeric_msa(
            cluster2data,
            subexon_table,
            cluster_index,
            output_folder,
            cluster,
            gene2speciesname,
            connected_subexons,
            cutoff=cutoff,
            min_col_number=min_col_number,
            aligner=aligner,
            padding=padding,
            species_list=species_list,
            keep_single_subexons=keep_single_subexons)
        while to_delete:
            delete = [
                subexon_table.loc[index, 'SubexonIDCluster'] in to_delete
                for index in cluster_index
            ]
            for index in cluster_index[delete]:
                subexon_table.at[index, 'Cluster'] = -1 * abs(
                    subexon_table.loc[index, 'Cluster'])
            cluster_index = cluster_index[np.logical_not(delete)]
            to_delete = _create_and_test_chimeric_msa(
                cluster2data,
                subexon_table,
                cluster_index,
                output_folder,
                cluster,
                gene2speciesname,
                connected_subexons,
                cutoff=cutoff,
                min_col_number=min_col_number,
                aligner=aligner,
                padding=padding,
                species_list=species_list,
                keep_single_subexons=keep_single_subexons)
    return cluster2data

def get_s_exons(  # noqa pylint: disable=too-many-arguments,too-many-locals
        output_folder,
        subexon_table,
        gene2speciesname,
        connected_subexons,
        minimum_len=4,
        coverage_cutoff=80.0,
        percent_identity_cutoff=30.0,
        gap_open_penalty=10,
        gap_extend_penalty=1,
        aligner='ProGraphMSA',
        padding='XXXXXXXXXX',
        movements=True,
        disintegration=True,
        species_list=None,
        keep_single_subexons=False):
    """Perform almost all the ThorAxe pipeline."""

    intermediate_output_path = utils.folders.create_subfolder(
        output_folder, '_intermediate')
    msa_output_path = utils.folders.create_subfolder(output_folder, 'msa')

    cluster2updated_data = {}
    cluster2data = create_chimeric_msa(
        intermediate_output_path,
        subexon_table,
        gene2speciesname,
        connected_subexons,
        cutoff=percent_identity_cutoff,
        min_col_number=minimum_len,
        aligner=aligner,
        padding=padding,
        species_list=species_list,
        keep_single_subexons=keep_single_subexons)
    modified_clusters = subexons.rescue.subexon_rescue_phase(
        cluster2data,
        subexon_table,
        minimum_len=minimum_len,
        coverage_cutoff=coverage_cutoff,
        percent_identity_cutoff=percent_identity_cutoff,
        gap_open_penalty=gap_open_penalty,
        gap_extend_penalty=gap_extend_penalty,
        substitution_matrix=None)
    cluster2data.update(
        create_chimeric_msa(intermediate_output_path,
                            subexon_table,
                            gene2speciesname,
                            connected_subexons,
                            clusters=modified_clusters,
                            cutoff=percent_identity_cutoff,
                            min_col_number=minimum_len,
                            aligner=aligner,
                            padding=padding,
                            species_list=species_list,
                            keep_single_subexons=keep_single_subexons))

    for (cluster, (subexon_df, chimerics, msa)) in cluster2data.items():
        if (msa is not None) and chimerics:
            gene_ids = subexons.alignment.get_gene_ids(msa)
            msa_matrix = subexons.alignment.create_msa_matrix(chimerics, msa)

            if movements:
                (msa_numpy, msa_matrix
                 ) = subexons.alignment.move_problematic_block_clusters(
                     msa, msa_matrix)
            else:
                msa_numpy = np.array([list(rec) for rec in msa], np.character)

            if disintegration:
                msa_numpy, msa_matrix = subexons.alignment.disintegration(
                    msa_numpy, msa_matrix)

            if movements or disintegration:
                msa = MultipleSeqAlignment([
                    SeqRecord(Seq(row.tostring().decode('utf-8'),
                                  generic_protein),
                              id=gene_id)
                    for (gene_id, row) in zip(gene_ids, msa_numpy)
                ])
                # cluster2data[cluster] = (subexon_df, chimerics, msa)
            with open(
                    _outfile(intermediate_output_path, "gene_ids_", cluster,
                             ".txt"), 'w') as outfile:
                for item in gene_ids:
                    outfile.write("%s\n" % item)

            np.savetxt(_outfile(intermediate_output_path, "msa_matrix_",
                                cluster, ".txt"),
                       msa_matrix,
                       delimiter=",")

            colclusters = subexons.alignment.column_clusters(
                subexons.alignment.column_patterns(msa_matrix))

            sequences = subexons.alignment.msa2sequences(
                msa, gene_ids, padding)
            subexon_df = subexons.alignment.save_s_exons(
                subexon_df, sequences, gene_ids, colclusters, msa_output_path)
        else:
            msa = None
            gene_ids = None
            msa_matrix = None
            colclusters = None

        cluster2updated_data[cluster] = (subexon_df, chimerics, msa, gene_ids,
                                         msa_matrix, colclusters)
        subexon_df.to_csv(
            _outfile(intermediate_output_path, "subexon_table_", cluster,
                     ".csv"))

    return cluster2updated_data

def update_subexon_table(subexon_table, cluster2data):
    """Update the subexon table by adding the s-exon information."""
    columns_to_add = [
        'S_exon_Lengths', 'S_exon_Sequences', 'S_exons', 'SubexonIndex'
    ]
    key_col = 'SubexonIDCluster'
    subexon2data = {}
    for (_, data) in cluster2data.items():
        cluster_df = data[0]
        if cluster_df.shape[0] > 0:
            # https://pandas.pydata.org/pandas-docs/stable/user_guide/
            # indexing.html#deprecate-loc-reindex-listlike
            subexon2data.update(
                cluster_df.reindex(columns=[key_col] +
                                   columns_to_add).drop_duplicates().set_index(
                                       key_col).to_dict('index'))

    columns = {colname: [] for colname in columns_to_add}
    for subexon in subexon_table[key_col]:
        for colname in columns_to_add:
            if subexon in subexon2data:
                columns[colname].append(subexon2data[subexon][colname])
            else:
                columns[colname].append('')

    for (colname, values) in columns.items():
        subexon_table[colname] = values

    return subexon_table

def add_s_exon_phases_and_coordinates(tbl):
    """
    Add s-exon genomic coordinates and phases to the tidy s-exon table.

    For each s-exon, it add the ``S_exon_Genomic_Sequence`` and their genomic
    coordinates, starting at ``1`` and defining the closed interval:
    ``[S_exon_CodingStart, S_exon_CodingEnd]``. Where ``S_exon_CodingStart``
    is greater or equal to ``S_exon_CodingEnd`` for genes in the reverse
    ``Strand``. It add also the ``S_exon_StartPhase`` and ``S_exon_EndPhase``
    for those intervals.

    An s-exon, defined at the protein level, can be an entire sub-exon or a
    part of it. In cases where an s-exon is identical to a sub-exon, the
    genomic coordinates and the phases are the same. However, when a sub-exon
    is split into several s-exons, coordinates and phases need to be calculated:

    **Example 1: s-exons in the positive strand.**

    The following sub-exon shares codons with the previous and following
    sub-exon. In this example, ``-`` represents introns and lower case
    characters represent exon nucleotides. The start and end phases are ``1``
    and ``2`` respectively:

    ``...v----vvwwwxxxyyyzz-----z...``

    Because we are in the positive strand, the residue ``V`` coming from the
    ``vvv`` shared codon is assigned to this sub-exon protein sequence. In the
    same way, the ``Z`` residue coming from the ``zzz`` shared exon is assigned
    to the beginning of the next sub-exon:

    ``VWXY``

    Let's say that this sub-exon is composed of two s-exons: ``VW`` and ``XY``.
    Then the corresponding genomic sequences are ``vvwww`` and ``xxxyyyzz`` and
    the ``(start, end)`` phases are ``(1, 0)`` and ``(0, 2)``.

    *Genomic coordinates.*

    Let's say that the first ``v`` has the genomic coordinate ``1``:

    ``v ---- vvwww xxxyyyzz ----- z``
    ``           1 11111111 12222 2``
    ``1 2345 67890 12345678 90123 4``

    In this way, the first s-exon has the genomic coordinates ``[6, 10]`` and
    the second has the genomic coordinates ``[11, 18]``. The first s-exon has
    ``2`` amino-acid residues, but ``5`` instead of ``6`` bases because the
    first codon is shared between two s-exons. In particular, the
    amino-acid residue ``V`` needs two bases in this s-exon to complete the
    codon started in the previous s-exon.

    **Example 2: s-exons in the negative strand.**

    ``v ---- vvwww xxxyyyzz ----- z``
    ``2 2222 11111 11111           ``
    ``4 3210 98765 43210987 65432 1``

    If the same sequence belongs to the negative strand, the sub-exon protein
    sequence is going to contain ``Z`` but not ``V``: ``WXYZ``. Therefore,
    possible s-exons are ``W`` and ``XYZ`` and the corresponding genomic
    sequences will be ``vvwww`` and ``xxxyyyzz`` and the ``(start, end)``
    phases will be ``(1, 0)`` and ``(0, 2)``.

    In this way, the first s-exon has the genomic coordinates ``[19, 15]`` and
    the second has the genomic coordinates ``[14, 7]``. The first s-exon has `
    `1`` amino-acid residue, but ``5`` instead of ``3`` bases because the first
    codon is shared between two s-exons. In particular, the amino-acid residue
    ``V`` needs two bases in this s-exon to complete the codon started in the
    previous s-exon.

    """
    tbl['S_exon_CodingStart'] = None
    tbl['S_exon_CodingEnd'] = None
    tbl['S_exon_StartPhase'] = None
    tbl['S_exon_EndPhase'] = None
    tbl['S_exon_Genomic_Sequence'] = None

    for _, group in tbl.groupby(['SubexonIDCluster', 'TranscriptIDCluster']):
        group_indices = group.index
        n_rows = len(group_indices)
        if n_rows > 1:
            end_coord = 0
            remaining_seq = ''
            for row_number, i in enumerate(group_indices):
                if row_number == 0:  # first s-exon
                    tbl.at[i, 'S_exon_StartPhase'] = tbl.loc[
                        i, 'SubexonStartPhase']
                    tbl.at[i, 'S_exon_EndPhase'] = 0
                    (end_coord, s_exon_seq,
                     remaining_seq) = _get_first_s_exon(tbl.loc[i, :])
                    tbl.at[i, 'S_exon_CodingStart'] = tbl.loc[
                        i, 'SubexonCodingStart']
                    tbl.at[i, 'S_exon_CodingEnd'] = end_coord
                    tbl.at[i, 'S_exon_Genomic_Sequence'] = s_exon_seq
                elif row_number == (n_rows - 1):  # last s-exon
                    tbl.at[i, 'S_exon_StartPhase'] = 0
                    tbl.at[i, 'S_exon_EndPhase'] = tbl.loc[i,
                                                           'SubexonEndPhase']
                    tbl.at[i, 'S_exon_CodingStart'] = end_coord + \
                        tbl.loc[i, 'Strand']
                    tbl.at[i, 'S_exon_CodingEnd'] = tbl.loc[i,
                                                            'SubexonCodingEnd']
                    tbl.at[i, 'S_exon_Genomic_Sequence'] = remaining_seq

                else:
                    tbl.at[i, 'S_exon_StartPhase'] = 0
                    tbl.at[i, 'S_exon_EndPhase'] = 0
                    (start_coord, end_coord, s_exon_seq,
                     remaining_seq) = _get_internal_s_exon(
                         tbl.loc[i, :], end_coord, remaining_seq)
                    tbl.at[i, 'S_exon_CodingStart'] = start_coord
                    tbl.at[i, 'S_exon_CodingEnd'] = end_coord
                    tbl.at[i, 'S_exon_Genomic_Sequence'] = s_exon_seq
        elif n_rows == 1:
            i = group_indices[0]
            tbl.at[i, 'S_exon_CodingStart'] = tbl.loc[i, 'SubexonCodingStart']
            tbl.at[i, 'S_exon_CodingEnd'] = tbl.loc[i, 'SubexonCodingEnd']
            tbl.at[i, 'S_exon_StartPhase'] = tbl.loc[i, 'SubexonStartPhase']
            tbl.at[i, 'S_exon_EndPhase'] = tbl.loc[i, 'SubexonEndPhase']
            tbl.at[i, 'S_exon_Genomic_Sequence'] = tbl.loc[i,
                                                           'SubexonSequence']

    return tbl

def _get_first_s_exon(row):
    """
    Calculate the genomic end coordinate of the first s-exon

    Also returns the genomic nucleotide sequences of the s-exon and the rest of
    the sub-exon.

    >>> row = {'SubexonSequence': 'TTGCCGTCATGAGCAG'}
    >>> row['SubexonCodingStart'] = 2379656
    >>> row['SubexonStartPhase'] = 1
    >>> row['S_exon_Sequence'] = 'AV'
    >>> row['Strand'] = -1
    >>> _get_first_s_exon(row)
    (2379649, 'TTGCCGTC', 'ATGAGCAG')
    >>> row = {'SubexonSequence': 'CACAGGTGCAGCAGTGA'}
    >>> row['SubexonCodingStart'] = 48434879
    >>> row['SubexonStartPhase'] = 1
    >>> row['S_exon_Sequence'] = 'AQ'
    >>> row['Strand'] = 1
    >>> _get_first_s_exon(row)
    (48434883, 'CACAG', 'GTGCAGCAGTGA')
    >>> # SNAP25 ENSE00000906745
    >>> seq = 'AGTAAAGATGCTGGTATCAGGACTTTGGTTATGTTGGATGAACAAGGAG'
    >>> row = {'SubexonSequence': seq}
    >>> row['SubexonCodingStart'] = 10284724
    >>> row['SubexonStartPhase'] = 0
    >>> row['S_exon_Sequence'] = 'S'
    >>> row['Strand'] = 1
    >>> _get_first_s_exon(row)
    (10284726, 'AGT', 'AAAGATGCTGGTATCAGGACTTTGGTTATGTTGGATGAACAAGGAG')
    """
    genomic_seq = row['SubexonSequence']
    start_coordinates = row['SubexonCodingStart']
    start_phase = row['SubexonStartPhase']
    s_exon_len = len(row['S_exon_Sequence'])

    needed_bases = s_exon_len * 3

    bases_to_complete_previous_codon = phases.bases_to_complete_previous_codon(
        start_phase)
    if bases_to_complete_previous_codon:
        bases_in_previous_codon = 3 - bases_to_complete_previous_codon
    else:
        bases_in_previous_codon = 0

    if row['Strand'] == 1:
        total_len = needed_bases - bases_in_previous_codon
        end_coordinates = start_coordinates + total_len - 1
    else:
        total_len = needed_bases + bases_to_complete_previous_codon
        end_coordinates = start_coordinates - total_len + 1

    return end_coordinates, genomic_seq[:total_len], genomic_seq[total_len:]

def _get_internal_s_exon(row, previous_end, seq):
    """
    Calculate the genomic coordinates of internal s-exons

    Also returns the genomic nucleotide sequences of the s-exon and the rest of
    the sub-exon.

    >>> row = {'S_exon_Sequence': 'MS'}
    >>> row['Strand'] = -1
    >>> _get_internal_s_exon(row, 2379649, 'ATGAGCAG')
    (2379648, 2379643, 'ATGAGC', 'AG')
    >>> row['Strand'] = 1
    >>> _get_internal_s_exon(row, 2379649, 'ATGAGCAG')
    (2379650, 2379655, 'ATGAGC', 'AG')
    """
    total_len = len(row['S_exon_Sequence']) * 3
    if row['Strand'] == 1:
        start_coordinates = previous_end + 1
        end_coordinates = previous_end + total_len
    else:
        start_coordinates = previous_end - 1
        end_coordinates = previous_end - total_len
    return start_coordinates, end_coordinates, seq[:total_len], seq[total_len:]

def get_s_exon_msas(output_folder):
    """
    Return a dict of the s_exon MSAs.
    """
    s_exon_msas = dict()
    msa_path = os.path.join(output_folder, 'msa')
    msa_files = os.listdir(msa_path)
    for msa_file in msa_files:
        if 'msa_s_exon_' in msa_file and '.fasta' in msa_file:
            fields = msa_file.split('_')
            s_exon = '{}_{}'.format(fields[3], fields[4].split('.')[0])
            s_exon_msas[s_exon] = subexons.alignment.read_msa_fasta(
                os.path.join(msa_path, msa_file))
    return s_exon_msas

def main():  # pylint: disable=too-many-locals
    """Perform Pipeline."""
    args = parse_command_line().parse_args()

    species_list = utils.species.get_species_list(args.specieslist)

    input_folder = os.path.abspath(args.inputdir)
    output_folder = input_folder if args.outputdir == '' else args.outputdir
    output_folder = utils.folders.create_subfolder(output_folder, 'thoraxe')

    intermediate_output_path = utils.folders.create_subfolder(
        output_folder, '_intermediate')

    transcript_table = get_transcripts(input_folder,
                                       float(args.maxtsl),
                                       species_list=species_list)
    subexon_table = get_subexons(transcript_table,
                                 minimum_len=args.minlen,
                                 coverage_cutoff=args.coverage,
                                 percent_identity_cutoff=args.identity,
                                 gap_open_penalty=args.gapopen,
                                 gap_extend_penalty=args.gapextend)

    transcript_table.to_csv(
        os.path.join(intermediate_output_path, "transcript_table.csv"))

    gene2speciesname = subexons.alignment.gene2species(transcript_table)
    connected_subexons = subexons.alignment.subexon_connectivity(subexon_table)

    cluster2data = get_s_exons(
        output_folder,
        subexon_table,
        gene2speciesname,
        connected_subexons,
        minimum_len=args.minlen,
        coverage_cutoff=args.coverage,
        percent_identity_cutoff=args.identity,
        gap_open_penalty=args.gapopen,
        gap_extend_penalty=args.gapextend,
        aligner=args.aligner,
        padding='X' * args.padding,
        movements=not args.no_movements,
        disintegration=not args.no_disintegration,
        species_list=species_list,
        keep_single_subexons=not args.rescue_unaligned_subexons)

    s_exon_msas = get_s_exon_msas(output_folder)

    subexon_table.to_csv(
        os.path.join(intermediate_output_path, "subexon_table.csv"))

    if args.plot_chimerics:
        subexons.plot.plot_msa_subexons(cluster2data, intermediate_output_path)

    subexon_table = update_subexon_table(subexon_table, cluster2data)
    subexon_table = subexons.alignment.impute_missing_s_exon(subexon_table)

    tidy_table = subexons.tidy.get_tidy_table(subexon_table, gene2speciesname)
    add_s_exon_phases_and_coordinates(tidy_table)
    tidy_table.to_csv(os.path.join(output_folder, "s_exon_table.csv"),
                      index=False)

    if args.phylosofs:
        s_exon_2_char = subexons.phylosofs.phylosofs_inputs(
            tidy_table, os.path.join(input_folder, 'Ensembl'), output_folder)
    else:
        s_exon_2_char = {}

    (node2genes, edge2genes, node2transcripts, edge2transcripts, edge2trx_cons
     ) = subexons.graph.nodes_and_edges2genes_and_transcripts(tidy_table)

    splice_graph_filename = os.path.join(output_folder, "splice_graph.gml")

    subexons.graph.splice_graph_gml(splice_graph_filename, s_exon_msas,
                                    node2genes, edge2genes, node2transcripts,
                                    edge2transcripts, edge2trx_cons,
                                    s_exon_2_char)

    path_table, ases_table = subexons.ases.conserved_ases(
        tidy_table,
        transcript_table,
        s_exon_msas,
        splice_graph_filename,
        min_genes=args.mingenes,
        min_transcripts=args.mintranscripts,
        column_order=args.canonical_criteria.split(",")
        if args.canonical_criteria else None)

    path_table.to_csv(os.path.join(output_folder, "path_table.csv"),
                      index=False)
    ases_table.to_csv(os.path.join(output_folder, "ases_table.csv"),
                      index=False)

if __name__ == '__main___':
    main()

max_stars_repo_name: PhyloSofS-Team/exonhomology

max_issues_repo_name: PhyloSofS-Team/exonhomology

max_forks_repo_name: PhyloSofS-Team/exonhomology


text:
import numpy as np
import gzip

class Reader(object):
    def __init__(self, directory=''):
        self.directory = directory

    def read_data(self, normalize=True, vectorize_label=False):
        X_train = read_images(self.directory + '/train-images-idx3-ubyte.gz')
        y_train = read_labels(self.directory + '/train-labels-idx1-ubyte.gz')
        X_test = read_images(self.directory + '/t10k-images-idx3-ubyte.gz')
        y_test = read_labels(self.directory + '/t10k-labels-idx1-ubyte.gz')
        if normalize:
            X_train = X_train / 255
            X_test = X_test / 255

        if vectorize_label:
            y_train = np.array(list(map(label_to_vec, y_train)))
            y_test = np.array(list(map(label_to_vec, y_test)))

        return X_train, y_train, X_test, y_test

def read_images(file):
    with gzip.open(file, 'rb') as f:
        magic_number = int.from_bytes(f.read(4), 'big')
        nimages = int.from_bytes(f.read(4), 'big')
        imagerows = int.from_bytes(f.read(4), 'big')
        imagecols = int.from_bytes(f.read(4), 'big')
        data = f.read()

    return np.frombuffer(data, dtype=np.uint8).reshape((nimages, imagerows, imagecols))

def read_labels(file):
    with gzip.open(file, 'rb') as f:
        magic_number = int.from_bytes(f.read(4), 'big')
        nlabels = int.from_bytes(f.read(4), 'big')
        data = f.read()

    return np.frombuffer(data, dtype=np.uint8)

def label_to_vec(label):
    vec = np.zeros((10, 1))
    vec[label] = 1.0
    return vec

max_stars_repo_name: IsaacSST/MNIST-reader

max_issues_repo_name: IsaacSST/MNIST-reader

max_forks_repo_name: IsaacSST/MNIST-reader


text:
#coding=utf-8
import time
import os
import cv2
import numpy as np
from numpy.linalg import inv
def sliding_window(img1, img2, patch_size=(100,302), istep=50):#, jstep=1, scale=1.0):
    """
get patches and thier upper left corner coordinates
The size of the sliding window is currently fixed.
patch_size: sliding_window's size'
istep: Row stride
    """
    Ni, Nj = (int(s) for s in patch_size)
    for i in range(0, img1.shape[0] - Ni+1, istep):
        #for j in range(0, img1.shape[1] - Nj, jstep):
            #patch = (img1[i:i + Ni, j:j + Nj], img2[i:i + Ni, j:j + Nj])
        patch = (img1[i:i + Ni, 39:341], img2[i:i + Ni, 39:341])
        yield (i, 39), patch

def predict(patches, DEBUG):
    """
predict zebra crossing for every patches 1 is zc 0 is background
    """
    #print(len(patches))
    labels = np.zeros(len(patches))
    index = 0
    for Amplitude, theta in patches:
        mask = (Amplitude>25).astype(np.float32)
        h, b = np.histogram(theta[mask.astype(np.bool)], bins=range(0,80,5))
        low, high = b[h.argmax()], b[h.argmax()+1]
        newmask = ((Amplitude>25) * (theta<=high) * (theta>=low)).astype(np.float32)
        value = ((Amplitude*newmask)>0).sum()

        if value > 1500:
            labels[index] = 1
        index += 1
        if(DEBUG):
            print(h)
            print(low, high)
            print(value)
            cv2.imshow("newAmplitude", Amplitude*newmask)
            cv2.waitKey(0)

    return labels

def preprocessing(img):
    """
Take the blue channel of the original image and filter it smoothly
    """
    kernel1 = np.ones((3,3),np.uint8)
    kernel2 = np.ones((5,5),np.uint8)
    gray = img[:,:,0]
    gray = cv2.medianBlur(gray,5)
    gray = cv2.morphologyEx(gray, cv2.MORPH_OPEN, kernel1,iterations=4)
    gray = cv2.morphologyEx(gray, cv2.MORPH_CLOSE, kernel2,iterations=3)
    return gray

def getGD(canny):
    """
return gradient mod and direction
    """
    sobelx=cv2.Sobel(canny,cv2.CV_32F,1,0,ksize=3)
    sobely=cv2.Sobel(canny,cv2.CV_32F,0,1,ksize=3)
    theta = np.arctan(np.abs(sobely/(sobelx+1e-10)))*180/np.pi
    Amplitude = np.sqrt(sobelx**2+sobely**2)
    mask = (Amplitude>30).astype(np.float32)
    Amplitude = Amplitude*mask
    return Amplitude, theta

def getlocation(indices, labels, Ni, Nj):
    """
return if there is a zebra cossing
if true, Combine all the rectangular boxes as its position
assume a picture has only one zebra crossing
    """
    zc = indices[labels == 1]
    if len(zc) == 0:
        return 0, None
    else:
        xmin = int(min(zc[:,1]))
        ymin = int(min(zc[:,0]))
        xmax = int(xmin + Nj)
        ymax = int(max(zc[:,0]) + Ni)
        return 1, ((xmin, ymin), (xmax, ymax))

if __name__ == "__main__":

    DEBUG = False #if False, won't draw all step

    Ni, Nj = (90, 1600)

    M = np.array([[-1.86073726e-01, -5.02678929e-01,  4.72322899e+02],
                 [-1.39150388e-02, -1.50260445e+00,  1.00507430e+03],
                 [-1.77785988e-05, -1.65517173e-03,  1.00000000e+00]])

    iM = inv(M)
    xy = np.zeros((640,640,2),dtype=np.float32)
    for py in range(640):
        for px in range(640):
            xy[py,px] = np.array([px,py],dtype=np.float32)
    ixy=cv2.perspectiveTransform(xy,iM)
    mpx,mpy = cv2.split(ixy)
    mapx,mapy=cv2.convertMaps(mpx,mpy,cv2.CV_16SC2)

    cap = cv2.VideoCapture("./video-02.mp4")
    time.sleep(1)
    NUM_FRAMES = int(cap.get(7))
    for ii in range(NUM_FRAMES):
        print("frame: ", ii)
        # Load frame from the camera
        ret, frame = cap.read()
        img=frame
        #img = cv2.remap(frame, mapx, mapy, cv2.INTER_LINEAR)
        #img = cv2.resize(img, (400,400))
        gray = preprocessing(img)
        # if DEBUG:
        #     cv2.imshow("gray", gray)

        canny = cv2.Canny(gray,30,90,apertureSize = 3)
        # if DEBUG:
        #     cv2.imshow("canny",canny)

        Amplitude, theta = getGD(canny)
        # if DEBUG:
        #     cv2.imshow("Amplitude", Amplitude)

        indices, patches = zip(*sliding_window(Amplitude, theta, patch_size=(Ni, Nj))) #use sliding_window get indices and patches
        labels = predict(patches, DEBUG) #predict zebra crossing for every patches 1 is zc 0 is background
        indices = np.array(indices)
        ret, location = getlocation(indices, labels, Ni, Nj)
        #draw
        # if DEBUG:
        #     for i, j in indices[labels == 1]:
        #         cv2.rectangle(img, (j, i), (j+Nj, i+Ni), (0, 0, 255), 3)
        if ret:
           cv2.rectangle(img, location[0], location[1], (255, 0, 255), 3)
        cv2.imshow("img", img)
        cv2.waitKey(1)

reason: import numpy,from numpy