import cv2
import numpy as np
import os
import shutil
import glob
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from keras.preprocessing.image import ImageDataGenerator
from main import _MAIN_PATH, DATA_PATH, content_directory, video_path, details, info_csv
from matplotlib import pyplot as plt
from keras import layers
import mediapipe as mp

total_files = 0
file_url = []
class_description = []

files_path =os.path.join(DATA_PATH,'Videos_Sentence_Level/')
for root, direct, files in os.walk(files_path):

    total_files += len(files)
    if root != files_path:

        for lab,url in zip([root.split('/')[-1] for i in files],files):
            class_description.append(lab)
            file_url.append(os.path.join(root,url))

video_labelled = pd.DataFrame(list(zip(file_url,class_description)),columns=["video location", "label"])

# ## import frames for words labelled

word_frame = pd.read_excel(os.path.join(DATA_PATH,'corpus_csv_files','ISL_CSLRT_Corpus_word_details.xlsx'))

frame_data = word_frame.copy()

frame_data["Frames path"] =  _MAIN_PATH +'/'+ frame_data["Frames path"]

def draw_styled_landmarks(image, results):
    mp_holistic = mp.solutions.holistic # Holistic model
    mp_drawing = mp.solutions.drawing_utils # Drawing utilities
    # Draw face connections
    mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACEMESH_TESSELATION,
                            mp_drawing.DrawingSpec(color=(80,110,10), thickness=1, circle_radius=1),
                            mp_drawing.DrawingSpec(color=(80,256,121), thickness=1, circle_radius=1)
                            )
    # Draw pose connections
    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS,
                            mp_drawing.DrawingSpec(color=(80,22,10), thickness=2, circle_radius=4),
                            mp_drawing.DrawingSpec(color=(80,44,121), thickness=2, circle_radius=2)
                            )
    # Draw left hand connections
    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
                            mp_drawing.DrawingSpec(color=(121,22,76), thickness=2, circle_radius=4),
                            mp_drawing.DrawingSpec(color=(121,44,250), thickness=2, circle_radius=2)
                            )
    # Draw right hand connections
    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
                            mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=4),
                            mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2)
                            )

def get_feature(sample,sample_label):
    mp_holistic = mp.solutions.holistic # Holistic model
    mp_drawing = mp.solutions.drawing_utils # Drawing utilities

    def mediapipe_detection(image, model):

        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB
        image.flags.writeable = False                  # Image is no longer writeable
        results = model.process(image)                 # Make prediction
        image.flags.writeable = True                   # Image is now writeable
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR
        return image, results

    cap = cv2.VideoCapture(sample)

    # Set mediapipe model
    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
        while cap.isOpened():

            # Read feed
            ret, frame = cap.read()
            if not ret:

                print("Can't receive frame (stream end?). Exiting ...")
                cap.release()
                cv2.destroyAllWindows()
                break
            # Make detections
            frame = cv2.resize(frame, (720, 480),interpolation = cv2.INTER_LINEAR)
            image, results = mediapipe_detection(frame, holistic)
            print(results)

            # Draw landmarks
            draw_styled_landmarks(image, results)
            # Show to screen

            org = (5, 40)

            # fontScale
            fontScale = 1

            # Blue color in BGR
            color = (38, 249, 255)

            # Line thickness of 2 px
            thickness = 1

            # Using cv2.putText() method
            font = cv2.FONT_HERSHEY_COMPLEX
            image = cv2.putText(image,f"Label: {sample_label}", org, font,
                            fontScale, color, thickness, cv2.LINE_AA)

            cv2.imshow('Joint Position & action ', image)

            # Break gracefully
            if cv2.waitKey(10) & 0xFF == ord('q'):
                break
        cap.release()
        cv2.destroyAllWindows()

sample = video_labelled['video location'].values[17]
sample_label = video_labelled['label'].values[17]

get_feature(sample,sample_label)

EXTRACT_FEATURE_VIDEO =False
def mediapipe_detection(image, model):

    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB
    image.flags.writeable = False                  # Image is no longer writeable
    results = model.process(image)                 # Make prediction
    image.flags.writeable = True                   # Image is now writeable
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR
    return image, results

def extract_keypoints(results):
    pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)
    face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)
    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)
    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)
    return np.concatenate([pose, face, lh, rh])

def get_results(sample):

    mp_holistic = mp.solutions.holistic # Holistic model
    mp_drawing = mp.solutions.drawing_utils # Drawing utilities
    cap = cv2.VideoCapture(sample)
    cap.set(cv2.CAP_PROP_FPS, 10)
    feat_list = []
    # Set mediapipe model
    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
        while cap.isOpened():

            # Read feed
            ret, frame = cap.read()

            try :
                frame = cv2.resize(frame, (720, 480),interpolation = cv2.INTER_LINEAR)
                image, results = mediapipe_detection(frame, holistic)
                feat_list.append(results)
            except:
                cap.release()
                return np.array([extract_keypoints(result) for result in feat_list])

def get_frame_feature(sample):
    mp_holistic = mp.solutions.holistic # Holistic model
    mp_drawing = mp.solutions.drawing_utils # Drawing utilities
    frame = cv2.imread(sample)
    # Set mediapipe model
    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
        try :
            frame = cv2.resize(frame, (720, 480),interpolation = cv2.INTER_LINEAR)
            image, results = mediapipe_detection(frame, holistic)
            return results
        except:
            return np.zeros((1,1662))

frame_meta = []
frame_label = []
for idx,(word, frame) in enumerate(zip(frame_data["Word"],frame_data["Frames path"])):
    if idx%10 == 0:
        print(f"processed frames {idx} remaining {frame_data.shape[0]-idx}")
    frame_meta.append(extract_keypoints(get_frame_feature(frame)))
    frame_label.append(word)

frame_meta = np.array(frame_meta)

if EXTRACT_FEATURE_VIDEO == True:

    lab_det = []
    feat_fname = []
    idx = 0
    for sample,label in zip(video_labelled['video location'],video_labelled['label']):

        lab_det.append(label)
        feat_fname.append(os.path.join('features',f"{idx}.npy"))
        print(f"file {sample} in progress")
        np.save(os.path.join('features',f"{idx}.npy"),get_results(sample))
        idx += 1
        # np.save(os.path.join('features',f"{idx}.npy"),feat)
    pd.DataFrame(np.array([lab_det,feat_fname]).transpose(),columns=['label','stored feature']).to_csv('feature_details.csv')

FEATURE_PATH =  'features'

feature_data = pd.read_csv('feature_details.csv')

label_video = feature_data['label']
meta_video= feature_data['stored feature']

meta_data = []
for file in meta_video:
    meta_data.append(np.load(file))

shp = [da.shape[0] for da in meta_data]

meta_data = np.array(meta_data)
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical

enc =  LabelEncoder()
word_enc = LabelEncoder()
y_int =enc.fit_transform(label_video)
y_int_frame = word_enc.fit_transform(frame_label)

y = to_categorical(y_int)
y_frame = to_categorical(y_int_frame)

fdata = np.expand_dims(frame_meta,axis=1)

X_train, X_test, y_train, y_test = train_test_split(fdata, y_frame, test_size=0.4)

from keras.models import Sequential
from keras.layers import LSTM, Dense
from keras.callbacks import TensorBoard

log_dir = os.path.join('Logs')
tb_callback = TensorBoard(log_dir=log_dir)

model = Sequential()
model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(1,1662)))
model.add(LSTM(128, return_sequences=True, activation='relu'))
model.add(LSTM(64, return_sequences=False, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dense(y_train.shape[1], activation='softmax'))

model.summary()

model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy'])

MODEL_RETRAIN = False

if MODEL_RETRAIN == True:
    model.fit(fdata, y_frame, epochs=1000, callbacks=[tb_callback])
else:
    model = tf.keras.models.load_model('model.h5')

sentence = []
print('Testing Models in sentence data')
for idx, test in enumerate(meta_data[1:276]):
    sentence.append(' '.join(list(word_enc.inverse_transform(np.argmax(model.predict(np.expand_dims(test, axis=1)),axis=1)))))

from confusion import confusion

met = confusion(np.array(label_video[1:276]), np.array(sentence))

cm = met.getmatrix()

[acc, pre, rec, fsc] = met.metrics()

if not os.path.exists('model.h5'):
    model.save('model.h5')

def read_test_data(sample, model, word_enc):
    mp_holistic = mp.solutions.holistic
    mp_drawing = mp.solutions.drawing_utils

    cap = cv2.VideoCapture(sample)

    fps = cap.get(cv2.CAP_PROP_FPS)

    frame_interval = round(fps / 24)
    list_frm = []

    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
        while cap.isOpened():
            ret, frame = cap.read()

            if ret:

                if cap.get(cv2.CAP_PROP_POS_FRAMES) % frame_interval == 0:
                    frame = cv2.resize(frame, (720, 480),interpolation = cv2.INTER_LINEAR)
                    image, results = mediapipe_detection(frame, holistic)
                    # print(extract_keypoints(results))
                    pred = word_enc.inverse_transform(np.argmax(model.predict(np.expand_dims(np.expand_dims(extract_keypoints(results),axis=0), axis=1)),axis=1))
                    draw_styled_landmarks(image,results)
                    org = (5, 40)
                    fontScale = 1
                    color = (38, 249, 255)
                    thickness = 1
                    font = cv2.FONT_HERSHEY_COMPLEX
                    image = cv2.putText(image,f"sign: {pred}", org, font,
                                    fontScale, color, thickness, cv2.LINE_AA)

                    cv2.imshow('test data', image)

                    if cv2.waitKey(10) & 0xFF == ord('q'):
                        cap.release()
                        cv2.destroyAllWindows()
                        break

            else:
                    cap.release()
                    cv2.destroyAllWindows()

                    break

SHOW_VIDEOS = False
if SHOW_VIDEOS == True:
    n = 120

    read_test_data(video_labelled['video location'][n], model, word_enc)

    print(f"original {video_labelled['label'][n]}")


import pytest
from freezegun import freeze_time

from imdbpie.auth import Auth

@pytest.fixture
def auth():
    auth_ = Auth()
    auth_._set_creds({'expirationTimeStamp': '2018-01-12T06:23:05Z'})
    yield auth_
    auth_.clear_cached_credentials()

@pytest.mark.parametrize(
    'current_datetime, exp_expired',
    [
        ('2018-01-12T06:23:05Z', True),  # matching expiry time
        ('2018-01-12T06:24:05Z', True),  # 1 min after expiry time
        ('2018-01-12T06:23:04Z', True),  # 1 sec before, must be 60+ before
        ('2018-01-12T06:22:06Z', True),  # 59 sec before, must be 60+ before
        ('2018-01-12T06:22:05Z', False),  # 60 sec before, must be 60+ before
        ('2018-01-11T06:23:05Z', False),  # 1 day before expiry time
        ('2017-11-11T06:23:05Z', False),  # a few weeks before expiry time
    ],
)
def test_creds_soon_expiring(auth, current_datetime, exp_expired):
    with freeze_time(current_datetime):
        assert auth._creds_soon_expiring()[1] is exp_expired


import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import TechNews_sql as sql
import TechNews_mongo as mg

def get_news_content(url):
    html = requests.get(url)
    soup = BeautifulSoup(html.text, 'html.parser')
    return soup.select_one('div.content div.indent')

# 把Ｐ標籤底下有Strong的Ｐ標籤移除
def extract_p_strong(content):
    for tag in content.findAll(['p','h3']):
        if(tag.name == 'p'):
            for strong in tag.findAll('strong'):
                strong.extract()

# sum的檔案名稱
def get_sum_file_name(df, i):
    return './'+'sum_'+df['category'][i]+'_'+str.replace(df['sum_title'][i][0:4], " ", "_")+'.txt'

# spot的檔案名稱
def get_spot_file_name(df, i, j):
    return './'+'spot_'+df['category'][i]+'_'+str.replace(df['spotlist'][i][j]['title'][0:4], " ", "_")+'.txt'

# 寫入檔案
def write_to_file(file_name):
    _content = ""
    with open(file_name, 'w') as file:
        for tag in content.findAll(['p','h3']):
            file.write(tag.text+'\n')
            _content += tag.text+'\n'

    return _content

#main1_2 start
df = pd.read_json('./exam1_1.json')

# 存sum的檔案
for i,url in enumerate(df['sum_title_url']):
    time.sleep(2)
    content = get_news_content(url)
    extract_p_strong(content)

    file_name = get_sum_file_name(df, i)
    print('write news to',file_name)
    # file
    content = write_to_file(file_name)
    # mysql
    sql.insert_news_content({"title":df['sum_title'][i], "content":content})
    # mongodb
    mg.insert_sum_content({"title":df['sum_title'][i], "content":content})

# 存spot的檔案
for i,spotlist in enumerate(df['spotlist']):
    for j,spot in enumerate(spotlist):
        time.sleep(2)
        content = get_news_content(spot['url'])
        extract_p_strong(content)

        file_name = get_spot_file_name(df, i, j)
        print('write news to',file_name)
        # file
        content = write_to_file(file_name)
        # mysql
        sql.insert_news_content({"title":df['spotlist'][i][j]['title'], "content":content})
        # mongodb
        mg.insert_spot_content({"title":df['spotlist'][i][j]['title'], "content":content, "index":j})


import socket
import logging
import pickle

from typing import List
from shared import *

# We maintain a module-level logger.
logger = logging.getLogger(__name__)

class GenericSocketUser(object):
    """ Class to standardize message send and receipt. """
    # The first portion of a message, the length, is of fixed size. (2^8 maximum message length in bytes)
    MESSAGE_LENGTH_BYTE_SIZE = 8

    def __init__(self, client_socket: socket.socket = None):
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if client_socket is None else client_socket

    def read_message(self, client_socket: socket.socket = None):
        """ Read message_length bytes from the specified socket, and deserialize our message. """
        working_socket = self.socket if client_socket is None else client_socket

        try:
            # TODO: Make this a tunable parameter... this is distinct from failure_time.
            working_socket_previous_timeout = working_socket.gettimeout()
            working_socket.settimeout(10)

            # Read our message length.
            chunks, bytes_read = [], 0
            while bytes_read < GenericSocketUser.MESSAGE_LENGTH_BYTE_SIZE:
                chunk = working_socket.recv(GenericSocketUser.MESSAGE_LENGTH_BYTE_SIZE - bytes_read)
                if chunk == b'':
                    raise EOFError("Working socket has been closed.")

                chunks.append(chunk)
                bytes_read += len(chunk)

            # Obtain our length.
            message_length = int.from_bytes(b''.join(chunks), byteorder='big')
            logger.debug(f'Reading message of length: {message_length}')

            # Repeat for the message content.
            chunks, bytes_read = [], 0
            while bytes_read < message_length:
                chunk = working_socket.recv(min(message_length - bytes_read, 2048))
                if chunk == b'':
                    raise EOFError("Working socket has been closed.")

                chunks.append(chunk)
                bytes_read += len(chunk)

            # Reconstruct message, and deserialize.
            received_message = pickle.loads(b''.join(chunks))
            logger.debug(f'Received message: {received_message}')
            working_socket.settimeout(working_socket_previous_timeout)
            return received_message

        except Exception as e:
            logger.warning(f"Exception caught: {e}")
            self.close(working_socket)
            return None

    def send_message(self, op_code: OpCode, contents: List, client_socket: socket.socket = None) -> bool:
        """ Correctly format a message to send to another socket user. """
        working_socket = self.socket if client_socket is None else client_socket
        message = [op_code] + contents

        serialized_message = pickle.dumps(message)
        message_length = len(serialized_message).to_bytes(GenericSocketUser.MESSAGE_LENGTH_BYTE_SIZE, byteorder='big')
        try:
            logger.debug(f"Sending message length: {len(serialized_message)} | {message_length}")
            working_socket.sendall(message_length)
            logger.debug(f"Sending message: {serialized_message}.")
            working_socket.sendall(serialized_message)
            return True

        except Exception as e:
            logger.warning(f"Exception caught: {e}")
            self.close(working_socket)
            return False

    def send_op(self, op_code: OpCode, client_socket: socket.socket = None) -> bool:
        """ Correctly format a OP code to send to another socket user. """
        working_socket = self.socket if client_socket is None else client_socket
        message = [op_code]

        serialized_message = pickle.dumps(message)
        message_length = len(serialized_message).to_bytes(GenericSocketUser.MESSAGE_LENGTH_BYTE_SIZE, byteorder='big')
        try:
            logger.debug(f"Sending message length: {len(serialized_message)} | {message_length}")
            working_socket.sendall(message_length)
            logger.debug(f"Sending message: {serialized_message}.")
            working_socket.sendall(serialized_message)
            return True

        except Exception as e:
            logger.warning(f"Exception caught: {e}")
            self.close(working_socket)
            return False

    def send_response(self, response_code: ResponseCode, client_socket: socket.socket = None) -> bool:
        """ Correctly format a response code to send to another socket user. """
        working_socket = self.socket if client_socket is None else client_socket
        message = [response_code]

        serialized_message = pickle.dumps(message)
        message_length = len(serialized_message).to_bytes(GenericSocketUser.MESSAGE_LENGTH_BYTE_SIZE, byteorder='big')
        try:
            logger.debug(f"Sending message length: {len(serialized_message)} | {message_length}")
            working_socket.sendall(message_length)
            logger.debug(f"Sending message: {serialized_message}.")
            working_socket.sendall(serialized_message)
            return True

        except Exception as e:
            logger.warning(f"Exception caught: {e}")
            self.close(working_socket)
            return False

    def close(self, client_socket: socket.socket = None):
        logger.info("'Close' called. Releasing socket(s).")
        try:
            if client_socket is not None:
                client_socket.close()
            if self.socket is not None:
                self.socket.close()

        except Exception as e:
            logger.error(f"Exception caught while trying to close the socket. Swallowing: {e}")


import numba
import numpy as np

from pgfa.math_utils import log_normalize

def get_sample_data_point(a, b, cn_major, cn_minor, cn_normal=2, error_rate=1e-3, tumour_content=1.0):
    cn_total = cn_major + cn_minor

    cn = []

    mu = []

    log_pi = []

    # Consider all possible mutational genotypes consistent with mutation before CN change
    for x in range(1, cn_major + 1):
        cn.append((cn_normal, cn_normal, cn_total))

        mu.append((error_rate, error_rate, min(1 - error_rate, x / cn_total)))

        log_pi.append(0)

    # Consider mutational genotype of mutation before CN change if not already added
    mutation_after_cn = (cn_normal, cn_total, cn_total)

    if mutation_after_cn not in cn:
        cn.append(mutation_after_cn)

        mu.append((error_rate, error_rate, min(1 - error_rate, 1 / cn_total)))

        log_pi.append(0)

    cn = np.array(cn, dtype=np.int)

    mu = np.array(mu, dtype=np.float)

    log_pi = log_normalize(np.array(log_pi, dtype=np.float64))

    return SampleDataPoint(int(a), int(b), cn, mu, log_pi, tumour_content)

class DataPoint(object):

    def __init__(self, sample_data_points):
        self.sample_data_points = sample_data_points

@numba.experimental.jitclass([
    ('b', numba.int64),
    ('d', numba.int64),
    ('cn', numba.int64[:, :]),
    ('mu', numba.float64[:, :]),
    ('log_pi', numba.float64[:]),
    ('tumour_content', numba.float64)
])
class SampleDataPoint(object):

    def __init__(self, a, b, cn, mu, log_pi, tumour_content=1.0):
        self.b = b
        self.d = a + b
        self.cn = cn
        self.mu = mu
        self.log_pi = log_pi
        self.tumour_content = tumour_content


from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect, \
        HttpResponseNotFound, HttpResponseForbidden
from django.views.generic import View, DetailView, ListView
import json
from friendship.models import Friend, Follow

from .models import User, List, ListItem, TopicTag, Comment, \
        Reblog, Like, Favorite, BrowseHistory

from messages.models import Message
from .forms import AddListForm
from django.views.decorators.cache import never_cache

def needs_view_permission(view_method):
    def check_permissions(self, request, slug=''):
        not_found_response = HttpResponseNotFound()
        try:
            ls = List.objects.get(slug=slug)
        except List.DoesNotExist:
            return not_found_response

        if ls.sufficent_view_permissions(request.user):
            return view_method(self, request, ls)
        else:
            return not_found_response
    return check_permissions

def needs_write_permission(write_method):
    def check_permissions(self, request, slug=''):
        not_found_response = HttpResponseNotFound()
        forbidden_response = HttpResponseForbidden()
        try:
            ls = List.objects.get(slug=slug)
        except List.DoesNotExist:
            return not_found_response

        if ls.owner == request.user:
            return write_method(self, request, ls)
        else:
            return forbidden_response
    return check_permissions

class ListDetailView(View):
    @needs_view_permission
    def get(self, request, ls):
        bh = BrowseHistory(user=request.user, list=ls)
        bh.save()
        context = self.context_data(request, ls)
        return render(request, 'lists/listdetail.html', context)

    def context_data(self, request, ls):
        context = {}
        context['l'] = ls
        user = request.user
        num_likes = Like.objects.filter(list=ls).count()
        context['num_likes'] = num_likes
        context['own_list'] = user == ls.owner

        try:
            like = Like.objects.get(owner=user, list=ls)
            context['user_likes'] = True
        except Like.DoesNotExist:
            context['user_likes'] = False

        try:
            favorite = Favorite.objects.get(owner=user, list=ls)
            context['user_favorited'] = True
        except Favorite.DoesNotExist:
            context['user_favorited'] = False

        try:
            reblog = Reblog.objects.get(owner=user, list=ls)
            context['user_reblogged'] = True
        except Reblog.DoesNotExist:
            context['user_reblogged'] = False

        return context

class HotListsView(ListView):
    model = List
    context_object_name= 'ls'
    template_name = 'lists/hotlists.html'

    def get_context_data(self, **kwargs):
        context = super(HotListsView, self).get_context_data(**kwargs)
        user = self.request.user
        context['ls'] = List.objects.filter(privacy=List.PUBLIC_VISIBILITY)
        return context

# Create your views here.
class AddListView(View):
    def get(self, request):
        current_user = request.user
        return render(request, 'lists/addlist/index.html')

    def post(self, request):
        ls = json.loads(request.body)
        # print ls
        # save list json from request into DB
        newList = List(owner=request.user, \
                title=ls["title"], \
                num_items=ls["number"], \
                privacy=ls["privacy"])

        newList.save()
        for listItem in ls["list"]:
            newListItem = ListItem(listid=newList, \
                    title=listItem["title"], \
                    descriptionhtml=listItem["description"], \
                    descriptionmeta=listItem["description_meta"])
            newListItem.save()

        slug_dict = {
            'slug': newList.slug
        }

        for tagChoiceID in ls["tags"]:
            newTopicTag = TopicTag(list=newList, topic=tagChoiceID)
            newTopicTag.save()

        friends = Friend.objects.friends(request.user)
        for friend in friends:
            list_notification = Message(type='LN', to_user=friend, from_user=request.user, content="I've added a new list called " + newList.title + ". Check it out!")
            list_notification.save()
        return HttpResponse(json.dumps(slug_dict), status=201, \
                content_type='application/json')

class DeleteView(View):

    @needs_write_permission
    def post(self, request, ls):
        ls.delete()
        return HttpResponseRedirect(request.META.get('HTTP_REFERER'))

class EditListView(View):
    @needs_write_permission
    def get(self, request, ls):
        return render(request, 'lists/editlist/index.html', {'listSlug': ls.slug})

    @needs_write_permission
    def post(self, request, ls):
        list = json.loads(request.body)
        ls.title = list['title']
        ls.num_items = list['number']
        ls.privacy = list['privacy']
        ls.save()

        for item in ListItem.objects.filter(listid=ls):
            item.delete()

        for item in list['list']:
            newItem = ListItem(listid=ls, title=item['title'],\
                    descriptionhtml=item['description'],
                    descriptionmeta=item['description_meta'])
            newItem.save()

        for topicTag in TopicTag.objects.filter(list=ls):
            topicTag.delete()

        for tagChoiceID in list['tags']:
            newTopicTag = TopicTag(list=ls, topic=tagChoiceID)
            newTopicTag.save()

        slug_dict = {
            'slug': ls.slug
        }

        return HttpResponse(json.dumps(slug_dict), status=201, \
                content_type='application/json')

class GetListData(View):
    @needs_view_permission
    def get(self, request, ls):
        items = ListItem.objects.filter(listid=ls)
        def get_item_info(item):
            return {
                'title': item.title,
                'description': item.descriptionhtml,
            }

        tags = TopicTag.objects.filter(list=ls)

        list_data = {
            'title': ls.title,
            'number': ls.num_items,
            'privacy': ls.privacy,
            'list': map(get_item_info, items),
            'tags': map(lambda tag: tag.topic, tags),
        }

        return HttpResponse(json.dumps(list_data), status=200, \
                content_type='application/json')

class DeleteComment(View):
    def get(self, request, comment_id):
        try:
            comment = Comment(id=comment_id)
            comment.delete()
        except Comment.DoesNotExist:
             return HttpResponseRedirect(request.META.get('HTTP_REFERER'), status=404)
        return HttpResponse(request.META.get('HTTP_REFERER'))

class PostComment(View):
    @needs_view_permission
    def post(self, request, ls):
        comment_content = request.POST.get('comment_content')
        comment = Comment(
                    list=ls,
                    owner=request.user,
                    content=comment_content
        )
        comment.save()
        return HttpResponseRedirect(request.META.get('HTTP_REFERER'))

'''
The following three view classes are
extremely similar, they act as toggles

e.g. for like whenever a user sends a request
for the like endpoints, the users likes
the list if the user hasn't already like it.

Sending a request to the same endpoint
counts as an unlike.
'''
class LikeList(View):
    @needs_view_permission
    @never_cache
    def get(self, request, ls):
        user = request.user
        redirect_response = HttpResponseRedirect(request.META.get('HTTP_REFERER'))
        try:
            like = Like.objects.get(list=ls, owner=user)
            like.delete()
        except Like.DoesNotExist:
            like = Like(list=ls, owner=user)
            like.save()
        return redirect_response

class ReblogList(View):
    @needs_view_permission
    @never_cache
    def get(self, request, ls):
        user = request.user
        forbidden_response = HttpResponseForbidden()
        redirect_response = HttpResponseRedirect(request.META.get('HTTP_REFERER'))

        if user == ls.owner:
            return forbidden_response

        try:
            reblog = Reblog.objects.get(list=ls, owner=user)
            reblog.delete()
        except Reblog.DoesNotExist:
            reblog = Reblog(list=ls, owner=user)
            reblog.save()
        return redirect_response

class FavoriteList(View):
    @needs_view_permission
    @never_cache
    def get(self, request, ls):
        user = request.user
        redirect_response = HttpResponseRedirect(request.META.get('HTTP_REFERER'))
        try:
            favorite = Favorite.objects.get(list=ls, owner=user)
            favorite.delete()
        except Favorite.DoesNotExist:
            favorite = Favorite(list=ls, owner=user)
            favorite.save()
        return redirect_response


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('rtiapp', '0042_blog_num_views'),
    ]

    operations = [
        migrations.AlterField(
            model_name='user_profile',
            name='reputation',
            field=models.FloatField(default=0),
            preserve_default=True,
        ),
    ]


#!/usr/bin/env python
# coding: utf-8
import textstat
from textstat import textstat as tx

def get_stat(test_data):
    a = textstat.flesch_reading_ease(test_data)
    b = textstat.flesch_kincaid_grade(test_data)
    c = textstat.smog_index(test_data)
    d = textstat.coleman_liau_index(test_data)
    e = textstat.automated_readability_index(test_data)
    f = textstat.dale_chall_readability_score(test_data)
    g = textstat.difficult_words(test_data)
    h = textstat.linsear_write_formula(test_data)
    i = textstat.gunning_fog(test_data)
    j = textstat.text_standard(test_data)
    k = textstat.fernandez_huerta(test_data)
    l = textstat.szigriszt_pazos(test_data)
    m = textstat.gutierrez_polini(test_data)
    n = textstat.crawford(test_data)
    o = textstat.gulpease_index(test_data)
    p = textstat.osman(test_data)
    x = tx.difficult_words_list(test_data)

    result = {"Flesch Reading Ease":a,
              "Flesch Kincaid Grade":b,
              "SMOG Index":c,
              "Coleman Liau Index":d,
              "Automated Readability Index":e,
              "Dale Chall Readability Score":f,
              "Difficult Words":g,
              "Linsear Write Formula":h,
              "Gunning Fog":i,
              "Text Standard":j,
              "Fernandez Huerta":k,
              "Szigriszt Pazos":l,
              "Gutierrez Polini":m,
              "Crawford":n,
              "Gulpease Index":o,
              "Osman":p,
              "Words List":x}
    return result


from setuptools import setup, find_packages

with open('README.rst') as readme_file:
    readme = readme_file.read()

with open('HISTORY.rst') as history_file:
    history = history_file.read()

requirements = [
    'cryptography==40.0.1',
    'wagtail>4.0,<5.1',
    'html2text==2020.1.16',

]
test_requirements = ['pytest>=3']

setup(
    author="Marco Westerhof",
    author_email='m.westerhof@lukkien.com',
    python_requires='>=3.7',
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Natural Language :: English',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
    ],
    description="Simplified A/B testing for wagtail",
    install_requires=requirements,
    license="BSD license",
    long_description=readme + '\n\n' + history,
    include_package_data=True,
    keywords='wagtune',
    name='wagtune',
    packages=find_packages(include=['wagtune', 'wagtune.*']),
    test_suite='tests',
    tests_require=test_requirements,
    url='https://github.com/mwesterhof/wagtune',
    version='0.1.0',
    zip_safe=False,
)


import json
from django.shortcuts import get_object_or_404, render
from rest_framework import status

from rest_framework import viewsets
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated
from ticker.models import Ticker

from users.models import User
from backtest.models import Backtest

from backtest.strategies.backtesting import backtest

from backtest.serializers import BacktestSerializer
from ticker.serializers import TickerSerializer, TimeSeriesSerializer

class BacktestViewSet( viewsets.GenericViewSet ):
    model = Backtest
    serializer_class = BacktestSerializer
    permission_classes = [IsAuthenticated]

    def get_object(self, pk):
        return get_object_or_404( self.model, pk=pk )

    def get_queryset(self, email):
        if self.queryset is None:
            self.queryset = self.model.objects.all()

        # self.queryset = self.model.objects.filter(email=email)
        return self.queryset

    def create(self, request):
        print(request.data )

        user = User.objects.filter( email=request.data.get( 'email' ) ).first()
        ticker = Ticker.objects.filter(symbol=request.data["ticker"]).first()
        margenData = request.data.get('margen')
        margen = None
        for item in Backtest.MARGENS:
            if item[1] == margenData:
                margen = item[0]
        backtest = Backtest(ticker=ticker,capital=request.data["capital"],
            date_start =request.data["dateStart"], date_end=request.data["dateEnd"],
            interval=request.data["interval"], indicators_data=request.data["indicatorsData"] ,
            user= user, margen=margen )

        backtest.save()

        return Response(
            {
                'message' : 'Backtest guardado exitosamente!....',
                "data": self.serializer_class(backtest).data
            }, status=status.HTTP_201_CREATED
        )
        """ return Response(
            {
                'message': 'Hay errores en el registro.',
                'errores': user_serializer.errors
            }, status=status.HTTP_400_BAD_REQUEST
        ) """

    def retrieve( self, request, pk=None ):
        user = self.get_object( pk )
        user_serializer = self.serializer_class( user )
        return Response( user_serializer.data )

    def update( self, request, pk=None ):
        user = self.get_object( pk )
        user_serializer = self.update_user_serializer( user, data=request.data )
        if user_serializer.is_valid():
            user_serializer.save()
            return Response(
                {
                    'message': 'Backtest actualizado correctamente.'
                }, status=status.HTTP_200_OK
            )
        return Response(
            {
                'message': 'Hay errores en la actualizacion.',
                'errores': user_serializer.errors
            }, status=status.HTTP_400_BAD_REQUEST
        )

    def destroy( self, request, pk=None ):
        self.model.objects.filter(pk=pk).delete()
        return Response(
                {
                    'message': 'Backtest eliminado correctamente.'
                }, status=status.HTTP_200_OK
            )

class BackTestAPIView( APIView ):
    permission_classes = [IsAuthenticated]

    def post( self, request):
        print( request.data )

        code, resumen, report= backtest(
            request.data["ticker"], request.data["interval"],
            request.data["capital"], request.data["dateStart"],
            request.data["dateEnd"], request.data["indicatorsData"],
            request.data["margen"],
        )

        if(code==1):

            data = { "resumen": resumen, "report": report }

            return Response(
                {
                    "code": code,
                    'message': 'Backtest Exitoso',
                    "data": data
                },
                status = status.HTTP_200_OK
            )
        if(code==2):
            return Response(
                { "code": code, 'message': 'El ticker seleccionado no posee datos historicos.' },
                status = status.HTTP_200_OK
            )
        if(code==3):
            return Response(
                { "code": code, 'message': 'La configuracion actual no consiguio ningun resultado.' },
                status = status.HTTP_200_OK
            )

class ListBackTestAPIView( APIView ):
    permission_classes = [IsAuthenticated]

    model = Backtest
    serializer_class = BacktestSerializer
    queryset = None

    def get_queryset(self, email):
        if self.queryset is None:
            user = User.objects.filter(email=email).first()
            self.queryset = self.model.objects.filter(user=user)
        return self.queryset

    def post( self, request):
        backtests = self.get_queryset( request.data.get('email') )
        backtests_serialzers = self.serializer_class(backtests, many=True)
        return Response( backtests_serialzers.data, status=status.HTTP_200_OK )


import gym
import numpy as np
from gym_yagw.envs.rendering import constants
import os

class YagwEnv(gym.Env):
    """
    Simple GridWorld implementation, below is an example of a 5x5 grid world

    |  X  | -1  | -1  | -1  | -1  |
    | -1  | -1  | -1  | -1  | -1  |
    | -1  | -1  | -1  | -1  | -1  |
    | -1  | -1  | -1  | -1  | -1  |
    | -1  | -1  | -1  | -1  | +1  |

    """

    def __init__(self, width=5, height=5):
        """
        Initializes the environment

        Observation:
            Type: Box(2)
            Num	Observation                 Min         Max
            0	column                      0           width-1
            1	row                         0           height-1

        Actions:
            Type: Discrete(4)
            Num	Action
            0	Left
            1	Right
            2   Up
            3   Down

        Reward:
            Reward is -1 for every step except when the agent moves to the goal state, where the reward is +1

        Starting State:
            (0, 0)

        Episode Termination:
            When agent reaches goal state

        :param width: the width of the grid
        :param height: the height of the grid
        """
        self.width = width
        self.height = height
        self.num_states = self.width * self.height
        self.x = 0
        self.y = 0
        self.action_descriptors = ["Left", "Right", "Up", "Down"]
        self.num_actions = len(self.action_descriptors)
        self.goal_state = [width - 1, height - 1]
        # Observation is a true Markov state of the form: [x,y] where x in [0, width) and y in [0, height)
        self.observation_space = gym.spaces.Box(low=np.array([0.0,0.0]),
                                                high=np.array([float(width-1), float(height-1)]), dtype=np.float32)
        # Action space is an integer in [0,4)
        self.action_space = gym.spaces.Discrete(4)
        self.viewer = None
        self.steps_beyond_done = None
        self.rect_size = 50
        self.metadata = {
        'render.modes': ['human', 'rgb_array'],
        'video.frames_per_second' : 50 # Video rendering speed
        }
        self.reward_range = (-float('inf'), float(1))

    def step(self, action):
        """
        Takes a step in the environment using the given action.

        When end of episode is reached, the caller is responsible for calling `reset()`
        to reset this environment's state.

        :param action: the action to take in the environment
        :return:
            observation (object): agent's observation of the current environment
            reward (float) : amount of reward returned after previous action
            done (bool): whether the episode has ended, in which case further step() calls will return undefined results
            info (dict): contains auxiliary diagnostic information (helpful for debugging, and sometimes learning)
        """
        if self.action_descriptors[action] == "Left":
            if self.x > 0:
                self.x = self.x-1
        elif self.action_descriptors[action] == "Right":
            if self.x < self.width-1:
                self.x = self.x + 1
        elif self.action_descriptors[action] == "Up":
            if self.y > 0:
                self.y = self.y - 1
        elif self.action_descriptors[action] == "Down":
            if self.y < self.height - 1:
                self.y = self.y + 1

        state = self.__get_state()
        reward = 1 if np.array_equal(state, self.goal_state) else -1 # Reward is -1 if not at goal state, otherwise it is 1
        done = False
        if reward == 1:
            done = True
            if self.steps_beyond_done is None:
                self.steps_beyond_done = 0
            else:
                gym.logger.warn("You are calling 'step()' even though this environment has already returned done = True. "
                                "You should always call 'reset()' once you receive 'done = True' -- "
                                "any further steps are undefined behavior.")
                self.steps_beyond_done +=1
        if self.viewer is not None:
            self.viewer.gridframe.set_state(self.__get_state())
        return state, reward, done, {}

    def reset(self):
        """
        Resets the environment and returns the initial state

        :return: the initial state
        """
        self.steps_beyond_done = None
        self.x = 0
        self.y = 0
        if self.viewer is not None:
            self.viewer.gridframe.reset()
        return self.__get_state()

    def render(self, mode='human'):
        """
        Renders the environment

        Supported rendering modes:

        - human: render to the current display or terminal and
          return nothing. Usually for human consumption.
        - rgb_array: Return an numpy.ndarray with shape (x, y, 3),
          representing RGB values for an x-by-y pixel image, suitable
          for turning into a video.

        :param mode: the rendering mode
        :return: True (if human mode) otherwise an rgb array
        """
        if mode not in self.metadata["render.modes"]:
            raise NotImplemented("mode: {} is not supported".format(mode))
        if self.viewer is None:
            self.__setup_viewer()
        return self.viewer.render(return_rgb_array = mode=='rgb_array')

    def __setup_viewer(self):
        """
        Setup for the viewer to use for rendering
        :return: None
        """
        from gym_yagw.envs.rendering.viewer import Viewer
        script_dir = os.path.dirname(__file__)
        resource_path = os.path.join(script_dir, './rendering/', constants.GRIDWORLD.RESOURCES_DIR)
        self.viewer = Viewer(width=self.width*self.rect_size,
                             height=(self.height*self.rect_size) + constants.GRIDWORLD.PANEL_HEIGHT,
                             rect_size=self.rect_size,
                             resources_dir=resource_path)
        self.viewer.agent_start()

    def close(self):
        """
        Closes the viewer (cleanup)

        :return: None
        """
        if self.viewer:
            self.viewer.close()
            self.viewer = None

    def __get_state(self):
        """
        Returns a feature representation of the state

        :return: [x,y] list
        """
        return np.array([self.x, self.y])

    def print_gridworld(self):
        """
        Utility function for printing the grid world and the agent's current position

        :return: None
        """
        for i in range(0, self.height):
            print("|", end='')
            for j in range(0, self.width):
                if i == self.y and j == self.x:
                    print("  X  ", end='')
                elif np.all(np.array([i,j]), self.goal_state):
                    print(" +1  ", end='')
                else:
                    print(" -1  ", end='')
                print("|", end='')
            print("")

    def get_state_index(self, state):
        """
        Utility function for getting the index of a state (useful for tabular algorithms)

        :param state: the feature representation of the state
        :return: the index of the state
        """
        return state[1]*self.width + state[0]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Oct 27 14:19:49 2019

@author: diego
"""
from selenium import webdriver
import os
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec

start=time.time()
#Codigo para manejar el buscador

chromedriver = "/home/diego/Documentos/chromedriver"
#chromedriver="/home/diego/anaconda3/bin/geckodriver"
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
#driver = webdriver.Firefox()

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}
my_url="https://www.eleconomista.com.mx/buscar/morena#"

driver.get(my_url)
#Create a new csv file
filename="economista_news.csv"
f=open(filename,"w")
#Define headers within the csv file
head="Id|Date|Title|Text|\n"
f.write(head)
#Define my flags and counters
i,flag,loop,top=0,1,0,100
aux=[]

while i<top:
    #print("The number of loop is: ",loop,flag)
    if loop>0:
        for r in range(loop):
            botton=driver.find_element_by_xpath('//*[@title="Buscar más noticias"]')
            driver.execute_script("arguments[0].click();", botton)
            time.sleep(3)
        my_url=driver.current_url
        driver.get(my_url)
        containers=driver.find_elements_by_xpath('//*[@class="entry-box-overlay"]/a[@title]')
        titles=[y.get_attribute("title") for y in containers]
        titles=[l for l in titles if l not in aux]
    else:
        containers=driver.find_elements_by_xpath('//*[@class="entry-box-overlay"]/a[@title]')
        titles=[y.get_attribute("title") for y in containers]
    #print("Longitudes: ", len(titles)," ", len(aux))

    for j in titles:
        data_text=""
        texto="//*"+"[@title="+'"'+j+'"'+"]"
        try:
            element=driver.find_element_by_xpath(texto)
        except:
            print("Entro")
            element=driver.find_element_by_xpath(texto)
        #wait = WebDriverWait(driver, 10)
        #result = wait.until(ec.element_selection_state_to_be(element, True))
        driver.execute_script("arguments[0].click();",element)
        my_url=driver.current_url
        driver.get(my_url)
        time.sleep(3)
        date=driver.find_element_by_xpath('//*[@class="article-date"]/time')
        txt=driver.find_elements_by_xpath('//*[@class="entry-body"]/p')
        for k in txt:
            data_text+=k.text+" "
        if len(date.text)<10:
            time.sleep(5)
            date=driver.find_element_by_xpath('//*[@class="article-date"]/time')

        if len(data_text)<25:
            time.sleep(3)
            txt=driver.find_elements_by_xpath('//*[@class="entry-body"]/p')
            for k in txt:
                data_text+=k.text+" "
        f.write(str(i)+"|"+date.text+"|"+j+"|"+data_text+"|\n")
        print(date.text,len(data_text),i)
        driver.back()
        time.sleep(5)
        my_url=driver.current_url
        driver.get(my_url)
        if loop>0:
            for h in range(loop+1):
                botton=driver.find_element_by_xpath('//*[@title="Buscar más noticias"]')
                driver.execute_script("arguments[0].click();", botton)
                time.sleep(2)
        my_url=driver.current_url
        driver.get(my_url)
        aux.append(j)
        i+=1
    #Define the number of loop.
    loop+=1
    driver.refresh()
    #flag=2*loop

f.close()
driver.close()
print("This code took :",time.time()-start)


import pytest
from account.models import Account
from tests.account.factories import AccountFactory

@pytest.mark.django_db
def test_user__str__():
    instance = AccountFactory()
    assert instance.id
    assert instance.email
    assert instance.first_name
    assert isinstance(instance, Account)


""" Create a Spotify playlist with the tracks from A3.30"""
from dotenv import load_dotenv
import bs4
import requests
import spotipy
from spotipy.oauth2 import SpotifyOAuth

def get_a3_30():
    """Get the tracks and artists from the Antena 3 website."""
    url = "https://media.rtp.pt/antena3/"
    result = requests.get(url, timeout=5)
    soup = bs4.BeautifulSoup(result.content, "html.parser")
    a330_url = soup.find("a", "results-right").get("href")

    result = requests.get(a330_url, timeout=5)
    soup = bs4.BeautifulSoup(result.content, "html.parser")

    tracks = [p.text for p in soup.find_all("p", "vote-track-name")]
    artists = [span.text for span in soup.find_all("span", "vote-band-title")]

    return tracks, artists

def create_playlist(tracks, artists):
    """Create a playlist on Spotify with the given tracks and artists."""
    scope = [
        "playlist-read-private",
        "playlist-modify-private",
        "playlist-read-collaborative",
        "playlist-modify-public",
    ]
    auth_manager = SpotifyOAuth(scope=scope)
    sp = spotipy.Spotify(auth_manager=auth_manager)

    playlists = sp.current_user_playlists()
    for playlist in playlists["items"]:
        if playlist["name"] == "A3.30":
            break

    track_ids = []
    for track, artist in zip(tracks, artists):
        print(f"Searching for {track} - {artist}")
        new_track = sp.search(q=f"{artist} {track}")["tracks"]["items"][0]
        track_ids.append(new_track["id"])

    sp.playlist_add_items(playlist["id"], items=track_ids)

def main():
    load_dotenv()
    tracks, artists = get_a3_30()
    create_playlist(tracks, artists)

if __name__ == "__main__":
    main()


import queue
import sounddevice as sd
import vosk
import json
import words
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from skills import *
import voice

q = queue.Queue()

model = vosk.Model("D:\sevase_GS\model_small\model_small")

device = sd.default.device          # sd.default.device == 1, 3  ////input, output [1, 4]
samplerate = int(sd.query_devices(device[0], 'input')['default_samplerate'])

def callback(indata, frames, time, status):

    q.put(bytes(indata))

def recognize(data, vectorizer, clf):
    trg = words.TRIGGERS.intersection(data.split()) #делает проверку, если тригерных слов нету то ретерн возращает проверку в начало
    if not trg:
        return
    data.replace(list(trg)[0], '') # удаляем имя бота из текста
    text_vector = vectorizer.transform([data]).toarray()[0]# получаем вектор полученного текста
    answer = clf.predict([text_vector])[0]# сравниваем с вариантами, получая наиболее подходящий ответ

    func_name = answer.split()[0]#получение имени функции из ответа из data_set
    voice.speaker(answer.replace(func_name, ''))#озвучка ответа из модели data_set + удаление опозновающ. слова
    exec(func_name + '()')#запуск функции из skills

def main():
    vectorizer = CountVectorizer()
    vectors = vectorizer.fit_transform(list(words.data_set.keys())) #получаем ключи списка, метод хэширует фразы и строит по ним векторы, какой либо масив из матрицы цифр. Похожие фразы, которые и будут содерж похожие слова , то их векторы будут совпадать => машина будет распозновать слова в любых похожих предложениях

    clf = LogisticRegression()
    clf.fit(vectors, list(words.data_set.values())) # берем ответы, а не ключи уже. После идет некая проверка, в которой опред, совпадение ключа и ответа и после выводится ответ :)

    del words.data_set

    with sd.RawInputStream(samplerate=samplerate, blocksize = 2000, device=device[0], dtype='int16',    # device[0] == micro
                            channels=1, callback=callback):      # callback записывает данные

        rec = vosk.KaldiRecognizer(model, samplerate)
        while True:
            data = q.get()
            if rec.AcceptWaveform(data):    # если остановится голос, начнется расшифровка вводимых данных
                data = json.loads(rec.Result())['text']
                recognize(data, vectorizer, clf)
            # else:
            #     print(rec.PartialResult())

if __name__ == '__main__':
     main()


# -*- coding: utf-8 -*-
import json
import datetime
import math
import re
import traceback
from copy import deepcopy
from decimal import Decimal
from functools import lru_cache

from django.apps import apps
from django.contrib import messages
from django.db import transaction
from django.forms.models import ModelFormMetaclass
from django.forms import *
from .fields import *
from django.http import HttpResponse, HttpResponseRedirect
from django.template.loader import render_to_string
from django.utils.safestring import mark_safe
from django.utils.text import slugify

from . import inputs
from .fields import *
from ..core.queryset import QuerySet
from sloth.api.exceptions import JsonReadyResponseException, ReadyResponseException, HtmlReadyResponseException
from ..utils import to_api_params, to_snake_case
from ..utils.formatter import format_value
from ..utils.http import FileResponse

ACTIONS = {}
EXPOSE = []

class PermissionChecker:

    def __init__(self, request, instance=None, instantiator=None, metaclass=None):
        self.request = request
        self.instance = instance
        self.instantiator = instantiator
        self.metaclass = metaclass

    def has_permission(self, user):
        return self and user

class ActionDefaultMetaClass:
    modal = True

class ActionMetaclass(ModelFormMetaclass):
    def __new__(mcs, name, bases, attrs):
        if 'Meta' in attrs:
            if hasattr(attrs['Meta'], 'model'):
                if isinstance(attrs['Meta'].model, str):
                    attrs['Meta'].model = apps.get_model(attrs['Meta'].model)
                bases += forms.ModelForm,
            else:
                bases += forms.Form,
            if not hasattr(attrs['Meta'], 'fields') and not hasattr(attrs['Meta'], 'exclude'):
                form_fields = []
                fieldsets = getattr(attrs['Meta'], 'fieldsets', {})
                if fieldsets:
                    for tuples in fieldsets.values():
                        for names in tuples:
                            if isinstance(names, str):
                                form_fields.append(names)
                            else:
                                form_fields.extend(names)
                    setattr(attrs['Meta'], 'fields', form_fields)
                else:
                    setattr(attrs['Meta'], 'exclude', ())
        else:
            bases += forms.Form,
            attrs['Meta'] = ActionDefaultMetaClass
        cls = super().__new__(mcs, name, bases, attrs)
        ACTIONS[name] = cls
        ACTIONS[to_snake_case(name)] = cls
        if 'ActionView' in [k.__name__ for k in bases]:
            EXPOSE.append(to_snake_case(name))
        return cls

class DecimalField(forms.DecimalField):
    def clean(self, value):
        if value:
            value = value.replace('.', '').replace(',', '.')
        value = super().clean(value)
        return value

class TextField(forms.CharField):
    def __init__(self, *args, **kwargs):
        kwargs.update(widget=forms.Textarea())
        super().__init__(*args, **kwargs)

class BooleanChoiceField(forms.ChoiceField):
    def __init__(self, *args, **kwargs):
        kwargs.update(widget=forms.Select(), choices=[['', ''], [1, 'Sim'], [0, 'Não']])
        super().__init__(*args, **kwargs)

    def clean(self, value):
        if value == '':
            if self.required:
                raise ValidationError('Selecione uma opção')
            return None
        return bool(int(value))

class RegionalDateWidget(DateInput):
    input_type = 'date'

    def render(self, name, value, attrs=None, renderer=None):
        if isinstance(value, datetime.date):
            value = value.isoformat()
        attrs = attrs or {}
        attrs.update({'autocomplete': 'off'})
        html = super().render(name, value, attrs)
        return mark_safe(html)

class RegionalDateTimeWidget(DateTimeInput):
    input_type = 'datetime-local'

    def render(self, name, value, attrs=None, renderer=None):
        if isinstance(value, datetime.datetime):
            value = value.isoformat().split('.')[0]
        attrs = attrs or {}
        attrs.update({'step': '1'})
        html = super().render(name, value, attrs)
        return mark_safe(html)

class Action(metaclass=ActionMetaclass):

    def __init__(self, *args, **kwargs):
        self.path = None
        self.request = kwargs.pop('request', None)
        self.instantiator = kwargs.pop('instantiator', None)
        self.queryset = kwargs.pop('queryset', None)
        self.instances = kwargs.pop('instances', None)
        self.metaclass = getattr(self, 'Meta')

        self.show_form = True
        self.can_be_closed = False
        self.can_be_reloaded = False
        self.content = dict(top=[], left=[], center=[], right=[], bottom=[], info=[], alert=[], danger=[])
        self.on_change_data = dict(show=[], hide=[], set=[], show_fieldset=[], hide_fieldset=[])

        if forms.ModelForm in self.__class__.__bases__:
            self.instance = kwargs.get('instance', None)
        else:
            self.instance = kwargs.pop('instance', None)

        if self.instance is None and self.instances is not None and self.instances.exists():
            self.instance = self.instances.first()

        if self.request.path.startswith('/api/') or self.request.path.startswith('/meta/'):
            if self.request.body and self.request.body.startswith(b'{') and self.request.body.endswith(b'}'):
                kwargs['data'] = json.loads(self.request.body)

        if self.has_url_posted_data():
            for k in self.request.GET:
                if k.startswith('post__'):
                    if 'data' not in kwargs:
                        kwargs['data'] = {}
                    kwargs['data'][k.split('__')[-1]] = self.request.GET[k]

        if 'data' not in kwargs:
            if self.get_api_name() in self.request.GET or self.get_api_name() in self.request.POST or not self.request.path.startswith('/app/'):
                # if self.base_fields or self.requires_confirmation():
                if self.request.method == 'GET' or self.requires_confirmation():
                    if self.get_method() == 'get':
                        data = self.request.GET or None
                    else:
                        data = self.request.POST or None
                else:
                    data = self.request.POST
                kwargs['data'] = data
                kwargs['files'] = self.request.FILES or None
            else:
                kwargs['data'] = None

        super().__init__(*args, **kwargs)
        if self.instance is not None and hasattr(self.instance, 'autouser') and not self.instance.autouser_id:
            self.instance.autouser = self.request.user
            if 'autouser' in self.fields:
                del self.fields['autouser']

        for name in self.fields:
            if name != 'data':
                setattr(self, name, self.initial.get(name, None))
        self.asynchronous = getattr(self.metaclass, 'asynchronous', None) and self.request.GET.get('synchronous') is None

        related_field_name = getattr(self.metaclass, 'related_field', None)
        if related_field_name:
            setattr(self.instance, related_field_name, self.instantiator)
            if related_field_name in self.fields:
                del self.fields[related_field_name]

        for field_name in self.fields:
            field = self.fields[field_name]
            if hasattr(field, 'queryset'):
                if getattr(field, 'username_lookup', None):
                    pks = list(field.queryset.filter(**{field.username_lookup: self.request.user}).values_list('pk', flat=True)[0:2])
                    if len(pks) == 1:
                        field.queryset = field.queryset.model.objects.filter(pk=pks[0])
                        field.initial = pks[0]
                        self.initial[field_name] = field.initial
                        # field.widget = forms.HiddenInput()
                        field.widget.attrs['class'] = '{} disabled'.format(field.widget.attrs.get('class', ''))
                        field.widget.attrs['readonly'] = 'readonly'
                else:
                    field.queryset = field.queryset#.contextualize(self.request).apply_role_lookups(self.request.user)

        self.response = {}
        self.fieldsets = {}
        self.one_to_one = {}
        self.one_to_many = {}

        confirmation = self.requires_confirmation()
        if confirmation:
            help_text = confirmation if isinstance(confirmation, str) else ''
            self.fields['confirmation'] = forms.BooleanField(
                label='', initial='on', required=False, help_text=help_text,
                widget=forms.HiddenInput()
            )

    @classmethod
    def get_api_doc(cls, detail=False):
        if detail and cls.base_fields:
            doc = []
            field_names = [field.label.lower() for field in cls.base_fields.values()]
            doc.append('Requer as seguintes informações: {}.'.format(', '.join(field_names)))
            return ' '.join(doc)
        return None

    def render(self, template_name, **context):
        return HttpResponse(render_to_string([template_name], context, request=self.request))

    def get_verbose_name(self):
        return self.get_metadata().get('name')

    def get_image(self):
        return self.get_metadata().get('image')

    def has_url_posted_data(self):
        return 'post__{}'.format(self.get_api_name()) in self.request.GET

    def closable(self, flag=True):
        self.can_be_closed = flag

    def reloadable(self, flag=True):
        self.can_be_reloaded = flag

    def info(self, text):
        if text:
            self.content['info'].append(text)

    def alert(self, text):
        if text:
            self.content['alert'].append(text)

    def danger(self, text):
        if text:
            self.content['danger'].append(text)

    def parameters(self):
        data = {}
        for token in self.request.path.split('/'):
            if '=' in token:
                k, v = token.split('=')
                data[k] = int(v) if v.isdigit() else v
        return data

    def parameter(self, name, default=None):
        return self.parameters().get(name, default)

    def objects(self, model_name):
        return apps.get_model(model_name).objects.queryset().contextualize(self.request).apply_role_lookups(self.request.user)

    def clear(self):
        self.show_form = False
        for k, v in self.content.items():
            if k != 'bottom':
                v.clear()

    def view(self):
        return None

    def append(self, item, position='center'):
        if hasattr(item, 'contextualize'):
            item.contextualize(self.request)
        self.content[position].append(item.html())

    def has_attr_permission(self, user, name):
        attr = getattr(self, 'has_{}_permission'.format(name), None)
        return attr is None or attr(user)

    def requires_confirmation(self):
        return getattr(self.metaclass, 'confirmation', False)

    @lru_cache
    def get_one_to_one_field_names(self):
        return self.instance.get_one_to_one_field_names() if self.instance else ()

    @lru_cache
    def get_one_to_many_field_names(self):
        return self.instance.get_one_to_many_field_names() if self.instance else ()

    def get_fieldsets(self):
        related_field = getattr(self.instance, 'related_field', None) if self.instance is not None else None
        if related_field and related_field in self.fields:
            del self.fields[related_field]
        fieldsets = getattr(self.metaclass, 'fieldsets', None)
        if fieldsets is None:
            fieldsets = {}
            if self.fields:
                field_names = [
                    name for name in self.fields.keys() if (
                            name not in self.get_one_to_one_field_names()
                            and name not in self.get_one_to_many_field_names()
                            and name not in ('confirmation', related_field)
                    )
                ]
                if field_names:
                    fieldsets[None] = field_names
        else:
            fieldsets = dict(fieldsets)
        return fieldsets

    def load_fieldsets(self):
        fieldsets = self.get_fieldsets()

        # extract one-to-one and one-to-many fields
        if self.instance:
            for name in list(self.fields):
                if name in self.get_one_to_one_field_names():
                    # remove one-to-one fields from the form
                    self.one_to_one[name] = self.fields.pop(name)
                if name in self.get_one_to_many_field_names():
                    # remove one-to-many fields from the form
                    self.one_to_many[name] = self.fields.pop(name)

        pop = []
        for name, names in fieldsets.items():
            if len(names) == 1 and (names[0] in self.one_to_one.keys() or names[0] in self.one_to_many.keys()):
                pop.append(name)
        for name in pop:
            del fieldsets[name]

        # configure one-to-one fields
        for one_to_one_field_name, one_to_one_field in self.one_to_one.items():
            field_list = []
            form_cls = one_to_one_field.queryset.model.add_form_cls()
            initial = one_to_one_field.queryset.model.objects.filter(
                pk=getattr(self.instance, '{}_id'.format(one_to_one_field_name))
            ).values(
                *form_cls.base_fields.keys()
            ).first() or {}
            key = one_to_one_field_name.upper()
            self.fields[key] = forms.BooleanField(
                required=one_to_one_field.required, initial=bool(initial) or one_to_one_field.required
            )
            self.fields[key].widget.attrs['class'] = 'field-controller'
            if one_to_one_field.required:
                self.fields[key].widget.attrs['class'] += ' d-none'
            for name, field in form_cls.base_fields.items():
                ont_to_one_key = '{}__{}'.format(one_to_one_field_name, name)
                field_list.append(ont_to_one_key)
                field.required = field.required and one_to_one_field.required or self.data.get(
                    one_to_one_field_name.upper()
                )
                field.widget.attrs['class'] = key
                self.fields[ont_to_one_key] = field
                self.initial[ont_to_one_key] = initial.get(name)

            # try to get field organization from form fieldsets if defined
            one_to_one_fieldsets = getattr(form_cls.Meta, 'fieldsets', None)
            if one_to_one_fieldsets is None:
                one_to_one_fieldsets = getattr(form_cls.Meta.model.metaclass(), 'fieldsets', None)
            if one_to_one_fieldsets is not None:
                field_list = []
                for fieldset in one_to_one_fieldsets.values():
                    for names in fieldset:
                        if isinstance(names, str):
                            field_list.append('{}__{}'.format(one_to_one_field_name, names))
                        else:
                            field_list.append(
                                ['{}__{}'.format(one_to_one_field_name, field_name) for field_name in names]
                            )
            field_list.append(one_to_one_field_name.upper())
            fieldsets[one_to_one_field.label] = field_list

        # configure one-to-many fields
        for one_to_many_field_name, one_to_many_field in self.one_to_many.items():
            field_list = []
            form_cls = one_to_many_field.queryset.model.add_form_cls()
            pks = []
            if self.instance.pk:
                pks.extend(getattr(self.instance, one_to_many_field_name).values_list('pk', flat=True))
            pks.extend(['' for _ in range(len(pks) + 1, one_to_many_field.max + 1)])
            for i, pk in enumerate(pks):
                initial = one_to_many_field.queryset.model.objects.filter(
                    pk=pk
                ).values(
                    *form_cls.base_fields.keys()
                ).first() if pk else {}
                key = '{}--{}'.format(one_to_many_field_name.upper(), i)
                required = i < one_to_many_field.min
                self.fields[key] = forms.CharField(
                    label='{} {}'.format(one_to_many_field.queryset.model.metaclass().verbose_name, i + 1),
                    required=required, initial=(pk or 'on') if required else pk, widget=forms.CheckboxInput()
                )
                self.fields[key].widget.attrs['class'] = 'field-controller'
                if required:
                    self.fields[key].widget.attrs['class'] += ' d-none'
                field_list.append(key)
                inline_field_list = []
                for name, field in form_cls.base_fields.items():
                    field = deepcopy(field)
                    one_to_many_key = '{}__{}__{}'.format(one_to_many_field_name, name, i)
                    inline_field_list.append(one_to_many_key)
                    field.required = field.required and required or False
                    field.widget.attrs['class'] = key
                    self.fields[one_to_many_key] = field
                    self.initial[one_to_many_key] = initial.get(name)

                field_list.append(inline_field_list)
            fieldsets[one_to_many_field.label] = field_list
        self.fieldsets = fieldsets
        self.parse_fieldsets()

    def parse_fieldsets(self):
        # configure ordinary fields
        for title, names in self.fieldsets.items():
            field_list = []
            for name in names:
                if isinstance(name, tuple) or isinstance(name, list):
                    for _name in name:
                        if _name in self.fields:
                            if not isinstance(self.fields[_name].widget, forms.HiddenInput):
                                field_list.append(dict(name=_name, width=100 // len(name)))
                else:
                    if name in self.fields:
                        if not isinstance(self.fields[name].widget, forms.HiddenInput):
                            field_list.append(dict(name=name, width=100))
            if field_list:
                self.fieldsets[title] = field_list

    def get_api_params(self):
        return to_api_params(self.fields.items())

    def save(self, *args, **kwargs):
        if hasattr(self.instance, '__roles__'):
            with transaction.atomic():
                self._save()
        else:
            self._save()

    def _save(self, *args, **kwargs):

        if hasattr(self.instance, 'pre_save'):
            self.instance.pre_save()

        # save one-to-one fields
        for name in self.one_to_one:
            instance = getattr(self.instance, name)
            if self.data.get(name.upper()):  # if checkbox is checked
                instance = instance or self.one_to_one[name].queryset.model()
                for field_name in self.fields:
                    if field_name.split('__')[0] == name:
                        setattr(instance, field_name.split('__')[1], self.cleaned_data[field_name])
                instance.save()
                setattr(self.instance, name, instance)
            else:  # if checkbox is NOT checked
                if instance:
                    instance.delete()
                setattr(self.instance, name, None)

        # save
        if hasattr(super(), 'save'):
            super().save(*args, **kwargs)

        # save one-to-many fields
        for name in self.one_to_many:
            qs = getattr(self.instance, name)
            pks = list(qs.values_list('pk', flat=True))
            for i in range(0, 10):
                key = '{}--{}'.format(name.upper(), i)
                pk = self.data.get(key)
                if pk:  # if checkbox is checked
                    if pk == 'on':
                        instance = qs.model()
                    else:
                        pks.remove(int(pk))
                        instance = qs.get(pk=pk)
                    for field_name in self.fields:
                        if field_name.startswith('{}__'.format(name)) and field_name.endswith('__{}'.format(i)):
                            setattr(instance, field_name.split('__')[1], self.cleaned_data[field_name])
                    instance.save()
                    qs.add(instance)
            qs.filter(pk__in=pks).delete()

        if hasattr(self.instance, 'post_save'):
            self.instance.post_save()

    def serialize(self, wrap=False):
        if self.response:
            return self.response
        if wrap:
            data = dict(type='form')
            form_fields = {}
            for field_name in self.fields:
                field = self.fields[field_name]
                form_fields[field_name] = dict(
                    label=field.label,
                    name=field_name,
                    type=type(field).__name__.replace('Field', '').lower(),
                    required=field.required,
                    value=self.data.get(field_name)
                )
            data.update(self.get_metadata())
            if hasattr(self, 'get_verbose_name'):
                data['name'] = hasattr(self, 'get_verbose_name')
            data.update(fields=form_fields, errors=self.errors)
            return data
        else:
            data = {field_name: self.fields[field_name].initial for field_name in self.fields}
            return data

    @lru_cache()
    def get_on_change_fields(self):
        on_change = []
        for attr_name in dir(self):
            if attr_name.startswith('on_') and attr_name.endswith('_change'):
                field_name = attr_name[3:-7]
                on_change.append(field_name)
        return on_change

    def get_allowed_attrs(self, recursive=True):
        return []

    @classmethod
    def get_api_name(cls):
        return to_snake_case(cls.__name__)

    @classmethod
    @lru_cache
    def get_metadata(cls, path=None, target=None):
        metaclass = getattr(cls, 'Meta', None)
        if metaclass:
            target = target
            name = getattr(metaclass, 'verbose_name', re.sub("([a-z])([A-Z])", "\g<1> \g<2>", cls.__name__))
            submit = getattr(metaclass, 'submit_label', name)
            icon = getattr(metaclass, 'icon', None)
            ajax = getattr(metaclass, 'ajax', True)
            modal = getattr(metaclass, 'modal', True)
            style = getattr(metaclass, 'style', 'primary')
            method = getattr(metaclass, 'method', 'post')
            auto_reload = getattr(metaclass, 'auto_reload', None)
            image = getattr(metaclass, 'image', None)
        else:
            name, submit, icon, ajax, modal, style, method, auto_reload, image = (
                'Enviar', 'Enviar', None, True, 'modal', 'primary', 'get', None, None
            )
        if path:
            path, *params = path.split('?')
            if target in ('queryset', 'instance'):
                path = '{}{{id}}/{}/'.format(path, cls.get_api_name())
            else:
                path = '{}{}/'.format(path, cls.get_api_name())
            if params:
                path = '{}?{}'.format(path, params[0])
        metadata = dict(
            type='form', key=cls.get_api_name(), name=name, submit=submit, target=target,
            method=method, icon=icon, style=style, ajax=ajax, path=path, modal=modal,
            auto_reload=auto_reload, image=image
        )
        return metadata

    def get_method(self):
        return getattr(self.metaclass, 'method', 'post') if hasattr(self, 'Meta') else 'post'

    def get_instances(self):
        if self.instances is not None:
            return self.instances
        elif self.queryset is not None:
            return self.queryset
        elif self.instance:
            return type(self.instance).objects.filter(pk=self.instance.pk)
        return []

    def is_modal(self):
        return getattr(self.metaclass, 'modal', True) if hasattr(self, 'Meta') else True

    def has_permission(self, user):
        return user.is_superuser

    def check_permission(self, user):
        return user.is_superuser or self.has_permission(user)

    @classmethod
    def check_fake_permission(cls, request, instance=None, instantiator=None):
        if request:  # and not request.user.is_superuser
            checker = PermissionChecker(request, instance, instantiator, getattr(cls, 'Meta', None))
            has_permission = cls.has_permission(checker, request.user)
            return has_permission
            ### return cls.check_permission(checker, request.user) if has_permission is None else has_permission
        return True

    def __str__(self):
        return self.html()

    def get_alternative_links(self):
        return []

    def html(self):

        for name, field in self.fields.items():
            classes = field.widget.attrs.get('class', '').split()
            if isinstance(field.widget, forms.CheckboxInput):
                classes.append('form-check-input')
            elif isinstance(field.widget, forms.widgets.Input):
                classes.append('form-control')

            if isinstance(field, forms.DateField):
                field.widget = RegionalDateWidget()
                classes.append('date-input')
                if name in self.initial and self.initial[name] and isinstance(self.initial[name], str) and '/' in self.initial[name]:
                    self.initial[name] = datetime.datetime.strptime(self.initial[name], '%d/%m/%Y').strftime('%Y-%m-%m')

            if isinstance(field, forms.DateTimeField):
                field.widget = RegionalDateTimeWidget()
                classes.append('date-time-input')
                if name in self.initial and self.initial[name] and isinstance(self.initial[name], str) and '/' in self.initial[name]:
                    self.initial[name] = datetime.datetime.strptime(self.initial[name], '%d/%m/%Y').strftime('%Y-%m-%m %H:%M')

            if isinstance(field, forms.DecimalField):
                field.widget.input_type = 'text'
                field.widget.rmask = getattr(field.widget, 'rmask', '#.##0,00')
                if name in self.initial and self.initial[name] is not None:
                    self.initial[name] =  Decimal('%.2f' % self.initial[name])

            if isinstance(field, forms.ImageField):
                classes.append('image-input')

            if isinstance(field, forms.ModelChoiceField) and isinstance(field.widget, Select):
                initial = self.initial.get(name)
                pks = []
                if initial:
                    if isinstance(initial, int):
                        pks.append(initial)
                    else:
                        if isinstance(initial[0], int):
                            pks.extend([pk for pk in initial])
                        else:
                            pks.extend([obj.pk for obj in initial])
                if self.data:
                    pks = [pk for pk in self.data.getlist(name) if pk]
                if getattr(field, 'picker', None) is None:
                    field.queryset = field.queryset.filter(pk__in=pks) if pks else field.queryset.none()
                field.widget.attrs['data-choices-url'] = '{}{}action_choices={}'.format(
                    self.get_full_path(), '&' if '?' in self.get_full_path() else '?', name
                )

            if getattr(field.widget, 'mask', None):
                classes.append('masked-input')
                field.widget.attrs['data-reverse'] = 'false'
                field.widget.attrs['data-mask'] = getattr(field.widget, 'mask')
            if getattr(field.widget, 'rmask', None):
                classes.append('masked-input')
                field.widget.attrs['data-reverse'] = 'true'
                field.widget.attrs['data-mask'] = getattr(field.widget, 'rmask')
            if getattr(field.widget, 'formatted', False):
                classes.append('html-input')

            if hasattr(field, 'picker'):
                grouper = field.picker if isinstance(field.picker, str) else None
                if isinstance(field, forms.ModelMultipleChoiceField):
                    field.widget = inputs.MultiplePickInput(field.queryset, grouper=grouper)
                else:
                    field.widget = inputs.PickInput(field.queryset, grouper=grouper)

            if getattr(field, 'addable', False):
                has_permission = field.queryset.model().has_add_permission(self.request.user)
                has_permission = has_permission or field.queryset.model().has_permission(self.request.user)
                help_text = '<div>{}</div>'.format(field.help_text) if field.help_text else ''
                link = '<a style="text-decoration:none" class="popup" href="/app/dashboard/{}/{}/add/">Adicionar</a>'.format(
                    field.queryset.model.metaclass().app_label, field.queryset.model.metaclass().model_name
                )
                help_text = '<div style="float:right">{}</div>'.format(link) + help_text
                if has_permission:
                    field.help_text = help_text

            field.widget.attrs['class'] = ' '.join(classes)
        return mark_safe(
            render_to_string(
                ['dashboard/form.html'], dict(
                    self=self, fieldsets=self.fieldsets
                ),
                request=self.request
            )
        )

    def show(self, *names):
        for name in names:
            if name.islower():
                if name in self.on_change_data['hide']:
                    self.on_change_data['hide'].remove(name)
                if name not in self.on_change_data['show']:
                    self.on_change_data['show'].append(name)
            else:
                name = slugify(name)
                if name in self.on_change_data['hide_fieldset']:
                    self.on_change_data['hide_fieldset'].remove(name)
                if name not in self.on_change_data['show_fieldset']:
                    self.on_change_data['show_fieldset'].append(name)

    def hide(self, *names):
        for name in names:
            if name.islower():
                if name in self.on_change_data['show']:
                    self.on_change_data['show'].remove(name)
                if name not in self.on_change_data['hide']:
                    self.on_change_data['hide'].append(name)
            else:
                name = slugify(name)
                if name in self.on_change_data['show_fieldset']:
                    self.on_change_data['show_fieldset'].remove(name)
                if name not in self.on_change_data['hide_fieldset']:
                    self.on_change_data['hide_fieldset'].append(name)

    def set(self, **kwargs):
        for k, v in kwargs.items():
            if hasattr(v, 'pk'):
                value = v.pk
                text = str(v)
            else:
                value = format_value(v)
                text = None
            self.on_change_data['set'].append(dict(name=k, value=value, text=text))

    def check_ouput(self, output, submit=False):
        from ..core.valueset import ValueSet
        position = 'bottom' if submit else 'center'
        if type(output) == dict:
            if self.request.path.startswith('/app/'):
                template_name = 'actions/{}.html'.format(self.get_api_name())
                self.content[position].append(render_to_string([template_name], output, request=self.request))
            else:
                raise JsonReadyResponseException(output)
        elif type(output) == str and output[-5:].split('.')[-1] in FileResponse.CONTENT_TYPES.keys():
            raise ReadyResponseException(FileResponse(output))
        elif type(output) in (str, int, float, Decimal, datetime.date):
            raise ReadyResponseException(HttpResponse(str(output)))
        elif isinstance(output, HttpResponse):
            raise ReadyResponseException(output)
        elif isinstance(output, ValueSet) or isinstance(output, QuerySet):
            if submit:
                path = '{}{}/'.format(self.request.path, 'submit') if self.request.POST else None
            else:
                path = '{}{}/'.format(self.request.path, 'view')
            self.content[position].append(output.readonly().contextualize(self.request).html(path=path))
        elif output is not None:
            raise Exception()
        if self.request.path.startswith('/app/') and self.response and submit:
            js = '<script>{}</script>'
            display_messages = True
            url = self.response.get('url')
            if self.response.get('dispose'):
                js = js.format('fade{}({});'.format(self.get_metadata().get('key'), self.response.get('dispose')))
            elif url == '.':
                if 'modal' in self.request.GET:
                    js = js.format('$(document).popup("{}");'.format(self.request.path))
                else:
                    js = js.format('$(document).refresh([]);'.format(self.get_metadata().get('key')))
            elif url == '..':
                display_messages = 'modal' in self.request.GET
                js = js.format('$(document).back().refresh([]);')
            elif url:
                display_messages = False
                js = js.format('$(document).redirect("{}");'.format(self.response['url']))
            html = render_to_string('dashboard/messages.html', request=self.request) if display_messages else ''
            raise ReadyResponseException(HttpResponse('<!---->{}{}<!---->'.format(js, html)))

        if self.request.path.startswith('/app/') and self.response and not submit:
            stack = self.request.session.get('stack', [])
            url = self.response.get('url')
            if url in ('.', '..') and len(stack) > 1:
                url = stack[-2]
            raise ReadyResponseException(HttpResponseRedirect(url))

        return output

    def is_valid(self):
        if self.asynchronous:
            return False
        self.check_ouput(self.view())
        self.load_fieldsets()
        if 'action_choices' in self.request.GET:
            raise JsonReadyResponseException(
                self.choices(self.request.GET['action_choices'], q=self.request.GET.get('term'))
            )
        if 'on_change_field' in self.request.POST:
            for data in self.on_change_data.values():
                data.clear()
            field_name = self.request.POST['on_change_field']
            value = self.request.POST.get('on_change_value')
            if value == 'true':
                value = True
            if value == 'false':
                value = False
            if value == 'unknown':
                value = None
            getattr(self, 'on_{}_change'.format(field_name))(value)
            raise JsonReadyResponseException(self.on_change_data)
        is_valid = super().is_valid()
        if not is_valid and not self.request.path.startswith('/app/'):
            raise JsonReadyResponseException(dict(errors=self.errors))
        return is_valid

    def choices(self, field_name, q=None):
        field = self.fields[field_name]
        if self.__class__.__name__ in ('Add', 'Edit'):
            attr = getattr(self.instance, 'get_{}_queryset'.format(field_name), None)
            if attr:
                for name in self.fields:
                    if isinstance(self.fields[name], MultipleChoiceField):
                        value = self.request.GET.getlist(name)
                    if isinstance(self.fields[name], ModelMultipleChoiceField):
                        value = self.request.GET.getlist(name)
                    else:
                        value = self.request.GET.get(name)
                        if value:
                            setattr(self.instance, name, self.fields[name].clean(value))
        else:
            attr = getattr(self, 'get_{}_queryset'.format(field_name), None)
        self.data.update(self.request.GET)
        qs = field.queryset if attr is None else attr(field.queryset)
        # qs = qs.contextualize(self.request).apply_role_lookups(self.request.user)
        total = qs.count()
        qs = qs.search(q=q) if q else qs
        items = [dict(id=value.id, text=str(value), html=value.get_select_display()) for value in qs[0:25]]
        return dict(total=total, page=1, pages=math.ceil((1.0 * total) / 25), q=q, items=items )

    def dispose(self, milleseconds=2000):
        self.response.update(dispose=milleseconds)

    def reload(self):
        return self.redirect('.')

    def message(self, text='Ação realizada com sucesso.', style='success', milleseconds=60000):
        level = dict(success=messages.SUCCESS, warning=messages.WARNING, info=messages.INFO)[style]
        if self.request.path.startswith('/app/'):
            messages.add_message(self.request, level, text)
        elif self.request.path.startswith('/meta/'):
            self.response.update(message=dict(text=text, style=style, milleseconds=milleseconds))
        else:
            self.response.update(message=text)

    def redirect(self, url=None):
        if url is None:
            url = '..' if getattr(self, 'fields', None) or self.is_modal() else '.'
        if not self.request.path.startswith('/api/'):
            self.response.update(type='redirect', url=url)
        if self.request.path.startswith('/app') and not self.get_metadata()['ajax']:
            raise ReadyResponseException(HttpResponseRedirect(url))

    def run(self, *tasks):
        for task in tasks:
            task.start(self.request)
        if len(tasks) > 1:
            self.redirect()
        else:
            self.redirect('/app/dashboard/api/task/{}/'.format(task.task_id))

    def submit(self):
        if self.instances:
            for instance in self.instances:
                self.instance = instance
                if self.has_permission(self.request.user):
                    self._post_clean()
                    self.save()
        else:
            self.save()
        self.message()
        self.redirect()

    def process(self):
        # print(self.request.path, {k:v for k, v in self.data.items() if k not in ['csrfmiddlewaretoken', self.get_metadata()['key']]})
        try:
            return self.check_ouput(self.submit(), True)
        except forms.ValidationError as e:
            if self.request.path.startswith('/app/'):
                message = 'Corrija os erros indicados no formulário'
                messages.add_message(self.request, messages.WARNING, message)
            self.add_error(None, e.message)
        except BaseException as e:
            if isinstance(e, ReadyResponseException):
                raise e
            if isinstance(e, JsonReadyResponseException):
                raise e
            if isinstance(e, HtmlReadyResponseException):
                raise e
            traceback.print_exc()
            message = 'Ocorreu um erro no servidor: {}'.format(e)
            if self.request.path.startswith('/app/'):
                messages.add_message(self.request, messages.WARNING, message)
            self.add_error(None, message)

    @classmethod
    def get_attr_metadata(cls, lookup):
        attr = getattr(cls, lookup)
        template = getattr(attr, '__template__', None)
        metadata = getattr(attr, '__metadata__', None)
        if template:
            if not template.endswith('.html'):
                template = '{}.html'.format(template)
            if not template.startswith('.html'):
                template = 'renderers/{}'.format(template)
        return getattr(attr, '__verbose_name__', lookup), False, template, metadata

    def value_set(self, *names):
        from sloth.core.base import ValueSet
        return ValueSet(self, names)

    @classmethod
    @lru_cache
    def action_form_cls(cls, action):
        return ACTIONS.get(action)

    def should_display_buttons(self):
        return self.fields or self.submit.__func__ != Action.submit

    def get_full_path(self):
        return self.path or self.request.get_full_path()

    def contextualize(self, request):
        return self

    def apply_role_lookups(self, user):
        return self

    def send_mail(self, to, subject, content, from_email=None):
        from sloth.api.models import Email
        Email.objects.send(to, subject, content, from_email)

    @classmethod
    def has_custom_view_method(cls):
        return not cls.view == Action.view

    @classmethod
    def has_custom_submit_method(cls):
        return not cls.submit == Action.submit

class ActionView(Action):
    pass

{'ValueSet': 'sloth.core.base.ValueSet', 'Email': 'sloth.api.models.Email'}.view

{'ValueSet': 'sloth.core.base.ValueSet', 'Email': 'sloth.api.models.Email'}.submit


import sys
import pygame
from bullet import Bullet

def ship_hit(ai_settings, stats, screen, ship, bullets):
    '''Обрабатывает столкновние корабля с пришельцем'''
    if stats.ship_left > 0:
        # уменьшает ship_left
        stats.ship_left -=1
        # очистка списков пришельцев и пуль
        bullets.empty()
        # создание нового флота и размешение корабля по центра
        update_bullets(ai_settings, stats, screen, ship, bullets)
        ship.center_ship()
    else:
        print('Конц игры')
        stats.game_active = False

def check_keydown_events(event, ai_settings, screen, ship):
    '''Реагирует на нажатие клавиш'''
    if event.key == pygame.K_RIGHT:
        ship.moving_right = True
    elif event.key == pygame.K_LEFT:
        ship.moving_left = True

def check_keyup_events(event, ship):
    '''Реагирует на отпускание клавиш'''
    if event.key == pygame.K_RIGHT:
        ship.moving_right = False
    elif event.key == pygame.K_LEFT:
        ship.moving_left = False

def check_events(ai_settings, screen, ship):
    '''отслеживание событий клавиатуры и мыши'''
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            check_keydown_events(event, ai_settings, screen, ship)
        elif event.type == pygame.KEYUP:
            check_keyup_events(event, ship)

def update_screen(ai_settings, screen, ship, bullets):
    '''Обновляет изображения на экране  и отоброжает новый экран'''
    # при каждом проходе цикла прорисовывается экран
    screen.fill(ai_settings.bg_color)

    for bullet in bullets.sprites():
        bullet.draw_bullet()

    ship.blitme()

    # отслеживание последнего прорисованного экрана
    pygame.display.flip()

def update_bullets(ai_settings, stats, screen, ship, bullets):
    """Обновите положение пуль и избавьтесь от старых пуль."""
    if len(bullets) == 0:
        new_bullet = Bullet(ai_settings, screen)
        bullets.add(new_bullet)

    bullets.update()

    for bullet in bullets.sprites():
        if bullet.rect.bottom >= ai_settings.screen_height:
            bullets.remove(bullet)
            ship_hit(ai_settings, stats, screen, ship, bullets)
            print('Не поймал')
        elif pygame.sprite.spritecollideany(ship, bullets):
            bullets.remove(bullet)
            update_bullets(ai_settings, stats, screen, ship, bullets)
            print('Поймал')

    # collisions = pygame.sprite.groupcollide(bullets, ship, True, True)