from django.http import request
from django.shortcuts import render, get_object_or_404
from .models import *
from users.models import CustomUser

from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
from django.contrib.auth.decorators import login_required
from django.views.generic import (
    ListView,
    DetailView,
    CreateView,
    UpdateView,
    DeleteView
)

import json

import urllib
from django.conf import settings
from django.contrib import messages
from django.shortcuts import render, redirect
from .decorators import allowed_users
from django.core.mail import *
from django.core.mail import send_mail
from django.utils.decorators import method_decorator
from django.core.paginator import Paginator
from .filters import ReporteFilter, BlogFilter
from django.views.decorators.http import require_http_methods

@require_http_methods(["GET"])
def home(request):

    return render(request, 'reporte/example.html')

#@require_http_methods(["GET"])
#def about(request):
#    context = {
#         'reportes': Reporte.objects.all()
#    }
#    return render(request, 'reporte/home.html', context)

class ReporteListView(ListView):
    model = Reporte
    template_name = 'reporte/home.html'  # <app>/<model>_<viewtype>.html
    context_object_name = 'reportes'
    ordering = ['-fecha_reporte']

class UserReporteListView(LoginRequiredMixin,ListView):
    model = Reporte
    template_name = 'reporte/user_reports.html'  # <app>/<model>_<viewtype>.html
    context_object_name = 'reportes'
    paginate_by = 4

    def get_queryset(self):

        user =  CustomUser.objects.filter(email = self.request.user).first()

        return Reporte.objects.filter(autor=user).order_by('-fecha_reporte')

class ReporteDetailView(DetailView):
    model = Reporte

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        reporte = self.get_object()
        recents =  Reporte.objects.filter(status='Visible').order_by('-fecha_reporte')

        lista_labels =  json.loads(reporte.labels)
        #lista_labels =  json.dumps(reporte.labels)
        if type(lista_labels) is dict:
            lista_labels=[" "]
        else:

            lista_labels.reverse()
        context['recents'] = recents[:4]
        context['labels'] = lista_labels
        return context

class ReporteCreateView(LoginRequiredMixin, CreateView):
    model = Reporte
    fields = ['titulo', 'descripcion','image','humedal','tipoReporte']

    def form_valid(self, form):
        form.instance.autor = self.request.user

        if form.is_valid():
            recaptcha_response = self.request.POST.get('g-recaptcha-response')
            url = 'https://www.google.com/recaptcha/api/siteverify'
            values = {
                'secret': settings.GOOGLE_RECAPTCHA_SECRET_KEY,
                'response': recaptcha_response
            }
            data = urllib.parse.urlencode(values).encode()
            req =  urllib.request.Request(url, data=data)
            response = urllib.request.urlopen(req)
            result = json.loads(response.read().decode())
            if result['success']:
                form.save()
                messages.success(self.request, 'Nuevo aporte subido con exito!')
            else:
                messages.error(self.request,
                 'reCAPTCHA Invalido. Por favor intenta otra vez.')
            return redirect('reporte-create')

        return super().form_valid(form)

class ReporteUpdateView(LoginRequiredMixin, UserPassesTestMixin, UpdateView):
    model = Reporte
    fields = ['titulo', 'descripcion','image','humedal','tipoReporte']

    def form_valid(self, form):
        #self.request.user
        form.instance.autor =  form.instance.autor
        return super().form_valid(form)

    def test_func(self):
        reporte = self.get_object()

        usuarios = CustomUser.objects.all()
        staff_list = []
        for user in usuarios:
            if len(user.groups.all()) != 0:
                staff_list.append(user.name)

        if self.request.user == reporte.autor or self.request.user.name in staff_list or self.request.user.is_staff:
            print(staff_list)
            return True
        return False

class ReporteStatusView(LoginRequiredMixin, UserPassesTestMixin, UpdateView):
    model = Reporte
    fields = ['status', 'importancia']

    def form_valid(self, form):
        form.instance.autor = form.instance.autor
        return super().form_valid(form)

    def test_func(self):
        reporte = self.get_object()

        usuarios = CustomUser.objects.all()
        staff_list = []
        for user in usuarios:
            if len(user.groups.all()) != 0:
                staff_list.append(user.name)

        if self.request.user == reporte.autor or self.request.user.name in staff_list:
            #print(staff_list)
            return True
        return False

class ReporteDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView):
    model = Reporte
    success_url = '/'

    def test_func(self):
        reporte = self.get_object()
        if self.request.user == reporte.autor or self.request.user.is_superuser or self.request.user.is_staff:
            return True
        return False

@require_http_methods(["GET","POST"])  # Sensitive
def involucrate(request):
    if request.method == 'POST':
        name = request.POST['name']
        email = request.POST['email']
        subject = request.POST['subject']
        message = request.POST['message']

        send_mail(
            name +" "+subject,
            message+" "+email,
            email,
            ['calihumedalesurbanos@gmail.com'],
        )

        return render(request, 'reporte/involucrate.html', {'title': 'Involucrate'})
    else:
        return render(request, 'reporte/involucrate.html', {'title': 'Involucrate'})

@require_http_methods(["GET"])  # Sensitive
def nuestraHistoria(request):
    return render(request, 'reporte/nuestraHistoria.html', {'title': 'About'})

@require_http_methods(["GET"])  # Sensitive
def LaBabilla(request):
    humedal = Humedal.objects.get(nombre= "La Babilla")
    return render(request, 'reporte/humedales/LaBabilla.html', {'title': 'LaBabilla','humedal':humedal})

@require_http_methods(["GET"])  # Sensitive
def ElRetiro(request):
    return render(request, 'reporte/humedales/ElRetiro.html', {'title': 'ElRetiro'})

@require_http_methods(["GET"])  # Sensitive
def EcoparqueLasGarzas(request):
    return render(request, 'reporte/humedales/EcoparqueLasGarzas.html', {'title': 'EcoparqueLasGarzas'})

####################### HUMEDAL LA BABILLA GALERIA
@require_http_methods(["GET"])  # Sensitive
def babilla_flora(request):
    humedal = Humedal.objects.get(nombre= "La Babilla")
    flora = humedal.flora.all()
    return render(request, 'reporte/humedales/galeria_flora_babilla.html', {'title': 'About','humedal': flora})
@require_http_methods(["GET"])  # Sensitive
def babilla_fauna(request):
    humedal = Humedal.objects.get(nombre= "La Babilla")
    fauna = humedal.faunaTerrestre.all()
    return render(request, 'reporte/humedales/galeria_fauna_babilla.html', {'title': 'About', 'humedal': fauna})
@require_http_methods(["GET"])  # Sensitive
def babilla_acuatica(request):
    humedal = Humedal.objects.get(nombre= "La Babilla")
    fauna_acutica = humedal.FaunaAcuatica.all()
    return render(request, 'reporte/humedales/galeria_acuatica_babilla.html', {'title': 'About','humedal': fauna_acutica})

####################### HUMEDAL LAS GARZAS GALERIA
@require_http_methods(["GET"])  # Sensitive
def garzas_flora(request):
    humedal = Humedal.objects.get(nombre= "Ecoparque Las Garzas")
    flora = humedal.flora.all()
    return render(request, 'reporte/humedales/galeria_flora_garzas.html', {'title': 'About','humedal':flora})
@require_http_methods(["GET"])  # Sensitive
def garzas_fauna(request):
    humedal = Humedal.objects.get(nombre="Ecoparque Las Garzas")
    fauna = humedal.faunaTerrestre.all()
    return render(request, 'reporte/humedales/galeria_fauna_garzas.html', {'title': 'About', 'humedal': fauna})

@require_http_methods(["GET"])  # Sensitive
def garzas_acuatica(request):
    humedal = Humedal.objects.get(nombre= "Ecoparque Las Garzas")
    fauna_acutica = humedal.FaunaAcuatica.all()
    return render(request, 'reporte/humedales/galeria_acuatica_garzas.html', {'title': 'About','humedal':fauna_acutica})

####################### HUMEDAL EL RETIRO GALERIA
@require_http_methods(["GET"])  # Sensitive
def retiro_flora(request):
    humedal = Humedal.objects.get(nombre="El Retiro")
    flora = humedal.flora.all()
    return render(request, 'reporte/humedales/galeria_flora_retiro.html', {'title': 'About','humedal':flora})

@require_http_methods(["GET"])  # Sensitive
def retiro_fauna(request):
    humedal = Humedal.objects.get(nombre= "El Retiro")
    fauna = humedal.faunaTerrestre.all()
    return render(request, 'reporte/humedales/galeria_fauna_retiro.html', {'title': 'About', 'humedal': fauna})

@require_http_methods(["GET"])  # Sensitive
def retiro_acuatica(request):
    humedal = Humedal.objects.get(nombre= "El Retiro")
    fauna_acutica = humedal.FaunaAcuatica.all()
    return render(request, 'reporte/humedales/galeria_acuatica_retiro.html', {'title': 'About','humedal': fauna_acutica})

#######################
@require_http_methods(["GET"])  # Sensitive
def planes_manejo(request):
    return render(request, 'reporte/recursos/planes_manejo.html', {'title': 'About'})

@require_http_methods(["GET"])  # Sensitive
def programas(request):
    return render(request, 'reporte/recursos/programas.html', {'title': 'About'})

@require_http_methods(["GET"])  # Sensitive
def videos(request):
    return render(request, 'reporte/recursos/videos.html', {'title': 'videos'})

#@require_http_methods(["GET", "POST"])  # Sensitive
#def registrate(request):
#    return render(request, 'reporte/registrate.html', {'title': 'registrate'})

@require_http_methods(["GET"])  # Sensitive
def quejas(request):
    return render(request, 'reporte/quejas.html', {'title': 'quejas'})
########################

#@require_http_methods(["GET", "POST"])  # Sensitive
#def hacer_reporte(request):
#    return render(request, 'reporte/hacer_reporte.html', {'title': 'About'})

class ReporteCreateViewInvitado(CreateView):
    model = Reporte
    fields = ['titulo', 'descripcion','image','humedal','tipoReporte']

    def form_valid(self, form):

        form.instance.autor = CustomUser.objects.filter(email='invitado@gmail.com').first()

        if form.is_valid():
            recaptcha_response = self.request.POST.get('g-recaptcha-response')
            url = 'https://www.google.com/recaptcha/api/siteverify'
            values = {
                'secret': settings.GOOGLE_RECAPTCHA_SECRET_KEY,
                'response': recaptcha_response
            }
            data = urllib.parse.urlencode(values).encode()
            req =  urllib.request.Request(url, data=data)
            response = urllib.request.urlopen(req)
            result = json.loads(response.read().decode())
            if result['success']:
                form.save()
                messages.success(self.request, 'Nuevo aporte subido con exito!')
            else:
                messages.error(self.request, 'Invalid reCAPTCHA. Please try again.')
            return redirect('reporte-hacer-reporte')

        return super().form_valid(form)

@require_http_methods(["GET"])  # Sensitive
def login1(request):
    return render(request, 'reporte/login1.html', {'title': 'About'})

@require_http_methods(["GET"])  # Sensitive
def blog(request):

    permitidos =  Reporte.objects.filter(status='Visible').order_by('importancia')

    paginator = Paginator(permitidos, 3) # Show 3 reports per page.
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    return render(request, 'reporte/blog.html',
    {'title': 'About','permitidos':permitidos,'page_obj': page_obj})

#@login_required
#def user(request):
#    return render(request, 'reporte/user.html', {'title': 'About'})

@require_http_methods(["GET"])  # Sensitive
@allowed_users(allowed_roles=['staff','admin'])
def gestionarBlog(request):

    reportes= Reporte.objects.filter(status='Visible').order_by('importancia')
    users = CustomUser.objects.all()
    f = BlogFilter(request.GET,queryset = reportes)
    paginator = Paginator(f.qs, 3) # Show 3 reports per page.

    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    context = {
             'title': 'Gestionar Blog',
             'reportes': reportes,
             'users': users,
             'filter': f,
             'page_obj': page_obj

              }
    return render(request, 'reporte/administrador/gestionarBlog.html', context)

@require_http_methods(["GET"])  # Sensitive
@allowed_users(allowed_roles=['staff','admin'])
def gestionarReportes(request):

    all_reports =  Reporte.objects.order_by('-fecha_reporte')
    #all_reports =  Reporte.objects.all()
    f = ReporteFilter(request.GET,queryset= all_reports)
    paginator = Paginator(f.qs, 3) # Show 3 reports per page.

    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    context = {
         'reportes':all_reports,
         'page_obj': page_obj,
         'filter': f
    }
    return render(request, 'reporte/administrador/gestionarReportes.html', context)

def count_posts_of(user):
    return Reporte.objects.filter(author=user.id).count()

@require_http_methods(["GET"])  # Sensitive
@allowed_users(allowed_roles=['staff','admin'])
def gestionarUsuarios(request):
    reportes= Reporte.objects.all()
    users = CustomUser.objects.all()
    num_reports_per_user= [0 for i in range(len(users))]
    total_reportes = len(reportes)
    if total_reportes == 0:
        total_reportes = 1

    porcentaje_reportes = [0 for i in range(len(users))]
    for i in range (len(users)):
        num_reports_per_user[i]=len(reportes.filter(autor=users[i].id))
        porcentaje_reportes[i] =  (num_reports_per_user[i]/total_reportes)*100

    return render(request, 'reporte/administrador/gestionarUsuarios.html', {'title': 'Gestionar Usuarios',
    'reportes': reportes,'users':users,'user_reports':num_reports_per_user,'porcentaje_reportes':porcentaje_reportes})


from imdbmovie_scrap import *
from pprint import pprint
from bs4 import BeautifulSoup
# import json
import requests
Director = []
language  = []
movie_time = []
def movie_detailes():
    c = 1
    for i in scrap_top_list():
       url = i['links']
       data=requests.get(url)
       soup=BeautifulSoup(data.text,'html.parser')
    #    direct = soup.find(class_="title-overview").find(id="title-overview-widget").find(class_="credit_summary_item").find("a").text.split(", ")
    #    output = "".join(direct)
    #    Director.append(output)

       runtime = soup.find(class_="title_wrapper").find("time").text.strip().replace("min","")
       if "min" not in runtime :
           number = int(runtime[0]) * 60
       else:
            number = int(runtime[0]) * 60 + int(runtime[3:])
       print(number)

        # num = str(number)+"minutes"
        # print(num, c)
        # c+=1
    #    movie_time.append(num)

    # return Director ,  movie_time
movie_detailes()


import json
import pandas as pd

def BrandCheck(result):
    brand_count = 0
    for product in result:
        if product['brand'] == "NA":
            brand_count += 1
    print("product without brand: %s" %brand_count, '\n')
    print("===========")

def PriceCheck(result):
    price_count = 0
    product_index = []
    product_name = []
    product_url = []
    for count, product in enumerate(result):
        if 'See low price in cart' == product['price']:
            product_index.append(count)
            product_name.append(product['name'])
            product_url.append(product['url'])
            df = pd.DataFrame({'product_index':product_index,'product_name':product_name,'product_url':product_url})
            df.to_csv("price_url.csv",index = False, sep=',')

        elif product['price'] == "NA":
            price_count += 1
            print(product['name'])
            print(product['url'],'\n')

    print("===========")
    print("product without price: %s" %price_count, '\n')
    print("===========")

def CategoryCheck(result):
    cat_count = 0
    for product in result:
        if product.get('category') == 'NA':
        #if len(product.get('category')) == 0:
            cat_count += 1
            print(product['keyword'])
            print(product['name'])
            print(product['url'],'\n')
    print("product without category: %s" %cat_count, '\n')
    print("===========")

def AtaGlance(result):
    ata_count = 0
    for product in result:
        if product['at_a_glance'] == 'NA':
            ata_count += 1
    print("product without at_a_glance: %s" %ata_count, '\n')

if __name__ == "__main__":
    with open('product_info_NY2_all.json', 'r', encoding="utf-8") as r:
        result = json.load(r)

    BrandCheck(result)
    PriceCheck(result)
    CategoryCheck(result)
    AtaGlance(result)


from django.conf.urls import url
from django.urls import path
from rest_framework import routers
from . import api, views

router = routers.DefaultRouter()
# router.register('api/user', api.UserViewSet, 'user')
router.register('api/usuario', api.UsuarioViewSet, 'usuario')
router.register('api/classificacao_usuario', api.ClassificacaoUsuarioViewSet, 'classficacao_usuario')
router.register('api/restaurante', api.RestauranteViewSet, 'restaurante')
router.register('api/cor', api.CorViewSet, 'cor')
router.register('api/classificacao_restaurante', api.ClassificacaoRestauranteViewSet, 'classificacao_restaurante')
router.register('api/comida', api.ComidaViewSet, 'comida')
router.register('api/ingredientes', api.IngredientesViewSet, 'ingredientes')
router.register('api/codimentos', api.CodimentosViewSet, 'codimentos')
router.register('api/codimentos_restaurante', api.CodimentosRestauranteViewSet, 'codimentos')
router.register('api/tamanho', api.TamanhoViewSet, 'tamanho')
router.register('api/pedido', api.PedidoViewSet, 'pedido')
router.register('api/pedido_restaurante', api.PedidoRestauranteViewSet, 'pedido_restaurante')
router.register('api/cardapio', api.CardapioViewSet, 'cardapio')
router.register('api/comentario', api.ComentarioViewSet, 'comentario')
router.register('api/tags', api.TagViewSet, 'tags')
router.register('api/tag_restaurante', api.TagRestauranteViewSet, 'tag_restaurante')

urlpatterns = router.urls

urlpatterns += [
	# retrieve
	path('api/classificacao_restaurante_final/<str:restaurante_slug>', api.ClassificacaoRestauranteFinal.as_view(), name='classificacao_restaurante_final'),
	path('api/classificacao_usuario_final/<int:pk>', api.ClassificacaoUsuarioFinal.as_view(), name='classificacao_usuario_final'),
	path('api/filtrar_restaurante/<str:nome>', api.FiltrarTagRestaurante.as_view(), name='tag_filtro'),
	path('api/filtrar_comida/<str:nome>', api.FiltrarTagComida.as_view(), name='tag_comida'),

	path('api/foto_restaurante', views.FotosRestauranteCloud.as_view(), name='foto_restaurante'),
	path('api/foto_restaurante/<int:pk>', views.FotosRestauranteCloudUD.as_view(), name='foto_restauranteUD'),

	path('api/foto_comida', views.FotosComidaCloud.as_view(), name='foto_comida'),
	path('api/foto_comida/<int:pk>', views.FotosComidaCloudUD.as_view(), name='foto_restauranteUD'),

	path('api/pegar_pedidos/<str:restaurante_slug>', api.PegarPedidosRestaurante.as_view(), name='pegar_pedidos'),
	path('api/pegar_comentarios/<str:restaurante_slug>', api.PegarComentariosRestaurante.as_view(), name='pegar_comentarios'),
	path('api/pegar_cardapio/<str:restaurante_slug>', api.PegarCardapioRestaurante.as_view(), name='pegar_cardapio'),
	path('api/pegar_foto_user/<int:pk>', api.PegarFotoUser.as_view(), name='pegar_foto'),
	path('api/pegar_destaques_restaurante/<str:restaurante_slug>', api.PegarDestaques.as_view(), name='pegar_destaques'),
	path('api/pegar_codimentos/<str:restaurante_slug>', api.PegarCodimentosRestaurante.as_view(), name='pegar_codimentos'),

	path('api/buscar/<str:nome>', api.Buscar.as_view(), name='buscar'),
]


from epubreader import EpubReader
from ibm_watson import TextToSpeechV1
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
import pickle
from tqdm import tqdm
import os
from os import system as sys
from Apikeys import *
import subprocess

bookname = 'Slime_ln11'
flderpath = 'Slime_LN11'
book = EpubReader(bookname)
adpath = f"/sdcard/Audiobooks/{flderpath}/"
flderpath = 'Slime_LN11'

apikey ,url = paid_apikey, paid_url
# apikey ,url = free_apikey, free_url

authenticator = IAMAuthenticator(apikey)
tts = TextToSpeechV1(authenticator=authenticator)
tts.set_service_url(url)

OLDMAXSIZE = 5120 -300
MAXSIZE = 5120 -1000
BITRATE = 142.02

_NAME = 'name'
_CONTENT = 'content'

get_size = lambda t: len(t.encode('utf-8'))

get_time = lambda b: (len(b) / BITRATE) / 3600

def split_by_size(items, max_size, get_size=len):
    buffer = []
    buffer_size = 0
    for item in items:
        item_size = get_size(item)
        if buffer_size + item_size + len(buffer)+5 <= max_size:
            buffer.append(item)
            buffer_size += item_size
        else:
            buffer = ' '.join(buffer)
            yield buffer
            buffer = [item]
            buffer_size = item_size
    if buffer_size > 0:
        yield ' '.join(buffer)

for k, chapter in book.chapters.items():
    c_name = chapter[_NAME]
    text = chapter[_CONTENT]
    print(f'====={c_name} : Size: {len(text)}=====')
    splits = list(split_by_size(text.split(),MAXSIZE, get_size))

    for split in tqdm(splits):
        res = tts.synthesize(split, accept='audio/mp3', voice='en-US_AllisonV3Voice').get_result()
        if content == None:
            content = res.content
        else:
            content += res.content
        if get_time(content) >= 30:
            print(f'====={c_name}.{part} finished')
            f_name = f'{flderpath}/{str(k).zfill(2)}_{book.bookname}_{c_name}.{part}.mp3'
            with open(f_name, 'wb') as audio_file:
                audio_file.write(content)
            sys(f"adb push {f_name} {adpath}")

            content = None
            part += 1
    print(f'====={c_name}.{part} finished')
    f_name = f'{flderpath}/{str(k).zfill(2)}_{book.bookname}_{c_name}.{part}.mp3'
    with open(f_name, 'wb') as audio_file:
        audio_file.write(content)
    sys(f"adb push {f_name} {adpath}")
    content = None
    part = 1

''' send files to android phone through ADB'''
# files = [ f for f in os.listdir(flderpath) if f.endswith('.mp3')]
# for f in files:
#     f = f".\{flderpath}\{f}"
#     print(f)
#     sys(f"adb push {f} {adpath}")
# print('hello')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 11 15:44:17 2019

@author: weetee
"""

import pickle
import os
import pandas as pd
import torch
import spacy
import re
from itertools import permutations
from tqdm import tqdm
from .preprocessing_funcs import load_dataloaders
from ..misc import save_as_pickle

import logging

tqdm.pandas(desc="prog-bar")
logging.basicConfig(format='%(asctime)s [%(levelname)s]: %(message)s', \
                    datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO)
logger = logging.getLogger('__file__')

def load_pickle(filename):
    completeName = os.path.join("./data/",\
                                filename)
    with open(completeName, 'rb') as pkl_file:
        data = pickle.load(pkl_file)
    return data

class infer_from_trained(object):
    def __init__(self, args=None, detect_entities=False):
        if args is None:
            self.args = load_pickle("args.pkl")
        else:
            self.args = args
        self.cuda = torch.cuda.is_available()
        self.detect_entities = detect_entities

        if self.detect_entities:
            self.nlp = spacy.load("en_core_web_lg")
        else:
            self.nlp = None
        self.entities_of_interest = ["PERSON", "NORP", "FAC", "ORG", "GPE", "LOC", "PRODUCT", "EVENT", \
                                     "WORK_OF_ART", "LAW", "LANGUAGE", 'PER']

        logger.info("Loading tokenizer and model...")
        from .train_funcs import load_state

        if self.args.model_no == 0:
            from ..model.BERT.modeling_bert import BertModel as Model
            model = args.model_size #'bert-base-uncased'
            lower_case = True
            model_name = 'BERT'
            self.net = Model.from_pretrained(model, force_download=False, \
                                         model_size=args.model_size,\
                                         task='classification', n_classes_=self.args.num_classes)
        elif self.args.model_no == 1:
            from ..model.ALBERT.modeling_albert import AlbertModel as Model
            model = args.model_size #'albert-base-v2'
            lower_case = False
            model_name = 'ALBERT'
            self.net = Model.from_pretrained(model, force_download=False, \
                                         model_size=args.model_size,\
                                         task='classification', n_classes_=self.args.num_classes)
        elif args.model_no == 2: # BioBert
            from ..model.BERT.modeling_bert import BertModel, BertConfig
            model = 'bert-base-uncased'
            lower_case = False
            model_name = 'BioBERT'
            config = BertConfig.from_pretrained('./additional_models/biobert_v1.1_pubmed/bert_config.json')
            self.net = BertModel.from_pretrained(pretrained_model_name_or_path='./additional_models/biobert_v1.1_pubmed/biobert_v1.1_pubmed.bin',
                                                 config=config,
                                                 force_download=False, \
                                                 model_size='bert-base-uncased',
                                                 task='classification',\
                                                 n_classes_=self.args.num_classes)

        self.tokenizer = load_pickle("%s_tokenizer.pkl" % model_name)
        self.net.resize_token_embeddings(len(self.tokenizer))
        if self.cuda:
            self.net.cuda()
        start_epoch, best_pred, amp_checkpoint = load_state(self.net, None, None, self.args, load_best=False)
        logger.info("Done!")

        self.e1_id = self.tokenizer.convert_tokens_to_ids('[E1]')
        self.e2_id = self.tokenizer.convert_tokens_to_ids('[E2]')
        self.pad_id = self.tokenizer.pad_token_id
        self.rm = load_pickle("relations.pkl")

    def get_all_ent_pairs(self, sent):
        if isinstance(sent, str):
            sents_doc = self.nlp(sent)
        else:
            sents_doc = sent
        ents = sents_doc.ents
        pairs = []
        if len(ents) > 1:
            for a, b in permutations([ent for ent in ents], 2):
                pairs.append((a,b))
        return pairs

    def get_all_sub_obj_pairs(self, sent):
        if isinstance(sent, str):
            sents_doc = self.nlp(sent)
        else:
            sents_doc = sent
        sent_ = next(sents_doc.sents)
        root = sent_.root
        #print('Root: ', root.text)

        subject = None; objs = []; pairs = []
        for child in root.children:
            #print(child.dep_)
            if child.dep_ in ["nsubj", "nsubjpass"]:
                if len(re.findall("[a-z]+",child.text.lower())) > 0: # filter out all numbers/symbols
                    subject = child; #print('Subject: ', child)
            elif child.dep_ in ["dobj", "attr", "prep", "ccomp"]:
                objs.append(child); #print('Object ', child)

        if (subject is not None) and (len(objs) > 0):
            for a, b in permutations([subject] + [obj for obj in objs], 2):
                a_ = [w for w in a.subtree]
                b_ = [w for w in b.subtree]
                pairs.append((a_[0] if (len(a_) == 1) else a_ , b_[0] if (len(b_) == 1) else b_))

        return pairs

    def annotate_sent(self, sent_nlp, e1, e2):
        annotated = ''
        e1start, e1end, e2start, e2end = 0, 0, 0, 0
        for token in sent_nlp:
            if not isinstance(e1, list):
                if (token.text == e1.text) and (e1start == 0) and (e1end == 0):
                    annotated += ' [E1]' + token.text + '[/E1] '
                    e1start, e1end = 1, 1
                    continue

            else:
                if (token.text == e1[0].text) and (e1start == 0):
                    annotated += ' [E1]' + token.text + ' '
                    e1start += 1
                    continue
                elif (token.text == e1[-1].text) and (e1end == 0):
                    annotated += token.text + '[/E1] '
                    e1end += 1
                    continue

            if not isinstance(e2, list):
                if (token.text == e2.text) and (e2start == 0) and (e2end == 0):
                    annotated += ' [E2]' + token.text + '[/E2] '
                    e2start, e2end = 1, 1
                    continue
            else:
                if (token.text == e2[0].text) and (e2start == 0):
                    annotated += ' [E2]' + token.text + ' '
                    e2start += 1
                    continue
                elif (token.text == e2[-1].text) and (e2end == 0):
                    annotated += token.text + '[/E2] '
                    e2end += 1
                    continue
            annotated += ' ' + token.text + ' '

        annotated = annotated.strip()
        annotated = re.sub(' +', ' ', annotated)
        return annotated

    def get_annotated_sents(self, sent):
        sent_nlp = self.nlp(sent)
        pairs = self.get_all_ent_pairs(sent_nlp)
        pairs.extend(self.get_all_sub_obj_pairs(sent_nlp))
        if len(pairs) == 0:
            print('Found less than 2 entities!')
            return
        annotated_list = []
        for pair in pairs:
            annotated = self.annotate_sent(sent_nlp, pair[0], pair[1])
            annotated_list.append(annotated)
        return annotated_list

    def get_e1e2_start(self, x):
        e1_e2_start = ([i for i, e in enumerate(x) if e == self.e1_id][0],\
                        [i for i, e in enumerate(x) if e == self.e2_id][0])
        return e1_e2_start

    def infer_one_sentence(self, sentence):
        self.net.eval()
        tokenized = self.tokenizer.encode(sentence); #print(tokenized)
        e1_e2_start = self.get_e1e2_start(tokenized); #print(e1_e2_start)
        tokenized = torch.LongTensor(tokenized).unsqueeze(0)
        e1_e2_start = torch.LongTensor(e1_e2_start).unsqueeze(0)
        attention_mask = (tokenized != self.pad_id).float()
        token_type_ids = torch.zeros((tokenized.shape[0], tokenized.shape[1])).long()

        if self.cuda:
            tokenized = tokenized.cuda()
            attention_mask = attention_mask.cuda()
            token_type_ids = token_type_ids.cuda()

        with torch.no_grad():
            classification_logits = self.net(tokenized, token_type_ids=token_type_ids, attention_mask=attention_mask, Q=None,\
                                        e1_e2_start=e1_e2_start)
            predicted = torch.softmax(classification_logits, dim=1).max(1)[1].item()
        print("Sentence: ", sentence)
        print("Predicted: ", self.rm.idx2rel[predicted].strip(), '\n')
        return predicted

    def infer_sentence(self, sentence, detect_entities=False):
        if detect_entities:
            sentences = self.get_annotated_sents(sentence)
            if sentences != None:
                preds = []
                for sent in sentences:
                    pred = self.infer_one_sentence(sent)
                    preds.append(pred)
                return preds
        else:
            return self.infer_one_sentence(sentence)

class FewRel(object):
    def __init__(self, args=None):
        if args is None:
            self.args = load_pickle("args.pkl")
        else:
            self.args = args
        self.cuda = torch.cuda.is_available()

        if self.args.model_no == 0:
            from ..model.BERT.modeling_bert import BertModel as Model
            from ..model.BERT.tokenization_bert import BertTokenizer as Tokenizer
            model = args.model_size #'bert-large-uncased' 'bert-base-uncased'
            lower_case = True
            model_name = 'BERT'
            self.net = Model.from_pretrained(model, force_download=False, \
                                             model_size=args.model_size,\
                                             task='fewrel')
        elif self.args.model_no == 1:
            from ..model.ALBERT.modeling_albert import AlbertModel as Model
            from ..model.ALBERT.tokenization_albert import AlbertTokenizer as Tokenizer
            model = args.model_size #'albert-base-v2'
            lower_case = False
            model_name = 'ALBERT'
            self.net = Model.from_pretrained(model, force_download=False, \
                                             model_size=args.model_size,\
                                             task='fewrel')
        elif args.model_no == 2: # BioBert
            from ..model.BERT.modeling_bert import BertModel, BertConfig
            from ..model.BERT.tokenization_bert import BertTokenizer as Tokenizer
            model = 'bert-base-uncased'
            lower_case = False
            model_name = 'BioBERT'
            config = BertConfig.from_pretrained('./additional_models/biobert_v1.1_pubmed/bert_config.json')
            self.net = BertModel.from_pretrained(pretrained_model_name_or_path='./additional_models/biobert_v1.1_pubmed/biobert_v1.1_pubmed.bin',
                                            config=config,
                                            force_download=False, \
                                            model_size='bert-base-uncased',
                                            task='fewrel')

        if os.path.isfile('./data/%s_tokenizer.pkl' % model_name):
            self.tokenizer = load_pickle("%s_tokenizer.pkl" % model_name)
            logger.info("Loaded tokenizer from saved file.")
        else:
            logger.info("Saved tokenizer not found, initializing new tokenizer...")
            if args.model_no == 2:
                self.tokenizer = Tokenizer(vocab_file='./additional_models/biobert_v1.1_pubmed/vocab.txt',
                                           do_lower_case=False)
            else:
                self.tokenizer = Tokenizer.from_pretrained(model, do_lower_case=False)
            self.tokenizer.add_tokens(['[E1]', '[/E1]', '[E2]', '[/E2]', '[BLANK]'])
            save_as_pickle("%s_tokenizer.pkl" % model_name, self.tokenizer)
            logger.info("Saved %s tokenizer at ./data/%s_tokenizer.pkl" %(model_name, model_name))

        self.net.resize_token_embeddings(len(self.tokenizer))
        self.pad_id = self.tokenizer.pad_token_id

        if self.cuda:
            self.net.cuda()

        if self.args.use_pretrained_blanks == 1:
            logger.info("Loading model pre-trained on blanks at ./data/test_checkpoint_%d.pth.tar..." % args.model_no)
            checkpoint_path = "./data/test_checkpoint_%d.pth.tar" % self.args.model_no
            checkpoint = torch.load(checkpoint_path)
            model_dict = self.net.state_dict()
            pretrained_dict = {k: v for k, v in checkpoint['state_dict'].items() if k in model_dict.keys()}
            model_dict.update(pretrained_dict)
            self.net.load_state_dict(pretrained_dict, strict=False)
            del checkpoint, pretrained_dict, model_dict

        logger.info("Loading Fewrel dataloaders...")
        self.train_loader, _, self.train_length, _ = load_dataloaders(args)

    def evaluate(self):
        counts, hits = 0, 0
        logger.info("Evaluating...")
        with torch.no_grad():
            for meta_input, e1_e2_start, meta_labels in tqdm(self.train_loader, total=len(self.train_loader)):
                attention_mask = (meta_input != self.pad_id).float()
                token_type_ids = torch.zeros((meta_input.shape[0], meta_input.shape[1])).long()

                if self.cuda:
                    meta_input = meta_input.cuda()
                    attention_mask = attention_mask.cuda()
                    token_type_ids = token_type_ids.cuda()

                outputs = self.net(meta_input, token_type_ids=token_type_ids, attention_mask=attention_mask, Q=None,\
                                  e1_e2_start=e1_e2_start)

                matrix_product = torch.mm(outputs, outputs.T)
                closest_idx = matrix_product[-1][:-1].argmax().cpu().item()

                if closest_idx == meta_labels[-1].item():
                    hits += 1
                counts += 1

        print("Results (%d samples): %.3f %%" % (counts, (hits/counts)*100))
        return meta_input, e1_e2_start, meta_labels, outputs


import pandas as pd

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.grid_search import GridSearchCV
from sklearn.cross_validation import ShuffleSplit
from sklearn.linear_model import SGDClassifier

dataset = pd.read_csv('data/cleaned_data.csv').dropna()

tfidf_vectorizer = TfidfVectorizer(analyzer = "word", ngram_range=(1, 4))

X = tfidf_vectorizer.fit_transform(dataset['source'])
y = dataset['language']

clf = SGDClassifier()
cv = ShuffleSplit(len(y), n_iter=5, test_size=0.2, random_state=0)

parameters = {'loss': ('log', 'hinge'),
              'penalty': ['none', 'l1', 'l2', 'elasticnet'],
              'alpha': [0.001, 0.0001, 0.00001, 0.000001]
}

gs_clf = GridSearchCV(clf, parameters, cv=cv, n_jobs=-1)
gs_clf.fit(X, y)

print('Best parameters:', gs_clf.best_params_)
print('Best score:', gs_clf.best_score_)


import socket
import time
import struct
import keyboard
import os
import datetime

UDP_IP = '10.5.5.1'
UDP_port = 7502

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind((UDP_IP, UDP_port))

# Create target Directory if don't exist
save_directory = "/home/nizar/test_in_the_lab/Bytes/Lidar"
if not os.path.exists(save_directory):
    os.mkdir(save_directory)
    print("Directory ", save_directory, " Created ")
else:
    print("Directory ", save_directory, " already exists")
os.chdir(save_directory)

now = datetime.datetime.now()
T = now.strftime("%Y-%m-%d %H:%M:%S")

file_name = 'Lidar_byte_packet_' + T + '.txt'
f = open(file_name, 'wb')
j = 0
running = True
start1 = time.time()
while running:
    data = sock.recv(12608)
    start2 = time.time()

    if int(struct.unpack('I', data[12:16])[0]) in range(33704, 56408):  # range (33792 , 56320) data[12 + 197 * 4 * 0:16 + 197 * 4 * 0])[0]
    #end = time.time()
        #elapse = end - start
        #print (elapse)
        for i in range(0, 16):
            f.write(data[0 + 788 * i: 16 + 788 * i])  # Frame ID data[(2*4+2) + 197 * 4 * i: (2*4+2 + 2) + 197 * 4 * i]

            for j in range(0, 16):

                f.write(data[(28 + 48*j + 788 * i):
                             (38 + 48*j + 788 * i)] )  # Range_mm_channel data[(((10+(4*3*j))*4) + 197 * 4 * i):
                                                                   #                       (((10+(4*3*j))*4+4) + 197 * 4 * i)

            f.write(data[(196*4) + 197 * 4 * i: 196*4 + (4) + 197 * 4 * i])  # Frame ID data[(196*4) + 197 * 4 * i: (8) + 197 * 4 * i]

        #print('catcha')
        #data = sock.recv(12608)

    if keyboard.is_pressed('q'):
        running = False
    end2 = time.time()
    elapse2 = end2 - start2
    print (elapse2)

end1 = time.time()
elapse1 = end1 - start1
sock.close()
print('Socket is closed!')
f.close()
print('\nFile is closed!\n')
print('time: ', elapse1)


import copy
import logging
from collections import OrderedDict
# pylint: disable=no-name-in-module, import-error
from qiskit.providers.ibmq.ibmqfactory import QX_AUTH_URL
from qiskit.providers.ibmq.credentials import Credentials
# pylint: disable=syntax-error
from qiskit.providers.ibmq.credentials.configrc import (read_credentials_from_qiskitrc,
                                                        write_qiskit_rc)
from qiskit.providers.ibmq.credentials.updater import is_directly_updatable, QE2_AUTH_URL

logger = logging.getLogger(__name__)

class IBMQCredentialsPreferences:
    """IBMQ Credential preferences"""
    _IBMQ_KEY = 'ibmq'
    _HUB_KEY = 'hub'
    _GROUP_KEY = 'group'
    _PROJECT_KEY = 'project'

    def __init__(self, preferences_dict):
        """Create IBMQCredentialsPreferences object."""
        self._ibmq_dict = preferences_dict.get(IBMQCredentialsPreferences._IBMQ_KEY, {})
        self._ibmq_changed = False
        self._credentials_changed = False
        self._credentials = None
        self._read_credentials()

    def _read_credentials(self):
        """Read first credential from file and attempt to convert to v2"""
        self._credentials = None
        try:
            credentials = read_credentials_from_qiskitrc()
            if credentials:
                credentials = list(credentials.values())[0]
                if credentials:
                    if is_directly_updatable(credentials):
                        self._credentials = Credentials(credentials.token,
                                                        QE2_AUTH_URL,
                                                        proxies=credentials.proxies,
                                                        verify=credentials.verify)
                    elif credentials.url == QE2_AUTH_URL:
                        self._credentials = credentials
                    elif credentials.is_ibmq():
                        self._credentials = Credentials(credentials.token,
                                                        QE2_AUTH_URL,
                                                        proxies=credentials.proxies,
                                                        verify=credentials.verify)
                        self._ibmq_dict[IBMQCredentialsPreferences._HUB_KEY] = credentials.hub
                        self._ibmq_dict[IBMQCredentialsPreferences._GROUP_KEY] = credentials.group
                        self._ibmq_dict[IBMQCredentialsPreferences._PROJECT_KEY] = \
                            credentials.project
                    else:
                        # Unknown URL - do not act on it.
                        logger.debug('The stored account with url "%s" could not be '
                                     'parsed.', credentials.url)
        except Exception as ex:  # pylint: disable=broad-except
            logger.debug("Failed to read IBM credentials: '%s'", str(ex))

    def save(self, preferences_dict):
        """Save credentials, always keep only one"""
        if self._credentials_changed:
            try:
                stored_credentials = OrderedDict()
                if self._credentials is not None:
                    stored_credentials[self._credentials.unique_id()] = self._credentials

                write_qiskit_rc(stored_credentials)
            except Exception as ex:  # pylint: disable=broad-except
                logger.debug("Failed to store IBM credentials: '%s'", str(ex))

            self._credentials_changed = False
            self._read_credentials()

        if self._ibmq_changed:
            preferences_dict[IBMQCredentialsPreferences._IBMQ_KEY] = self._ibmq_dict
            self._ibmq_changed = False

    @property
    def credentials(self):
        """ returns credentials """
        return self._credentials

    def set_credentials(self, token, proxy_urls=None):
        """ set credentials """
        if token is not None:
            proxies = {} if proxy_urls is None else {'urls': proxy_urls}
            cred = Credentials(token, QX_AUTH_URL, proxies=proxies)
            if self._credentials is None or self._credentials != cred:
                self._credentials = cred
                self._credentials_changed = True
        else:
            if self._credentials is not None:
                self._credentials_changed = True

            self._credentials = None
            self.hub = None
            self.group = None
            self.project = None

        return self._credentials

    @property
    def url(self):
        """ returns URL """
        if self._credentials is not None:
            return self._credentials.url

        return None

    @property
    def token(self):
        """ returns token """
        if self._credentials is not None:
            return self._credentials.token

        return None

    @property
    def proxies(self):
        """ returns proxies """
        if self._credentials is not None:
            return copy.deepcopy(self._credentials.proxies)

        return None

    @property
    def proxy_urls(self):
        """ returns proxy URL list """
        if self._credentials is not None and \
                'urls' in self._credentials.proxies:
            return copy.deepcopy(self._credentials.proxies['urls'])

        return None

    @property
    def hub(self):
        """ return hub """
        return self._ibmq_dict.get(IBMQCredentialsPreferences._HUB_KEY)

    @hub.setter
    def hub(self, hub):
        """Set hub."""
        self._ibmq_dict[IBMQCredentialsPreferences._HUB_KEY] = hub
        self._ibmq_changed = True

    @property
    def group(self):
        """ returns group """
        return self._ibmq_dict.get(IBMQCredentialsPreferences._GROUP_KEY)

    @group.setter
    def group(self, group):
        """Set group."""
        self._ibmq_dict[IBMQCredentialsPreferences._GROUP_KEY] = group
        self._ibmq_changed = True

    @property
    def project(self):
        """ returns project """
        return self._ibmq_dict.get(IBMQCredentialsPreferences._PROJECT_KEY)

    @project.setter
    def project(self, project):
        """Set project."""
        self._ibmq_dict[IBMQCredentialsPreferences._PROJECT_KEY] = project
        self._ibmq_changed = True


from selenium import webdriver
import urllib.request
import time, re, os

def get_num_from_string(string):
    match = re.findall(r'\d+', string)
    return match[0]

def create_folder_for_section(section):
    pathString = "./Answers/Section{}".format(section)
    #only create the directory if it already is not present
    if not os.path.exists(pathString):
        os.makedirs(pathString)

# root url for the slader answers
slader_root_url = 'http://slader.com'
# linear applications textbook identifier
linear_apps_textbook_url = '/textbook/9780321982384-linear-algebra-and-its-applications-5th-edition'
# use the chrome driver
driver = webdriver.Chrome()

driver.get(slader_root_url + linear_apps_textbook_url)
time.sleep(2)  # wait for page to load

# chapter headers to click
chapter_headers = []
# urls for the sections to get answers for
section_urls = []

chapter_headers = driver.find_elements_by_class_name('toc-item')

# expand the chapter section
chapter_headers[0].click()

# get all the rows that have the pg nums
rows = driver.find_elements_by_xpath(
    '//section[@class="toc-item-expanded"][contains(@style,\'display: block\')]/table/tbody/tr')

# save the urls to the sections
for row in rows:
    section_urls.append(row.get_attribute('data-url'))

# navigate to the section
driver.get(slader_root_url + section_urls[0])

# wait for page to load
time.sleep(2)

# urls for the exercises
exercise_urls = []

# name of the section
section_name = driver.find_element_by_xpath('//span[@class="select2-chosen"]').text
#remove the forward slash in the section name
section_name = section_name.replace('/', '-')

# save the urls to the exercises
exercises = driver.find_elements_by_xpath('//div[@class=\"list-item exercise-in-group-item\"]')
for exercise in exercises:
    exercise_urls.append(exercise.get_attribute('data-url'))

# navigate to the exercies
driver.get(slader_root_url + exercise_urls[0])

#number of the current exercise
exercise_num = driver.find_element_by_xpath('//span[@class="exercises-by-group"]').text
exercise_num = get_num_from_string(exercise_num)

print("Now capturing section {}, exercise {}".format(section_name, exercise_num))

# find the answers to the exercises
answers = driver.find_element_by_xpath(
    '//section[@class=\"solutions-list reloadable\"]')
answer_imgs = driver.find_elements_by_xpath('//img[@class=\"image\"]')

#create the folder for the current chapter and section
create_folder_for_section(section_name)

for idx, answer_img in enumerate(answer_imgs):
    urllib.request.urlretrieve(
        answer_img.get_attribute('src'), "./Answers/Section{}/section{}-#{}_{}.png".format(section_name,section_name,exercise_num,idx))

driver.quit()


from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
import pandas as pd

cloud_config= {
  'secure_connect_bundle': 'secure-connect-big-data.zip'
}

auth_provider = PlainTextAuthProvider("SiZbFNHCwZqpLJUKuUzPqqLZ", "aLWCZKsmZDBW9f86gltP6nlcMkR1L8A2Z-Mb8sXYOiZXKMuk,b3d2oCqMiMDoAj4ZO4XtsEubgTMC0cg_hDmvqLPrbLhD21hworKC1_lRPJ758Y-goOOoe1WgP6WFQNB")
cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)
session = cluster.connect('big_data')

insert = session.prepare("INSERT INTO recipes_by_category(category, average_rating, recipe_id, name ) VALUES (?, ?, ?, ?)")

df = pd.read_csv (r'split_recipes_by_category.csv', nrows=10, encoding="utf-8")

for index,row in df.iterrows():
        category=str(row[0])
        average_rating=float(row[1])
        recipe_id=int(row[2])
        name=str(row[3])

        session.execute(insert, [category, average_rating, recipe_id, name])

# Close the Cassandra session and cluster connection
session.shutdown()
cluster.shutdown()


import molgrid
import torch
import argparse
import os

from openbabel import pybel

p = argparse.ArgumentParser(description="Visualize point cloud (PCD file) with Open3D")
p.add_argument("sdf", type=str, help="SDF file")
p.add_argument("-r", "--resolution", type=float, default=0.5, help="Grid resolution")
p.add_argument(
    "-d", "--dimension", type=float, default=23.5, help="Grid dimension (1D)"
)
p.add_argument(
    "-m", "--ligmap", type=str, default="files/ligmap", help="Ligand types file"
)
p.add_argument("-o", "--output", type=str, default=None, help="Output file")
p.add_argument("--dx", action="store_true", help="Output grids as DX files")

args = p.parse_args()

system = os.path.splitext(os.path.basename(args.sdf))[0]

if args.output is None:
    args.output = f"{system}.pcd"

resolution = args.resolution
dimension = args.dimension

gm = molgrid.GridMaker(resolution=resolution, dimension=dimension)

t = molgrid.FileMappedGninaTyper(args.ligmap)

# Grid dimensions (including types)
gdims = gm.grid_dimensions(t.num_types())

# Pre-allocate grid
# Only one example (batch size is 1)
grid = torch.zeros(1, *gdims, dtype=torch.float32, device="cuda:0")

obmol = next(pybel.readfile("sdf", args.sdf))
obmol.addh()
print(obmol, end="")

# Use OpenBabel molecule object (obmol.OBmol) instead of PyBel molecule (obmol)
cs = molgrid.CoordinateSet(obmol.OBMol, t)

ex = molgrid.Example()
ex.coord_sets.append(cs)

c = ex.coord_sets[0].center()  # Only one coordinate set
print("center:", tuple(c))

# https://gnina.github.io/libmolgrid/python/index.html#the-transform-class
transform = molgrid.Transform(
    c, random_translate=0.0, random_rotation=False,  # float  # bool
)
transform.forward(ex, ex)

# Compute grid
gm.forward(ex, grid[0])

print("grid.shape:", grid.shape)

if args.dx:
    # https://gnina.github.io/libmolgrid/python/index.html#molgrid.write_dx_grids
    # Grid4f is different from Grid4fCUDA
    # If a function takes Grid4f as input, torch.Tensor need to be moved to the CPU
    molgrid.write_dx_grids(
        f"grids/{system}",
        t.get_type_names(),
        grid[0].cpu(),
        ex.coord_sets[0].center(),
        0.5,
    )

cloud = torch.logical_and(grid[0] >= 0.4, grid[0] <= 0.6)

def grid_lims(c):
    """
    Compute grid limits given coordinate of center.
    """
    return c - dimension / 2.0, c + dimension / 2.0

# Number of voxels in one direction
n_steps = round(dimension / resolution) + 1

x1, x2 = grid_lims(c[0])
x = torch.linspace(x1, x2, steps=n_steps)

y1, y2 = grid_lims(c[1])
y = torch.linspace(y1, y2, steps=n_steps)

z1, z2 = grid_lims(c[2])
z = torch.linspace(z1, z2, steps=n_steps)

X, Y, Z = torch.meshgrid(x, y, z)

# Total number of points in the cloud
n_points = torch.sum(cloud)

with open(args.output, "w") as fout:

    fout.write(
        f"""VERSION .7
    FIELDS x y z rgb
    SIZE 8 8 8 4
    TYPE F F F F
    COUNT 1 1 1 1
    WIDTH {n_points}
    HEIGHT 1
    VIEWPOINT 0 0 0 1 0 0 0
    POINTS {n_points}
    DATA ascii\n"""
    )

    for i, name in enumerate(t.get_type_names()):
        idx = cloud[i]  # Indices of cloud points

        for x, y, z in zip(
            X[idx].cpu().numpy(), Y[idx].cpu().numpy(), Z[idx].cpu().numpy()
        ):
            # Miltiplication if to differentiate colors more
            fout.write(f"{x:.5f} {y:.5f} {z:.5f} {i * 1000}\n")


import sys
import  os.path
import  re
import logging
import traceback
import copy
from types import TracebackType
from typing import Optional, OrderedDict, Union
from omegaconf import DictConfig
from scabha.exceptions import ScabhaBaseException, FormattedTraceback
from scabha.substitutions import SubstitutionNS, forgiving_substitutions_from
import rich.progress
import rich.logging
from rich.tree import Tree
from rich import print as rich_print
from rich.markup import escape
from rich.padding import Padding

from . import task_stats
from .task_stats import declare_subtask, declare_subtask_attributes, \
                        declare_subcommand, update_process_status, \
                        run_process_status_update

class FunkyMessage(object):
    """Class representing a message with two versions: funky (with markup), and boring (no markup)"""
    def __init__(self, funky, boring=None, prefix=None):
        self.funky = funky
        self.boring = boring if boring is not None else funky
        self.prefix = prefix
    def __str__(self):
        return self.funky
    def __add__(self, other):
        if isinstance(other, FunkyMessage):
            return FunkyMessage(f"{self}{other}", f"{self.boring}{other.boring}")
        else:
            return FunkyMessage(f"{self}{other}", f"{self.boring}{other}")

def defunkify(arg: Union[str, FunkyMessage]):
    return arg.boring if isinstance(arg, FunkyMessage) else arg

class MultiplexingHandler(logging.Handler):
    """handler to send INFO and below to stdout, everything above to stderr"""
    def __init__(self, info_stream=sys.stdout, err_stream=sys.stderr):
        super(MultiplexingHandler, self).__init__()
        self.info_handler = logging.StreamHandler(info_stream)
        self.err_handler = logging.StreamHandler(err_stream)
        self.multiplex = True

    def emit(self, record):
        # does record come with its own handler? Rather use that
        if hasattr(record, 'custom_console_handler'):
            handler = record.custom_console_handler
        else:
            handler = self.err_handler if record.levelno > logging.INFO and self.multiplex else self.info_handler
        handler.emit(record)
        # ignore broken pipes, this often happens when cleaning up and exiting
        try:
            handler.flush()
        except BrokenPipeError:
            pass

    def flush(self):
        try:
            self.err_handler.flush()
            self.info_handler.flush()
        except BrokenPipeError:
            pass

    def close(self):
        self.err_handler.close()
        self.info_handler.close()

    def setFormatter(self, fmt):
        self.err_handler.setFormatter(fmt)
        self.info_handler.setFormatter(fmt)

class StimelaLogFormatter(logging.Formatter):
    DEBUG_STYLE   = "dim", ""
    WARNING_STYLE = "", "yellow"
    ERROR_STYLE   = "bold", "red"
    CRITICAL_STYLE  = "bold", "red"

    _ansi_escape = re.compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]')

    def __init__(self, boring=False):
        datefmt = "%Y-%m-%d %H:%M:%S"
        if not boring:
            super().__init__("{asctime} {name} [{style}]{levelname}: {message}[/{style}]", datefmt, style="{")
            self._prefix_fmt = logging.Formatter("[{style}]{prefix} {message}[/{style}]", style="{")
        else:
            super().__init__("{asctime} {name} {levelname}: {message}", datefmt, style="{")
            self._prefix_fmt = logging.Formatter("{prefix} {message}", style="{")
        self.boring = boring

    def format(self, record):
        # apply styling
        record = copy.copy(record)
        if not hasattr(record, 'style'):
            if record.levelno <= logging.DEBUG:
                font, color = self.DEBUG_STYLE
            elif record.levelno >= logging.CRITICAL:
                font, color = self.SEVERE_STYLE
            elif record.levelno >= logging.ERROR:
                font, color = self.ERROR_STYLE
            elif record.levelno >= logging.WARNING:
                font, color = self.WARNING_STYLE
            else:
                font = color = ""
            if getattr(record, 'color', None):
                color = record.color.lower()
            if hasattr(record, 'bold') or hasattr(record, 'boldface'):
                if "bold" not in font:
                    font = f"bold {font}"
            setattr(record, 'style', f"{font} {color}".strip() or "normal")
        # in boring mode, make funky messages boring, and strip ANSI codes
        if self.boring:
            record.msg = self._ansi_escape.sub('', defunkify(record.msg))
        # select format based on whether we have prefix or not
        if hasattr(record, 'prefix'):
            if self.boring:
                record.prefix = defunkify(record.prefix)
            return self._prefix_fmt.format(record)
        else:
            return super().format(record)

_logger = None
log_console_handler = log_formatter = log_file_formatter = log_boring_formatter = log_colourful_formatter = None

LOG_DIR = '.'

def is_logger_initialized():
    return _logger is not None

def declare_chapter(title: str, **kw):
    progress_console.rule(title, **kw)

def logger(name="STIMELA", propagate=False, boring=False, loglevel="INFO"):
    """Returns the global Stimela logger (initializing if not already done so, with the given values)"""
    global _logger
    if _logger is None:
        _logger = logging.getLogger(name)
        if type(loglevel) is str:
            loglevel = getattr(logging, loglevel)
        _logger.setLevel(loglevel)
        _logger.propagate = propagate

        global log_console_handler, log_formatter, log_file_formatter, log_boring_formatter, log_colourful_formatter
        global progress_console, progress_bar

        log_boring_formatter = StimelaLogFormatter(boring=True)
        log_colourful_formatter = StimelaLogFormatter(boring=False)

        log_formatter = log_boring_formatter if boring else log_colourful_formatter

        progress_bar, progress_console = task_stats.init_progress_bar()

        log_console_handler = rich.logging.RichHandler(console=progress_console,
                            highlighter=rich.highlighter.NullHighlighter(), markup=True,
                            show_level=False, show_path=False, show_time=False, keywords=[])

        log_console_handler.setFormatter(log_formatter)
        log_console_handler.setLevel(loglevel)

        _logger.addHandler(log_console_handler)
        _logger_console_handlers[_logger.name] = log_console_handler

        import scabha
        scabha.set_logger(_logger)

    return _logger

_logger_file_handlers = {}
_logger_console_handlers = {}

def has_file_logger(log: logging.Logger):
    return log.name in _logger_file_handlers

def disable_file_logger(log: logging.Logger):
    current_logfile, fh = _logger_file_handlers.get(log.name, (None, None))
    if fh is not None:
        fh.close()
        log.removeHandler(fh)
        del _logger_file_handlers[log.name]

class DelayedFileHandler(logging.FileHandler):
    """A version of FileHandler that also handles directory and symlink creation in a delayed way"""
    def __init__(self, logfile, symlink, mode):
        self.symlink, self.logfile = symlink, logfile
        self.is_open = False
        super().__init__(logfile, mode, delay=True)

    def get_logfile_dir(self):
        """Gets name of logfile and ensures the directory exists"""
        if not self.is_open:
            self.is_open = True
            logdir = os.path.dirname(self.logfile)
            if logdir and not os.path.exists(logdir):
                os.makedirs(logdir)
                if self.symlink:
                    symlink_path = os.path.join(os.path.dirname(logdir.rstrip("/")) or ".", self.symlink)
                    # remove existing symlink
                    if os.path.islink(symlink_path):
                        os.unlink(symlink_path)
                    # Make symlink to logdir. If name exists and is not a symlink, we'll do nothing
                    if not os.path.exists(symlink_path):
                        os.symlink(os.path.basename(logdir), symlink_path)
        return os.path.dirname(self.logfile)

    def emit(self, record):
        self.get_logfile_dir()
        return super().emit(record)

def setup_file_logger(log: logging.Logger, logfile: str, level: Optional[Union[int, str]] = logging.INFO, symlink: Optional[str] = None):
    """Sets up logging to file

    Args:
        log (logging.Logger): Logger object
        logfile (str): logfile. May contain dirname, which will be created as needed.
        level (Optional[Union[int, str]], optional): Logging level, defaults to logging.INFO.
        symlink (Optional[str], optional): if set, and logfile contains a dirname that is created, sets named symlink to point to it
            (This is useful for patterns such as logfile="logs-YYMMDD/logfile.txt", then logs -> logs-YYMMDD)

    Returns:
        [logging.Logger]: logger object
    """
    current_logfile, fh = _logger_file_handlers.get(log.name, (None, None))

    # does the logger need a new FileHandler created
    if current_logfile != logfile:
        log.debug(f"will switch to logfile {logfile} (previous was {current_logfile})")
        # remove old FH if so
        if fh is not None:
            fh.close()
            log.removeHandler(fh)

        # create new FH
        fh = DelayedFileHandler(logfile, symlink, 'w')
        fh.setFormatter(log_boring_formatter)
        log.addHandler(fh)

        _logger_file_handlers[log.name] = logfile, fh

        # if logging to console, disable propagation from this sub-logger, and add a console handler
        # This ensures that parent loggers that log to files to not get repeated messages
        if log_console_handler:
            log.propagate = False
            if log.name not in _logger_console_handlers:
                _logger_console_handlers[log.name] = log_console_handler
                log.addHandler(log_console_handler)

    # resolve level
    if level is not None:
        if type(level) is str:
            level = getattr(logging, level, logging.INFO)
        fh.setLevel(level)

    return log

def update_file_logger(log: logging.Logger, logopts: DictConfig, nesting: int = 0, subst: Optional[SubstitutionNS] = None, location=[]):
    """Updates logfiles associated with given logger based on option settings

    Args:
        log (logging.Logger):                          Logger object
        nesting (int):                                 nesting level of this logger
        logopts (Union[StimelaLogConfig, DictConfig]): config settings
        subst (Dict[str, Any]):                        dictionary of substitutions for pathnames in logopts
        location (List[str]):                          location of this logger in the hierarchy

    Returns:
        [type]: [description]
    """

    if logopts.enable and logopts.nest >= nesting:
        path = os.path.join(logopts.dir or ".", logopts.name + logopts.ext)

        if subst is not None:
            with forgiving_substitutions_from(subst, raise_errors=False) as context:
                path = context.evaluate(path, location=location + ["log"])
                if context.errors:
                    for err in context.errors:
                        log.error(f"bad substitution in log path: {err}")
                    return None

        # substitute non-filename characters for _
        path = re.sub(r'[^a-zA-Z0-9_./-]', '_', path)

        # setup the logger
        setup_file_logger(log, path, level=logopts.level, symlink=logopts.symlink)
    else:
        disable_file_logger(log)

def get_logfile_dir(log: logging.Logger):
    """Returns filename associated with the logger, or None if not logging to file"""
    logfile, fh = _logger_file_handlers.get(log.name, (None, None))
    if logfile is None:
        return None
    return fh.get_logfile_dir()

def log_exception(*errors, severity="error", log=None):
    """Logs one or more error messages or exceptions (unless they are marked as already logged), and
    pretty-prints them to the console  as appropriate.
    """
    def exc_message(e):
        if isinstance(e, ScabhaBaseException):
            return escape(e.message)
        elif type(e) is str:
            return escape(e)
        else:
            return escape(f"{type(e).__name__}: {e}")

    if severity == "error":
        colour = "bold red"
        message_dispatch = (log or logger()).error
    else:
        colour = "yellow"
        message_dispatch = (log or logger()).warning

    trees = []
    do_log = False
    messages = []

    def add_dict(dd, tree):
        for field, value in dd.items():
            if isinstance(value, (dict, OrderedDict, DictConfig)):
                subtree = tree.add(escape(f"{field}:"))
                add_dict(value, subtree)
            else:
                tree.add(escape(f"{field}: {value}"))

    def add_nested(excs, tree):
        for exc in excs:
            if isinstance(exc, Exception):
                subtree = tree.add(f"{exc_message(exc)}")
                # tbtree = subtree.add("Traceback:")
                # for line in traceback.format_exception(exc):
                #     tbtree.add(line)
                if isinstance(exc, ScabhaBaseException) and exc.nested:
                    add_nested(exc.nested, subtree)
            elif type(exc) is TracebackType:
                subtree = tree.add(f"[dim]Traceback:[/dim]")
                for line in traceback.format_tb(exc):
                    subtree.add(f"[dim]{escape(line.rstrip())}[/dim]")
            elif type(exc) is FormattedTraceback:
                subtree = tree.add(f"[dim]Traceback:[/dim]")
                for line in exc.lines:
                    subtree.add(f"[dim]{escape(line)}[/dim]")
            elif isinstance(exc, (dict, OrderedDict, DictConfig)):
                add_dict(exc, tree)
            else:
                tree.add(str(exc))

    has_nesting = False

    for exc in errors:
        if isinstance(exc, ScabhaBaseException):
            messages.append(exc.message)
            if not exc.logged:
                do_log = exc.logged = True
            tree = Tree(f"[{colour}]:warning: {exc_message(exc)}[/{colour}]", guide_style="dim")
            trees.append(tree)
            if exc.nested:
                add_nested(exc.nested, tree)
                has_nesting = True
        else:
            tree = Tree(f"[{colour}]:warning: {exc_message(exc)}[/{colour}]", guide_style="dim")
            trees.append(tree)
            do_log = True
            messages.append(str(exc))

    if do_log:
        message_dispatch(": ".join(messages))

    printfunc = task_stats.progress_bar.console.print if task_stats.progress_bar is not None else rich_print

    if has_nesting:
        declare_chapter("detailed error report follows", style="red")
        for tree in trees:
            printfunc(Padding(tree, pad=(0,0,0,8)))


#filepath = "testFileNames.txt"
import os
import csv
# from scipy.integrate import simps
import scipy.integrate # https://github.com/conda/conda/issues/6396#issuecomment-350254762
# import scipy.integrate as simps
import math

def readlines(filepath):

    with open(filepath) as fp:
        lines = fp.readlines()

    # Clean the commands by stripping the newline characters (\n) and the \r
    filenames = []
    for line in lines:
        line = line.strip("\n").strip("\r")
        filenames.append(line)

    return filenames

def splitOnOffFileNames(filepath):

    # First remove the file it if exists
    removeFile("OnList.txt")
    removeFile("OffList.txt")

    fileNames = readlines(filepath)
    onList = []
    offList = []
    filepath = ""
    for file in fileNames:
        if "On" in file :
            filepath = "OnList.txt"
            onList.append(file)
        elif "Off" in file :
            filepath = "OffList.txt"
            offList.append(file)
        else:
            filepath = "errorPath.txt"
            print("Error in splitOnOffFileNames")

        with open(filepath, "a+") as f:
            f.write(file)
            f.write("\n")

    return onList, offList

def writeValues(outputFileName, listItems):

    removeFile(outputFileName) # Remove the file if it already exists

    for item in listItems:
        with open(outputFileName, "a+") as file:
            file.write(str(item))
            file.write("\n")

def removeFile(fileName):
    # Handle errors while calling os.remove()
    try:
        os.remove(fileName)
    except:
        print("")

# https://stackoverflow.com/questions/26082718/python-write-two-lists-into-two-column-text-file/33324278
def exportToCSV(myTuple, filename, headerTuple) :
    removeFile(filename)    # If the file exists already, remove it.
    with open(filename, 'w+', newline='') as f:
        writer = csv.writer(f, delimiter='\t')
        writer.writerows(headerTuple)
        writer.writerows(myTuple)

def diffList(list1,list2):
    differenceList = []

    zip_object = zip(list1, list2)

    for list1_i, list2_i in zip_object:
        differenceList.append(list1_i-list2_i)

    return differenceList

def calc_list_avg(list_of_lists, numAverages) :
    zipped_lists = zip(*list_of_lists)
    avg_list = []
    for list in zipped_lists :
        avg_list.append(sum(list)/float(numAverages))

    return avg_list

def calc_area(trace_points_dBm, freq_array):
    # Calculate area under the curve
    # Convert dBm to mW
    points_mW = []
    for point in trace_points_dBm:
        point_W = 10.0**(point/10.0)    # Convert dBm to W
        point_mW = 0.001*point_W        # Convert W to mW
        points_mW.append(point_mW)

    area = scipy.integrate.simps(points_mW, freq_array) # scipy.integrate.

    areadB = 10.0*math.log10(area)

    return areadB

def calc_max_min(listofLists):
    zipped_lists = zip(*listofLists)
    max_list = []
    min_list = []
    for list in zipped_lists :
        max_list.append(max(list))
        min_list.append(min(list))

    return max_list, min_list


import numpy as np
import pandas
from scipy.stats import norm
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier

def c2s_test(a: pandas.DataFrame, b: pandas.DataFrame, classifier='nn') -> float:
    """
    Based on
    "Revisiting Classifier Two-Sample Tests", Lopez 2016
    See page 3

    Parameters
    ----------
    a : pandas.DataFrame
        LHS data set
    b : pandas.DataFrame
        RHS data set
    classifier : str or object
        'nn' for Neural Network, 'knn' for nearest neighbor, or a classifier instance
    Returns
    -------
    p-value : float
    """
    if isinstance(classifier, str):
        if classifier == 'knn':
            classifier = KNeighborsClassifier()
        elif classifier == 'nn':
            classifier = MLPClassifier()
        else:
            raise ValueError(f'Unknown classifier {classifier}')
    assert hasattr(classifier, 'fit')
    assert hasattr(classifier, 'predict')

    x = np.concatenate([a, b])
    y = np.concatenate([np.zeros(len(a)), np.ones(len(b))])

    # Second: Shuffle and split into train/test
    x_train, x_test, y_train, y_test = train_test_split(x, y, shuffle=True)

    # Third: Train
    classifier.fit(x_train, y_train)

    # Fourth: Test statistic is accuracy
    score = classifier.score(x_test, y_test)

    # Under H0, the score can be approximated by N(1/2, 1/(4|X_test|))
    mean, std = 0.5, np.sqrt(1 / (4 * len(x_test)))
    p = norm.cdf(score, mean, std)
    if p > 0.5:
        return 2 * (1 - p)
    return 2 * p

def c2s_nn_test(a: pandas.DataFrame, b: pandas.DataFrame) -> float:
    return c2s_test(a, b, 'nn')

def c2s_knn_test(a: pandas.DataFrame, b: pandas.DataFrame) -> float:
    return c2s_test(a, b, 'knn')


# Import dependencies
import cv2

# Read Images
image = cv2.imread('media/test.jpg', cv2.IMREAD_COLOR)

# Print error message if image is null
if image is None:
    print('Could not read image')

# Print X
pAZ_1 = (10, 10)
pBZ_1 = (100, 100)

pAZ_2 = (100, 10)
pBZ_2 = (10, 100)

cv2.line(image, pAZ_1, pBZ_1, (255, 255, 0), thickness=3, lineType=cv2.LINE_AA)
cv2.line(image, pAZ_2, pBZ_2, (255, 255, 0), thickness=3, lineType=cv2.LINE_AA)

# Print Y
pAY_1 = (120, 10)
pBY_1 = (170, 60)

pAY_2 = (220, 10)
pBY_2 = (140, 100)

cv2.line(image, pAY_1, pBY_1, (255, 255, 0), thickness=3, lineType=cv2.LINE_AA)
cv2.line(image, pAY_2, pBY_2, (255, 255, 0), thickness=3, lineType=cv2.LINE_AA)

# Print Z
pAZ_1 = (240, 10)
pBZ_1 = (290, 10)

pAZ_2 = (290, 10)
pBZ_2 = (240, 100)

pAZ_3 = (240, 100)
pBZ_3 = (290, 100)

cv2.line(image, pAZ_1, pBZ_1, (255, 255, 0), thickness=3, lineType=cv2.LINE_AA)
cv2.line(image, pAZ_2, pBZ_2, (255, 255, 0), thickness=3, lineType=cv2.LINE_AA)
cv2.line(image, pAZ_3, pBZ_3, (255, 255, 0), thickness=3, lineType=cv2.LINE_AA)

# define the center of circle
circle_center = (55, 55)
# define the radius of the circle
radius = 55
#  Draw a circle using the circle() Function
cv2.circle(image, circle_center, radius, (0, 0, 255), thickness=3, lineType=cv2.LINE_AA)

# define the center of circle
circle_center = (290 + 20 + radius, 55)
# define the radius of the circle
#  Draw a circle using the circle() Function
cv2.circle(image, circle_center, radius, (0, 0, 255), thickness=-1, lineType=cv2.LINE_AA)

# define the center point of ellipse
ellipse_center = (415, 190)
# define the major and minor axes of the ellipse
axis1 = (100, 50)
axis2 = (125, 50)
# draw the ellipse
# Horizontal
cv2.ellipse(image, ellipse_center, axis1, 0, 30, 330, (255, 0, 0), thickness=3)
# Vertical
cv2.ellipse(image, ellipse_center, axis2, 90, 0, 360, (0, 0, 255), thickness=3)

# let's write the text you want to put on the image
text = 'I am a Happy dog!'
# org: Where you want to put the text
org = (10, 200)
# write the text on the input image
cv2.putText(image, text, org, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=2, color=(250, 225, 100))

cv2.imshow('Image Line', image)
cv2.waitKey(0)


from abc import ABC, abstractmethod
from typing import Type

import numpy as np

from pyxc.core.container import Container2D
from ..core.processor.arrays import (
    image_notator,
    image_serializer,
    reshape_if_1dim,
    xyd_splitter,
)
from numpy.typing import ArrayLike

class DataLoaderBase(ABC):
    """
    An abstract base class for data loaders.

    Initializes a container with preprocessed data and returns the container when called. Works with a Layer object.
    Subclasses should implement the `prep` method to specify the preprocessing of data.

    Parameters
    ----------
    container : Container2D
        The container class to be initialized with the preprocessed data.
    data : array_like
        The raw data to be preprocessed.

    Attributes
    ----------
    container : Container2D
        The container that was initialized with the preprocessed data.
    """

    def __init__(self, container: Type["Container2D"], data: ArrayLike):
        x_raw, y_raw, data = self.prep(data)
        self.container = container(x_raw=x_raw, y_raw=y_raw, data=data)

    def __call__(self) -> Type["Container2D"]:
        """
        Return the container when the instance is called.

        Returns
        -------
        container : object
            The container that was initialized with the preprocessed data.

        Notes
        -----
        This method was implemented since this is a straightforward way to provide the prepped container object.
        """
        return self.container

    @abstractmethod
    def prep(self, data):
        """
        Process the provided data.

        This method should be implemented in subclasses to specify the preprocessing steps.

        Parameters
        ----------
        data : any
            The raw data to be preprocessed.

        Returns
        -------
        x_raw: np.ndarray
            1-dimensional numpy array with shape of (n, ).
        y_raw: np.ndarray
            1-dimensional numpy array with shape of (n, ).
        data: np.ndarray
            Structured array with proper column names.
        """
        pass

class ImageLoader(DataLoaderBase):
    def prep(self, data: ArrayLike) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
        """
        Process the provided image data.

        Provided dimension of the image data should be 2 or 3. If the dimension is 2, then the image is assumed to be
        grayscale. If the dimension is 3, then the image is assumed to be multi-channel. The number of channels is not
        limited.

        Parameters
        ----------
        data : any
            The raw image data to be preprocessed.

        Returns
        -------
        x_serial, y_serial, prepped_data : tuple
            The preprocessed image data. x_serial and y_serial are the flattened arrays
            for the X and Y coordinates, and prepped_data is the serialized and reshaped image data.
        """
        x_serial, y_serial, image = image_notator(data)  # Get X and Y coordinates.
        if image.ndim == 3:
            prepped_data = image_serializer(image)
        elif image.ndim == 2:
            prepped_data = reshape_if_1dim(image_serializer(image))
        else:
            raise ValueError("Error: Image dimension error.")
        return x_serial.flatten(), y_serial.flatten(), prepped_data

class XYDLoader(DataLoaderBase):
    def prep(self, data: ArrayLike) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
        """
        Process the provided XYD data.

        Parameters
        ----------
        data : array_like or structured array
            The raw XYD data to be preprocessed. The first and second columns should be
            x- and y-coordinates, and the third and subsequent columns should be data columns.

        Returns
        -------
        x_raw, y_raw, data : tuple
            The preprocessed XYD data. x_raw and y_raw are the x- and y-coordinates,
            and data contains the rest of the columns.
        """
        x_raw, y_raw, data = xyd_splitter(data)
        return x_raw, y_raw, data


import argparse

def string_sum(string):
    """Return sum of digits in string"""
    digits = [int(char) for char in string if char.isdigit()]
    return sum(digits)

def ladder(string):
    """
    Return ladder that looks like:
                  #
                 ##
                ###
    """
    n_steps = int(string) if string.isdigit() else 1
    result = ''
    for i in range(1, n_steps+1):
        step = '#'*i
        result += f'{step:{n_steps}}'[::-1] + '\n'
    return result

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('-t', '--task', help='Type of task', type=int)
    parser.add_argument('-v', '--value', help='Task argument value')

    args = parser.parse_args()

    try:
        tasks = {
            1: string_sum,
            2: ladder
        }
        print(tasks[args.task](args.value))

    except KeyError as e:
        print('Task type error, check your input and try again.')

if __name__ == '__main__':
    main()


import requests, time, sys
# python3 post_api.py API_KEY

if len(sys.argv) < 2:
    print("Missing API Key!")
    quit()

key = sys.argv[1]
filename = "wav_files/sample.wav"

fh = open(filename, "rb")
data = fh.read()
fh.close()

multipart_form_data = {
    'file': data,
    'key': key,
    'model': 'fast'
}

print("Sending wav file %s" % (filename,))
start = time.time()

response = requests.post('http://api.privox.io/cgi-bin/stt.py', files=multipart_form_data)

took = int(time.time() - start)

print("Took %s seconds" % (took,))
print(response.content.decode('utf-8').strip())


import LTP
import json

def cws(jsondata):
    data = jsondata['data']
    text = data['defualt']
    result = LTP.cws(text)
    cwslist = result[0]
    cwsobj = {"defualt":cwslist}
    jsonstr = json.dumps(cwsobj,ensure_ascii=False)
    return jsonstr


import re
from collections import Counter
from math import log

from ComputeGraph import ComputeGraph

def main():
    split_word = ComputeGraph.ComputeGraph(docs='data/text_corpus.txt')
    count_docs = ComputeGraph.ComputeGraph(docs='data/text_corpus.txt')
    count_idf = ComputeGraph.ComputeGraph(docs=split_word)
    calc_index = ComputeGraph.ComputeGraph(docs=split_word, save='results/tf-idf.txt')

    def tokenizer_mapper(line):
        tokens = line['text'].split()
        for token in tokens:
            word = re.search('([a-zA-Z])+', token)
            if word and len(word.group()) > 4:
                yield {
                    'doc_id': line['doc_id'],
                    'word': word.group().lower()
                }

    split_word.Map(tokenizer_mapper)

    def columns_number_folder(state, _):
        for column in state:
            state[column] += 1
        return state

    count_docs.Fold(columns_number_folder, {'docs_count': 0})

    def unique_columns_reducer(records):
        yield {
            'doc_id': records[0]['doc_id'],
            'word': records[0]['word']
        }

    count_idf.Reduce(unique_columns_reducer, 'doc_id', 'word')
    count_idf.Join(count_docs, [], strategy='outer left')

    def docs_contain_word_reducer(records):
        yield {
            'word': records[0]['word'],
            'docs_contain_word': len(records),
            'docs_count': records[0]['docs_count']
        }

    count_idf.Reduce(docs_contain_word_reducer, 'word')

    def term_frequency_reducer(records):
        word_count = Counter()
        for record in records:
            word_count[record['word']] += 1

        total = sum(word_count.values())
        for word, count in word_count.items():
            yield {
                'doc_id': record['doc_id'],
                'word': word,
                'tf': count / total
            }

    calc_index.Reduce(term_frequency_reducer, 'doc_id')

    calc_index.Join(count_idf, ['word'], strategy='left outer')

    def invert_reducer(records):
        for i, record in enumerate(records):
            records[i]['tf-idf'] = record['tf'] * \
                                   log(record['docs_count'] / record['docs_contain_word'])
        records = sorted(records, reverse=True, key=lambda i: i['tf-idf'])
        top = [tuple([record['doc_id'], record['tf-idf']])
               for record in records[:3]]
        yield {
            'term': records[0]['word'],
            'index': top
        }

    calc_index.Reduce(invert_reducer, 'word')

    calc_index.Compute(True)

if __name__ == '__main__':
    main()


"""
Snipe command.

(C) 2022-2023 - B1ue-Dev
"""

import logging
import asyncio
from datetime import datetime
import interactions
from interactions.ext.hybrid_commands import (
    hybrid_slash_command,
    HybridContext,
)
from src.utils.utils import handle_username

class Snipe(interactions.Extension):
    """Extension for /snipe command."""

    def __init__(self, client: interactions.Client) -> None:
        self.client: interactions.Client = client
        self._cached: dict[str, str] = {}

    @interactions.listen(interactions.events.MessageDelete)
    async def on_message_delete(
        self, msg: interactions.events.MessageDelete
    ) -> None:
        """Listen to MESSAGE_DELETE and write to cache."""

        message: interactions.Message = msg.message
        if not isinstance(message, interactions.Message):
            return

        _channel_id = str(message.channel.id)

        if not self._cached.get(_channel_id, None):
            self._cached[_channel_id] = []

        if len(self._cached[_channel_id]) == 5:
            self._cached[_channel_id].pop(0)

        self._cached[_channel_id].append(
            {
                "content": message.content,
                "author_id": str(message.author.id),
                "timestamp": str(round(datetime.now().timestamp())),
            }
        )

        async def delete_message():
            try:
                _item = self._cached[_channel_id][-1:][0]
                await asyncio.sleep(300)
                self._cached[_channel_id].remove(_item)
            except ValueError:
                pass

            if len(self._cached[_channel_id]) == 0:
                self._cached.pop(_channel_id)

        asyncio.create_task(delete_message())

    @hybrid_slash_command(
        name="snipe",
        description="Snipes the last deleted message from the current channel.",
        dm_permission=False,
    )
    async def snipe(self, ctx: HybridContext) -> None:
        """Snipes the last deleted message from the current channel."""

        channel_id = str(ctx.channel_id)
        if not self._cached.get(channel_id, None):
            return await ctx.send("No message to snipe.")

        embed = interactions.Embed(
            footer=interactions.EmbedFooter(
                text=f"Requested by {handle_username(ctx.author.user.username, ctx.author.user.discriminator)}",
                icon_url=ctx.author.avatar.url,
            ),
            description="",
        )
        for i in self._cached[channel_id]:
            embed.description += f"""<@{i["author_id"]}>: {i["content"]} - <t:{i["timestamp"]}:R>\n"""

        await ctx.send(embed=embed)

def setup(client) -> None:
    """Setup the extension."""
    Snipe(client)
    logging.info("Loaded Snipe extension.")


from rich.console import Console
from rich.progress import track
from rich.theme import Theme
import gspread
import os
import time
from settings import spreadsheet, worksheet, sa_name
from PyDictionary import PyDictionary

dictionary = PyDictionary()

custom_theme = Theme({'success': 'bold underline green', 'warning': 'underline yellow', 'error': 'bold red', 'error_api': 'blue'})
console = Console(theme=custom_theme)

os.system('cls' if os.name == 'nt' else 'clear')

#TODO implement undrelines if word appears in definition
#TODO add synonyms & antonyms
#TODO add examples <- create my own method

service_account = gspread.service_account(filename=sa_name)
spread_sheet = service_account.open(spreadsheet)
work_sheet = spread_sheet.worksheet(worksheet)

possibles = ['Noun', 'Verb', 'Adjective', 'Adverb']
not_single = []
review = []
cur_pos = 2
rows_count = 1
ran = False

def county():
    global rows_count, ran
    cnt_track = 0
    for i in range(rows_count, work_sheet.row_count+1):
        if cnt_track >=10: # checking for the end of spreadsheet
            rows_count -= 10
            break
        if (work_sheet.cell(i, 1).value == None):
            cnt_track += 1
            rows_count += 1
        elif (work_sheet.cell(i, 1).value != None):
            cnt_track = 0
            rows_count += 1
        time.sleep(0.1)
    console.print('Words counted!', style='success')
    ran = True
    return rows_count-1

def dict(i):
    result = ''
    definition = dictionary.meaning(f'{work_sheet.cell(i, 1).value}', disable_errors=True)
    # print(definition)
    for i in possibles:
        try:
            definition[i]
            if len(definition[i]) == 1:
                result += f'{i}: ' + definition[i][0] + '\n'
            elif len(definition[i]) >=3:
                result += f'{i}: '
                for pos in range(3):
                    result += f'{pos+1}) ' + definition[i][pos] + '\n'
            else:
                result += f'{i}: '
                for pos in range(len(definition[i])+1):
                    result += f'{pos+1}) ' + definition[i][pos] + '\n'
        except:
            pass

    return result.strip()

def search(beg=2,end=rows_count): # current position
    global cur_pos, review
    for i in track(range(beg, rows_count), description='Processing...'):
        try:
            if len((work_sheet.cell(i, 1).value).split()) > 1:
                not_single.append(f'A{i}')
        except:
            pass
        if (work_sheet.cell(i, 1).value != None) and (work_sheet.cell(i, 2).value == None):
            work_sheet.update(f'B{i}', dict(i))
        elif (work_sheet.cell(i, 1).value == None) and (work_sheet.cell(i, 2).value != None):
            work_sheet.update(f'A{i}', 'REVIEW!!!')
            review.append(f'A{i}:B{i}')
            # print it using rich
        cur_pos += 1
        time.sleep(0.1)

def main():
    global ran
    if not ran:
        with console.status('Counting all words in cells...', spinner='material'):
            county()
        with console.status('Cooldown', spinner='clock'):
            time.sleep(5)
    search(beg=cur_pos)
    console.print('Program sucessfully executed!', style='success')
    if review:
        console.print('\nDuring the execution these cells missing word but having definition were found:\n', style='warning')
        print(', '.join(review))
    if not_single:
        console.print('\nDuring the execution these cells containing multiple words were found:\n', style='warning')
        print(', '.join(not_single))

if __name__ == '__main__':
    while True:
        try:
            main()
            break
        except gspread.exceptions.APIError as err:
            console.print('Error on the API side occured!', style='error_api')
            with console.status("Google API's has some restrictions! We're sorry!", spinner='monkey'):
                time.sleep(20)
            # print(err)
        except gspread.exceptions.CellNotFound as err:
            console.print('Cell was not found!', style='error')
            pass
        except gspread.exceptions.WorksheetNotFound as err:
            console.print('Worksheet does not exist or not acessible!', style='error')


import os
from setuptools import setup

with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f:
    readme_rst = f.read()

setup(
    name='logmod',
    url='https://github.com/digitalmensch/logmod',
    download_url='https://pypi.python.org/pypi/logmod',
    description='Log all calls to a module',
    long_description=readme_rst,
    keywords='logging, module, debugging',
    platforms='any',
    license="MIT",
    version='0.1',
    author='Tobias Ammann',
    py_modules=['logmod'],
    install_requires=['structlog'],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3.6'
    ],
)


from pprint import pprint

import numpy as np
import mlebe.training.dataio.torchsample_transforms as ts
# import torchvision.transforms as tv
from torchio.transforms.interpolation import Interpolation

from .imageTransformations import RandomElasticTransform, RandomAffineTransform, RandomNoiseTransform, \
    RandomFlipTransform, RandomBiasFieldTransform, get_normalization, Scale_mlebe

class Transformations:

    def __init__(self, name):
        self.name = name

        # Input patch and scale size
        self.scale_size = (192, 192, 1)
        self.patch_size = (128, 128, 1)

        # Affine and Intensity Transformations
        self.shift_val = (0, 0)
        self.rotate_val = 15.0
        self.inten_val = (1.0, 1.0)
        self.random_flip_prob = 0.0
        self.random_affine_prob = 0.0
        self.random_elastic_prob = 0.0
        self.random_noise_prob = 0.0
        self.bias_field_prob = 0.0
        self.scale_range = [1, 1]
        self.scale_proba = 0

        # Divisibility factor for testing
        self.division_factor = (16, 16, 1)

        # Maximum allowed for elastic transform
        self.max_deform = (7.5, 7.5, 7.5)

    def get_transformation(self):
        return {
            'gsd_pCT': self.get_gsd_pCT_transformer(),
            'mlebe': self.get_gsd_pCT_transformer()
        }[self.name]

    def print(self):
        print('\n\n############# Augmentation Parameters #############')
        pprint(vars(self))
        print('###################################################\n\n')

    def initialise(self, opts, max_output_channels=10):
        t_opts = getattr(opts, self.name)
        self.max_output_channels = max_output_channels

        # Affine and Intensity Transformations
        if hasattr(t_opts, 'scale_size'):       self.scale_size = t_opts.scale_size
        if hasattr(t_opts, 'scale_range'):        self.scale_range = t_opts.scale_range
        if hasattr(t_opts, 'scale_proba'):
            if self.scale_range == [1, 1]:
                self.scale_proba = 0
            else:
                self.scale_proba = t_opts.scale_proba
        if hasattr(t_opts, 'patch_size'):       self.patch_size = t_opts.patch_size
        if hasattr(t_opts, 'shift_val'):        self.shift_val = t_opts.shift_val
        if hasattr(t_opts, 'rotate'):           self.rotate_val = t_opts.rotate
        if hasattr(t_opts, 'max_deform'):       self.max_deform = t_opts.max_deform
        if hasattr(t_opts, 'inten_val'):        self.inten_val = t_opts.intensity
        if hasattr(t_opts, 'random_flip_prob'): self.random_flip_prob = t_opts.random_flip_prob
        if hasattr(t_opts, 'random_affine_prob'): self.random_affine_prob = t_opts.random_affine_prob
        if hasattr(t_opts, 'random_elastic_prob'): self.random_elastic_prob = t_opts.random_elastic_prob
        if hasattr(t_opts, 'bias_field_prob'):  self.bias_field_prob = t_opts.bias_field_prob
        if hasattr(t_opts, 'bias_magnitude_range'):  self.bias_magnitude_range = t_opts.bias_magnitude_range
        if hasattr(t_opts, 'division_factor'):  self.division_factor = t_opts.division_factor
        if hasattr(t_opts, 'random_noise_prob'):  self.random_noise_prob = t_opts.random_noise_prob
        if hasattr(t_opts, 'normalization'):  self.normalization = t_opts.normalization

    def get_gsd_pCT_transformer(self):
        return {'train': self.gsd_pCT_train_transform, 'valid': self.gsd_pCT_valid_transform,
                'bids': self.bids_transform}

    def gsd_pCT_train_transform(self, seed=None):
        if seed is None:
            # seed must be an integer for torch
            seed = np.random.randint(0, 9999)

        train_transform = ts.Compose([
            Scale_mlebe(self.scale_range, self.scale_size, self.scale_proba),
            ts.ToTensor(),
            ts.Pad(size=self.scale_size),
            ts.TypeCast(['float', 'float']),

            RandomFlipTransform(axes=(1), p=self.random_flip_prob, seed=seed,
                                max_output_channels=self.max_output_channels),
            RandomElasticTransform(seed=seed, p=self.random_elastic_prob, image_interpolation=Interpolation.BSPLINE,
                                   max_displacement=self.max_deform,
                                   max_output_channels=self.max_output_channels),

            RandomAffineTransform(scales=[1, 1], degrees=(self.rotate_val), isotropic=True, default_pad_value=0,
                                  image_interpolation=Interpolation.BSPLINE, seed=seed, p=self.random_affine_prob,
                                  max_output_channels=self.max_output_channels),

            RandomNoiseTransform(p=self.random_noise_prob, seed=seed, max_output_channels=self.max_output_channels),

            # somehow applies shift even if "shift": [0,0]
            ts.RandomAffine(translation_range=self.shift_val),

            RandomBiasFieldTransform(p=self.bias_field_prob, is_tensor=True,
                                     bias_magnitude_range=self.bias_magnitude_range),

            ts.ChannelsFirst(),
            # ts.NormalizeMedicPercentile(norm_flag=(True, False)),
            get_normalization(self.normalization),
            # Normalize_mlebe(),
            # ts.NormalizeMedic(norm_flag=(True, False)),
            # Todo fork torchsample and fix the Random Crop bug
            # ts.ChannelsLast(), # seems to be needed for crop
            # ts.RandomCrop(size=self.patch_size),
            ts.TypeCast(['float', 'long'])
        ])

        return train_transform

    def gsd_pCT_valid_transform(self, seed=None):
        valid_transform = ts.Compose([
            Scale_mlebe([0, 0], self.scale_size, 0),
            ts.ToTensor(),
            ts.Pad(size=self.scale_size),
            ts.ChannelsFirst(),
            ts.TypeCast(['float', 'float']),
            # ts.NormalizeMedicPercentile(norm_flag=(True, False)),
            # ts.NormalizeMedic(norm_flag=(True, False)),
            get_normalization(self.normalization),
            # ts.ChannelsLast(),
            # ts.SpecialCrop(size=self.patch_size, crop_type=0),
            ts.TypeCast(['float', 'long'])
        ])
        return valid_transform

    def bids_transform(self):
        bids_transform = ts.Compose([
            Scale_mlebe([0, 0], self.scale_size, 0, bids=True),
            ts.ToTensor(),
            ts.Pad(size=self.scale_size),
            ts.ChannelsFirst(),
            ts.TypeCast(['float', 'float']),
            # ts.NormalizeMedicPercentile(norm_flag=(True, False)),
            # ts.NormalizeMedic(norm_flag=(True, False)),
            get_normalization(self.normalization),
            # ts.ChannelsLast(),
            # ts.SpecialCrop(size=self.patch_size, crop_type=0),
            ts.TypeCast(['float', 'long'])
        ])

        return bids_transform


import torch
import sys
import pathlib
import math

sys.path.append(pathlib.Path(__file__).parent.parent)

from models.alibi_attn import AliBiAttention

import torch
import torch.nn.functional as F
import torch.nn as nn
import math

def test_alibi_biases():
    seq_len = 5
    hid_dim = 4
    n_head = 2
    dropout = 0.0

    alibi_attn = AliBiAttention(hid_dim, n_head, dropout)
    alibi_biases_output = alibi_attn.alibi_biases(seq_len)

    desired_bias = torch.tensor(
        [
            [0, 0, 0, 0, 0],
            [-1, 0, 0, 0, 0],
            [-2, -1, 0, 0, 0],
            [-3, -2, -1, 0, 0],
            [-4, -3, -2, -1, 0],
        ],
        dtype=torch.int64,
    )[None]
    assert alibi_biases_output.shape == (1, seq_len, seq_len)
    assert alibi_biases_output.allclose(desired_bias)

def test_prepare_mask():
    batch_size = 2
    seq_len = 3
    hid_dim = 128
    n_head = 8
    dropout = 0.0

    alibi_attn = AliBiAttention(hid_dim, n_head, dropout)
    mask = torch.ones(batch_size, seq_len)
    prepared_mask = alibi_attn.prepare_mask(mask)

    assert prepared_mask.shape == (batch_size, 1, 1, seq_len)

if __name__ == "__main__":
    test_alibi_biases()
    test_prepare_mask()