# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from datetime import datetime, timedelta
from django.utils import timezone

import requests
from django.db.models import Q, Sum
from django.core.files.base import ContentFile, File
from django.http import JsonResponse, HttpResponse
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from django.template.loader import render_to_string
from django.utils import timezone

from entidades.models import Gauser_extra
from entidades.views import decode_selectgcs
from gauss.funciones import html_to_pdf, pass_generator, usuarios_ronda
from gauss.rutas import RUTA_MEDIA, MEDIA_VUT, RUTA_BASE
from autenticar.control_acceso import permiso_required
from mensajes.models import Aviso
from autenticar.models import Permiso, Gauser
from mensajes.views import encolar_mensaje, crear_aviso
from domotica.models import *
from domotica.mqtt import client
from vut.models import Vivienda, DomoticaVUT
from vut.views import viviendas_autorizado

# @permiso_required('acceso_dispositivos')
def dispositivos(request):
    g_e = request.session['gauser_extra']
    Etiqueta_domotica.objects.get_or_create(entidad=g_e.ronda.entidad, nombre='General')
    gpds = GauserPermitidoDispositivo.objects.filter(dispositivo__entidad=g_e.ronda.entidad,
                                                     gauser=g_e.gauser).values_list('dispositivo__id', flat=True)
    q = Q(id__in=gpds) | Q(propietario=g_e.gauser)
    disps = Dispositivo.objects.filter(Q(entidad=g_e.ronda.entidad), q).distinct()
    etiquetas = Etiqueta_domotica.objects.filter(entidad=g_e.ronda.entidad)
    if request.method == 'POST' and request.is_ajax():
        if request.POST['action'] == 'ver_formulario_crear' and g_e.has_permiso('crea_dispositivos_domotica'):
            try:
                d = Dispositivo(propietario=g_e.gauser, nombre='Nombre del dispositivo', texto='Texto descripción')
                html = render_to_string('dispositivos_fieldset_crear.html', {'g_e': g_e, 'etiquetas': etiquetas,
                                                                             'dispositivo': d})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'crea_dispositivo' and g_e.has_permiso('crea_dispositivos_domotica'):
            try:
                try:
                    etiqueta = Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad,
                                                             id=request.POST['select_etiqueta'])
                except:
                    etiqueta = Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad, nombre='General')
                d = Dispositivo.objects.create(entidad=g_e.ronda.entidad, propietario=g_e.gauser, etiqueta=etiqueta,
                                               plataforma=request.POST['plataforma'], texto=request.POST['texto'],
                                               mqtt_port=request.POST['mqtt_port'], nombre=request.POST['nombre'],
                                               mqtt_id=request.POST['mqtt_id'], mqtt_topic=request.POST['mqtt_topic'],
                                               ifttt=request.POST['ifttt'], mqtt_broker=request.POST['mqtt_broker'],
                                               tipo=request.POST['tipo'])
                html = render_to_string('dispositivos_table_tr.html', {'disps': [d], 'g_e': g_e})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'ver_formulario_crear_etiqueta' and g_e.has_permiso('crea_carpetas'):
            try:
                html = render_to_string("dispositivos_fieldset_etiqueta.html", {'etiquetas': etiquetas})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'ver_formulario_editar_carpeta' and g_e.has_permiso('edita_carpetas'):
            try:
                e = Etiqueta_domotica.objects.get(id=request.POST['etiqueta'])
                html = render_to_string("dispositivos_fieldset_etiqueta_editar.html",
                                        {'etiquetas': etiquetas, 'etiqueta': e})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'modifica_etiqueta' and g_e.has_permiso('edita_carpetas'):
            try:
                nombre = request.POST['nombre']
                try:
                    Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad, nombre__iexact=nombre)
                    return JsonResponse({'ok': False, 'mensaje': 'Ya existe una etiqueta/carpeta con ese nombre.'})
                except:
                    e = Etiqueta_domotica.objects.get(id=request.POST['etiqueta'])
                    e.nombre = nombre
                    try:
                        e.padre = Etiqueta_domotica.objects.get(id=request.POST['padre'])
                    except:
                        e.padre = None
                    e.save()
                    html = render_to_string('dispositivos_table_tr.html', {'disps': disps, 'g_e': g_e})
                    return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'ver_formulario_buscar':
            try:
                html = render_to_string("dispositivos_fieldset_buscar.html", {'etiquetas': etiquetas, 'g_e': g_e})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'crea_etiqueta' and g_e.has_permiso('crea_carpetas'):
            try:
                nombre = request.POST['nombre']
                try:
                    Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad, nombre__iexact=nombre)
                    return JsonResponse({'ok': False, 'mensaje': 'Ya existe una etiqueta/carpeta con ese nombre.'})
                except:
                    if request.POST['padre']:
                        padre = Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad, id=request.POST['padre'])
                        Etiqueta_domotica.objects.create(entidad=g_e.ronda.entidad, padre=padre, nombre=nombre)
                    else:
                        Etiqueta_domotica.objects.create(entidad=g_e.ronda.entidad, nombre=nombre)
                    return JsonResponse({'ok': True})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'borra_etiqueta' and g_e.has_permiso('borra_cualquier_carpeta'):
            try:
                Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad, id=request.POST['etiqueta']).delete()
                html = render_to_string('dispositivos_table_tr.html', {'disps': disps, 'g_e': g_e})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'busca_disps_manual':
            try:
                try:
                    inicio = datetime.strptime(request.POST['inicio'], '%Y-%m-%d').date()
                except:
                    inicio = datetime.strptime('1900-1-1', '%Y-%m-%d').date()
                try:
                    fin = datetime.strptime(request.POST['fin'], '%Y-%m-%d').date()
                except:
                    fin = datetime.now().date()
                texto = request.POST['texto']
                try:
                    etiqueta = Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad, id=request.POST['etiqueta'])
                except:
                    etiqueta = None
                if etiqueta:
                    q = Q(propietario__ronda__entidad=g_e.ronda.entidad) & Q(
                        creado__gte=inicio) & Q(creado__lte=fin) & Q(
                        nombre__icontains=texto) & Q(etiqueta__in=etiqueta.hijos)
                else:
                    q = Q(propietario__ronda__entidad=g_e.ronda.entidad) & Q(
                        creado__gte=inicio) & Q(creado__lte=fin) & Q(
                        nombre__icontains=texto)

                disps_search = disps.filter(q)
                html = render_to_string('dispositivos_table_tr.html', {'disps': disps_search, 'g_e': g_e})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'ver_formulario_editar':
            try:
                disp = disps.get(id=request.POST['disp'])
                if g_e.has_permiso('edita_todos_archivos') or disp.permiso_w(g_e.gauser) or disp.permiso_x(g_e.gauser):
                    hoy = datetime.now().date()
                    subentidades = Subentidad.objects.filter(entidad=g_e.ronda.entidad, fecha_expira__gte=hoy)
                    cargos = Cargo.objects.filter(entidad=g_e.ronda.entidad)
                    d = {'g_e': g_e, 'cargos': cargos, 'subentidades': subentidades, 'etiquetas': etiquetas, 'd': disp}
                    html = render_to_string("dispositivos_table_tr_device_edit.html", d)
                    return JsonResponse({'ok': True, 'html': html})
                else:
                    return JsonResponse({'ok': False, 'mensaje': 'No tienes los permisos necesarios.'})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'update_archivo':
            try:
                disp = disps.get(id=request.POST['disp'])
                if g_e.has_permiso('edita_todos_archivos') or disp.permiso_w(g_e.gauser) or disp.permiso_x(g_e.gauser):
                    disp.nombre = request.POST['nombre']
                    disp.etiqueta = Etiqueta_domotica.objects.get(id=request.POST['etiqueta'])
                    disp.cargos.clear()
                    disp.acceden.clear()
                    cargos = Cargo.objects.filter(entidad=g_e.ronda.entidad, id__in=request.POST.getlist('cargos[]'))
                    subs = Subentidad.objects.filter(entidad=g_e.ronda.entidad, id__in=request.POST.getlist('subs[]'))
                    disp.cargos.add(*cargos)
                    disp.acceden.add(*subs)
                    disp.save()
                    html = render_to_string('dispositivos_table_tr_device.html', {'d': disp, 'g_e': g_e})
                    return JsonResponse({'ok': True, 'html': html})
                else:
                    return JsonResponse({'ok': False, 'mensaje': 'No tienes los permisos necesarios.'})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'borrar_dispositivo':
            disp = disps.get(id=request.POST['disp'])
            if disp.permiso_x(g_e.gauser) or disp.propietario == g_e or g_e.has_permiso('borra_cualquier_archivo'):
                GauserPermitidoDispositivo.objects.filter(dispositivo=disp, gauser=g_e.gauser).delete()
                otros = GauserPermitidoDispositivo.objects.filter(dispositivo=disp).count()
                if otros == 0:
                    disp.delete()
                return JsonResponse(
                    {'ok': True, 'mensaje': 'Dispositivo borrado. Ahora pueden verlo %s personas' % otros})
            else:
                return JsonResponse({'ok': False, 'mensaje': 'No ha sido posible borrar el dispositivo.'})
        elif request.POST['action'] == 'borrar_disp_completamente':
            disp = disps.get(id=request.POST['disp'])
            if g_e.has_permiso('borra_cualquier_archivo'):
                todos = GauserPermitidoDispositivo.objects.filter(dispositivo=disp)
                todos.delete()
                disp.delete()
                return JsonResponse({'ok': True, 'mensaje': 'Dispositivo borrado por completo.'})
            else:
                return JsonResponse({'ok': False, 'mensaje': 'No ha sido posible borrar el dispositivo.'})
        else:
            return JsonResponse({'ok': False, 'mensaje': 'No tiene los permisos necesarios.'})

    elif request.method == 'POST':
        if request.POST['action'] == 'crea_dispositivo43254432':
            n_files = int(request.POST['n_files'])
            if g_e.has_permiso('crea_dispositivos_domotica'):
                try:
                    for i in range(n_files):
                        fichero = request.FILES['fichero_xhr' + str(i)]
                        try:
                            etiqueta = Etiqueta_domotica.objects.get(entidad=g_e.ronda.entidad,
                                                                     id=request.POST['etiqueta'])
                        except:
                            etiqueta, c = Etiqueta_domotica.objects.get_or_create(entidad=g_e.ronda.entidad,
                                                                                nombre='General')
                        disp = Dispositivo.objects.create(propietario=g_e, content_type=fichero.content_type,
                                                          etiqueta=etiqueta, nombre=fichero.name, fichero=fichero)
                        GauserPermitidoDispositivo.objects.create(gauser=g_e.gauser, dispositivo=disp, permiso='x')
                        html = render_to_string('dispositivos_table_tr.html', {'disps': [disp], 'g_e': g_e})
                        return JsonResponse({'ok': True, 'html': html, 'mensaje': False})
                except:
                    return JsonResponse({'ok': False, 'mensaje': 'Se ha producido un error.'})
            else:
                mensaje = 'No tienes permiso para cargar programaciones.'
                return JsonResponse({'ok': False, 'mensaje': mensaje})

        elif request.POST['action'] == 'descargar_disp':
            try:
                d = disps.get(id=request.POST['dispositivo'])
                fichero = d.fichero.read()
                response = HttpResponse(fichero, content_type=d.content_type)
                response['Content-Disposition'] = 'attachment; filename=' + d.fich_name
                return response
            except:
                crear_aviso(request, False, 'Error. No se ha podido descargar el archivo.')

    return render(request, "dispositivos.html",
                  {
                      'iconos':
                          ({'tipo': 'button', 'nombre': 'plus', 'texto': 'Añadir',
                            'permiso': 'crea_dispositivos_domotica', 'title': 'Anadir un nuevo dispositivo'},
                           {'tipo': 'button', 'nombre': 'folder', 'texto': 'Nueva', 'permiso': 'crea_carpetas',
                            'title': 'Crear una nueva carpeta/etiqueta'},
                           {'tipo': 'button', 'nombre': 'search', 'texto': 'Buscar/Filtrar',
                            'permiso': 'libre',
                            'title': 'Busca/Filtra resultados entre los diferentes archivos'},
                           ),
                      'g_e': g_e,
                      'etiquetas': etiquetas,
                      'disps': disps,
                      'carpetas': Etiqueta_domotica.objects.filter(entidad=g_e.ronda.entidad),
                      'formname': 'dispositivos',
                      'carpeta_id': 'todas',  # Este identificador implica leer todas las carpetas
                      'avisos': Aviso.objects.filter(usuario=g_e, aceptado=False),
                  })

@permiso_required('acceso_grupos_domotica')
def grupos_domotica(request):
    g_e = request.session['gauser_extra']
    grupos = Grupo.objects.filter(propietario=g_e.gauser)
    return render(request, "grupos_domotica.html",
                  {
                      'formname': 'grupos_domotica',
                      'iconos':
                          ({'tipo': 'button', 'nombre': 'plus', 'texto': 'Nuevo grupo',
                            'permiso': 'crea_grupos_domotica', 'title': 'Crear un nuevo grupo para domótica'},
                           ),
                      'avisos': Aviso.objects.filter(usuario=g_e, aceptado=False),
                      'grupos': grupos,
                  })

@login_required()
def ajax_grupos_domotica(request):
    g_e = request.session["gauser_extra"]
    if request.is_ajax():
        if request.method == 'POST':
            if request.POST['action'] == 'add_grupo' and g_e.has_permiso('crea_grupos_domotica'):
                try:
                    grupo = Grupo.objects.create(nombre='Nuevo grupo/ubicación', propietario=g_e.gauser)
                    html = render_to_string('grupos_domotica_accordion.html', {'grupos': [grupo]})
                    return JsonResponse({'html': html, 'ok': True})
                except:
                    return JsonResponse({'ok': False})
            elif request.POST['action'] == 'open_accordion':
                try:
                    grupos_id = GauserPermitidoGrupo.objects.filter(gauser=g_e.gauser).values_list('grupo', flat=True)
                    grupos = Grupo.objects.filter(id__in=grupos_id)
                    grupo = grupos.get(id=request.POST['grupo'])
                    html = render_to_string('grupos_domotica_accordion_content.html', {'grupo': grupo, 'g_e': g_e})
                    return JsonResponse({'ok': True, 'html': html})
                except:
                    return JsonResponse({'ok': False})
            elif request.POST['action'] == 'delete_grupo':
                try:
                    grupo = Grupo.objects.get(id=request.POST['grupo'])
                    if grupo.propietario == g_e.gauser or g_e.has_permiso('borra_grupos_domotica'):
                        grupo.delete()
                        return JsonResponse({'ok': True, 'mensaje': "El grupo se ha borrado sin incidencias."})
                    else:
                        return JsonResponse({'ok': False, 'mensaje': "Error al tratar de borrar el grupo."})
                except:
                    return JsonResponse({'ok': False, 'mensaje': "Error al tratar de borrar el grupo."})

            elif request.POST['action'] == 'update_campo':
                try:
                    grupo = Grupo.objects.get(id=request.POST['grupo'])
                    if grupo.propietario == g_e.gauser or g_e.has_permiso('edita_grupos_domotica'):
                        campo = request.POST['campo']
                        valor = request.POST['valor']
                        setattr(grupo, campo, valor)
                        grupo.save()
                        return JsonResponse({'ok': True, 'campo': campo, 'valor': valor})
                    else:
                        return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el grupo."})
                except:
                    return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el grupo."})
            elif request.POST['action'] == 'update_select':
                try:
                    grupo = Grupo.objects.get(id=request.POST['grupo'])
                    grupo_padre = Grupo.objects.get(id=request.POST['grupo_padre'])
                    q = grupo.propietario == g_e.gauser and grupo_padre.propietario == g_e.gauser
                    if q or g_e.has_permiso('edita_grupos_domotica'):
                        grupo.grupo_padre = grupo_padre
                        grupo.save()
                        return JsonResponse({'ok': True})
                    else:
                        return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el grupo."})
                except:
                    return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el grupo."})
            elif request.POST['action'] == 'update_usuarios_autorizados':
                try:
                    grupo = Grupo.objects.get(id=request.POST['grupo'])
                    operacion = request.POST['operacion']
                    if operacion == 'add':
                        autorizado = Gauser_extra.objects.get(id=request.POST['autorizado'][1:]).gauser
                    elif operacion == 'delete' or operacion == 'change_permiso':
                        gpg = GauserPermitidoGrupo.objects.get(id=request.POST['autorizado'])
                        autorizado = gpg.gauser
                    else:
                        autorizado = None
                    # Condiciones con las que tendría permiso para editar el grupo y añadir autorizados:
                    c1 = grupo.gauserpermitidogrupo_set.filter(gauser=g_e.gauser, permiso__icontains='VUE').count() > 0
                    c2 = g_e.has_permiso('edita_grupos_domotica')
                    c3 = grupo.propietario == g_e.gauser
                    if (c1 or c2 or c3) and request.POST['operacion'] == 'add':
                        GauserPermitidoGrupo.objects.get_or_create(gauser=autorizado, grupo=grupo)
                    elif (c1 or c2 or c3) and request.POST['operacion'] == 'delete':
                        gpg.delete()
                    elif (c1 or c2 or c3) and request.POST['operacion'] == 'change_permiso':
                        gpg.permiso = request.POST['permiso']
                        gpg.save()
                    html = render_to_string('grupos_domotica_accordion_content_autorizados.html', {'grupo': grupo})
                    return JsonResponse({'ok': True, 'html': html, 'permiso': c1 or c2 or c3})
                except:
                    return JsonResponse({'ok': False})

# @permiso_required('acceso_configura_domotica')
def configura_domotica(request):
    g_e = request.session['gauser_extra']
    # gauser_permitido = GauserPermitido.objects.filter(gauser=g_e.gauser)
    # dispositivos_permitidos = gauser_permitido.values_list('dispositivo__id', flat=True)
    # secuencias_permitidas = gauser_permitido.values_list('secuencia__id', flat=True)
    # conjuntos_permitidos = gauser_permitido.values_list('conjunto__id', flat=True)
    # dispositivos = Dispositivo.objects.filter(Q(propietario=g_e.gauser) | Q(id__in=dispositivos_permitidos))
    # secuencias = Secuencia.objects.filter(Q(propietario=g_e.gauser) | Q(id__in=secuencias_permitidas))
    # conjuntos = Conjunto.objects.filter(Q(propietario=g_e.gauser) | Q(id__in=conjuntos_permitidos))

    # if request.method == 'POST':
    #     action = request.POST['action']
    #     if action == 'libro_contabilidad_vut':
    #         permiso = Permiso.objects.get(code_nombre='genera_libro_registro_policia')
    #         grupo = Grupo.objects.get(id=request.POST['id_grupo'])
    #         if has_permiso_on_grupo(g_e, grupo, permiso):
    #             fecha_anterior_limite = datetime.today().date() - timedelta(1100)
    #             viajeros = Viajero.objects.filter(reserva__grupo=grupo,
    #                                               reserva__entrada__gte=fecha_anterior_limite)
    #             c = render_to_string('libro_registro_policia.html',
    #                                  {'grupo': grupo, 'viajeros': viajeros, 'ruta_base': RUTA_BASE})
    #             ruta = '%s%s/%s/' % (MEDIA_VUT, grupo.gpropietario.id, grupo.id)
    #             fich = html_to_pdf(request, c, fichero='libro_registros', media=ruta,
    #                                title=u'Libro de registro de viajeros', tipo='sin_cabecera')
    #             response = HttpResponse(fich, content_type='application/pdf')
    #             response['Content-Disposition'] = 'attachment; filename=Libro_registro_viajeros.pdf'
    #             return response
    #     elif action == 'download_file_asiento':
    #         asiento = AsientoVUT.objects.get(id=request.POST['asiento'])
    #         permiso = Permiso.objects.get(code_nombre='edita_asiento_vut')
    #         a = AutorizadoContabilidadVut.objects.filter(contabilidad=asiento.partida.contabilidad,
    #                                                      autorizado=g_e.gauser, permisos__in=[permiso]).count()
    #         if asiento.partida.contabilidad.propietario == g_e.gauser or a > 0:
    #             fich = open(RUTA_BASE + asiento.fichero.url)
    #             response = HttpResponse(fich, content_type=asiento.content_type)
    #             response['Content-Disposition'] = 'attachment; filename=' + asiento.fich_name
    #             return response

    grupos_id = GauserPermitidoGrupo.objects.filter(gauser=g_e.gauser).values_list('grupo', flat=True)
    return render(request, "domotica.html",
                  {
                      'formname': 'domotica',
                      # 'iconos':
                      #     ({'tipo': 'button', 'nombre': 'plus', 'permiso': 'crea_dispositivos_domotica',
                      #       'texto': 'Nuevo dispositivo', 'title': 'Crear un nuevo dispositivo domótico'},
                      #      ),
                      'grupos': Grupo.objects.filter(id__in=grupos_id),
                      'configuraciones_enlace': EnlaceDomotica.objects.filter(propietario=g_e.gauser),
                      'avisos': Aviso.objects.filter(usuario=g_e, aceptado=False)
                  })

def pulsador_domotico(d):
    if d.plataforma == 'IFTTT':
        try:
            s = requests.Session()
            s.verify = False
            p = s.post(d.ifttt, timeout=5)
            return {'ok': True, 'response': p.status_code}
        except:
            return {'ok': False}
    elif d.plataforma == 'ESPURNA':
        if d.tipo == 'SELFLOCKING':
            topic = '{0}/relay/0/set'.format(d.mqtt_topic)
            client.publish(topic, 1)
        elif d.tipo == 'ONOFF':
            topic = '{0}/relay/0/set'.format(d.mqtt_topic)
            client.publish(topic, 2)
        else:
            return {'ok': False, 'mensaje': 'No detecta tipo'}
        # client.disconnect()
        return {'ok': True}
    else:
        return {'ok': False, 'mensaje': 'No detecta plataforma'}

@login_required()
def ajax_configura_domotica(request):
    g_e = request.session["gauser_extra"]
    if request.is_ajax():
        if request.POST['action'] == 'open_accordion':
            try:
                grupos_id = GauserPermitidoGrupo.objects.filter(gauser=g_e.gauser).values_list('grupo', flat=True)
                grupos = Grupo.objects.filter(id__in=grupos_id)
                grupo = grupos.get(id=request.POST['grupo'])
                gdispositivos = GauserPermitidoDispositivo.objects.filter(dispositivo__grupo=grupo, gauser=g_e.gauser)
                if g_e.gauser.username == 'gauss':
                    viviendas_posibles = Vivienda.objects.filter(borrada=False)
                else:
                    viviendas_posibles = viviendas_autorizado(g_e)
                html = render_to_string('domotica_accordion_content.html',
                                        {'gdispositivos': gdispositivos, 'grupo': grupo, 'g_e': g_e,
                                         'grupos': grupos, 'viviendas_posibles': viviendas_posibles})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'add_dispositivo_domotica':
            try:
                if g_e.has_permiso('crea_dispositivos_domotica'):
                    grupos_id = GauserPermitidoGrupo.objects.filter(gauser=g_e.gauser).values_list('grupo', flat=True)
                    grupos = Grupo.objects.filter(id__in=grupos_id)
                    grupo = grupos.get(id=request.POST['grupo'])
                    d = Dispositivo.objects.create(propietario=g_e.gauser, nombre='Nombre del dispositivo', grupo=grupo)
                    gdispositivo = GauserPermitidoDispositivo.objects.get(gauser=g_e.gauser, dispositivo=d)
                    html = render_to_string('domotica_accordion_content_dispositivo.html',
                                            {'g_e': g_e, 'gdispositivo': gdispositivo, 'grupos': grupos})
                    return JsonResponse({'ok': True, 'html': html})
                else:
                    return JsonResponse({'ok': False})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'delete_dispositivo_domotica':
            try:
                d = Dispositivo.objects.get(id=request.POST['dispositivo'])
                if d.permiso(g_e.gauser) == 'BORRAR' or g_e.has_permiso('borra_dispositivos_domotica'):
                    d.delete()
                    return JsonResponse({'ok': True})
                else:
                    return JsonResponse({'ok': False})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'update_campo':
            try:
                d = Dispositivo.objects.get(id=request.POST['dispositivo'])
                if 'E' in d.permiso(g_e.gauser) or g_e.has_permiso('edita_dispositivos_domotica'):
                    campo = request.POST['campo']
                    valor = request.POST['valor']
                    setattr(d, campo, valor)
                    d.save()
                    return JsonResponse({'ok': True, 'campo': campo, 'valor': valor})
                else:
                    return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
            except:
                return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
        elif request.POST['action'] == 'update_plataforma':
            try:
                d = Dispositivo.objects.get(id=request.POST['dispositivo'])
                if 'E' in d.permiso(g_e.gauser) or g_e.has_permiso('edita_dispositivos_domotica'):
                    d.plataforma = request.POST['valor']
                    d.save()
                    return JsonResponse({'ok': True, 'campo': 'plataforma', 'valor': request.POST['valor']})
                else:
                    return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
            except:
                return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
        elif request.POST['action'] == 'update_tipo_dispositivo':
            try:
                d = Dispositivo.objects.get(id=request.POST['dispositivo'])
                if 'E' in d.permiso(g_e.gauser) or g_e.has_permiso('edita_dispositivos_domotica'):
                    d.tipo = request.POST['valor']
                    d.save()
                    return JsonResponse({'ok': True, 'campo': 'tipo', 'valor': request.POST['valor']})
                else:
                    return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
            except:
                return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
        elif request.POST['action'] == 'update_grupo_dispositivo':
            try:
                d = Dispositivo.objects.get(id=request.POST['dispositivo'])
                if 'E' in d.permiso(g_e.gauser) or g_e.has_permiso('edita_dispositivos_domotica'):
                    grupo = Grupo.objects.get(id=request.POST['valor'])
                    d.grupo = grupo
                    d.save()
                    return JsonResponse({'ok': True, 'valor': request.POST['valor']})
                else:
                    return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
            except:
                return JsonResponse({'ok': False, 'mensaje': "Error al tratar de editar el dispositivo."})
        elif request.POST['action'] == 'pulsador_domotico':
            d = Dispositivo.objects.get(id=request.POST['dispositivo'])
            respuesta = pulsador_domotico(d)
            return JsonResponse(respuesta)
            if d.plataforma == 'IFTTT':
                try:
                    s = requests.Session()
                    s.verify = False
                    p = s.post(d.ifttt, timeout=5)
                    return JsonResponse({'ok': True, 'response': p.status_code})
                except:
                    return JsonResponse({'ok': False})
            elif d.plataforma == 'ESPURNA':
                if d.tipo == 'SELFLOCKING':
                    topic = '{0}/relay/0/set'.format(d.mqtt_topic)
                    client.publish(topic, 1)
                elif d.tipo == 'ONOFF':
                    topic = '{0}/relay/0/set'.format(d.mqtt_topic)
                    client.publish(topic, 2)
                else:
                    return JsonResponse({'ok': False, 'mensaje': 'No detecta tipo'})
                # client.disconnect()
                return JsonResponse({'ok': True})
            else:
                return JsonResponse({'ok': False, 'mensaje': 'No detecta plataforma'})
        elif request.POST['action'] == 'copiar_dispositivo':
            try:
                if g_e.gauser.username == 'gauss':
                    viviendas = Vivienda.objects.filter(borrada=False)
                else:
                    viviendas = viviendas_autorizado(g_e)
                vivienda = viviendas.get(id=request.POST['vivienda'])
                dispositivo = Dispositivo.objects.get(id=request.POST['id'])
                dv, c = DomoticaVUT.objects.get_or_create(vivienda=vivienda, dispositivo=dispositivo)
                dv.propietario = g_e.gauser
                dv.url = dispositivo.ifttt
                dv.nombre = dispositivo.nombre
                dv.texto = dispositivo.texto
                dv.tipo = dispositivo.tipo
                dv.save()
                return JsonResponse({'ok': True})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'add_enlace_domotica':
            enlace = EnlaceDomotica.objects.create(propietario=g_e.gauser, nombre='', valido_desde=timezone.now(),
                                                   valido_hasta=timezone.now())
            html = render_to_string('enlace_accordion.html', {'enlace': enlace})
            return JsonResponse({'ok': True, 'html': html})
        elif request.POST['action'] == 'open_accordion_enlace':
            try:
                enlace = EnlaceDomotica.objects.get(id=request.POST['enlace'], propietario=g_e.gauser)
                gdispositivos = GauserPermitidoDispositivo.objects.filter(gauser=g_e.gauser).order_by(
                    'dispositivo__grupo')
                html = render_to_string('enlace_accordion_content.html',
                                        {'enlace': enlace, 'gdispositivos': gdispositivos})
                return JsonResponse({'ok': True, 'html': html})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'enlace_dispositivos':
            try:
                enlace = EnlaceDomotica.objects.get(id=request.POST['id'], propietario=g_e.gauser)
                enlace.dispositivos.clear()
                enlace.dispositivos.add(*request.POST.getlist('valor[]'))
                return JsonResponse({'ok': True})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'enlace_nombre':
            try:
                enlace = EnlaceDomotica.objects.get(id=request.POST['id'], propietario=g_e.gauser)
                enlace.nombre = request.POST['valor']
                enlace.save()
                return JsonResponse({'ok': True, 'nombre': enlace.nombre})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'enlace_valido_hasta':
            try:
                enlace = EnlaceDomotica.objects.get(id=request.POST['id'], propietario=g_e.gauser)
                enlace.valido_hasta = timezone.make_aware(datetime.strptime(request.POST['valor'], '%H:%M %d-%m-%Y'))
                enlace.save()
                return JsonResponse({'ok': True})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'enlace_valido_desde':
            try:
                enlace = EnlaceDomotica.objects.get(id=request.POST['id'], propietario=g_e.gauser)
                enlace.valido_desde = timezone.make_aware(datetime.strptime(request.POST['valor'], '%H:%M %d-%m-%Y'))
                enlace.save()
                return JsonResponse({'ok': True})
            except:
                return JsonResponse({'ok': False})
        elif request.POST['action'] == 'del_enlace':
            try:
                enlace = EnlaceDomotica.objects.get(id=request.POST['id'], propietario=g_e.gauser)
                enlace.delete()
                return JsonResponse({'ok': True})
            except:
                return JsonResponse({'ok': False})

def lnk(request):
    ahora = timezone.make_aware(datetime.today())
    if request.method == 'GET':
        try:
            secret = request.GET['s']
            enlace = EnlaceDomotica.objects.get(secret=secret)
            if enlace.valido_desde > ahora:
                return render(request, "enlace_domotica_error.html", {'tipo': 'temprano'})
            elif enlace.valido_hasta < ahora:
                return render(request, "enlace_domotica_error.html", {'tipo': 'pasado'})
            else:
                return render(request, "enlace_domotica.html",
                              {
                                  'formname': 'enlace_domotica',
                                  'enlace': enlace
                              })
        except:
            return render(request, "enlace_domotica_error.html", {'tipo': 'noexiste'})
    elif request.method == 'POST' and request.is_ajax():
        if request.POST['action'] == 'boton_domotico':
            try:
                enlace = EnlaceDomotica.objects.get(secret=request.POST['secret'])
                dispositivo = Dispositivo.objects.get(id=request.POST['id'])
                if dispositivo in enlace.dispositivos.all():
                    respuesta = pulsador_domotico(dispositivo)
                    return JsonResponse(respuesta)
                else:
                    return JsonResponse({'ok': False, 'mensaje': 'Error dispositivo'})
            except:
                return JsonResponse({'ok': False, 'mensaje': 'Error getting object'})

    return JsonResponse({'ok': False})


from sklearn.tree import DecisionTreeRegressor
import pandas as pd
import pickle
from datetime import datetime
from sklearn.model_selection import train_test_split
import csv
import numpy as np

N = 1000
modelFile = 'XTX_Save_Model//trained_model' + str(N) + '.sav'
splitSize = 0.7
X = []
y = []
train_x = []
test_x = []
train_y = []
test_y = []

def print_TXT(timespend1, modelscore1):
    global N, splitSize

    printText = "Desicion Tree Model SkLearn \n " \
                "Time spend: " + str(timespend1) + "\n " \
                   "Model Score: " + str(modelscore1) + "\n " \
                     "Split Size: " + str(splitSize) + "\n " \
                        "N: " + str(N) + "\n "

    filepath = open("XTX_DT_REG_Results//Score" + str(N) + ".txt", "w+")
    filepath.write(printText)
    filepath.close()

def load_data():
    global train_x, test_x, train_y, test_y, X, y, N, splitSize

    filepath = "Data//XTXData{}K.csv".format(str(N/1000))

    with open(filepath, 'r') as f:
        global dataset, headers
        reader = csv.reader(f, delimiter=',')
        headers = next(reader)
        dataset = list(reader)
        dataset = np.array(dataset).astype(float)
        dataset = np.nan_to_num(dataset)

    print("Data Loaded")
    print(datetime.now() - startTime)

    y = dataset.iloc[:, 60].astype(float)
    X = dataset.iloc[:, 0:60].astype(float)
    print(X.shape)
    print(y.shape)
    print("")

    (train_x, test_x, train_y, test_y) = \
        train_test_split(X, y, train_size=splitSize, random_state=1)

    print("Data Splited")
    print(datetime.now() - startTime)
    print("Train Data")
    print(train_y.shape)
    print(train_x.shape)
    print("Test Data")
    print(test_y.shape)
    print(test_x.shape)
    print("")

def regmodel():
    global test_x, test_y, X, y
    # create a regressor object
    regressor = DecisionTreeRegressor(random_state=0, min_samples_leaf=3)
    print("Model created")
    print(datetime.now() - startTime)
    print("")

    # fit the regressor with X and Y data

    regressor.fit(train_x, train_y)
    # regressor.fit(X, y)

    print("Trained")
    print(datetime.now() - startTime)
    print("")

    ModelScore = regressor.score(test_x, test_y)
    print("Accuracy: ", ModelScore)
    # print("Accuracy: ", regressor.score(X, y))
    timeSpend = datetime.now() - startTime

    #Saves model on .sav file
    pickle.dump(regressor, open(modelFile, 'wb'))

    print_TXT(timeSpend, ModelScore)

    print("")

def numpy_predictor():
    global modelFile
    load_Regresor = pickle.load(open(modelFile, 'rb'))

    N = 5000
    filepath = "Data//XTXData{}K.csv".format(str(N/1000))
    dataset = pd.read_csv(filepath)
    dataset.fillna(0, inplace=True)

    y = dataset.iloc[:, 60].astype(float)
    X = dataset.iloc[:, 0:60].astype(float)

    X_Np = X.to_numpy()
    #X_Np = X.as_matrix()

    print("Data Loaded")
    print(datetime.now() - startTime)
    print("")

    sumA = 0
    sumB = 0

    for i in range(0, N):
        if (i % 100000) == 0:
            print("Checking " + str(i))
            print(datetime.now() - startTime)
            print("")

        newX = [X_Np[i, :]]
        realY = y.iloc[i]

        newY = load_Regresor.predict(newX)
        newY = newY[0]

        sumA = sumA + (realY - newY) ** 2
        sumB = sumB + realY ** 2

    modelScore = 1 - (sumA / sumB)

    print("Model Score \n is: " + str(modelScore))
    print(datetime.now() - startTime)
    print("")

    timeSpend = datetime.now() - startTime
    print_TXT(timeSpend, modelScore)

def main_predictor():
    global modelFile
    load_Regresor = pickle.load(open(modelFile, 'rb'))

    N = 1000000
    filepath = "Data//XTXData{}K.csv".format(str(N/1000))
    dataset = pd.read_csv(filepath)
    dataset.fillna(0, inplace=True)

    y = dataset.iloc[:, 60].astype(float)
    X = dataset.iloc[:, 0:60].astype(float)

    print("Data Loaded")
    print(datetime.now() - startTime)
    print("")

    sumA = 0
    sumB = 0

    for i in range(0, N):
        if (i % 100000) == 0:
            print("Checking "+str(i))
            print(datetime.now() - startTime)
            print("")

        newX = [X.iloc[i, :]]
        realY = y.iloc[i]

        newY = load_Regresor.predict(newX)
        newY = newY[0]

        sumA = sumA + (realY - newY) ** 2
        sumB = sumB + realY ** 2

    modelScore = 1 - (sumA / sumB)

    print("Model Score \n is: " + str(modelScore))
    print(datetime.now() - startTime)
    print("")

    timeSpend = datetime.now() - startTime
    print_TXT(timeSpend, modelScore)

def plot_tree():
    from sklearn.tree import export_graphviz
    load_regresor = pickle.load(open(modelFile, 'rb'))
    print("Model loaded")
    print(datetime.now() - startTime)
    print("")

    export_graphviz(load_regresor, out_file='tree_XTX_Model.dot')
    print("Model plotted")
    print(datetime.now() - startTime)
    print("")

def main_train():
    load_data()
    regmodel()
    numpy_predictor()

startTime = datetime.now()
print("Script Start")

load_data()
#main_train()
#plot_tree()
#main_predictor()
#numpy_predictor()

print(datetime.now() - startTime)


#此代码无法正常使用

import httpx,os
from io import BytesIO
from PIL import Image

from more_action import send_msg
from action_sql import plugins_sql, qu_key
def get(text):
    req=httpx.post("https://api.xingzhige.com/API/Qrcode/",data={"text":text})
    print(req.content)
    image=Image.open(BytesIO(req.content))
    fileName="qrcode_cache"+'.'+image.format.lower()
    cache=os.path.join(os.getcwd(),"cache",fileName)
    with open(cache,'wb') as f:
        f.write(req.content)
    return cache

def main(l):
    text=l["qu"][8:]
    wxid=l["wxid"]
    cache=get(text)
    send_msg(wxid,cache,3)

if __name__=="__main__":
    plugins_sql.inf("qrcode",0.01,"zwx08","生成qrcode")
    qu_key.write("qrcode","&qrcode",1,"{plugin}.main",1)


import pandas as pd
import streamlit as st
import pandas as pd
import io
import numpy as np
import zipfile
import base64
import os
from pandas_profiling import ProfileReport
from streamlit_pandas_profiling import st_profile_report
import neattext as nt
from autocorrect import Speller
import spacy
en_core = spacy.load('en_core_web_sm')
spell = Speller(lang='en')
import plotly.express as px
from PIL import Image
from textblob import TextBlob, Word
from textblob import TextBlob
from streamlit_option_menu import option_menu
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import plotly.graph_objects as go
import seaborn as sns
import matplotlib.pyplot as plt
from io import StringIO
from wordcloud import WordCloud
from matplotlib import colors
from nltk.corpus import stopwords
color_list=['#4d4d4d','#ffcc00','#000000',]
colormap=colors.ListedColormap(color_list)
import matplotlib.pyplot as plt

def main_page():
    st.markdown("# Main page 🎈")
    st.sidebar.markdown("# Main page 🎈")

def page2():
    st.markdown("# Filter 🎈")
    st.sidebar.markdown("# Filter Text 🎈")

    multiple_files = st.file_uploader(" ",type="csv", accept_multiple_files=True)
    for file in multiple_files:
        df = pd.read_csv(file)
        file.seek(0)

        gb = GridOptionsBuilder.from_dataframe(df)
        gb.configure_pagination(paginationAutoPageSize=True) #Add pagination
        gb.configure_side_bar() #Add a sidebar
        gb.configure_selection('multiple') #Enable multi-row selection
        gridOptions = gb.build()

        grid_response = AgGrid(
            df,
            gridOptions=gridOptions,
            data_return_mode='AS_INPUT',
            update_mode='MODEL_CHANGED',
            fit_columns_on_grid_load=False,
            theme='blue', #Add theme color to the table
            enable_enterprise_modules=True,
            height=600,
            width='100%',
            reload_data=True
        )

        data = grid_response['data']
        selected = grid_response['selected_rows']
        df = pd.DataFrame(selected) #Pass the selected rows to a new dataframe df

        ###################################################################

def page3():
    st.markdown("# Merge  ❄")
    st.sidebar.markdown("# Merge Files ❄")

    # Excel file merge function
    def excel_file_merge(zip_file_name):
        df = pd.DataFrame()
        archive = zipfile.ZipFile(zip_file_name, 'r')
        with zipfile.ZipFile(zip_file_name, "r") as f:
            for file in f.namelist():
              xlfile = archive.open(file)
              if file.endswith('.xlsx'):
                # Add a note indicating the file name that this dataframe originates from
                df_xl = pd.read_excel(xlfile, engine='openpyxl')
                df_xl['Note'] = file
                # Appends content of each Excel file iteratively
                df = df.append(df_xl, ignore_index=True)
        return df

    # Upload CSV data
    with st.sidebar.header('1. Upload your ZIP file'):
        uploaded_file = st.sidebar.file_uploader("Excel-containing ZIP file", type=["zip"])

    # File download
    def filedownload(df):
        csv = df.to_csv(index=False)
        b64 = base64.b64encode(csv.encode()).decode()  # strings <-> bytes conversions
        href = f'Download Merged File as CSV'
        return href

    # Main panel
    if st.sidebar.button('Submit'):
        #@st.cache
        df = excel_file_merge(uploaded_file)
        st.header('**Merged data**')
        st.write(df)
        st.markdown(filedownload(df), unsafe_allow_html=True)
    else:
        st.info('Awaiting for ZIP file to be uploaded.')

        ########################################################################

def page4():
    st.markdown("# Profiling Report 🎉")
    st.sidebar.markdown("# Profiling Report 🎉")

    # Upload CSV data
    with st.sidebar.header('Upload your CSV data'):
        uploaded_file = st.sidebar.file_uploader(" ", type=["csv"])
        st.sidebar.markdown("""

    """)

    # Pandas Profiling Report
    if uploaded_file is not None:
        @st.cache
        def load_csv():
            csv = pd.read_csv(uploaded_file)
            return csv
        df = load_csv()
        pr = ProfileReport(df, explorative=True)
        st.header('**Input DataFrame**')
        st.write(df)
        st.write('---')
        st.header('**Pandas Profiling Report**')
        st_profile_report(pr)
    else:
        st.info('Awaiting for CSV file to be uploaded.')
            # Example data
        @st.cache
        def load_data():
            df = load_data()
            pr = ProfileReport(df, explorative=True)
            st.header('**Input DataFrame**')
            st.write(df)
            st.write('---')
            st.header('**Pandas Profiling Report**')
            st_profile_report(pr)

            ##################################################################

def page5():
    st.markdown("# Renaming Columns 🎉")
    st.sidebar.markdown("# Renaming Columns 🎉")

    text_file = st.file_uploader("Upload CSV File", type=['csv'])
    if text_file is not None:
        df= pd.read_csv(text_file)
        st.write(df)

        with st.form(key="form"):
            col_to_change = st.selectbox("Column to change", df.columns)
            new_col_name = st.text_input("New name", value="")
            submit_button = st.form_submit_button(label='Rename')

        if submit_button:
            df = df.rename(columns={col_to_change: new_col_name})

            st.write(df)
            clean_text = df['Text']

            st.download_button(label='Download Dataset',data=clean_text.to_csv(),file_name='renamedfile.csv',mime='text/csv')

        #####################################################################

def page6():
    st.markdown("# Text Cleaner 3 🎉")
    st.sidebar.markdown("# Clean text 🎉")

    def main():

        text_file = st.file_uploader("Upload CSV File",type=['csv'])

        if text_file is not None:
            cleanerdf= pd.read_csv(text_file)
            cleanerdf['Cleaned_Text'] = cleanerdf['Text'].apply(str.lower)
            cleanerdf["Cleaned_Text"] = cleanerdf['Cleaned_Text'].apply(lambda x: " ".join([y.lemma_ for y in en_core(x)]))

            cdf1 = cleanerdf.copy()

            cdf1['Cleaned_Text'] = [' '.join([spell(i) for i in x.split()]) for x in cdf1['Cleaned_Text']]
            cdf1['Cleaned_Text'] = cdf1['Cleaned_Text'].apply(nt.remove_punctuations)
            cdf1['Cleaned_Text'] = cdf1['Cleaned_Text'].apply(nt.remove_numbers)
            cdf1['Cleaned_Text'] = cdf1['Cleaned_Text'].apply(nt.remove_special_characters)
            cdf1['Cleaned_Text'] = cdf1['Cleaned_Text'].apply(nt.remove_dates)
            cdf1['Cleaned_Text'] = cdf1['Cleaned_Text'].apply(nt.remove_emojis)
            cdf1['Cleaned_Text'] = cdf1['Cleaned_Text'].apply(nt.remove_stopwords)

            st.write(cdf1)
            clean_text = cdf1['Cleaned_Text']

            st.download_button(label='Download Cleaned Text',data=clean_text.to_csv(),file_name='cleantext.csv',mime='text/csv')

    if __name__ == '__main__':
        main()

                #####################################################################

def page7():
    st.markdown("# Word Frequency 🎉")
    st.sidebar.markdown(" ")

    text_file = st.file_uploader("Upload CSV File",type=['csv'])

    col1, col2 = st.columns(2)

    with col1:
        if text_file is not None:
            st.markdown('''
            # **Cleaned Text**
            ---
            ''')
            wdf= pd.read_csv(text_file)
            col1.write(wdf)

            wdf["text"] = wdf['Cleaned_Text']
            new_df = wdf.text.str.split(expand=True).stack().value_counts().reset_index()
            new_df.columns = ['Word', 'Frequency']
            col1.write(new_df)

            st.download_button(label='Download Word Count' ,data=new_df.to_csv(),file_name='words.csv',mime='text/csv')

            with col2:
                st.markdown('''
                # **Word Frequency Graph**
                ---
                ''')
                fig = px.bar(
                new_df,
                x = "Frequency",
                y = "Word",
                color="Frequency", width=800, height=1000,
                orientation = 'h' #Optional Parameter
                )
                fig.update_layout(yaxis={'categoryorder':'total ascending'})
                st.plotly_chart(fig)

            st.markdown('''
            # **Word Cloud**
            ---
            ''')

            from wordcloud import WordCloud
            from matplotlib import colors
            from nltk.corpus import stopwords
            color_list=['#4d4d4d','#ffcc00','#000000',]
            colormap=colors.ListedColormap(color_list)
            import matplotlib.pyplot as plt

            new_df2 =  new_df.copy()

            mask = np.array(Image.open("bee.png"))
            words = ' '.join([Word for Word in new_df2['Word']])
            wordCloud = WordCloud(background_color=None, colormap='OrRd', min_font_size=4, max_font_size=100, width=3000, height=800, mask=mask, mode = 'RGB').generate(words)
            plt.figure(figsize=(50, 50), facecolor='k')
            plt.imshow(wordCloud, interpolation='bilinear')
            plt.axis("off")
            st.pyplot()

                    #####################################################################

def page8():
    st.markdown("# Sentiment Analyzer 🎉")
    st.sidebar.markdown("# Text Sentiments 🎉")

    #adding a file uploader

    file = st.file_uploader("Please choose a file")

    if file is not None:
        df= pd.read_csv(file)
        #Can be used wherever a "file-like" object is accepted:

        df['Polarity'] = df.apply(lambda x: TextBlob(x['Cleaned_Text']).sentiment.polarity, axis=1)
        df['Subjectivity'] = df.apply(lambda x: TextBlob(x['Cleaned_Text']).sentiment.subjectivity, axis=1)

        def condition(x):
            if x>0.01:
                return "Positive"
            elif x<0:
                return "Negative"
            else:
                return 'Neutral'

        df['Sentiment'] = df['Polarity'].apply(condition)

        st.write(df)
        st.download_button(label='Download Sentiment Report ',data=df.to_csv(),file_name='sentiments.csv',mime='text/csv')

        col1, col2 = st.columns(2)

        with col1:
            col1.header = "Sentiment"
            st.title = "Scatter"
            fig = px.scatter(df, x="Polarity", y="Subjectivity", color="Sentiment", width=750, height=700, opacity=0.7, color_discrete_sequence=["#FCE6C9","#00BFFF","#CD2626"])
            fig.update_traces(marker=dict(size=20, line=dict(width=2,color='DarkSlateGrey')), selector=dict(mode='markers'))
            st.plotly_chart(fig)

            fig = px.histogram(df, x="Subjectivity")
            st.plotly_chart(fig)

            fig = px.histogram(df, x="Polarity")
            st.plotly_chart(fig)

        with col2:
            col2.header = "Subjectivity"
            fig = px.pie(df, values='Subjectivity', names='Sentiment', color_discrete_sequence=["#00BFFF", "#CD2626", "#FCE6C9"])
            st.plotly_chart(fig)

            from wordcloud import WordCloud
            from matplotlib import colors
            from nltk.corpus import stopwords
            color_list=['#4d4d4d','#ffcc00','#000000',]
            colormap=colors.ListedColormap(color_list)
            import matplotlib.pyplot as plt
            st.set_option('deprecation.showPyplotGlobalUse', False)

            words = ' '.join([Text for Text in df[df['Sentiment']=='Negative']['Cleaned_Text']])
            wordCloud = WordCloud(background_color='black', colormap='Reds', min_font_size=4, max_font_size=200, width=3000, height=1000).generate(words)
            plt.figure(figsize=(50, 50))
            plt.imshow(wordCloud, interpolation='bilinear')
            plt.axis("off")
            st.pyplot()
            st.set_option('deprecation.showPyplotGlobalUse', False)

            words = ' '.join([Text for Text in df[df['Sentiment']=='Positive']['Cleaned_Text']])
            wordCloud = WordCloud(background_color='black', colormap='Blues', min_font_size=4, max_font_size=200, width=3000, height=1000).generate(words)
            plt.figure(figsize=(50, 50))
            plt.imshow(wordCloud, interpolation='bilinear')
            plt.axis("off")
            st.pyplot()
            st.set_option('deprecation.showPyplotGlobalUse', False)

            from wordcloud import WordCloud
            from matplotlib import colors
            from nltk.corpus import stopwords
            color_list=['#4d4d4d','#ffcc00','#000000',]
            colormap=colors.ListedColormap(color_list)
            import matplotlib.pyplot as plt

            words = ' '.join([Text for Text in df[df['Sentiment']=='Neutral']['Cleaned_Text']])
            wordCloud = WordCloud(background_color='black', colormap='YlOrBr', min_font_size=4, max_font_size=200, width=3000, height=1000).generate(words)
            plt.figure(figsize=(50, 50))
            plt.imshow(wordCloud, interpolation='bilinear')
            plt.axis("off")
            st.pyplot()
            st.set_option('deprecation.showPyplotGlobalUse', False)

            senti = df['Sentiment'].value_counts()
            st.write(senti)

            color = ['bisque','deepskyblue','firebrick']

            fig = plt.figure(figsize=(10, 4))
            sns.set_style("darkgrid")
            sns.countplot(x = 'Sentiment', data=df, palette=color)
            st.pyplot(fig)

                    #####################################################################

def page9():
    st.markdown("# Counter 🎉")
    st.sidebar.markdown("# Count instances 🎉")

    text_file = st.file_uploader("Upload CSV File",type=['csv'])

    if text_file is not None:

        col1, col2 = st.columns(2)

        with col1:
            df= pd.read_csv(text_file)
            st.write(df)

        with col2:
            df["Count"] = df['Text']
            new_df = df.Count.value_counts()
            st.write(new_df)

page_names_to_funcs = {
    "Main Page": main_page,
    "Filter": page2,
    "Merge": page3,
    "EDA": page4,
    "Rename": page5,
    "Cleaner": page6,
    "Word Frequency": page7,
    "Sentiment Analyzer": page8,
    "Counter": page9
}

selected_page = st.sidebar.selectbox("Choose Here", page_names_to_funcs.keys())
page_names_to_funcs[selected_page]()


import sqlite3

try:

    # Veritabanı açma
    conn = sqlite3.connect('library.db')

    # Tablo oluşturma
    conn.execute('''CREATE TABLE `yazar` (
        `yazarNo`  INTEGER,
        `ad`   TEXT,
        PRIMARY KEY(`yazarNo`)
            )''')

    conn.execute('''CREATE TABLE `kitap` (
        `kitapNo`  INTEGER,
        `ad`   TEXT,
        `miktar`  TEXT,
        `yazar` INTEGER,
        PRIMARY KEY(`kitapNo`)
            )''')

    # Veritabanına yazma
    conn.commit()

    # Veritabanını kapatma
    conn.close()

    print("DB Olusturuldu")

except Exception as e:
    print(f"DB Olusturulmadı, Hata: {e}")

#yazar girişlerinin yapılması için fonksiyon
def yazarGiris():
    yazar_ad=input("Yazar adı girin: ")
    # bağlantıyı aç
    with sqlite3.connect("library.db") as conn:
        cur=conn.cursor()
        try:
            #kayıt et
            cur.execute('''INSERT INTO yazar (ad) VALUES(?)''',[yazar_ad])
            conn.commit()
            print("Yazar eklendi.")
        except Exception as e:
            print(e)

def kitapGiris():
    kitap_ad=input("Kitap adı girin: ")
    stok_adedi=input("Stok adedi girin: ")
    with sqlite3.connect("library.db") as conn:
        cur=conn.cursor()
        try:
            #yazar tablosundan yazarları alır
            cur.execute('''SELECT * FROM yazar ''')
            data=cur.fetchall()
        except Exception as e:
            print(e)
    print("yazar seçinmi yapın: ")
    #yazar seçimi için yazarları listeler
    for i in range(len(data)):
        print(f"{data[i][0]}:{data[i][1]}")
    yazarno=int(input())

    with sqlite3.connect("library.db") as conn:
        cur=conn.cursor()
        try:
            #kitap eklemek için sql
            cur.execute('''INSERT INTO kitap (ad,miktar,yazar) VALUES(?,?,?)''',[kitap_ad,stok_adedi,yazarno])
            conn.commit()
            print("Kitap eklendi.")
        except Exception as e:
            print(e)

def arama():
    aranacak = input("Aramak Istenilen: ").lower()

    with sqlite3.connect("library.db") as conn:
        cur = conn.cursor()
        try:
            #aranacak kitap veya yazarı arar
            cur.execute("""select k.kitapNo, k.ad, y.ad, k.miktar from yazar y, kitap k where y.yazarNo == k.yazar""")
            data_all=cur.fetchall()
        except Exception as e:
            print(e)

    print('{:^20} {:^20} {:^20} {:^20}'.format("No","Kitap Adı","Yazarı","Stok"))
    print('{:^20} {:^20} {:^20} {:^20}'.format("---------","---------","---------","---------"))

    for i in range(len(data_all)):
        if aranacak in data_all[i][1].lower() or aranacak in data_all[i][2].lower():
            print('{:^20} {:^20} {:^20} {:^20}'.format(data_all[i][0],data_all[i][1],data_all[i][2],data_all[i][3]))

def satis():
    print('{:^20} {:^20} {:^20} {:^20}'.format("No","Kitap Adı","Yazarı","Stok"))
    print('{:^20} {:^20} {:^20} {:^20}'.format("---------","---------","---------","---------"))

    with sqlite3.connect("library.db") as conn:
        cur = conn.cursor()
        try:
            #kitapları çeker ve listeleme yapar
            cur.execute("""SELECT * FROM kitap ORDER BY ad ASC""")
            data_k=cur.fetchall()
            #yazarları çeker
            cur.execute("""SELECT * FROM yazar""")
            data_y=cur.fetchall()
        except Exception as e:
            print(e)
    #ekrana listeleme yapar
    for i in range(len(data_k)):
        for j in range(len(data_y)):
            if str(data_y[j][0]) == str(data_k[i][3]):
                print('{:^20} {:^20} {:^20} {:^20}'.format(data_k[i][0],data_k[i][1],data_y[j][1],data_k[i][2]))

    numara = input("Satılacak kitabın numarası: ")
    with sqlite3.connect("library.db") as conn:
        cur = conn.cursor()
        try:
            #kitapları çeker ve listeler
            cur.execute("""SELECT kitapNo,miktar FROM kitap""")
            kitaplar = cur.fetchall()
        except Exception as e:
            print(e)

    for i in range(len(kitaplar)):
        #eğer seçilen kitabın numarası i ye eşitse
        if int(numara) == int(kitaplar[i][0]):
            if int(kitaplar[i][1]) != 0:
                with sqlite3.connect("library.db") as conn:
                    cur = conn.cursor()
                    try:
                        #kitabı stoktan 1 azalt
                        cur.execute("""UPDATE kitap SET miktar = miktar-1 WHERE kitapNo = ?""", numara)
                        conn.commit()
                    except Exception as e:
                        print(e)

# switch case yapısı için dictionary kalıbı
def indirect(i):
    switcher={
            "y":yazarGiris,
            "k":kitapGiris,
            "a":arama,
            "s":satis
            }
    return switcher[i]()

while 1:
    print("\n")
    deger = input('{:^20} {:^20} {:^20} {:^20}'.format("[Y]azar Girişi","[K]itap Girişi","[A]rama","[S]atış")).lower()
    indirect(deger)


"""
    Name : Pratibha Singh (pratibhasingh@kgpian.iitkgp.ac.in)
    Roll No : 20CS60R12

    File name : task1.py
    Date : Mar 25, 2021
    Problem Statement : Creating grammer and parsing the files and saving the result in logfile.txt
"""

#importing required modules
import ply.lex as lex
import ply.yacc as yacc
import sys
import unicodedata
import task1
import re

# creating dictionary class
class my_dictionary(dict):

    def __init__(self):
        self = dict()

    # function to add key value pair
    def add(self, key, value):
        self[key] = value

# creating a dictionary
result = my_dictionary()

field_list = ['Movie_Name', 'Director', 'Writer', 'Producer', 'Original Language', 'Cast with the Character name', 'Storyline', 'Box office collection', 'Runtime']
#list to store all cast and crew names
cast_char_name = []
dir_name = []

# initialzing all key value of dictonary as ''(empty string)
for i in field_list:
    result.add(i, '')

#print(result)

# token list used
tokens = [
    'MOVIE_S', 'MOVIE_E',
    'DIRECTOR', 'DIR_CLOSE',
    'WHITE_SPACE',
    'NAME',
    'WRITER', 'CLOSE',
    'ESTART',
    'PRODUCER',
    'LANG',
    'STORY',
    'BOXOFFICE',
    'RUNTIME', 'REND',
    'RNAME', 'CEND', 'MID'
]

# Regular expression for each tokens
def t_MOVIE_S(t):
    r'<meta\sproperty=\"og:title\"\scontent=\"'
    return t

def t_MOVIE_E(t):
    r'\">'
    return t

def t_DIRECTOR(t):
    r'\"movie-info-director\">'
    return t

def t_ESTART(t):
    r'\s*<a\shref=\"[\/a-zA-Z_0-9-:]+\">'
    return t

def t_DIR_CLOSE(t):
    r'<\/a>\s*,*'
    return t

def t_WRITER(t):
    r'Writer:<\/div>\s*.*\">\s*'
    return t

def t_PRODUCER(t):
    r'Producer:<\/div>\s*.*\">\s*'
    return t

def t_LANG(t):
    r'Original\sLanguage:<\/div>\s*.*\">'
    return t

def t_STORY(t):
    r'data-qa=\"movie-info-synopsis\">\s*'
    return t

def t_BOXOFFICE(t):
    r'Box\sOffice\s\(Gross\sUSA\):<\/div>\s*.*\">'
    return t

def t_RUNTIME(t):
    r'<time\sdatetime=\"P.*M\">\s*'
    return t

def t_REND(t):
    r'\s*<\/time>'
    return t

def t_RNAME(t):
    r'span\stitle=\"[\u00C0-\u00CF\u00D0-\u00DF\u00E0-\u00EF\u00F0-\u00FFa-zA-z0-9\s\-_&#;\'.]+\">\s*'
    return t

def t_MID(t):
    r'\s*<\/span>\s*.*\s*<span\sclass=\"characters\ssubtle\ssmaller\"\stitle=\"[\u00C0-\u00CF\u00D0-\u00DF\u00E0-\u00EF\u00F0-\u00FFa-zA-Z0-9&#;\s\-_&\'.]+\">\s*<br\/>\s*'
    return t

def t_CEND(t):
    r'\s*<(br\/|\/span)>'
    return t

def t_CLOSE(t):
    r'\s*<\/div>'
    return t

def t_NAME(t):
    #r'[a-zA-Z]+'
    r'[\u00C0-\u00CF\u00D0-\u00DF\u00E0-\u00EF\u00F0-\u00FFa-zA-z0-9,.\(\)$\-&!?:_\';#]+'
    return t

t_WHITE_SPACE = r'[\ \t]+'

def t_error(t):
    #print("Illegal Characters")
    t.lexer.skip(1)

# Parser Rules
def p_start(var):
    '''
    start : movie_name
          | dir_name
          | writer_name
          | producer_name
          | language
          | cast
          | cast_only_name
          | storyline
          | boxoffice
          | runtime
    '''

# Extracting movie name
def p_movie_name(var):
    'movie_name : MOVIE_S name MOVIE_E'
    result.add('Movie_Name', var[2])

# Extracting director names
def p_dir_name(var):
    'dir_name : DIRECTOR name DIR_CLOSE'
    res = var[2]
    dir_name.insert(len(dir_name), res)

# Extracting Writer names
def p_writer_name(var):
    'writer_name : WRITER expression CLOSE'
    result.add('Writer', var[2])

# Extracting producer names
def p_producer_name(var):
    'producer_name : PRODUCER expression CLOSE'
    result.add('Producer', var[2])

# Extracting language
def p_language(var):
    'language : LANG name CLOSE'
    result.add('Original Language', var[2])

# Extracting Cast and Crew names
def p_cast(var):
    'cast : RNAME name MID name CEND'
    res = var[2] + ' as ' + var[4]
    cast_char_name.insert(len(cast_char_name), res)

def p_cast_only_name(var):
    'cast_only_name : RNAME name MID CEND'
    res = var[2]
    cast_char_name.insert(len(cast_char_name), res)

# Extracting storyline
def p_storyline(var):
    'storyline : STORY name CLOSE'
    result.add('Storyline', var[2])

# Extracting boxoffice
def p_boxoffice(var):
    'boxoffice : BOXOFFICE name CLOSE'
    result.add('Box office collection', var[2])

# Extracting runtime
def p_runtime(var):
    'runtime : RUNTIME name REND'
    result.add('Runtime', var[2])

# Recurssive way of extraction
def p_expression_multi(var):
    'expression : expression expression'
    var[0] = var[1] + ',' + var[2]

def p_expression_single(var):
    'expression : expression'
    var[0] = var[1]

def p_expression(var):
    'expression : ESTART name DIR_CLOSE'
    var[0] = var[2]

# Extracting names using recurssive method
def p_name_one(var):
    'name : NAME'
    var[0] = var[1]

def p_name_mid(var):
    'name : NAME name'
    var[0] = var[1] + ' ' + var[2]

def p_name_multi(var):
    'name : NAME wspaces name'
    var[0] = var[1] + var[2] + var[3]

def p_wspaces(var):
    '''
    wspaces : WHITE_SPACE
            | WHITE_SPACE wspaces
    '''
    var[0] = ' '

def p_error(var):
    pass

# building lexer and parser
lexer = lex.lex()
parser = yacc.yacc()

# opening the html file of movie from task1
f = open(task1.filename)
txt = f.read()

lex.input(txt)

#for tok in iter(lex.token, None):
#    print(repr(tok.type), repr(tok.value))

parser.parse(txt)

result.add('Director', dir_name )
result.add('Cast with the Character name', cast_char_name)

print('\t\tExtract Following Fields:')
print('1. Movie Name')
print('2. Director')
print('3. Writers')
print('4. Producer')
print('5. Original Language')
print('6. Cast with the character name')
print('7. Storyline')
print('8. Box Office Collection')
print('9. Runtime')

# opening file in append mode to write all the query fields
file_name = open('logfile.txt', 'a')

while(True):
    val = input('\nEnter Any one of the fields(Enter digit) or press \'E\' to exit:   ')

    if val == '1' or val == '3' or val == '4' or val == '5' or val == '8' or val == '9':
        user_ip = int(val)
        idx = field_list[user_ip-1]
        print('\n', idx, ':')
        if(bool(result[idx])):
            val_list = result[idx].split(',')
            for i in val_list:
                print(i)
                file_name.write('<%s> <%s> <%s> <%s>\n' %(task1.indx, task1.movie_name, idx, i))
        else:
            print('Data Unavailable for this field')
            file_name.write('<%s> <%s> <%s> <%s>\n' %(task1.indx, task1.movie_name, idx, 'Data Unavailable'))

    elif val == '7':
        user_ip = int(val)
        idx = field_list[user_ip-1]
        print('\n', idx, ':')
        if(bool(result[idx])):
            print(result[idx])
            file_name.write('<%s> <%s> <%s> <%s>\n' %(task1.indx, task1.movie_name, idx, result[idx]))
        else:
            print('Data Unavailable for this field')
            file_name.write('<%s> <%s> <%s> <%s>\n' %(task1.indx, task1.movie_name, idx, 'Data Unavailable'))

    elif val == '2' or val == '6':
        user_ip = int(val)
        idx = field_list[user_ip-1]
        print('\n', idx, ':')
        if(bool(result[idx])):
            for i in result[idx]:
                print(i)
                file_name.write('<%s> <%s> <%s> <%s>\n' %(task1.indx, task1.movie_name, idx, i))
        else:
            print('Data Unavailable for this field')
            file_name.write('<%s> <%s> <%s> <%s>\n' %(task1.indx, task1.movie_name, idx, 'Data Unavailable'))

    elif val == 'E':
        print("Closing......")
        break

    else:
        print('Incorrect format... choose a valid number')

# closing all the files
file_name.close
f.close


import cv2
import os

# setting the path to the folder having eyes pics
folder_path = 'C:\\Users\\harsh\\Downloads\\Eyes'

# looping over all the images in the folder
for file_name in os.listdir(folder_path):
    # for all the jpg images (the folder has only jpg images still to filter any other format)
    if file_name.endswith('.jpg'):
        # loading and displaying the image
        img = cv2.imread(os.path.join(folder_path, file_name))
        cv2.imshow('Image', img)
        cv2.waitKey(0)

# destroy all windows after displaying the images
cv2.destroyAllWindows()


import matplotlib
import os

class Report:
    def __init__(self, title, report_path, figures_folder='./figures'):
        # Open output file
        self.mdfile = open(report_path, 'w+')

        # Setup folder for figures
        self.figures_relpath = figures_folder
        self.figures_abspath = os.path.join(os.path.abspath(os.path.dirname(report_path)), self.figures_relpath)
        if not os.path.exists(self.figures_abspath):
            os.makedirs(self.figures_abspath)

        # Print title
        self.print_line('# %s' % title)

        # Use type 1 fonts in PDFs
        matplotlib.rcParams['ps.useafm'] = True
        matplotlib.rcParams['pdf.use14corefonts'] = True
        matplotlib.rcParams['text.usetex'] = True

    def print_line(self, text):
        self.mdfile.write(text+'\n\n')

    def print_figure(self, fig, name, description):
        # For paper
        fig.savefig('%s/%s.pdf' % (self.figures_abspath, name), bbox_inches='tight')

        self.print_line('#### Figure: %s \n\n' % description)
        # For MD Report
        fig.savefig('%s/%s.png' % (self.figures_abspath, name), bbox_inches='tight')
        self.print_line('![%s](%s/%s.png)\n\n' %(description, self.figures_relpath, name))
        return

    def print_table_from_series(self, series):
        return self.print_table_from_df(series.reset_index())

    def print_table_from_df(self, df):
        import pandas as pd
        fmt = ['---' for i in range(len(df.columns))]
        df_fmt = pd.DataFrame([fmt], columns=df.columns)
        df_formatted = pd.concat([df_fmt, df])
        self.print_line(df_formatted.to_csv(sep="|", index=False))


from django import template
from webapp.settings import TRANSLATION_FIELDS

register = template.Library()

class LocalizedContent(template.Node):
    def __init__(self, model, language_code):
        self.model = model
        self.lang = language_code

    def render(self, context):
        model = template.resolve_variable(self.model, context)
        lang = template.resolve_variable(self.lang, context)
        for f in TRANSLATION_FIELDS:
            try:
                setattr(model, f, getattr(model, '%s_%s' % (f, lang)))
            except AttributeError:
                pass
        return ''

@register.tag(name='get_localized_content')
def get_localized_content(parser, token):
    bits = list(token.split_contents())
    if len(bits) != 3:
        raise template.TemplateSyntaxError("'get_localized_content' tag takes exactly 2 arguments")
    return LocalizedContent(model=bits[1], language_code=bits[2])


from klein import run, route, Klein
app = Klein()
import json
import treq

import extruct

from bs4 import BeautifulSoup
from twisted.web.client import getPage
from twisted.internet import reactor, defer

def merge_dicts(current_product_info, new_dict):
    returned_info = dict(current_product_info)
    for x in new_dict:
        if  x not in current_product_info:
            returned_info[x] = new_dict[x]
        else:
            if new_dict[x] is not None:
                returned_info[x] = new_dict[x]
    return returned_info

def parse_product_schema(product_details):
    product = {}
    product['name'] = product_details.get('name',None)
    product['image'] = product_details.get('image',None)
    if product['image'] is not None and isinstance(product['image'],str):
        product['image'] = [product['image']]
    product['brand'] = product_details.get('brand',None)
    product['description'] = product_details.get('description',None)
    product['condition'] = product_details.get('itemCondition',None)
    product['manufacturer'] = product_details.get('manufacturer',None)
    product['color'] = product_details.get('color',None)
    return product

def parse_offers_schema(offers):
    product = {}
    product['price'] = offers.get('price',None) if not isinstance(offers.get('price',None),list) else offers['price'][0]
    product['availability'] = offers.get('availability',None)
    if product['availability'] is not None and 'schema.org' in product['availability']:
        product['availability'] = product['availability'].split('/')[3]
    product['currency'] = offers.get('priceCurrency',None)
    product['currency'] = offers.get('currency',product.get('currency',None))
    product['condition'] = offers.get('itemCondition',product.get('condition',None))
    product['condition'] = offers.get('condition',product.get('condition',None))
    return product

def fix_if_more_contexts(structure):
    if isinstance(structure,str) or isinstance(structure,int):
        return structure

    if isinstance(structure,dict):
        d = {}
        for key in structure:
            result = fix_if_more_contexts(structure[key])
            if ':' in key:
                new_key = key.split(':')[-1]
                d[new_key] = result
            else:
                d[key] = result
        return d
    else:
        d = []
        for key in structure:
            d.append(fix_if_more_contexts(key))
        return d

def check_if_type(structure, value):
    if isinstance(structure, list):
        for x in structure:
            if x.endswith(value):
                return True
    elif isinstance(structure, str):
        return structure.endswith(value)

def get_breadcrumb(breadcrumb_list):
    product = {}
    product['category'] = ' // '.join([x.get('name',
                                             x.get('title',
                                                 x.get('value',
                                                       x['item']['name'] if isinstance(x.get('item',None),dict)
                                                       else ''
                                                             ))) for x in breadcrumb_list])
    return product

def get_info_from_meta(text,url):
    product  = {}
    all_meta = extruct.extract(text,url, uniform=True)
    #print (all_meta['microdata'])
    opengraph = all_meta['opengraph']
    product['offers'] = []
    try:
        if len(opengraph) >=1:
            opengraph = opengraph[0]
            product['description'] = opengraph.get('og:description',None)
            product['image'] = opengraph.get('og:image',None)
            product['brand'] = opengraph.get('product:brand',None)
            product['name'] = opengraph.get('og:title',None)
            offer = {'price':opengraph.get('product:price:amount',
                            opengraph.get('og:price:amount',None)),
                     'currency': opengraph.get('product:price:currency',
                            opengraph.get('og:price:currency',None))}
            if offer['price'] is not None:
                product['offers'].append(offer)

    except Exception as e:
        print(e)
    microdata = all_meta['microdata']
    try:
        product_details = [x for x in microdata if check_if_type(x.get('@type',''),'Product')]
        if len(product_details) == 1:
            product_details = product_details[0]
            product['multiple_products'] = False
            product = merge_dicts(product,parse_product_schema(product_details))
            #product.update(parse_product_schema(product_details))

            if 'offers' in  product_details:
                offers = product_details['offers']
                if isinstance(offers,list):
                    for x in offers:
                        product['offers'].append(parse_offers_schema(x))
                else:
                    product['offers'].append(parse_offers_schema(offers))
        elif len(product_details)>1:
            product['multiple_products'] = True

    except Exception as e:
        print(e)

    jsonld = all_meta['json-ld']
    try:
        graphdetails = [x for x in jsonld if '@graph' in x]
        if len(graphdetails)>0:
            jsonld = graphdetails[0]['@graph']

        itemOffered = [x for x in jsonld if check_if_type(x.get('@type',''),'s:Offer')]

        if len(itemOffered)>0:
            jsonld = fix_if_more_contexts(jsonld)
            itemOffered = [x for x in jsonld if check_if_type(x.get('@type',''),'Offer')]
            product_details = itemOffered[0]
            if 'offers' in  product_details:
                offers = product_details['offers']
                if isinstance(offers,list):
                    for x in offers:
                        product['offers'].append(parse_offers_schema(x))
                else:
                    product['offers'].append(parse_offers_schema(offers))

        product_details = [x for x in jsonld if check_if_type(x.get('@type',''),'Product') ]
        if len(product_details) == 1:
            product_details = product_details[0]
            #product.update(parse_product_schema(product_details))
            product['multiple_products'] = False

            product = merge_dicts(product,parse_product_schema(product_details))

            if 'offers' in  product_details:
                offers = product_details['offers']
                if isinstance(offers,list):
                    for x in offers:
                        product['offers'].append(parse_offers_schema(x))
                else:
                    product['offers'].append(parse_offers_schema(offers))
        elif len(product_details)>1:
            product['multiple_products'] = True

    except Exception as e:
        print(e)

    try:
        if len(product) > 0:
            for p in [jsonld,microdata]:
                breadcrumb_details = [x for x in p if x.get('@type','').endswith('BreadcrumbList')]
                breadcrumb_list = [x for x in p if x.get('@type','').endswith('Breadcrumb')]
                if len(breadcrumb_details)>0:
                    breadcrumb_list = breadcrumb_details[0]['itemListElement']
                if len(breadcrumb_list)>0:
                    #product.update(get_breadcrumb(breadcrumb_list))
                    product = merge_dicts(product,get_breadcrumb(breadcrumb_list))
    except Exception as e:
        print(e)

    product['offers'] = [x for x in product['offers'] if x['price'] is not None]
    if len(product['offers']) == 0:
        del product['offers']
    return product

# not_price_words = ['od','zyskaj','powyżej','odbierz','from','get']

# def get_price(text):
#     import re
#     q = [x for x in re.findall(r'([A-z]*?)\s*([\d,\.]{2,})\s*(pln|zl|zł|€|eur|euro|gbp|£)', text)]
#     prices = []
#     for x in q:
#         try:
#             price = float(x[1].strip().replace(',','.'))
#         except:
#             continue
#         t = x[0].strip()
#         currency = x[2]
#         if len([p for p in not_price_words if t.endswith(p)])>0:
#             continue

#         if price == 0:
#             continue
#         index = text.find(x[1])
#         prices.append((price,currency,index))

#     if len(prices)>=2 and (prices[1][2]-prices[0][2])<30:
#         return {'price':min(prices[0][0],prices[1][0]), 'currency':prices[0][1]}
#     elif len(prices) > 0:
#         return {'price':prices[0][0], 'currency':prices[0][1]}
#     else:
#         return None

# def clean_html(html):
#     soup = BeautifulSoup(html,'lxml')
#     for s in soup(['script', 'style']):
#         s.decompose()
#     return (' '.join(soup.stripped_strings)).lower()

not_price_words = ['od','zyskaj','powyżej','odbierz','from','get']

def get_price(text):
    import re
    r = re.compile(r'(?P<text>.*?)((?P<price1>\d+[\.,]*\d*\s*(pln|zl|zł|€|eur|euro|gbp|£|\$))|(?P<price2>[£€$]\s*\d+[\.,]*\d*))')
    p = [m.groupdict() for m in r.finditer(text)]
    prices = []
    for x in p:
        bb = x['price2'] if x['price1'] is None else x['price1']
        try:
            pot_price = re.search(r'\d+[\.,]*\d*',bb).group(0)
            price = float(pot_price.strip().replace(',','.'))
        except:
            continue

        t = x['text'].strip()

        if len([p for p in not_price_words if t.endswith(p)])>0:
            continue
        currency = bb.replace(pot_price,'').strip()
        if price < 0.00001:
            continue
        index = text.find(bb)
        prices.append((price,currency,index))

    if len(prices)>=2 and (prices[1][2]-prices[0][2])<30:
        return {'price':min(prices[0][0],prices[1][0]), 'currency':prices[0][1]}
    elif len(prices) > 0:
        return {'price':prices[0][0], 'currency':prices[0][1]}
    else:
        return None

def clean_html(html):
    soup = BeautifulSoup(html,'lxml')
    for s in soup(['script', 'style']):
        s.decompose()
    return (' '.join(soup.stripped_strings)).lower()

def get_products_details(text,url):
    try:
        product = get_info_from_meta(text,url)
        if 'offers' not in product:
            price = get_price(clean_html(text))
            if price is not None:
                product['offers'] = [price]

        return product
    except Exception as e:
        print(e)

    return {}

@defer.inlineCallbacks
def return_info(response):
    text = yield response.content()
    url = response.request.absoluteURI.decode("utf-8")
    v = get_products_details(text, url)#response.request.absoluteURI)
    return json.dumps(v)

@app.route("/parse", methods=['POST'])
def get_category(request):
    content = json.loads(request.content.read())
    url = content['url']
    d = treq.get(url)
    d.addCallback(return_info)
    return d

resource = app.resource

app.run("0.0.0.0", 5005)


import pandas as pd
import numpy as np
import os
from tqdm import tqdm
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from sklearn import metrics
import warnings
import matplotlib.pyplot as plt

pd.set_option('display.max_columns', 100)
warnings.filterwarnings('ignore')

def group_feature(df, key, target, aggs):
    agg_dict = {}
    for ag in aggs:
        agg_dict[f'{target}_{ag}'] = ag
    print(agg_dict)
    t = df.groupby(key)[target].agg(agg_dict).reset_index()
    return t

def extract_feature(df, train):
    t = group_feature(df, 'ship', 'x', ['max', 'min', 'mean', 'std', 'skew', 'sum'])
    train = pd.merge(train, t, on='ship', how='left')
    t = group_feature(df, 'ship', 'x', ['count'])
    train = pd.merge(train, t, on='ship', how='left')

    t = group_feature(df, 'ship', 'y', ['max', 'min', 'mean', 'std', 'skew', 'sum'])
    train = pd.merge(train, t, on='ship', how='left')

    t = group_feature(df, 'ship', 'v', ['max', 'min', 'mean', 'std', 'skew', 'sum'])
    train = pd.merge(train, t, on='ship', how='left')

    t = group_feature(df, 'ship', 'd', ['max', 'min', 'mean', 'std', 'skew', 'sum'])
    train = pd.merge(train, t, on='ship', how='left')

    train['x_max_x_min'] = train['x_max'] - train['x_min']
    train['y_max_y_min'] = train['y_max'] - train['y_min']
    train['y_max_x_min'] = train['y_max'] - train['x_min']
    train['x_max_y_min'] = train['x_max'] - train['y_min']
    train['slope'] = train['y_max_y_min'] / np.where(train['x_max_x_min'] == 0, 0.001, train['x_max_x_min'])
    train['area'] = train['x_max_x_min'] * train['y_max_y_min']

    mode_hour = df.groupby('ship')['hour'].agg(lambda x: x.value_counts().index[0]).to_dict()
    train['mode_hour'] = train['ship'].map(mode_hour)

    t = group_feature(df, 'ship', 'hour', ['max', 'min'])
    train = pd.merge(train, t, on='ship', how='left')

    hour_nunique = df.groupby('ship')['hour'].nunique().to_dict()
    date_nunique = df.groupby('ship')['date'].nunique().to_dict()
    train['hour_nunique'] = train['ship'].map(hour_nunique)
    train['date_nunique'] = train['ship'].map(date_nunique)

    t = df.groupby('ship')['time'].agg({'diff_time': lambda x: np.max(x) - np.min(x)}).reset_index()
    t['diff_day'] = t['diff_time'].dt.days
    t['diff_second'] = t['diff_time'].dt.seconds
    train = pd.merge(train, t, on='ship', how='left')
    # print(train)
    # input()
    return train

def extract_dt(df):
    df['time'] = pd.to_datetime(df['time'], format='%m%d %H:%M:%S')
    # df['month'] = df['time'].dt.month
    # df['day'] = df['time'].dt.day
    df['date'] = df['time'].dt.date
    df['hour'] = df['time'].dt.hour
    # df = df.drop_duplicates(['ship','month'])
    df['weekday'] = df['time'].dt.weekday
    return df

path = r'D:\a_zhy\MachineLearning\game\input'
train = pd.read_csv(path + r'\train.csv')
# train = df.drop_duplicates(['ship','type'])
test = pd.read_csv(path + r'\test.csv')

train = extract_dt(train)
test = extract_dt(test)
# print(train)
# 去除特定列下面的重复行
train_label = train.drop_duplicates('ship')
test_label = test.drop_duplicates('ship')

# train_label['type'].value_counts(1)

type_map = dict(zip(train_label['type'].unique(), np.arange(3)))
type_map_rev = {v:k for k,v in type_map.items()}
train_label['type'] = train_label['type'].map(type_map)
# print(train_label)
# input()
train_label = extract_feature(train, train_label)

test_label = extract_feature(test, test_label)

features = [x for x in train_label.columns if x not in ['ship','type','time','diff_time','date']]
target = 'type'
print(len(features), ','.join(features))

X = train_label[features].copy()
# X = (X-X.mean())/X.std()
# X = (X-X.min())/(X.max()-X.min())
X_test = test_label[features].copy()
# X_test = (X_test-X_test.mean())/X_test.std()
# X_test = (X_test-X_test.min())/(X_test.max()-X_test.min())
y = train_label[target]
# X: (7000, 45)
# y: (7000,)

# 自定义F1评价函数
def f1_score_vail(pred, data_vail):
    labels = data_vail.get_label()
    pred = np.argmax(pred.reshape(3, -1), axis=0)      # lgb的predict输出为各类型概率值
    score_vail = metrics.f1_score(y_true=labels, y_pred=pred, average='macro')
    return 'f1_score', score_vail, True

'''模型优化———————————————————'''
from sklearn.model_selection import train_test_split
X_train_, X_val_, y_train_, y_val_ = train_test_split(X, y, random_state=0, test_size=0.2)

### 数据转换
print('数据转换')
lgb_train = lgb.Dataset(X_train_, y_train_, free_raw_data=False)
lgb_eval = lgb.Dataset(X_val_, y_val_, reference=lgb_train, free_raw_data=False)

print('设置参数')
params = {
    'metric': 'multi_logloss',
    'nthread': 4,
    'n_estimators': 5000,
    'boosting_type': 'gbdt',
    'objective': 'multiclass',
    'num_class': 3,
    'early_stopping_rounds': 100,
}

# 交叉验证(调参)
print('交叉验证')
max_auc = float('1')
best_params = {}

# 准确率
# print("调参1：提高准确率")
# for num_leaves in range(10, 11, 20):
#     for max_depth in range(3, 4, 2):
#         params['num_leaves'] = num_leaves
#         params['max_depth'] = max_depth
#
#         cv_results = lgb.cv(
#             params,
#             lgb_train,
#             seed=1,
#             nfold=5,
#             # metrics=['multi_logloss'],
#             feval=f1_score_vail,
#             early_stopping_rounds=10,
#             verbose_eval=True
#         )
#         # print(cv_results)
#         # input()
#         mean_auc = pd.Series(cv_results['multi_logloss-mean']).min()
#         boost_rounds = pd.Series(cv_results['multi_logloss-mean']).idxmin()
#
#         if mean_auc <= max_auc:
#             max_auc = mean_auc
#             best_params['num_leaves'] = num_leaves
#             best_params['max_depth'] = max_depth
# if 'num_leaves' and 'max_depth' in best_params.keys():
#     params['num_leaves'] = best_params['num_leaves']
#     params['max_depth'] = best_params['max_depth']

# # 过拟合
# print("调参2：降低过拟合")
# for max_bin in range(5, 256, 10):
#     for min_data_in_leaf in range(1, 102, 10):
#         params['max_bin'] = max_bin
#         params['min_data_in_leaf'] = min_data_in_leaf
#
#         cv_results = lgb.cv(
#             params,
#             lgb_train,
#             seed=1,
#             nfold=5,
#             metrics=['multi_logloss'],
#             early_stopping_rounds=10,
#             verbose_eval=True
#         )
#
#         mean_auc = pd.Series(cv_results['multi_logloss-mean']).max()
#         boost_rounds = pd.Series(cv_results['multi_logloss-mean']).idxmax()
#
#         if mean_auc >= max_auc:
#             max_auc = mean_auc
#             best_params['max_bin'] = max_bin
#             best_params['min_data_in_leaf'] = min_data_in_leaf
# if 'max_bin' and 'min_data_in_leaf' in best_params.keys():
#     params['min_data_in_leaf'] = best_params['min_data_in_leaf']
#     params['max_bin'] = best_params['max_bin']
#
# print("调参3：降低过拟合")
# for feature_fraction in [0.6, 0.7, 0.8, 0.9, 1.0]:
#     for bagging_fraction in [0.6, 0.7, 0.8, 0.9, 1.0]:
#         for bagging_freq in range(0, 50, 5):
#             params['feature_fraction'] = feature_fraction
#             params['bagging_fraction'] = bagging_fraction
#             params['bagging_freq'] = bagging_freq
#
#             cv_results = lgb.cv(
#                 params,
#                 lgb_train,
#                 seed=1,
#                 nfold=5,
#                 metrics=['multi_logloss'],
#                 early_stopping_rounds=10,
#                 verbose_eval=True
#             )
#
#             mean_auc = pd.Series(cv_results['multi_logloss-mean']).max()
#             boost_rounds = pd.Series(cv_results['multi_logloss-mean']).idxmax()
#
#             if mean_auc >= max_auc:
#                 max_auc = mean_auc
#                 best_params['feature_fraction'] = feature_fraction
#                 best_params['bagging_fraction'] = bagging_fraction
#                 best_params['bagging_freq'] = bagging_freq
#
# if 'feature_fraction' and 'bagging_fraction' and 'bagging_freq' in best_params.keys():
#     params['feature_fraction'] = best_params['feature_fraction']
#     params['bagging_fraction'] = best_params['bagging_fraction']
#     params['bagging_freq'] = best_params['bagging_freq']
#
# print("调参4：降低过拟合")
# for lambda_l1 in [1e-5, 1e-3, 1e-1, 0.0, 0.1, 0.3, 0.5, 0.7, 0.9, 1.0]:
#     for lambda_l2 in [1e-5, 1e-3, 1e-1, 0.0, 0.1, 0.4, 0.6, 0.7, 0.9, 1.0]:
#         params['lambda_l1'] = lambda_l1
#         params['lambda_l2'] = lambda_l2
#         cv_results = lgb.cv(
#             params,
#             lgb_train,
#             seed=1,
#             nfold=5,
#             metrics=['multi_logloss'],
#             early_stopping_rounds=10,
#             verbose_eval=True
#         )
#
#         mean_auc = pd.Series(cv_results['multi_logloss-mean']).max()
#         boost_rounds = pd.Series(cv_results['multi_logloss-mean']).idxmax()
#
#         if mean_auc >= max_auc:
#             max_auc = mean_auc
#             best_params['lambda_l1'] = lambda_l1
#             best_params['lambda_l2'] = lambda_l2
# if 'lambda_l1' and 'lambda_l2' in best_params.keys():
#     params['lambda_l1'] = best_params['lambda_l1']
#     params['lambda_l2'] = best_params['lambda_l2']
#
# print("调参5：降低过拟合2")
# for min_split_gain in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
#     params['min_split_gain'] = min_split_gain
#
#     cv_results = lgb.cv(
#         params,
#         lgb_train,
#         seed=1,
#         nfold=5,
#         metrics=['multi_logloss'],
#         early_stopping_rounds=10,
#         verbose_eval=True
#     )
#
#     mean_auc = pd.Series(cv_results['multi_logloss-mean']).max()
#     boost_rounds = pd.Series(cv_results['multi_logloss-mean']).idxmax()
#
#     if mean_auc >= max_auc:
#         max_auc = mean_auc
#
#         best_params['min_split_gain'] = min_split_gain
# if 'min_split_gain' in best_params.keys():
#     params['min_split_gain'] = best_params['min_split_gain']

# print(best_params)
# 参数
# params = {
#     'n_estimators': 5000,
#     'boosting_type': 'gbdt',
#     'objective': 'multiclass',
#     'num_class': 3,
#     'early_stopping_rounds': 100,
# }

'''模型训练———————————————————'''
fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

models = []
pred = np.zeros((len(test_label),3))
oof = np.zeros((len(X), 3))
for index, (train_idx, val_idx) in enumerate(fold.split(X, y)):

    train_set = lgb.Dataset(X.iloc[train_idx], y.iloc[train_idx])
    val_set = lgb.Dataset(X.iloc[val_idx], y.iloc[val_idx])

    model = lgb.train(params, train_set, valid_sets=[train_set, val_set],
                      feval=f1_score_vail, verbose_eval=100)
    models.append(model)
    val_pred = model.predict(X.iloc[val_idx])
    oof[val_idx] = val_pred
    val_y = y.iloc[val_idx]
    val_pred = np.argmax(val_pred, axis=1)
    print(index, 'val f1', metrics.f1_score(val_y, val_pred, average='macro'),'\n')
    # 0.8695539641133697
    # 0.8866211724839532

    test_pred = model.predict(X_test)
    pred += test_pred/5

# print("参数：\n",params)
oof = np.argmax(oof, axis=1)
print('oof f1', metrics.f1_score(oof, y, average='macro'))
# 0.8701544575329372

pred = np.argmax(pred, axis=1)
sub = test_label[['ship']]
sub['pred'] = pred

print(sub['pred'].value_counts(1))
sub['pred'] = sub['pred'].map(type_map_rev)
sub.to_csv('result_3.csv', index=None, header=None)

# ret = []
# for index, model in enumerate(models):
#     df = pd.DataFrame()
#     df['name'] = model.feature_name()
#     df['score'] = model.feature_importance()
#     df['fold'] = index
#     ret.append(df)
#
# df = pd.concat(ret)
#
# df = df.groupby('name', as_index=False)['score'].mean()
# df = df.sort_values(['score'], ascending=False)
# print(df)