from django.conf import settings
from django.shortcuts import render_to_response, get_object_or_404
from django import forms
from django.template import RequestContext
from django.views.generic import list_detail
from django.http import Http404, HttpResponseRedirect
from colddirt.dirty.models import *
from colddirt.forms import *
import datetime, colddirt.captcha

def live_update(request):
    from django.http import HttpResponse
    from django.core import serializers
    # To get my designed feature set, plus not kill the load
    # on my server, this is all a bit hackery.  If you see
    # a better way to handle it, *especially* the JS, let
    # me know!
    if request.method == 'GET':
        new_pk = request.GET.get('pk')
        if new_pk:
            data = serializers.serialize("json", Dirt.objects.filter(pk__gte=new_pk).order_by('-publish_date')[:10])
    return HttpResponse(data, mimetype='application/json')

def dirt_count(request):
    from django.http import HttpResponse
    countd = str(Dirt.objects.all().count())
    return HttpResponse(countd, mimetype='text/plain')

def live(request):
    dirt_form = DirtyForm()
    html_captcha = colddirt.captcha.displayhtml(settings.RECAPTCHA_PUB_KEY)
    return render_to_response("dirty/live_dirt.html", {
                                "form": dirt_form.as_ul(),
                                'html_captcha': html_captcha})

def index(request):
    import re
    from django.template.defaultfilters import slugify
    # I display the form on every page, so I don't need an if/else here.  However, You would normall put the below two lines after an else:
    dirt_form = DirtyForm()
    html_captcha = colddirt.captcha.displayhtml(settings.RECAPTCHA_PUB_KEY)
    if request.method == 'POST':
        check_captcha = colddirt.captcha.submit(request.POST['recaptcha_challenge_field'], request.POST['recaptcha_response_field'], settings.RECAPTCHA_PRIVATE_KEY, request.META['REMOTE_ADDR'])
        if check_captcha.is_valid is False:
            # Captcha is wrong show a error ...
            return render_to_response('dirty/dirt_list.html',
                                      { 'form': dirt_form.as_ul(),
                                       'html_captcha': html_captcha,
                                       'errorlist_captcha': True},
                                      context_instance=RequestContext(request))
        dirt_form = DirtyForm(request.POST)
        if dirt_form.is_valid():
            # I opted not to create an m2m relationship for several
            # reasons. Note: the latest_word is some random word.
            latest_word = Word.objects.filter(is_taken__exact=False).order_by('?')[:1]
            latest_word[0].is_taken=True
            latest_word[0].save()
            new_dirt = Dirt(description=dirt_form.cleaned_data['description'],
                            dirtword=latest_word[0].dirtyword)
            new_dirt.save()
            # Credit for the tag parsing goes to Ubernostrum (James B)
            # from the Cab (great for learning!) project
            # I opted to not store tag_list in each entry
            # Splitting to get the new tag list is tricky, because people
            # will stick commas and other whitespace in the darndest places.
            new_tag_list = [t for t in re.split('[\s,]+', dirt_form.cleaned_data['tag_list']) if t]
            # Now add the tags
            for tag_name in new_tag_list:
                tag, created = Tagling.objects.get_or_create(name=slugify(tag_name), slug=slugify(tag_name))
                new_dirt.tags.add(tag)
            return HttpResponseRedirect(new_dirt.get_absolute_url())
    return list_detail.object_list(
           request,
           queryset = Dirt.objects.order_by('?')[:100],
           template_name = "dirty/dirt_list.html",
           template_object_name = "dirty",
           paginate_by = 10,
           extra_context = {'form': dirt_form.as_ul(),
                            'html_captcha': html_captcha},
        )

def submit(request):
    import re
    from django.template.defaultfilters import slugify
    dirt_form = DirtyForm()
    html_captcha = colddirt.captcha.displayhtml(settings.RECAPTCHA_PUB_KEY)
    if request.method == 'POST':
        check_captcha = colddirt.captcha.submit(request.POST['recaptcha_challenge_field'], request.POST['recaptcha_response_field'], settings.RECAPTCHA_PRIVATE_KEY, request.META['REMOTE_ADDR'])
        if check_captcha.is_valid is False:
            # Captcha is wrong show a error ...
            return render_to_response('dirty/dirt_list.html',
                                      { 'form': dirt_form.as_ul(),
                                       'html_captcha': html_captcha,
                                       'errorlist_captcha': True},
                                      context_instance=RequestContext(request))
        dirt_form = DirtyForm(request.POST)
        if dirt_form.is_valid():
            # I opted not to create an m2m relationship for several
            # reasons. Note: the latest_word is some random word.
            latest_word = Word.objects.filter(is_taken__exact=False).order_by('?')[:1]
            latest_word[0].is_taken=True
            #latest_word[0].save()
            new_dirt = Dirt(description=dirt_form.cleaned_data['description'],
                            dirtword=latest_word[0].dirtyword)
            new_dirt.save()
            # Credit for the tag parsing goes to Ubernostrum (James B)
            # from the Cab (great for learning!) project
            # I opted to not store tag_list in each entry
            # Splitting to get the new tag list is tricky, because people
            # will stick commas and other whitespace in the darndest places.
            new_tag_list = [t for t in re.split('[\s,]+', dirt_form.cleaned_data['tag_list']) if t]
            # Now add the tags
            for tag_name in new_tag_list:
                tag, created = Tagling.objects.get_or_create(name=slugify(tag_name), slug=slugify(tag_name))
                new_dirt.tags.add(tag)
            return HttpResponseRedirect(new_dirt.get_absolute_url())
    return render_to_response('dirty/dirt_list.html',
                              { 'form': dirt_form.as_ul(),
                               'html_captcha': html_captcha},
                              context_instance=RequestContext(request))

def tag_list(request, slug):
    dirts = Dirt.objects.filter(tags__slug__iexact=slug)
    html_captcha = colddirt.captcha.displayhtml(settings.RECAPTCHA_PUB_KEY)
    return list_detail.object_list(
           request,
           queryset = dirts,
           template_name = "dirty/dirt_list.html",
           template_object_name = "dirty",
           paginate_by = 25,
           extra_context = {'form': DirtyForm().as_ul(),
                            'html_captcha': html_captcha,
                            'tag_search': slug }
        )

def search(request):
    from django.db.models import Q
    q = request.GET.get("q", "")
    if q and len(q) >= 3:
        clause = Q(dirtword__icontains=q)               \
               | Q(description__icontains=q)       \
               | Q(tags__name__icontains=q)
        site_search = Dirt.objects.filter(clause).distinct()
    else:
        site_search = Dirt.objects.order_by('?')[:100]
    return list_detail.object_list(
        request              = request,
        queryset             = site_search,
        template_name        = "dirty/search.html",
        template_object_name = "dirty",
        paginate_by          = 20,
        extra_context        = {"q" : q},
    )

def tag_cloud(request):
    #make tag cloud
    populartags = Tagling.objects.popular_tags()
    maxtags = []
    for tag in populartags:
        maxtags.append(tag.tagcount)
    #we need max count for the tagcloud
    maxtagcount = max(maxtags)
    html_captcha = colddirt.captcha.displayhtml(settings.RECAPTCHA_PUB_KEY)
    return render_to_response("dirty/tag_cloud.html", {
                                "populartags" : populartags,
                                "maxtagcount" : maxtagcount,
                                'html_captcha': html_captcha,
                                'form': DirtyForm().as_ul()})

def huh(request, template="dirty/huh.html",redirect_to="/"):
    from django.core.mail import send_mail
    contact_form = ContactForm()
    user = request.user
    description = ''
    status = ''
    subject = ''
    from_email = ''
    if request.method == 'POST':
        contact_form = ContactForm(request.POST)
        if contact_form.is_valid():
            description = contact_form.cleaned_data['description']
            status = contact_form.cleaned_data['status']
            subject = contact_form.cleaned_data['subject']
            from_email = contact_form.cleaned_data['from_email']
            send_mail('Cold Dirt ' + subject,
                      'Type: ' + status + description,
                      from_email,
                      ['kelvin@kelvinism.com'], fail_silently=True)
    return render_to_response(template, {'contact_form':contact_form.as_ul(),
                                      'description': description,
                                      'status': status,
                                      'subject': subject,
                                      'from_email': from_email,
                                      'user': user})

def report(request, report_id):
    # This is super ghetto, but works for now
    new_report = Report(item=report_id)
    new_report.save()
    return HttpResponseRedirect("/reported/success/")


#!bin/python3

import sympy
import sympy.physics.mechanics as mech
from sympy.physics.continuum_mechanics.truss import Truss
from cmath import inf
from sympy.core.add import Add
from sympy.core.mul import Mul
from sympy.core.symbol import Symbol
from sympy.core.sympify import sympify
from sympy import Matrix, pi
from sympy.functions.elementary.miscellaneous import sqrt
from sympy.matrices.dense import zeros
from sympy import sin, cos
import matplotlib.pyplot as plt
import matplotlib.colors as mcm
import labellines
import numpy as np
import os
import cv2
import copy
import pickle
from scipy.interpolate import interp1d
import pandas as pd

ff = os.path.join(os.path.dirname(__file__), 'frames')
if not os.path.isdir(ff):
    os.mkdir(ff)

class Truss2(Truss):

    def __init__(self):
        """
        Initializes the class
        """
        self._nodes = []
        self._members = {}
        self._loads = {}
        self._supports = {}
        self._node_labels = []
        self._node_positions = []
        self._node_position_x = []
        self._node_position_y = []
        self._nodes_occupied = {}
        self._reaction_loads = {}
        self._internal_forces = {}
        self._node_coordinates = {}
        self._history = {'time': [], 'max_comp': [], 'max_tens': []}
        self._sf = 3e3

    def add_node(self, label, x, y):
        """
        This method adds a node to the truss along with its name/label and its location.
        Parameters
        ==========
        label:  String or a Symbol
            The label for a node. It is the only way to identify a particular node.
        x: Sympifyable
            The x-coordinate of the position of the node.
        y: Sympifyable
            The y-coordinate of the position of the node.
        Examples
        ========
        >>> from sympy.physics.continuum_mechanics.truss import Truss
        >>> t = Truss()
        >>> t.add_node('A', 0, 0)
        >>> t.nodes
        [('A', 0, 0)]
        >>> t.add_node('B', 3, 0)
        >>> t.nodes
        [('A', 0, 0), ('B', 3, 0)]
        """
        x = sympify(x)
        y = sympify(y)

        if label in self._node_labels:
            raise ValueError("Node needs to have a unique label")

        elif x in self._node_position_x and y in self._node_position_y and self._node_position_x.index(x)==self._node_position_y.index(y):
            raise ValueError("A node already exists at the given position")

        else :
            self._nodes.append((label, x, y))
            self._node_labels.append(label)
            self._node_positions.append((x, y))
            self._node_position_x.append(x)
            self._node_position_y.append(y)
            self._node_coordinates[label] = [x, y]

    def remove_node(self, label):
        """
        This method removes a node from the truss.
        Parameters
        ==========
        label:  String or Symbol
            The label of the node to be removed.
        Examples
        ========
        >>> from sympy.physics.continuum_mechanics.truss import Truss
        >>> t = Truss()
        >>> t.add_node('A', 0, 0)
        >>> t.nodes
        [('A', 0, 0)]
        >>> t.add_node('B', 3, 0)
        >>> t.nodes
        [('A', 0, 0), ('B', 3, 0)]
        >>> t.remove_node('A')
        >>> t.nodes
        [('B', 3, 0)]
        """
        for i in range(len(self.nodes)):
            if self._node_labels[i] == label:
                x = self._node_position_x[i]
                y = self._node_position_y[i]

        if label not in self._node_labels:
            raise ValueError("No such node exists in the truss")

        else:
            members_duplicate = self._members.copy()
            for member in members_duplicate:
                if label == self._members[member][0] or label == self._members[member][1]:
                    raise ValueError("The node given has members already attached to it")
            self._nodes.remove((label, x, y))
            self._node_labels.remove(label)
            self._node_positions.remove((x, y))
            self._node_position_x.remove(x)
            self._node_position_y.remove(y)
            if label in list(self._loads):
                self._loads.pop(label)
            if label in list(self._supports):
                self._supports.pop(label)
            self._node_coordinates.pop(label)

    def change_node_label(self, label, new_label):
        """
        This method changes the label of a node.
        Parameters
        ==========
        label: String or Symbol
            The label of the node for which the label has
            to be changed.
        new_label: String or Symbol
            The new label of the node.
        Examples
        ========
        >>> from sympy.physics.continuum_mechanics.truss import Truss
        >>> t = Truss()
        >>> t.add_node('A', 0, 0)
        >>> t.add_node('B', 3, 0)
        >>> t.nodes
        [('A', 0, 0), ('B', 3, 0)]
        >>> t.change_node_label('A', 'C')
        >>> t.nodes
        [('C', 0, 0), ('B', 3, 0)]
        """
        if label not in self._node_labels:
            raise ValueError("No such node exists for the Truss")
        elif new_label in self._node_labels:
            raise ValueError("A node with the given label already exists")
        else:
            for node in self._nodes:
                if node[0] == label:
                    self._nodes[self._nodes.index((label, node[1], node[2]))] = (new_label, node[1], node[2])
                    self._node_labels[self._node_labels.index(node[0])] = new_label
                    self._node_coordinates[new_label] = self._node_coordinates[label]
                    self._node_coordinates.pop(label)
                    if node[0] in list(self._supports):
                        self._supports[new_label] = self._supports[node[0]]
                        self._supports.pop(node[0])
                    if new_label in list(self._supports):
                        if self._supports[new_label] == 'pinned':
                            if 'R_'+str(label)+'_x' in list(self._reaction_loads) and 'R_'+str(label)+'_y' in list(self._reaction_loads):
                                self._reaction_loads['R_'+str(new_label)+'_x'] = self._reaction_loads['R_'+str(label)+'_x']
                                self._reaction_loads['R_'+str(new_label)+'_y'] = self._reaction_loads['R_'+str(label)+'_y']
                                self._reaction_loads.pop('R_'+str(label)+'_x')
                                self._reaction_loads.pop('R_'+str(label)+'_y')
                            self._loads[new_label] = self._loads[label]
                            for load in self._loads[new_label]:
                                if load[1] == 90:
                                    load[0] -= Symbol('R_'+str(label)+'_y')
                                    if load[0] == 0:
                                        self._loads[label].remove(load)
                                    break
                            for load in self._loads[new_label]:
                                if load[1] == 0:
                                    load[0] -= Symbol('R_'+str(label)+'_x')
                                    if load[0] == 0:
                                        self._loads[label].remove(load)
                                    break
                            self.apply_load(new_label, Symbol('R_'+str(new_label)+'_x'), 0)
                            self.apply_load(new_label, Symbol('R_'+str(new_label)+'_y'), 90)
                            self._loads.pop(label)
                        elif self._supports[new_label] == 'roller':
                            self._loads[new_label] = self._loads[label]
                            for load in self._loads[label]:
                                if load[1] == 90:
                                    load[0] -= Symbol('R_'+str(label)+'_y')
                                    if load[0] == 0:
                                        self._loads[label].remove(load)
                                    break
                            self.apply_load(new_label, Symbol('R_'+str(new_label)+'_y'), 90)
                            self._loads.pop(label)
                    else:
                        if label in list(self._loads):
                            self._loads[new_label] = self._loads[label]
                            self._loads.pop(label)
                    for member in list(self._members):
                        if self._members[member][0] == node[0]:
                            self._members[member][0] = new_label
                            self._nodes_occupied[(new_label, self._members[member][1])] = True
                            self._nodes_occupied[(self._members[member][1], new_label)] = True
                            self._nodes_occupied.pop(tuple([label, self._members[member][1]]))
                            self._nodes_occupied.pop(tuple([self._members[member][1], label]))
                        elif self._members[member][1] == node[0]:
                            self._members[member][1] = new_label
                            self._nodes_occupied[(self._members[member][0], new_label)] = True
                            self._nodes_occupied[(new_label, self._members[member][0])] = True
                            self._nodes_occupied.pop(tuple([self._members[member][0], label]))
                            self._nodes_occupied.pop(tuple([label, self._members[member][0]]))

    def solve(self):
        """
        This method solves for all reaction forces of all supports and all internal forces
        of all the members in the truss, provided the Truss is solvable.
        A Truss is solvable if the following condition is met,
        2n >= r + m
        Where n is the number of nodes, r is the number of reaction forces, where each pinned
        support has 2 reaction forces and each roller has 1, and m is the number of members.
        The given condition is derived from the fact that a system of equations is solvable
        only when the number of variables is lesser than or equal to the number of equations.
        Equilibrium Equations in x and y directions give two equations per node giving 2n number
        equations. However, the truss needs to be stable as well and may be unstable if 2n > r + m.
        The number of variables is simply the sum of the number of reaction forces and member
        forces.
        .. note::
           The sign convention for the internal forces present in a member revolves around whether each
           force is compressive or tensile. While forming equations for each node, internal force due
           to a member on the node is assumed to be away from the node i.e. each force is assumed to
           be compressive by default. Hence, a positive value for an internal force implies the
           presence of compressive force in the member and a negative value implies a tensile force.
        Examples
        ========
        >>> from sympy.physics.continuum_mechanics.truss import Truss
        >>> t = Truss()
        >>> t.add_node("node_1", 0, 0)
        >>> t.add_node("node_2", 6, 0)
        >>> t.add_node("node_3", 2, 2)
        >>> t.add_node("node_4", 2, 0)
        >>> t.add_member("member_1", "node_1", "node_4")
        >>> t.add_member("member_2", "node_2", "node_4")
        >>> t.add_member("member_3", "node_1", "node_3")
        >>> t.add_member("member_4", "node_2", "node_3")
        >>> t.add_member("member_5", "node_3", "node_4")
        >>> t.apply_load("node_4", magnitude=10, direction=270)
        >>> t.apply_support("node_1", type="pinned")
        >>> t.apply_support("node_2", type="roller")
        >>> t.solve()
        >>> t.reaction_loads
        {'R_node_1_x': 0, 'R_node_1_y': 20/3, 'R_node_2_y': 10/3}
        >>> t.internal_forces
        {'member_1': 20/3, 'member_2': 20/3, 'member_3': -20*sqrt(2)/3, 'member_4': -10*sqrt(5)/3, 'member_5': 10}
        """
        count_reaction_loads = 0
        for node in self._nodes:
            if node[0] in list(self._supports):
                if self._supports[node[0]]=='pinned':
                    count_reaction_loads += 2
                elif self._supports[node[0]]=='roller':
                    count_reaction_loads += 1
        if 2*len(self._nodes) != len(self._members) + count_reaction_loads:
            raise ValueError("The given truss cannot be solved")
        coefficients_matrix = [[0 for i in range(2*len(self._nodes))] for j in range(2*len(self._nodes))]
        load_matrix = zeros(2*len(self.nodes), 1)
        load_matrix_row = 0
        for node in self._nodes:
            if node[0] in list(self._loads):
                for load in self._loads[node[0]]:
                    if load[0] != Symbol('R_'+str(node[0])+'_x') and load[0] != Symbol('R_'+str(node[0])+'_y'):
                        load_matrix[load_matrix_row] -= load[0]*cos(pi*load[1]/180)
                        load_matrix[load_matrix_row + 1] -= load[0]*sin(pi*load[1]/180)
            load_matrix_row += 2
        cols = 0
        row = 0
        for node in self._nodes:
            if node[0] in list(self._supports):
                if self._supports[node[0]]=='pinned':
                    coefficients_matrix[row][cols] += 1
                    coefficients_matrix[row+1][cols+1] += 1
                    cols += 2
                elif self._supports[node[0]]=='roller':
                    coefficients_matrix[row+1][cols] += 1
                    cols += 1
            row += 2
        for member in list(self._members):
            start = self._members[member][0]
            end = self._members[member][1]
            length = sqrt((self._node_coordinates[start][0]-self._node_coordinates[end][0])**2 + (self._node_coordinates[start][1]-self._node_coordinates[end][1])**2)
            start_index = self._node_labels.index(start)
            end_index = self._node_labels.index(end)
            horizontal_component_start = (self._node_coordinates[end][0]-self._node_coordinates[start][0])/length
            vertical_component_start = (self._node_coordinates[end][1]-self._node_coordinates[start][1])/length
            horizontal_component_end = (self._node_coordinates[start][0]-self._node_coordinates[end][0])/length
            vertical_component_end = (self._node_coordinates[start][1]-self._node_coordinates[end][1])/length
            coefficients_matrix[start_index*2][cols] += horizontal_component_start
            coefficients_matrix[start_index*2+1][cols] += vertical_component_start
            coefficients_matrix[end_index*2][cols] += horizontal_component_end
            coefficients_matrix[end_index*2+1][cols] += vertical_component_end
            cols += 1
        forces_matrix = (Matrix(coefficients_matrix)**-1)*load_matrix
        self._reaction_loads = {}
        i = 0
        min_load = inf
        for node in self._nodes:
            if node[0] in list(self._loads):
                for load in self._loads[node[0]]:
                    if type(load[0]) not in [Symbol, Mul, Add]:
                        min_load = min(min_load, load[0])
        for j in range(len(forces_matrix)):
            if type(forces_matrix[j]) not in [Symbol, Mul, Add]:
                if abs(forces_matrix[j]/min_load) <1E-10:
                    forces_matrix[j] = 0
        for node in self._nodes:
            if node[0] in list(self._supports):
                if self._supports[node[0]]=='pinned':
                    self._reaction_loads['R_'+str(node[0])+'_x'] = forces_matrix[i]
                    self._reaction_loads['R_'+str(node[0])+'_y'] = forces_matrix[i+1]
                    i += 2
                elif self._supports[node[0]]=='roller':
                    self._reaction_loads['R_'+str(node[0])+'_y'] = forces_matrix[i]
                    i += 1
        for member in list(self._members):
            self._internal_forces[member] = forces_matrix[i]
            i += 1
        return

    def get_member_types(self):
        self.member_types = {i: 'road' if i.find('road') > -1 else 'beam' for i in self.members}

    def get_member_lengths(self):
        def rel_f(l):
            if l < 1.847:
                return 1. - 0.14633 * l**2
            else:
                return 1.7094 / l**2
        self.member_lengths = {i: ((self._node_position_x[self._node_labels.index(self.members[i][0])] -
                                    self._node_position_x[self._node_labels.index(self.members[i][1])]) ** 2 +
                                   (self._node_position_y[self._node_labels.index(self.members[i][0])] -
                                    self._node_position_y[self._node_labels.index(self.members[i][1])]) ** 2
                                   ) ** 0.5 for i in self.members}
        self.compressive_weaknesses = {i[0]: rel_f(i[1]) for i in self.member_lengths.items()}
        # wood: E = 6.9 GPa, density = 0.7 g/cc, sigy = 34.5 MPa
        # calculate critical length
        # kg m s N Pa J
        # calculate critical load(length)
        # mass / m (if aluminum, 5cm by 5cm square rod, = 6.75 kg / m ??? (5 cm ~ 2 in
        # density = 2.7 g / cm**3  (
        #  A = 25cm**2 (2.5e-3 m**2)
        # I = 52.08 cm**4 = 5.21e-7 m**4
        # R = (I/A)**0.5 = 1.44 cm 1.44e-2 m
        # E = 69 GPa (6.9e10 Pa)
        # Sigma_yield = 83 MPa (8.3e7 Pa)
        # Scrit = 128
        # L crit = 1.84704 m
        # Tensile Failure: 8.3e7*2.5e-3 = 207.5 kN
        # Johnson's Formula (short columns): F = A*sig* [1-(sig/E)*(L/(2*pi*R))**2]
        # F = 207.5 kN * [ 1 - 0.14633 * L**2]
        # Euler's Formula (above L = 1.847 m) F = pi**2 * E * I/ L**2
        # F = 6.9e10 * pi**2 * 5.208e-7 /L**2
        # F = 1.7094 * 207.5 kN / L**2
        # self.member_buckling_loads = {i[0]: i[1]}

    def get_member_masses(self, masses):
        self.member_masses = {i: self.member_lengths[i] * masses[self.member_types[i]] for i in self.members}

    def get_node_masses(self, masses):
        self.node_masses = {i[0]: masses['node'] for i in self.nodes}

    def update_history(self, time):
        self._history['time'].append(time)
        mt = max(self._internal_forces.values())
        mc = min([self._internal_forces[i]/self.compressive_weaknesses[i] for i in self._internal_forces])
        self._history['max_comp'].append(-mc)
        self._history['max_tens'].append(mt)

    def plotter(self, time=0, name='', group=''):
        scale = 10000
        cdict = {'red': [(0.0, 0.0, 0.0),
                         (0.5, 0.0, 0.0),
                         (1.0, 1.0, 1.0)],

                 'green': [(0.0, 0.0, 0.0),
                           (1.0, 0.0, 0.0)],

                 'blue': [(0.0, 1.0, 1.0),
                          (0.5, 0.0, 0.0),
                          (1.0, 0.0, 0.0)]}
        bx = [[float(self._node_position_x[self._node_labels.index(self.members[i][0])]),
               float(self._node_position_x[self._node_labels.index(self.members[i][1])])]
              for i in self.members]
        rbx = [[float(self._node_position_x[self._node_labels.index(self.members[i][0])]),
               float(self._node_position_x[self._node_labels.index(self.members[i][1])])]
              for i in self.members if i.find('road') > -1]
        xmin = np.min(rbx)
        xmax = np.max(rbx)
        by = [[float(self._node_position_y[self._node_labels.index(self.members[i][0])]),
               float(self._node_position_y[self._node_labels.index(self.members[i][1])])]
              for i in self.members]
        rx = [float(i[1]) for i in self.nodes if i[0].find('road') > -1]
        ry = [float(i[2]) for i in self.nodes if i[0].find('road') > -1]
        ymin, ymax = [f(by) for f in [np.min, np.max]]
        dn = mcm.TwoSlopeNorm(vcenter=0, vmin=-1*self._sf,
                              vmax=self._sf)
        cmap = mcm.LinearSegmentedColormap('stress', cdict)

        colors = [cmap(dn(float(i[1]))) if float(i[1]) > 0 else
                  cmap(dn(float(i[1])/float(self.compressive_weaknesses[i[0]])))
                  for i in self.internal_forces.items()]
        labs = ['%.0f' % self.internal_forces[i] for i in self.members]
        fig, ax = plt.subplots(2, 1, figsize=[12, 12])
        plt.sca(ax[0])
        plt.plot(self._history['time'], self._history['max_comp'], 'b-', label='Compression')
        plt.plot(self._history['time'], self._history['max_tens'], 'r-', label='Tension')
        plt.plot([0, 1], [self._sf, self._sf], 'k:', label='Load Limit')
        plt.xlim((0, 1))
        plt.ylim((0, 2e4))
        props = dict(boxstyle='round', facecolor='white', alpha=0.5)
        maxf = max([max(self._history['max_comp']), max(self._history['max_tens'])]) / self._sf
        textstr = "Maximum Stress Reached %.4f" % maxf + r' $\times$ design limit'
        # place a text box in upper left in axes coords
        ax[0].text(0.05, 0.95, textstr, transform=ax[0].transAxes, fontsize=14,
                verticalalignment='top', bbox=props)
        if time > 0:
            labellines.labelLines(plt.gca().get_lines(), xvals=[time*i for i in [0.2, 0.6, 0.5/time]])
        plt.sca(ax[1])
        [plt.plot(x, y, '-', c=c, label=l) for x, y, c, l in zip(bx, by, colors, labs)]
        massx = time*xmax
        f = interp1d(rx, ry)
        massy = f(massx)+0.2*ymax
        labellines.labelLines(plt.gca().get_lines(), xvals=[np.mean(bi) for bi in bx])
        plt.scatter(massx, massy, marker=r'$\Phi$', c='k', s=70)
        plt.plot(np.array(bx).T, np.array(by).T, 'ok')

        for i in self.reaction_loads:
            if self.reaction_loads[i] == 0:
                continue
            ni = self._node_labels.index(i[2:-2])
            x, y = self._node_positions[ni]
            dir = np.sign(self.reaction_loads[i])
            if i[-2:].find('_x'):
                # horiz
                dx = dir * 0.2
                dy = 0
            elif i[-2:].find('_y'):
                # vert
                dx = 0
                dy = dir * 0.2
            w = self.reaction_loads[i]/scale
            plt.arrow(x, y, dy, dx, width=w,
                      head_width=w*1.5, head_length=w*0.5)
        for i in self.loads:
            for ii in self.loads[i]:
                mags = []
                dirs = []
                if isinstance(ii[0], sympy.core.numbers.Float):
                    mags.append(ii[0])
                    dirs.append(ii[1])
            dx = [0.2 * np.sin(np.radians(float(d))) for d in dirs]
            dy = [0.2 * np.cos(np.radians(float(d))) for d in dirs]
            ww = np.divide(mags, sum(mags))
            dx = np.multiply(dx, ww).sum()
            dy = np.multiply(dy, ww).sum()

            x, y = self._node_positions[self._node_labels.index(i)]
            w = float(np.sum(mags)/scale)
            plt.arrow(float(x), float(y), float(dy), float(dx), width=w,
                          head_width=w * 1.5, head_length=w * 0.5, fc='green')
        if (ymax-ymin)*2 > xmax - xmin:
            buf = (ymax - ymin)*2 - (xmax - xmin)
        else: buf = 0.5
        plt.xlim((float(xmin)-buf, float(xmax)+buf))
        plt.ylim((float(ymin)-buf/2, float(ymin) + float(xmax)/2 + buf/2))
        plt.title(name + ' ' + group)
        plt.savefig(os.path.join(ff, '%04i.png' % (time*200)), dpi=120)
        plt.close()

def make_bridge(length=8, segments=4, height=3):
    tt = Truss2()
    # build road
    for i in range(segments + 1):
        tt.add_node('road_joint_%i' % i, i * length/segments, 0)
    for i in range(1, segments + 1):
        tt.add_node('truss_node_%i' % i, (i - 0.5) * length/segments, height)
        tt.add_member('beam_%ia' % i, 'road_joint_%i' % (i-1), 'truss_node_%i' % i)
        tt.add_member('beam_%ib' % i, 'road_joint_%i' % i, 'truss_node_%i' % i)
    for i in range(2, segments + 1):
        tt.add_member('beam_%ic' % i, 'truss_node_%i' % (i - 1), 'truss_node_%i' % i)
    for i in range(1, segments + 1):
        tt.add_member('road_section_%i' % i, 'road_joint_%i' % (i-1), 'road_joint_%i' % i)
    tt.apply_support('road_joint_0', 'pinned')
    tt.apply_support('road_joint_%i' % segments, 'roller')
    return tt

def make_f_bridge(length=8, segments=4, f_y_road=0, f_y_pins=3, f_x_pins=0):
    """This is one approach to defining node positions by function.

    This function will always place one less horizontal support beam than road segments.
    You can make much better bridges if you discover other valid truss designs...
    Parameters:
        length: float or int
        segments: int
        f_y_road: int, float, or Callable
        f_y_pins: int, float, or Callable
        f_x_pins: int, float, or Callable

        Any Callable passed to the parameters above needs to have the signature:
         f(x, xmax) -> float or int
         where x is the current x position along the bridge (assuming equal spacing)
         and xmax is the length of the bridge."""
    tt = Truss2()
    if isinstance(f_y_road, (int, float)):
        val = f_y_road

        def f_y_road(x, xmax):
            return val

    if isinstance(f_y_pins, (int, float)):
        val = f_y_pins

        def f_y_pins(x, xmax):
            return val

    if isinstance(f_x_pins, (int, float)):
        val = f_x_pins
        def f_x_pins(x, xmax):
            return val + x

    # build road
    for i in range(segments + 1):
        tt.add_node('road_joint_%i' % i, i * length/segments, f_y_road(i * length/segments, length))
    for i in range(1, segments + 1):
        xp = f_x_pins(i * length/segments, length)
        tt.add_node('truss_node_%i' % i, xp, f_y_pins(xp, length))
        tt.add_member('beam_%ia' % i, 'road_joint_%i' % (i-1), 'truss_node_%i' % i)
        tt.add_member('beam_%ib' % i, 'road_joint_%i' % i, 'truss_node_%i' % i)
    for i in range(2, segments + 1):
        tt.add_member('beam_%ic' % i, 'truss_node_%i' % (i - 1), 'truss_node_%i' % i)
    for i in range(1, segments + 1):
        tt.add_member('road_section_%i' % i, 'road_joint_%i' % (i-1), 'road_joint_%i' % i)
    tt.apply_support('road_joint_0', 'pinned')
    tt.apply_support('road_joint_%i' % segments, 'roller')
    return tt

def apply_self_load(truss, masses):
    g = 9.81 #(m/s**2)
    truss.get_member_lengths()
    truss.get_member_types()
    truss.get_member_masses(masses)
    truss.get_node_masses(masses)
    efm = {i[0]: truss.node_masses[i[0]] for i in truss.nodes}
    for member in truss.members:
        a, b = truss.members[member]
        efm[a] += truss.member_masses[member] / 2
        efm[b] += truss.member_masses[member] / 2
    for node in truss._node_labels:
        truss.apply_load(node, magnitude=efm[node]*g, direction=270)
    return

def lever_rule(fun, road, pos, v):
    g = 9.81
    keylist = list(road.keys())
    dist = {i: abs(road[i][1]-pos) for i in keylist}

    dl2 = [dist[i] for i in dist]
    dl2.sort()
    lim = max(dl2[0:2])
    nodes = {i: dist[i] for i in dist if dist[i] <= lim}
    nl = [(i[0], i[1]) for i in nodes.items()]
    ws = {nl[i][0]: nl[-(i+1)][1]/sum(dl2[0:2]) for i in range(len(nodes))}
    # w0 = d1/(d0+d1)
    # w1 = d0/(d0+d1)
    load = g * v
    for i in ws:
        fun(road[keylist[i]][0], load * ws[i], 270)
    return

def apply_moving_loads(truss, t, v, tprev=None):
    road_pins = [i for i in truss._node_labels if i.find('road') > -1]
    road_pin_numbers = [int(i.split('_')[-1]) for i in road_pins]
    road_nodes = {i[0]: truss.nodes[truss._node_labels.index(i[1])] for i in zip(road_pin_numbers, road_pins)}
    road_xvals = [i[1] for i in road_nodes.values()]
    load_pos = t * (max(road_xvals) - min(road_xvals))
    if tprev is None:
        pass
    else:
        prev_pos = tprev * (max(road_xvals) - min(road_xvals))
        lever_rule(truss.remove_load, road_nodes, prev_pos, v)
    lever_rule(truss.apply_load, road_nodes, load_pos, v)
    return

def save_truss(tt, name=None, group=None):
    if name is None:
        name = 'truss' + input("What # round is this? ")
    if group is None:
        group = input("What is your group name? ")
    h = open(name + group + '.pickle', 'wb')
    pickle.dump(tt, h)
    h.close()

def smart_pfind(path):
    ll = os.listdir(path)
    ll2 = [li for li in ll if li.find('.pickle') > -1]
    return ll2[int(input('Pick by number:\n' +
                         '\n'.join(['%i: %s' % (i, li) for i, li in enumerate(ll2)])+'\n'))]

def get_truss(name=None):
    if name is None:
        name = smart_pfind('.')
    h = open(name, 'rb')
    tt = pickle.load(h)
    h.close()
    return tt

def truss_simulator(truss, moving_mass=50, fixed_loads=None, ligament_masses=None, tsteps=None, fps=2, show=True, name=None, group=None):
    """Different Rounds may have different lengths, number of segments, or arbitrary requirements on shape (truss below road,
    truss above road, """
    if name is None:
        name = 'truss' + input("What # round is this? ")
    if group is None:
        group = input("What is your group name? ")
    if fixed_loads is None:
        fixed_loads = {}
    if ligament_masses is None:
        ligament_masses = {'road': 20,  # kg / m
                           'beam': 4,  # kg / m
                           'node': 2}  # kg
    if tsteps is None:
        rnodes = len([i for i in truss._node_labels if i.find('road') > -1])
        tsteps = np.linspace(0, 1, 2*rnodes-1)
    apply_self_load(truss, ligament_masses)
    apply_moving_loads(truss, tsteps[0], moving_mass)
    truss.solve()
    truss.update_history(tsteps[0])
    truss.plotter(tsteps[0], name=name, group=group)
    for ii, t in enumerate(tsteps[1:]):
        oldloads = copy.deepcopy(truss.loads)
        apply_moving_loads(truss, t, moving_mass, tsteps[ii])
        changes = ['%s: %s -> %s' % (i, oldloads[i][-1], truss.loads[i][-1]) for i in truss.loads if oldloads[i] != truss.loads[i]]
        print(ii + 1, t, tsteps[ii])
        print(*changes, sep='\n')
        truss.solve()
        truss.update_history(t)
        truss.plotter(t, name=name, group=group)
    if show:
        make_video(fps, )
        if input('Save this result? (y/N) ').lower() in ['y', 'yes', 'ye']:
            save_truss(truss, name, group)

def make_video(fps, vn='video.mp4'):
    # image_folder = 'frames'
    video_name = vn
    images = [img for img in os.listdir(ff) if img.endswith(".png")]
    images.sort()
    frame = cv2.imread(os.path.join(ff, images[0]))
    height, width, layers = frame.shape
    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
    video = cv2.VideoWriter(video_name, fourcc, fps, (width, height))

    for image in images:
        video.write(cv2.imread(os.path.join(ff, image)))

    cv2.destroyAllWindows()
    video.release()


import logging
from pathlib import Path
from typing import Literal

from pydantic_settings import BaseSettings

class Settings(BaseSettings):
    # ---------------------------------------------------
    # Startup parameters
    # ---------------------------------------------------
    restore_persisted_state: bool = True
    log_level: int = logging.DEBUG

    # ---------------------------------------------------
    # CORS
    # ---------------------------------------------------
    cors_origins: list = ["*"]

    # ---------------------------------------------------
    # Redis
    # ---------------------------------------------------
    redis_host: str = "localhost"
    redis_port: int = 6379

    # ---------------------------------------------------
    # Travel
    # ---------------------------------------------------
    travel_tick_seconds: float = 0.3

    # ---------------------------------------------------
    # Sonar
    # ---------------------------------------------------
    serenity_name: str = "Serenity"
    serenity_hp: int = 3
    sonar_starting_distance: int = 8
    sonar_map_width: int = 15
    sonar_map_height: int = 15
    sonar_map_file_name: Literal["default"] = "default"
    sonar_spawn_margin: int = 2
    sonar_torpedo_damage: int = 2
    sonar_torpedo_reach: int = 4
    sonar_torpedo_radius: int = 2
    sonar_mine_damage: int = 2
    sonar_mine_reach: int = 4
    sonar_mine_radius: int = 2
    sonar_player_default_hp: int = 4
    sonar_use_control_panel: bool= True

    # ---------------------------------------------------
    # Paths
    # ---------------------------------------------------
    package_dir: Path = Path(__file__).parents[1]
    planetary_config_path: Path = package_dir / "defaults/planet_graph.json"
    asteroid_map_dir: Path = package_dir / "defaults/maps/"
    switches_path: Path = package_dir / "defaults/switches.json"

    class Config:
        env_prefix = "serenity_"
        env_file = ".env"
        env_file_encoding = "utf-8"

settings = Settings()


import json

from time import sleep
from threading import Thread
from logger import Logger
log = Logger.getInstance().getLogger()

class DeviceEvents(Thread):

    def __init__(self,
                 lock,
                 event_queue,
                 device_hash):
        Thread.__init__(self)
        self.setDaemon(True)
        self.lock = lock
        self.event_queue = event_queue
        self.device_hash = device_hash

    def process_entry(self, msg) -> int:
        topic = msg.topic

        if topic not in self.device_hash:
            # log.debug(("{} is not "
            #           "in device hash").format(topic))
            return 0

        payload = json.loads(msg.payload.decode("utf-8"))
        self.device_hash[topic].refresh_device_data(payload)
        device = self.device_hash[topic]
        log.debug("\n{0}\n".format(device))

        rc = 0
        try:
            rc = device.process()
        except Exception as e:
            log.error(e)
            rc = -1

        return rc

    def process_events(self):
        if not self.event_queue.empty():
            self.lock.acquire()
            msg = self.event_queue.get()
            self.lock.release()
            rc = self.process_entry(msg)
            if rc != 0:
                log.error("Failed to process entry {0}".format(
                    rc
                ))

    def run(self):
        while True:
            if self.event_queue:
                self.process_events()
            sleep(1)


import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

# Define the image name
inputimage = 'static/img/1.tif'
# Load the imagef
img = Image.open(inputimage)

img = np.asarray(img)
img2 = img
for i in range(3):
    img2 = np.concatenate((img2, np.rot90(img, i+1)), axis=1)
# Prepare figure window
plt.figure()
# Show image on the window
plt.imshow(img2, cmap='gray')
# Show the window
plt.show()


'''Automatic preparation, submission and consolidation
of AWS batch tasks; as a single Luigi Task.
'''

from abc import ABC
from abc import abstractmethod
from collections import defaultdict
from nesta.core.luigihacks import batchclient
from subprocess import check_output
from subprocess import CalledProcessError
import time
from datetime import datetime as dt
import luigi
from nesta.core.luigihacks.misctools import get_config
import logging
import os

# Define a global timeout, set to 95% of the timeout time
# in order to give the Luigi worker some grace
_config = get_config("luigi.cfg", "worker")

def command_line(command, verbose=False):
    '''Execute command line tasks and return the final output line.
    This is particularly useful for extracting the AWS access keys
    directly from the OS; as well as executing the environment
    preparation script (:code:`core/scripts/nesta_prepare_batch.sh`).
    '''
    # Execute the command and decode the output
    out = check_output([command], shell=True)
    out_lines = out.decode("utf-8").split("\n")
    if verbose:
        for line in out_lines:
            logging.info(f"{os.getpid()}: "
                         ">>>\t'{}'".format(line.replace("\r", ' ')))
    # The second last output is the actual final output
    # (ignoring the status code, which is the last output)
    return out_lines[-2]

class AutoBatchTask(luigi.Task, ABC):
    '''A base class for automatically preparing and submitting AWS batch tasks.

    Unlike regular Luigi :code:`Tasks`, which require the user
    to override the :code:`requires`, :code:`output` and :code:`run`
    methods, :code:`AutoBatchTask` instead effectively replaces
    :code:`run` with two new abstract methods: :code:`prepare`
    and :code:`combine`, which are repectively documented. With these abstract
    methods specified, :code:`AutoBatchTask` will automatically prepare,
    submit, and combine one batch task (specified in
    :code:`core.batchables`) per parameter set specified in the
    :code:`prepare` method. The :code:`combine` method will subsequently
    combine the outputs from the batch task.

    Args:
        batchable (str): Path to the directory containing the run.py batchable
        job_def (str): Name of the AWS job definition
        job_name (str): Name given to this AWS batch job
        job_queue (str): AWS batch queue
        region_name (str): AWS region from which to batch
        env_files (:obj:`list` of :obj:`str`, optional): List of names
                  pointing to local environmental files (for example local
                  imports or scripts) which should be zipped up with the
                  AWS batch job environment. Defaults to [].
        vcpus (int, optional): Number of CPUs to request for the AWS batch job.
              Defaults to 1.
        memory (int, optional): Memory to request for the AWS batch job.
               Defaults to 512 MiB.
        max_runs (int, optional): Number of batch jobs to run, which is useful
                 for testing a subset of the full pipeline, or making cost
                 predictions for AWS computing time. Defaults to `None`,
                 implying that all jobs should be run.
        poll_time (int, optional): Time in seconds between querying the AWS
                  batch job status. Defaults to 60.
        success_rate (float, optional): If the fraction of FAILED jobs exceeds
                     :code:`success_rate` then the entire Task, along with
                     any submitted AWS batch jobs, is killed. The fraction is
                     calculated with respect to any jobs with RUNNING,
                     SUCCEEDED or FAILED status. Defaults to 0.75.
    '''
    batchable = luigi.Parameter()
    job_def = luigi.Parameter()
    job_name = luigi.Parameter()
    job_queue = luigi.Parameter()
    region_name = luigi.Parameter()
    env_files = luigi.ListParameter(default=[])
    vcpus = luigi.IntParameter(default=1)
    memory = luigi.IntParameter(default=512)
    max_runs = luigi.IntParameter(default=None)  # For testing
    timeout = luigi.IntParameter(default=21600)
    poll_time = luigi.IntParameter(default=60)
    success_rate = luigi.FloatParameter(default=0.95)
    test = luigi.BoolParameter(default=True)
    max_live_jobs = luigi.IntParameter(default=25)
    worker_timeout = float('inf')
    date = luigi.DateParameter(default=dt.now())

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.failed_jobs = set()

    def run(self):
        '''DO NOT OVERRIDE THIS METHOD.

        An implementation of the :code:`Luigi.Task.run` method which is a
        wrapper around the :code:`prepare`, :code:`execute` and
        :code:`combine` methods. Instead of overriding this method, you
        should implement :code:`prepare` and :code:`combine` methods in
        your class.
        '''
        pid = os.getpid()
        self.TIMEOUT = time.time() + int(_config["timeout"])

        # Generate the parameters for batches
        job_params = self.prepare()
        if self.test:
            if len(job_params) > 2:
                job_params = job_params[0:2]
                logging.info(f"Test mode ({pid}): running {len(job_params)} jobs")

        # Prepare the environment for batching
        env_files = " ".join(self.env_files)
        try:
            if self.test:
                logging.info(f"Test mode ({pid}): Preparing batch")
            s3file_timestamp = command_line("nesta_prepare_batch "
                                            "{} {}".format(self.batchable,
                                                           env_files), self.test)
            if self.test:
                logging.info(f"Test mode ({pid}): Prepared batch")
        except CalledProcessError:
            raise batchclient.BatchJobException("Invalid input "
                                                "or environment files")
        # Execute batch jobs
        self.execute(job_params, s3file_timestamp)
        # Combine the outputs
        self.combine(job_params)

    @abstractmethod
    def prepare(self):
        '''You should implement a method which returns a :code:`list`
        of :code:`dict`, where each :code:`dict` corresponds to inputs
        to the batchable. Each row of the output must at least
        contain the following keys:

        - **done** (`bool`): indicating whether the job has already been
          finished.
        - **outinfo** (`str`): Text indicating e.g. the location of the output,
          for use in the batch job and for `combine` method

        Returns:
            :obj:`list` of :obj:`dict`
        '''
        pass

    @abstractmethod
    def combine(self, job_params):
        '''You should implement a method which collects the outputs specified by
        the **outinfo** key of :code:`job_params`, which is the output from the
        :code:`prepare` method. This method should finally write to the
        :code:`luigi.Target` output.

        Parameters:
            job_params (:obj:`list` of :obj:`dict`): The batchable job
                       parameters, as returned from the :code:`prepare` method.
        '''
        pass

    def execute(self, job_params, s3file_timestamp):
        ''' The secret sauce, which automatically submits and monitors
        the AWS batch jobs. Your AWS access key and id are automatically
        retrieved via the AWS CLI.

        Parameters:
            job_params (:obj:`list` of :obj:`dict`): The batchable job
                       parameters, as returned from the :code:`prepare` method.
                       Each job is submitted from every item in this
                       :code:`list`. Each `dict` key-value per is converted
                       into an environmental variable in the batch job, with
                       the variable
                       name formed from the key, prefixed by `BATCHPAR_`.
            s3file_timestamp (str): The timestamp of the batchable zip file
                             to be found on S3 by the AWS batch job.
        '''
        pid = os.getpid()
        # Get AWS info to pass to the batch jobs
        aws_id = command_line("aws --profile default configure "
                              "get aws_access_key_id", self.test)
        aws_secret = command_line("aws --profile default configure "
                                  "get aws_secret_access_key", self.test)

        # Build a set of environmental variables to send to the jobs
        env_variables = [{"name": "AWS_ACCESS_KEY_ID", "value": aws_id},
                         {"name": "AWS_SECRET_ACCESS_KEY",
                          "value": aws_secret},
                         {"name": "BATCHPAR_S3FILE_TIMESTAMP",
                          "value": s3file_timestamp}]
                         #{"name": "PYTHONIOENCODING", "value": "latin1"}]

        if self.test:
            logging.info(f"Test mode ({pid}): Got env variables")

        # Set up batch client, and check that we haven't
        # already hit the time limit
        batch_client = batchclient.BatchClient(poll_time=self.poll_time,
                                               region_name=self.region_name)
        self._assert_timeout(batch_client, job_ids=[])
        if self.test:
            logging.info(f"Test mode ({pid}): Ready to batch")

        all_job_kwargs = []
        for i, params in enumerate(job_params):
            if params["done"]:
                continue
            # Break in case of testing
            if (self.max_runs is not None) and (i >= self.max_runs):
                break

            _env_variables = env_variables.copy()
            for k, v in params.items():
                new_row = dict(name="BATCHPAR_{}".format(k), value=str(v))
                _env_variables.append(new_row)
            # Add the environmental variables to the container overrides
            overrides = dict(environment=_env_variables,
                             memory=self.memory, vcpus=self.vcpus)
            job_kwargs = dict(jobDefinition=self.job_def,
                              jobName=self.job_name,
                              jobQueue=self.job_queue,
                              timeout=dict(attemptDurationSeconds=self.timeout),
                              containerOverrides=overrides)
            all_job_kwargs.append(job_kwargs)

        # Wait for jobs to finish
        self._run_batch_jobs(batch_client, all_job_kwargs)

    def _run_batch_jobs(self, batch_client, all_job_kwargs):
        '''Monitor AWS batch jobs until finished or failed.

        Parameters:
            batch_client (:obj:`BatchClient`)
            job_ids (:obj:`list` of :obj:`str`): List of AWS batch
                    job IDs to monitor.
        '''

        # Keep submitting until all submitted
        all_job_ids = set()
        done_job_ids = set()
        submitted_job_idxs = set()
        logging.info(f"{os.getpid()}: "
                     "{} jobs to run".format(len(all_job_kwargs)))
        while len(all_job_kwargs) > len(all_job_ids):
            # Get the number of live jobs
            running_job_ids = all_job_ids - done_job_ids
            n_live = len(running_job_ids)
            n_done = len(done_job_ids)
            n_left = len(all_job_kwargs) - n_done - n_live
            logging.info(f"{os.getpid()}: "
                         "{} jobs are live, "
                         "{} are finished, "
                         "and {} are yet to be submitted".format(n_live, n_done, n_left))

            if n_live > 1:
                self._assert_timeout(batch_client, running_job_ids)
                self._assert_success(batch_client, all_job_ids, done_job_ids)
            # Submit some jobs until `self.max_live_jobs` reached
            for ijob, job_kwargs in enumerate(all_job_kwargs):
                if n_live >= self.max_live_jobs:
                    break
                if ijob in submitted_job_idxs:
                    continue
                # Submit a new job
                id_ = batch_client.submit_job(**job_kwargs)
                all_job_ids.add(id_)
                submitted_job_idxs.add(ijob)
                n_live += 1
            # Wait before continuing
            logging.info(f"{os.getpid()}: Not done submitting...")
            time.sleep(self.poll_time)

        # Wait until all finished
        running_job_ids = all_job_ids - done_job_ids
        while len(running_job_ids) > 0:
            running_job_ids = all_job_ids - done_job_ids
            self._assert_timeout(batch_client, running_job_ids)
            self._assert_success(batch_client, all_job_ids, done_job_ids)
            # Wait before continuing
            #logging.info("Not finished waiting...")
            time.sleep(self.poll_time)

    def _assert_success(self, batch_client, job_ids, done_jobs):
        '''Assert that success rate has not been breached.'''

        stats = defaultdict(int)  # Collection of failure vs total statistics
        # Check status for each job
        for id_ in job_ids:
            status = batch_client.get_job_status(id_)
            if id_ not in done_jobs:
                logging.debug(f"{os.getpid()}: "
                              "{} {}".format(id_, status))
            if status == "FAILED":
                self.failed_jobs.add(id_)
            if status not in ("SUCCEEDED", "FAILED", "RUNNING"):
                continue
            stats[status] += 1
            if status != "RUNNING":
                done_jobs.add(id_)

        # Ignore if jobs are simply stalling
        if len(stats) == 0:
            logging.info(f"{os.getpid()}: "
                         "No jobs are currently running")
            return

        # Calculate the failure rate
        total = sum(stats.values())
        failure_rate = stats["FAILED"] / total
        if failure_rate <= (1 - self.success_rate):
            return
        reason = "Exiting due to high failure rate: {}%".format(int(failure_rate*100))
        reason += "\nFailed jobs are: {}".format(self.failed_jobs)
        batch_client.hard_terminate(job_ids=job_ids, reason=reason)

    def _assert_timeout(self, batch_client, job_ids):
        '''Assert that timeout has not been breached.'''
        logging.info(f"{os.getpid()}: "
                     "{} seconds left".format(self.TIMEOUT - time.time()))
        if time.time() < self.TIMEOUT:
            return
        reason = f"{os.getpid()}: "
        reason += "Impending worker timeout, so killing live tasks"
        reason += "\nFailed jobs are: {}".format(self.failed_jobs)
        batch_client.hard_terminate(job_ids=job_ids, reason=reason)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jun 18 20:49:41 2020

@author: chenshihti
"""

import requests #引入api函數
import json
#想要爬資料的 標網址
r= requests.get('https://www.dcard.tw/_api/forums/job/posts?popular=true')

response1=r.text
response2=json.loads(r.text)

for i in response2:
    print (i['excerpt'])


import time
import requests
from scrapy import Selector

headers = {
    "Accept": 'text/html, application/xhtml+xml, application/xml; q=0.9, */*; q=0.8',
    # "Accept-Encoding": 'gzip, deflate, br',
    "Accept-Language": 'zh-CN,zh;q=0.9',
    "Cache-Control": 'max-age=0',
    "Cookie": 'cf_clearance=ea2b3e04bded706ff7978fbd92b61bf163c91469-1579604232-0-250; __cfduid=d5da5233bff7d49a3203df0474113ca6e1579604232; ALs=dc5f533d31714acc3cd3672cc582d2c1; _ga=GA1.2.1517880621.1579604339; _gid=GA1.2.1272238848.1579604339',
    "Host": 'www.addustour.com',
    "Upgrade-Insecure-Requests": '1',
    "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'
    }

def get_news(news_url, prefix):
    s = requests.Session()
    s.keep_alive = False  # 关闭多余连接
    repeat = 10
    while repeat:
        try:
            resp = s.get(news_url, headers=headers, verify=False, timeout=20)
            break
        except:
            print('trying:', repeat)
            repeat -= 1
            time.sleep(3)
    if repeat == 0:
        return
    resp.encoding = 'UTF-8'
    sel = Selector(resp)
    title = sel.xpath('//*[@id="article_title"]/h2/text()').extract_first()
    contents = sel.xpath('//*[@id="article_text"]/p//text()').extract()
    content = "\n".join(contents) if contents else ""
    target_file = 'addustour/' + prefix + '.txt'
    print(news_url)
    print(title)
    print("==================")
    with open(target_file, 'a', encoding='utf-8') as t:
        if title:
            t.write(title)
            t.write('\n')
        if content:
            t.write(content)
            t.write('\n')
    now_day = time.strftime("%Y-%m-%d", time.localtime())
    target_file = 'addustour/' + 'count_' + now_day + '.txt'
    with open(target_file, 'a', encoding='utf-8') as t:
        if content:
            t.write(news_url)
            t.write('\n')

def get_urls(url):
    s = requests.Session()
    s.keep_alive = False  # 关闭多余连接
    repeat = 10
    while repeat:
        try:
            resp = s.get(url, headers=headers, verify=False, timeout=20)
            break
        except:
            print('trying:', repeat)
            repeat -= 1
            time.sleep(3)
    if repeat == 0:
        return
    resp.encoding='UTF-8'
    sel = Selector(resp)
    info_urls = sel.xpath('//*[@id="al-right"]/div[2]/ul/li/a/@href').extract()
    if not info_urls:
        raise ValueError("cookie 失效！")

    news_urls = []
    for info_url in info_urls:
        news_url = "https://www.addustour.com" + info_url.replace("../..","")
        news_urls.append(news_url)

    return news_urls

if __name__ == '__main__':
    requests.adapters.DEFAULT_RETRIES = 10  # 增加重连次数
    requests.packages.urllib3.disable_warnings()
    categories = [('1-%D9%85%D8%AD%D9%84%D9%8A%D8%A7%D8%AA', 17895),
                  ('2-%D9%85%D8%AD%D8%A7%D9%81%D8%B8%D8%A7%D8%AA', 3056),
                  ('4-%D8%A7%D9%82%D8%AA%D8%B5%D8%A7%D8%AF', 6062),
                  ('5-%D8%B9%D8%B1%D8%A8%D9%8A-%D9%88%D8%AF%D9%88%D9%84%D9%8A', 9024),
                  ('7-%D8%B1%D9%8A%D8%A7%D8%B6%D8%A9', 7476),
                  ('3-%D8%AB%D9%82%D8%A7%D9%81%D8%A9', 2719),
                  ('6-%D8%AF%D8%B1%D9%88%D8%A8', 4520),
                  ('32-%D8%B4%D8%A8%D8%A7%D8%A8', 9),
                  ('33-%D8%AC%D8%A7%D9%85%D8%B9%D8%A7%D8%AA', 58)]
    for cid, category in enumerate(categories):
        base_url = 'https://www.addustour.com/categories/' + category[0]
        if cid == 0:
            start = 4502
        else:
            start = 1
        for i in range(start, category[1]+1):
            url = base_url + "/page" + str(i)
            prefix = category[0].split('-')[0] + '_' + str(i)
            news_urls = get_urls(url)
            for news_url in news_urls:
                get_news(news_url, prefix)
                time.sleep(1)
            time.sleep(1)


import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
import pickle
import pandas as pd
from sklearn.metrics import accuracy_score

def feature_scaling(X, a, b):
    X = np.array(X)
    Normalized_X = np.zeros((X.shape[0], X.shape[1]))
    for i in range(X.shape[1]):
        Normalized_X[:, i] = ((X[:, i] - min(X[:, i])) / (max(X[:, i]) - min(X[:, i]))) * (b - a) + a
    return Normalized_X

def feature_encoder(X):
    lbl = LabelEncoder()
    lbl.fit(list(X.values))
    X = lbl.transform(list(X.values))
    return X

data = pd.read_csv("VideoLikesDatasetClassification.csv")

#data.dropna(how='any', inplace=True)
data["tags"].fillna("[none]", inplace=True)
data['views'].fillna(int((data['views'].mean())), inplace=True)
data['comment_count'].fillna(int((data['comment_count'].mean())), inplace=True)
data['category_id'].fillna((data['category_id'].max()), inplace=True)

data["video_id"] = feature_encoder(data["video_id"])
data["title"] = feature_encoder(data["title"])
data["channel_title"] = feature_encoder(data["channel_title"])
data["comments_disabled"] = feature_encoder(data["comments_disabled"])
data["ratings_disabled"] = feature_encoder(data["ratings_disabled"])
data["video_error_or_removed"] = feature_encoder(data["video_error_or_removed"])
data["tags"] = feature_encoder(data["tags"])
data["VideoPopularity"] = feature_encoder(data["VideoPopularity"])

# Correlation matrix to help us in features selection
corr_matrix = data.corr()
best_features = corr_matrix.index[abs(corr_matrix['VideoPopularity']) > 0.04]
best_features = best_features.delete(-1)
X = data[best_features]
X = feature_scaling(X, 0, 100)
Y = data[['VideoPopularity']].iloc[:, :]
Y = feature_scaling(Y, 0, 100)
Y = Y.flatten()
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.20, random_state=1)

filename = 'LogisticRegression.sav'
# load the model from disk
LogisticRegression = pickle.load(open(filename, 'rb'))
y_pred = LogisticRegression.predict(X_test)
logisticRegressionAccuracy = accuracy_score(y_pred, y_test)
print('Logistic Regression Accuracy', logisticRegressionAccuracy)
print("-----------------------------------------------------------------------------")

filename = 'SVMPolynomial.sav'
# load the model from disk
SVMPolynomial = pickle.load(open(filename, 'rb'))
y_pred = SVMPolynomial.predict(X_test)
SVMPolykernelAccuracy = accuracy_score(y_pred, y_test)
print('SVM with Polynomial kernel Accuracy', SVMPolykernelAccuracy)
print("-----------------------------------------------------------------------------")

filename = 'decisionTree.sav'
# load the model from disk
decisionTree = pickle.load(open(filename, 'rb'))
y_pred = decisionTree.predict(X_test)
decisionTreeAccuracy = accuracy_score(y_pred, y_test)
print('Decision Tree Accuracy', decisionTreeAccuracy)

print("-----------------------------------------------------------------------------")
filename = 'SVMRBFkernel.sav'
# load the model from disk
loaded_model = pickle.load(open(filename, 'rb'))
y_pred = loaded_model.predict(X_test)
SVMRBFkernelAccuracy = accuracy_score(y_pred, y_test)
print('SVM with Gaussian(RBF) kernel Accuracy', SVMRBFkernelAccuracy)


from rest_framework import permissions
from rest_framework.decorators import action
from rest_framework.exceptions import NotFound
from rest_framework.response import Response
from rest_framework.views import APIView

from apps.owm.serializers import GetWeatherSerializer, WeatherSerializer
from utils.owm import OWMHelper

from drf_yasg.utils import swagger_auto_schema

from drf_yasg import openapi

class WeatherView(APIView):
    permission_classes = [permissions.AllowAny]
    serializer_class = GetWeatherSerializer

    @swagger_auto_schema(
        operation_description="Get actual weather data",
        method='get',
        query_serializer=GetWeatherSerializer(),
        responses={200: openapi.Response('', WeatherSerializer(many=True))},
        operation_id='current_weather_data'
    )
    @action(methods=['get'], detail=False)
    def get(self, request, *args, **kwargs):
        data = self.request.query_params
        serializer = self.serializer_class(data=data)
        serializer.is_valid(raise_exception=True)
        owm = OWMHelper(**serializer.data)
        owm.get_owm_data()
        if owm.ok:
            response_serializer = WeatherSerializer(data=owm.transform_response())
            response_serializer.is_valid(raise_exception=True)
            return Response(response_serializer.data)
        raise NotFound(detail=owm.error(), code=404)


#!python3
#klipper.py - a module for modifying a klipbook data file

import shelve, pyperclip

#klipperInsert(key, fileName | default(KlipBook)) - inserts clipboard content using specified keyword and filename
def klipperInsert(wordKey,book_Name='KlipBook'):
    df = shelve.open(book_Name)
    df[wordKey]=pyperclip.paste()
    df.close()
    return wordKey + ' was added to ' + book_Name

#klipperErase(key, fileName | default(KlipBook)) - deletes the key content from the specified klipbook file
def klipperErase(wordKey, book_Name='KlipBook'):
    df = shelve.open(book_Name)
    del df[wordKey]
    df.close()
    return wordKey + ' was deleted from ' + book_Name

#klipperGet(key, fileName | default(KlipBook)) - pastes the content of the key from the specified klipbook file
def klipperGet(wordKey, book_Name='KlipBook'):
    df = shelve.open(book_Name)
    if(wordKey in df):
        pyperclip.copy(df[wordKey])
        df.close()
        return wordKey + " from " + book_Name + " was pasted to the clipboard "
    else:
        df.close()
        return wordKey + " from " + book_Name + " was missing "

#klipperKeys(fileName | default(KlipBook)) - returns the keys from the specified klipbook file
def klipperKeys(book_Name='KlipBook'):
    df = shelve.open(book_Name)
    keys = str(list(df.keys()))
    df.close()
    return keys


import adi # pip install pyadi-iio
import matplotlib.pyplot as plt
import numpy as np

# Create radio
sdr = adi.Pluto()

# Configure properties
sdr.rx_rf_bandwidth = 4000000
sdr.rx_lo = 2000000000

# Read properties
print("RX LO %s" % (sdr.rx_lo))

# Collect data
sig = sdr.rx()
plt.plot(np.real(sig))
plt.plot(np.imag(sig))
plt.xlabel("Samples")
plt.ylabel("Amplitude [dbFS]")
plt.show()


import urllib.request
import urllib.parse
import json

# Base URL for the API endpoint
base_url = "http://py4e-data.dr-chuck.net/json?"

# Prompt for a location
location = "Kokshetau Institute of Economics and Management"

# Encode the location parameter
parameters = {"address": location}
encoded_params = urllib.parse.urlencode(parameters)

# Construct the full URL
url = base_url + encoded_params

try:
    # Call the API and retrieve the JSON data
    response = urllib.request.urlopen(url)
    json_data = response.read().decode()

    # Parse the JSON data
    data = json.loads(json_data)

    # Retrieve the first place_id from the JSON
    place_id = data["results"][0]["place_id"]

    # Print the place_id
    print("Place id:", place_id)

except Exception as e:
    print("An error occurred:", str(e))


import matplotlib.pyplot as plt
import numpy
import pickle
import argparse
import os

def main(args):

    with open(os.path.join(args.hist_dir, "hist_stage_1.pickle"), "rb") as f:
        history1 = pickle.load(f)

    with open(os.path.join(args.hist_dir, "hist_stage_2.pickle"), "rb") as f:
        history2 = pickle.load(f)

    loss = history1['loss'] + history2['loss']
    val_loss = history1['val_loss'] + history2['val_loss']

    for i in range(len(loss)):
        loss[i] = float(loss[i])

    for i in range(len(val_loss)):
        val_loss[i] = float(val_loss[i])

    # Skip the loss for first 2 epochs
    plt.plot(loss[2:])
    plt.plot(val_loss[2:])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'val'], loc='upper left')
    plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Visualise loss")
    # filenames
    parser.add_argument("-d", "--hist_dir", type = str, required = True, help = "Directory to history files")
    args = parser.parse_args()

    main(args)


from datetime import datetime
from venv import logger
from model.backend_logs_model import BackendLogModel
from util import file_util
from util import mysql_util
from config import projrct_config as conf
import time
from util import logging_util

# 创建日志对象
logger = logging_util.init_logger('backend_logs')

# 1、获取后台访问日志文件夹下有哪些日志文件
all_file_list = file_util.get_dir_files_list(conf.backend_logs_data_root_path)

# 2、查询元数据库表中已经采集的日志文件，用来对比要采集的新的访问日志文件

# 创建元数据库连接
metadata_util = mysql_util.get_mysql_util()

# 获取已经被采集的日志文件
processed_file_list = mysql_util.get_processed_files(
    util=metadata_util,
    db_name=conf.metadata_db,
    tb_name=conf.logs_monitor_meta_table_name,
    tb_cols=conf.logs_monitor_meta_table_create_cols
)

# 对比要采集的日志文件
new_file_list = file_util.get_new_by_compare_lists(processed_file_list,all_file_list)

# 3、进行日志数据采集
csv_file = open(conf.logs_output_csv_root_path + conf.logs_output_csv_file_name, 'a')
# 向csv文件中写入表头
csv_file.write(BackendLogModel.generate_csv_header())

# 创建数据库连接
target_util = mysql_util.get_mysql_util()
target_util.check_table_exists_and_create(
    db_name=conf.target_db,
    tb_name=conf.target_logs_table_name,
    tb_cols=conf.target_logs_table_create_cols
)

if len(new_file_list)>0:
    # 遍历要采集的日志文件
    start_time = time.time()
    for file_name in new_file_list:
        # 开启事务
        target_util.start_transaction()

        try:
            count_lines = 0
            for row in open(file_name,'r',encoding='utf-8'):
                # 逐行写入到数据库中
                backend_logs_model = BackendLogModel(row)
                target_util.insert_single_sql_without_commit(backend_logs_model.generate_insert_sql())

                # 逐行写入到CSV文件中
                csv_file.write(backend_logs_model.generate_csv_str())
                # 每处理一行，记录加一
                count_lines += 1

            # 将已处理的文件插入元数据表中，进行标记，防止重复采集
            mysql_util.process_file_insert_matedata_sql(file_name,count_lines,datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        except Exception as e:
            target_util.rollback_transaction()   # 执行异常，回滚事务
            print('事务已经回滚！')
            raise e
        else:
            target_util.commit_transaction()    # 正常执行，提交事务
            print(f'日志{file_name}已经采集完毕!')
    # 记录日志信息
    end_time = time.time()
    logger.info(f'本次总共采集了{len(new_file_list)}个文件，本次采集共消耗了{end_time-start_time}s')

else:
    print('日志已经全部采集完毕!')

# 关闭连接
csv_file.close()
target_util.close()
metadata_util.close()


"""Module for making a sting-answer from the data"""
from itertools import product
from process_flights import get_price

def sort_by_price(data):
    """Sorts list by the price of the tickets"""
    data[0].sort(key=lambda i: float(i.price.replace(',', '.')))
    return data

def make_to_string(data):
    """Makes a sting-answer from the data"""
    data = sort_by_price(data)
    sum_results = []
    for res in data:
        results = []
        for result in res:
            results.append('Date: {}. Departs in {}, Arrives in {}. '
                           'Duration of flight = {}, {}. '
                           'Price: {} in {}. Class: {}'.format(result.date,
                                                               result.depart_time,
                                                               result.arrive_time,
                                                               result.duration,
                                                               result.stops,
                                                               result.price,
                                                               result.value,
                                                               result.fl_class))
        sum_results.append(results)
    return sum_results

def combine_results(data):
    """Combines results to get all variants of round trip"""
    list_of_combinations = list(product(data[0], data[1]))
    return list_of_combinations

def get_total_price(combination):
    """Counts a total price of two flights in combination"""
    return float(get_price(combination[0]).replace(',', '.')) +\
           float(get_price(combination[1]).replace(',', '.'))


from common.datetime_ import datetime_

__author__ = 'SLCIT, Inc.'
__email__ = 'simoncharest@gmail.com'
__copyright__ = f'Copyright © {datetime_.get_years(2020)} {__author__} <{__email__}>. All rights reserved.'
__project__ = 'Genealopy'
__credits__ = {
    'Simon Charest': {
        'organization': [f'{__author__}', 'SLCIT, Inc.'],
        'email': [f'{__email__}', 'simoncharest@gmail.com'],
        'Facebook': 'https://www.facebook.com/simon.charest/',
        'GitHub': 'https://github.com/Simon-Charest',
        'LinkedIn': 'https://www.linkedin.com/in/simoncharest/',
        'Twitter': 'https://twitter.com/scharest'
    },
    'Francis-Olivier Gradel, Eng.': {
        'organization': 'Retronic Design',
        'product': 'DB9 to USB Game Controller Adapter',
        'email': 'info@retronicdesign.com',
        'website': 'retronicdesign.com',
        'eBay': 'https://www.ebay.com/usr/retronicdesign',
        'Facebook': 'https://www.facebook.com/retronicdesign',
        'LinkedIn': 'https://ca.linkedin.com/in/francis-gradel-ing-b620591a',
        'Twitter': 'https://twitter.com/fogradel',
        'AtariAge': 'https://atariage.com/forums/profile/37766-nitz1976/',
        'Famille Gradel': 'https://www.retronicdesign.com/genealogie/'
    },
    'Gilles Charest': {},
    'Gilles Deguire': {'email': 'deguire@mesancetres.ca', 'organization': 'Mes ancêtres', 'website': 'http://www.mesancetres.ca/'},
    'Louise Charest': {'email': 'charestl53@videotron.ca'},
    'Marc Charest': {'email': 'marc@lourobin.com'},
    'Michel Charest': {'email': 'mic6349@gmail.com'},
    'Pierre Charest': {'email': 'charestp@videotron.ca', 'organization': '9116-8872 Québec inc.', 'website': 'https://www.myheritage.fr/site-205814691/charest'},
    'Richard Côté': {'email': 'richardcoste@sogetel.net', 'organization': "Le Centre de généalogie francophone d'Amérique", 'website': 'http://www.genealogie.org/login/'},
    'Suzanne Coderre': {'email': 'suzanne1809@gmail.com'},
    'The Charest Family': {}
}
__license__ = 'GNU'
__maintainer__ = 'Simon Charest'
__status__ = 'Developement'
__version__ = '2.0.0'

"""
Filenames examples:
    ma01_dubreuil_lague.json   : first-degree (01) ascendant (a) of the mother's side (m) of the family
    fa01_tremblay_charest.json : first-degree (01) ascendant (a) of the father's side (f) of the family
    00_lague_charest.json      : Root (00) of both sides of the family
    m00_dubreuil_lague.json    : Root (00) of the mother's side (m) of the family
    f00_tremblay_charest.json  : Root (00) of the father's side (f) of the family
    md01_dubreuil_lague.json   : first-degree (01) descendant (d) of the mother's side (m) of the family
    fd01_charest_drouin.json   : first-degree (01) descendant (d) of the father's side (f) of the family
"""

# Data files
DATA_DIRECTORY = 'data/'
INPUT_DIRECTORY = f'{DATA_DIRECTORY}input/'
OUTPUT_DIRECTORY = f'{DATA_DIRECTORY}output/'
ALL_FILENAMES = [f'{INPUT_DIRECTORY}**/*.json']
INPUT_FILENAMES = map(lambda filename: INPUT_DIRECTORY + str(filename), [
        # Laguë
        # 'Laguë/far_ascendance/*.json',  # Marie-Anne Lucas dit Francoeur's ascendance
        # 'Laguë/*.json',  # Suzanne Laguë's ascendance
        # 'Laguë/descendance/*.json',

        # Charest
        "Charest/far_ascendence/fa99_delorraine.json",  # Charest noble ascendence
        'Charest/far_ascendence/*.json',  # Delphis Charest's ascendence
        'Charest/tanguay_charest_siblings/*.json',  # Delphis Charest's siblings
        'Charest/dion_charette_ascendence/*.json',  # Aurèle Charette (Charest-Charette)'s ascendence
        'Charest/little_cousins/*.json',  # Clément Charest siblings' descendence
        'Charest/little_cousin_descendence/*.json',  # Clément Charest siblings' descendence
        'Tremblay/*.json',  # Rita Lacombe Tremblay's ascendence
        'Charest/*.json',
        'Charest/descendence/*.json',

        # Laguë-Charest
        '0?_lague_charest.json'
    ]
)
OUTPUT_FILENAME = f'{OUTPUT_DIRECTORY}data.txt'

# Encryption
SECURITY_DIRECTORY = f'{DATA_DIRECTORY}security/'
KEY_FILENAME = f'{SECURITY_DIRECTORY}key.txt'
SALT_FILENAME = f'{SECURITY_DIRECTORY}salt.txt'

# Visuals
GRAPH_DIRECTORY = f'{DATA_DIRECTORY}graph/'
GRAPH_FILENAME = f'{GRAPH_DIRECTORY}{__project__.lower()}.gv'
GRAPH_FORMAT = 'png'
RANK_DIRECTION = 'TB'  # TB, LR, BT or RL
SHAPE = 'box'
STYLE = 'filled'
NAME_UNKNOWN = '(inconnu)'

# Filters
GENDER = ['M', 'F']
RELATIONSHIP = ['grandfather', 'father', 'grandmother', 'mother', 'union']

# Colors
DARKEN_INCOMPLETE = True
FEMALE_COLOR = 'pink'
FEMALE_INCOMPLETE_COLOR = 'deeppink'
MALE_COLOR = 'lightblue'
MALE_INCOMPLETE_COLOR = 'deepskyblue'
UNDEFINED_COLOR = 'grey'
SEARCH_COLOR = 'orange'
PARENT_LINK_STYLE = 'solid'
UNDEFINED_LINK_STYLE = 'dashed'

# Debugging
DEBUG = False


"""changed user.id to type string

Revision ID: 712f2c5f8c2c
Revises:
Create Date: 2023-10-28 21:10:20.093259

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '712f2c5f8c2c'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('user', schema=None) as batch_op:
        batch_op.alter_column('id',
               existing_type=sa.INTEGER(),
               type_=sa.String(length=11),
               existing_nullable=False)
        batch_op.create_unique_constraint('unique_user_id', ['id'])

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('user', schema=None) as batch_op:
        batch_op.drop_constraint('unique_user_id', type_='unique')
        batch_op.alter_column('id',
               existing_type=sa.String(length=11),
               type_=sa.INTEGER(),
               existing_nullable=False)

    # ### end Alembic commands ###


import logging
import requests

from typing import List
from concurrent.futures import ThreadPoolExecutor

from v2.scrapping_books.pages.books_page import BooksPage
from v2.scrapping_books.parsers.book import BookParser

logging.basicConfig(
    format="%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s",
    datefmt="%d-%m-%Y %H:%M:%S",
    level=logging.INFO,
    filename="logs.txt",
)

logger = logging.getLogger("scraping")

class BookManager:
    URL = "https://books.toscrape.com/"

    def __init__(self):
        # self.books = self.retrieve_books()
        self.books = self.retrieve_books_mul()
        self.books_gen = (book for book in self.books)

    def fetch_url(self, url) -> str:
        response_content = requests.get(url).content
        print(f"reading page {url}")
        return response_content

    def retrieve_books(self) -> List[BookParser]:
        """retrieve all books sequentially"""
        # Fetch the first page to determine the total number of pages
        first_page_content = self.fetch_url(self.URL + "catalogue/page-1.html")
        first_page = BooksPage(first_page_content)
        total_pages = first_page.page_count

        # Initialize the books list with books from the first page
        _books = first_page.books

        # Fetch books from the remaining pages
        for page in range(2, total_pages + 1):
            url = self.URL + f"catalogue/page-{page}.html"
            page_content = self.fetch_url(url)
            page = BooksPage(page_content)
            _books.extend(page.books)

        return _books

    def retrieve_books_mul(self) -> List[BookParser]:
        """retrieve all books concurrently"""
        first_page_content = self.fetch_url(self.URL + "catalogue/page-1.html")
        first_page = BooksPage(first_page_content)
        total_pages = first_page.page_count

        _books = first_page.books
        URLs = [
            self.URL + f"catalogue/page-{i}.html" for i in range(2, total_pages + 1)
        ]

        with ThreadPoolExecutor() as executor:
            contents = list(executor.map(self.fetch_url, URLs))
            for content in contents:
                page = BooksPage(content)
                _books.extend(page.books)

        return _books

    def look_top_books(self):
        top_books = [book for book in self.books if book.stars == 5][:20]
        for book in top_books:
            print(book)

    def look_cheapest_books(self):
        cheapest_books = sorted(self.books, key=lambda x: x.price)[:20]
        for book in cheapest_books:
            print(book)

    # show 10 books sorted by starts (desc) and price (asc)
    # @dev: first sorts by starts, then sorts by price
    # best_cheapest_books = sorted(books, key=lambda x: (x.stars * -1, x.price))[:10]
    # for book in best_cheapest_books:
    #     print(book)

    def get_next_available_book(self):
        print(next(self.books_gen))

    @property
    def get_user_choice(self) -> str:
        return f"""Enter one of the following:
            - 'b' to look at 5-star books
            - 'c' to look at the cheapest books
            - 'n' to just get the next available book on the catalogue
            - 'q' to exit
        """

    def menu(self):
        user_options = {
            "b": self.look_top_books,
            "c": self.look_cheapest_books,
            "n": self.get_next_available_book,
        }
        user_input = input(self.get_user_choice)
        while user_input != "q":
            if user_input in user_options:
                selected_function = user_options[user_input]
                selected_function()
            else:
                print("Unknown command. Please try again.")
            user_input = input(self.get_user_choice)

def test():
    logger.info("Loading books list...")
    manager = BookManager()
    manager.menu()

    # print('ALL BOOKS')
    # for book in books:
    #     print(book)
    # print('-------- TOP 10 best books --------')
    # print_best_books(books)
    # print('-------- TOP 10 cheapest books --------')
    # print_cheapest_books(books)
    # print('-------- TOP 10 best cheapest books --------')
    # print_best_cheapest_books(books)


"""
        PRACTICA 2 (PRPA)

Laura Cano Gómez

Solution to the one-way tunnel

"""
import time
import random
from multiprocessing import Lock, Condition, Process
from multiprocessing import Value
from multiprocessing import Manager

NORTH = 0
SOUTH = 1

NCARS = 5
NPED = 3     # Como los peatones si pueden pasar en ambos sentidos, no hace falta distinguir su sentido

TIME_CARS_NORTH = 0.5   # a new car enters each 0.5s
TIME_CARS_SOUTH = 0.5   # a new car enters each 0.5s
TIME_PED = 5            # a new pedestrian enters each 5s

TIME_IN_BRIDGE_CARS = (1, 0.5) # normal 1s, 0.5s
TIME_IN_BRIDGE_PEDESTRIAN = (30, 10) # normal 1s, 0.5s

class Monitor():
    def __init__(self):
        self.mutex = Lock()
        self.patata = Value('i',0)

        self.cNortePasando= Value('i', 0)       # Numero de coches del norte dentro del puente
        self.cSurPasando= Value('i', 0)         # Numero de coches del sur dentro del puente
        self.peatonesPasando= Value('i', 0)     # Numero de peatones dentro del puente

        self.permisoCoches= Condition(self.mutex)       # Variable condicion
        self.permisoPeatones= Condition(self.mutex)     # Variable condicion

    '''
    INVARIANTE: (demo en Desarrollo monitor.pdf)
        cNortePasando >= 0
        cSurPasando >= 0
        peatonesPasando >= 0
        cNortePasando > 0 => cSurPasando == 0 /\ peatonesPasando == 0
        cSurPasando > 0 => cNortePasando == 0 /\ peatonesPasando == 0
        peatonesPasando > 0 => cNortePasando == 0 /\ cSurPasando == 0

        Hay inanicion porque podrian estar pasando continuamente  2 de ellos sin que el otro pueda entrar. Ademas, no hoy justicia porque si no paran de llegar de un tipo, hasta que no acaben de salir los otros 2 se quedan bloqueados. Hay que poner alguna limitacion de paso -> turnos (PUENTE POR TURNOS.py)
    '''

    def puenteEstaVacio(self):
        return self.cNortePasando.value == 0 and self.cSurPasando == 0 and self.peatonesPasando == 0

    # Permiso para coches que vienen del norte
    def puedenPasarCN(self):
        # Pueden pasar si no hay coches en sentido contrario ni peatones pasando, o si hay coches en su mismo sentido
        puedenPasar= (self.cSurPasando.value == 0 and self.peatonesPasando.value == 0) or (self.cNortePasando.value>0)

        return  puedenPasar or self.puenteEstaVacio()

    # Permiso para coches que vienen del sur
    def puedenPasarCS(self):
        puedenPasar= (self.cNortePasando.value == 0 and self.peatonesPasando.value == 0) or (self.cSurPasando.value>0)

        return  puedenPasar or self.puenteEstaVacio()

    # Permiso para peatones
    def puedenPasarPeatones(self):
        puedePasar= (self.cNortePasando.value == 0 and self.cSurPasando.value == 0) or (self.peatonesPasando.value>0)
        return puedePasar or self.puenteEstaVacio()

    def wants_enter_car(self, direction: int) -> None:
        self.mutex.acquire()
        self.patata.value += 1
        #### código

        # Espera a tener permiso
        if direction==NORTH:
            self.permisoCoches.wait_for(self.puedenPasarCN)

        else:
            self.permisoCoches.wait_for(self.puedenPasarCS)

        # Entra al puente
        if direction == NORTH:
            self.cNortePasando.value += 1
        else:
            self.cSurPasando.value += 1

        # Sale y abre el mutex
        self.mutex.release()

    def leaves_car(self, direction: int) -> None:
        self.mutex.acquire()
        self.patata.value += 1
        #### código

        # Sale del puente
        if direction == NORTH:
            self.cNortePasando.value -= 1
        else:
            self.cSurPasando.value -= 1

        # Avisa a todos los demas coches y peatones de que ha salido, para que comprueben si pueden pasar
        self.permisoCoches.notify_all()
        self.permisoPeatones.notify_all()

        self.mutex.release()

    def wants_enter_pedestrian(self) -> None:
        self.mutex.acquire()
        self.patata.value += 1
        #### código

        # Espera a tener permiso y entra
        self.permisoPeatones.wait_for(self.puedenPasarPeatones)
        self.peatonesPasando.value += 1

        self.mutex.release()

    def leaves_pedestrian(self) -> None:
        self.mutex.acquire()
        self.patata.value += 1
        #### código

        # Sale del puente
        self.peatonesPasando.value -= 1

        # Avisa a todos los demas coches y peatones de que ha salido, para que comprueben si pueden pasar
        self.permisoCoches.notify_all()
        self.permisoPeatones.notify_all()

        self.mutex.release()

    def __repr__(self) -> str:
        #return f'Monitor: {self.patata.value}'
        return f'\n   Monitor-> Coches Norte: {self.cNortePasando.value}, Coches Sur: {self.cSurPasando.value}, Peatones :  {self.peatonesPasando.value}.\n\n'

# Funciones para simular el tiempo del paso por el puente
# Con normales
'''
def delay_car_north() -> None:
    time.sleep(random.normalvariate(1,0.5))

def delay_car_south() -> None:
    time.sleep(random.normalvariate(1,0.5))

def delay_pedestrian() -> None:
    time.sleep(random.normalvariate(30,10))
'''
# Sin normales
def delay_car_north(factor = 3) -> None:
    time.sleep(random.randint(1,4)/factor)

def delay_car_south(factor = 3) -> None:
    time.sleep(random.randint(1,4)/factor)

def delay_pedestrian(factor = 2) -> None:
    time.sleep(random.randint(3,6)/factor)

# Creación de peatones y coches
def car(cid: int, direction: int, monitor: Monitor)  -> None:
    print(f"Coche {cid} rumbo {direction} quiere entrar. {monitor}")

    monitor.wants_enter_car(direction)
    print(f"Coche {cid} rumbo {direction} entrando... {monitor}")
    if direction==NORTH :
        delay_car_north()
    else:
        delay_car_south()
    #print(f"Coche {cid} rumbo {direction} abandonando el puente. {monitor}")

    monitor.leaves_car(direction)
    print(f"El coche {cid} rumbo {direction} ha salido. {monitor}")

def pedestrian(pid: int, monitor: Monitor) -> None:
    print(f"Peatón {pid} quiere entrar. {monitor}")
    monitor.wants_enter_pedestrian()
    print(f"Peatón {pid} entrando... {monitor}")
    delay_pedestrian()

    #print(f"Peatón {pid} abandonando el puente. {monitor}")
    monitor.leaves_pedestrian()
    print(f"El peatón {pid} ha salido. {monitor}")

def gen_pedestrian(monitor: Monitor) -> None:
    pid = 0
    plst = []
    for _ in range(NPED):
        pid += 1
        p = Process(target=pedestrian, args=(pid, monitor))
        p.start()
        plst.append(p)
        time.sleep(random.expovariate(1/TIME_PED))

    for p in plst:
        p.join()

def gen_cars(direction: int, time_cars, monitor: Monitor) -> None:
    cid = 0
    plst = []
    for _ in range(NCARS):
        cid += 1
        p = Process(target=car, args=(cid, direction, monitor))
        p.start()
        plst.append(p)
        time.sleep(random.expovariate(1/time_cars))

    for p in plst:
        p.join()

def main():
    monitor = Monitor()

    gcars_north = Process(target=gen_cars, args=(NORTH, TIME_CARS_NORTH, monitor))
    gcars_south = Process(target=gen_cars, args=(SOUTH, TIME_CARS_SOUTH, monitor))
    gped = Process(target=gen_pedestrian, args=(monitor,))

    gcars_north.start()
    gcars_south.start()
    gped.start()

    gcars_north.join()
    gcars_south.join()
    gped.join()

if __name__ == '__main__':
    main()


import networkx as nx
import matplotlib.pyplot as plt
import numpy as np
import ast

INF = 100000010
SIZE = 100000 + 1
distances = [INF] * SIZE
visited = [False] * SIZE
vertex = [[] for i in range(SIZE)]
road = [0] * SIZE

global G

def dijkstra(initialNode):
    global G

    distances[initialNode] = 0
    s = [(0,initialNode)]

    while len(s) is not 0:

        p = s.pop()

        x = p[1]
        w = p[0]

        if visited[x] is True:
            continue
        visited[x] = True

        for i in range(len(vertex[x])):
            edge = vertex[x][i][1]
            weight = vertex[x][i][0]

            if (distances[x] + weight) < distances[edge]:
                road[edge] = x

                distances[edge] = distances[x] + weight
                s.append((distances[edge], edge))

#############
def main(s, t):
    global G

    rf = open("copyLugares.txt", "r")
    cities = ast.literal_eval(rf.read())
    rf.close()

    rf = open("copyCoords.txt", "r")
    pos = ast.literal_eval(rf.read())
    rf.close()

    G = nx.Graph()
    rf = open("copyconnect.txt", "r")

    n, m = map(int, rf.readline().split())

    for i in range(m):
        a, b, peso = map(float, rf.readline().split())
        vertex[int(a)].append((peso, int(b)))
        vertex[int(b)].append((peso, int(a)))

    node = s  # De donde empieza
    destiny = t # Destino

    dijkstra(node)

    for i in range(0, n + 1):
        print('[ ', end='')
        print(distances[i], end=' ]')

    print()

    i = destiny
    while (i is not node):
        print(road[i], end=" ")
        formatted_weight = "{:.2f}".format(distances[i])
        G.add_edge(road[i], i, weight=formatted_weight)

        G[road[i]][i]['weight'] = formatted_weight
        i = road[i]

    img = plt.imread("MapaCarreteraBolivia.PNG")
    fig, ax = plt.subplots(figsize=(15, 15))

    ax.imshow(img, extent=[-150, 200, -150, 200], alpha=0.7, resample=False)

    list_of_nodes = list(G.nodes())

    new_dict = "{"
    new_pos = "{"

    for i in range(G.number_of_nodes()):
        new_dict += str(list_of_nodes[i]) + ':"' + str(cities.get(list_of_nodes[i])) + '",'
        new_pos += "\"" + str(cities.get(list_of_nodes[i])) + "\":" + str(pos.get(cities.get(list_of_nodes[i]))) + ','

    new_dict += "}"
    new_pos += '}'

    dicty = ast.literal_eval(new_dict)
    poss = ast.literal_eval(new_pos)

    nx.relabel_nodes(G, mapping=dicty, copy=False)

    labels = nx.get_edge_attributes(G, 'weight')

    # G.remove_node('None')

    nx.draw_networkx(G, poss, with_labels=True, node_color="cyan", node_size=80, width=1)

    labels = nx.get_edge_attributes(G, 'weight')
    nx.draw_networkx_edge_labels(G, poss, edge_labels=labels, font_size=7)

    plt.savefig("path_Dijkstra.png")

    #plt.show()

    rf.close()


#-----------------------------------------------------
# Mimas: conference submission and review system
# (c) Allan Kelly 2016-2020 http://www.allankelly.net
# Licensed under MIT License, see LICENSE file
# -----------------------------------------------------

# System imports

# Google imports

from google.appengine.ext import ndb

# Local imports
from scaffold import tags
import basehandler

class EditTagsPage(basehandler.BaseHandler):
    def get(self):
        speaker = None
        talk = None
        tag_list = []

        if self.request.params.has_key("spk"):
            tag_type = "Speaker"
            speaker = ndb.Key(urlsafe=self.request.get("spk")).get()
            tags_list = tags.TagList(speaker.key).pretty_tag_list([tag_type])
            parent_key = speaker.key.urlsafe()

        if self.request.params.has_key("talk"):
            tag_type="Talk"
            talk = ndb.Key(urlsafe=self.request.get("talk")).get()
            tags_list = tags.TagList(talk.key).pretty_tag_list([tag_type])
            parent_key = talk.key.urlsafe()

        template_values = {
            "speaker":      speaker,
            "talk":         talk,
            "parent_key":   parent_key,
            "tags":         tags_list,
            "tag_type":     tag_type,
            "all_tags":     tags.retrieve_all_unique_tags(),
        }

        self.write_page("scaffold/edittags.html", template_values)

    def post(self):
        parent_key = ndb.Key(urlsafe=self.request.get("parent_key"))
        if self.request.get("remove_tag"):
            tags.TagList(parent_key).remove_tag(self.request.get("newtag"), [self.request.get("tag_type")])

        if self.request.get("tag_type") == "Speaker":
            self.redirect("/edittags?spk=" + parent_key.urlsafe())
        else:
            self.redirect("/edittags?talk=" + parent_key.urlsafe())

class AddTagPost(basehandler.BaseHandler):
    def post(self):
        k = ndb.Key(urlsafe=self.request.get("key"))
        tags.TagList(k).add_tag(self.request.get("tag"), self.request.get("tag_type"))


from django.contrib.sessions.backends.db import SessionStore as DBStore
from django.utils.functional import cached_property
from django.conf import settings
from account.middleware import get_current_request

class SessionStore(DBStore):

    @classmethod
    def get_model_class(cls):
        from account.models import CustomSession
        return CustomSession

    def create_model_instance(self, data):

        request = get_current_request()
        print('request: %s' % request)
        # print('ip: %s' % ip)
        # print('request.user.username: %s' % request.user.username)
        print('hello')

        custom_session_instance = self._get_session_from_db()
        print(custom_session_instance)

        email = None
        name = None
        if not custom_session_instance:
            date_created = None
            ip = request.META.get('REMOTE_ADDR')
            print('new_ip: %s' % ip)

        else:
            date_created = custom_session_instance.created_at
            ip = request.META.get('REMOTE_ADDR')
            stored_ip = custom_session_instance.ip
            if request.user.is_authenticated:
                name = request.user.username
                email = request.user.email
            if stored_ip == ip:
                print('stored_ip')
                ip = ip
            else: print('ip is changed')

        return self.model(
            session_key=self._get_or_create_session_key(),
            session_data=self.encode(data),
            expire_date=self.get_expiry_date(),
            created_at = date_created,
            ip = ip,
            name =name,
            email = email,

       )