"""Basic Linear Algebra utility functions implemented as gufuncs,
broadcasting

This file contains the wrappers for several basic linear algebra
functions as gufuncs. The underlying implementation is BLAS based.

- inner1d: inner product (dot product) over the inner dimension,
  broadcasting

- dotc1d: inner product by conjugate (dot product) over the inner
  dimension, broadcasting

- innerwt: weighted inner product over the inner dimension,
  broadcasting

- matrix_multiply: matrix_multiply over the 2 inner dimensions,
  broadcasting

- matvec_multiply: matvec_multiply over the 2 inner dimensions,
  broadcasting

- quadratic form: quadratic form uQv over the inner dimensions,
  broadcasting

- update_rank1: rank1 update over the inner dimensions,
  broadcasting

- update_rankk: rankk update over the 2 inner dimensions,
  broadcasting

"""

from __future__ import division, absolute_import, print_function

import contextlib
import multiprocessing

import numpy as np

from . import _impl

@contextlib.contextmanager
def _setup_gulinalg_threads(workers):
    if workers == -1:
        workers = multiprocessing.cpu_count()
    elif workers < 1 or workers % 1 != 0:
        raise ValueError("num_threads must be a non-negative integer or -1")

    orig_workers = _impl.get_gufunc_threads()
    if workers != orig_workers:
        _impl.set_gufunc_threads(workers)

    try:
        yield workers
    finally:
        if workers != orig_workers:
            _impl.set_gufunc_threads(orig_workers)

def inner1d(a, b, workers=1, **kwargs):
    """
    Compute the dot product of vectors over the inner dimension, with
    broadcasting.

    Parameters
    ----------
    a : (..., N) array
        Input array
    b : (..., N) array
        Input array
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Returns
    -------
    inner : (...) array
        dot product over the inner dimension.

    Notes
    -----
    Numpy broadcasting rules apply when matching dimensions.

    Implemented for types single, double, csingle and cdouble. Numpy
    conversion rules apply.

    For single and double types this is equivalent to dotc1d.

    Maps to BLAS functions sdot, ddot, cdotu and zdotu.

    See Also
    --------
    dotc1d : dot product conjugating first vector.
    innerwt : weighted (i.e. triple) inner product.

    Examples
    --------
    >>> a = np.arange(1,5).reshape(2,2)
    >>> b = np.arange(1,8,2).reshape(2,2)
    >>> res = inner1d(a,b)
    >>> res.shape == (2,)
    True
    >>> print (res)
    [ 7. 43.]

    """
    with _setup_gulinalg_threads(workers):
        out = _impl.inner1d(a, b, **kwargs)
    return out

def dotc1d(a, b, workers=1, **kwargs):
    """
    Compute the dot product of vectors over the inner dimension, conjugating
    the first vector, with broadcasting

    Parameters
    ----------
    a : (..., N) array
        Input array
    b : (..., N) array
        Input array

    Returns
    -------
    dotc : (...) array
        dot product conjugating the first vector over the inner
        dimension.
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Notes
    -----
    Numpy broadcasting rules apply when matching dimensions.

    Implemented for types single, double, csingle and cdouble. Numpy
    conversion rules apply.

    For single and double types this is equivalent to inner1d.

    Maps to BLAS functions sdot, ddot, cdotc and zdotc.

    See Also
    --------
    inner1d : dot product
    innerwt : weighted (i.e. triple) inner product.

    Examples
    --------
    >>> a = np.arange(1,5).reshape(2,2)
    >>> b = np.arange(1,8,2).reshape(2,2)
    >>> res = inner1d(a,b)
    >>> res.shape == (2,)
    True
    >>> print (res)
    [ 7. 43.]

    """
    with _setup_gulinalg_threads(workers):
        out = _impl.dotc1d(a, b, **kwargs)
    return out

def innerwt(a, b, c, workers=1, **kwargs):
    """
    Compute the weighted (i.e. triple) inner product, with
    broadcasting.

    Parameters
    ----------
    a, b, c : (..., N) array
        Input arrays
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Returns
    -------
    inner : (...) array
        The weighted (i.e. triple) inner product.

    Notes
    -----
    Numpy broadcasting rules apply.

    Implemented for types single, double, csingle and cdouble. Numpy
    conversion rules apply.

    See Also
    --------
    inner1d : inner product.
    dotc1d : dot product conjugating first vector.

    Examples
    --------
    >>> a = np.arange(1,5).reshape(2,2)
    >>> b = np.arange(1,8,2).reshape(2,2)
    >>> c = np.arange(0.25,1.20,0.25).reshape(2,2)
    >>> res = innerwt(a,b,c)
    >>> res.shape == (2,)
    True
    >>> res
    array([ 3.25, 39.25])

    """
    with _setup_gulinalg_threads(workers):
        out = _impl.innerwt(a, b, c, **kwargs)
    return out

def matrix_multiply(a, b, workers=1, **kwargs):
    """
    Compute matrix multiplication, with broadcasting

    Parameters
    ----------
    a : (..., M, N) array
        Input array.
    b : (..., N, P) array
        Input array.
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Returns
    -------
    r : (..., M, P) array matrix multiplication of a and b over any number of
        outer dimensions

    Notes
    -----
    Numpy broadcasting rules apply.

    Matrix multiplication is computed using BLAS _gemm functions.

    Implemented for single, double, csingle and cdouble. Numpy conversion
    rules apply.

    Examples
    --------
    >>> a = np.arange(1,17).reshape(2,2,4)
    >>> b = np.arange(1,25).reshape(2,4,3)
    >>> res = matrix_multiply(a,b)
    >>> res.shape == (2, 2, 3)
    True
    >>> res
    array([[[  70.,   80.,   90.],
            [ 158.,  184.,  210.]],
    <BLANKLINE>
           [[ 750.,  792.,  834.],
            [1030., 1088., 1146.]]])

    """
    with _setup_gulinalg_threads(workers):
        out = _impl.matrix_multiply(a, b, **kwargs)
    return out

def matvec_multiply(a, b, workers=1, **kwargs):
    """
    Compute matrix vector multiplication, with broadcasting

    Parameters
    ----------
    a : (..., M, N) array
        Input array.
    b : (..., N) array
        Input array
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Returns
    -------
    r : (..., M) matrix vector multiplication of a and b over any number of
        outer dimensions

    Notes
    -----
    Numpy broadcasting rules apply.

    Matrix vector multiplication is computed using BLAS _gemv functions.

    Implemented for single, double, csingle and cdouble. Numpy conversion
    rules apply.

    Examples
    --------
    >>> a = np.arange(1,17).reshape(2,2,4)
    >>> b = np.arange(1,9).reshape(2,4)
    >>> res = matvec_multiply(a,b)
    >>> res.shape == (2,2)
    True
    >>> res
    array([[ 30.,  70.],
           [278., 382.]])

    """
    with _setup_gulinalg_threads(workers):
        out =  _impl.matvec_multiply(a, b, **kwargs)
    return out

def quadratic_form(u, Q, v, workers=1, **kwargs):
    """
    Compute the quadratic form uQv, with broadcasting

    Parameters
    ----------
    u : (..., M) array
        The u vectors of the quadratic form uQv
    Q : (..., M, N) array
        The Q matrices of the quadratic form uQv
    v : (..., N) array
        The v vectors of the quadratic form uQv
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Returns
    -------
    qf : (...) array
        The result of the quadratic forms

    Notes
    -----
    Numpy broadcasting rules apply.

    Implemented for types single, double, csingle and cdouble. Numpy
    conversion rules apply.

    This is similar to PDL inner2

    Examples
    --------

    The result in absence of broadcasting is just as np.dot(np.dot(u,Q),v)
    or np.dot(u, np.dot(Q,v))

    >>> u = np.array([2., 3.])
    >>> Q = np.array([[1.,1.], [0.,1.]])
    >>> v = np.array([1.,2.])
    >>> quadratic_form(u,Q,v)
    12.0

    >>> np.dot(np.dot(u,Q),v)
    12.0

    >>> np.dot(u, np.dot(Q,v))
    12.0

    """
    with _setup_gulinalg_threads(workers):
        out = _impl.quadratic_form(u, Q, v, **kwargs)
    return out

def update_rank1(a, b, c, conjugate=True, workers=1, **kwargs):
    """
    Compute rank1 update, with broadcasting

    Parameters
    ----------
    a : (..., M) array
        Input array.
    b : (..., N) array
        Input array
    c : (..., M, N) array
        Input array.
    conjugate : bool (default True)
        For complex numbers, use conjugate transpose of b instead of normal
        transpose. If false, use normal transpose.
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Returns
    -------
    r : (..., M, N) rank1 update of a, b and c over any number of
        outer dimensions

    Notes
    -----
    Numpy broadcasting rules apply.

    Rank1 update is computed using BLAS _ger functions for real numbers.
    For complex number, uses _gerc and _geru for conjuate and normal transpose
    variants respectively.

    Implemented for single, double, csingle and cdouble. Numpy conversion
    rules apply.

    Examples
    --------
    >>> a = np.array([1, 2])
    >>> b = np.array([3, 4])
    >>> c = np.array([[1, 2], [3, 4]])
    >>> res = update_rank1(a, b, c)
    >>> res.shape == (2, 2)
    True
    >>> res
    array([[ 4.,  6.],
           [ 9., 12.]])

    """
    if conjugate:
        gufunc = _impl.update_rank1_conjugate
    else:
        gufunc = _impl.update_rank1

    with _setup_gulinalg_threads(workers):
        out = gufunc(a, b, c, **kwargs)
    return out

def update_rankk(a, c=None, UPLO='U', transpose_type='T', sym_out=True,
                 workers=1, **kwargs):
    """
    Compute symmteric rank-k update, with broadcasting

    Parameters
    ----------
    a : (..., N, K) or (..., K, N) array
        Input array. If `transpose_type` is 'N', `a` should be shape
        (..., N, K) otherwise it should be shape (..., K, N)
    c : (..., N, N) array, optional
        Input array. If None, `c` will be a zeros matrix.
    UPLO : {'U', 'L'}, optional
        Specifies whether the calculation is done with the lower
        triangular part of the elements in `a` ('L', default) or
        the upper triangular part ('U').
    transpose_type : {'N', 'T', 'C'}, optional
        Transpose type which decides equation to be solved.
        N => No transpose i.e. C = alpha * A * A.T + beta * C
        T => Transpose i.e. C = alpha * A.T * A + beta * C
        C => Conjugate transpose i.e. C = alpha * A.T * A + beta * C
    sym_out: bool, optional
        If True, create a symmetric output by copying the upper (lower)
        triangular entries into the lower (upper) triangle.
    workers : int, optional
        The number of parallel threads to use along gufunc loop dimension(s).
        If set to -1, the maximum number of threads (as returned by
        ``multiprocessing.cpu_count()``) are used.

    Returns
    -------
    r : (..., M, N) rank-k update of a and c over any number of outer
        dimensions

    Notes
    -----
    Numpy broadcasting rules apply.

    Implemented for single, double. Numpy conversion rules apply.

    Rank-k update is computed using BLAS _syrk functions.

    Examples
    --------
    >>> a = np.array([[1., 0.],
    ...               [0., -2.],
    ...               [2., 3.]])
    >>> c = np.zeros((3, 3))
    >>> res = update_rankk(a, c, transpose_type='N', sym_out=False)
    >>> res.shape == (3, 3)
    True
    >>> res
    array([[ 1.,  0.,  2.],
           [ 0.,  4., -6.],
           [ 0.,  0., 13.]])
    >>> res = update_rankk(a, transpose_type='N', sym_out=True)
    >>> res.shape == (3, 3)
    True
    >>> res
    array([[ 1.,  0.,  2.],
           [ 0.,  4., -6.],
           [ 2., -6., 13.]])
    """
    uplo_choices = ['U', 'L']
    transpose_choices = ['N', 'T', 'C']

    if UPLO not in uplo_choices:
        raise ValueError("Invalid UPLO argument '%s', valid values are: %s" %
                         (UPLO, uplo_choices))

    if transpose_type not in transpose_choices:
        raise ValueError(("'Invalid transpose_type argument '%s', "
                          "valid values are: %s") %
                         (transpose_type, transpose_choices))

    if a.dtype.kind == 'c' or (c is not None and c.dtype.kind == 'c'):
        raise NotImplementedError(
            "complex-value support not currently implemented")

    if transpose_type == 'T':
        # transpose the input and then call with transpose_type='N'
        a = a.swapaxes(-1, -2)  # tranpose the last two dimensions
        transpose_type = 'N'
    elif transpose_type == 'C':
        # gufunc = _impl.update_rank1_conjugate
        raise NotImplementedError("transpose_type='C' unimplemented")

    if transpose_type == 'N':
        if UPLO == 'U':
            if c is None:
                if sym_out:
                    gufunc = _impl.update_rankk_no_c_up_sym
                else:
                    gufunc = _impl.update_rankk_no_c_up
            else:
                if sym_out:
                    gufunc = _impl.update_rankk_up_sym
                else:
                    gufunc = _impl.update_rankk_up
        else:
            if c is None:
                if sym_out:
                    gufunc = _impl.update_rankk_no_c_down_sym
                else:
                    gufunc = _impl.update_rankk_no_c_down
            else:
                if sym_out:
                    gufunc = _impl.update_rankk_down_sym
                else:
                    gufunc = _impl.update_rankk_down

    with _setup_gulinalg_threads(workers):
        out = gufunc(a, c, **kwargs)

    if c is None and not sym_out:
        # Have to swap here because update_rankk_no_c* returns with the last
        # two axes transposed for efficiency (due to BLAS Fortran order).
        out = out.swapaxes(-1, -2)
    if not out.flags.c_contiguous:
        out = np.ascontiguousarray(out)
    return out


import requests
from bs4 import BeautifulSoup

indeed_resour = requests.get("https://www.indeed.com/jobs?q=python&limit=50")

indeed_soup = BeautifulSoup(indeed_resour.text, "html.parser")

pagination = indeed_soup.find("div", {"class": "pagination"})

links = pagination.find_all('a')
pages = []
for link in links[:-1]:
    pages.append(int(link.string))

max_page = pages[-1]

for n in range(max_page):
    print(f"start={n*50}")


import itertools
from functools import reduce

import coreapi
import coreschema
from django.conf import settings
from django.db.models import Q
from django.utils import timezone
from django_filters import rest_framework as filters
from guardian.shortcuts import get_objects_for_user
from rest_framework import mixins, viewsets, parsers, status
from rest_framework.decorators import action
from rest_framework.exceptions import NotFound, ValidationError, PermissionDenied
from rest_framework.response import Response

from alerts import engine
from alerts.models import (
    Ticket, TicketLog, Broadcast, TicketComment, PublicAlertMessage,
    AuthorizationRequest, AlertDisconnection, ManualAlertLog,
)
from alerts.modules.base_states import ALERT_STATES, EVENT_STATES, AUTHORIZATION_LEVELS, ALL_TICKET_STATES
from alerts.modules.conditions import get_next_authorization
from api.v1.handlers.alert_utils import get_ticket_bucket
from api.v1.serializers.ticket_serializers import (
    TicketListSerializer,
    TicketSerializer,
    TicketLogSerializer,
    BroadcastSerializer,
    TicketCommentSerializer,
    TicketArchiveActionSerializer,
    PublicAlertMessageSerializer,
    AuthorizationRequestSerializer,
    AuthorizationRequestTicketSerializer,
    AuthorizationRequestCreateSerializer,
    AuthorizationRequestUpdateSerializer,
    AlertDisconnectionSerializer,
    ManualAlertLogSerializer
)
from api.v1.serializers.user_serializers import UserSerializer, serialize_author
from base.filters import FilterListBackend
from base.permissions import Authenticated, UserPermissionChecker
from base import schemas
from documents.models import Document
from documents.utils import serialize_docs_for_upload, upload_docs
from documents.views import DocumentReadOnlyViewSet, DocumentUploadMixin
from remotes.dispatch import send_simple_message, send_simple_smc_message
from targets.mixins import NestedInTargetMixin
from targets.models import Target

class TicketFilter(filters.FilterSet):
    open = filters.CharFilter(
        method='open_filter',
        help_text="Tickets still open",
    )
    archived = filters.BooleanFilter(
        help_text="Archived tickets, which might or might not be open",
    )
    evaluable = filters.BooleanFilter(
        help_text="Evaluable tickets, which might or might not be open",
    )
    module = filters.CharFilter(
        help_text="The exact name of the module specification",
    )
    level = filters.NumberFilter(method='level_filter')
    level_lte = filters.NumberFilter(method='level_filter')
    level_lt = filters.NumberFilter(method='level_filter')
    level_gte = filters.NumberFilter(method='level_filter')
    level_gt = filters.NumberFilter(method='level_filter')
    group = filters.CharFilter(
        method='group_filter',
        help_text="The comma-separated groups the ticket belongs to",
    )

    class Meta:
        model = Ticket
        fields = ('open', 'archived', 'evaluable', 'module')

    def open_filter(self, queryset, name, value):
        if name == 'open' and value is not None and \
                value.lower() in ('true', 'yes', '1', 'y'):
            return queryset.exclude(state__exact=Ticket.TicketState.CLOSED)
        if name == 'open' and value is not None and \
                value.lower() in ('false', 'no', '0', 'n'):
            return queryset.filter(state__exact=Ticket.TicketState.CLOSED)
        return queryset

    def level_filter(self, queryset, name, value):
        op = {
            'level': '',
            'level_lte': '__lte',
            'level_lt': '__lt',
            'level_gte': '__gte',
            'level_gt': '__gt',
        }.get(name)
        if op is None:
            return queryset
        return queryset.filter(**{f'result_state__level{op}': int(value)})

    def group_filter(self, queryset, name, value):
        if name != 'group':
            return queryset
        groups = frozenset(filter(bool, (value or "").split(",")))
        if not groups:
            return queryset
        return queryset.filter(reduce(
            lambda acc, q: acc & q,
            map(lambda g: Q(groups__contains=f"/{g}/"), groups),
        ))

class TicketsView(viewsets.ModelViewSet):
    permission_classes = (Authenticated,)
    filter_backends = [FilterListBackend]
    filterset_class = TicketFilter
    serializer_class = TicketListSerializer

    def has_target_perm(self, target, codename):
        checker = getattr(self, '_checker', None)
        if checker is None:
            checker = UserPermissionChecker(self.request.user)
            checker.prefetch_perms(Target.objects.all())
            setattr(self, '_checker', checker)
        return checker.has_perm(codename, target)

    def get_queryset(self):
        perms = [
            f'targets.ticket.{state}.read'
            for state in ALL_TICKET_STATES
        ]
        targets = get_objects_for_user(
            user=self.request.user,
            perms=perms,
            klass=Target.objects.all(),
            any_perm=True
        )
        if len(targets) > 0:
            ticket_filters = []
            for target in targets:
                state_filters = [
                    Q(state__startswith=state)
                    for state in ALL_TICKET_STATES
                    if self.has_target_perm(target, f'targets.ticket.{state}.read')
                ]
                ticket_filters.append(
                    Q(target=target) & reduce(lambda acc, q: acc | q, state_filters)
                )
            queryset = Ticket.objects.filter(
                reduce(lambda acc, q: acc | q, ticket_filters)
            ).prefetch_related('target')
        else:
            queryset = Ticket.objects.none()
        return queryset

class TicketView(NestedInTargetMixin, viewsets.ReadOnlyModelViewSet):
    permission_classes = (Authenticated,)
    filter_backends = [FilterListBackend]
    filterset_class = TicketFilter

    def get_queryset(self):
        state_filters = [
            Q(state__startswith=state)
            for state in ALL_TICKET_STATES
            if self.has_target_perm(f'ticket.{state}.read')
        ]
        if len(state_filters) > 0:
            queryset = Ticket.objects.filter(
                target=self.target
            ).prefetch_related(
                'target'
            ).filter(
                reduce(lambda acc, q: acc | q, state_filters)
            )
        else:
            queryset = Ticket.objects.none()
        return queryset

    def get_serializer_class(self):
        if self.action == 'list':
            return TicketListSerializer
        return TicketSerializer

    def retrieve(self, request, *args, **kwargs):
        instance = self.get_object()
        state = instance.state_group
        serializer = TicketSerializer(instance, context={
            "children": self.has_target_perm(f'ticket.{state}.children.read'),
            "logs": self.has_target_perm(f'ticket.{state}.log.read'),
            "event_data": self.has_target_perm(f'ticket.{state}.event_data.read'),
        })
        return Response(serializer.data)

    @action(methods=["post"], detail=True, serializer_class=TicketArchiveActionSerializer)
    def archive(self, request, **kwargs):
        ticket = self.get_object()
        action_serializer = TicketArchiveActionSerializer(
            ticket,
            data=request.data,
            context=self.get_serializer_context(),
        )
        action_serializer.is_valid(raise_exception=True)
        action_serializer.save()
        ticket.refresh_from_db()
        return Response(data=self.get_serializer_class()(ticket).data)

class NestedInTicketMixin(NestedInTargetMixin):
    _ticket = None

    def resolve_ticket(self):
        pk = self.kwargs.get("ticket_pk")
        ticket = Ticket.objects.filter(target=self.target, pk=pk).first()
        if ticket is None or not self.has_target_perm(f'ticket.{ticket.state_group}.read'):
            raise NotFound(detail="Ticket not found.")
        return ticket

    @property
    def ticket(self):
        if self._ticket is None:
            self._ticket = self.resolve_ticket()
        return self._ticket

    def has_ticket_perm(self, suffix):
        return self.has_target_perm(f'ticket.{self.ticket.state_group}.{suffix}')

class TicketLogView(NestedInTicketMixin, viewsets.ReadOnlyModelViewSet):
    permission_classes = (Authenticated,)
    serializer_class = TicketLogSerializer
    parser_classes = (parsers.JSONParser, parsers.MultiPartParser)

    def get_queryset(self):
        if not self.has_ticket_perm('log.read'):
            raise PermissionDenied()
        return self.ticket.logs.all()

    def get_serializer_context(self):
        ctx = super().get_serializer_context()
        return {**ctx, "kwargs": self.kwargs}

    def create(self, request, *args, **kwargs):
        if not self.has_ticket_perm('log.write'):
            raise PermissionDenied()

        serializer = self.serializer_class(data=request.data)
        serializer.is_valid(raise_exception=True)
        message = serializer.validated_data.pop('message')
        log = serializer.save(
            ticket=self.ticket,
            author=serialize_author(request.user),
            meta={'message': message}
        )
        folder = self.ticket.get_document_folder('log', log.id)
        for file in request.FILES.values():
            log.documents.add(
                Document.objects.create(
                    folder=folder,
                    file=file,
                    name=str(file),
                    uploaded_by=request.user
                )
            )
        return Response(data=self.serializer_class(log).data, status=201)

class TicketLogDocumentView(NestedInTicketMixin, DocumentReadOnlyViewSet):
    def get_queryset(self):
        if not self.has_ticket_perm('log.read'):
            raise PermissionDenied()
        try:
            authorization = AuthorizationRequest.objects.get(
                ticket=self.ticket, pk=self.kwargs.get("authorization_pk"),
            )
        except AuthorizationRequest.DoesNotExist:
            raise NotFound(detail="AuthorizationRequest not found.")
        return authorization.documents.all()

class BroadcastView(mixins.CreateModelMixin, viewsets.ReadOnlyModelViewSet):
    permission_classes = (Authenticated,)
    serializer_class = BroadcastSerializer
    queryset = Broadcast.objects.all()

    def get_queryset(self):
        return self.queryset.filter(
            ticket=self.kwargs.get("ticket_pk"),
            ticket__target__canonical_name=self.kwargs.get("target_canonical_name"),
        )

    def get_serializer_context(self):
        ctx = super().get_serializer_context()
        return {**ctx, "kwargs": self.kwargs}

    def create(self, request, *args, **kwargs):
        serializer = self.serializer_class(data=request.data, context=self.get_serializer_context())
        serializer.is_valid(raise_exception=True)
        broadcast = serializer.save()
        return Response(data=self.serializer_class(broadcast).data, status=201)

class AuthorizationRequestViewSchema(schemas.CustomSchema):

    @schemas.serializers.post('/authorization/')
    def authorization_fields(self):
        return [
            coreapi.Field(
                name='action',
                required=True,
                location='body',
                schema=coreschema.String(
                    description='Action for which the authorization will be created (close, escalate, archive)',
                ),
            ),
            coreapi.Field(
                name='to_state',
                required=False,
                location='body',
                schema=coreschema.String(
                    description='escalation target state, required only when action=escalate',
                ),
            ),
        ]

class TicketAuthorizationRequestView(NestedInTicketMixin, mixins.CreateModelMixin, viewsets.ReadOnlyModelViewSet):
    permission_classes = (Authenticated,)
    serializer_class = AuthorizationRequestSerializer
    schema = AuthorizationRequestViewSchema.as_schema()

    def get_authorization_read_filters(self):
        state = self.ticket.state_group
        if state in ALERT_STATES:
            states = ALERT_STATES
        elif state in EVENT_STATES:
            states = EVENT_STATES
        else:
            raise ValidationError('Ticket state group unknown')

        permissions = []
        for level in AUTHORIZATION_LEVELS:
            permissions.extend([
                f'close.authorization.{level}.read',
                f'archive.authorization.{level}.read'
            ])
        permissions.extend([
            f'escalate.{to_state}.authorization.{level}.read'
            for to_state, level in itertools.product(states, AUTHORIZATION_LEVELS)
            if state != to_state
        ])

        readable_authorizations = [
            f'ticket.{self.ticket.state_group}.{perm[:-5]}'
            for perm in permissions
            if self.has_ticket_perm(perm)
        ]
        if len(readable_authorizations) > 0:
            return reduce(
                lambda acc, q: acc | q,
                [
                    Q(authorization=auth)
                    for auth in readable_authorizations
                ]
            )

        return None

    def get_queryset(self):
        auth_filters = self.get_authorization_read_filters()
        if auth_filters is None:
            raise PermissionDenied()
        return self.ticket.authorizations.filter(auth_filters)

    def create(self, request, *args, **kwargs):
        ticket = self.ticket
        _action = request.data.get('action', '')
        if _action == 'close':
            conditions = ticket.close_conditions
        elif _action == 'archive':
            conditions = ticket.archive_conditions
        elif _action == 'escalate':
            to_state = request.data.get('to_state', None)
            if to_state is None:
                raise ValidationError({'to_state': 'required'})
            if to_state not in ticket.escalate_conditions:
                raise ValidationError({'to_state': f'no escalate to {to_state} conditions'})
            conditions = ticket.escalate_conditions[to_state]
        else:
            raise ValidationError({'action': f'Action {_action} not supported'})
        authorization = get_next_authorization(conditions, request.user)
        if authorization is None:
            raise ValidationError({'action': 'No new authorization required'})
        if not self.has_target_perm(f'{authorization}.request'):
            raise PermissionDenied()
        if self.get_queryset().filter(
                status=AuthorizationRequest.Status.PENDING,
                authorization=authorization
        ).exists():
            raise ValidationError({'action': 'there is a pending authorization request'})
        auth_request = AuthorizationRequest.objects.create(
            authorization=authorization,
            ticket=ticket,
            created_by=serialize_author(request.user)
        )
        # Creating a new log entry
        log = TicketLog.new_request_entry(
            ticket=ticket,
            author=request.user,
            authorization_string=authorization
        )
        log.save()

        if settings.STACK_IS_SML and '.authority-' in authorization:
            send_simple_smc_message(
                'alerts.ticket.authorization.create',
                AuthorizationRequestCreateSerializer(auth_request).data
            )
        else:
            # update ticket conditions
            engine.update_ticket_conditions(ticket)
        return Response(data=AuthorizationRequestSerializer(auth_request).data, status=201)

    def _resolve_authorization_request(self, request, authorization_request, new_status):
        ticket = self.ticket
        if authorization_request.status != AuthorizationRequest.Status.PENDING:
            raise ValidationError(f'only pending authorizations can be {new_status}')
        serializer = AuthorizationRequestSerializer(
            authorization_request,
            data=request.data,
        )
        serializer.is_valid(raise_exception=True)
        serializer.save(
            resolved_by=serialize_author(request.user),
            resolved_at=timezone.now(),
            status=new_status
        )
        authorization_request.refresh_from_db()
        ticket_id = authorization_request.ticket.id
        folder = f'documents/ticket/{ticket_id}/authorization/{authorization_request.id}/'
        for file in request.FILES.values():
            authorization_request.documents.add(
                Document.objects.create(
                    folder=folder,
                    file=file,
                    name=str(file),
                    uploaded_by=request.user,
                    meta={'request_id': {'value': authorization_request.id}}
                )
            )
        # Creating a new log entry
        log = TicketLog.new_authorization_entry(
            ticket=ticket,
            author=request.user,
            authorization_request=authorization_request,
            new_status=new_status,
        )
        log.save()
        for doc in authorization_request.documents.all():
            log.documents.add(doc)

        if not settings.STACK_IS_SML and authorization_request.origin != settings.NAMESPACE:
            remote = authorization_request.ticket.target.remote
            if remote is not None:
                documents = list(serialize_docs_for_upload(authorization_request.documents.all(), folder))
                bucket = get_ticket_bucket(authorization_request.ticket)
                upload_docs(documents, bucket)
                send_simple_message(
                    'alerts.ticket.authorization.update',
                    body=AuthorizationRequestUpdateSerializer(
                        authorization_request,
                        context={'documents': documents}
                    ).data,
                    exchange=remote.exchange
                )
            else:
                raise ValidationError(f'target remote is missing')
        else:
            # update ticket conditions
            engine.update_ticket_conditions(authorization_request.ticket)
        return Response(data=AuthorizationRequestSerializer(authorization_request).data)

    @action(
        methods=["post"],
        detail=True,
        parser_classes=(parsers.FormParser, parsers.MultiPartParser, parsers.JSONParser),
        serializer_class=AuthorizationRequestSerializer,
    )
    def approve(self, request, **kwargs):
        auth_request = self.get_object()
        if not self.has_target_perm(f'{auth_request.authorization}.resolve'):
            raise PermissionDenied()
        new_status = AuthorizationRequest.Status.APPROVED
        return self._resolve_authorization_request(request, auth_request, new_status)

    @action(
        methods=["post"],
        detail=True,
        parser_classes=(parsers.FormParser, parsers.MultiPartParser, parsers.JSONParser),
        serializer_class=AuthorizationRequestSerializer,
    )
    def deny(self, request, **kwargs):
        auth_request = self.get_object()
        if not self.has_target_perm(f'{auth_request.authorization}.resolve'):
            raise PermissionDenied()
        new_status = AuthorizationRequest.Status.DENIED
        return self._resolve_authorization_request(request, auth_request, new_status)

class AuthorizationRequestDocumentView(DocumentReadOnlyViewSet):
    # TODO use nested in ticket view and check authorization read permissions
    def get_queryset(self):
        try:
            authorization = AuthorizationRequest.objects.get(
                ticket=self.kwargs.get("ticket_pk"),
                ticket__target__canonical_name=self.kwargs.get("target_canonical_name"),
                pk=self.kwargs.get("authorization_pk"),
            )
        except AuthorizationRequest.DoesNotExist:
            raise NotFound(detail="AuthorizationRequest not found.")
        return authorization.documents.all()

class AuthorizationRequestView(viewsets.ReadOnlyModelViewSet):
    permission_classes = (Authenticated,)
    serializer_class = AuthorizationRequestTicketSerializer
    # TODO fix this queryset based on permissions
    queryset = AuthorizationRequest.objects.all()

class TicketCommentView(NestedInTicketMixin, viewsets.ModelViewSet):
    serializer_class = TicketCommentSerializer

    def get_queryset(self):
        state_group = self.ticket.state_group
        read_perms = [
            (
                f'ticket.{state_group}.close_report_comment.read',
                TicketComment.CommentType.CLOSE_REPORT
            ),
            (
                f'ticket.{state_group}.alert_management_comment.read',
                TicketComment.CommentType.ALERT_MANAGEMENT
            ),
            (
                f'ticket.{state_group}.alert_complementary_comment.read',
                TicketComment.CommentType.ALERT_COMPLEMENTARY
            ),
            (
                f'ticket.{state_group}.event_management_comment.read',
                TicketComment.CommentType.EVENT_MANAGEMENT
            )
        ]
        type_filters = [
            Q(comment_type=c_type)
            for c_perm, c_type in read_perms
            if self.has_target_perm(c_perm)
        ]
        if len(type_filters) > 0:
            queryset = self.ticket.comments.filter(
                reduce(lambda acc, q: acc | q, type_filters)
            )
        else:
            raise PermissionDenied()
        return queryset

    def perform_create(self, serializer):
        comment = serializer.save(ticket=self.ticket, created_by=UserSerializer(self.request.user).data)
        log = TicketLog.new_comment_entry(
            ticket=self.ticket,
            author=self.request.user,
            meta={
                "comment": comment.content,
                "description": "comment",
                "comment_id": comment.id,
                "comment_type": comment.comment_type
            },
        )
        log.save()
        folder = self.ticket.get_document_folder('comment', comment.id)
        for file in self.request.FILES.values():
            doc = Document.objects.create(
                folder=folder,
                file=file,
                name=str(file),
                uploaded_by=self.request.user
            )
            comment.documents.add(doc)
            log.documents.add(doc)
        engine.update_ticket_conditions(self.ticket)
        return log

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        data = serializer.validated_data
        types = list(c_type for c_type, _ in TicketComment.CommentType.choices)
        c_type = data.get('comment_type', 'None')
        if c_type not in types or not self.has_ticket_perm(f'{c_type}_comment.write'):
            raise PermissionDenied()
        self.perform_create(serializer)
        return Response(serializer.data, status=status.HTTP_201_CREATED)

class TicketCommentDocumentView(NestedInTicketMixin, DocumentReadOnlyViewSet):
    def get_queryset(self):
        comment_pk = self.kwargs.get('comment_pk')
        comment = self.ticket.comments.filter(pk=comment_pk).first()
        if comment is None:
            raise NotFound(detail="Ticket comment not found.")
        if not self.has_ticket_perm(f'{comment.comment_type}_comment.read'):
            raise PermissionDenied()
        queryset = comment.documents.all()
        return queryset

class PublicAlertMessageFilter(filters.FilterSet):
    alert_type = filters.CharFilter(method='alert_type_filter')
    scope = filters.CharFilter(method='scope_filter')

    class Meta:
        model = PublicAlertMessage
        fields = ['alert_type', 'scope']

    def alert_type_filter(self, queryset, name, value):
        if name == 'alert_type' and value is not None:
            return queryset.filter(**{name: value, })
        return queryset

    def scope_filter(self, queryset, name, value):
        if name == 'scope' and value is not None:
            return queryset.filter(**{name: value, })
        return queryset

class PublicAlertMessageView(viewsets.ModelViewSet):
    serializer_class = PublicAlertMessageSerializer
    filter_backends = [FilterListBackend]
    filter_class = PublicAlertMessageFilter

    def get_queryset(self):
        target_canonical_name = self.kwargs.get('target_canonical_name')
        try:
            target = Target.objects.get(canonical_name=target_canonical_name)
        except Target.DoesNotExist:
            raise NotFound(detail="Target not found.")
        queryset = target.alert_messages.all()
        return queryset

    def perform_create(self, serializer):
        target_canonical_name = self.kwargs.get('target_canonical_name')
        try:
            target = Target.objects.get(canonical_name=target_canonical_name)
        except Target.DoesNotExist:
            raise NotFound(detail="Target not found.")
        serializer.save(target=target, created_by=UserSerializer(self.request.user).data)

class AlertDisconnectionFilter(filters.FilterSet):
    scope = filters.CharFilter(method='scope_filter')

    class Meta:
        model = AlertDisconnection
        fields = ['scope']

    def scope_filter(self, queryset, name, value):
        if name == 'scope' and value is not None:
            return queryset.filter(**{name: value, })
        return queryset

class AlertDisconnectionsView(viewsets.ModelViewSet):
    permission_classes = (Authenticated,)
    filter_backends = [FilterListBackend]
    filterset_class = AlertDisconnectionFilter
    serializer_class = AlertDisconnectionSerializer
    queryset = AlertDisconnection.objects.all()

class TargetAlertDisconnectionsView(NestedInTargetMixin, viewsets.ModelViewSet):
    permission_classes = (Authenticated,)
    filter_backends = [FilterListBackend]
    filter_class = AlertDisconnectionFilter
    serializer_class = AlertDisconnectionSerializer

    def get_queryset(self):
        queryset = self.target.alert_disconnections.all()
        return queryset

    def perform_create(self, serializer):
        target_canonical_name = self.kwargs.get('target_canonical_name')

        # scope = serializer.data
        # if not self.has_target_perm(f'ticket.{scope}.alert_disconnection.write'):
        #     raise PermissionDenied()

        try:
            target = Target.objects.get(canonical_name=target_canonical_name)
        except Target.DoesNotExist:
            raise NotFound(detail="Target not found.")
        serializer.save(target=target, created_by=UserSerializer(self.request.user).data)

class AlertDisconnectionDocumentView(DocumentReadOnlyViewSet, DocumentUploadMixin, mixins.DestroyModelMixin):
    permission_classes = (Authenticated,)

    def get_queryset(self):
        disconnection_pk = self.kwargs.get('disconnection_pk')
        try:
            disconnection = AlertDisconnection.objects.get(pk=disconnection_pk)
        except AlertDisconnection.DoesNotExist:
            raise NotFound(detail="Alert disconnection not found.")
        queryset = disconnection.documents.all()
        return queryset

    def perform_document_create(self, file_obj, description, meta, request, type):
        doc = super().perform_document_create(file_obj, description, meta, request, type)
        disconnection_pk = self.kwargs.get('disconnection_pk')
        try:
            disconnection = AlertDisconnection.objects.get(pk=disconnection_pk)
        except AlertDisconnection.DoesNotExist:
            raise NotFound(detail="Alert disconnection not found.")
        disconnection.documents.add(doc)
        return doc

    def get_folder(self):
        disconnection_pk = self.kwargs.get('disconnection_pk')
        return f'disconnection/{disconnection_pk}'

class TargetManualAlertLogsView(NestedInTicketMixin, viewsets.ModelViewSet):
    permission_classes = (Authenticated,)
    filter_backends = [FilterListBackend]
    # filter_class = ManualAlertLog
    serializer_class = ManualAlertLogSerializer

    def get_queryset(self):
        queryset = self.target.manual_alerts.all()
        return queryset

    def perform_create(self, serializer):
        # scope = serializer.data
        # if not self.has_target_perm(f'ticket.{scope}.alert_disconnection.write'):
        #     raise PermissionDenied()

        serializer.save(target=self.target, ticket=self.ticket, created_by=UserSerializer(self.request.user).data)

class ManualAlertLogDocumentView(DocumentReadOnlyViewSet, DocumentUploadMixin, mixins.DestroyModelMixin):
    permission_classes = (Authenticated,)

    def get_queryset(self):
        manual_pk = self.kwargs.get('manual_pk')
        try:
            manual_alert_log = ManualAlertLog.objects.get(pk=manual_pk)
        except ManualAlertLog.DoesNotExist:
            raise NotFound(detail="Manual alert log not found.")
        queryset = manual_alert_log.documents.all()
        return queryset

    def perform_document_create(self, file_obj, description, meta, request):
        doc = super().perform_document_create(file_obj, description, meta, request)
        manual_pk = self.kwargs.get('manual_pk')
        try:
            manual_alert_log = ManualAlertLog.objects.get(pk=manual_pk)
        except AlertDisconnection.DoesNotExist:
            raise NotFound(detail="Manual alert log not found.")
        manual_alert_log.documents.add(doc)
        return doc

    def get_folder(self):
        manual_pk = self.kwargs.get('manual_pk')
        return f'manual/{manual_pk}'


from enum import Enum
import json

class PROTOCOL(Enum):
    """
    游戏协议定义
    根据实际游戏协议扩展
    """

    C2SHeartbeat = 100  # 心跳包
    C2SSignup = 101  # 报名参加游戏
    C2SStart = 102  # 游戏开始
    C2SGameover = 103  # 主播主动结束游戏
    C2SPlayerScore = 104  # 玩家上报得分

    S2CHeartbeat = 200  # 心跳响应包
    S2CRoomClosed = 201  # 游戏房间已经结束关闭
    S2CGameStart = 202  # 游戏开始
    S2CGameOver = 203  # 游戏结束
    S2CPlayerJoin = 204  # 玩家进入
    S2CPlayerSignup = 205  # 玩家报名
    S2CRealtimeRank = 206  # 实时分数排行榜
    S2CTeams = 207  # 队伍信息


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Tue Aug 28 14:52:26 2018

@author: arje
"""

import utility
import numpy as np
from sklearn import metrics
import cloudpickle as pickle
import dim2_keras as dim2

"""
hyper-parameters
"""
learning_rate = [0.001, 0.0001,0.00001, 0.000001] #parameter used in gradient descent optimization
epochs_schedule = [50, 50, 50, 50] #how many times to go over the training obs during optimizing
regularizer = 0.0001 #?
target_shape = [64, 64] # the size of the patches given to the network
batch_size = 128 #the number of training obs to use at once

"""
data
"""
train_images = np.load('./data/train_new.npy') #which training set to load
train_annotation = '../data/train_2class_new.txt' #img id, coordinates of bb, class
val_images = np.load('../data/val_new.npy') #which validation set to load
val_annotation = '../data/val_2class_new.txt'
test_images = np.load('../data/test_new.npy')
test_annotation = '../data/test_2class_new.txt'

#Creates generators for object patches
train_gen, train_steps = utility.get_patch_generator_1class_new(train_images, train_annotation, target_shape, batch_size, True) #creates train patches from the original images
val_gen, val_steps = utility.get_patch_generator_1class_new(val_images, val_annotation, target_shape, batch_size, False)

"""
Model setup
"""
#Read in the model structure
model = dim2.dim2_model((target_shape[0],target_shape[1],3), 2)
#model = md.get_flower_model((target_shape[0],target_shape[1],3),2)

#Initialize the model
current_weights = model.get_weights()
optimal_weights = model.get_weights() #initialization
optimal_val_acc = 0
cb = utility.CustomCallback()
history = {}

""" network training """

for lr, epoch in zip(learning_rate, epochs_schedule): #zip goes over two lists at the same time

    model.optimizer.lr = lr #sets learning rate for model
    model.set_weights(current_weights) #sets weights for model

    h = model.fit_generator(train_gen, train_steps, epoch, callbacks=[cb,],
                            validation_data=val_gen, validation_steps=val_steps) #a generator is used so we don't keep unnecessary things in memory

    history = utility.merge_history(history, h.history) #history returns training and validation loss and other metrics, so basically the fit of the model

    if optimal_val_acc < cb.monitor_quantity:
        optimal_weights = cb.model_weights #saves weights of the best model

    current_weights = model.get_weights() #updates the model weights and goes to next lr & epoch

"""
evaluate the best model with validation set
"""
model.set_weights(optimal_weights) #use the best possible weights
val_gen, val_steps = utility.get_patch_generator_1class_scale(val_images, val_annotation, target_shape, batch_size, False) #same as before

y_pred = np.zeros((1,2)) #initialize zero arrays of length 5
y_true = np.zeros((1,2))

for iteration in range(val_steps):
    #python 3 version:
    x, y = next(val_gen)
    #x, y = val_gen.next() #gives values for these both
    y_true = np.concatenate((y_true, y), axis=0) #concatenate adds the new value to the array
    y_pred = np.concatenate((y_pred, model.predict(x)), axis=0)

y_true = np.argmax(y_true[1:], axis=-1) #leaves out the first row and searches maximums for all other rows, y_true's are arrays with 1 for the true class and 0 for the others
y_pred = np.argmax(y_pred[1:], axis=-1) #y_pred's are arrays with the probabilities of all classes. This takes just the most probable label.

confusion = metrics.confusion_matrix(y_true, y_pred)
performance = {'val_confusion':confusion} #creating dictionary
performance['val_accuracy'] = metrics.accuracy_score(y_true, y_pred) #adding to the dictionary
performance['val_precision'] = metrics.precision_score(y_true, y_pred, average='macro')
performance['val_recall'] = metrics.recall_score(y_true, y_pred, average='macro')
performance['val_f1'] = metrics.f1_score(y_true, y_pred, average='macro')
performance['history'] = history
performance['weights'] = optimal_weights

"""
evaluate the best model with test set
"""
model.set_weights(optimal_weights) #use the best possible weights
test_gen, test_steps = utility.get_patch_generator_1class_scale(test_images, test_annotation, target_shape, batch_size, False) #same as before

y_pred = np.zeros((1,2)) #initialize zero arrays of length 5
y_true = np.zeros((1,2))

for iteration in range(test_steps):
    #python 3 version:
    x, y = next(test_gen)
    #x, y = test_gen.next() #gives values for these both
    y_true = np.concatenate((y_true, y), axis=0) #concatenate adds the new value to the array
    y_pred = np.concatenate((y_pred, model.predict(x)), axis=0)

y_true = np.argmax(y_true[1:], axis=-1) #leaves out the first row and searches maximums for all other rows, y_true's are arrays with 1 for the true class and 0 for the others
y_pred = np.argmax(y_pred[1:], axis=-1) #y_pred's are arrays with the probabilities of all classes. This takes just the most probable label.

confusion = metrics.confusion_matrix(y_true, y_pred)
performance['test_confusion'] = confusion #creating dictionary
performance['test_accuracy'] = metrics.accuracy_score(y_true, y_pred) #adding to the dictionary
performance['test_precision'] = metrics.precision_score(y_true, y_pred, average='macro')
performance['test_recall'] = metrics.recall_score(y_true, y_pred, average='macro')
performance['test_f1'] = metrics.f1_score(y_true, y_pred, average='macro')

print(performance['val_accuracy'])
print(performance['val_confusion'])
print(performance['test_accuracy'])
print(performance['test_confusion'])

output_file = './result/model_2class_dim_keras_new.pickle'
fid = open(output_file,'wb')
pickle.dump(performance, fid) #just a way of saving the performance results
fid.close()


# -*- coding: utf-8 -*-
import os
import json
import re

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request
from scrapy.utils.url import urljoin_rfc, add_or_replace_parameter
from scrapy.utils.response import get_base_url

from product_spiders.utils import extract_price_eu
from product_spiders.items import Product, ProductLoaderWithNameStrip as ProductLoader

HERE = os.path.abspath(os.path.dirname(__file__))

class Home24Spider(BaseSpider):
    name = 'made_de-home24.de'
    allowed_domains = ['home24.de']
    start_urls = ['http://www.home24.de/moebel/?order=name_desc',
                  'http://www.home24.de/lampen-shop/?order=name_desc',
                  'http://www.home24.de/wohnaccessoires/?order=name_desc']

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        base_url = get_base_url(response)

        next_page = response.xpath('//div[contains(@class,"pagination")]/a[@rel="next"]/@href').extract()
        if next_page:
            yield Request(urljoin_rfc(base_url, next_page[0]))

        products = response.xpath('//div[@class="article-tile__wrap"]')
        for product in products:
            url = product.select('.//a/@href')[0].extract()
            image_url = product.select('.//div[@class="article-tile__images"]/div/img/@data-echo')[0].extract()
            price = product.select('.//span[contains(@class,"article__price ")]/text()')[0].extract()
            yield Request(urljoin_rfc(base_url, url), callback=self.parse_product, meta={'image_url': image_url, 'price': price})

    def parse_product(self, response):
        hxs = HtmlXPathSelector(response)
        base_url = get_base_url(response)

        for url in response.xpath('//meta[@itemprop="isSimilarTo"]/@content').extract():
            yield Request(url, meta=response.meta, callback=self.parse_product)
            yield Request(url.replace('undefinedundefined', '.de/'), callback=self.parse_product)

        sku = response.xpath('//*[@itemprop="sku"]/@content').extract()
        name = response.xpath('//script/text()').re('"productName": "(.+)"')
        brand = response.xpath('//script/text()').re('"productBrand": "(.+)"')
        categories = response.css('.breadcrumbs').xpath('.//span[@itemprop="name"]/text()').extract()[1:]

        options = response.css('.mvc-configurator__options')
        data = re.findall('window.__INITIAL_STATE__ = (.+)<', response.body)[0]
        data = json.loads(data)

        if options:
            for option in data['children'].itervalues():

                loader = ProductLoader(item=Product(), selector=hxs)
                loader.add_value('identifier', option['trackingData']['sku'])
                loader.add_value('sku', sku)
                loader.add_value('url', response.url)
                loader.add_value('name', option['trackingData']['name'])
                loader.add_value('image_url', response.urljoin(option['images'][0]['big']['src']))
                loader.add_value('price', option['trackingData']['price'])
                loader.add_value('brand', brand)
                for category in categories:
                    loader.add_value('category', category)

                yield loader.load_item()
        else:
            #name = response.xpath('//div[@itemprop="name"]/h1/text()').extract()
            sub_name = response.xpath('//div[@itemprop="name"]/span/text()').extract()
            name = name[0].strip() if name else ''
            loader = ProductLoader(item=Product(), selector=hxs)
            identifier = re.sub('-C$', '', sku[0])
            loader.add_value('identifier', identifier)
            loader.add_value('sku', sku)
            loader.add_value('url', response.url)
            loader.add_value('name', name)
            loader.add_xpath('price', '//script/text()', re='"productPrice": (.+),')
            loader.add_value('brand', brand)
            loader.add_value('image_url', response.urljoin(data['children'].values()[0]['images'][0]['big']['src']))
            for category in categories:
                loader.add_value('category', category)

            yield loader.load_item()


from django.db import models

FOOD_CHOICES = (
  ("beef", "Beef"),
  ("lamb", "Lamb"),
  ("cheese", "Cheese"),
  ("farmed salmon", "Farmed Salmon"),
  ("turkey", "Turkey"),
  ("chicken", "Chicken"),
  ("canned tuna", "Canned Tuna"),
  ("eggs", "Eggs"),
  ("potatoes", "Potatoes"),
  ("rice", "Rice"),
  ("peanut butter", "Peanut Butter"),
  ("nuts", "Nuts"),
  ("yogurt", "Yogurt"),
  ("broccoli", "Broccoli"),
  ("tofu", "Tofu"),
  ("dry beans", "Dry Beans"),
  ("milk (2%)", "Milk (2%)"),
  ("tomatoes", "Tomatoes"),
  ("lentils", "Lentils"),
)

# Create your models here.

class food(models.Model):
  name = models.CharField(max_length=200)
  emissons = models.IntegerField(default=0)

class transport(models.Model):
  name = models.CharField(max_length=200)
  emissons = models.IntegerField(default=0)


'''

http://archive.ics.uci.edu/ml/datasets/Ecoli
'''
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl
import warnings

from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
from sklearn.preprocessing import LabelEncoder
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import RandomForestClassifier
mpl.rcParams['font.sans-serif'] = [u'SimHei']
mpl.rcParams['axes.unicode_minus'] = False
warnings.filterwarnings('ignore')

def R2_score(y_true, y_pred):
    assert y_pred.shape == y_true.shape, '输入的{}与{}不相等'.format(y_pred.shape, y_true.shape)
    return np.sum(np.square(y_pred - y_true)) / np.sum(np.square(y_pred - np.mean(y_true)))

# 数据加载
names = ['mcg', 'gvh', 'lip', 'chg', 'aac', 'alm1', 'alm2', 'classtic']
df = pd.read_csv('./datas/ecoli.data', sep='  ', names=names)
le = LabelEncoder()
X = df.iloc[:, 0:-1]
Y = pd.DataFrame(le.fit_transform(df.iloc[:, -1]))
# print(Y)

# 数据分割

x_train2,x_test2,y_train2,y_test2 = train_test_split(X, Y, test_size=0.5, random_state=0)

print ("训练样本数量%d，测试样本数量:%d" % (x_train2.shape[0], x_test2.shape[0]))
## 比较
estimators = [1,50,100,500]
depth = [1,2,3,7,15]
err_list = []
for es in estimators:
    es_list = []
    for d in depth:
        tf = RandomForestClassifier(n_estimators=es, criterion='gini', max_depth=d, random_state=0)
        tf.fit(x_train2, y_train2)
        st = tf.score(x_test2, y_test2)
        err = 1 - st
        es_list.append(err)
        print ("%d决策树数目，%d最大深度，正确率:%.2f%%" % (es, d, st * 100))
    err_list.append(es_list)

## 画图
plt.figure(facecolor='w')
i = 0
colors = ['r','b','g','y']
lw = [1,2,4,3]
max_err = 0
min_err = 100
for es,l in zip(estimators,err_list):
    plt.plot(depth, l, c=colors[i], lw=lw[i], label=u'树数目:%d' % es)
    max_err = max((max(l),max_err))
    min_err = min((min(l),min_err))
    i += 1
plt.xlabel(u'树深度', fontsize=16)
plt.ylabel(u'错误率', fontsize=16)
plt.legend(loc='upper left', fancybox=True, framealpha=0.8, fontsize=12)
plt.grid(True)
plt.xlim(min(depth),max(depth))
plt.ylim(min_err * 0.99, max_err * 1.01)
plt.title(u'随机森林中树数目、深度和错误率的关系图', fontsize=18)
plt.show()

项目实战-蛋白质定位点分析/基于随机森林 + 决策树对蛋白质定位点分析(准确性0.23).py

基于随机森林 + 决策树对蛋白质定位点分析(准确性0.23).py


# crutial import for backend to run py itself
import os, sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from server import db
from backend.user_db import *
from backend.auth import *
import random
import base64

from .progression_bar import printProgressBar
import time

image_dir = 'data/images'

def load_image(n):
    images = EateryImages()
    printProgressBar(0, n, prefix = 'Progress:', suffix = '', length = 50)
    for eid in range(0,n):
        time.sleep(0.1)
        for img in images.get_images(8):
            image = Image(eid, img)
            db.session.add(image)
            db.session.commit()
        printProgressBar(eid + 1, n, prefix = 'Loading Images:', suffix = '', length = 50)
    pass

class EateryImages:
    def __init__(self) -> None:
        self.data = self.read_images()

    def read_images(self):
        res = []
        for subdir, dirs, files in os.walk(image_dir):
            for file in files:
                img = self.read_img(os.path.join(subdir, file))
                res.append(img)
        return res

    # read one images
    def read_img(self, path):
        with open(path, "rb") as img_file:
            my_string = 'data:image/png;base64,' + base64.b64encode(img_file.read()).decode('utf-8')
        return my_string

    def get_images(self, num):
        image_list = self.data
        length = len(image_list)
        num = min(num, length)
        idx_list = random.sample(range(0,length),num)
        res = []
        for i in idx_list:
            res.append(image_list[i])
        return res


#!/home/scollis/anaconda/bin/python
from __future__ import print_function
import netCDF4
#import time

def grab_hrrr_slice(outname, reftime_index, time1_index):
    """ Grab select variables and reftime/time1 slice from HRRR file. """

    ncep_hrrr = netCDF4.Dataset('http://thredds-jumbo.unidata.ucar.edu/thredds/dodsC/grib/NCEP/HRRR/CONUS_2p5km/TwoD')

    # create the output NetCDF file, open for writing
    out_dset = netCDF4.Dataset(outname, 'w')

    # create the dimensions in the output netCDF file
    for dim in ['x', 'y', 'isobaric']:
        out_dset.createDimension(dim, len(ncep_hrrr.dimensions[dim]))

    # copy selected data for select variables to output NetCDF file
    var_names = ['Planetary_boundary_layer_height_surface',
                 'Dewpoint_temperature_isobaric',
                 'Temperature_isobaric',
                 'Pressure_reduced_to_MSL_msl',
                 'u-component_of_wind_isobaric',
                 'v-component_of_wind_isobaric']

    for var_name in var_names:
        print(var_name)
        in_var = ncep_hrrr.variables[var_name]
        in_units = ncep_hrrr.variables['time'].units
        print(in_units)
        out_var = out_dset.createVariable(var_name, in_var.dtype, in_var.dimensions[2:])
        out_units = out_dset.createVariable(var_name + 'timestamp', 'S1')
        out_units[:] = in_units[:]
        out_var[:] = in_var[reftime_index, time1_index][:]
        out_var.units = ncep_hrrr.variables['time'].units
    pblh = out_dset.variables['Planetary_boundary_layer_height_surface'][0, 0:20]
    print(pblh)
    out_dset.timestamp = str(ncep_hrrr.variables['time'][reftime_index, time1_index]) + " :: " + ncep_hrrr.variables['time'].units
    out_dset.close()

if __name__ == "__main__":
    file_num = open('/data/san_store/dap_hrrr/current.txt', 'r+')
    cnum = file_num.readlines()
    num = int(cnum[-1])
    print('hello world')
    # reftime index to copy to output file
    time1_index = 0     # time1 index to copy to output file
    for time1_index in range(10):
        grab_hrrr_slice('/data/san_store/dap_hrrr/selected_data_%03d_%03d.nc' %(num, time1_index), -1, time1_index)
    #time.sleep(3600)
    num += 1
    file_num.write(str(num) + '\n')
    file_num.close()


from flask import Flask, jsonify, request, render_template, redirect, url_for
from flask_restless import APIManager
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.orm import sessionmaker
import os , hashlib
from base64 import encodestring
from model import *

template_dir = os.path.abspath( "../template/" )
static_dir = os.path.abspath( "../static/" )

app = Flask( "Server" , template_folder = template_dir , static_folder = static_dir)
# Connecting to the database
app.config.from_pyfile ( 'config.py' )
db = SQLAlchemy( app ) # database object

# Adding routes over here

# Restaurant
@app.route( "/restaurants/<name>" , methods=['GET' , 'POST' , 'PUT','DELETE'] )
def restaurants(name):
    if request.method == 'GET':
        restaurant = Restaurant.query.filter( name == name ).first()
        location = Location.query.filter( Location.business_id == restaurant.business_id).first()
        restaurant.location = location
        result = restaurant_schema.dump( restaurant )

    elif request.method == 'PUT':
        update_rest = Restaurant.query.filter( name == name ).first()
        update_rest.update( request.args )
        db.session().commit()
        result = restaurant_schema.dump( update_rest )

    elif request.method == 'POST':
        args =  dict(request.args)
        new_rest = Restaurant(business_id= 'RNANDOM_STR' ,
                            name=name,
                            review_count=0 ,
                            is_open=1 ,
                            stars=0.0 )
        db.session().add( new_rest )
        db.session().commit()
        result = restaurant_schema.dump( new_rest )

    elif request.method == 'DELETE':
        deleted_rest = Restaurant.query.filter( name == name ).first()
        Restaurant.query.filter( name == name ).delete()
        result  = restaurant_schema.dump( deleted_rest)
    return jsonify(result.data )

@app.route("/restaurants/" , methods=['GET'])
def restaurant():
    restaurants = Restaurant.query.limit( 10 ).all()
    result=[]
    for rest in restaurants:
        rest.location = Location.query.filter(
            Location.business_id == rest.business_id).first()
    result = restaurants_schema.dump( restaurants )
    return jsonify( result.data )

@app.route("/restaurants/location/<name>", methods=['POST'])
def open_new_location( name ):
    restaurants = Restaurant.query.filter( name==name).first()
    args = request.args

    new_location = Location(address=args['address'],
                            city = args['city'],
                            postal_code=args['postcode'],
                            phone_number=args['phone_number'],
                            business_id=restaurant.business_id)
    db.session().add( new_location )
    db.session().commit()

    result = restaurant_schema.dump( restaurants )
    return jsonify( result.data )

@app.route("/locations/<id>", methods=['GET' ,'PUT','DELETE'])
def location(id):
    location = Location.query.filter( Location.location_id == id ).first()
    result = location_schema.dump( location )
    if request.method == 'GET':
        return jsonify( result.data )
    elif request.method == 'PUT':
        location.update( request.args )
        db.session().commit()
    elif request.method == 'DELETE':
        location.delete()
        db.session().commit()

    return jsonify( result.data )

if __name__ == "__main__":
    app.run( debug=True , port= 5050)


#!/usr/bin/env python
# coding: utf-8

# In[20]:

import numpy as np
from numpy import genfromtxt
from sklearn import linear_model
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

# In[21]:

# 读入数据
data = genfromtxt(r"/Users/han/Desktop/query_result.csv",delimiter=',')
print(data)

# In[22]:

# 切分数据
x_data = data[:,1:-1]
y_data = data[:,-1]
print(x_data)
print(y_data)

# In[23]:

# 创建模型
model = linear_model.LinearRegression()
model.fit(x_data, y_data)

# In[24]:

# 系数
print("coefficients:",model.coef_)

# 截距
print("intercept:",model.intercept_)

# 测试
# 8.83	77.9689	7.95	8.73	76.2129	54.51	2971.3401	6.67
#x_test = [[8.83,77.9689,7.95,8.73,76.2129,54.51,2971.3401,]]

x_test = [[5.89,34.70,5.66,5.05,25.47,41.50,1722.25,]]
predict = model.predict(x_test)
print("predict:",predict)

# In[25]:

# 1082	5.57	31.02	4.52	4.11	16.85	41.50	1722.25	6.09
#x_test = [[5.89,34.70,5.66,5.05,25.47,41.50,1722.25,]]
#x_test = [[5.57,31.02,4.52,4.11,16.85,41.50,1722.25,]]
x_test = x_data
predict = model.predict(x_test)

print("predict:",predict)

# In[19]:

plt.scatter(predict, y_data)
plt.show()

# In[ ]:

# In[ ]:

# In[7]:

print(model.score(x_data, y_data))

# In[ ]:


#encoding=utf-8
# dialog thread
# ganben
# object: target a asyn threading handle conversation and context;

import threading
import logging
import time
import Queue
# from .. protocol.omessage import Omessage
# system wide imported module
import jieba
import random
import time
# from .. models.converitem import ConversationItem
from models.conversation import Conversation
from models.dbcon import DBconn
from models.profile import Profile
from modechecker import Checkers

class Dthread(threading.Thread):
    # db = DBConn().DB
    def __init__(self, threadID, conn, db):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.in_que = Queue.Queue()  # input queue of this instance;
        self.o_que = Queue.Queue()  #out put queuqe to response to socket client;
        self.conn = conn
        # self.conv = conv
        # self.item = item
        self.logger = logging.getLogger(__name__)
        self.profile = Profile()
        self.profile.intent = []
        self.state = 0 # for initial state, is 0, for mode, is 1, 2, 3 etc.
        self.db = db
        self.status = True
        self.checker = Checkers(self.profile)
        self.flag = time.time()
        self.conv = Conversation(threadID, self.db)
        self.qstate = 0 #0 = initial, 1, HQ, 2, HA, 3, AQ, 4, AA, with mode 1 2 3 and profile data, make it a dict
        # print "start thread %s " % threadID
        self.conn.send('welcome to chatserver with default profile! \n')
        self.logger.info('thread created %s ', self.threadID)

#TODO implement run and stop method, make it input ouput split

    def run(self):
        while True:
            if self.status:
                #receive msg here
                data = self.conn.recv(4096)
                if not data:
                    break

                self.in_que.put(data)
                self.conv.add_dialog(data, 'HQ')

                if not self.in_que.empty():
            # sr = jieba.cut("这是一条有味道的回复")
            #     o = [self.threadID, self.in_que.get()]
                    r = self.checker.rescheck(self.in_que.get())

                    self.o_que.put(r)
                if not self.o_que.empty():

                    try:
                        self.conn.send(self.o_que.get())
                    except:
                        self.logger.warning('send msg error, thread quit', self.threadID)
                        self.status = False
                # TODO: here we need to update the NN status, profile intent or whatsoever
                # self.flag = time.time() #currently no time count is needed

            else:
                self.logger.debug('turn off thread')
                self.stop()
            #     delta =  time.time() - self.flag
            #     if delta > 10:
            #         print "quit thread %s " % self.threadID
            #         self.stop()

            # print sr
            # print "thread - %s" % self.threadID

    def put_msg(self, message):  #meaningless method
        # self.in_que.put(" [Re]: %s" % message)
        self.conv.add_dialog(message, "HQ")

        d_table = modejudge(message)    ### check self.state if in any mode (1, 2, 3)
        self.logger.info('modejudge finished ')
        if d_table[0]:
            self.o_que.put("Entering command mode: ")
            self.state = 1
        else:
            if d_table[1]:
                self.o_que.put("Entering command mode: ")
                self.state = 2

        self.in_que.put(message)     #after the state is determined, go to in queue and let the runner process the answer

    def stop(self):
        self.status = False
        self.conv.close()
        # self.item.close()
        # print "thread %s stop called" % self.threadID
        self.logger.info('thread %s is stopping', self.threadID)
        super(Dthread, self).join(0.1)

#private methodes;

def modejudge(self, input):   #Step 1 : mode check, generate a parallel mode judge result array: Q: dose it inlet self, private or public
    res = []
    res.append( self.checker.mode1check(input))  # split words with command dict, #later with profile dict, call a class mode
    res.append( self.checker.mode2check(input))   # split words with namespace(pronoun) check and question check, and dict

    return res


"""
SSAGroup spider created on the top of ATSSpider

scrapy crawl ssagroup -a mining_job_id=9999 -a iteration=1 -a extract=1 -a url="http://www.ssaltd.com"

Sample URL:
    http://www.ssaltd.com
"""

from zlib import crc32
from scrapy.http import Request
from scrapy.selector import Selector
from urlparse import urljoin

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import NormalizedJoin, Prefix, RemoveBadElements

class SSAGroup(ATSSpider):

    name = 'ssagroup'

    def parse(self, response):
        """
            Parse all country and get call to each one
        """
        sel = Selector(response)
        for option in sel.xpath(
            '//div/select[@class="select-region"]/option'
        ):
            href = option.xpath('./@value').extract()
            if href:
                url = urljoin(response.url, '%svacancies/' % href[0])
                yield Request(
                    callback=self.parse_jobs_list,
                    meta={
                        'country': option.xpath('./text()').extract(),
                    },
                    url=url
                )

    def parse_jobs_list(self, response):
        sel = Selector(response)
        for job_url in sel.xpath(
            '//section/article[contains(@class, "vacancies")]/h2/a/@href'
        ).extract():
            yield Request(
                callback=self.parse_job_callback(),
                meta=response.meta,
                url=job_url
            )

        # pagination
        next_page = sel.xpath(
            '//ol[@class="wp-paginate"]/li/a[@class="next"]/@href'
        ).extract()
        if next_page:
            yield Request(
                callback=self.parse_jobs_list,
                meta=response.meta,
                url=next_page[0]
            )

    def parse_job(self, response):
        """
        Extract all required information.
        """
        sel = Selector(response)
        location = sel.xpath(
            '//ul/li[@class="location"]/text()'
        ).extract()
        if location:
            location.extend(response.meta.get('country', []))

        loader = BrightcorpItemLoader(selector=sel)
        loader.add_xpath(
            'title',
            '//div[@class="main-title"]/div/h2/text()'
        )
        loader.add_value(
            'location', location, NormalizedJoin(', ')
        )
        loader.add_value(
            'referencenumber',
            str(crc32(response.url)),
            Prefix('%s-' % self.name)
        )
        loader.add_value('url', response.url)
        loader.add_xpath(
            'description',
            '//div[@class="container"]/article/div[@class="text"]/node()[following-sibling::h1]',
            RemoveBadElements(['a', 'img', ])
        )
        loader.add_xpath(
            'jobtype',
            '//ul/li[@class="type"]/text()'
        )
        loader.add_xpath(
            'jobcategory',
            '//ul/li[@class="sectors"]/text()'
        )
        loader.add_xpath(
            'baseSalary',
            '//ul/li[@class="salary"]/text()'
        )
        loader.add_value('apply_url', response.url)

        yield loader.load_item()


from functools import cached_property
from typing import Optional

from models_library.basic_types import BootModeEnum, LogLevel
from pydantic import Field, SecretStr
from pydantic.class_validators import validator
from pydantic.networks import HttpUrl
from settings_library.base import BaseCustomSettings
from settings_library.logging_utils import MixinLoggingSettings
from settings_library.postgres import PostgresSettings

# SERVICES CLIENTS --------------------------------------------

class _UrlMixin:
    def _build_url(self, prefix: str) -> str:
        prefix = prefix.upper()
        return HttpUrl.build(
            scheme="http",
            host=getattr(self, f"{prefix}_HOST"),
            port=f"{getattr(self, f'{prefix}_PORT')}",
            path=f"/{getattr(self, f'{prefix}_VTAG')}",
        )

class WebServerSettings(BaseCustomSettings, _UrlMixin):
    WEBSERVER_HOST: str = "webserver"
    WEBSERVER_PORT: int = 8080
    WEBSERVER_VTAG: str = "v0"

    WEBSERVER_SESSION_SECRET_KEY: SecretStr
    WEBSERVER_SESSION_NAME: str = "osparc.WEBAPI_SESSION"

    @cached_property
    def base_url(self) -> str:
        return self._build_url("WEBSERVER")

# TODO: dynamically create types with minimal options?
class CatalogSettings(BaseCustomSettings, _UrlMixin):
    CATALOG_HOST: str = "catalog"
    CATALOG_PORT: int = 8000
    CATALOG_VTAG: str = "v0"

    @cached_property
    def base_url(self) -> str:
        return self._build_url("CATALOG")

class StorageSettings(BaseCustomSettings, _UrlMixin):
    STORAGE_HOST: str = "storage"
    STORAGE_PORT: int = 8080
    STORAGE_VTAG: str = "v0"

    @cached_property
    def base_url(self) -> str:
        return self._build_url("STORAGE")

class DirectorV2Settings(BaseCustomSettings, _UrlMixin):
    DIRECTOR_V2_HOST: str = "director-v2"
    DIRECTOR_V2_PORT: int = 8000
    DIRECTOR_V2_VTAG: str = "v2"

    @cached_property
    def base_url(self) -> str:
        return self._build_url("DIRECTOR")

# MAIN SETTINGS --------------------------------------------

class AppSettings(BaseCustomSettings, MixinLoggingSettings):
    # pylint: disable=no-self-use
    # pylint: disable=no-self-argument

    # DOCKER
    SC_BOOT_MODE: Optional[BootModeEnum]

    # LOGGING
    LOG_LEVEL: LogLevel = Field(
        LogLevel.INFO.value,
        env=["API_SERVER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"],
    )

    # POSTGRES
    API_SERVER_POSTGRES: Optional[PostgresSettings]

    # SERVICES with http API
    API_SERVER_WEBSERVER: Optional[WebServerSettings]
    API_SERVER_CATALOG: Optional[CatalogSettings]
    API_SERVER_STORAGE: Optional[StorageSettings]
    API_SERVER_DIRECTOR_V2: Optional[DirectorV2Settings]

    API_SERVER_DEV_FEATURES_ENABLED: bool = Field(
        False, env=["API_SERVER_DEV_FEATURES_ENABLED", "FAKE_API_SERVER_ENABLED"]
    )

    API_SERVER_REMOTE_DEBUG_PORT: int = 3000

    @cached_property
    def debug(self) -> bool:
        """If True, debug tracebacks should be returned on errors."""
        return self.SC_BOOT_MODE in [
            BootModeEnum.DEBUG,
            BootModeEnum.DEVELOPMENT,
            BootModeEnum.LOCAL,
        ]

    @validator("LOG_LEVEL", pre=True)
    @classmethod
    def _validate_loglevel(cls, value) -> str:
        return cls.validate_log_level(value)


from django import forms
from django.forms import ModelForm, FileInput
from models import Profile, Project, Category, Message, Post
from django.contrib.auth.forms import UserCreationForm
from django.contrib.auth.models import User
from django.contrib.admin import widgets

from functools import partial
DateInput = partial(forms.DateInput, {'class': 'datepicker'})

class ProfileForm(ModelForm):
	class Meta:
		model = Profile
		exclude = ('user', 'confirmation_code',
			'message_email', 'notification_email', 'weekly_email')
		widgets = {
			'picture': FileInput()
		}

class AccountForm(ModelForm):
	class Meta:
		model = Profile
		fields = ('message_email', 'notification_email')

	def __init__(self, *args, **kwargs):
		super(AccountForm, self).__init__(*args, **kwargs)
		self.fields['message_email'].label = 'Receive emails for new messages'
		self.fields['notification_email'].label = 'Receive emails for new notifications'

class ProjectForm(ModelForm):
	start_date = forms.DateField(widget=DateInput())
	end_date = forms.DateField(widget=DateInput())

	class Meta:
		model = Project
		#exclude = ('user',)
		fields = '__all__'
		widgets = {
			'picture': FileInput()
		}

	def __init__(self, user, *args, **kwargs):
		super(ProjectForm, self).__init__(*args, **kwargs)
		self.fields['start_date'].required = False
		self.fields['end_date'].required = False
		#staff_query = Profile.objects.exclude(user__id=user.id).order_by('last_name')
		#self.fields['staff'].queryset = staff_query
		self.fields['category'].queryset = Category.objects.all().order_by('name')
		self.fields['is_paid'].label = 'Paid project'

class RegisterForm(UserCreationForm):
	email = forms.EmailField(required=True)

	class Meta:
		model = User
		fields = ( "username", "email" )

	def clean_email(self):
		email = self.cleaned_data.get('email')
		username = self.cleaned_data.get('username')
		if email == 'nmasur@wesleyan.edu': return email
		if User.objects.filter(email=email).exclude(username=username).count():
			raise forms.ValidationError("That email address has already been taken.")
		#if not email.endswith('@wesleyan.edu'):
		#	raise forms.ValidationError("You must enter a wesleyan.edu email address.")
		return email

class MessageForm(ModelForm):
	class Meta:
		model = Message
		fields = '__all__'

	def __init__(self, *args, **kwargs):
		super(MessageForm, self).__init__(*args, **kwargs)
		self.fields['content'].label = ''

class PostForm(ModelForm):
	class Meta:
		model = Post
		fields = '__all__'

	def __init__(self, *args, **kwargs):
		super(PostForm, self).__init__(*args, **kwargs)
		self.fields['content'].label = ''
		self.fields['public'].label = 'Make public'


import numpy as np
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")

##
def genGridWorld():
    O = -1e5  # Dangerous places to avoid
    D = 35    # Dirt
    W = -100  # Water
    C = -3000 # Cat
    T = 1000  # Toy
    grid_list = {0:'', O:'O', D:'D', W:'W', C:'C', T:'T'}
    grid_world = np.array([[0, O, O, 0, 0, O, O, 0, 0, 0],
        [0, 0, 0, 0, D, O, 0, 0, D, 0],
        [0, D, 0, 0, 0, O, 0, 0, O, 0],
        [O, O, O, O, 0, O, 0, O, O, O],
        [D, 0, 0, D, 0, O, T, D, 0, 0],
        [0, O, D, D, 0, O, W, 0, 0, 0],
        [W, O, 0, O, 0, O, D, O, O, 0],
        [W, 0, 0, O, D, 0, 0, O, D, 0],
        [0, 0, 0, D, C, O, 0, 0, D, 0]])
    return grid_world, grid_list

##
def showWorld(grid_world, tlt):
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    ax.set_title(tlt)
    ax.set_xticks(np.arange(0.5,10.5,1))
    ax.set_yticks(np.arange(0.5,9.5,1))
    ax.grid(color='b', linestyle='-', linewidth=1)
    ax.imshow(grid_world, interpolation='nearest', cmap='copper')
    return ax

##
def showTextState(grid_world, grid_list, ax):
    for x in range(grid_world.shape[0]):
        for y in range(grid_world.shape[1]):
            if grid_world[x,y] >= -3000:
                ax.annotate(grid_list.get(grid_world[x,y]), xy=(y,x), horizontalalignment='center')

##
def showPolicy(policy, ax):
    for x in range(policy.shape[0]):
        for y in range(policy.shape[1]):
            if policy[x,y] == 0:
                ax.annotate('$\downarrow$', xy=(y,x), horizontalalignment='center')
            elif policy[x,y] == 1:
                ax.annotate(r'$\rightarrow$', xy=(y,x), horizontalalignment='center')
            elif policy[x,y] == 2:
                ax.annotate('$\u2191$', xy=(y,x), horizontalalignment='center')
            elif policy[x,y] == 3:
                ax.annotate('$\leftarrow$', xy=(y,x), horizontalalignment='center')
            elif policy[x,y] == 4:
                ax.annotate('$\perp$', xy=(y,x), horizontalalignment='center')

##
def prob_model(row, col, action, use_prob_model=False):
    if not use_prob_model:
        if action == 0:
            # down
            row_next = row + 1
            col_next = col
            if row_next > 8:
                row_next = row
        elif action == 1:
            # right
            row_next = row
            col_next = col + 1
            if col_next > 9:
                col_next = col
        elif action == 2:
            # up
            row_next = row - 1
            col_next = col
            if row_next < 0:
                row_next = row
        elif action == 3:
            # left
            row_next = row
            col_next = col - 1
            if col_next < 0:
                col_next = col
        elif action == 4:
            # prep -> stay.
            row_next = row
            col_next = col
        return row_next, col_next
    elif use_prob_model:
        row_next = np.zeros(4)
        col_next = np.zeros(4)
        prob = np.zeros(4)
        if action == 0:
            # down
            row_next[0], col_next[0] = prob_model(row, col, 0)
            prob[0] = np.array([0.7])
            row_next[1], col_next[1] = prob_model(row, col, 1) # right
            prob[1] = 0.1
            row_next[2], col_next[2] = prob_model(row, col, 3) # left
            prob[2] = 0.1
            row_next[3], col_next[3] = prob_model(row, col, 4) # stay
            prob[3] = 0.1
        elif action == 1:
            # right
            row_next[0], col_next[0] = prob_model(row, col, 1)
            prob[0] = np.array([0.7])
            row_next[1], col_next[1] = prob_model(row, col, 0)  # down
            prob[1] = 0.1
            row_next[2], col_next[2] = prob_model(row, col, 2)  # up
            prob[2] = 0.1
            row_next[3], col_next[3] = prob_model(row, col, 4)  # stay
            prob[3] = 0.1
        elif action == 2:
            # up
            row_next[0], col_next[0] = prob_model(row, col, 2)
            prob[0] = np.array([0.7])
            row_next[1], col_next[1] = prob_model(row, col, 1)  # right
            prob[1] = 0.1
            row_next[2], col_next[2] = prob_model(row, col, 3)  # left
            prob[2] = 0.1
            row_next[3], col_next[3] = prob_model(row, col, 4)  # stay
            prob[3] = 0.1
        elif action == 3:
            # left
            row_next[0], col_next[0] = prob_model(row, col, 3)
            prob[0] = np.array([0.7])
            row_next[1], col_next[1] = prob_model(row, col, 0)  # down
            prob[1] = 0.1
            row_next[2], col_next[2] = prob_model(row, col, 2)  # up
            prob[2] = 0.1
            row_next[3], col_next[3] = prob_model(row, col, 4)  # stay
            prob[3] = 0.1
        elif action == 4:
            # prep
            row_next = np.array([row])
            col_next = np.array([col])
            prob = np.array([1])
        return row_next, col_next, prob

def ValIter(R, discount=None, maxSteps=None, infHor=False, probModel=False):
    #R = grid_world matrix! state metric for reward
    if infHor==False:
        T = maxSteps
        # down, right, up, left, prep
        actions = np.array([0, 1, 2, 3, 4])
        Q = np.zeros((R.shape[0], R.shape[1], actions.shape[0], T))
        V = np.zeros((R.shape[0], R.shape[1], T))
        # r = np.zeros((R.shape[0], R.shape[1], actions.shape[0], T))
        # r[:,:,T-1] = R
        V[:, :, T - 1] = R
        # iterate over all ts
        # iterate backwards in time! for schleife rückwärts
        for t in range(T - 2, -1, -1):
            # iterate over all state rows
            for row in range(R.shape[0]):
                # iterate over all columns
                for col in range(R.shape[1]):
                    # iterate over all actions
                    for action in range(actions.shape[0]):
                        if not probModel:
                            row_next, col_next = prob_model(row, col, action)
                            Q[row, col, action, t] = R[row, col] + V[row_next, col_next, t + 1]
                        elif probModel:
                            #Task 2.d) : each next state or action has additionally a probability value
                            row_next, col_next, prob = prob_model(row, col, action, use_prob_model=True)
                            Q[row, col, action, t] = R[row, col]
                            for i in range(row_next.shape[0]):
                                Q[row, col, action, t] = Q[row, col, action, t]+prob[i] * V[int(row_next[i]), int(col_next[i]), t + 1]
                    V[row, col, t] = max(Q[row, col, :, t])
    else:
        # infinite horizon:
        # down, right, up, left, prep
        actions = np.array([0, 1, 2, 3, 4])
        max_k = maxSteps
        Q = np.zeros((R.shape[0], R.shape[1], actions.shape[0], max_k))
        V = np.zeros((R.shape[0], R.shape[1], max_k))
        k = 0
        # for schleife vorwärts:
        for k in range(max_k-1):
            # iterate over all state rows
            for row in range(R.shape[0]):
                # iterate over all columns
                for col in range(R.shape[1]):
                    # iterate over all actions
                    for action in range(actions.shape[0]):
                        row_next, col_next = prob_model(row, col, action)
                        Q[row, col, action, k+1] = R[row, col] + discount*V[row_next, col_next, k]
                    V[row, col, k+1] = max(Q[row, col, :, k+1])

            #set to zero:
            #brich ab wenn konvergiert!
            if (np.abs(V[:,:,k]-V[:,:,k+1])<0.01).all():
                V = V[:,:,0:k+1]
                Q = Q[:,:,:,0:k+1]
                break
        V = np.flip(V, axis=2)
        Q = np.flip(Q, axis=3)

    return V, Q

##
def maxAction(V, R, discount, probModel=None):
    return 0

##
def findPolicy(V, Q, probModel=None):
    pi = np.zeros((Q.shape[0], Q.shape[1], Q.shape[3]))
    for t in range(Q.shape[3]):
        for row in range(Q.shape[0]):
            for col in range(Q.shape[1]):
                pi[row, col, t] = np.argmax(Q[row, col, :,t])
    return pi

############################

saveFigures = False

data = genGridWorld()
grid_world = data[0]
grid_list = data[1]

#probModel = ...

ax = showWorld(grid_world, 'Environment')
showTextState(grid_world, grid_list, ax)
if saveFigures:
    plt.savefig('gridworld.pdf')

# Finite Horizon
V, Q = ValIter(grid_world, maxSteps=10)
V = V[:,:,0];
showWorld(np.maximum(V, 0), 'Value Function - Finite Horizon')
if saveFigures:
    plt.savefig('value_Fin_15.pdf')

policy = findPolicy(V, Q)
ax = showWorld(grid_world, 'Policy - Finite Horizon')
showPolicy(policy[:,:,0], ax)
if saveFigures:
    plt.savefig('policy_Fin_15.pdf')

#Infinite Horizon
V, Q = ValIter(grid_world, maxSteps=100, discount=0.1, infHor=True)
V = V[:,:,0];
showWorld(np.maximum(V, 0), 'Value Function - Infinite Horizon')
if saveFigures:
    plt.savefig('value_Inf_08.pdf')

policy = findPolicy(V, Q);
ax = showWorld(grid_world, 'Policy - Infinite Horizon')
showPolicy(policy[:,:,0], ax)
if saveFigures:
    plt.savefig('policy_Inf_08.pdf')

# Finite Horizon with Probabilistic Transition
V, Q = ValIter(grid_world, maxSteps=15, probModel=True)
V = V[:,:,0];
showWorld(np.maximum(V, 0), 'Value Function - Finite Horizon with Probabilistic Transition')
if saveFigures:
    plt.savefig('value_Fin_15_prob.pdf')

policy = findPolicy(V, Q)
ax = showWorld(grid_world, 'Policy - Finite Horizon with Probabilistic Transition')
showPolicy(policy[:,:,0], ax)
if saveFigures:
    plt.savefig('policy_Fin_15_prob.pdf')

plt.show()


import argparse
import os
import sys

import tensorflow as tf

parser = argparse.ArgumentParser()

parser.add_argument('--batch_size', type=int, default=100, help='Number of images to process in a batch')
parser.add_argument('--data_dir', type=str, default='/tmp/mnist_data', help='Path to the MNIST data directory.')
parser.add_argument('--model_dir', type=str, default='/tmp/mnist_model',
                    help='The directory where the model will be stored.')
parser.add_argument('--train_epochs', type=int, default=40, help='Number of epochs to train.')
parser.add_argument(
    '--data_format', type=str, default=None, choices=['channels_first', 'channels_last'],
    help='A flag to override the data format used in the model. channels_first '
         'provides a performance boost on GPU but is not always compatible '
         'with CPU. If left unspecified, the data format will be chosen '
         'automatically based on whether TensorFlow was built for CPU or GPU.')

_NUM_IMAGES = {
    'train': 50000,
    'validation': 10000
}

def input_fn(is_training, filename, batch_size=1, num_epochs=1):
    def example_parser(serialized_example):
        features = tf.parse_single_example(
            serialized_example,
            features={
                'image_raw': tf.FixedLenFeature([], tf.string),
                'label': tf.FixedLenFeature([], tf.int64)
            })
        image = tf.decode_raw(features['image_raw'], tf.uint8)
        image.set_shape([28 * 28])

        image = tf.cast(image, tf.float32) / 255 - 0.5
        label = tf.cast(features['label'], tf.int32)
        return image, tf.one_hot(label, 10)

    dataset = tf.data.TFRecordDataset([filename])

    if is_training:
        dataset = dataset.shuffle(buffer_size=_NUM_IMAGES['train'])

    dataset = dataset.repeat(num_epochs)
    dataset = dataset.map(example_parser).prefetch(batch_size)
    dataset = dataset.batch(batch_size)
    iterator = dataset.make_one_shot_iterator()
    images, labels = iterator.get_next()
    return images, labels

def mnist_model(inputs, mode, data_format):
    inputs = tf.reshape(inputs, [-1, 28, 28, 1])
    if data_format is None:
        data_format = ('channels_first' if tf.test.is_built_with_cuda() else 'channels_last')

    conv1 = tf.layers.conv2d(inputs=inputs,
                             filters=32,
                             kernel_size=[5, 5],
                             padding='same',
                             activation=tf.nn.relu,
                             data_format=data_format)
    pool1 = tf.layers.max_pooling2d(inputs=conv1,
                                    pool_size=[2, 2],
                                    strides=2,
                                    data_format=data_format)

    conv2 = tf.layers.conv2d(inputs=pool1,
                             filters=64,
                             kernel_size=[5, 5],
                             padding='same',
                             activation=tf.nn.relu,
                             data_format=data_format)
    pool2 = tf.layers.max_pooling2d(inputs=conv2,
                                    pool_size=[2, 2],
                                    strides=2,
                                    data_format=data_format)

    pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])

    dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)

    dropout = tf.layers.dropout(dense, rate=0.4, training=(mode == tf.estimator.ModeKeys.TRAIN))

    logits = tf.layers.dense(inputs=dropout, units=10)
    return logits

def mnist_model_fn(features, labels, mode, params):
    logits = mnist_model(features, mode, params['data_format'])
    predictions = {
        'classes': tf.argmax(input=logits, axis=1),
        'probabilities': tf.nn.softmax(logits, name='softmax_tensor')
    }

    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)

    loss = tf.losses.softmax_cross_entropy(onehot_labels=labels, logits=logits)

    if mode == tf.estimator.ModeKeys.TRAIN:
        optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
        train_op = optimizer.minimize(loss, tf.train.get_or_create_global_step())
    else:
        train_op = None

    accuracy = tf.metrics.accuracy(tf.argmax(labels, axis=1), predictions['classes'])
    metrics = {'accuracy': accuracy}

    tf.identity(accuracy[1], name='train_accuracy')
    tf.summary.scalar('train_accuracy', accuracy[1])

    return tf.estimator.EstimatorSpec(
        mode=mode,
        predictions=predictions,
        loss=loss,
        train_op=train_op,
        eval_metric_ops=metrics
    )

def main(unused_argv):
    train_file = os.path.join(FLAGS.data_dir, 'train.tfrecords')
    test_file = os.path.join(FLAGS.data_dir, 'test.tfrecords')
    assert (tf.gfile.Exists(train_file) and tf.gfile.Exists(test_file)), (
        'Run covert_to_records.py to convert the MNIST data to TFRecord file format.')

    mnist_classfier = tf.estimator.Estimator(model_fn=mnist_model_fn, model_dir=FLAGS.mode_dir,
                                             params={'data_format': FLAGS.data_format})

    tensors_to_log = {
        'train_accuracy': 'train_accuracy'
    }
    logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=100)

    mnist_classfier.train(input_fn=lambda: input_fn(True, train_file, FLAGS.batch_size, FLAGS.train_epochs),
                          hooks=[logging_hook])

    eval_results = mnist_classfier.evaluate(input_fn=lambda: input_fn(False, test_file, FLAGS.batch_size))
    print()
    print('Evaluation results:\n\t%s' % eval_results)

if __name__ == '__main__':
    tf.logging.set_verbosity(tf.logging.INFO)
    FLAGS, unparsed = parser.parse_known_args()
    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


"""
Created on Fri Jun  5 12:06:30 2020

@author: enolasengeissen
"""
import psycopg2

from veloroutes_voies_vertes.test.base_test_database import DatabaseTestCase

__copyright__ = "Copyright 2019, 3Liz"
__license__ = "GPL version 3"
__email__ = "info@3liz.org"
__revision__ = "$Format:%H$"

class TestSqlFunctions(DatabaseTestCase):

    def test_trigger_repere_numero_serie(self):
        """ Test the trigger on repere(numero_serie)"""
        # Contrainte sur le champs repere(numero_serie) avec la fonction numserie()
        msg = 'numero_serie ne peut être NULL si type_noeud vaut CPT'
        with self.assertRaises(psycopg2.InternalError, msg=msg):
            self.cursor.execute(
                "INSERT INTO veloroutes.repere(type_noeud) VALUES ('CPT')")

    def test_insertok_numero_serie(self):
        """Test that we can insert a row in repere with good items"""
        self.cursor.execute("TRUNCATE TABLE veloroutes.repere CASCADE")
        sql = """
            INSERT INTO veloroutes.repere(libelle,numero_serie,type_noeud,geom)
            VALUES ('libellé test', 'num_serie_test', 'DFE',
                    ST_GeomFromText('POINT(0 0)',2154))
        """
        self.cursor.execute(sql)
        res = """
            SELECT veloroutes.repere.libelle FROM veloroutes.repere
            WHERE veloroutes.repere.numero_serie='num_serie_test'
        """
        self.cursor.execute(res)
        record = self.cursor.fetchall()
        self.assertEqual('libellé test', record[0][0])

    def test_trigger_segment_projet_revetement(self):
        """ Test the trigger on segment(revetement) case project"""
        # Contraintes sur le champs segment(revetement) avec la fonction revet()
        msg = 'revêtement ne peut pas prendre de valeur si avancement vaut 1'
        with self.assertRaises(psycopg2.InternalError, msg=msg):
            self.cursor.execute(
                "INSERT INTO veloroutes.segment(avancement, revetement, statut) VALUES (1,'LIS','VV')")

    def test_trigger_segment_fictif_revetement(self):
        """ Test the trigger on segment(revetement) case fictive geometry"""
        msg = 'revêtement ne peut pas prendre de valeur si la geom est fictive'
        with self.assertRaises(psycopg2.InternalError, msg=msg):
            self.cursor.execute(
                "INSERT INTO veloroutes.segment(geometrie_fictive, revetement, statut)"
                "VALUES ('T','LIS','VV')")

    def test_insertok_revetement(self):
        """Test that we can insert a row in segment with good items"""
        self.cursor.execute("TRUNCATE TABLE veloroutes.segment CASCADE")

        sql = """
            INSERT INTO veloroutes.segment(annee_ouverture,date_saisie,
                                           src_geom,src_annee,avancement,
                                           revetement,statut,gestionnaire,
                                           proprietaire,geom,precision,
                                           sens_unique,geometrie_fictive)
            VALUES ('2010-01-01', '2013-09-09', 'src_geom_test',
                    '2010',4,'LIS', 'ASP', 'gestion_test', 'GOLBEY',
                    ST_GeomFromText('LINESTRING(0 0,1 1)',2154), 'DC', 'F', 'F')
        """
        self.cursor.execute(sql)
        res = """
            SELECT veloroutes.segment.src_geom
            FROM veloroutes.segment
            WHERE veloroutes.segment.gestionnaire='gestion_test';
        """
        self.cursor.execute(res)
        record = self.cursor.fetchall()
        self.assertEqual('src_geom_test', record[0][0])