# Coding assignment: traffic flow

import numpy as np
import matplotlib.pyplot as plt

# matplotlib style
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16

# Problem parameters
Vmax = 136 / 3.6  # speed of traffic (m/s)
rhomax = 250 / 1000  # the number of cars per unit length of highway (cars/m)
L = 11 * 1000  # logitude of the highway (m)

nx = 51  # Points but not subintervals
dx = L / (nx - 1)  # (m)
dt = 0.001 * 3600  #  (s)

# Initial conditions
x = np.linspace(0, L, nx)
rho0 = np.ones(nx) * (20 / 1000)
rho0[10:20] = 50 / 1000

# Boundary condition
# rho(0, t) = 20 / 1000  (cars/m)

def V(rho_t):
    """
    Average velocity in (m/s)
    """
    return Vmax * (1 - rho_t / rhomax)

def F(rho):
    """
    F rate of cars per hour (cars/s)
    """
    return Vmax * rho * (1 - rho / rhomax)

def rho(t):
    """
    Solves for density (cars/m) with time in (s)
    """
    nt = int(t / dt)

    rho = np.copy(rho0)

    for n in range(nt):
        nrho = rho.copy()
        rho[1:] = nrho[1:] - dt / dx * (F(nrho[1:]) - F(nrho[: -1]))
        rho[0] = 20 / 1000  # Boundary condition (cars/m)

    return rho

if __name__ == '__main__':

    print('The minimum velocity at time t = 0 in (m/s) is: %.2f (m/s)' %
          (V(np.max(rho(0)))))

    print()

    print('The average velocity at time = 3 (min) in (m/s) is: %.2f (m/s)' %
          (V(np.mean(rho(3 * 60)))))

    print()

    print('The minimum velocity at time t = 3 (min) in (m/s) is: %.2f (m/s)' %
          (V(np.max(rho(3 * 60)))))

    plt.plot(x, rho(3 * 60), color='#003366', ls='--', lw=3)

    plt.show()


#!/usr/bin/python
import ConfigParser
import logging
import subprocess
import sys

import requests
from argo_poem_tools.config import Config
from argo_poem_tools.packages import Packages
from argo_poem_tools.repos import build_api_url, create_yum_repo_file, \
    get_repo_data

def main():
    logger = logging.getLogger('argo-poem-packages')
    logger.setLevel(logging.INFO)

    # setting up logging to file
    logfile = logging.FileHandler('/var/log/messages')
    logfile.setLevel(logging.INFO)
    logfile.setFormatter(logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    ))

    # add the handler to the root logger
    logger.addHandler(logfile)

    try:
        subprocess.call(['yum', 'clean', 'all'])

        config = Config()
        token = config.get_token()
        hostname = config.get_hostname()
        profiles = config.get_profiles()

        logger.info(
            'Sending request for profile(s): ' + ', '.join(profiles)
        )
        data = get_repo_data(
            build_api_url(hostname),
            token,
            profiles
        )

        if not data:
            logger.warning(
                'No data for given metric profile(s): ' +
                ', '.join(profiles)
            )
            sys.exit(2)

        else:
            logger.info('Creating YUM repo files...')
            create_yum_repo_file(data)

            pkg = Packages(data)

            installed, not_installed, downgraded, not_downgraded = \
                pkg.install_packages()

            warn_msg = []
            info_msg = []
            if pkg.get_packages_not_found():
                warn_msg.append(
                    'Packages not found: ' +
                    ', '.join(pkg.get_packages_not_found())
                )

            if installed:
                new_installed = \
                    pkg.get_packages_installed_with_versions_as_requested(
                        installed
                    )

                if new_installed:
                    info_msg.append(
                        'Packages installed: ' + ', '.join(new_installed)
                    )

                installed_diff = \
                    pkg.get_packages_installed_with_different_version()

                if installed_diff:
                    info_msg.append(
                        'Packages installed with different version: ' +
                        ', '.join(installed_diff)
                    )

            if not_installed:
                warn_msg.append(
                    'Packages not installed: ' + ', '.join(not_installed)
                )

            if downgraded:
                info_msg.append(
                    'Packages downgraded: ' + ', '.join(downgraded)
                )

            if not_downgraded:
                warn_msg.append(
                    'Packages not downgraded: ' + ', '.join(not_downgraded)
                )

            if info_msg:
                logger.info('; '.join(info_msg))

            if warn_msg:
                logger.warning('; '.join(warn_msg))
                sys.exit(1)

            else:
                logger.info('ok!')
                sys.exit(0)

    except requests.exceptions.ConnectionError as err:
        logger.error(err)
        sys.exit(2)

    except requests.exceptions.RequestException as err:
        logger.error(err)
        sys.exit(2)

    except ConfigParser.ParsingError as err:
        logger.error(err)
        sys.exit(2)

    except ConfigParser.NoSectionError as err:
        logger.error(err)
        sys.exit(2)

    except ConfigParser.NoOptionError as err:
        logger.error(err)
        sys.exit(2)

if __name__ == '__main__':
    main()


import pygame

class Ship():
    def __init__(self,ai_settings,screen):
        """Initialize the ship and set its starting position."""
        self.screen=screen
        self.ai_settings=ai_settings
        # Load the ship image and get its rect.
        self.image = pygame.image.load("ship.bmp")
        self.rect=self.image.get_rect()
        self.screen_rect = screen.get_rect()

        # Start each new ship at the bottom center of the screen.
        self.rect.centerx = self.screen_rect.centerx
        self.rect.bottom = self.screen_rect.bottom

        # Store a decimal value of the ship's center, to update ship's centre value not the rec
        self.center=float(self.rect.centerx)

        # Movement flags
        self.moving_right , self.moving_left = False,False

    def blitme(self):
        """Draw the ship at its current location."""
        self.screen.blit(self.image,self.rect)

    def update(self):
        if self.moving_right and self.rect.right < self.screen_rect.right :
            self.center += self.ai_settings.speed_ship_factor
        if self.moving_left and self.rect.left > 0 :
            self.center -= self.ai_settings.speed_ship_factor

        self.rect.centerx = self.center


import os
import numpy as np
from tifffile import imread
from skimage.measure import label
from scipy.ndimage.filters import gaussian_filter

def random_image(shape=(128, 128)):
    img = gaussian_filter(np.random.normal(size=shape), min(shape) / 20)
    img = img > np.percentile(img, 80)
    img = label(img)
    img[img > 255] = img[img > 255] % 254 + 1
    return img

def circle_image(shape=(128, 128), center=None):
    if center is None:
        center = (0,)*len(shape)
    xs = tuple(np.linspace(-1, 1, s) for s in shape)
    Xs = np.meshgrid(*xs, indexing="ij")
    R = np.sqrt(np.sum([(X - c) ** 2 for X, c in zip(Xs, center)], axis=0))
    img = R < .5
    return img

def overlap_image(shape=(128, 128)):
    img1 = circle_image(shape, center=(0.1,) * len(shape))
    img2 = circle_image(shape, center=(-0.1,) * len(shape))
    img = np.maximum(img1, 2 * img2)
    overlap = np.count_nonzero(np.bitwise_and(img1 > 0, img2 > 0))
    A1 = np.count_nonzero(img1 > 0)
    A2 = np.count_nonzero(img2 > 0)

    iou = overlap / min(A1, A2)
    return img, iou

def real_image2d():
    root = os.path.dirname(os.path.abspath(__file__))
    img = imread(os.path.join(root, 'data', 'mask2d.tif'))
    return img

def real_image3d():
    root = os.path.dirname(os.path.abspath(__file__))
    img = imread(os.path.join(root, 'data', 'mask3d.tif'))
    return img

def check_similar(x, y):
    delta = np.abs(x - y)
    debug = 'avg abs err = %.10f, max abs err = %.10f' % (np.mean(delta), np.max(delta))
    assert np.allclose(x, y), debug


from django.test import TestCase
from django.db.utils import IntegrityError
from .factories import (
    CategoryFactory,
    EntryFactory,
    CollectionWithConflictingEntriesFactory,
    CollectionWithSortedEntriesFactory,
)

class EntryTestCase(TestCase):

    def test_entry_auto_publication_date(self):
        obj = EntryFactory.create(published=True)

        self.assertIsNotNone(obj.publication_date)

    def test_entry_categories(self):
        CategoryFactory.create_batch(4)
        obj = EntryFactory.create(categories=CategoryFactory.create_batch(2))

        self.assertEqual(obj.categories.count(), 2)

class EntryCollectionTestCase(TestCase):

    def test_entrycollection_fails_with_duplicate_order(self):
        with self.assertRaises(IntegrityError):
            CollectionWithConflictingEntriesFactory.create()

    def test_entrycollection_succeeds_with_different_order(self):
        obj = CollectionWithSortedEntriesFactory.create()
        self.assertEqual(obj.entries.count(), 2)


import json
import difflib
#help(difflib.SequenceMatcher)

with open('data.json') as file:
    data=json.load(file)

#print(data['rain'])

def nyckel(nyckel):
    return data[nyckel]

def skriva_ut():
    while True:
        word= input('skriv ett ord')
        word=word.lower()
        for i in data.keys():
            if i==word:
                return i
            elif i== word.title():
                return i
            elif i == word.upper():
                return i

        else:
            reko_ord= difflib.get_close_matches(word,data.keys())

            if len(difflib.get_close_matches(word,data.keys()))>0:
                rätt_reko= input('menade du ordet: ' + reko_ord[0] + ' ja eler nej')
                if rätt_reko=='ja':
                    return reko_ord[0]
                else:
                    continue

            else:
                print('ordet finns inte försök igen')

def huvudprogram():
    ord_lista=skriva_ut()

    if type(nyckel(ord_lista))==list:
        for i in nyckel(ord_lista):
            print(i)
    else:
        print(nyckel(ord_lista))

huvudprogram()


import sys
import numpy as np
import cv2

cap = cv2.VideoCapture("image/PETS2000.mp4")

if not cap.isOpened() :
    print("Video load failed")
    sys.exit()

# bs = cv2.createBackgroundSubtractorMOG2()
bs = cv2.createBackgroundSubtractorKNN()
while True :
    ret, frame = cap.read()

    if not ret : break

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    fgmask = bs.apply(gray)
    back = bs.getBackgroundImage()

    _,_,stats,_ = cv2.connectedComponentsWithStats(fgmask)

    for stat in stats[1:] :
        x,y,w,h,s = stat
        if s < 100 : continue

        cv2.rectangle(frame, (x,y,w,h), (0,0,255),2)

    cv2.imshow('frame',frame)
    cv2.imshow('fgmask',fgmask)
    cv2.imshow('back',back)

    if cv2.waitKey(30) == 27 :
        break

cap.release()
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 18 11:40:25 2015

@author: mamauer
"""

import GPy
import numpy as np

class GPMulti(GPy.models.GPRegression):

    """ Gaussian Process Model for Multidimensional output

    Parameters
    ---------
    * X : numpy.ndarray
        input observations
    * Y : numpy.ndarray
        output observations
    * kern : (optional) GPy.kern.kern.kern
        The kernel, defaults to (ARD*ARD*ARD)**ARD

    """

    def __init__(self, X, Y, kern=None):
        self.X = X
        self.Y = Y
        _X, _Y = self._get_multidim_training_pairs(X, Y)
        self.original_outputdim = Y.shape[1]
        if kern is None:
            self.kern = (GPy.kern.RBF(input_dim=X.shape[1], ARD=True) *
                         GPy.kern.RBF(input_dim=X.shape[1], ARD=True) *
                         GPy.kern.RBF(input_dim=X.shape[1], ARD=True)) **\
                GPy.kern.RBF(input_dim=Y.shape[1], ARD=True)

        super(GPMulti, self).__init__(_X, _Y[:, None], kernel=self.kern)

    def _get_multidim_training_pairs(self, X, Y):
        """Convert X to X_i

        Parameters
        ----------
        * X : numpy.ndarray
        input observations
        * Y : numpy.ndarray
        output observations

        Return
        ------
        _X_new, _Y_new : numpy.ndarrays of new observations
        """

        _X_new = []
        _Y_new = []
        ident = np.identity(Y.shape[1])

        for x_i, y_i in zip(X, Y):
            for i in xrange(Y.shape[1]):
                _X_new.append(x_i.tolist() + ident[i].tolist())
                _Y_new.append(y_i[i])
        return np.array(_X_new), np.array(_Y_new)

    def get_multidim_training_input(self, Xnew):
        """Returns list of x, i from a single x,
        similar to _get_multidim_training_pairs

        Parameters
        ----------
        Xnew : numpy.ndarray
        \tThe X to transform

        Returns
        -------

        _X_new : numpy.ndarray
        \tTransformed Xnew
        """
        _X_new = []
        ident = np.identity(self.original_outputdim)
        for extension in ident:
            _X_new.append(Xnew.tolist() + extension.tolist())
        return np.array(_X_new)

    def predict(self, Xnew, full_cov=True,
                **likelihood_args):
        """ Yields the prediction of the GP at Xnew

        Parameters
        ----------
        * Xnew : numpy.ndarray
        \tThe X to predict at (original dimension)
        * full_cov : (optional) Boolean
        \tWhether a full covariance matrix should be returned. default to True

        Returns
        -------
        * mu, cov : numpy.ndarrays
        \tThe mean and covariance matrix of the predicted gaussian distribution
        """

        _Xnew = self.get_multidim_training_input(np.ravel(Xnew))
        return super(GPMulti, self).predict(_Xnew, full_cov, **likelihood_args)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('fin', '0018_auto_20160324_1133'),
    ]

    operations = [
        migrations.AddField(
            model_name='contractor',
            name='debet_allowed',
            field=models.BooleanField(default=False, verbose_name='Zgoda na debet'),
        ),
    ]


import sqlite3

con = sqlite3.connect(r'/home/dad/Documents/db/chinook.db')
cursorObj = con.cursor()
'''
command = "SELECT albumid, COUNT(trackid)\
        FROM tracks\
        GROUP BY albumid;"

doesnt work
command = "SELECT albumid, COUNT(trackid) \
FROM tracks \
WHERE COUNT(albumid) BETWEEN 18 AND 20;"
# GROUP BY albumid \
# HAVING albumid = 1;"
'''
command = '''\
SELECT
    tracks.AlbumId,
    title,
    SUM(Milliseconds) AS length
FROM
    tracks
INNER JOIN albums ON albums.AlbumId = tracks.AlbumId
GROUP BY
    tracks.AlbumId
HAVING
    length > 60000000;'''

# SELECT
#    albumid,
#    COUNT(trackid)
# FROM
#    tracks
# GROUP BY
#    albumid
# HAVING
#    COUNT(albumid) BETWEEN 18 AND 20
# ORDER BY albumid;'''

cursorObj.execute(command)
rows = cursorObj.fetchall()

for row in rows:
    print(row)

con.commit()


# Copyright 2018 Dong-Hyun Lee, Kakao Brain.
# (Strongly inspired by original Google BERT code and Hugging Face's code)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Transformer Model Classes & Config Class """
import pdb
import math
import json
from typing import NamedTuple
import random

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from utils.utils import split_last, merge_last, mixup_op

class Config(NamedTuple):
    "Configuration for BERT model"
    vocab_size: int = None # Size of Vocabulary
    dim: int = 768 # Dimension of Hidden Layer in Transformer Encoder
    n_layers: int = 12 # Numher of Hidden Layers
    n_heads: int = 12 # Numher of Heads in Multi-Headed Attention Layers
    dim_ff: int = 768*4 # Dimension of Intermediate Layers in Positionwise Feedforward Net
    #activ_fn: str = "gelu" # Non-linear Activation Function Type in Hidden Layers
    p_drop_hidden: float = 0.1 # Probability of Dropout of various Hidden Layers
    p_drop_attn: float = 0.1 # Probability of Dropout of Attention Layers
    max_len: int = 512 # Maximum Length for Positional Embeddings
    n_segments: int = 2 # Number of Sentence Segments

    @classmethod
    def from_json(cls, file):
        return cls(**json.load(open(file, "r")))

def gelu(x):
    "Implementation of the gelu activation function by Hugging Face"
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

class LayerNorm(nn.Module):
    "A layernorm module in the TF style (epsilon inside the square root)."
    def __init__(self, cfg, variance_epsilon=1e-12):
        super().__init__()
        self.gamma = nn.Parameter(torch.ones(cfg.dim)) # (preload)
        self.beta  = nn.Parameter(torch.zeros(cfg.dim)) # (preload)
        self.variance_epsilon = variance_epsilon

    def forward(self, x):
        u = x.mean(-1, keepdim=True)
        s = (x - u).pow(2).mean(-1, keepdim=True)
        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
        return self.gamma * x + self.beta

class Embeddings(nn.Module):
    "The embedding module from word, position and token_type embeddings."
    def __init__(self, cfg):
        super().__init__()
        self.tok_embed = nn.Embedding(cfg.vocab_size, cfg.dim) # (preload)
        self.pos_embed = nn.Embedding(cfg.max_len, cfg.dim) # (preload)
        self.seg_embed = nn.Embedding(cfg.n_segments, cfg.dim) # (preload)

        self.norm = LayerNorm(cfg) # (preload)
        self.drop = nn.Dropout(cfg.p_drop_hidden)

    def forward(
        self, x, seg, mixup, shuffle_idx, l, clone_ids, mixup_layer, simple_pad, no_grad_clone
        ):
        seq_len = x.size(1)
        pos = torch.arange(seq_len, dtype=torch.long, device=x.device)
        pos = pos.unsqueeze(0).expand_as(x) # (S,) -> (1, S) -> (B, S)  이렇게 외부에서 생성되는 값

        token_e = self.tok_embed(x)
        pos_e = self.pos_embed(pos)
        seg_e = self.seg_embed(seg)

        if mixup and 'word' in mixup:
            if simple_pad:
                if mixup_layer == 0:
                    token_e = mixup_op(token_e, l, shuffle_idx)
            else:
                if no_grad_clone:
                    with torch.no_grad():
                        c_token_e = self.tok_embed(clone_ids)
                else:
                    c_token_e = self.tok_embed(clone_ids)

                if mixup_layer == 0:
                    embeds_a, embeds_b = token_e, c_token_e[shuffle_idx]
                    token_e = l * embeds_a + (1-l) * embeds_b
                else:
                    e = token_e + pos_e + seg_e
                    ec = c_token_e + pos_e + seg_e

                    h = self.drop(self.norm(e))

                    if no_grad_clone:
                        with torch.no_grad():
                            hc = self.drop(self.norm(ec))
                    else:
                        hc = self.drop(self.norm(ec))

                    return h, hc

        e = token_e + pos_e + seg_e
        return self.drop(self.norm(e)), None

class MultiHeadedSelfAttention(nn.Module):
    """ Multi-Headed Dot Product Attention """
    def __init__(self, cfg):
        super().__init__()
        self.proj_q = nn.Linear(cfg.dim, cfg.dim) #(preload)
        self.proj_k = nn.Linear(cfg.dim, cfg.dim) #(preload)
        self.proj_v = nn.Linear(cfg.dim, cfg.dim) #(preload)
        self.drop = nn.Dropout(cfg.p_drop_attn)
        self.scores = None # for visualization
        self.n_heads = cfg.n_heads

    def forward(self, x, mask):
        """
        x, q(query), k(key), v(value) : (B(batch_size), S(seq_len), D(dim))
        mask : (B(batch_size) x S(seq_len))
        * split D(dim) into (H(n_heads), W(width of head)) ; D = H * W
        """
        # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)
        q, k, v = self.proj_q(x), self.proj_k(x), self.proj_v(x)
        q, k, v = (split_last(x, (self.n_heads, -1)).transpose(1, 2)
                   for x in [q, k, v])
        # (B, H, S, W) @ (B, H, W, S) -> (B, H, S, S) -softmax-> (B, H, S, S)
        scores = q @ k.transpose(-2, -1) / np.sqrt(k.size(-1))
        if mask is not None:
            mask = mask[:, None, None, :].float()
            scores -= 10000.0 * (1.0 - mask)
        scores = self.drop(F.softmax(scores, dim=-1))
        # (B, H, S, S) @ (B, H, S, W) -> (B, H, S, W) -trans-> (B, S, H, W)
        h = (scores @ v).transpose(1, 2).contiguous()
        # -merge-> (B, S, D)
        h = merge_last(h, 2)
        self.scores = scores
        return h

class PositionWiseFeedForward(nn.Module):
    """ FeedForward Neural Networks for each position """
    def __init__(self, cfg):
        super().__init__()
        self.fc1 = nn.Linear(cfg.dim, cfg.dim_ff) #(preload)
        self.fc2 = nn.Linear(cfg.dim_ff, cfg.dim) #(preload)
        #self.activ = lambda x: activ_fn(cfg.activ_fn, x)

    def forward(self, x):
        # (B, S, D) -> (B, S, D_ff) -> (B, S, D)
        return self.fc2(gelu(self.fc1(x)))

class Block(nn.Module):
    """ Transformer Block """
    def __init__(self, cfg):
        super().__init__()
        self.attn = MultiHeadedSelfAttention(cfg)
        self.proj = nn.Linear(cfg.dim, cfg.dim) # (preload)
        self.norm1 = LayerNorm(cfg) #(preload)
        self.pwff = PositionWiseFeedForward(cfg) #(preload)
        self.norm2 = LayerNorm(cfg) #(preload)
        self.drop = nn.Dropout(cfg.p_drop_hidden)

    def forward(self, x, mask):
        h = self.attn(x, mask)
        h = self.norm1(x + self.drop(self.proj(h)))
        h = self.norm2(h + self.drop(self.pwff(h)))
        return h

class Transformer(nn.Module):
    """ Transformer with Self-Attentive Blocks"""
    def __init__(self, cfg):
        super().__init__()
        self.embed = Embeddings(cfg)
        self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg.n_layers)])   # h 번 반복

    def forward(
            self,
            x=None, seg=None, mask=None,
            clone_ids=None, mixup=None, shuffle_idx=None, l=1,
            mixup_layer=-1, simple_pad=False, no_grad_clone=False
        ):
        h, hc = self.embed(
            x, seg, mixup, shuffle_idx, l, clone_ids, mixup_layer, simple_pad, no_grad_clone
        )

        layer = 1
        for block in self.blocks:
            h = block(h, mask)

            if hc is not None:
                if no_grad_clone:
                    with torch.no_grad():
                        hc = block(hc, mask)
                else:
                    hc = block(hc, mask)

            if mixup_layer == layer and (mixup == 'word' or mixup == 'word_cls'):
                if hc is not None:
                    h_a, h_b = h, hc[shuffle_idx]
                    h = l * h_a + (1-l) * h_b
                    hc = None
                else:
                    h = mixup_op(h, l, shuffle_idx)

            layer += 1
        return h

class Classifier(nn.Module):
    """ Classifier with Transformer """
    def __init__(self, cfg, n_labels):
        super().__init__()
        self.transformer = Transformer(cfg)
        self.fc = nn.Linear(cfg.dim, cfg.dim)
        self.activ = nn.Tanh()
        self.drop = nn.Dropout(cfg.p_drop_hidden)
        self.classifier = nn.Linear(cfg.dim, n_labels)
        self.layers = cfg.n_layers

    def forward(
            self,
            input_ids=None,
            segment_ids=None,
            input_mask=None,
            output_h=False,
            input_h=None,
            mixup=None,
            shuffle_idx=None,
            clone_ids=None,
            l=1,
            manifold_mixup=None,
            simple_pad=False,
            no_grad_clone=False
        ):
        if input_h is None:

            if mixup == 'word':
                mixup_layer = random.randint(0, self.layers) if manifold_mixup else 0
            elif mixup == 'word_cls':
                mixup_layer = random.randint(0, self.layers+1) if manifold_mixup else 0
            elif mixup == 'cls':
                mixup_layer = self.layers + 1
            elif mixup == 'word_cls_only':
                mixup_layer = random.choice([0, self.layers + 1])
            else:
                mixup_layer = -1

            h = self.transformer(
                x=input_ids, seg=segment_ids, mask=input_mask,
                clone_ids=clone_ids, mixup=mixup, shuffle_idx=shuffle_idx, l=l,
                mixup_layer = mixup_layer, simple_pad=simple_pad, no_grad_clone=no_grad_clone
            )

            # only use the first h in the sequence
            # h shape = [16, 128, 768]
            # h[h:, 0] = [16, 768]

            pooled_h = self.activ(self.fc(h[:, 0]))

            # pooled_h = [16, 768]

            if mixup_layer == self.layers+1:
                pooled_h = mixup_op(pooled_h, l, shuffle_idx)

            if output_h:
                return pooled_h
        else:
            pooled_h = input_h
        logits = self.classifier(self.drop(pooled_h))
        return logits


import pygame
from PIL import Image

def pygame_img_save(path):
    img = pygame.image.load(path)
    pygame.image.save(img, path)

def pygame_img_scale(path, scale=(64, 64)):
    img = pygame.image.load(path)
    img = pygame.transform.scale(img, scale)
    new_path = path.split('.')
    new_path[-1] = f'_{scale[0]}x{scale[1]}.' + new_path[-1]
    pygame.image.save(img, ''.join(new_path))

def pil_crop(path, box):
    im = Image.open(path)
    width, height = im.size
    count = 1
    for i in range(0, width, box[0]):
        for j in range(0, height, box[1]):
            new_im = im.crop((j, i, j + box[0], i + box[1]))
            new_path = path.split('.')
            new_path[-1] = f'_{count}.' + new_path[-1]
            new_im.save(''.join(new_path))
            count += 1

def pil_merge(path_to_image, path_to_temp_file):
    with open(path_to_temp_file) as temp:
        rl = temp.readlines()
        width, height = len(rl[0].split()) * 64, len(rl) * 64
        img = Image.open(r'assets\2dhandpainted_towntileset\Assets\64x64\Town@64x64_crop\Town@64x64_2.png')
        img = img.resize((width, height))
    with open(path_to_temp_file) as temp:
        x = 0
        y = 0
        for line in temp:
            for i in line.split():
                element = Image.open(path_to_image.format(int(i)))
                img.paste(element, (64 * x, 64 * y))
                x += 1
            x = 0
            y += 1
    return img

# for i in range(1, 10):
#     pygame_img_save(f'assets/7soulsrpggraphics_sprites/Assets/Human (Back)/Full/full_0{i}.png')
# for i in range(10, 241):
#     pygame_img_save(img, f'assets/7soulsrpggraphics_sprites/Assets/Human (Back)/Full/player_{i}.png')

# side_list = ['Front', 'Back', 'Side']
# for side in side_list:
#     path_list = [f'Human ({side})/Bodies/body_', f'Human ({side})/Heads/head_', f'Human ({side})/Full/player_']
#     for path in path_list:
#         n = 32
#         for i in range(1, 10):
#             pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/{path}0{i}.png', (n, n))
#         n = 64
#         for i in range(1, 10):
#             pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/{path}0{i}.png', (n, n))
#         n = 32
#         for i in range(10, 241):
#             pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/{path}{i}.png', (n, n))
#         n = 64
#         for i in range(10, 241):
#             pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/{path}{i}.png', (n, n))
#         if 'Heads' in path:
#             n = 32
#             for i in range(241, 321):
#                 pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/{path}{i}.png', (n, n))
#             n = 64
#             for i in range(241, 321):
#                 pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/{path}{i}.png', (n, n))

# filename = r'assets/guiicons/Assets/GUI_Icons/GUI_Icons_png/transparent/arrow_s_t.png'
# img = Image.open(filename)
# img.save('assets/arrow_s_t.ico')

# n = 32
# for i in range(1, 10):
#     pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/Monster/Monsters/monster_0{i}.png', (n, n))
#
# n = 64
# for i in range(1, 10):
#     pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/Monster/Monsters/monster_0{i}.png', (n, n))

# n = 32
# for i in range(10, 229):
#     try:
#         pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/Monster/Monsters/monster_{i}.png', (n, n))
#     except:
#         print(f'nie ma {i}')
#
# n = 64
# for i in range(10, 229):
#     try:
#         pygame_img_scale(f'assets/7soulsrpggraphics_sprites/Assets/Monster/Monsters/monster_{i}.png', (n, n))
#     except:
#         print(f'nie ma {i}')

# pil_crop('assets/2dhandpainted_towntileset/Assets/64x64/Town@64x64.png', (64, 64))

# img = Image.open(r'assets\2dhandpainted_towntileset\Assets\64x64\Town@64x64_crop\Town@64x64_2.png')
# img = img.resize((2048, 2048))
# img.save(r'C:\Users\Damian\Desktop\Programowanie\Python\Projekt_gry\game\test.png')
# img = pil_merge(r'assets\2dhandpainted_towntileset\Assets\64x64\Town@64x64_crop\Town@64x64_{}.png', r'screen2')
# img.save(r'assets\2dhandpainted_towntileset\Assets\64x64\grass.png')


#!/usr/bin/python -Ot
# -*- coding: utf-8 -*-
#
# Copyright (C) 2012-2015 Andreas Lang-Nevyjel
#
# Send feedback to: <lang-nevyjel@init.at>
#
# This file is part of webfrontend
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License Version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

""" boot views """

import json
import logging
import time

from django.contrib.auth.decorators import login_required
from django.db import transaction
from django.db.models import Q
from django.http import HttpResponse
from django.utils.decorators import method_decorator
from django.views.generic import View
from initat.cluster.backbone.models import device, cd_connection, cluster_timezone, \
    kernel, image, partition_table, status, network, DeviceLogEntry
from initat.cluster.backbone.serializers import device_serializer_boot
from initat.cluster.backbone.render import render_me
from initat.cluster.frontend.helper_functions import contact_server, xml_wrapper
from rest_framework.authentication import SessionAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView
import logging_tools
import server_command

logger = logging.getLogger("cluster.boot")

class show_boot(View):
    @method_decorator(login_required)
    def get(self, request):
        return render_me(
            request, "boot_overview.html", {
            }
        )()

class get_boot_info_json(View):
    @method_decorator(login_required)
    @method_decorator(xml_wrapper)
    def post(self, request):
        _post = request.POST
        sel_list = _post.getlist("sel_list[]")
        dev_result = device.objects.filter(Q(pk__in=sel_list)).prefetch_related(
            "bootnetdevice__net_ip_set__network__network_device_type",
            "categories",
            "domain_tree_node",
        ).select_related(
            "device_group",
            "device_type",
        )
        cd_cons = cd_connection.objects.filter(Q(child__in=sel_list) | Q(parent__in=sel_list)).select_related(
            "child__device_group",
            "child__device_type",
            "child__domain_tree_node",
            "parent__device_group",
            "parent__device_type",
            "parent__domain_tree_node",
        )
        call_mother = True if int(_post["call_mother"]) else False
        # to speed up things while testing
        if call_mother:
            srv_com = server_command.srv_command(command="status")
            srv_com["devices"] = srv_com.builder(
                "devices",
                *[srv_com.builder("device", pk="{:d}".format(cur_dev.pk)) for cur_dev in dev_result])
            result = contact_server(request, "mother", srv_com, timeout=10, log_result=False, connection_id="webfrontend_status")
        else:
            result = None
        # print result.pretty_print()
        for cur_dev in dev_result:
            cur_dev.cd_cons = cd_cons
            # recv/reqstate are written by mother, here we 'salt' this information with the device XML (pingstate)
            if call_mother:
                if result is not None:
                    # copy from mother
                    dev_node = result.xpath(".//ns:device[@pk='{:d}']".format(cur_dev.pk), smart_strings=False)
                    if len(dev_node):
                        dev_node = dev_node[0]
                    else:
                        dev_node = None
                else:
                    dev_node = None
                cur_dev.mother_xml = dev_node
            else:
                cur_dev.mother_xml = False
        if result is not None and result.xpath(".//ns:cd_ping_list/ns:cd_ping"):
            cd_result = {}
            for cd_ping in result.xpath(".//ns:cd_ping_list/ns:cd_ping"):
                cd_result[int(cd_ping.attrib["pk"])] = True if int(cd_ping.attrib["reachable"]) else False
            request.xml_response["cd_response"] = json.dumps(cd_result)
        ctx = {
            "request": request,
            "fields": ["partition_table", "act_partition_table"]
        }
        request.xml_response["response"] = JSONRenderer().render(device_serializer_boot(dev_result, many=True, context=ctx).data)

class update_device(APIView):
    authentication_classes = (SessionAuthentication,)
    permission_classes = (IsAuthenticated,)

    def put(self, request, pk):
        _change_lut = {
            "b": "change_dhcp_mac",
            "t": "change_target_state",
            "p": "change_partition_table",
            "i": "change_new_image",
            "k": "change_new_kernel",
        }
        dev_data = request.DATA
        # import pprint
        # pprint.pprint(dev_data)
        _en = dev_data["bo_enabled"]
        _changed = False
        _mother_commands = set()
        _all_update_list = set()
        if int(pk):
            pk_list = [int(pk)]
            # many = False
        else:
            pk_list = dev_data["device_pks"]
            # many = True
            # update enabled list
            for short_key in _en.iterkeys():
                if short_key in _change_lut:
                    if not dev_data.get(_change_lut[short_key], False):
                        _en[short_key] = False
        with transaction.atomic():
            all_devs = list(device.objects.filter(Q(pk__in=pk_list)).select_related(
                "domain_tree_node", "device_group"
            ).prefetch_related(
                "categories"
            ).order_by("device_group__name", "name"))
            for cur_dev in all_devs:
                # print "**", cur_dev
                update_list = set()
                if _en["t"]:
                    new_new_state, new_prod_link = (None, None)
                    if dev_data["new_state"]:
                        new_new_state = status.objects.get(Q(pk=dev_data["new_state"]))
                        if dev_data["prod_link"]:
                            new_prod_link = network.objects.get(Q(pk=dev_data["prod_link"]))  # @UndefinedVariable
                    if new_new_state != cur_dev.new_state or new_prod_link != cur_dev.prod_link:
                        cur_dev.new_state = new_new_state
                        cur_dev.prod_link = new_prod_link
                        update_list.add("target state")
                        _mother_commands.add("refresh")
                if _en["i"]:
                    new_image = image.objects.get(Q(pk=dev_data["new_image"])) if dev_data.get("new_image", None) else None
                    if new_image != cur_dev.new_image:
                        cur_dev.new_image = new_image
                        update_list.add("image")
                if _en["k"]:
                    new_kernel = kernel.objects.get(Q(pk=dev_data["new_kernel"])) if dev_data.get("new_kernel", None) else None
                    new_stage1_flavour = dev_data.get("stage1_flavour", "")
                    new_kernel_append = dev_data.get("kernel_append", "")
                    if new_kernel != cur_dev.new_kernel or new_stage1_flavour != cur_dev.stage1_flavour or new_kernel_append != cur_dev.kernel_append:
                        cur_dev.new_kernel = new_kernel
                        cur_dev.stage1_flavour = new_stage1_flavour
                        cur_dev.kernel_append = new_kernel_append
                        update_list.add("kernel")
                        _mother_commands.add("refresh")
                if _en["p"]:
                    new_part = partition_table.objects.get(Q(pk=dev_data["partition_table"])) if dev_data["partition_table"] else None
                    if new_part != cur_dev.partition_table:
                        cur_dev.partition_table = new_part
                        update_list.add("partition")
                if _en["b"]:
                    new_dhcp_mac, new_dhcp_write = (dev_data["dhcp_mac"], dev_data["dhcp_write"])
                    _bc = False
                    if new_dhcp_mac != cur_dev.dhcp_mac or new_dhcp_write != cur_dev.dhcp_write:
                        cur_dev.dhcp_mac = new_dhcp_mac
                        cur_dev.dhcp_write = new_dhcp_write
                        _bc = True
                    if cur_dev.bootnetdevice:
                        new_driver, new_macaddr = (dev_data["driver"], dev_data["macaddr"])
                        if new_driver != cur_dev.bootnetdevice.driver or new_macaddr != cur_dev.bootnetdevice.macaddr:
                            cur_dev.bootnetdevice.driver = new_driver
                            # ignore empty macaddr (for many changes)
                            if new_macaddr.strip():
                                cur_dev.bootnetdevice.macaddr = new_macaddr
                            _bc = True
                    if _bc:
                        update_list.add("bootdevice")
                        _mother_commands.add("refresh")
                if update_list:
                    _changed = True
                    _all_update_list |= update_list
                if _changed:
                    cur_dev._no_bg_job = True
                    cur_dev.save()
                    if cur_dev.bootnetdevice:
                        # promote no_bg_job flag
                        cur_dev.bootnetdevice.device = cur_dev
                        cur_dev.bootnetdevice.save()
                    cur_dev._no_bg_job = False
                    # print cur_dev.new_kernel, cur_dev.new_image
        _lines = []
        if _mother_commands:
            for _mother_com in _mother_commands:
                srv_com = server_command.srv_command(command=_mother_com)
                srv_com["devices"] = srv_com.builder(
                    "devices",
                    *[
                        srv_com.builder("device", name=cur_dev.name, pk="{:d}".format(cur_dev.pk)) for cur_dev in all_devs
                    ]
                )
                _res, _log_lines = contact_server(request, "mother", srv_com, timeout=10, connection_id="webfrontend_refresh")
                # print "*", _mother_com, _log_lines
                _lines.extend(_log_lines)
        if _all_update_list:
            if len(all_devs) > 1:
                dev_info_str = "{}: {}".format(
                    logging_tools.get_plural("device", len(all_devs)),
                    ", ".join([unicode(cur_dev) for cur_dev in all_devs]))
            else:
                dev_info_str = unicode(all_devs[0])
            _lines.append((logging_tools.LOG_LEVEL_OK, "updated {} for '{}'".format(", ".join(_all_update_list), dev_info_str)))
        response = Response(
            {
                "log_lines": _lines
            }
        )
        return response

class get_devlog_info(View):
    @method_decorator(login_required)
    @method_decorator(xml_wrapper)
    def post(self, request):
        _post = request.POST
        # import pprint
        _pk_log_list = json.loads(_post["sel_list"])
        lp_dict = dict([(key, latest) for key, latest in _pk_log_list])
        devs = device.objects.filter(Q(pk__in=lp_dict.keys()))
        oldest_pk = min(lp_dict.values() + [0])
        logger.info("request devlogs for {}, oldest devlog_pk is {:d}".format(
            logging_tools.get_plural("device", len(lp_dict)),
            oldest_pk))
        _lines = []
        dev_logs = DeviceLogEntry.objects.filter(
            Q(pk__gt=oldest_pk) & Q(device__in=devs)
        ).select_related(
            "source",
            "level",
            "user",
        ).order_by("pk")
        logs_transfered = dict([(key, 0) for key in lp_dict.iterkeys()])
        for dev_log in dev_logs:
            if dev_log.pk > lp_dict[dev_log.device_id]:
                logs_transfered[dev_log.device_id] += 1
                _lines.append([
                    dev_log.pk,
                    dev_log.device_id,
                    dev_log.source_id,
                    dev_log.user_id,
                    dev_log.level_id,
                    dev_log.text,
                    time.mktime(cluster_timezone.normalize(dev_log.date).timetuple()),
                ])
        return HttpResponse(json.dumps({"devlog_lines": _lines}), content_type="application/json")

class soft_control(View):
    @method_decorator(login_required)
    @method_decorator(xml_wrapper)
    def post(self, request):
        _post = request.POST
        dev_pk_list = json.loads(_post["dev_pk_list"])
        cur_devs = device.objects.filter(Q(pk__in=dev_pk_list))
        soft_state = _post["command"]
        logger.info("sending soft_control '{}' to {}: {}".format(
            soft_state,
            logging_tools.get_plural("device", len(dev_pk_list)),
            ", ".join(sorted([unicode(cur_dev) for cur_dev in cur_devs]))
        ))
        srv_com = server_command.srv_command(command="soft_control")
        srv_com["devices"] = srv_com.builder(
            "devices",
            *[
                srv_com.builder("device", soft_command=soft_state, pk="{:d}".format(cur_dev.pk)) for cur_dev in cur_devs
            ]
        )
        result = contact_server(request, "mother", srv_com, timeout=10, log_result=False)
        if result:
            request.xml_response.info("sent {} to {}".format(soft_state, unicode(cur_dev)), logger)

class hard_control(View):
    @method_decorator(login_required)
    @method_decorator(xml_wrapper)
    def post(self, request):
        _post = request.POST
        cd_con_pks = json.loads(_post["cd_pk_list"])
        cur_cd_cons = cd_connection.objects.select_related("child", "parent").filter(Q(pk__in=cd_con_pks))
        command = _post["command"]
        logger.info("got hc command '{}' for {}:".format(
            command,
            logging_tools.get_plural("device", len(cd_con_pks))))
        for cur_cd_con in cur_cd_cons:
            logger.info("  device {} (controlling device: {})".format(
                unicode(cur_cd_con.child),
                unicode(cur_cd_con.parent)))
        srv_com = server_command.srv_command(command="hard_control")
        srv_com["devices"] = srv_com.builder(
            "devices",
            *[
                srv_com.builder(
                    "device",
                    command=command,
                    pk="{:d}".format(cur_cd_con.parent_id),
                    cd_con="{:d}".format(cur_cd_con.pk),
                    bootserver_hint="{:d}".format(cur_cd_con.child.bootserver_id),
                )
                for cur_cd_con in cur_cd_cons
            ])
        contact_server(request, "mother", srv_com, timeout=10)


from __future__ import print_function, division, absolute_import

import os
import inspect
from collections import Mapping
from contextlib import contextmanager
from six.moves import cPickle, builtins

from odin.utils import is_string, is_path, as_tuple
from odin.config import (auto_config, get_floatX, get_session,
                         get_rng, randint)
auto_config()
floatX = get_floatX()
# ==================== import utilities modules ==================== #
from odin.backend.helpers import *
from odin.backend.tensor import *
from odin.backend.activations import *
from odin.backend import role
from odin.backend import metrics
from odin.backend import losses
from odin.backend import optimizers
from odin.backend import rand
from odin.backend import rnn_cell

@contextmanager
def variable_dtype(dtype):
  """Temporary change the default dtype for creating variable using ODIN"""
  global floatX
  _old_floatX = str(floatX)
  floatX = dtype
  yield
  floatX = _old_floatX

def initialize_all_variables(vars=None):
  """ This function will automatically check if the variables
  are initialized, and only perform initialization for
  un-initialized variables.

  Note
  ----
  Re-initialize an initialized variable will give it random values
  """
  if vars is None:
    vars = get_all_variables()
  else:
    vars = [v for v in as_tuple(vars)
            if is_variable(v)]
  # ====== check if variable not initialized ====== #
  init_info = eval([tf.is_variable_initialized(v) for v in vars])
  vars = [v for v, inited in zip(vars, init_info)
          if not inited]
  # ====== build mapping graph -> list of vars ====== #
  graph = defaultdict(list)
  for v in vars:
    graph[v.graph].append(v)
  # ====== run the initialization ====== #
  for g, v in graph.items():
    get_session(graph=g).run([i.initializer for i in v])

def is_variable_initialized(var):
  if not is_variable(var):
    raise ValueError("`var` must be instance of tensorflow.Variable, "
                     "but given type: %s" % type(var))
  return eval(tf.is_variable_initialized(var))

def placeholder(shape=None, dtype=floatX, name=None, roles=[]):
  if shape is None and name is None:
    raise ValueError("shape and name arguments cannot be None at the same time.")
  # ====== check duplicated placeholder ====== #
  if name is not None:
    all_placeholders = [
        o._outputs[0] for o in get_all_operations(otype='Placeholder')]
    for v in all_placeholders:
      v_shape = tuple(v.shape.as_list())
      if v.name == name + ':0': # found duplicated variable
        # set new value for variable
        if shape is not None:
          if v_shape == shape:
            return role.add_roles(v, roles)
          else:
            raise ValueError("Pre-defined placeholder with name: %s and"
                " shape: %s, which is different from given shape: %s"
                % (name, v_shape, shape))
        # just get the variable
        else:
          return role.add_roles(v, roles)
  # ====== Modify add name prefix ====== #
  plh = tf.placeholder(dtype=dtype, shape=shape, name=name)
  return role.add_roles(plh, roles)

# ===========================================================================
# Fast evaluation
# ===========================================================================
def _validate_updates(updates):
  if updates is not None:
    if isinstance(updates, Mapping):
      updates = updates.items()
    # create updates ops
    if not isinstance(updates, tf.Operation):
      updates_ops = []
      for u in updates:
        if isinstance(u, (tuple, list)):
          p, new_p = u
          updates_ops.append(tf.assign(p, new_p))
        elif is_operation(u): # assumed already an assign op
          updates_ops.append(u)
      updates = tf.group(*updates_ops)
  return updates

def eval(x, feed_dict=None,
         update_before=None, update_after=None,
         options=None, run_metadata=None):
  ''' Generalized version of code evaluation, it
  could evaluate python and tensorflow expression.

  Parameters
  ----------
  x : list, tuple, dictionary, `Tensor`
      tensorfow `Tensor` for evaluation
  feed_dict : dict
      Input dictionary, mapping placeholder -> values
  update_before: {None, list, or dict}
      mapping from `Tensor` to its new value which is `Tensor` or
      real value, the updates is runned before evaluating
  update_after: {None, list, or dict}
      same as `updates_before`, but run the `updates` after
      evaluate `x`
  options: tensorflow.RunOptions
      thhe options allow controlling the behavior of
      this particular step (e.g. turning tracing on).
  run_metadata: tensorflow.RunMetadata
      When appropriate, the non-Tensor output of this
      step will be collected there. For example,
      when users turn on tracing in options, the
      profiled info will be collected into
      this argument and passed back.

  Example
  -------
  >>> import tensorflow as tf
  >>> from odin import backend as K
  >>> run_metadata = tf.RunMetadata()
  >>> K.eval(...,
  ...        options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE,
  ...                              output_partition_graphs=True),
  ...        run_metadata=run_metadata)
  >>> with open('log_path', 'w') as f:
  >>>   f.write(str(run_metadata))

  Note
  ----
  If "Couldn't open CUDA library libcupti.so.8.0" appears when you
  adding RunOptions, try adding "/usr/local/cuda/extras/CUPTI/lib64/"
  to your LD_LIBRARY_PATH
  '''
  results = ()
  update_before = _validate_updates(update_before)
  update_after = _validate_updates(update_after)
  # ====== run updates before ====== #
  if update_before is not None:
    get_session(update_before.graph).run(update_before, feed_dict=feed_dict,
                                   options=options,
                                   run_metadata=run_metadata)
  # ====== list of Tensor or string ====== #
  if isinstance(x, (tuple, list)):
    string_eval = []
    tensor_eval = []
    tensor_idx = []
    # evaluate string expression
    for i, j in enumerate(x):
      if is_string(j):
        string_eval.append(builtins.eval(j))
      else:
        tensor_eval.append(j)
        tensor_idx.append(i)
    # evaluate tensor
    if len(tensor_eval) > 0:
      graph = [i.graph for i in tensor_eval]
      if len(set(graph)) > 1:
        raise RuntimeError("Cannot evaluate multiple `Tensor` come from "
                           "different `Graph`.")
      tensor_eval = get_session(graph[0]).run(tensor_eval,
                                              feed_dict=feed_dict,
                                              options=options,
                                              run_metadata=run_metadata)
    results = tuple([tensor_eval.pop(0) if i in tensor_idx else string_eval.pop(0)
                     for i in range(len(x))])
  # ====== mapping ====== #
  elif isinstance(x, Mapping):
    results = {}
    tensor_eval_key = []
    tensor_eval_value = []
    for k, v in x.items():
      if is_string(v):
        results[k] = builtins.eval(v)
      else:
        tensor_eval_key.append(k)
        tensor_eval_value.append(v)
    # evaluate tensor
    if len(tensor_eval) > 0:
      graph = [i.graph for i in tensor_eval_value]
      if len(set(graph)) > 1:
        raise RuntimeError("Cannot evaluate multiple `Tensor` come from "
                           "different `Graph`.")
      tensor_eval_value = get_session(graph[0]).run(tensor_eval_value,
                                                    feed_dict=feed_dict,
                                                    options=options,
                                                    run_metadata=run_metadata)
    # update results
    for k, v in zip(tensor_eval_key, tensor_eval_value):
      results[k] = v
  # ====== just a string ====== #
  elif is_string(x):
    results = builtins.eval(x)
  # ====== just a Tensorflow object ====== #
  elif isinstance(x, tf.Operation) or \
  is_tensor(x, inc_distribution=True, inc_variable=True):
    results = get_session(x.graph).run(x, feed_dict=feed_dict,
                                       options=options,
                                       run_metadata=run_metadata)
  # ====== exception ====== #
  else:
    raise RuntimeError("Cannot evaluate object of type: %s" % type(x))
  # ====== run updates after ====== #
  if update_after is not None:
    get_session(update_after.graph).run(update_after, feed_dict=feed_dict,
                                        options=options,
                                        run_metadata=run_metadata)
  return results

# ===========================================================================
# IO helper
# ===========================================================================
_saver = {}

def save_variables(var_list, path, session=None):
  """ This function only apply for trainable parameters """
  if session is None:
    session = get_session()
  var_list = [v for v in set(as_tuple(var_list)) if is_variable(v)]
  name = '|'.join(sorted([v.name for v in var_list]))
  if name in _saver:
    saver = _saver[name]
  else:
    saver = tf.train.Saver(var_list=var_list, restore_sequentially=False,
        allow_empty=False)
  # ====== save the variables ====== #
  checkpoint = saver.save(session, path, global_step=None,
      write_meta_graph=False, write_state=False)
  # ====== save meta-info for recreate variable ====== #
  var_meta = []
  for v in var_list:
    name = v.name.split(':')[0]
    dtype = v.dtype.base_dtype.name
    shape = v.shape.as_list()
    var_meta.append((name, dtype, shape))
  # ====== save the collections ====== #
  collections = {var.name: role.get_roles(var, return_string=True)
                 for var in var_list}
  with open(path + '.collections', 'wb') as f:
    cPickle.dump([collections, var_meta], f,
                 protocol=cPickle.HIGHEST_PROTOCOL)
  return checkpoint

def restore_variables(path, session=None):
  if session is None:
    session = get_session()
  # ====== load and check var meta ====== #
  with open(path + '.collections', 'rb') as f:
    collections, var_meta = cPickle.load(f)
  var_list = []
  allvars = {v.name.split(':')[0]: v for v in get_all_variables()}
  for name, dtype, shape in var_meta:
    if name in allvars: # found predefined variable
      var_list.append(allvars[name])
    else: # create new variable
      if tf.get_variable_scope().name:
        raise RuntimeError("The current variable scope is: %s, you can "
            "only restore variables from default scope."
            % tf.get_variable_scope().name)
      var_list.append(tf.get_variable(
          shape=shape, name=name, dtype=dtype))
  # ====== restore the variables ====== #
  name = '|'.join(sorted([v.name for v in var_list]))
  if name in _saver:
    saver = _saver[name]
  else:
    saver = tf.train.Saver(var_list=var_list, restore_sequentially=False,
        allow_empty=False)
  saver.restore(session, path)
  # ====== restore the collections ====== #
  for v in var_list:
    role.add_roles(v, collections[v.name])

def save_graph(path, graph=None):
  g = tf.summary.FileWriter(path)
  if graph is None:
    graph = get_session().graph
  elif isinstance(graph, tf.Session):
    graph = graph.graph
  g.add_graph(graph)
  g.flush()
  g.close()


import time
import numpy as np
from numpy import linspace
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import LinearLocator, FormatStrFormatter
from matplotlib import cm

np.seterr(all='raise')

class Solver(object):
    """
        At the basis of:
        https://github.com/hplgit/num-methods-for-PDEs/blob/master/src/diffu/diffu1D_u0.py
    """

    def __init__(self, x_max, t_max, k):
        """
        :param a: Variable coefficient (constant).
        :param L: Length of the domain ([0,L]).
        :param N: The total number of mesh cells; mesh points are numbered
                  from 0 to N.
        :param F: The dimensionless number a*dt/dx**2, which implicitly
                  specifies the time step.
        :param T: The stop time for the simulation.
        """

        # Boundary condition in U(x=0, t) and U(x=x_max, 0)
        self.boundary_condition = 0.0

        # Step/differential
        self.dx = 0.05
        self.dt = 0.008

        # Option pricing parameters
        self.sigma = 0.05
        self.r = 0.01
        self.k = k
        self.beta = 0.9

        # CFL condition:
        cfl = (self.sigma ** 2 * self.dt) / (2.0 * self.dx ** 2)
        c = self.dt / self.dx ** 2

        # Max x, max t
        self.x_max = x_max
        self.t_max = t_max

        self.x_array_size = int(self.x_max / self.dx) + 1
        self.t_array_size = int(self.t_max / self.dt) + 1

        # Mesh points in space
        self.x = linspace(0, self.x_max, self.x_array_size)

        # Mesh points in time
        self.t = linspace(0, self.t_max, self.t_array_size)
        self.U_xt = np.zeros((self.x_array_size, self.t_array_size), dtype=np.float)
        self.U_max = 0.0

    def initialFunction(self, x):
        """
        U(x, t=0)
        """

        initial_value = np.math.sin(x)
        return initial_value

    def pdeSolver(self):
        """
        Pde solver
        :return: self.U_xt, self.x, self.t, t1 - self.t0
        """

        # Set initial condition u(x,t=0) = initialFunction(x)

        for x_i in range(self.x_array_size):
            self.U_xt[x_i, 0] = self.initialFunction(self.x[x_i])

        try:
            # Calculate for t + dt
            for t_i in range(1, self.t_array_size):

                # Starting from t_1 (for t_0 we have used initial condition

                # Value for t_i -> t
                t = self.t[t_i]

                # Compute U[x0...x_max, t_i]
                for x_i in range(self.x_array_size - 1):
                    # Compute U at spatial mesh points U(x_i, t_n)
                    # R stands for spatial discretitized ODE
                    x = self.x[x_i]
                    # R = 0.1*np.math.tanh(x)*(self.U_xt[x_i -1, t_i -1] -2 *self.U_xt[x_i, t_i -1] + self.U_xt[x_i +1, t_i -1])/(self.dx*self.dx)

                    R1 = self.r * self.U_xt[x_i, t_i - 1] - self.r * x * (
                                self.U_xt[x_i + 1, t_i - 1] - self.U_xt[x_i, t_i - 1]) / self.dx
                    R2 = -0.5 * (self.sigma ** 2) * (x ** 2) * (
                                self.U_xt[x_i - 1, t_i - 1] - 2 * self.U_xt[x_i, t_i - 1] + self.U_xt[
                            x_i + 1, t_i - 1]) / (self.dx ** 2)

                    R = R1 + R2
                    # Runge-Kutta4 coeficients
                    k1 = self.dt * R
                    k2 = self.dt * (R + 0.5 * k1)
                    k3 = self.dt * (R + 0.5 * k2)
                    k4 = self.dt * (R + k3)

                    # Deriviated function in time domain
                    self.U_xt[x_i, t_i] = self.U_xt[x_i, t_i - 1] + (1.0 / 6.0) * (k1 + 2.0 * k2 + 2.0 * k3 + k4)

                    # Max conditions: current asset price vs strike price at t=T
                    if t == self.t_max and x > k:
                        self.U_xt[x_i, t_i] = 0

                    # Find max U_xt - important for plot function
                    if self.U_max < self.U_xt[x_i, t_i]:
                        self.U_max = self.U_xt[x_i, t_i]

                # Insert boundary conditions / Runge-Kutta at x = 0, and x = x_max
                self.U_xt[0, t_i] = self.boundary_condition
                self.U_xt[self.x_array_size - 1, t_i] = self.boundary_condition

            return self.U_xt, self.x, self.t
        except FloatingPointError:
            print("Exception at t = {} and x = {}".format(t, x))

    def plot(self):
        fig = plt.figure(figsize=(12, 8))
        # fig.tight_layout()
        fig.subplots_adjust(left=0, bottom=0, right=1, top=0.86)

        # 3D Plot - Runge-Kutta Method
        ax = fig.gca(projection='3d')
        ax.view_init(azim=-128, elev=18)
        self.x, self.t = np.meshgrid(self.x, self.t)
        surf = ax.plot_surface(self.x, self.t, np.transpose(self.U_xt), cmap=cm.jet,
                               antialiased=True)

        ax.set_xlabel('S')
        ax.set_ylabel('t')
        ax.set_zlabel('V[S,t]')
        ax.text2D(0.05, 0.95, "Method of lines: Linear Black-Scholes equation."
                              "\nStrike price: k= {},"
                              "\nInitial condition: V(S, t=0)=Sin(S),"
                              "\nBoundary condition: V(S=0, t)=0, V(S={:0.2f}, t)=0,"
                              "\nWhere $x\in$(0, {:0.2f}) and $t\in$(0, {}),"
                              "\n$\sigma=$ {}, $r=$ {}."
                  .format(self.k, self.x_max, self.x_max, self.t_max, self.sigma, self.r),size=14, transform=ax.transAxes)

        # Customize the z axis.
        ax.set_zlim(0, self.U_max)
        ax.zaxis.set_major_locator(LinearLocator(10))
        ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))

        # Add a color bar which maps values to colors.
        fig.colorbar(surf, shrink=0.5, aspect=5)
        plt.show()

if __name__ == '__main__':
    x_max = np.math.pi
    k = 3.5
    t_max = 0.2
    p = Solver(x_max, t_max, k)
    p.pdeSolver()
    p.plot()


from rest_framework import status
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated

from rest_framework.generics import get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView

from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema

from .managers import BinanceManager
from .serializers import WithdrawSerializer, DepositSerializer

binanceManager = BinanceManager()

#custom views, create deposit address

@swagger_auto_schema(methods=['POST'],
    request_body=openapi.Schema(
        type=openapi.TYPE_OBJECT,
        required=['asset'],
        properties={
            'asset': openapi.Schema(type=openapi.TYPE_STRING),
        },
    ),
    operation_description='create deposit endpoint ',

    responses={
            200: openapi.Response('get deposit address'),
        }, tags=['Trading']
    )
@api_view(['POST'])
def get_deposit_address(request):
    """get deposit address for trading"""

    result = binanceManager.create_deposit(request.data['asset'])
    return Response(result)

@swagger_auto_schema(methods=['post'],
    request_body=openapi.Schema(
        type=openapi.TYPE_OBJECT,
        required=['amount', 'asset', 'address'],
        properties={
            'amount': openapi.Schema(type=openapi.TYPE_INTEGER),
            'asset': openapi.Schema(type=openapi.TYPE_STRING),
            'address': openapi.Schema(type=openapi.TYPE_STRING)
        },
    ),
    operation_description='create withdrawal endpoint for given address and balance',

    responses={
        200: openapi.Response('create withdraw'),
    }, tags=['Trading']
)
@api_view(['POST'])
@permission_classes([IsAuthenticated])
def create_withdraw(request):
    """create withdraw endpoint"""
    serializer = WithdrawSerializer(data=request.data)
    if serializer.is_valid():
        data = serializer.data

        result = binanceManager.create_withdraw(data['asset'], data['address'], data['amount'])
        return Response(result)

    return Response(serializer.errors, status=400)

@swagger_auto_schema(methods=['GET'],
    operation_description='dummy operation for getting sub accounts ',

    responses={
            200: openapi.Response('get sub accs'),
        }, tags=['Trading']
    )
@api_view(['GET'])
@permission_classes([IsAuthenticated])
def get_sub_accounts(request):
    """get trading sub accounts"""

    result = binanceManager.get_sub_accounts()
    return Response(result)

@swagger_auto_schema(methods=['GET'],
    operation_description='dummy operation for getting coin information ',

    responses={
            200: openapi.Response('get coin information '),
        }, tags=['Trading']
    )
@api_view(['GET'])
@permission_classes([])
def coin_info(request):
    """get coin information """

    result = binanceManager.get_exchange_info()
    return Response(result)


import cv2
import os

from utilities.file_utilities import get_file_name
from utilities.path import root
from utilities.tools import flatten
from constants.path_manager import IMAGE_TEXT_DATA_PATH, IMAGE_OUTPUT_PATH

def get_table_lines(image, kernel_size):
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, kernel_size)
    erosion = cv2.erode(image, kernel, iterations=1)
    dilation = cv2.dilate(erosion, kernel, iterations=1)
    return dilation

def find_left_right_conner(x_coos, y_coos):
    coo_x = {}
    coo_y = {}
    i = 0
    j = 0
    min_margin = 10
    for x in x_coos:
        if x in coo_x:
            coo_x[x].append(y_coos[i])
        else:
            coo_x[x] = []
            coo_x[x].append(y_coos[i])
        i += 1

    for y in y_coos:
        if y in coo_y:
            coo_y[y].append(x_coos[j])
        else:
            coo_y[y] = []
            coo_y[y].append(x_coos[j])
        j += 1
    sorted_x = sorted(x_coos)
    sorted_y = sorted(y_coos)
    x_min = sorted_x[0]
    x_max = sorted_x[-1]
    y_min = sorted_y[0]
    y_max = sorted_y[-1]
    bottom_y = coo_x[x_min]
    top_y = coo_x[x_max]
    left_x = coo_y[y_min]
    right_x = coo_y[y_max]
    if len(bottom_y) >= 2 and \
            len(top_y) >= 2 and \
            len(left_x) >= 2 and \
            len(right_x) >= 2 and \
            abs(x_max - max(right_x)) <= 1 and \
            abs(x_min - min(left_x)) <= 1 and \
            abs(y_max - max(top_y)) <= 1 and\
            abs(y_min - min(bottom_y)) <= 1 and\
            y_max - y_min >= min_margin and \
            x_max - x_min >= min_margin:
        y1 = sorted(bottom_y)
        x_y_min = y1[0]
        y2 = sorted(top_y, reverse=True)
        x_y_max = y2[0]
        return x_min, x_y_min, x_max, x_y_max
    else:
        return [0, 0, 0, 0]

def extract_table_from_img(input_img_name, output_img_path=None, show_tables=False,
                           save_small_tables=False, get_test_tables=False):
    """
    table extracted from img will be saved in table_info
    if want draw rectangles to directly show tables of the img set show_tables=True

    """
    print('I am working on extracting table from image')
    os.path.isfile(input_img_name)
    img = cv2.imread(input_img_name)
    max_area = img.shape[0] * img.shape[1]
    max_area_condition = max_area * 3 / 4
    gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    edge_img = cv2.Canny(gray_img, 50, 150)
    dilate_kernel = cv2.getStructuringElement(cv2.MORPH_OPEN, (7, 3))
    dilate_image = cv2.dilate(edge_img, dilate_kernel, iterations=1)
    res, binary_img = cv2.threshold(dilate_image, 45, 255, cv2.THRESH_BINARY)

    horizontal_dilation = get_table_lines(binary_img, kernel_size=(50, 1))
    vertical_dilation = get_table_lines(binary_img, kernel_size=(1, 50))
    table_dilation = horizontal_dilation + vertical_dilation
    # table_dilation = cv2.dilate(table_dilation, dilate_kernel, iterations=1)
    table_dilation, contours, hierarchy = cv2.findContours(table_dilation, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    # print(len(contours))
    rec_coo = []
    for i in range(len(contours)):
        contour_coordinates = contours[i]
        x_coordinates = flatten(contour_coordinates[:, :, 0].tolist())
        y_coordinates = flatten(contour_coordinates[:, :, 1].tolist())
        x_no_repeat_list = list(set(x_coordinates))
        y_no_repeat_list = list(set(y_coordinates))
        same_x_num = len(x_coordinates) - len(x_no_repeat_list)
        same_y_num = len(y_coordinates) - len(y_no_repeat_list)
        if same_x_num >= 2 and same_y_num >= 2:
            rec_x_min, rec_y_min, rec_x_max, rec_y_max = find_left_right_conner(x_coordinates, y_coordinates)
            find_area = (rec_x_max - rec_x_min)*(rec_y_max - rec_y_min)
            if find_area is not 0 and find_area < max_area_condition:
                # print('find left right conner')
                rec_coo.append([rec_x_min, rec_y_min, rec_x_max, rec_y_max])
                if show_tables or get_test_tables:
                    cv2.rectangle(img, (rec_x_min, rec_y_min), (rec_x_max, rec_y_max), (0, 255, 0), 3)
                    f_name = get_file_name(input_img_name)
                    cv2.imwrite(os.path.join(f_name + '_Draw.png'), img)
    table_num = 0
    # extract table from img_name
    rec_list = []
    for x_y_coo in rec_coo:
        rec = img[x_y_coo[1]:x_y_coo[3] + 1, x_y_coo[0]:x_y_coo[2] + 1]
        rec_list.append(rec)
        if save_small_tables:
            table_label = os.path.join(output_img_path, 'g1_0_table_cut' + str(table_num) + '.png')
            cv2.imwrite(table_label, rec)
            table_num += 1
    return rec_list, rec_coo

if __name__ == '__main__':
    import time

    start_time = time.time()
    pdf_test_path = os.path.join(root, 'image_handlers', 'data', 'image4experiment')
    pdf_output_path = os.path.join(root, 'image_handlers', 'data', 'output4experiment')
    gerber_pdf_input_file = os.path.join(root, 'image_handlers', 'data', 'gerber_pdf_images', 'g1-0.png')

    file_name = 'test-gerber.png'
    # input a png
    input_file = os.path.join(pdf_test_path, file_name)
    # output
    output_file = os.path.join(pdf_output_path, file_name)
    # out_file = os.path.join(IMAGE_OUTPUT_PATH, file_name)

    # output a dict of locations
    recs, recs_coo = extract_table_from_img(input_file, show_tables=True, save_small_tables=False,
                                            output_img_path=IMAGE_OUTPUT_PATH)
    print('tests path is: {}'.format(input_file))
    print('time cost is: {}s'.format(time.time()-start_time))
    # print(recs_coo[0])
    # cv2.imshow('img_tables', recs[0])
    # cv2.waitKey(0)
    # cv2.destroyAllWindows()
    # print(recs)
    print(recs_coo)


"""
Update the "last_opening_date" of the project
"""
import sqlite3
from datetime import datetime

def run(project_id):
    conn = sqlite3.connect('api/databases_files/projects_db/all_project.db')
    cursor = conn.cursor()
    date_today = datetime.now()
    query = """UPDATE projects SET last_opening_date = ? WHERE id = ?"""
    data = (date_today, str(project_id))
    cursor.execute(query, data)
    conn.commit()


import os
import pymongo
import pytest
if os.path.exists("env.py"):
    import env

# checks for environment variables
def test_vars_exist():
    envs = [
        "IP",
        "PORT",
        "SECRET_KEY",
        "MONGO_URI",
        "MONGO_DBNAME",
        "EMAIL_LOGIN",
        "EMAIL_PASS"]
    if os.path.exists("env.py"):
        for x in envs:
            assert os.getenv(x) is not None, \
                "Please set environment variable '{}'".format(x)
    else:
        assert os.path.exists("env.py"), "Create an env.py file"

client = pymongo.MongoClient(
    os.environ.get("MONGO_URI"),
    serverSelectionTimeoutMS=5000)

# tests mongo connection
def test_mongo():
    try:
        client.server_info()
    except BaseException:
        raise ConnectionError("Invalid Mongo URI Connection String")

# checks if database exists
def test_db():
    try:
        assert os.environ.get("MONGO_DBNAME") in client.list_database_names(
        ), "Database '{}' does not exist".format(
                os.environ.get("MONGO_DBNAME"))
    except BaseException:
        raise ConnectionError("Connection could not be made")

if os.path.exists("env.py"):
    db = client[os.environ.get("MONGO_DBNAME")]
else:
    db = client["None"]

# checks if collections all collections required exist
def test_collections():
    colls = ["customers", "defects", "departments", "form_fields", "origins",
             "pack_info", "products", "roles"]
    for coll in colls:
        try:
            assert coll in db.list_collection_names(
            ), "Collection '{}' does not exist in database".format(coll)
        except BaseException:
            raise ConnectionError("Connection could not be made")

# checks if product structure is correct
def test_product():
    product = db.products.find_one()
    keys = [
        "product_name",
        "department",
        "customer",
        "status",
        "start_date",
        "created_by"]
    if product is not None:
        for key in keys:
            assert key in product.keys(), \
                "Product object is missing '{}' key".format(key)
    else:
        raise FileNotFoundError("No product objects in collection")

# checks if form fields structure is correct
def test_form_fields():
    field_list = db.form_fields.find_one()
    keys = [
        "customer",
        "department",
        "commercial_details",
        "packaging_details",
        "operations_details",
        "technical_details",
        "prophet_details"]
    if field_list is not None:
        for key in keys:
            for key in keys:
                assert key in field_list.keys(), \
                    "Form field object is missing '{}' key".format(key)
    else:
        raise FileNotFoundError("No Form Field objects in collection")


from functools import lru_cache
import glob
import io
import os
import re
from pathlib import Path
from typing import Tuple, Union
from urllib import request
from PIL import Image

import requests
import torch as th
from guided_diffusion.script_util import (create_model_and_diffusion,
                                          model_and_diffusion_defaults)
from torch.nn import functional as tf
from torchvision.transforms import functional as tvf
from tqdm.autonotebook import tqdm

from data.diffusion_model_flags import DIFFUSION_LOOKUP

CACHE_PATH = os.path.expanduser("~/.cache/clip-guided-diffusion")
ALPHANUMERIC_REGEX = r"[^\w\s]"

# modified from https://github.com/lucidrains/DALLE-pytorch/blob/d355100061911b13e1f1c22de8c2b5deb44e65f8/dalle_pytorch/vae.py
def download(url: str, filename: str, root: str = CACHE_PATH) -> str:
    os.makedirs(root, exist_ok=True)
    download_target = Path(os.path.join(root, filename))
    download_target_tmp = download_target.with_suffix('.tmp')
    if os.path.exists(download_target) and not os.path.isfile(download_target):
        raise RuntimeError(
            f"{download_target} exists and is not a regular file")
    if os.path.isfile(download_target):
        return str(download_target)
    with request.urlopen(url) as source, open(download_target_tmp, "wb") as output:
        with tqdm(total=int(source.info().get("Content-Length")), ncols=80) as loop:
            while True:
                buffer = source.read(4096)
                if not buffer:
                    break
                output.write(buffer)
                loop.update(len(buffer))
    os.rename(download_target_tmp, download_target)
    return str(download_target)

def download_guided_diffusion(image_size: int, class_cond: bool, checkpoints_dir: str = CACHE_PATH, overwrite: bool = False) -> str:
    cond_key = 'cond' if class_cond else 'uncond'
    diffusion_model_info = DIFFUSION_LOOKUP[cond_key][image_size]
    if not overwrite:
        target_path = Path(checkpoints_dir).joinpath(diffusion_model_info["filename"])
        if target_path.exists():
            return str(target_path)
    return download(diffusion_model_info["url"], diffusion_model_info["filename"], checkpoints_dir)

@lru_cache(maxsize=1)
def load_guided_diffusion(
    checkpoint_path: str,
    image_size: int,
    class_cond: bool,
    diffusion_steps: int = None,
    timestep_respacing: str = None,
    use_fp16: bool = True,
    device: str = '',
    noise_schedule: str = 'linear',
    dropout: float = 0.0,
):
    '''
    checkpoint_path: path to the checkpoint to load.
    image_size: size of the images to be used.
    class_cond: whether to condition on the class label
    diffusion_steps: number of diffusion steps
    timestep_respacing: whether to use timestep-respacing or not
    '''
    if not (len(device) > 0): raise ValueError("device must be set")
    if not (noise_schedule in [ "linear", "cosine"]): raise ValueError("linear_or_cosine must be set")

    cond_key = 'cond' if class_cond else 'uncond'
    diffusion_model_info = DIFFUSION_LOOKUP[cond_key][image_size]
    model_config: dict = model_and_diffusion_defaults()
    model_config.update(diffusion_model_info['model_flags'])
    model_config.update(**{  # Custom params from user
        'diffusion_steps': diffusion_steps,
        'timestep_respacing': timestep_respacing,
        "use_fp16": use_fp16,
        "noise_schedule": noise_schedule,
        "dropout": dropout,
    })
    model, diffusion = create_model_and_diffusion(**model_config)
    model.load_state_dict(th.load(checkpoint_path, map_location='cpu'))
    model.requires_grad_(False).eval().to(device)
    for name, param in model.named_parameters():
        if "qkv" in name or "norm" in name or "proj" in name:
            param.requires_grad_()
    if model_config["use_fp16"]:
        model.convert_to_fp16()
    return model.to(device), diffusion

def fetch(url_or_path):
    if str(url_or_path).startswith('http://') or str(url_or_path).startswith('https://'):
        r = requests.get(url_or_path)
        r.raise_for_status()
        fd = io.BytesIO()
        fd.write(r.content)
        fd.seek(0)
        return fd
    return open(url_or_path, 'rb')

def alphanumeric_filter(s: str) -> str:
    return re.sub(ALPHANUMERIC_REGEX, "", s).replace(" ", "_")

def clean_and_combine_prompts(base_path, txts, batch_idx, max_length=255) -> str:
    clean_txt = "_".join([alphanumeric_filter(txt) for txt in txts])[:max_length]
    return os.path.join(base_path, clean_txt, f"{batch_idx:02}")

def log_image(image: th.Tensor, base_path: str, txts: list, current_step: int, batch_idx: int) -> str:
    dirname = clean_and_combine_prompts(base_path, txts, batch_idx)
    os.makedirs(dirname, exist_ok=True)
    stem = f"{current_step:04}"
    filename = os.path.join(dirname, f'{stem}.png')
    pil_image = tvf.to_pil_image(image.add(1).div(2).clamp(0, 1))
    pil_image.save(os.path.join(os.getcwd(), f'current.png'))
    pil_image.save(filename)
    return str(filename)

def create_gif(base, prompts, batch_idx):
    io_safe_prompts = clean_and_combine_prompts(base, prompts, batch_idx)
    images_glob = os.path.join(io_safe_prompts, "*.png")
    imgs = [Image.open(f) for f in sorted(glob.glob(images_glob))]
    gif_filename = f"{io_safe_prompts}_{batch_idx:02}.gif"
    imgs[0].save(fp=gif_filename, format='GIF', append_images=imgs, save_all=True, duration=200, loop=0)
    return gif_filename

def resize_image(image: th.Tensor, out_size: Union[int, Tuple[int, int]]) -> th.Tensor:
    """(Katherine Crowson) - Resize image"""
    outsize_x = out_size if isinstance(out_size, int) else out_size[0]
    outsize_y = out_size if isinstance(out_size, int) else out_size[1]
    ratio = image.size(1) / image.size(1)
    area = min(image.size(0) * image.size(1), outsize_x * outsize_y)
    size = round((area * ratio)**0.5), round((area / ratio)**0.5)
    return image.reshape(size)


# -*- coding: utf-8 -*-
from django.shortcuts import render
from django import forms
from testapi.models import SmsLog

class SmsForm(forms.Form):
    name = forms.CharField(
        max_length=100,
        label='Ваше имя:',
        widget=forms.TextInput(attrs={'class': 'form-control'})
    )
    phone = forms.CharField(
        max_length=100,
        required=False,
        label='Телефон абонента:',
        widget=forms.TextInput(attrs={'class': 'form-control'})
    )
    message = forms.CharField(
        label='Ваше сообщение:',
        widget=forms.Textarea(attrs={'class': 'form-control', 'rows': '12'})
    )

def home(request):
    form = SmsForm()
    log = SmsLog.objects.all()
    return render(request, 'site/home.html', dict(form=form, log=log))


# Shocking1 - first experiments with the DS3502 DigiPot

import sys
import random
import pdb
import readchar
import time
from adafruit_motorkit import MotorKit
import board
import adafruit_ds3502
i2c = board.I2C()
ds3502 = adafruit_ds3502.DS3502(i2c)

emulation=0
max_drink_counter=6 # one bad drink in each group of six
bad = 0
drink_counter = 0
# not sure what I am doing wrong

mh1 = MotorKit()

# recommended for auto-disabling motors on shutdown!
def turnOffMotors():
    mh1.motor1.throttle = None
    mh1.motor2.throttle = None
    mh1.motor3.throttle = None
    mh1.motor4.throttle = None
    #mh2.motor1.throttle = None
    #mh2.motor2.throttle = None
    #mh2.motor3.throttle = None
    #mh2.motor4.throttle = None

#atexit.register(turnOffMotors)

m = [None]
m.append(mh1.motor1)
m.append(mh1.motor2)
m.append(mh1.motor3)
m.append(mh1.motor4)
#m.append(mh2.motor1)
#m.append(mh2.motor2)
#m.append(mh2.motor3)
#m.append(mh2.motor4)

try:
    import RPi.GPIO as GPIO # Import Raspberry Pi GPIO library

    GPIO.setwarnings(False) # Ignore warning for now
    #GPIO.setmode(GPIO.BOARD)
    GPIO.setmode(GPIO.BCM)
    #GPIO.setup(13, GPIO.IN)
    #GPIO.setup(19, GPIO.IN)

    # Set pins 10 and 12 to be an input pin and set initial value to be pulled low (off)
    #GPIO.setup(13, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
    GPIO.setup(13, GPIO.IN, pull_up_down=GPIO.PUD_UP)
    GPIO.setup(19, GPIO.IN, pull_up_down=GPIO.PUD_UP)

except:
    print('Raspberry pi not responding, continuing in emulation mode')
    emulation=1

def next_drink():
    """ increment the counter and if need be select the next bad shot"""
    global drink_counter
    global bad
    if bad == 0:
        bad = random.randrange(6)+1
    drink_counter += 1
    if drink_counter > max_drink_counter:
        drink_counter=1
        bad = random.randrange(6)+1

    return

def dispense(player):
    """ dispense for a player
    player 1 is pumps 1 and 3
    player 2 is pumps 5 and 7

    player 1 - Motor 1 forward good stuff
    player 1 - Motor 2 forward bad stuff
    player 2 - Motor 3 forward good stuff
    player 2 - Motor 4 forward bad stuff
    """

    global drink_counter
    global bad
    print("\nIn dispense player=",player)
    for i in range(player):
        # get drink number and bad number
        next_drink()
        print('\tdispense a drink player=%i drink_counter=%i bad=%i ' % (i,drink_counter, bad), end='')
        if drink_counter==bad:
            print('\tYou are not going into space today.')
            time.sleep(1)
            # serve the icky drink
            # reset drink_counter and bad. Or should I? One bad out of six shots
            drink_counter=0
            bad=0
        else:
            # serve a fine shot
            time.sleep(1)
            print('\tYou get to go to space. Congratulations')

    print('service complete\n')

# We don't know the state of the drink counter or of anything.
# you push the button and 1 or 2 shots are poured.
# 1 out of 6 shots should be bad. But there are edge cases one should
# keep in mind.

print("Starting loop. Press a button, or if you are not on a pi  the keys '1', '2', or 'Q'")
flag1=False
wiper1 = 0
#while True:
#    for i in range (12):
#        print(i*10)
#        ds3502.wiper = i*10
#        time.sleep(1)

while True: # Run forever
    # event loops don't need to run at system speed
    time.sleep(.01)
    t=time.time()
    if emulation == 1:
        key = readchar.readkey()
        if key.upper()=='Q':
            sys.exit(2)

        key=int(key)
        if key in (1,2):
            dispense(key)

    else:
        if GPIO.input(13) == GPIO.LOW:
            #dispense(1)
            if flag1==False:
                wiper1 += 10
                print('Was False now true.')
                print('Was False now true. wiper1: ', wiper1)
                ds3502.wiper = wiper1
                if wiper1 > 128:
                    wiper1 = 0
            flag1=True
        else:
            #GPIO.input(13) == GPIO.HIGH
            if flag1==True:
                print('Was True now false.')
                #ds3502.wiper = wiper1
            flag1 = False

        if GPIO.input(19) == GPIO.LOW:
            print('button 2 pushed')
            #dispense(2)


__author__ = "Manouchehr Rasouli"
__date__ = "22/Aug/2017"

from selenium import webdriver
import time

class ChromeLoadTest:
    def __init__(self, url, result):
        self.driver = webdriver.Chrome()
        self.result = result
        self.url = url

    def do_test(self):
        start = time.time()
        self.driver.get(self.url["url"])
        end = time.time()

        navigation_start = self.driver.execute_script("return window.performance.timing.navigationStart")
        response_start = self.driver.execute_script("return window.performance.timing.responseStart")
        dom_complete = self.driver.execute_script("return window.performance.timing.domComplete")

        back_end_performance = response_start - navigation_start
        front_end_performance = dom_complete - response_start

        self.result["chrome_check_time"] = str(end - start)
        self.result["chrome_back_end_performance"] = str(back_end_performance / 1000)
        self.result["chrome_front_end_performance"] = str(front_end_performance / 1000)
        self.driver.quit()
        return self.result


from glob import glob
import os, shutil
import pikepdf, pdfplumber

PATH = input('Please enter the path: ')

EXT = "*.pdf"
temp_cfn = [file
                 for path, subdir, files in os.walk(PATH)
                 for file in glob(os.path.join(path, EXT))]

for file in temp_cfn:

    pdf = pdfplumber.open(file)
    page = pdf.pages[0]
    text = page.extract_text()
    print( "\n\n", text )
    pdf.close()

    pdf_file = pikepdf.Pdf.open(file)

    for page  in pdf_file.pages:
        for annots in page.get('/Annots'):
            uri = annots.get('/A').get('/URI')
            if(uri is not None):
                print(  '[FILE] '+ str(file) +'\n [+] URL FOUND ' , uri, "\n\n" )


import numpy as np
import scipy as sc
from scipy import special

from sklearn.preprocessing import PolynomialFeatures

class FourierFeatures:

    def __init__(self, d_state, nb_feat, band):
        self.d_state = d_state
        self.nb_feat = nb_feat

        self.freq = np.random.multivariate_normal(mean=np.zeros(self.d_state),
                                                  cov=np.diag(1.0 / band),
                                                  size=self.nb_feat)
        self.shift = np.random.uniform(-np.pi, np.pi, size=self.nb_feat)

    def fit_transform(self, x):
        phi = np.sin(np.einsum('nk,...k->...n', self.freq, x) + self.shift)
        return phi

class Policy:

    def __init__(self, d_state, dm_act, **kwargs):
        self.d_state = d_state
        self.dm_act = dm_act

        if 'band' in kwargs:
            self.band = kwargs.get('band', False)
            self.nb_feat = kwargs.get('nb_feat', False)
            self.basis = FourierFeatures(self.d_state, self.nb_feat, self.band)
        else:
            self.degree = kwargs.get('degree', False)
            self.nb_feat = int(sc.special.comb(self.degree + self.d_state, self.degree)) - 1
            self.basis = PolynomialFeatures(self.degree, include_bias=False)

        self.K = 1e-8 * np.random.randn(self.dm_act, self.nb_feat)
        self.cov = np.eye(dm_act)

    def features(self, x):
        return self.basis.fit_transform(x.reshape(-1, self.d_state)).squeeze()

    def mean(self, x):
        feat = self.features(x)
        return np.einsum('...k,mk->...m', feat, self.K)

    def actions(self, x, stoch):
        mean = self.mean(x)
        if stoch:
            return np.random.multivariate_normal(mean, self.cov)
        else:
            return mean

    def grad(self, x, u):
        return np.einsum('nl,ll,nk->nk', u - self.mean(x), np.linalg.inv(self.cov), self.features(x))

class GPOMDP:

    def __init__(self, env, n_episodes, nb_steps, discount, alpha):
        self.env = env

        self.d_state = self.env.observation_space.shape[0]
        self.dm_act = self.env.action_space.shape[0]

        self.ulim = self.env.action_space.high

        self.n_episodes = n_episodes
        self.nb_steps = nb_steps
        self.discount = discount

        self.alpha = alpha

        self.ctl = Policy(self.d_state, self.dm_act, degree=1)
        self.ctl.cov = 0.01 * self.ctl.cov

        self.rollouts = []

    def sample(self, n_episodes, nb_steps, stoch=True):
        rollouts = []

        for _ in range(n_episodes):
            roll = {'x': np.empty((0, self.d_state)),
                    'u': np.empty((0, self.dm_act)),
                    'r': np.empty((0,))}

            x = self.env.reset()

            for _ in range(nb_steps):
                u = self.ctl.actions(x, stoch)

                roll['x'] = np.vstack((roll['x'], x))
                roll['u'] = np.vstack((roll['u'], u))

                x, r, done, _ = self.env.step(np.clip(u, - self.ulim, self.ulim))
                roll['r'] = np.hstack((roll['r'], r))

            rollouts.append(roll)

        return rollouts

    def baseline(self, rewards, gradient):
        _norm = np.zeros((self.nb_steps, self.ctl.nb_feat))
        for g in gradient:
            _norm += np.cumsum(g**2, axis=0)

        _b = np.zeros((self.nb_steps, self.ctl.nb_feat))
        for r, g in zip(rewards, gradient):
            _b += np.cumsum(g**2, axis=0) * r[:, np.newaxis] / _norm

        return _b

    def run(self, nb_iter=100, verbose=False):
        _trace = {'ret': []}

        for it in range(nb_iter):
            self.rollouts = self.sample(n_episodes=self.n_episodes, nb_steps=self.nb_steps)

            _disc = np.hstack((1.0, np.cumprod(self.discount * np.ones((self.nb_steps,))[:-1])))

            _reward = []
            for roll in self.rollouts:
                _reward.append(_disc * roll['r'])

            _grad = []
            for roll in self.rollouts:
                _g = self.ctl.grad(roll['x'], roll['u'])
                _grad.append(_g)

            _baseline = self.baseline(_reward, _grad)

            _wgrad = np.zeros((self.ctl.nb_feat, ))
            for r, g in zip(_reward, _grad):
                _wgrad += np.sum(np.cumsum(g, axis=0) * (r[:, np.newaxis] - _baseline), axis=0) / len(_reward)

            self.ctl.K += self.alpha * _wgrad

            ret = 0.0
            for r in _reward:
                ret += r.sum() / len(_reward)

            _trace['ret'].append(ret)

            if verbose:
                print('it=', it, f'ret={ret:{5}.{4}}')

        return _trace


"""Camera Calibration"""
import glob

import cv2
import matplotlib.pyplot as plt
import numpy as np

class CameraCalibrator:
    """A class that provides image calibration and undistortion."""

    def __init__(self):
        self._cam_matrix = None
        self._dist_coeffs = None
        self._is_calib = False
        self._objpoints = None
        self._imgpoints = None

    def load_calibration_points(self, images_path, visualize=False):
        """prepare object points and image points from all the images"""

        if len(images_path) == 0:
            raise ValueError('No images path provided')

        images = glob.glob(images_path)

        objp = np.zeros((6*9, 3), np.float32)
        objp[:, :2] = np.mgrid[0:9, 0:6].T.reshape(-1, 2)

        # Arrays to store object points and image points from all the images.
        objpoints = [] # 3d points in real world space
        imgpoints = [] # 2d points in image plane.

        # Step through the list and search for chessboard corners
        for fname in images:
            img = cv2.imread(fname)
            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

            # Find the chessboard corners
            ret, corners = cv2.findChessboardCorners(gray, (9, 6), None)

            # If found, add object points, image points
            if ret:
                objpoints.append(objp)
                imgpoints.append(corners)

                if visualize:
                    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
                    ax1.set_title('Original Image {}'.format(img.shape))
                    ax1.imshow(img)

                    # Draw and display the corners
                    img = cv2.drawChessboardCorners(img, (9, 6), corners, ret)
                    ax2.set_title('Chessboard corners {}'.format(img.shape))
                    ax2.imshow(img)
                    plt.show()
            else:
                if visualize:
                    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
                    ax1.set_title('Original Image {}'.format(img.shape))
                    ax1.imshow(img)

                    ax2.set_title('Chessboard not found')
                    plt.show()

        self._objpoints = objpoints
        self._imgpoints = imgpoints

    def calibrate_camera(self, src_img):
        """Calculate camera matrix and distortion coeffs given a source image. """

        if self._objpoints is None or self._imgpoints is None:
            raise ValueError('Image points do not exist')

        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(
            self._objpoints,
            self._imgpoints,
            (src_img.shape[1], src_img.shape[0]), # (w, h) for internal cam mtx
            None,   # Cam mtx
            None    # distortion coeffs
        )
        self._cam_matrix = mtx
        self._dist_coeffs = dist
        self._is_calib = True

    def undistort(self, distorted_img):
        """Transform an image to compensate for lens distortion."""

        if not self._is_calib:
            self.calibrate_camera(distorted_img)

        return cv2.undistort(
            distorted_img,
            self._cam_matrix,
            self._dist_coeffs,
            None,               # pointer to output image
            self._cam_matrix    # new matrix - for additional scaling/shifting
        )


import numpy as np
from scipy import signal
from matplotlib import pyplot as plt

nx = 50
D = 2

x1 = np.random.randn(nx)
x2 = np.roll(x1, D)
x3 = np.roll(x1, -D)

crr12 = signal.correlate(x1, x2)
crr13 = signal.correlate(x1, x3)

lags = np.arange(-nx + 1, nx)
delay12 = lags[np.argmax(crr12)]
delay13 = lags[np.argmax(crr13)]

fig = plt.figure()
plt.plot(x1, 'b', label='$x_1$')
plt.plot(x2, 'g', label='$x_2$: delayed')
plt.plot(x3, 'r', label='$x_3$: advanced')
plt.grid()
plt.legend()
plt.savefig('signals.jpg')

fig = plt.figure(figsize=(10, 4))
axes = fig.add_subplot(1, 2, 1)
axes.plot(lags, crr12)
axes.set_xlabel("lag (samples)")
axes.set_ylabel("xcorr (x1, x2)")
axes.grid()
axes.set_title("delay: {:d} samples".format(delay12))
axes = fig.add_subplot(1, 2, 2)
axes.plot(lags, crr13)
axes.set_xlabel("lag (samples)")
axes.set_ylabel("xcorr (x1, x3)")
axes.grid()
axes.set_title("delay: {:d} samples".format(delay13))
plt.savefig('xcorrs.jpg')

plt.show()

def mfreqz(b, a=1):
    w, h = signal.freqz(b,a)
    h_dB = 20 * np.log10 (abs(h))
    plt.subplot(211)
    plt.plot(w / max(w), h_dB)
    plt.ylim(-150, 5)
    plt.ylabel('Magnitude (db)')
    plt.xlabel(r'Normalized Frequency (x$\pi$rad/sample)')
    plt.title(r'Frequency response')
    plt.subplot(212)
    h_Phase = np.unwrap(np.arctan2(np.imag(h), np.real(h)))
    plt.plot(w/max(w),h_Phase)
    plt.ylabel('Phase (radians)')
    plt.xlabel(r'Normalized Frequency (x$\pi$rad/sample)')
    plt.title(r'Phase response')
    plt.subplots_adjust(hspace=0.5)

n = 201
fs = 32
ft = signal.firwin(n, cutoff = [1, 4], window = "hanning", pass_zero=False, fs=fs)
mfreqz(ft)
plt.show()