#! /usr/bin/env python3
"""This demo uses Aboleth for approximate Gaussian process regression."""
import logging

import numpy as np
import bokeh.plotting as bk
import bokeh.palettes as bp
import tensorflow as tf
# from sklearn.gaussian_process.kernels import Matern as kern
from sklearn.gaussian_process.kernels import RBF as kern

import aboleth as ab
from aboleth.datasets import gp_draws

# Set up a python logger so we can see the output of MonitoredTrainingSession
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# Set up a consistent random seed in Aboleth so we get repeatable, but random
# results
RSEED = 666
ab.set_hyperseed(RSEED)

# Data settings
N = 1000  # Number of training points to generate
Ns = 400  # Number of testing points to generate
kernel = kern(length_scale=0.5)  # Kernel to use for making a random GP draw
true_noise = 0.1  # Add noise to the GP draws, to make things a little harder

# Model settings
n_samples = 5  # Number of random samples to get from an Aboleth net
p_samples = 30  # Number of samples for prediction
n_epochs = 200  # how many times to see the data for training
batch_size = 10  # mini batch size for stochastric gradients
config = tf.ConfigProto(device_count={'GPU': 0})  # Use GPU? 0 is no

# Model initialisation
NOISE = 1.  # Likelihood st. dev. initialisation, and learning

# Random Fourier Features
kern = ab.RBF(learn_lenscale=True)  # keep the length scale positive

# Variational Fourier Features -- length-scale setting here is the "prior", we
# can choose to optimise this or not
# lenscale = 1.
# kern = ab.RBFVariational(lenscale=lenscale)  # This is VAR-FIXED kernel from
# Cutjar et. al. 2017

# This is how we make the "latent function" of a Gaussian process, here
# n_features controls how many random basis functions we use in the
# approximation. The more of these, the more accurate, but more costly
# computationally. "full" indicates we want a full-covariance matrix Gaussian
# posterior of the model weights. This is optional, but it does greatly improve
# the model uncertainty away from the data.
n_samples_ = tf.placeholder(tf.int32)
net = (
    ab.InputLayer(name="X", n_samples=n_samples_) >>
    ab.RandomFourier(n_features=200, kernel=kern) >>
    ab.DenseVariational(output_dim=1, learn_prior=True, full=True)
)

def main():
    """Run the demo."""
    n_iters = int(round(n_epochs * N / batch_size))
    print("Iterations = {}".format(n_iters))

    # Get training and testing data
    Xr, Yr, Xs, Ys = gp_draws(N, Ns, kern=kernel, noise=true_noise)

    # Prediction points
    Xq = np.linspace(-20, 20, Ns).astype(np.float32)[:, np.newaxis]
    Yq = np.linspace(-4, 4, Ns).astype(np.float32)[:, np.newaxis]

    # Set up the probability image query points
    Xi, Yi = np.meshgrid(Xq, Yq)
    Xi = Xi.astype(np.float32).reshape(-1, 1)
    Yi = Yi.astype(np.float32).reshape(-1, 1)

    _, D = Xr.shape

    # Name the "data" parts of the graph
    with tf.name_scope("Input"):
        # This function will make a TensorFlow queue for shuffling and batching
        # the data, and will run through n_epochs of the data.
        Xb, Yb = batch_training(Xr, Yr, n_epochs=n_epochs,
                                batch_size=batch_size)
        X_ = tf.placeholder_with_default(Xb, shape=(None, D))
        Y_ = tf.placeholder_with_default(Yb, shape=(None, 1))

    # This is where we build the actual GP model
    with tf.name_scope("Deepnet"):
        phi, kl = net(X=X_)
        noise = ab.pos_variable(NOISE)
        ll = tf.distributions.Normal(loc=phi, scale=noise).log_prob(Y_)
        loss = ab.elbo(ll, kl, N)

    # Set up the training graph
    with tf.name_scope("Train"):
        optimizer = tf.train.AdamOptimizer()
        global_step = tf.train.create_global_step()
        train = optimizer.minimize(loss, global_step=global_step)

    # This is used for building the predictive density image
    with tf.name_scope("Predict"):
        logprob = ab.sample_mean(ll)

    # Logging learning progress
    log = tf.train.LoggingTensorHook(
        {'step': global_step, 'loss': loss},
        every_n_iter=1000
    )

    # This is the main training "loop"
    with tf.train.MonitoredTrainingSession(
            config=config,
            save_summaries_steps=None,
            save_checkpoint_secs=None,
            hooks=[log]
    ) as sess:
        try:
            while not sess.should_stop():
                sess.run(train, feed_dict={n_samples_: n_samples})
        except tf.errors.OutOfRangeError:
            print('Input queues have been exhausted!')
            pass

        # Prediction, the [[None]] is to stop the default placeholder queue
        Ey = sess.run(phi,
                      feed_dict={X_: Xq, Y_: [[None]], n_samples_: p_samples})
        logPY = sess.run(logprob,
                         feed_dict={Y_: Yi, X_: Xi, n_samples_: p_samples})

    Eymean = Ey.mean(axis=0)  # Average samples to get mean predicted funtion
    Py = np.exp(logPY.reshape(Ns, Ns))  # Turn log-prob into prob

    # Plot
    im_min = np.amin(Py)
    im_size = np.amax(Py) - im_min
    img = (Py - im_min) / im_size
    f = bk.figure(tools='pan,box_zoom,reset', sizing_mode='stretch_both')
    f.image(image=[img], x=-20., y=-4., dw=40., dh=8,
            palette=bp.Plasma256)
    f.circle(Xr.flatten(), Yr.flatten(), fill_color='blue', legend='Training')
    f.line(Xs.flatten(), Ys.flatten(), line_color='blue', legend='Truth')
    for y in Ey:
        f.line(Xq.flatten(), y.flatten(), line_color='red', legend='Samples',
               alpha=0.2)
    f.line(Xq.flatten(), Eymean.flatten(), line_color='green', legend='Mean')
    bk.show(f)

def batch_training(X, Y, batch_size, n_epochs):
    """Batch training queue convenience function."""
    data_tr = tf.data.Dataset.from_tensor_slices({'X': X, 'Y': Y}) \
        .shuffle(buffer_size=1000, seed=RSEED) \
        .repeat(n_epochs) \
        .batch(batch_size)
    data = data_tr.make_one_shot_iterator().get_next()
    return data['X'], data['Y']

if __name__ == "__main__":
    main()


from pathlib import PurePosixPath
from typing import Dict, Any, Union, Sequence, Iterable, Tuple
import re

from cw.cached import cached

TreeNodeKey = Union[str, int]
TreeNodePath = Union[PurePosixPath, TreeNodeKey, Sequence[TreeNodeKey]]

int_re = re.compile(r"^\s*\[(\d+)\]\s*$")

class TreeNode:
    """

    """
    def __init__(self, *, value: Any=None, key: TreeNodePath = None, parent: "TreeNode"=None, default=None):
        self.children: Dict[TreeNodeKey, TreeNode] = {}
        if key is None and (parent is not None):
            raise ValueError("Only the root node may not have a key.")

        # At this point NoneType is allowed because it may only be the root node, which is unnamed.
        if not isinstance(key, (str, int, type(None))):
            raise TypeError("The key may only be of type str or int.")

        # Set the parent. This will be None for the root node.
        self.parent = parent

        if parent is not None:
            self.key = key
            self.root = parent.root
            self.parent.add_child(self)
        else:
            self.root = self
            self.key = ""

        self.value = value
        self.__default = default

    @classmethod
    def from_path_value_pairs(cls, path_value_pairs: Iterable[Tuple[TreeNodePath, Any]]):
        root = cls(default=lambda:  None)
        for path, value in path_value_pairs:
            root[path] = value
        root.set_default(None)
        return root

    @classmethod
    def from_dict(cls, path_value_pairs: Dict[TreeNodePath, Any]):
        return cls.from_path_value_pairs(path_value_pairs.items())

    @cached
    def default(self):
        # If __default is None, then return the default of the parent node.
        # If this is the root Node, return None.
        if self.__default is None:
            if self.is_root:
                return None
            else:
                return self.parent.default
        else:
            return self.__default

    def set_default(self, default):
        self.__default = default
        del self.default
        for _, child in self.items():
            if hasattr(child, "default"):
                del child.default

    @cached
    def path(self):
        node = self
        path_parts = []
        while node.parent is not None:
            path_parts.append(node.path_key)
            node = node.parent
        return PurePosixPath("/", *reversed(path_parts))

    @cached
    def path_key(self):
        return self.escape_key(self.key)

    @cached
    def is_root(self):
        return self is self.root

    @staticmethod
    def escape_key(key):
        if isinstance(key, int):
            return f"[{key}]"
        elif isinstance(key, str):
            return key
        else:
            raise TypeError("The key may only be of type str or int.")

    @staticmethod
    def unescape_key(escaped_key):
        match = int_re.match(escaped_key)
        if match:
            return int(match.group(1))
        else:
            return escaped_key

    @staticmethod
    def generate_path_obj(path: TreeNodePath) -> PurePosixPath:
        if isinstance(path, PurePosixPath):
            path = path
        elif isinstance(path, (list, tuple)):
            path = PurePosixPath(*(f"[{x}]" if isinstance(x, int) else x for x in path))
        else:
            path = PurePosixPath(f'[{path}]' if isinstance(path, int) else path)
        return path

    def add_child(self, child: "TreeNode"):
        self.children[child.key] = child

    def node_from_path(self, path: TreeNodePath):
        path = self.generate_path_obj(path)
        path_parts = path.parts

        # If the path_parts tuple is empty, then the path points to this object.
        if not path_parts:
            return self

        next_node_key = path_parts[0]

        # If the path is absolute set the search node it the root node.
        # Otherwise set the child node to the next child in line.
        if path.is_absolute():
            next_node = self.root
        else:
            # If the next node name is "..", go to the parent.
            if next_node_key == "..":
                next_node = self.parent or self
            else:
                if next_node_key in self.children:
                    next_node = self.children[self.escape_key(path_parts[0])]
                else:
                    if callable(self.default):
                        next_node = TreeNode(key=self.unescape_key(next_node_key), parent=self, value=self.default())
                    else:
                        raise KeyError(f"Node with key '{next_node_key}' in path '{path}' does not exist.")

        # Keep searching for the target node in at the next_node.
        return next_node.node_from_path(PurePosixPath(*path_parts[1:]))

    def __getitem__(self, path: TreeNodePath):
        return self.node_from_path(path).value

    def __setitem__(self, path: TreeNodePath, value):
        node = self.node_from_path(path)
        node.value = value

    def __iter__(self):
        yield from self.children

    def __len__(self):
        return len(self.children)

    def items(self):
        yield from self.children.items()

    def path_values(self):
        for _, node in self.children:
            yield node.path, node.weight

    def path_nodes(self):
        for _, node in self.children:
            yield node.path, node

    # It finally happened, this function ran without bugs on the first try.
    # This only means one thing. Programming karma will be striking back soon.
    #
    # Never mind, found two bugs already. Three. Four.
    # Seventh try's the charm.
    def object_hierarchy(self):
        # If all the child nodes have integers as keys and all of these keys are
        # positive ranging from 0 to the largest key value without gaps, then
        # it's a list.

        # If this node has children (and is not a list) then it's a dictionary

        # Otherwise this node is an endpoint node and contains a value.

        if len(self):
            # The node has children, so it's a container (either list or dictionary).
            key_set = set(self)
            if all((isinstance(key, int) for key in key_set)):
                # All children have integer keys, so it might be a list.
                # Check if all the keys range from 0 to max key value without gaps.
                if key_set == set(range(max(key_set) + 1)):
                    # It's a list, get the children's object_hierarchy() value, make sure
                    # they are put in the list in the correct order
                    return list((self.children[i].object_hierarchy() for i in range(max(key_set) + 1)))

            # If this code was reached, it's a dictionary.
            # Create a dictionary with the values being the return value of the
            # children's object_hierarchy() function.
            return dict(((key, child.object_hierarchy()) for key, child in self.items()))
        else:
            # The node has no children, so return the value.
            return self.value

    def __str__(self):
        return f"<TreeNode path='{self.path}' key='{self.key}' value='{self.value}'>"


from plone.app.testing import PloneSandboxLayer
from plone.app.testing import applyProfile
from plone.app.testing import PLONE_FIXTURE
from plone.app.testing import IntegrationTesting
from plone.app.testing import FunctionalTesting

from plone.testing import z2

from zope.configuration import xmlconfig

class VnccollabPortalLayer(PloneSandboxLayer):

    defaultBases = (PLONE_FIXTURE,)

    def setUpZope(self, app, configurationContext):
        # Load ZCML
        import vnccollab.portal
        xmlconfig.file(
            'configure.zcml',
            vnccollab.portal,
            context=configurationContext
        )

        # Install products that use an old-style initialize() function
        #z2.installProduct(app, 'Products.PloneFormGen')

#    def tearDownZope(self, app):
#        # Uninstall products installed above
#        z2.uninstallProduct(app, 'Products.PloneFormGen')

    def setUpPloneSite(self, portal):
        applyProfile(portal, 'vnccollab.portal:default')

VNCCOLLAB_PORTAL_FIXTURE = VnccollabPortalLayer()

VNCCOLLAB_PORTAL_INTEGRATION_TESTING = IntegrationTesting(
    bases=(VNCCOLLAB_PORTAL_FIXTURE,),
    name="VnccollabPortalLayer:Integration"
)
VNCCOLLAB_PORTAL_FUNCTIONAL_TESTING = FunctionalTesting(
    bases=(VNCCOLLAB_PORTAL_FIXTURE, z2.ZSERVER_FIXTURE),
    name="VnccollabPortalLayer:Functional"
)


import csv

import openpyxl
import requests

def main():
    data = requests.get('https://www.edraak.org/api/marketing/explore/').json()
    rows = []
    for category, courses in data['courses_by_category'].items():
        for course in courses:
            row = {
                "CourseEN": course['name_en'],
                "CourseAR": course['name_ar'],
                "URL": course['course_url'],
                # "Cost": data['data']['master_course_profile']['master_course_bar_info']['price'],
                "DetailsEN": course['description_en'],
                "DetailsAR": course['description_ar'],
                "Language": course['language'],
                "Platform": "Edraak",
                "Logo": course['cover_image'],
                "Filter": f"{category} > {course['name_en']}".title(),
            }
            rows.append(row)
    for course in data['specializations']:
        row = {
            "CourseEN": course['name_en'],
            "CourseAR": course['name_ar'],
            "URL": f"https://www.edraak.org/{course['lms_url']}",
            # "Cost": data['data']['master_course_profile']['master_course_bar_info']['price'],
            "DetailsEN": course['description_en'],
            "DetailsAR": course['description_ar'],
            "Language": course['language'],
            "Platform": "Edraak",
            "Logo": course['cover_image'],
            "Filter": f"Specialization > {course['name_en']}".title(),
        }
        rows.append(row)
    with open('edraak.csv', 'w',encoding=encoding,newline='') as f:
        c = csv.DictWriter(f, fieldnames=rows[0].keys())
        c.writeheader()
        c.writerows(rows)
    convert('edraak.csv')

encoding = 'utf-8'
def convert(filename):
    wb = openpyxl.Workbook()
    ws = wb.active
    with open(filename, encoding=encoding) as f:
        reader = csv.reader(f, delimiter=',')
        for row in reader:
            ws.append(row)
    wb.save(filename.replace("csv", "xlsx"))

if __name__ == '__main__':
    main()


# Import librraries
import numpy as np
import tensorflow as tf
from tensorflow import keras
from matplotlib import pyplot as plt
import os
import cv2
from pathlib import Path

#Function to Get Images
def getImages(data_dir):
    # Set the image size
    img_size_val = (32, 32)
    # Create empty lists for the images and labels
    images_val = []
    labels_val = []
    # Loop over each image in the folder
    for file in os.listdir(data_dir):
        file_dir = os.path.join(data_dir, file)
        if file_dir.endswith(('.tiff','.bmp')):
            # Load the image and resize it to the desired size
            img = cv2.imread(file_dir, cv2.IMREAD_GRAYSCALE)
            img = cv2.resize(img, img_size_val)
            # Append the image and label to the lists
            images_val.append(img)
            labels_val.append(label)
    # Convert the lists to NumPy arrays
    images_val = np.array(images_val)
    labels_val = np.array(labels_val)
    return images_val, labels_val

# Set the path to the folder containing the 'train' and 'val' folders
train_dir = Path('./charts/train_val')
val_dir = Path('./charts/train_val')
test_dir = Path('./charts/test')
# Load the train and validation images and labels
X_train, y_train = getImages(train_dir)
X_val, y_val = getImages(val_dir)
X_test, y_test = getImages(test_dir)

#creating model
model = keras.Sequential([

    keras.layers.Conv2D(30, (3,3), activation='relu', input_shape=(28, 28, 1)),
    keras.layers.MaxPooling2D((2,2)),

    keras.layers.Flatten(),
    keras.layers.Dense(100, activation='relu'),
    keras.layers.Dense(10, activation='sigmoid')
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

test_loss, test_acc = model.evaluate(X_test, steps=len(y_test))

print('Test accuracy:', test_acc)
print('Test loss:', test_loss)


#!/usr/bin/python
# coding=utf-8

import numpy as np
from pyspark import SparkContext, SparkConf

def intToBase(n):
        if n==1: return "A"
        if n==2: return "C"
        if n==3: return "G"
        return "T"

def generateString(length):
        ints=np.random.random_integers(1,4,length)
        result="".join(list(map(lambda x: intToBase(x),ints)))
        return result

if __name__ == "__main__":
	conf = SparkConf().setAppName("ACGTGenerator").setMaster("local[8]")
	sc=SparkContext.getOrCreate(conf=conf)

	dataset=sc.parallelize(range(40)).map(lambda x:generateString(1000))
	dataset.saveAsTextFile("dataset.txt")

	sc.stop()


import torch.utils.data as data
import torchvision.transforms as transforms

import os
import random
import torch

from PIL import Image

from utils import  make_dataset

class BaseDataset(data.Dataset):
    def __init__(self):
        super(BaseDataset, self).__init__()

    def name(self):
        return 'BaseDataset'

    def initialize(self, opt):
        pass

class PairDataset(BaseDataset):
    def initialize(self, opt):
        self.opt = opt
        self.root = opt.dataroot
        self.dir_A = os.path.join(opt.dataroot, opt.phase + 'A')
        self.dir_B = os.path.join(opt.dataroot, opt.phase + 'B')

        self.A_paths = make_dataset(self.dir_A)
        self.B_paths = make_dataset(self.dir_B)

        self.A_paths = sorted(self.A_paths)
        self.B_paths = sorted(self.B_paths)
        self.A_size = len(self.A_paths)
        self.B_size = len(self.B_paths)

        transform_list = []

        transform_list += [transforms.ToTensor(),
                           transforms.Normalize((0.5, 0.5, 0.5),
                                                (0.5, 0.5, 0.5))]
        # transform_list = [transforms.ToTensor()]

        self.transform = transforms.Compose(transform_list)
        # self.transform = get_transform(opt)

    def __getitem__(self, index): # 这个方法如何，何时调用？
        A_path = self.A_paths[index % self.A_size]
        B_path = self.B_paths[index % self.B_size]

        A_img = Image.open(A_path).convert('RGB')
        B_img = Image.open(A_path.replace("low", "normal").replace("A", "B")).convert('RGB')

        A_img = self.transform(A_img)
        B_img = self.transform(B_img)

        w = A_img.size(2)
        h = A_img.size(1)
        # w_offset = random.randint(0, max(0, w - self.opt.fineSize - 1))
        # h_offset = random.randint(0, max(0, h - self.opt.fineSize - 1))

        # A_img = A_img[:, h_offset:h_offset + self.opt.fineSize,
        #         w_offset:w_offset + self.opt.fineSize]
        # B_img = B_img[:, h_offset:h_offset + self.opt.fineSize,
        #         w_offset:w_offset + self.opt.fineSize]

        if self.opt.resize_or_crop == 'no':
            r, g, b = A_img[0] + 1, A_img[1] + 1, A_img[2] + 1
            A_gray = 1. - (0.299 * r + 0.587 * g + 0.114 * b) / 2.
            A_gray = torch.unsqueeze(A_gray, 0)
            input_img = A_img
            # A_gray = (1./A_gray)/255.
        else:

            # A_gray = (1./A_gray)/255.
            if (not self.opt.no_flip) and random.random() < 0.5:
                idx = [i for i in range(A_img.size(2) - 1, -1, -1)]
                idx = torch.LongTensor(idx)
                A_img = A_img.index_select(2, idx)
                B_img = B_img.index_select(2, idx)
            if (not self.opt.no_flip) and random.random() < 0.5:
                idx = [i for i in range(A_img.size(1) - 1, -1, -1)]
                idx = torch.LongTensor(idx)
                A_img = A_img.index_select(1, idx)
                B_img = B_img.index_select(1, idx)
            if (not self.opt.no_flip) and random.random() < 0.5:
                times = random.randint(self.opt.low_times, self.opt.high_times) / 100.
                input_img = (A_img + 1) / 2. / times
                input_img = input_img * 2 - 1
            else:
                input_img = A_img
            r, g, b = input_img[0] + 1, input_img[1] + 1, input_img[2] + 1
            A_gray = 1. - (0.299 * r + 0.587 * g + 0.114 * b) / 2.
            A_gray = torch.unsqueeze(A_gray, 0)
        return {'A': A_img, 'B': B_img, 'A_gray': A_gray, 'input_img': input_img,
                'A_paths': A_path, 'B_paths': B_path}

    def __len__(self):
        return self.A_size

    def name(self):
        return 'PairDataset'

class Structure():
    def __init__(self, opt):
        self.__dict__.update(opt)

def Get_paired_dataset(batch_size):
    data = PairDataset()
    opt = {"dataroot": "../final_dataset", "phase": "train", "resize_or_crop": False, "isTrain": True, "no_flip": True,
           "vary": 1, "lighten": True, "fineSize": 400}
    opt = Structure(opt)
    data.initialize(opt)
    dataloader = torch.utils.data.DataLoader(data, batch_size=batch_size)
    return dataloader


from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
import pandas as pd
import pickle
#import matplotlib.pyplot as plt
import numpy as np

def print_scores(y_true, y_hat, method_list):
    methods = [accuracy_score, recall_score, precision_score]
    for method in methods:
        score = method(y_test, y_hat)
        print('test {} = {:0.3f}'.format(method.__name__, score))

if __name__=='__main__':
    # load data
    df = pickle.load( open( 'pkl/aspen_d2_imputemean_alldays.p', 'rb'))
    df.drop('N_AVY', axis=1, inplace=True)
    # fill na with zero in case any not imputed
    df.fillna(0, inplace=True)

    ''' case : slab or wet '''
    cases = [['SLAB','WET'], ['WET','SLAB']]

    ''' run case   '''
    case = cases[1] # chose case here, 0 or 1
    data_df = df.copy() # copy to read all columns after dropping

    # drop other binary and probability column
    c_drop = [c for c in list(df.columns) if case[1] in c]
    data_df.drop(c_drop, axis=1, inplace=True)

    # train test split in time
    splitdate = pd.to_datetime('2016-06-01')
    train_df = data_df[data_df.index <= splitdate]
    test_df = data_df[data_df.index > splitdate]

    # oversample train data
    #train_shuffle, counts, factors = oversample(train_df, 'AVY', n=n)
    #print('oversample to n = {}'.format(n))
    #pickle.dump( oversamp_df, open( "pkl/aspen_oversamp6.p", "wb" ) )
    train_shuffle = train_df

    ''' select features and target X,y '''
    # train set
    X_train = train_shuffle.copy()
    y_train = X_train.pop(case[0])
    # test set
    X_test = test_df.copy()
    y_test = X_test.pop(case[0])
    # datetime for plot
    test_datetime = pd.to_datetime(X_test.index)

    # train model

    param_grid = {
        'loss':['deviance'],
        'learning_rate':[0.025,0.05,0.075],
        'n_estimators': [500, 600, 700],
        'criterion': ['friedman_mse'],
        'max_features': ['log2'],
        'min_samples_split': [4, 5, 6],
        'min_samples_leaf': [4, 5, 6],
        'subsample': [0.2, 0.3, 0.4, 0.5, 0.6],
        'verbose': [1]
        }

    est = GradientBoostingClassifier()

    grid = GridSearchCV(est, param_grid, scoring='recall')
    grid.fit(X_train, y_train)

    y_hat = grid.predict(X_test)

    print('case: {}'.format(case[0]))

    score_a = accuracy_score(y_test, y_hat)
    print('test accuracy = {:0.3f}'.format(score_a))

    score_r = recall_score(y_test, y_hat)
    print('test recall = {:0.3f}'.format(score_r))

    print(grid.best_params_)

    best_est = grid.best_estimator_

    pickle.dump(best_est, open("best-ests/best_est_gbc_{}.p".format(case[0]), "wb"))


from django.core.exceptions import ValidationError
from django.db import models
from django_countries import Countries
from django_countries.fields import CountryField

from customer.models import Customer
from property.models import Property

class Name(models.Model):
    first_name = models.CharField(
        max_length=20,
        help_text="First Name"
    )
    mid_name = models.CharField(
        max_length=10,
        null=True,
        blank=True,
        help_text="Mid Name"
    )
    last_name = models.CharField(
        max_length=15,
        help_text="Last Name"
    )

class EUCountries(Countries):
    only = [
        'AT', 'BE', 'BG', 'CY', 'CZ', 'DK', 'EE', 'ES', 'FI', 'GR', 'HR',
        'FR', 'DE', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'MT', 'NL', 'PT',
        'RO', 'SE', 'SI', 'SK'
    ]

class Address(models.Model):
    property = models.OneToOneField(
        Property, related_name="address", help_text="Property",
        on_delete=models.CASCADE, null=True)
    customer = models.OneToOneField(
        Customer, related_name="address", help_text="Customer",
        on_delete=models.CASCADE, null=True)
    street = models.CharField(help_text="Street", max_length=128)
    city = models.CharField(
        help_text="City", max_length=15, default="Eindhoven")
    country = CountryField(countries=EUCountries,
                           help_text="Countries in EU Union", default='NL')
    zip_code = models.CharField(
        help_text="Postal code", max_length=12, default="5611KT")

    class Meta:
        verbose_name = 'Address'

class Image(models.Model):
    property = models.ForeignKey(
        Property, related_name="Image", help_text="Property Images",
        on_delete=models.CASCADE, null=True)

    url = models.URLField("url", max_length=200)
    alt = models.CharField("alt", max_length=20)

    class Meta:
        verbose_name = 'Property Image'

    def clean(self):
        total_images = Image.objects.filter(property=self.property).count()
        if(total_images > 10):
            raise ValidationError(
                "The total number of images for a property cannot exceed 10")


import sys
sys.path.append('.')  # to run from the project root dir

import argparse
import pickle
import torch
import numpy as np
from scipy import linalg
from tqdm import tqdm
import models

def calc_fid(sample_mean, sample_cov, real_mean, real_cov, eps=1e-6):
    cov_sqrt, _ = linalg.sqrtm(sample_cov @ real_cov, disp=False)

    if not np.isfinite(cov_sqrt).all():
        print('product of cov matrices is singular')
        offset = np.eye(sample_cov.shape[0]) * eps
        cov_sqrt = linalg.sqrtm((sample_cov + offset) @ (real_cov + offset))

    if np.iscomplexobj(cov_sqrt):
        if not np.allclose(np.diagonal(cov_sqrt).imag, 0, atol=1e-3):
            m = np.max(np.abs(cov_sqrt.imag))

            raise ValueError(f'Imaginary component {m}')

        cov_sqrt = cov_sqrt.real

    mean_diff = sample_mean - real_mean
    mean_norm = mean_diff @ mean_diff

    trace = np.trace(sample_cov) + np.trace(real_cov) - 2 * np.trace(cov_sqrt)

    fid = mean_norm + trace

    return fid

def extract_feature_from_samples():
    import math
    n_batch = math.ceil(args.n_sample * 1. / args.batch_size / hvd.size())
    features = None

    with torch.no_grad():
        for _ in tqdm(range(n_batch), disable=hvd.rank() != 0):
            latent = torch.randn(args.batch_size, 1, 512, device=device)
            img, _ = generator(latent)
            img = img.clamp(min=-1., max=1.)
            feat = inception(img)[0].view(img.shape[0], -1)  # the img will be automatically resized
            if features is None:
                features = feat.to('cpu')
            else:
                features = torch.cat((features, feat.to('cpu')), dim=0)

    return features

def compute_fid():
    pass

if __name__ == '__main__':
    import horovod.torch as hvd

    device = 'cuda'

    parser = argparse.ArgumentParser()

    parser.add_argument("--config", type=str, help='config name of the pretrained generator')
    parser.add_argument('--batch_size', type=int, default=16)
    parser.add_argument('--n_sample', type=int, default=50000)
    parser.add_argument('--inception', type=str, default=None, required=True)

    parser.add_argument('--channel_ratio', type=float, default=None)
    parser.add_argument('--target_res', type=int, default=None)

    args = parser.parse_args()

    hvd.init()
    torch.cuda.set_device(hvd.local_rank())

    generator = models.get_pretrained('generator', args.config).to(device)
    generator.eval()

    # set sub-generator
    if args.channel_ratio:
        from models.dynamic_channel import set_uniform_channel_ratio, CHANNEL_CONFIGS

        assert args.channel_ratio in CHANNEL_CONFIGS
        set_uniform_channel_ratio(generator, args.channel_ratio)

    if args.target_res is not None:
        generator.target_res = args.target_res

    # compute the flops of the generator (is possible)
    if hvd.rank() == 0:
        try:
            from torchprofile import profile_macs

            macs = profile_macs(generator, torch.rand(1, 1, 512).to(device))
            params = sum([p.numel() for p in generator.parameters()])
            print(' * MACs: {:.2f}G, Params: {:.2f}M'.format(macs / 1e9, params / 1e6))
        except:
            print(' * Profiling failed. Passed.')

    inception = models.get_pretrained('inception').to(device)
    inception.eval()

    inception_features = extract_feature_from_samples()
    # now perform all gather
    inception_features = hvd.allgather(inception_features, name='inception_features').numpy()[:args.n_sample]

    if hvd.rank() == 0:
        print(f'extracted {inception_features.shape[0]} features')

    if hvd.rank() == 0:
        sample_mean = np.mean(inception_features, 0)
        sample_cov = np.cov(inception_features, rowvar=False)

        with open(args.inception, 'rb') as f:
            embeds = pickle.load(f)
            real_mean = embeds['mean']
            real_cov = embeds['cov']

        fid = calc_fid(sample_mean, sample_cov, real_mean, real_cov)
        print(args.inception)
        print('fid:', fid)


#pylint:disable=no-member

import cv2 as cv

def rescaleFrame(frame, scale = 0.7): # this function works for images , video and live video
    width = int(frame.shape[1]*scale)
    height = int(frame.shape[0]*scale)

    dimensions = (width,height)

    return cv.resize(frame, dimensions, interpolation=cv.INTER_AREA)

# # Reading Images

# img = cv.imread('C:/Users/ayonb/VS_Code_practice/opencv-course-master/Resources/Photos/cat_large.jpg')
# resized_img = rescaleFrame(img)
# cv.imshow('Resized Cat', resized_img)

# cv.waitKey(0)

# # Reading Videos
# def changeRes(weight,height): # this function works for live video only
#     capture.set(3,width)
#     capture.set(4,height)

capture = cv.VideoCapture('opencv-course-master\Resources\Videos\dog.mp4')

while True:
    isTrue, frame = capture.read()

    frame_resized = rescaleFrame(frame, scale=.2)

    cv.imshow('Video', frame)
    cv.imshow('Resized Video', frame_resized)

    if cv.waitKey(20) & 0xFF==ord('d'):
        break

capture.release()
cv.destroyAllWindows()


import numpy as np
from collections import defaultdict
from .searcher import btree
DEFAULT_SEARCHER = btree.Searcher()

class RidgeDitect(object):
    def __init__(self, searcher=DEFAULT_SEARCHER):
        self._searcher = searcher
    def init(self):
        self.graph = {}
    def fit(self, vectors, sample=4):

        nearbys = self._searcher.search(vectors, sample)

        nearby_vectors = vectors[nearbys] - np.expand_dims(vectors, axis=1)

        nearby_penalties = np.sum(nearby_vectors ** 2, 2)

        nearby_lengths = nearby_penalties ** 0.5

        guide_vectors = np.einsum('ijk,ij->ik', nearby_vectors, 1 / nearby_penalties)
        scores = np.einsum('ijk,ik -> ij', nearby_vectors, guide_vectors) / (nearby_lengths * np.linalg.norm(guide_vectors, 2))
        sorted_args = np.argsort(scores, axis=1)[:, ::-1]
        sorted_nearbys = np.take_along_axis(nearbys, sorted_args, axis=1)
        connectable_map = dict(zip(*np.unique(sorted_nearbys[:,0], return_counts=True)))
        masks = self._get_masks(sorted_nearbys, scores, sorted_args, nearby_vectors, vectors, sample)

        self.graph = {ind:nearby[mask][0:min(connectable_map.get(ind, 2), sample, np.count_nonzero(mask))] for ind, mask, nearby in zip(range(vectors.shape[0]), masks, sorted_nearbys)}

        vectors_length = vectors.shape[0]

        reverse_nodes = defaultdict(dict)
        for node, towords in self.graph.items():
            for toword in towords:
                reverse_nodes[toword][node] = True

        checked = {}
        clusters = {}
        cluster_number = 0
        cluster ={}
        for node in range(vectors_length):
            if node in checked:

                continue

            cluster[node] = True
            is_next_exist = True
            targets = {node:True}

            while is_next_exist:
                next_targets = {}
                is_next_exist = False
                for target in targets:
                    cluster[target] = True
                    checked[target] = True
                    canditates_list =  [self.graph.get(target, {}), reverse_nodes.get(target, {})]

                    for  canditates in canditates_list:
                        for canditate in canditates:
                            if canditate in checked:
                                continue
                            next_targets[canditate] = True
                            is_next_exist = True
                targets = next_targets
            self._process_cluster(cluster, clusters, cluster_number)
            cluster_number += 1
            cluster = {}

        self.clusters = clusters

    def _process_cluster(self, cluster, clusters, clusters_number):
        clusters[clusters_number] = np.array(list(cluster))
    def _get_masks(self, sorted_nearbys, scores, sorted_args, samples, nearby_vectors, vectors):
        return np.take_along_axis(scores, sorted_args, axis=1) >= 0


#!/bin/python3

import io
import os
import json
import pprint
import argparse
import numpy as np
from pyquaternion import Quaternion

from builders.simple_builder import SimpleBuilder
from towers.empty_tower import EmptyTower
from blocks.simple_block import SimpleBlock
from scenes.block_scene import BlockScene
from utils.json_encoders import TowerEncoder

# rotations = [Quaternion(axis=[1, 0, 0], angle = np.pi/2),
#              Quaternion(axis=[0, 1, 0], angle = np.pi/2),
#              Quaternion(axis=[0, 0, 1], angle = np.pi/2),
#              Quaternion(axis=[1, 0, 0], angle = 0)]

rotations = [Quaternion(axis=[1, 0, 0], angle = 0),]
             # Quaternion(axis=[0, 1, 0], angle = np.pi/2)]

def main():
    parser = argparse.ArgumentParser(description = ('Tests `SimpleBuilder` '+\
                                                   'and associated classes'))

    parser.add_argument('out', type = str, help = 'Path to save render.')
    parser.add_argument('--number', type = int, default = 2,
                        help = 'Number of blocks.')
    parser.add_argument('--height', type = float, default = 10,
                        help = 'Height of the tower.')

    args = parser.parse_args()

    print('creating starting tower')
    base_dims = [2, 2]
    empty_tower = EmptyTower(base_dims)
    print('creating block for building')
    block_dims = [2, 1, 1]
    block = SimpleBlock(block_dims)

    max_blocks = args.number
    max_height = args.height
    builder = SimpleBuilder(max_blocks, max_height)

    new_tower = builder(empty_tower, block)

    tower_json = repr(new_tower)

    pprint.pprint(json.loads(repr(new_tower)))
    with open(os.path.join(args.out, 'test.json'), 'w') as f:
        json.dump(json.loads(tower_json), f, indent = 4, sort_keys = True)

    scene = BlockScene(tower_json)
    img_out = os.path.join(args.out, 'test_render')
    scene.render(img_out, [1])
    scene_out = os.path.join(args.out, 'test_scene.blend')
    scene.save(scene_out)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
https://leetcode.com/problems/coloring-a-border/

The key is to find the border, use BFS.

Time complexity: O(mn)
"""
import collections
class Solution:
    def colorBorder(self, grid, r0, c0, color):
        m, n = len(grid), len(grid[0])
        ref_color = grid[r0][c0]
        border = []
        queue = collections.deque([(r0, c0)])
        visited = {(r0, c0)}
        while queue:
            x, y = queue.popleft()
            nei = []
            for dx, dy in [(1, 0), (0, 1), (-1, 0), (0, -1)]:
                new_x, new_y = x + dx, y + dy
                if 0 <= new_x < m and 0 <= new_y < n:
                    nei.append((new_x, new_y))
            cnt = 0
            for a, b in nei:
                if grid[a][b] == ref_color: # same component
                    cnt += 1
                    if (a, b) not in visited:
                        visited.add((a, b))
                        queue.append((a, b))

            if cnt != 4: # (x, y) is border
                border.append((x, y))

        for x, y in border:
            grid[x][y] = color
        return grid


from django.urls import path

from . import views

app_name = 'olive'

urlpatterns = [
    path('', views.IndexView.as_view(), name='index'),
    path('shop/', views.ShopView.as_view(), name='shop'),
    # path('shop/<category>', views.ShopView.as_view(), name='shop_category'),
    path('<int:olive_id>/', views.DetailView.as_view(), name='detail'),
    path('order/<int:olive_id>/', views.OrderView.as_view(), name='order'),
    path('checkout/<int:olive_id>/', views.CustomerFormView.as_view(), name='checkout'),

]


import requests
import folium
import json
from graphqlclient import GraphQLClient
from .serviceAPIDistance import callDistance

clientBorne = GraphQLClient(endpoint="https://api.placetoplug.com/go/graphql")
queryBorne = """query Query ($query: FindChargingZone!) {
                    findChargingZones(query: $query) {
                        chargingZones {
                        id
                        address {
                            street
                            city
                            country
                        }
                        status
                        coordinates
                        vehicleTypes
                        isFastCharge
                        plugTypes
                        }
                    }
                }"""

def callAPIBorne(trajet, my_map):
    # Call API to find the city
    # callVille1 = requests.get("https://api.openrouteservice.org/geocode/search?api_key=5b3ce3597851110001cf6248ae8b2b8cd4f943509643db9544603784&text="+ ville1 +"&boundary.country=FR")
    # callVille2 = requests.get("https://api.openrouteservice.org/geocode/search?api_key=5b3ce3597851110001cf6248ae8b2b8cd4f943509643db9544603784&text="+ ville2 +"&boundary.country=FR")

    # depart = callVille1.json()['features'][0]['geometry']['coordinates']
    # dest = callVille2.json()['features'][0]['geometry']['coordinates']
    res = []
    i=1

    for t in trajet:
        print(f"{i}/{trajet.__len__()}")
        i+=1
        variablesBorne = {
            "query": {
                "limit": 1000,
                "filters": {
                "bounds": {
                    "minLat": t[1] - .5,
                    "minLon": t[0] - .5,
                    "maxLat": t[1] + .5,
                    "maxLon": t[0] + .5
                },
                "isPublic": True
                }
            }
        }

        # Call API to find the bornes
        dataBorne = json.loads(clientBorne.execute(query=queryBorne, variables=variablesBorne))

        if dataBorne["data"]["findChargingZones"]["chargingZones"].__len__() == 0:
            print("No charging station found")
            res.append(None)
            continue
        else:
            bornePlusProche = dataBorne["data"]["findChargingZones"]["chargingZones"][0]["coordinates"]

        dmin = callDistance([t, bornePlusProche])

        for b in dataBorne["data"]["findChargingZones"]["chargingZones"][1:]:
            tmp = callDistance([t, b["coordinates"]])
            if tmp < dmin:
                bornePlusProche = b["coordinates"]
                dmin = tmp

        res.append(bornePlusProche)

    return res
    # FinalBorne = dataBorne["data"]["findChargingZones"]["chargingZones"][0]
    # for b in dataBorne["data"]["findChargingZones"]["chargingZones"]:
    #     if callDistance(temp, b["coordinates"]) < callDistance(temp, FinalBorne["coordinates"]):
    #         FinalBorne = b

    # temp = FinalBorne["coordinates"]


import psycopg2

class DbModel(object):
    """ DbModel
        Takes care of storing and loading records
        to PostgreSQL database
    """
    def __init__(self, configuration):
        self.config = configuration
        self.connection = None
        self._prepare_connection()
        self._create_table()

    def __del__(self):
        if self.connection:
            self.connection.close()
            self.connection = None

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.connection.close()
        self.connection = None

    def _prepare_connection(self):
        """ Creates a new connection if it was not
            created yet or was closed
        """
        try:
            if self.connection is None or self.connection.closed:
                self.connection = psycopg2.connect(**self.config)
            print("Database connected successfully........")
        except (Exception, psycopg2.DatabaseError) as error:
            print(error)

    def _create_table(self):
        """ Creates a new table if table doesn't exists
        """
        try:
            # Creating a cursor object using the cursor() method
            cursor = self.connection.cursor()

            # Creating table as per requirement
            sql = '''CREATE TABLE IF NOT EXISTS ETL(
                key VARCHAR(255) PRIMARY KEY,
                value VARCHAR(255) NOT NULL,
                timestamp VARCHAR(255) NOT NULL
            )'''
            cursor.execute(sql)
            print("Table created successfully........")
        except (Exception, psycopg2.DatabaseError) as error:
            print(error)

    def insert_data(self, data):
        """ Inserts a record in the database
        """
        try:
            # Creating a cursor object using the cursor() method
            cursor = self.connection.cursor()

            # Preparing SQL queries to INSERT a record into the database.
            cursor.execute('''INSERT INTO ETL(key, value, timestamp)
            VALUES (%s,%s,%s)''', (data['key'], data['value'], data['ts'],))
            # Commit changes in the database
            self.connection.commit()
            print("Record inserted........")
        except (Exception, psycopg2.DatabaseError) as error:
            print(error)


import png

def load_glyph_data_from_png(file_path: str) -> tuple[list[list[int]], int, int]:
    width, height, bitmap, _ = png.Reader(filename=file_path).read()
    data = []
    for bitmap_row in bitmap:
        data_row = []
        for x in range(0, width * 4, 4):
            alpha = bitmap_row[x + 3]
            if alpha > 127:
                data_row.append(1)
            else:
                data_row.append(0)
        data.append(data_row)
    return data, width, height

def save_glyph_data_to_png(data: list[list[int]], file_path: str):
    bitmap = []
    for data_row in data:
        bitmap_row = []
        for x in data_row:
            bitmap_row.append(0)
            bitmap_row.append(0)
            bitmap_row.append(0)
            if x == 0:
                bitmap_row.append(0)
            else:
                bitmap_row.append(255)
        bitmap.append(bitmap_row)
    png.from_array(bitmap, 'RGBA').save(file_path)


import argparse
from pathlib import Path

from tqdm import tqdm

from processor.asr.yitu_asr_processor import yitu_asr_wrapper
from util.log_util import getLogger
from util.redis_util import getRedisClient
from util.util import mapLineCount

logger = getLogger('read_selenium_output')

from postprocess.line_processors import *

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--input', required=True)
    parser.add_argument('--pattern')
    parser.add_argument('--postfix')

    args = parser.parse_args()

    input_path = Path(args.input)

    assert input_path.exists()

    if input_path.is_dir():
        assert args.pattern is not None
        logger.info(f'find files in {input_path}/{args.pattern}')
        files = [file for file in input_path.rglob(args.pattern)]
    else:
        files = [input_path]

    output_path = input_path.parent.joinpath(f'{input_path.stem}{args.postfix}')
    fw = output_path.open('w')
    logger.info(f'write file into {output_path}')

    if len(files) == 0:
        logger.warning(f'no file found, exit')
        exit(1)

    redis_cli = getRedisClient(db=0)

    for file in files:
        total_line_num = mapLineCount(str(file))
        fw = file.parent.joinpath(f'{file.name}_{args.postfix}').open('w')

        with file.open('rb') as fr:
            for lineno, line in tqdm(enumerate(fr, start=1), total=total_line_num):

                line = line.decode('utf-8', errors='ignore')
                line = line.strip()
                if line == '':
                    continue

                line_key = f"{file}:{lineno}"

                redis_line = redis_cli.get(line_key)
                if redis_line is not None:
                    continue

                res = yitu_asr_wrapper(line, line_key)

                fw.write(res + '\n')


import xml.etree.ElementTree as ET

class Parser:
    def __init__(self, name):
        self.tree = ET.parse(name)
        self.iter_ = self.tree.iter()
        self.functions = {}
        self.bufer = {}

    def parse_childs(self,data):

        if data.getchildren():
            self.bufer[data.attrib['count']] = []
            self.parse_childs(self, data.getchildren())
        else:
            if data.tag=='integer':
                val=0
            elif data.tag=='string':
                val = ''
            elif data.tag=='double':
                val = 1.1
            if data.attrib['name'] in self.bufer:
                self.bufer[data.attrib['name']]=val

    def parse_it(self):
        for elem in self.iter_:
            inp={}
            out={}
            #print(elem.attrib)
            for fn in elem.getchildren():
                if fn.tag=='input':
                    self.bufer={}
                    #print(fn.getchildren())
                if fn.tag == 'output':
                    break
            if 'name' in elem.keys() and 'rpcf_' in elem.attrib['name']:
                self.functions[elem.attrib['name']] = {'id': elem.attrib['id'], 'input': inp,'output': out}
        #print(self.functions)

    def runner(self, attr, *args, **kwargs):
        '''метод, который будет имитировать все прочие методы'''
        function = attr
        if function in self.functions.keys():
            print(function, *args, **kwargs)
        else:
            print('No function found')

    def __getattr__(self,attr):
        def wrap(*args,**kwargs):
            return self.runner(attr,*args,**kwargs)
        return wrap

obj = Parser('api_53-004.xml')
obj.parse_it()
obj.rpcf_get_stats({'type':1235})


#!/usr/bin/python
## coding:utf-8

#import sys
#sys.path.append("..")
#import main
#
#print("main.sqrt(72) is %f"%main.square_root(72))

import logging, mymodule

logging.basicConfig()
logger = logging.getLogger("MyApp")

logger.setLevel(logging.DEBUG)
logger.info("Start my app")
logger = logging.getLogger("Another App")
logger.info("another.. my app")
logger.info("Start my app")
logger = logging.getLogger("MyApp")
logger.info("another.. my app")
logger.info("Start my app")

logger = logging.getLogger("Another App")

try:
    mymodule.doIt()
except Exception as e:
    logger.exception("There was a problem")

logger.info("Ending my app")


"""empty message

Revision ID: 1a39e327a561
Revises: b6f61b196b53
Create Date: 2019-05-07 12:37:20.846345

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1a39e327a561'
down_revision = 'b6f61b196b53'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('image', sa.Column('file', sa.String(length=100), nullable=True))
    op.add_column('image', sa.Column('uploaded_by', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'image', 'user', ['uploaded_by'], ['id'])
    op.drop_column('image', 'owner_id')
    op.drop_column('image', 'path')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('image', sa.Column('path', sa.VARCHAR(length=255), nullable=True))
    op.add_column('image', sa.Column('owner_id', sa.INTEGER(), nullable=True))
    op.drop_constraint(None, 'image', type_='foreignkey')
    op.drop_column('image', 'uploaded_by')
    op.drop_column('image', 'file')
    # ### end Alembic commands ###


# -*- coding: utf-8 -*-
"""Tests for the old block devices."""

import os
from pytest import mark

from raspi_maker.device import get_devices
from raspi_maker.device import _get_blk_size_bytes
from raspi_maker.device import _get_hr_size
from raspi_maker.device import _get_lsblk_output
from raspi_maker.device import _get_model
from raspi_maker.device import _get_vendor
from raspi_maker.device import _partition_details
from raspi_maker.device import _device_partitions
from raspi_maker.device import _get_partitions
from raspi_maker.device import Device

@mark.unit
def test__get_blk_size_bytes_test_inputs():
    assert 16008609792 == _get_blk_size_bytes('sdb', '31266816\n'), (
        'Block size is off on test string.')

@mark.unit
def test__get_hr_size():
    test_cases = [
        ('0', '0B'),
        ('1', '1B'),
        ('-1', '1B'),
        ('1000', '1000B'),
        ('100000', '97KB'),
        ('-10000', '9KB'),
        ('100000000', '95MB'),
        ('1000000000000', '931GB'),
        ('100000000000000000', '88PB'),
        ('900000000000000000', '799PB'),
    ]
    for x in test_cases:
        size = _get_hr_size(x[0])
        assert type(size) == str, 'Got wrong type back :('
        assert size == x[1], 'Got bad answer back. Learn to math!'
        assert len(size) < 6, 'Didnt trim enough :('

@mark.unit
def test__partition_details():
    sample_input = '''Model: SanDisk Cruzer Fit (scsi)
Disk /dev/sdc: 31266816s
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Disk Flags:

Number  Start    End       Size      Type     File system  Flags
 1      8192s    137215s   129024s   primary  fat16        lba
 2      137216s  8538111s  8400896s  primary  ext4
'''
    test1 = {
        'Number'
        'Start'
        'End'
        'Size'
        'Type'
        'File system'
        'Flags'
    }
    assert '1' == _partition_details(sample_input, 1)['Number']
    assert '8192s' == _partition_details(sample_input, 1)['Start']
    assert '137215s' == _partition_details(sample_input, 1)['End']
    assert '129024s' == _partition_details(sample_input, 1)['Size']
    assert 'primary' == _partition_details(sample_input, 1)['Type']
    assert 'fat16' == _partition_details(sample_input, 1)['File system']
    assert 'lba' == _partition_details(sample_input, 1)['Flags']

    assert '2' == _partition_details(sample_input, 2)['Number']
    assert '137216s' == _partition_details(sample_input, 2)['Start']
    assert '8538111s' == _partition_details(sample_input, 2)['End']
    assert '8400896s' == _partition_details(sample_input, 2)['Size']
    assert 'primary' == _partition_details(sample_input, 2)['Type']
    assert 'ext4' == _partition_details(sample_input, 2)['File system']
    assert None == _partition_details(sample_input, 2)['Flags']

lsblk_example = '''NAME                    MAJ:MIN RM   SIZE RO TYPE  MOUNTPOINT
sda                       8:0    0 931.5G  0 disk
├─sda1                    8:1    0   512M  0 part  /boot/efi
├─sda2                    8:2    0   488M  0 part  /boot
└─sda3                    8:3    0 930.5G  0 part
  └─sda3_crypt          252:0    0 930.5G  0 crypt
    ├─ubuntu--vg-root   252:1    0 898.7G  0 lvm   /
    └─ubuntu--vg-swap_1 252:2    0  31.9G  0 lvm   [SWAP]
sdc                       8:32   1  14.9G  0 disk
├─sdc1                    8:33   1    63M  0 part
└─sdc2                    8:34   1     4G  0 part
sr0                      11:0    1  1024M  0 rom
mmcblk0                 179:0    0    29G  0 disk
└─mmcblk0p1             179:1    0    63M  0 part
'''

@mark.unit
def test__get_partitions_simple():
    assert _get_partitions(lsblk_example, 'sdc') == ['sdc1', 'sdc2']

@mark.unit
def test__get_partitions_full():
    assert _get_partitions(lsblk_example, 'sdc', full_paths=True) == ['/dev/sdc1', '/dev/sdc2']

@mark.unit
def test__get_partitions_fancy():
    assert _get_partitions(lsblk_example, 'mmcblk0', full_paths=True) == ['/dev/mmcblk0p1']

@mark.functional
def test_get_devices():
    assert get_devices(), 'Cant find any devices, maybe this is broke.'

@mark.functional
def test_get_blk_size_path_makes_sense():
    assert os.path.exists('/sys/block/sda/size'), (
        'I have made a terrible mistake about the filesystem, size file does not exist.')

@mark.functional
def test__get_blk_size_bytes_test_functions():
    assert 100 < _get_blk_size_bytes('sda'), 'SDA should really be bigger than 100 bytes.'

@mark.functional
def test__get_vendor_functional():
    assert _get_vendor('sda')

@mark.functional
def test__get_vendor_missing_functional():
    fake_device = 'natoheunstahoeunsth'
    assert fake_device == _get_vendor(fake_device)

@mark.functional
def test__get_model_functional():
    assert _get_model('sda')

@mark.functional
def test__get_model_missing_functional():
    fake_device = 'natoheunstahoeunsth'
    assert fake_device == _get_model(fake_device)

@mark.functional
def test_device_init():
    assert Device('sda')

@mark.functional
def test_device_partitions_property():
    assert Device('sda').partitions()

@mark.functional
def test__device_partitions():
    assert _device_partitions('sda')

@mark.functional
def test__get_lsblk_output():
    assert _get_lsblk_output()


from django.db import models
from stores.models import Store
from utils.image_upload_helper import upload_space_image_path
from utils.helpers import model_cleaner
from utils.helpers import autoslugFromUUID

@autoslugFromUUID()
class Space(models.Model):
    name = models.CharField(max_length=150)
    store = models.ForeignKey(Store, on_delete=models.CASCADE, related_name="store_spaces")
    slug = models.SlugField(unique=True, max_length=254)
    image_1 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_1_reference = models.CharField(max_length=254, blank=True, null=True)
    image_1_comment = models.CharField(max_length=254, blank=True, null=True)
    image_2 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_2_reference = models.CharField(max_length=254, blank=True, null=True)
    image_2_comment = models.CharField(max_length=254, blank=True, null=True)
    image_3 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_3_reference = models.CharField(max_length=254, blank=True, null=True)
    image_3_comment = models.CharField(max_length=254, blank=True, null=True)
    image_4 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_4_reference = models.CharField(max_length=254, blank=True, null=True)
    image_4_comment = models.CharField(max_length=254, blank=True, null=True)
    image_5 = models.ImageField(upload_to=upload_space_image_path, blank=True, null=True)
    image_5_reference = models.CharField(max_length=254, blank=True, null=True)
    image_5_comment = models.CharField(max_length=254, blank=True, null=True)
    equipment_details = models.TextField(blank=True, null=True)
    created_at = models.DateField(auto_now_add=True)
    updated_at = models.DateField(auto_now=True)

    class Meta:
        verbose_name = "Space"
        verbose_name_plural = "Spaces"
        ordering = ["-created_at"]

    def __str__(self):
        return self.name

    def clean(self, initialObject=None, requestObject=None):
        store_id = initialObject.store.id if initialObject else self.store.id if self.store else None
        qsFieldObjectList = [
            {
                "qs": self.__class__.objects.filter(name__iexact=self.name.lower(), store__id=store_id),
                "field": "name"
            }
        ]
        model_cleaner(selfObj=self, qsFieldObjectList=qsFieldObjectList, initialObject=initialObject)


from keyboards.user_keyboards.subscribe_channels import create_subscribe
from utils.json_worker.channels import get_channels, del_channel
from utils.misc.markreplace import markdowned

async def action_cd(callback):
    try:
        data = await get_channels()
        channel_id = callback.data.split('?')[1]
        channel_title = data[channel_id]['title']
        await del_channel(channel_id)
        await callback.message.answer(await markdowned(f'Канал *{channel_title}* успешно удален!'),
                                      reply_markup=await create_subscribe(url=False, cancel=True),
                                      parse_mode="MarkdownV2")

    except Exception as e:
        await callback.message.reply('ERROR!\n' + str(e))


from typing import Iterator, Sequence, Tuple

import torch
from torch import device, Tensor
from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler

from gym.spaces import Space

class RolloutStorage(object):
    def __init__(
        self,
        num_steps: int,
        num_processes: int,
        obs_shape: Sequence[int],
        action_space: Space,
    ):
        self.obs = torch.zeros(num_steps + 1, num_processes, *obs_shape)
        if action_space.__class__.__name__ == "Discrete":
            action_shape = 1
        else:
            action_shape = action_space.shape[0]
        self.actions = torch.zeros(num_steps, num_processes, action_shape)
        if action_space.__class__.__name__ == "Discrete":
            self.actions = self.actions.long()
        self.action_log_probs = torch.zeros(num_steps, num_processes, 1)
        self.rewards = torch.zeros(num_steps, num_processes, 1)
        self.value_preds = torch.zeros(num_steps + 1, num_processes, 1)
        self.returns = torch.zeros(num_steps + 1, num_processes, 1)
        self.masks = torch.ones(num_steps + 1, num_processes, 1)
        self.levels = torch.LongTensor(num_steps + 1, num_processes).fill_(0)

        self.num_steps = num_steps
        self.step = 0

    def __getitem__(self, key: str):
        return getattr(self, key)

    def to(self, device: device):
        self.obs = self.obs.to(device)
        self.actions = self.actions.to(device)
        self.action_log_probs = self.action_log_probs.to(device)
        self.rewards = self.rewards.to(device)
        self.value_preds = self.value_preds.to(device)
        self.returns = self.returns.to(device)
        self.masks = self.masks.to(device)
        self.levels = self.levels.to(device)

    def insert(
        self,
        obs: Tensor,
        actions: Tensor,
        action_log_probs: Tensor,
        rewards,
        value_preds: Tensor,
        masks: Tensor,
        levels: Tensor,
    ):
        self.obs[self.step + 1].copy_(obs)
        self.actions[self.step].copy_(actions)
        self.action_log_probs[self.step].copy_(action_log_probs)
        self.rewards[self.step].copy_(rewards)
        self.value_preds[self.step].copy_(value_preds)
        self.masks[self.step + 1].copy_(masks)
        self.levels[self.step + 1].copy_(levels)

        self.step = (self.step + 1) % self.num_steps

    def after_update(self):
        self.obs[0].copy_(self.obs[-1])
        self.masks[0].copy_(self.masks[-1])
        self.levels[0].copy_(self.levels[-1])

    def compute_returns(self, next_value: Tensor, gamma: float, gae_lambda: float):
        self.value_preds[-1] = next_value
        gae = 0
        for step in reversed(range(self.rewards.size(0))):
            delta = (
                self.rewards[step]
                + gamma * self.value_preds[step + 1] * self.masks[step + 1]
                - self.value_preds[step]
            )
            gae = delta + gamma * gae_lambda * self.masks[step + 1] * gae
            self.returns[step] = gae + self.value_preds[step]

    def compute_advantages(self):
        self.advantages = self.returns[:-1] - self.value_preds[:-1]
        mean = self.advantages.mean()
        std = self.advantages.std()
        self.advantages = (self.advantages - mean) / (std + 1e-5)

    def feed_forward_generator(
        self,
        num_mini_batch: int = None,
        mini_batch_size: int = None,
    ) -> Iterator[Tuple[Tensor, ...]]:
        num_steps, num_processes = self.rewards.size()[0:2]
        batch_size = num_processes * num_steps

        if mini_batch_size is None:
            assert batch_size >= num_mini_batch, "[ERROR] num mini batch is too large"
            mini_batch_size = batch_size // num_mini_batch

        sampler = BatchSampler(
            SubsetRandomSampler(range(batch_size)),
            mini_batch_size,
            drop_last=True,
        )

        for indices in sampler:
            obs_batch = self.obs[:-1].view(-1, *self.obs.size()[2:])[indices]
            next_obs_batch = self.obs[1:].view(-1, *self.obs.size()[2:])[indices]
            actions_batch = self.actions.view(-1, self.actions.size(-1))[indices]
            old_action_log_probs_batch = self.action_log_probs.view(-1, 1)[indices]
            rewards_batch = self.rewards.view(-1, 1)[indices]
            value_preds_batch = self.value_preds[:-1].view(-1, 1)[indices]
            returns_batch = self.returns[:-1].view(-1, 1)[indices]
            masks_batch = self.masks[1:].view(-1, 1)[indices]
            levels_batch = self.levels[:-1].view(-1, 1)[indices]

            if hasattr(self, "advantages"):
                adv_targs = self.advantages.view(-1, 1)[indices]
            else:
                adv_targs = None

            yield (
                obs_batch,
                next_obs_batch,
                actions_batch,
                old_action_log_probs_batch,
                rewards_batch,
                value_preds_batch,
                returns_batch,
                masks_batch,
                adv_targs,
                levels_batch,
            )


"""
blobGame.py
Izabele Bauzyte
4/28/2019

This game has a player move around the screen
to capture food particles, and once they have
captured all food, they reach final form and
have to capture the enemies
"""

"""
Attributions:

All art by Izabele Bauzyte

Babylon.ogg [Modified]
    "Babylon" Kevin MacLeod (incompetech.com)
    Licensed under Creative Commons: By Attribution 3.0 License
    http://creativecommons.org/licenses/by/3.0/
Ringing_Ambient_Background.ogg [Modified]
    Youtube Audio Library
Soft_Blossom.ogg [Modified]
    Youtube Audio Library
    Ann Annie
Splat.ogg [Modified]
    soundbible.com/1728-Spit-Splat-2.html
    Mike Koenig
    02/09/11
"""

#import and initialize
import pygame, random, time
pygame.init()

screen = pygame.display.set_mode((600, 600))

class Player(pygame.sprite.Sprite):
    """
    The central blob character
    stays in the middle of the screen
    """

    def __init__(self):

        pygame.sprite.Sprite.__init__(self)
        #img and rect
        self.scale = 30
        self.imgSrc = "player.png"
        self.image = pygame.transform.scale(pygame.image.load(self.imgSrc).convert_alpha(), (self.scale, self.scale))
        self.rect = self.image.get_rect()
        self.rect.centerx = screen.get_width() / 2
        self.rect.centery = screen.get_height() / 2

        # for animation of rotation
        self.degrees = 0

        self.finalForm = False

    def rotate(self):
        """
        Animates image
        """

        if (self.degrees + 1 > 360):

            self.degrees = 0

        else:

            self.degrees += 1

        self.image = pygame.transform.rotate(
                        pygame.transform.scale(
                                pygame.image.load(self.imgSrc).convert_alpha(), (self.scale, self.scale)), self.degrees)

    def update(self):

        self.rotate()

    def Feed(self, increase = 2):
        """
        Increases player size
        """

        self.scale += increase
        self.image = pygame.transform.scale(pygame.image.load(self.imgSrc), (self.scale, self.scale))
        self.rect = self.image.get_rect()
        self.rect.centerx = screen.get_width() / 2
        self.rect.centery = screen.get_height() / 2

        pygame.mixer.Sound("Splat.ogg").play()

    def FinalForm(self):
        """
        Changes player image
        """

        self.imgSrc = "playerOP.png"
        self.image = pygame.transform.scale(pygame.image.load(self.imgSrc).convert_alpha(), (self.scale, self.scale))
        self.finalForm = True

        pygame.mixer.Sound("Ringing_Ambient_Background.ogg").play()

class Food(pygame.sprite.Sprite):
    """
    Randomly generated food particles
    """

    def __init__(self, xStart, xEnd, yStart, yEnd):

        pygame.sprite.Sprite.__init__(self)

        self.imgSrc = "food.png"
        self.image = pygame.image.load(self.imgSrc).convert_alpha()
        self.image = pygame.transform.rotate(self.image, random.randint(0, 360))
        self.rect = self.image.get_rect()

        # area in which food can spawn due to borders
        self.xStart = xStart
        self.xEnd = xEnd
        self.yStart = yStart
        self.yEnd = yEnd

        self.rect.centerx = random.randint(self.xStart, self.xEnd)
        self.rect.centery = random.randint(self.yStart, self.yEnd)

        # x and y movement on screen
        self.maxMove = 5

        # rotation animation
        self.rotationSpeed = random.randint(0, 5)
        self.degrees = 0

    def moveVert(self, speed):
        """
        Moves food when player moves vertically
        """

        self.rect.centery += speed

    def moveHor(self, speed):
        """
        Moves food when player moves hor
        """

        self.rect.centerx += speed

    def rotate(self):

        if (self.degrees + self.rotationSpeed > 360):

            self.degrees = 0

        else:

            self.degrees += self.rotationSpeed

        self.image = pygame.transform.rotate(
                                pygame.image.load(self.imgSrc).convert_alpha(), self.degrees)

    def update(self):
        """
        Random movement contained inside moveable area
        TODO : still occasionally spawns/moves outside of playeable area

            All food particles spawn inside play area but are able to move outside
            so the problem is in this function.
            Discrepancy between the moveHor/moveVert functions and this one?
        """

        self.rotate()

        if (self.rect.centerx - self.maxMove > self.xStart) or (self.rect.centerx + self.maxMove < self.xEnd):

            self.rect.centerx += random.randint(-self.maxMove, self.maxMove)

        if (self.rect.centery - self.maxMove > self.yStart) or (self.rect.centery + self.maxMove < self.yEnd):

                self.rect.centery += random.randint(-self.maxMove, self.maxMove)

class Enemy(pygame.sprite.Sprite):
    """
    Enemy class chases player initially then changes to random
    movement when the player hits final form
    """

    def __init__(self, background, player):

        pygame.sprite.Sprite.__init__(self)
        self.imgSrc = "enemy.png"
        self.image = pygame.image.load(self.imgSrc).convert_alpha()
        self.rect = self.image.get_rect()
        self.rect.centerx = random.randint(0, background.get_width())
        self.rect.centery = random.randint(0, background.get_height())
        self.player = player

        # rotation animation
        self.degrees = 0
        self.rotationSpeed = random.randint(0, 5)

    def moveVert(self, speed):
        """
        updates vert with player movement
        """

        self.rect.centery += speed

    def moveHor(self, speed):
        """
        updates hor with player movement
        """

        self.rect.centerx += speed

    def rotate(self):

        if (self.degrees + self.rotationSpeed > 360):

            self.degrees = 0

        else:

            self.degrees += self.rotationSpeed

        self.image = pygame.transform.rotate(
                                pygame.image.load(self.imgSrc).convert_alpha(), self.degrees)

    def update(self):
        """
        Moves closer to the player every update
        or changes to random motion when they are
        in final form
        """

        self.rotate()

        if self.player.finalForm:

            self.randomMovement()

        else:

            self.chasePlayer()

    def chasePlayer(self):
        """
        Moves closer to the player
        """

        if self.player.rect.centerx > self.rect.centerx:

            self.rect.centerx += 1
        else:
            self.rect.centerx -= 1

        if self.player.rect.centery > self.rect.centery:

            self.rect.centery += 1

        else:
            self.rect.centery -= 1

    def randomMovement(self):

        self.rect.centerx += random.randint(-3, 3)
        self.rect.centery += random.randint(-3, 3)

class Label(pygame.sprite.Sprite):
    """
    Object to display text
    """

    def __init__(self, text, loc, alignment, color = (255, 255, 255)):
        """
        string text
        tuple location
        int alignment (-1 = left, 0 = center, 1 = right)
        3 tuple for color default is white
        """

        pygame.sprite.Sprite.__init__(self)

        self.font = pygame.font.SysFont("None", 30)
        self.text = text
        self.color = color
        self.loc = loc
        self.alignment = alignment

    def update(self):

        # image
        self.image = self.font.render(self.text, 1, self.color)
        self.rect = self.image.get_rect()

        # left aligned
        if self.alignment == -1:

            self.rect.left = self.loc[0]
            self.rect.centery = self.loc[1]

        # center aligned
        elif self.alignment == 0:

            self.rect.center = self.loc

        # right aligned
        else:

            self.rect.right = self.loc[0]
            self.rect.centery = self.loc[1]

class Border(pygame.sprite.Sprite):
    """
    Constrains player from moving outside of playable area
    and conceals black background from player with appropriate size
    """

    def __init__(self, startX, startY, width, height):

        pygame.sprite.Sprite.__init__(self)

        self.image = pygame.Surface((width, height))
        self.image.fill((160, 160, 160))
        self.rect = self.image.get_rect()

        self.rect.left = startX
        self.rect.top = startY

    def moveVert(self, speed):
        """
        updates with player motion
        """

        self.rect.centery += speed

    def moveHor(self, speed):
        """
        updates with player motion
        """

        self.rect.centerx += speed

def checkKeysMoveViewArea(keys, x, y, movementRate, player, movementGroup, background, borders):
    """
    keys: list of keys pressed during the game
    x, y: x and y to move all objects and background by when player moves
    movementRate: the rate at which objects and bg are moved
    movementGroup: all non-BG objects which need to be shifted
    background:
    borders: list of 4 borders in clockwise format that constrain player and objects

    This function checks whether a key is currently being pressed
    and moves objects on screen accordingly

    returns x, y so they can be constantly updated
    """

    if keys[pygame.K_UP]:

        # top border
        if not player.rect.colliderect(borders[0].rect):

            y += movementRate
            for sprite in movementGroup:

                sprite.moveVert(movementRate)

    if keys[pygame.K_RIGHT]:

        # right border
        if not player.rect.colliderect(borders[1].rect):

            x -= movementRate
            for sprite in movementGroup:

                sprite.moveHor(-movementRate)

    if keys[pygame.K_DOWN]:

        # bottom border
        if not player.rect.colliderect(borders[2].rect):

            y -= movementRate
            for sprite in movementGroup:

                sprite.moveVert(-movementRate)

    if keys[pygame.K_LEFT]:

        # left border
        if not player.rect.colliderect(borders[3].rect):

            x += movementRate
            for sprite in movementGroup:

                sprite.moveHor(movementRate)

    screen.blit(background, (x, y))
    return x, y

def getScore(lives, foodCollected, enemiesKilled, secondsRemain):

    lifeMult = 100
    foodMult = 10
    enemyMult = 20
    secondsMult = 30

    return (lifeMult * lives) + (foodMult * foodCollected) + (enemyMult * enemiesKilled) + (secondsMult * secondsRemain)

def game():

    #display

    pygame.display.set_caption("basic blob game")

    #entities

    # game setup
    startSeconds = 20
    seconds = startSeconds
    lives = 3

    # bg
    background = pygame.image.load("background.png").convert_alpha()

    # shadow
    shadowBase = pygame.image.load("shadow.png").convert_alpha()
    shadow = shadowBase.copy()
    shadowAlpha = 255 - (startSeconds * 4) # opacity of on screen shadow
    shadow.fill((255, 255, 255, shadowAlpha), None, pygame.BLEND_RGBA_MULT)
    screen.blit(shadow, (0,0))

    # num food and enemies
    numEnemies = 15
    numFood = 15
    numFoodCollected = 0
    numEnemiesKilled = 0

# borders
    # setting up game borders
    borderThickness = 20 # thickness of each border
    offset = (screen.get_width() / 2) - 30 # how far from background edge each border is
    """
    Used in this case to hide the edge of the background from the screen
    """

    # the area remaining for game use after offset is set
    playAreaStart = borderThickness + offset
    playAreaEnd = background.get_height() - borderThickness - offset

    borderTop = Border(offset, offset, background.get_height() - (2 * offset), borderThickness)
    borderRight = Border(background.get_width() - offset - borderThickness, offset, borderThickness, background.get_height() - (2 * offset))
    borderBottom = Border(offset, background.get_height() - offset - borderThickness, background.get_width() - (2 * offset), borderThickness)
    borderLeft = Border(offset, offset, borderThickness, background.get_height() - (2 * offset))
    # clockwise array of border sides
    borders = [borderTop, borderRight, borderBottom, borderLeft]

# player
    player = Player()
    playerGroup = pygame.sprite.Group(player)

# enemies
    enemyGroup = pygame.sprite.Group()
    for i in range(numEnemies):

        enemyGroup.add(Enemy(background, player))

# food
    foodGroup = pygame.sprite.Group()
    for i in range(numFood):

        foodGroup.add(Food(playAreaStart, playAreaEnd, playAreaStart, playAreaEnd))

# screen labels
    timeLabelLoc = (screen.get_width() / 2, 20)
    collisionLabelLoc = (20 , screen.get_height() - 50)
    livesLabelLoc = (20, screen.get_height() - 20)

    timeLabelText = "%d Seconds Left"

    foodCollisionText = "%d/%d Food Collected"
    enemyCollisionText = "%d/%d Enemies Killed"

    livesLabelText = "Lives Remaining: %d"

    timeLabel = Label("", timeLabelLoc, 0)
    collisionLabel = Label(foodCollisionText % (numFoodCollected, numFood), collisionLabelLoc, -1)
    livesLabel = Label(livesLabelText %(lives), livesLabelLoc, -1)

# groupings
    labelGroup = pygame.sprite.Group()
    labelGroup.add(timeLabel)
    labelGroup.add(collisionLabel)
    labelGroup.add(livesLabel)

    # for collision detection
    entitiesGroup = pygame.sprite.Group()
    entitiesGroup.add(enemyGroup)
    entitiesGroup.add(foodGroup)

    # for movement on/off screen
    movementGroup = pygame.sprite.Group()
    movementGroup.add(enemyGroup)
    movementGroup.add(foodGroup)
    movementGroup.add(borders)

    #assign
    clock = pygame.time.Clock()
    keepGoing = True

    # for keeping track of background location
    x = 0
    y = 0

    # rate at which player moves
    movementRate = 10

    start = time.time()

    # score info
    previousScore = 0

    #loop
    while keepGoing:

        #time
        clock.tick(30)

        keys = pygame.key.get_pressed()

        # moves the items around the screen based on player keys
        x, y = checkKeysMoveViewArea(keys, x, y, movementRate, player, movementGroup, background, borders)

        #events
        for event in pygame.event.get():

            if event.type == pygame.QUIT:

                keepGoing = False
                previousScore = getScore(lives, numFoodCollected, numEnemiesKilled, seconds)

        collisions = pygame.sprite.spritecollide(player, entitiesGroup, True)

        for sprite in collisions:

            # player is still collecting food
            if numFoodCollected < numFood:

                # player has hit an enemy while they are not yet in final mode
                if isinstance(sprite, Enemy):

                    numEnemies -= 1
                    lives -= 1
                    livesLabel.text = livesLabelText % (lives)

                    thing = pygame.Surface((screen.get_width(), screen.get_height()))
                    thing.fill((255, 0, 0))
                    screen.blit(thing, (0,0))

                    if lives == 0:

                        keepGoing = False
                        previousScore = getScore(lives, numFoodCollected, numEnemiesKilled, seconds)

                # player has collected another food sprite
                if isinstance(sprite, Food):

                    player.Feed()

                    # change lighting effect to indicate more energy
                    shadowAlpha -= 10
                    shadow = shadowBase.copy()
                    shadow.fill((255, 255, 255, shadowAlpha), None, pygame.BLEND_RGBA_MULT)

                    numFoodCollected += 1

                    if numFoodCollected == numFood:

                        player.FinalForm()
                        labelGroup.add(Label("Final Form", (20,20), -1, (255, 0, 0)))
                        collisionLabel.text = enemyCollisionText % (numEnemiesKilled, numEnemies)

                    else:

                        collisionLabel.text = foodCollisionText % (numFoodCollected, numFood)

            # player has collected all food
            else:

                # eat enemy and increase in size
                if isinstance(sprite, Enemy):

                    numEnemiesKilled += 1
                    collisionLabel.text = enemyCollisionText % (numEnemiesKilled, numEnemies)

                    player.Feed(5)

                    # player has eaten all enemies
                    if numEnemies == numEnemiesKilled:

                        keepGoing = False
                        previousScore = getScore(lives, numFoodCollected, numEnemiesKilled, seconds)

        elapsedTime = time.time() - start

        # change screen shadow opacity every half second
        if (elapsedTime > 0.5):

            start = time.time()

            alphaShift = 2
            shadowAlpha += alphaShift

            # reset to valid color argument
            if shadowAlpha <= 0:

                shadowAlpha = 0

            # player ran out of time
            if shadowAlpha + alphaShift >= 255:

                keepGoing = False
                previousScore = time.time() - start

            # update label
            else:

                seconds = (255 - shadowAlpha) / (alphaShift / 0.5)
                timeLabel.text = timeLabelText % (seconds)

            # update shadow overlay
            shadow = shadowBase.copy()
            shadow.fill((255, 255, 255, shadowAlpha), None, pygame.BLEND_RGBA_MULT)

        # update and move sprites in layers
            # moving group
        movementGroup.update()
        movementGroup.draw(screen)
            # player
        playerGroup.update()
        playerGroup.draw(screen)
            # screen
        screen.blit(shadow, (0,0))
            # labels on top of shadow to be visible
        labelGroup.update()
        labelGroup.draw(screen)

        #refresh
        pygame.display.flip()

    return previousScore

def instructions(previousScore, highScore):
    """
    Tells user how to play
    and shows previous scores
    """

    allSprites = pygame.sprite.Group(Player())

    # some interesting graphics for the intro screen
    for i in range(10):

        food = Food(0, screen.get_width() / 2, 500, screen.get_height() - 30)

        # using a food class and changing image to avoid the following enemy behaviour
        foodWithEnemyPic = Food(screen.get_width() / 2, screen.get_width(), 500, screen.get_height() - 30)
        foodWithEnemyPic.imgSrc = "enemy.png"

        allSprites.add(food)
        allSprites.add(foodWithEnemyPic)

    insFont = pygame.font.SysFont(None, 30)

    # really awful instruction string that tells user how to play
    instructions = ["",
    "Blob Game",
    "Collect the food and grow into your final form",
    "Then eat the enemies pursuing you",
    "Press space to begin", ("Previous Score: %d" % previousScore), ("Session High Score: %d" % highScore), "", "",
                    "                                       You ","","", "", "", "",
                    "        Food                                        Enemy"
        ]

    insLabels = []

    bg = pygame.image.load("background.png").convert_alpha()

    screen.blit(bg, (0,0))

    for line in instructions:

        insLabels.append(insFont.render(line, 1, (255, 255, 0)))

    keepGoing = True
    clock = pygame.time.Clock()

    while keepGoing:

        clock.tick(30)

        for event in pygame.event.get():

            if event.type == pygame.QUIT:

                # exit game completely
                keepGoing = False
                donePlaying = True

            if event.type == pygame.KEYDOWN:

                if event.key == pygame.K_SPACE:

                    # play again
                    keepGoing = False
                    donePlaying = False

        allSprites.clear(screen, bg)
        allSprites.update()
        allSprites.draw(screen)

        for i in range(len(insLabels)):

            screen.blit(insLabels[i], (50, 30 * i))

        pygame.display.flip()

    return donePlaying

def main():

    pygame.mouse.set_visible(False)

    previousScore = 0
    highScore = 0

    donePlaying = False

    while not donePlaying:

        pygame.mixer.stop()
        pygame.mixer.Sound("Soft_Blossom.ogg").play(-1)

        donePlaying = instructions(previousScore, highScore)

        if not donePlaying:

            pygame.mixer.stop()
            pygame.mixer.Sound("Babylon.ogg").play(-1)

            print("game")
            previousScore = game()
            if previousScore > highScore:

                highScore = previousScore

    pygame.quit()

if __name__ == "__main__":
    main()


import sys
from usando_qtdesigner148.design import *
from PyQt5.QtWidgets import QMainWindow, QApplication, QFileDialog
from PyQt5.QtGui import QPixmap

class Redimensionamento(QMainWindow, Ui_MainWindow):
    def __init__(self, parent=None):
        super().__init__(parent)
        super().setupUi(self)  # para que ele execute a instancia dessa classe
        self.btnEscolheArq.clicked.connect(self.abrir_imagem)
        self.btnRedim.clicked.connect(self.redimensionar)
        self.btnSalvar.clicked.connect(self.salvar)

    def abrir_imagem(self):
        image, _ = QFileDialog.getOpenFileName(
            self.centralwidget,
            'Abrir imagem'
            '/home/adriana/Downloads/',
            # options=QFileDialog.DontUseNativeDialog  #se nao funcionar a janela, descomentar essa parte

        )
        self.inputArquivo.setText(image)
        self.original_img = QPixmap(image)
        self.labelImg.setPixmap(self.original_img)
        self.inputLargura.setText(str(self.original_img.width()))
        self.inputAltura.setText(str(self.original_img.height()))

    def redimensionar(self):
        largura = int(self.inputLargura.text())
        self.nova_imagem = self.original_img.scaledToWidth(largura)  # calcula a altura automaticamente
        self.labelImg.setPixmap(self.nova_imagem)
        self.inputLargura.setText(str(self.nova_imagem.width()))
        self.inputAltura.setText(str(self.nova_imagem.height()))

    def salvar(self):
        image, _ = QFileDialog.getSaveFileName(
            self.centralwidget,
            'Salvar imagem'
            '/home/adriana/',
        )
        self.nova_imagem.save(image)

if __name__ == '__main__':
    qt = QApplication(sys.argv)
    novo = Redimensionamento()
    novo.show()
    qt.exec_()

5 - interface grafica/usando_qtdesigner148/novo.py


"""empty message

Revision ID: 0770e1a3baea
Revises: 6525b4aac66c
Create Date: 2018-04-13 22:40:38.476991

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '0770e1a3baea'
down_revision = '6525b4aac66c'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('logs',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('subject', sa.String(length=30), nullable=False),
    sa.Column('content', sa.Text(), nullable=False),
    sa.Column('level', sa.String(length=20), nullable=False),
    sa.Column('created_at', sa.DateTime(), nullable=False),
    sa.Column('solved_at', sa.DateTime(), nullable=False),
    sa.Column('created_by', sa.Integer(), nullable=True),
    sa.Column('solved_by', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['created_by'], ['users.id'], ),
    sa.ForeignKeyConstraint(['solved_by'], ['users.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('logs')
    # ### end Alembic commands ###


##############################################################
##
##  mode_shares.py
##
##
##  Daniel Jimenez, Metro, March 15, 2016
##
##############################################################

import openpyxl
from openpyxl import load_workbook
from collections import defaultdict
import sys
import os

def load_vector(path):
    #Folder ../../model/reports
    folder = os.path.dirname(os.path.realpath(path))
    #Notebook = ../../model/reports/notebook
    notebook_folder = os.path.join(folder,"notebook")
    for root, dirnames, filenames in os.walk(notebook_folder):
        for file in filenames:
            if "mode_share_summary_vector" in file:
                return [float(x.strip()) for x in open(os.path.join(notebook_folder,file),'r')]

def load_data(sheet, name, vector):
    sheet["C3"].value = name
    dic_data = defaultdict(int)
    reference = [int(x) for x in open("V:/tbm/kate/programs_v1.0/src/py/automation_files/joan_summary_cell_references.csv", 'r')]
    data = zip(reference,vector)

    #Zip reference cell and value. Reference cell should point to where value should go in the report
    for r,v in data:
        dic_data[r] += v

    #Write value referencing the key cell and the value
    for k,v in dic_data.iteritems():
        sheet["C{}".format(k)].value = int(round(v,0))

def main():
    #Passed arguments
    excel_file = sys.argv[1]
    run_name = sys.argv[2]

    #Load mode_share_summary_vector into a list
    summary_vector = load_vector(excel_file)

    des_excel = excel_file
    des_book = load_workbook(des_excel)
    mode_sheet = des_book.get_sheet_by_name("Mode Choice Summary")
    load_data(mode_sheet,run_name,summary_vector)
    des_book.save(des_excel)

if __name__ == "__main__":
    main()


import numpy as np
import pandas as pd
import os
from sklearn.preprocessing import LabelEncoder  # кодирование категориальных
from sklearn.feature_selection import SelectKBest  # выбор признаков
from sklearn.feature_selection import chi2  # выбор по Хи квадрат
from sklearn.model_selection import train_test_split  # деление на тест и обучение
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from catboost import CatBoostClassifier

class Settings:

    def __init__(self):
        self.df_row_count = 2001
        self.df_subject_count = 6
        self.df_min_grade = 0
        self.df_max_grade = 99
        self.df_file = 'students.csv'
        self.model_file = 'people_university_model.cbm'
        self.model_file_eff = 'people_university_model_eff.cbm'
        self.resource_folder = 'ml'
        self.final_result_list = ['удовлетворительно', 'хорошо', 'отлично']

        if not os.path.exists(self.resource_folder):
            os.mkdir(self.resource_folder)

    def df_path(self):
        return self.resource_folder + '/' + self.df_file

    def model_path(self):
        return self.resource_folder + '/' + self.model_file

    def model_eff_path(self):
        return self.resource_folder + '/' + self.model_file_eff

class DataframeGenerator:

    def __init__(self, settings: Settings):
        self.__settings = settings

    def generate(self):
        subjects = [f"subject_{num + 1}" for num in range(self.__settings.df_subject_count)]

        grades = np.random.randint(
            self.__settings.df_min_grade,
            self.__settings.df_max_grade + 1,
            (self.__settings.df_row_count, self.__settings.df_subject_count)
        )
        student_scores = pd.DataFrame(grades, columns=subjects)

        student_scores.sum(axis=0)
        student_scores['mean_score'] = np.round((student_scores.sum(axis=1) / self.__settings.df_subject_count), 3)

        student_scores['final_lab'] = student_scores.apply(self.__generate_final, axis=1)

        return student_scores

    def __generate_final(self, row):
        score = row['mean_score']
        if score < 35:
            val = self.__settings.final_result_list[0]
        elif 35 <= score < 60:
            val = self.__settings.final_result_list[1]
        else:
            val = self.__settings.final_result_list[2]
        return val

class StudentRepository:

    def __init__(self, settings: Settings):
        self.__data_generator = DataframeGenerator(settings)
        self.__settings = settings

    def create(self):
        return self.__data_generator.generate()

    def store(self):
        df = self.create()
        df.to_csv(self.__settings.df_path(), index=False)

    def load(self):
        return pd.read_csv(self.__settings.df_path())

    def loadOrCreate(self, override=False):
        if override or not os.path.exists(self.__settings.df_path()):
            self.store()
        return self.load()

class ModelService:

    @staticmethod
    def train_model(x_train, y_train, iterations=100, depth=4, learning_rate=0.1, file_to_save=''):
        model = CatBoostClassifier(iterations=iterations, depth=depth, learning_rate=learning_rate)
        model.fit(x_train, y_train)
        model.save_model(file_to_save)

    @staticmethod
    def preprocess_dataset(
            df,
            to_drop=['final_lab', 'mean_score'],
            y_name='final_lab',
            need_to_encode_y=True,
            need_best_features=True,
            test_size=0.2,
            random_state=1234
    ):
        X = df.drop(columns=to_drop, axis=1)
        y = df[y_name]  # Series

        if need_to_encode_y:
            label_encoder = LabelEncoder()
            temp = label_encoder.fit_transform(y)
            new_result = pd.DataFrame([y, temp], columns=['y', 'encoded_y'])
            new_result.to_csv('ml/result_labels.csv', index=False)
            y = temp

        if need_best_features:
            # 2. Выберите лучшие 3 признака для обучения
            selector = SelectKBest(chi2, k=3)
            X = selector.fit_transform(X, y)

        # 3. Разбейте датасет на тестовую и обучающую выборку train_test_split (для лучших признаков)
        # 80% обучающей выборки, 20% тестовой
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state)

        return X_train, X_test, y_train, y_test

    @staticmethod
    def model_report(model, x_test, y_test, average='weighted'):
        y_pred = model.predict(x_test)
        return {
            'accuracy': accuracy_score(y_test, y_pred),
            'precision': precision_score(y_test, y_pred, average=average),
            'recall': recall_score(y_test, y_pred, average=average),
            'f1': f1_score(y_test, y_pred, average=average)
        }

    @staticmethod
    def load_model(path_to_model=''):
        new_model = CatBoostClassifier()
        new_model.load_model(path_to_model)
        return new_model


from django.conf.urls import url

from keywords import views

app_name = "keywords"

urlpatterns = [
    url(r'^$',
        views.keyword_list,
        name="keyword_list"),

    url(r'^(?P<keyword_slug>[-\w]*)/$',
        views.keyword_detail,
        name="keyword_detail"),
]


import numpy as np
import pandas as pd
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from nltk.corpus import stopwords
import string

## getting training data

df = pd.read_csv("0000000000002747_training_twitter_x_y_train.csv")
y = list(df["airline_sentiment"])
text = list(df["text"])

## training documents
documents = []
for i in range(len(y)):
    documents.append((text[i].split(" "),y[i]))
documents[0:5] ## in the form of tuple with 1st element as a list of words and second as the sentiment, ie positive, negative or neutral

## processing the words:

## making a list of stopwords and punctuations to be removed from the tweets

stop = stopwords.words("english")
punctuations = list(string.punctuation)
stop = stop + punctuations ## to remove punctuations

## using nltk library

from nltk.corpus import wordnet

## fucntion to get pos_tag of a word
def get_simple_pos(tag):
    if tag.startswith('J'):
        return wordnet.ADJ
    elif tag.startswith('V'):
        return wordnet.VERB
    elif tag.startswith('N'):
        return wordnet.NOUN
    elif tag.startswith('R'):
        return wordnet.ADV
    else:
        return wordnet.NOUN

from nltk import pos_tag
from nltk.stem import WordNetLemmatizer
lt = WordNetLemmatizer()

## fucntion to clean the words in the text
## removing stop words and punctuations
## performing lemmatization to get better insight on the meaning of the words

def clean_review(words):
    output_words = []
    for w in words:
        if w.lower() not in stop:
            try:
                pos = pos_tag([w])
                clean_word = lt.lemmatize(w,pos = get_simple_pos(pos[0][1]))
                output_words.append(clean_word.lower())
            except:
                continue
    return output_words

documents = [(clean_review(i),category) for i,category in documents]

y_train = [category for words,category in documents]
x_train = [" ".join(words) for words, categories in documents]

## using tfidf Vectorizer to remove words which do not hold much importance during classification

from sklearn.feature_extraction.text import TfidfVectorizer
token_vec = TfidfVectorizer(max_features = 3000, ngram_range = (1,3))
x_train_t = token_vec.fit_transform(x_train) ## we can use the sparse matrix directyly as training and testing data in our sklearn classifiers

## forming x_test

df = pd.read_csv("0000000000002747_test_twitter_x_test.csv")
text = list(df["text"])
x_test = []
for i in range(len(text)):
    x_test.append(text[i])
x_test

x_test_t = token_vec.transform(x_test) ## we can use the sparse matrix directyly as training and testing data in our sklearn classifiers

## both training and testing data now ready
from sklearn.model_selection import train_test_split
x_train, x_val, y_train, y_val = train_test_split(x_train_t, y_train)
## classification using MultiNomial Naive Bayes
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB()
clf.fit(x_train, y_train)
print(clf.score(x_val, y_val)) ## checking the score on the validation data

## saving the predictions in a csv file
y_pred = clf.predict(x_test_t)
predictions = np.array(y_pred)
pd.DataFrame(predictions).to_csv("finall.csv") ## saving file