#!/usr/bin/python

import sys
import os
import shutil
import json
import pickle
import warnings
import datetime
import numpy as np
import pandas as pd
from tqdm import tqdm
import joblib

from sklearn.isotonic import IsotonicRegression

import xgboost as xgb
import catboost as cb
from sklearn import ensemble
with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    import lightgbm as lgb

def load_data(path_data):
    # Load preprocessed data

    df = pd.read_csv(path_data, header=[0,1], index_col=[0,1], parse_dates=True)

    return df

class Trial():
    def __init__(self, params_json):
        self.params_json = params_json

        # Mandatory input variables
        self.trial_name = params_json['trial_name']
        self.trial_comment = params_json['trial_comment']
        self.path_result = params_json['path_result']
        self.path_preprocessed_data = params_json['path_preprocessed_data']
        self.splits = params_json['splits']
        self.sites = params_json['sites']
        self.features = params_json['features']
        self.target = params_json['target']
        self.model_params = params_json['model_params']
        self.regression_params = params_json['regression_params']
        self.save_options = params_json['save_options']

        if 'parallel_processing' in params_json:
            self.parallel_processing = params_json['parallel_processing']
        else:
            self.parallel_processing = {'backend': 'threading',
                                        'n_workers': 1}

        if 'quantile' in self.regression_params['type']:
            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])
            if len(alpha_q) == 0:
                raise ValueError('Number of quantiles needs to be larger than zero.')

        # Optional input variables
        if 'variables_lags' in params_json:
            self.variables_lags = params_json['variables_lags']
        else:
            self.variables_lags = None
        if 'diff_target_with_physical' in params_json:
            self.diff_target_with_physical = params_json['diff_target_with_physical']
        else:
            self.diff_target_with_physical = False
        if 'target_smoothing_window' in params_json:
            self.target_smoothing_window = params_json['target_smoothing_window']
        else:
            self.target_smoothing_window = 1
        if 'train_only_zenith_angle_below' in params_json:
            self.train_only_zenith_angle_below = params_json['train_only_zenith_angle_below']
        else:
            self.train_only_zenith_angle_below = False
        if 'weight_params' in params_json:
            self.weight_params = params_json['weight_params']
        else:
            self.weight_params = False
        # Checks

        # runtime
        self.parallel_backend = params_json.get("parallel_backend", "threading")

    def generate_dataset(self, df, split, site):

        def add_lags(df, variables_lags):
            # Lagged features
            vspec = pd.DataFrame([(k, lag) for k, v in variables_lags.items() for lag in v],
                                 columns=["Variable", "Lag"])\
                              .set_index("Variable")\
                              .sort_values("Lag")
            for lag, variables in vspec.groupby("Lag").groups.items():
                shifted = df.loc[:, sorted(variables)].groupby('ref_datetime').shift(lag)
                shifted.columns = ['%s_lag%s' % (variable, lag) for variable in sorted(variables)]
                df = pd.concat([df, shifted], axis=1)
            return df

        # Make target into list if not already
        if self.diff_target_with_physical and not ('Physical_Forecast' in self.features):
            df_X = df[site].loc[pd.IndexSlice[:, split[0]:split[1]], self.features+['Physical_Forecast']]
        else:
            df_X = df[site].loc[pd.IndexSlice[:, split[0]:split[1]], self.features]

        df_y = df[site].loc[pd.IndexSlice[:, split[0]:split[1]], [self.target]]

        # Add lagged variables
        if self.variables_lags is not None:
            df_X = add_lags(df_X, self.variables_lags)

        # Remove samples where either all features are nan or target is nan
        is_nan = df_X.isna().all(axis=1) | df_y.isna().all(axis=1)
        df_model = pd.concat([df_X, df_y], axis=1)[~is_nan]

        # Keep all timestamps for which zenith <= prescribed value (day timestamps)
        if self.train_only_zenith_angle_below:
            idx_day = df_model[df_model['zenith'] <= self.train_only_zenith_angle_below].index
            df_model = df_model.loc[idx_day, :]

        # Create target and feature DataFrames
        if self.diff_target_with_physical:
            df_model[self.target] = df_model[self.target]-df_model['Physical_Forecast']

        # Use mean window to smooth target
        df_model[self.target] = df_model[self.target].rolling(self.target_smoothing_window, win_type='boxcar', center=True, min_periods=0).mean()

        # Apply sample weighting
        if self.weight_params:
            weight_end = self.weight_params['weight_end']
            weight_shape = self.weight_params['weight_shape']
            valid_times = df_model.index.get_level_values('valid_datetime')
            days = np.array((valid_times[-1]-valid_times).total_seconds()/(60*60*24))
            weight = (1-weight_end)*np.exp(-days/weight_shape)+weight_end
        else:
            weight = None

        return df_X, df_y, df_model, weight

    def generate_dataset_split_site(self, df, split_set='train'):
        # Generate train and valid splits

        print('Generating dataset...')
        dfs_X_split, dfs_y_split, dfs_model_split, weight_split = [], [], [], []
        with tqdm(total=len(self.splits[split_set])*len(self.sites)) as pbar:
            for split in self.splits[split_set]:
                dfs_X_site, dfs_y_site, dfs_model_site, weight_site = [], [], [], []
                for site in self.sites:

                    df_X, df_y, df_model, weight = self.generate_dataset(df, split, site)

                    dfs_X_site.append(df_X)
                    dfs_y_site.append(df_y)
                    dfs_model_site.append(df_model)
                    weight_site.append(weight)

                    pbar.update(1)

                dfs_X_split.append(dfs_X_site)
                dfs_y_split.append(dfs_y_site)
                dfs_model_split.append(dfs_model_site)
                weight_split.append(weight_site)

        return dfs_X_split, dfs_y_split, dfs_model_split, weight_split

    def build_model_dataset(self, df_model_train, model, df_model_valid=None, weight=None):
        # Build up dataset adapted to models
        train_set, valid_sets = {}, {}
        if model == 'lightgbm':
            train_set = lgb.Dataset(df_model_train[self.features], label=df_model_train[[self.target]], weight=weight, params={'verbose': -1}, free_raw_data=False)
            if df_model_valid is not None:
                valid_set = lgb.Dataset(df_model_valid[self.features], label=df_model_valid[[self.target]], params={'verbose': -1}, free_raw_data=False)
                valid_sets = [train_set, valid_set]
            else:
                vaild_sets['lightgbm'] = [train_set_lgb]
        elif model == 'xgboost':
            train_set = xgb.DMatrix(df_model_train[self.features], label=df_model_train[[self.target]], weight=weight)
            if df_model_valid is not None:
                valid_set = xgb.DMatrix(df_model_valid[self.features], label=df_model_valid[[self.target]])
                valid_sets = [(train_set, 'train'), (valid_set, 'valid')]
            else:
                valid_sets = [(train_set, 'train')]
        elif model == 'catboost':
            train_set = cb.Pool(df_model_train[self.features], label=df_model_train[[self.target]], weight=weight)
            if df_model_valid is not None:
                valid_set = cb.Pool(df_model_valid[self.features], label=df_model_valid[[self.target]])
                valid_sets = [valid_set]
        elif model == 'skboost' in self.model_params:
            train_set = [df_model_train[self.features], df_model_train[self.target], weight]

        return train_set, valid_sets

    def train_on_objective(self, train_set, valid_sets, model, objective='mean', alpha=None):

        if model == 'lightgbm':
            with warnings.catch_warnings():
                if self.model_params['lightgbm']['verbose'] == -1:
                    warnings.simplefilter("ignore")
                if objective == 'mean':
                    objective_lgb = 'mean_squared_error'
                    eval_key_name = 'l2'
                elif objective == 'quantile':
                    objective_lgb = 'quantile'
                    eval_key_name = 'quantile'
                    self.model_params['lightgbm']['alpha'] = alpha
                else:
                    raise ValueError("'objective' for lightgbm must be either 'mean' or 'quantile'")
                self.model_params['lightgbm']['objective'] = objective_lgb

                evals_result = {}
                gbm = lgb.train(self.model_params['lightgbm'],
                                train_set,
                                valid_sets=valid_sets,
                                valid_names=None,
                                evals_result=evals_result,
                                verbose_eval=False,
                                callbacks=None)
                evals_result = {key: value[eval_key_name] for key, value in evals_result.items()}

        elif model == 'xgboost':
            if objective == 'mean':
                self.model_params['xgboost']['objective'] = 'reg:squarederror'
            else:
                raise ValueError("'objective' for xgboost must be 'mean'.")
            evals_result = {}
            gbm = xgb.train(self.model_params['xgboost'],
                            train_set,
                            self.model_params['xgboost']['num_round'],
                            evals=valid_sets,
                            evals_result=evals_result,
                            verbose_eval=False)
            evals_result = None #TODO Add evals for xgboost

        elif model=='catboost':
            if objective == 'mean':
                objective_cb = 'Lq:q=2'
            elif objective == 'quantile':
                objective_cb = 'Quantile:alpha={0:g}'.format(alpha)
            else:
                raise ValueError("'objective' must be one of ['mean', 'quantile']")
            self.model_params['catboost']['objective'] = objective_cb

            gbm = cb.train(pool=train_set,
                           params=self.model_params['catboost'],
                           eval_set=valid_sets,
                           verbose=False)
            evals_result = {key: value[objective_cb] for key, value in gbm.evals_result_.items()}

        elif model=='skboost':
            if objective == 'mean':
                self.model_params['skboost']['loss'] = 'ls'
                self.model_params['skboost']['criterion'] = 'friedman_mse'
            elif objective == 'quantile':
                self.model_params['skboost']['loss'] = 'quantile'
                self.model_params['skboost']['alpha'] = alpha
                self.model_params['skboost']['criterion'] = 'mae' #TODO Check how `criterion` affects quantile loss.
            else:
                raise ValueError("'objective' must be one of ['mean', 'quantile']")

            gbm = ensemble.GradientBoostingRegressor(**self.model_params['skboost'])
            gbm.fit(train_set[0], train_set[1], sample_weight=train_set[2])
            evals_result = None #TODO Add evals for skboost

        else:
            raise ValueError("'objective' for skboost must be either 'mean' or 'quantile'")

        return gbm, evals_result

    def train(self, train_set, valid_sets, model):

        gbm_q, evals_result_q = {}, {}
        if 'mean' in self.regression_params['type']:
            # Train model for mean
            gbm, evals_result = self.train_on_objective(train_set, valid_sets, model, objective='mean')

            gbm_q['mean'] = gbm
            evals_result_q['mean'] = evals_result #TODO change this similarly to quantile case.

        if 'quantile' in self.regression_params['type']:
            # Train models for different quantiles
            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])

            with joblib.parallel_backend(self.parallel_processing['backend']):
                results = joblib.Parallel(n_jobs=self.parallel_processing['n_workers'])(
                                joblib.delayed(self.train_on_objective)(
                                    train_set, valid_sets,
                                    model, objective='quantile', alpha=alpha)
                                    for alpha in alpha_q)
            for (gbm, evals_result), alpha in zip(results, alpha_q):
                gbm_q['quantile{0:.2f}'.format(alpha)] = gbm
                evals_result_q['quantile{0:.2f}'.format(alpha)] = evals_result

        if not (('mean' in self.regression_params['type']) or ('quantile' in self.regression_params['type'])):
            raise ValueError('Value of regression parameter "objective" not recognized.')

        return gbm_q, evals_result_q

    def train_model_split_site(self, dfs_model_train_split, dfs_model_valid_split=None, weight_train_split=None):

        print('Training...')
        gbm_model, evals_result_model = {}, {}
        with tqdm(total=len(self.model_params.keys())*len(dfs_model_train_split)*len(dfs_model_train_split[0])) as pbar:
            for model in self.model_params.keys():
                gbm_split, evals_result_split = [], []
                for idx_split, dfs_model_train_site in enumerate(dfs_model_train_split):

                    gbm_site, evals_result_site = [], []
                    for idx_site, df_model_train in enumerate(dfs_model_train_site):

                        if dfs_model_valid_split is not None:
                            df_model_valid = dfs_model_valid_split[idx_split][idx_site]
                        else:
                            df_model_valid = None

                        if weight_train_split is not None:
                            weight = weight_train_split[idx_split][idx_site]
                        else:
                            weight = None

                        train_set, valid_sets = self.build_model_dataset(df_model_train, model, df_model_valid=df_model_valid, weight=weight)
                        gbm_q, evals_result_q = self.train(train_set, valid_sets, model) #TODO Make it possible to train starting from an existing model. E.g. LightGBM has a `input_model` option.

                        #TODO Add support for categorical_features.
                        gbm_site.append(gbm_q)
                        evals_result_site.append(evals_result_q)

                        pbar.update(1)

                    gbm_split.append(gbm_site)
                    evals_result_split.append(evals_result_site)

                gbm_model[model] = gbm_split
                evals_result_model[model] = evals_result_split

        return gbm_model, evals_result_model

    def predict(self, df_X, gbm_q, model):
        # Use trained models to predict
        #TODO Use SHAP to estimate contribution of different features. https://github.com/slundberg/shap

        def post_process(y_pred):

            if self.diff_target_with_physical:
                y_pred = y_pred+df_X['Physical_Forecast'].values

            if not self.regression_params['target_min_max'] == [None, None]:
                target_min_max = self.regression_params['target_min_max']

                if target_min_max[1] == 'clearsky':
                    idx_clearsky = y_pred > df_X['Clearsky_Forecast'].values
                    y_pred[idx_clearsky] = df_X['Clearsky_Forecast'].values[idx_clearsky]

                    if not target_min_max[0] == None:
                        y_pred = y_pred.clip(min=target_min_max[0], max=None)

                else:
                    y_pred = y_pred.clip(min=target_min_max[0], max=target_min_max[1])

            return y_pred

        # Make DataFrame to store the predictions in
        idx_q_start = 0
        columns = []
        if 'mean' in self.regression_params['type']:
            idx_q_start += 1
            columns.append('mean')

        if 'quantile' in self.regression_params['type']:
            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])
            columns.extend(['quantile{0}'.format(int(round(100*alpha))) for alpha in alpha_q])

        df_index = pd.DataFrame(index=df_X.index, columns=columns)

        # Keep all timestamps for which zenith <= prescribed value (day timestamps)
        if self.train_only_zenith_angle_below:
            idx_day = df_X['zenith'] <= self.train_only_zenith_angle_below
            idx_night = df_X['zenith'] > self.train_only_zenith_angle_below
            df_X = df_X[idx_day]

        df_y_pred_qs = {}

        y_pred_q = []
        for q in gbm_q.keys():
            if model == 'lightgbm':
                y_pred = gbm_q[q].predict(df_X[self.features])
            elif model == 'xgboost':
                if self.regression_params['type'][0] == 'mean':
                    y_pred = gbm_q[q].predict(xgb.DMatrix(df_X[self.features]))
            elif model == 'catboost':
                y_pred = gbm_q[q].predict(df_X[self.features])
            elif model == 'skboost':
                y_pred = gbm_q[q].predict(df_X[self.features])
            else:
                raise ValueError()

            y_pred = post_process(y_pred)
            y_pred_q.append(y_pred)

        # Convert list to numpy 2D-array
        y_pred_q = np.stack(y_pred_q, axis=-1)

        if 'quantile_postprocess' in self.regression_params.keys():
            if self.regression_params['quantile_postprocess'] == 'none':
                pass
            elif self.regression_params['quantile_postprocess'] == 'sorting':
                # Lazy post-sorting of quantiles
                y_pred_q = np.sort(y_pred_q, axis=-1)
            elif self.regression_params['quantile_postprocess'] == 'isotonic_regression':
                # Isotonic regression
                regressor = IsotonicRegression()
                y_pred_q = np.stack([regressor.fit_transform(alpha_q, y_pred_q[sample,:]) for sample in range(idx_q_start, y_pred_q.shape[0])])

        # Create prediction output dataframe
        df_y_pred_q = df_index
        if self.train_only_zenith_angle_below:
            df_y_pred_q[idx_day] = y_pred_q
            df_y_pred_q[idx_night] = 0
        else:
            df_y_pred_q.values[:] = y_pred_q

        df_y_pred_q = df_y_pred_q.astype('float64')

        return df_y_pred_q

    def predict_model_split_site(self, dfs_X_split, gbm_model):
        # Use trained models to predict for their corresponding split

        dfs_y_pred_model = {}
        print('Predicting...')
        with tqdm(total=len(self.model_params.keys())*len(dfs_X_split[0])*len(dfs_X_split)) as pbar:
            for model in self.model_params.keys():
                dfs_y_pred_split = []
                gbm_split = gbm_model[model]
                for dfs_X_site, gbm_site in zip(dfs_X_split, gbm_split):
                    dfs_y_pred_site = []
                    for dfs_X, gbm_q, in zip(dfs_X_site, gbm_site):
                        df_y_pred_q = self.predict(dfs_X, gbm_q, model)
                        dfs_y_pred_site.append(df_y_pred_q)

                        pbar.update(1)

                    dfs_y_pred_split.append(dfs_y_pred_site)

                dfs_y_pred_model[model] = dfs_y_pred_split

        return dfs_y_pred_model

    def calculate_loss(self, dfs_y_true_split, dfs_y_pred_model):

        print('Calculating loss...')
        if 'mean' in self.regression_params['type']:

            dfs_loss_model = {}
            for model in self.model_params.keys():
                dfs_loss_split = []
                dfs_y_pred_split = dfs_y_pred_model[model]
                for dfs_y_true_site, dfs_y_pred_site in zip(dfs_y_true_split, dfs_y_pred_split):
                    dfs_loss_site = []
                    for df_y_true, df_y_pred in zip(dfs_y_true_site, dfs_y_pred_site):
                        y_true = df_y_true[[self.target]].values
                        y_pred = df_y_pred.values

                        loss = (y_pred-y_true)**2

                        df_loss = pd.DataFrame(data=loss, index=df_y_pred.index, columns=df_y_pred.columns)

                        dfs_loss_site.append(df_loss)

                    dfs_loss_split.append(dfs_loss_site)

                dfs_loss_model[model] = dfs_loss_split

        if 'quantile' in self.regression_params['type']:
            # Evaluation using pinball loss function

            alpha_q = np.arange(self.regression_params['alpha_range'][0],
                                self.regression_params['alpha_range'][1],
                                self.regression_params['alpha_range'][2])
            a = alpha_q.reshape(1,-1)

            dfs_loss_model = {}
            for model in self.model_params.keys():

                dfs_loss_split = []
                dfs_y_pred_split = dfs_y_pred_model[model]
                for dfs_y_true_site, dfs_y_pred_site in zip(dfs_y_true_split, dfs_y_pred_split):
                    dfs_loss_site = []
                    for df_y_true, df_y_pred in zip(dfs_y_true_site, dfs_y_pred_site):
                        y_true = df_y_true[[self.target]].values
                        y_pred = df_y_pred.values

                        # Pinball loss with nan if true label is nan
                        with np.errstate(invalid='ignore'):
                            loss = np.where(np.isnan(y_true),
                                            np.nan,
                                            np.where(y_true < y_pred,
                                                    (1-a)*(y_pred-y_true),
                                                    a*(y_true-y_pred)))

                            df_loss = pd.DataFrame(data=loss, index=df_y_pred.index, columns=df_y_pred.columns)

                        dfs_loss_site.append(df_loss)

                    dfs_loss_split.append(dfs_loss_site)

                dfs_loss_model[model] = dfs_loss_split

        return dfs_loss_model

    def calculate_score(self, dfs_loss_model):

        flatten = lambda l: [item for sublist in l for item in sublist]
        score_model = {}
        for model in self.model_params.keys():
            score_model[model] = pd.concat(flatten(dfs_loss_model[model])).mean().mean()

        return score_model

    def save_result(self, params_json, result_data, result_prediction, result_model, result_evals, result_loss):

        print('Saving results...')
        trial_path = self.path_result+self.trial_name
        if os.path.exists(trial_path):
            shutil.rmtree(trial_path)
        os.makedirs(trial_path)

        file_name_json = '/params_'+self.trial_name+'.json'
        with open(trial_path+file_name_json, 'w') as file:
            json.dump(params_json, file, indent=4)

        if self.save_options['data'] == True:
            for key in result_data.keys():
                os.makedirs(trial_path+'/'+key)
                for split in range(len(result_data[key])):
                    file_name = key+'_split_{0}.csv'.format(split)
                    df = pd.concat(result_data[key][split], axis=1, keys=self.sites)
                    df.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['prediction'] == True:
            for key in result_prediction.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_prediction[key][model])):
                            file_name = key+'_'+model+'_split_{0}.csv'.format(split)
                            df = pd.concat(result_prediction[key][model][split], axis=1, keys=self.sites)
                            df.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['model'] == True:
            for key in result_model.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_model[key][model])):
                        for site in range(len(result_model[key][model][0])):
                            for q in result_model[key][model][0][0].keys():
                                if model in ['lightgbm', 'xgboost', 'catboost']:
                                    file_name = key+'_'+model+'_q_'+q+'_split_{0}_site_{1}.txt'.format(split, site)
                                    result_model[key][model][split][site][q].save_model(trial_path+'/'+key+'/'+file_name)
                                if model == 'skboost':
                                    file_name = key+'_'+model+'_q_'+q+'_split_{0}_site_{1}.pkl'.format(split, site)
                                    with open(trial_path+'/'+key+'/'+file_name, 'wb') as f:
                                        pickle.dump(result_model[key][model][split][site][q], f)
        if self.save_options['evals'] == True:
            for key in result_evals.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_evals[key][model])):
                        file_name = key+'_'+model+'_split_{0}.csv'.format(split)
                        data = result_evals[key][model][split]
                        data = {(level1_key, level2_key, level3_key): pd.Series(values)
                                for level1_key, level2_dict in zip(self.sites,data)
                                for level2_key, level3_dict in level2_dict.items()
                                for level3_key, values in level3_dict.items()}
                        df = pd.DataFrame(data)
                        df.index.name = 'trees'
                        df.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['loss'] == True:
            for key in result_loss.keys():
                os.makedirs(trial_path+'/'+key)
                for model in self.model_params.keys():
                    for split in range(len(result_loss[key][model])):
                        file_name = key+'_'+model+'_split_{0}.csv'.format(split)
                        df_loss = pd.concat(result_loss[key][model][split], axis=1, keys=self.sites)
                        df_loss.to_csv(trial_path+'/'+key+'/'+file_name)
        if self.save_options['overall_score'] == True:
            score_train_model = self.calculate_score(result_loss['dfs_loss_train'])
            score_valid_model = self.calculate_score(result_loss['dfs_loss_valid'])
            file_name = self.path_result+'/trial-scores.txt'

            for model in score_train_model.keys():
                if not os.path.exists(file_name):
                    with open(file_name, 'w') as file:
                        file.write('Name: {0}; Comment: {1}; Model: {2}; Train score {3}; valid score {4};\n'.format(self.trial_name, self.trial_comment, model, score_train_model[model], score_valid_model[model]))
                else:
                    with open(file_name, 'a') as file:
                        file.write('Name: {0}; Comment: {1}; Model: {2}; Train score {3}; valid score {4};\n'.format(self.trial_name, self.trial_comment, model, score_train_model[model], score_valid_model[model]))
        else:
            score_train_model = None
            score_valid_model = None
        print('Results saved to: '+trial_path)

        return score_train_model, score_valid_model

    def run(self, df):

        print('Running trial pipeline for trial: {0}...'.format(self.trial_name))
        dfs_X_train_split, dfs_y_train_split, dfs_model_train_split, weight_train_split = self.generate_dataset_split_site(df, split_set='train')
        dfs_X_valid_split, dfs_y_valid_split, dfs_model_valid_split, _ = self.generate_dataset_split_site(df, split_set='valid')

        gbm_model, evals_result_model = self.train_model_split_site(dfs_model_train_split, dfs_model_valid_split=dfs_model_valid_split, weight_train_split=weight_train_split)

        dfs_y_pred_train_model = self.predict_model_split_site(dfs_X_train_split, gbm_model)
        dfs_y_pred_valid_model = self.predict_model_split_site(dfs_X_valid_split, gbm_model)

        dfs_loss_train_model = self.calculate_loss(dfs_y_train_split, dfs_y_pred_train_model)
        dfs_loss_valid_model = self.calculate_loss(dfs_y_valid_split, dfs_y_pred_valid_model)

        result_data = {'dfs_X_train': dfs_X_train_split,
                    'dfs_X_valid': dfs_X_valid_split,
                    'dfs_y_train': dfs_y_train_split,
                    'dfs_y_valid': dfs_y_valid_split}
        result_model = {'gbm_model': gbm_model}
        result_evals = {'evals_result': evals_result_model}
        result_prediction = {'dfs_y_pred_train': dfs_y_pred_train_model,
                                'dfs_y_pred_valid': dfs_y_pred_valid_model}
        result_loss = {'dfs_loss_train': dfs_loss_train_model,
                    'dfs_loss_valid': dfs_loss_valid_model}

        score_train_model, score_valid_model = self.save_result(self.params_json, result_data, result_prediction, result_model, result_evals, result_loss)

        return score_train_model, score_valid_model

if __name__ == '__main__':
    params_path = sys.argv[1]
    with open(params_path, 'r', encoding='utf-8') as file:
        params_json = json.loads(file.read())

    df = load_data(params_json['path_preprocessed_data']+params_json['filename_preprocessed_data'])
    trial = Trial(params_json)
    trial.run(df)


### Script to parse reads from SRR files
### 17.03.2020

import os
import gzip
import shutil
import re
from Bio import SeqIO
from KMer_counter import kmer_read_counter

SRR_initial_loc = 'SRR_Files_GZ/'
Unzip_SRR_dest = 'SRR_Files_Unzipped/'

# UNZIP THE .GZ FILES HERE
for file in os.listdir(SRR_initial_loc):  # This will need to be modified for multiple .gz files within the dir (regex)
    # Regex to build file_out name
    pattern = '^SRR[0-9]+'
    search_string = file
    SRR_tag = re.match(pattern, search_string)
    # print(SRR_tag.group(0)) # Output test
    with gzip.open(f'{SRR_initial_loc}{file}', 'rb') as file_in, open(f'{Unzip_SRR_dest}{SRR_tag.group(0)}.fastq',
                                                                      'wb') as file_out:
        shutil.copyfileobj(file_in, file_out)

# print(os.listdir(SRR_initial_loc)) # Output test

# COUNT TOTAL NUMBER OF READS IN FASTQ HERE
count = 0
read_lst = []
for file in os.listdir(Unzip_SRR_dest):
    for read in SeqIO.parse(f'{Unzip_SRR_dest}{file}', 'fastq'):
        read_lst.append(read)
        count += 1

# Determine average read length from fastq file here
print('%i reads' % count)

# REMOVE READS WITH PHRED SCORE < 20 HERE
good_reads = []
for file in os.listdir(Unzip_SRR_dest):
    pattern = '^SRR[0-9]+' # Start RegEx
    search_string = file
    SRR_tag = re.match(pattern, search_string) # End RegEx
    for read in SeqIO.parse(f'{Unzip_SRR_dest}{file}', 'fastq'):
        if min(read.letter_annotations['phred_quality']) >= 20:
            good_reads.append(read)
    count = SeqIO.write(good_reads, f'{SRR_tag.group(0)}_goodPHREDscores.fastq', 'fastq')
print('Saved %i reads with PHRED score >= 20' % count)

quality_seq = []
for seq in good_reads:
    seq_data = str(seq.seq)
    quality_seq.append(seq_data)

print(quality_seq) # Output test
# print(type(quality_seq[0])) # Output test

iteration = 1
for seq in quality_seq:
    read_names = ['SRR020192_read', str(iteration)] # Implement regex here too for read_names[0]
    kmer_read_counter(seq, 9, ''.join(read_names), ''.join(read_names))
    iteration += 1


import sys
import os, errno
import argparse
import shutil
sys.path.insert(0, '.\\lib')
sys.path.insert(0, '.\\lines')

import scipy.io
from skimage import io
import time, string

from lib.UUID import *
from lib.render import *
from lib.read_write import *
from lib.processing import *
import lines.process_cell_channel as cell_line
import lines.process_mito_channel as mito_line
from lines import mito_counter

def blockPrint():
	sys.stdout = open(os.devnull, 'w')

def enablePrint():
	sys.stdout = sys.__stdout__

def get_args(args):
	parser = argparse.ArgumentParser(description = 'Script for analyzing mitochondria skeletonization')
	parser.add_argument('-r',
						dest = 'read_dir',
						help = 'Raw data read directory',
						required = False,
						default = ".\\testing_environment\\")
	parser.add_argument('-w',
						dest = 'save_dir',
						help = 'Save directory for segmentation and skeletonization data',
						required = False,
						default = ".\\testing_environment\\test_run\\")

	options = vars(parser.parse_args())
	if os.path.exists(options['save_dir']):
		if options['save_dir'] == ".\\test_run":
			shutil.rmtree(options['save_dir'])
	return options

def main(args):
	os.system('cls' if os.name == 'nt' else 'clear')
	options = get_args(args)
	root_read_dir = options['read_dir']
	save_dir = options['save_dir']

	print("> Parent Read Directory : {}\r".format(root_read_dir))
	print("> Save Directory : {}\r".format(save_dir))

	save_dir_cell = os.path.join(save_dir, 'cell')
	save_dir_mito = os.path.join(save_dir, 'mito')
	save_dir_anal = os.path.join(save_dir, 'analysis')

	mkdir_check(save_dir_cell)
	mkdir_check(save_dir_mito)
	mkdir_check(save_dir_anal)

	start = time.time()
	filenames = get_img_filenames(root_read_dir)
	num_images = len(filenames)
	end = time.time()
	print("> {} images detected, time taken: {}".format(num_images, end - start))

	mito_stats = []
	cell_stats = []

	print("> Processing IDs saved here: {}\r".format(save_dir))

	file_list_ID = open(os.path.join(save_dir, "UUID_LUT.txt"),'w')
	for UID, img_name, img_fname, path_diff, img_loc, img_path in filenames:
		file_list_ID.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(UID, img_name, img_fname, path_diff, img_loc, img_path))
	file_list_ID.close()

	img_num = 1
	for UID, img_name, _, _, _, img_path in filenames:
		print("> ==========================================================================================\r")
		print("\r> Currently Processing : {}\r".format(img_name))
		print("> \tImage Unique ID: {}\r".format(UID))
		print("> \tImage/Total Number of Images: {}/{}\r".format(img_num, num_images))
		if '1488' in img_name:
			# continue
			print("> Image ID: 1488 - Cell TD\r")
			blockPrint()
			start = time.time()
			cell_line.analyze(UID, img_path, save_dir_cell)
			end = time.time()
			enablePrint()
			print("> Time to Compete: {}".format(end - start))
			mito_stats.append(end - start)
			img_num += 1

		elif '2561' in img_name:
			# continue
			print("> Image ID: 2561 - Mitochondria\r")
			blockPrint()
			start = time.time()
			mito_line.analyze(UID, img_path, save_dir_mito)
			end = time.time()
			enablePrint()
			print("> Time to Compete: {}".format(end - start))
			cell_stats.append(end - start)
			img_num += 1

	print("> ==========================================================================================\r")
	print("> Prelim Analysis completed")
	save_data(mito_stats, "mito_processing_RT", save_dir)
	save_data(cell_stats, "cell_processing_RT",  save_dir)

	# Start merge of MC_analyzer
	cell_filelist = get_just_filenames(save_dir_cell, suffix = '_dat.mat')
	mito_filelist = get_just_filenames(save_dir_mito, suffix = '.mat')

	UUID_datatable = read_txt_file(os.path.join(save_dir, "UUID_LUT.txt"))

	C_M_UUID_pairs, UUID_pairs = create_pairTable(cell_filelist, UUID_datatable, save_dir)

	filename_pairs = []
	for cell_UUID, mito_UUID in UUID_pairs:
		filename_pairs.append(["C_" + cell_UUID + "_dat.mat",
								"M_" + mito_UUID + "_bin.mat",
								"M_" + mito_UUID + "_skel.mat"])
	print("> Creating UUID Filename Pairs")
	write_list_txt(save_dir_anal, "Cell_mito_UUID_Pairs.txt", C_M_UUID_pairs)
	write_list_txt(save_dir_anal, "UUID_paired_filenames.txt", filename_pairs)

	for filenames in filename_pairs:
		save_fileID = get_UUID(filenames[0])
		cell_img = scipy.io.loadmat(os.path.join(save_dir_cell, filenames[0]))['data']
		mito_stack = scipy.io.loadmat(os.path.join(save_dir_mito, filenames[1]))['data']
		mito_skel = scipy.io.loadmat(os.path.join(save_dir_mito, filenames[2]))['data']

		labeled_mito_bin = stack_multiplier(cell_img, mito_stack)
		labeled_mito_skel = stack_multiplier(cell_img, mito_skel)

		save_data(labeled_mito_bin, "CM_" + save_fileID + "_bin", save_dir_anal)
		save_data(labeled_mito_skel, "CM_" + save_fileID + "_skel", save_dir_anal)

	# Start merge of mitocounter
	mito_counter.main(save_dir)
if __name__ == "__main__":
	main(sys.argv)


import pygame
import random as rnd
import scipy as sp

from Variables import *

from Goals import *
from Boid import *

from Noise import Va

from Room import *

pygame.init()                                           # Initialise game

clock = pygame.time.Clock()                             # Set clock

boidfunc(Divide)                                        # Check for special scenarios
roomfunc(Room)

count=0                                                 # Set Count
Count=[]

mainloop = True                                         # Run game

while mainloop:

    clock.tick(30)                                          # Set FPS

    for event in pygame.event.get():                        # Quitting mechanism
        if event.type == pygame.QUIT:
            mainloop = False
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                mainloop = False
        if event.type == pygame.KEYDOWN:                    # Screenshot mechanism
            if event.key == pygame.K_s:
                pygame.image.save(screen,"Screenshot.jpg")

    count += 1                                              # Count each frame
    Count.append(count)

    if len(all_sprites)==0:
        mainloop = False

    if Frames != 0:                                         # Automated quitting function
        if count == Frames:
            mainloop = False
    if mainloop == False:                                       # Print Frame Count
        print("Total Frame Count: {0}".format ( count ))

        for sprite in goals:                                    # Print boids at each goal
            print("Goal at {0}:".format(sprite.rect.x))
            print("{0} boids collected".format (sprite.number))

    for sprite in all_sprites:                              # Check wall collisions
        sprite.collide(room)

    for sprite in leaders:
        sprite.collide(room)

    all_sprites.update()                                    # Get new movement direction
    leaders.update()
    informed.update()

    pygame.display.set_caption("Frame {0}".format(count))   # Frame count as window title

    screen.fill((255,255,255))                              # Recolour screen to remove sprite traces

    all_sprites.draw(screen)                                # Draw all objects onto the screen
    leaders.draw(screen)
    goals.draw(screen)
    room.draw(screen)
    informed.draw(screen)

    pygame.display.flip()                                   # Update screen

pygame.quit()


import argparse
from getpass import getpass
import logging
import sys

from . api import QuayCon
from . errors import (
    MissingTokenError,
    UnknownOrganization,
)
from . utils import QUAYIO_REGISTRY
from . config import load_config
from . utils import ask_confirmation, parse_repository

def touch_command(repository, config_file, **kwargs):
    registry, org, name, tag = parse_repository(repository)
    config = load_config(config_file)
    quaycon = QuayCon(config)
    return list(quaycon.touch(registry, org, name, tag, **kwargs))

def discover_command(organizations, config_file, force=False, **kwargs):
    config = load_config(config_file)
    quaycon = QuayCon(config)
    for org in organizations:
        try:
            quaycon.discover(org, incremental=not force)
        except (MissingTokenError, UnknownOrganization):  # pragma: no cover
            if not kwargs.setdefault('interactive', True):
                raise
            msg = "Unknown organization '{}'. Do you wish to add it?"
            if ask_confirmation(msg.format(org)):
                token = getpass("Please enter API token: ")
                quaycon.add_organization(QUAYIO_REGISTRY, org, token)
                quaycon.save()
                quaycon.discover(org)
    if kwargs.setdefault('interactive', True):
        quaycon.save()

def version_command(*args, **kwargs):  # pragma: no cover
    from . import __version__
    version = '.'.join(map(str, __version__))
    sys.stdout.write(version + '\n')

def main(*args):  # pragma: no cover
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-v', '--verbose',
        action='count',
        help='Verbose mode, -vv for more details'
    )
    parser.add_argument(
        '-c', '--config',
        dest='config_file',
        help='Specify configuration file'
    )
    subparsers = parser.add_subparsers(help='sub-command help')
    version_parser = subparsers.add_parser(
        'version', help='print software version'
    )
    version_parser.set_defaults(func=version_command)

    touch_parser = subparsers.add_parser(
        'touch', help='trigger dependant builds'
    )
    touch_parser.add_argument(
        '-w', '--wait',
        type=int,
        default=0,
        help='Delay in seconds between build status checks. Default is 0'
             'meaning that command does not wait for build completion'
    )
    touch_parser.add_argument(
        '-r', '--recursive',
        action='store_true',
        default=False,
        help='Rebuild entire sub-tree dependencies.'
    )
    touch_parser.add_argument('repository')
    touch_parser.set_defaults(func=touch_command)

    discover_parser = subparsers.add_parser(
        'discover', help='Discover dependencies between repositories'
    )
    discover_parser.add_argument(
        '-f', '--force', action='store_true', default=False,
        help='Rescan the entire organization, '
             'not only new repositories and triggers'
    )
    discover_parser.add_argument(
        'organizations', metavar='ORG', nargs='+',
        default=None, help='Organizations sub-set to inspect'
    )
    discover_parser.set_defaults(func=discover_command)

    args = parser.parse_args()
    logging_level = logging.WARN
    if args.verbose == 1:
        logging_level = logging.INFO
    elif args.verbose == 2:
        logging_level = logging.DEBUG
    elif args.verbose > 2:
        logging_level = logging.TRACE
    logging.basicConfig(level=logging_level)
    for logger in ['requests']:
        logging.getLogger(logger).setLevel(logging.WARNING)
    args.func(**vars(args))


from django.contrib import admin
from django.urls import path , include

urlpatterns = [
    path('admin/', admin.site.urls),

    path('', include('Profile.urls', namespace="Profile")),
    path('api/', include('api.urls', namespace="api")),
    path('', include('docdata.urls', namespace="docdata")),

]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 12 17:23:43 2018

@author: ppxee
"""

### Import required libraries ###
import matplotlib.pyplot as plt #for plotting
import matplotlib.animation as animation
#plt.rcParams['animation.ffmpeg_path'] = '/usr/bin/ffmpeg'
from astropy.io import fits #for handling fits
#from astropy.table import Table #for handling tables
from photutils import CircularAperture, aperture_photometry
import numpy as np #for handling arrays
#import math
#from astropy.stats import median_absolute_deviation
#import vari_funcs_no06 #my module to help run code neatly
plt.close('all') #close any open plots
hdr08B = fits.getheader('Images/extra_clean_no06_UDS_08B_K.fits') # random year (same in all)
const = -hdr08B['CD1_1'] # constant that defines unit conversion for FWHM

n=1
binarr = range(11)
#jmag = np.empty(len(binarr))
#flarejmag = np.empty(len(binarr))
#jflux = np.empty(len(binarr))
#flarejflux = np.empty(len(binarr))
sejmag = np.empty(len(binarr))
sejmagerr = np.empty(len(binarr))
for n in binarr:
    if n == 0:
        tbdata = fits.open('SE_outputs_yearstacks/07B_output_J.fits')[1].data
        obdata = tbdata[tbdata['NUMBER']==57824]
        sejmag[n] = obdata['MAG_APER'][0][4]
        sejmagerr[n] = obdata['MAGERR_APER'][0][4]
    elif n == binarr[-2]:
#        tbdata = fits.open('SE_outputs_yearstacks/09B_output_J.fits')[1].data
#        obdata = tbdata[tbdata['NUMBER']==62243]
#        sejmag[n] = obdata['MAG_APER'][0][4]
#        sejmagerr[n] = obdata['MAGERR_APER'][0][4]
        sejmag[n] = np.nan
        sejmagerr[n] = np.nan
    elif n == binarr[-1]:
#        tbdata = fits.open('SE_outputs_yearstacks/10B_output_J.fits')[1].data
##        obdata = tbdata[tbdata['NUMBER']==62243]
#        sejmag[n] = tbdata['MAG_APER'][62243][4]
#        sejmagerr[n] = tbdata['MAGERR_APER'][62243][4]
        sejmag[n] = np.nan
        sejmagerr[n] = np.nan
    else:
        tbdata = fits.open('SE_outputs_shortstacks/08B_J_output_cam1_8bin_'+str(n-1)+'.fits')[1].data
        obdata = tbdata[tbdata['NUMBER']==13363]
        sejmag[n] = obdata['MAG_APER'][0][4]
        sejmagerr[n] = obdata['MAGERR_APER'][0][4]

binarr = range(13)
#kmag = np.empty(len(binarr))
#flarekmag = np.empty(len(binarr))
#kflux = np.empty(len(binarr))
#flarekflux = np.empty(len(binarr))
sekmag = np.empty(len(binarr))
sekmagerr = np.empty(len(binarr))
for n in binarr:
    if n == 0:
        tbdata = fits.open('SE_outputs_yearstacks/07B_output.fits')[1].data
        obdata = tbdata[tbdata['NUMBER']==62242]
        sekmag[n] = obdata['MAG_APER'][0][4]
        sekmagerr[n] = obdata['MAGERR_APER'][0][4]
    elif n == binarr[-2]:
        tbdata = fits.open('SE_outputs_yearstacks/09B_output.fits')[1].data
        obdata = tbdata[tbdata['NUMBER']==62242]
        sekmag[n] = obdata['MAG_APER'][0][4]
        sekmagerr[n] = obdata['MAGERR_APER'][0][4]
    elif n == binarr[-1]:
        tbdata = fits.open('SE_outputs_yearstacks/10B_output.fits')[1].data
        obdata = tbdata[tbdata['NUMBER']==62242]
        sekmag[n] = obdata['MAG_APER'][0][4]
        sekmagerr[n] = obdata['MAGERR_APER'][0][4]
    else:
        tbdata = fits.open('SE_outputs_shortstacks/08B_output_cam1_10bin_'+str(n-1)+'.fits')[1].data
        obdata = tbdata[tbdata['NUMBER']==15100]
        sekmag[n] = obdata['MAG_APER'][0][4]
        sekmagerr[n] = obdata['MAGERR_APER'][0][4]

### Get H data ###
binarr = range(9)
sehmag = np.empty(len(binarr))
sehmagerr = np.empty(len(binarr))
for n in binarr:
    if n == 0:
#        tbdata = fits.open('SE_outputs_yearstacks/07B_output.fits')[1].data
#        obdata = tbdata[tbdata['NUMBER']==62242]
#        sekmag[n] = obdata['MAG_APER'][0][4]
#        sekmagerr[n] = obdata['MAGERR_APER'][0][4]
        sehmag[n] = np.nan
        sehmagerr[n] = np.nan
    elif n == binarr[-2]:
#        tbdata = fits.open('SE_outputs_yearstacks/09B_output.fits')[1].data
#        obdata = tbdata[tbdata['NUMBER']==62242]
#        sekmag[n] = obdata['MAG_APER'][0][4]
#        sekmagerr[n] = obdata['MAGERR_APER'][0][4]
        sehmag[n] = np.nan
        sehmagerr[n] = np.nan
    elif n == binarr[-1]:
#        tbdata = fits.open('SE_outputs_yearstacks/10B_output.fits')[1].data
#        obdata = tbdata[tbdata['NUMBER']==62242]
#        sekmag[n] = tbdata['MAG_APER'][62243][4]
#        sekmagerr[n] = tbdata['MAGERR_APER'][62243][4]
        sehmag[n] = np.nan
        sehmagerr[n] = np.nan
    else:
        tbdata = fits.open('SE_outputs_shortstacks/08B_H_output_cam1_6bin_'+str(n-1)+'.fits')[1].data
        obdata = tbdata[tbdata['NUMBER']==11900]
        sehmag[n] = obdata['MAG_APER'][0][4]
        sehmagerr[n] = obdata['MAGERR_APER'][0][4]

### figure out date stamps ###
### create list of date stamps between 20/9 and 25/11
initdates = np.empty(67).astype(str)
for x in range(67):
    if x <= 10:
        initdates[x] = str(20+x)+'/9'
    elif x > 10 and x <= 41:
        initdates[x] = str(x-10)+'/10'
    elif x > 41:
        initdates[x] = str(x-41)+'/11'

dates = initdates
#sem07 = np.repeat('07B',30)
#dates = np.append(sem07, initdates)
#sem09 = np.repeat('09B',30)
#dates = np.append(dates, sem09)
#sem10 = np.repeat('10B',30)
#dates = np.append(dates, sem10)
#Jdates = ['3/10','16/10','18/10','18/10','19/10','19/10',]
#Hdates = ['20/9','21/9','22/9','27/9','1/10','10/10','21/10','27/10','1/11',
#          '8/11','16/11','21/11']
Jxdata = np.array([18, 28, 29, 30, 33, 41, 42, 49])#+30
#Jxdata = np.append(0, Jxdata)
#Jxdata = np.append(Jxdata, 220)
Hxdata = np.array([0,3,16,35,45,61])#+30
#Hxdata = np.append(0, Hxdata)
#Hxdata = np.append(Hxdata, 220)
Kxdata = np.array([0, 1, 9, 11, 12, 13, 16, 21, 39, 56])#+30
#Kxdata = np.append(0, Kxdata)
#Kxdata = np.append(Kxdata, 220)
initticks = np.array([0, 10, 20, 30, 40, 50, 60])
ticks= initticks#+30
#ticks = np.append(0, ticks)
#ticks = np.append(ticks, [126])#,156])

#plt.figure(figsize=[17,7])
#plt.plot([0,260,290], [flarejflux[0],flarejflux[-2],flarejflux[-1]], 'bs')#, label='J semester flux')
#plt.plot([0,260,290], [flarehflux[0],flarehflux[-2],flarehflux[-1]], 'ks')#, label='H semester flux')
#plt.plot([0,260,290], [flarekflux[0],flarekflux[-2],flarekflux[-1]], 'rs')#, label='K semester flux')
#
#plt.plot(Jxdata, flarejflux[1:-2], 'bo', label='J flare flux')
#plt.plot(Hxdata, flarehflux[1:-2], 'ko', label='H flare flux')
#plt.plot(Kxdata, flarekflux[1:-2], 'ro', label='K flare flux')
#plt.legend()
#plt.xticks(ticks, dates[ticks], rotation='vertical')
#plt.ylabel('Flux of flare in 1.4" aperture')
#plt.xlabel('Date')
#plt.tight_layout()
#
#plt.figure(figsize=[17,7])
#plt.plot([0,260,290], [jflux[0],jflux[-2],jflux[-1]], 'bs')#, label='J semester flux')
#plt.plot([0,260,290], [hflux[0],hflux[-2],hflux[-1]], 'ks')#, label='H semester flux')
#plt.plot([0,260,290], [kflux[0],kflux[-2],kflux[-1]], 'rs')#, label='K semester flux')
#plt.plot(Jxdata, jflux[1:-2], 'bo', label='J object flux')
#plt.plot(Hxdata, hflux[1:-2], 'ko', label='H object flux')
#plt.plot(Kxdata, kflux[1:-2], 'ro', label='K object flux')
#plt.legend()
#plt.xticks(ticks, dates[ticks], rotation='vertical')
#plt.ylabel('Flux of object in 3" aperture')
#plt.xlabel('Date')
#plt.tight_layout()
##
##
##plt.figure(figsize=[17,7])
##plt.plot(Jxdata, flarejflux, 'o', label='J flare flux')
##plt.legend()
##plt.xticks(ticks, dates[ticks], rotation='vertical')
##
##plt.figure(figsize=[17,7])
##plt.plot(Hxdata, flarehflux, 'o', label='H flare flux')
##plt.legend()
##plt.xticks(ticks, dates[ticks], rotation='vertical')
##
##plt.figure(figsize=[17,7])
##plt.plot(Kxdata, flarekflux, 'o', label='K flare flux')
##plt.legend()
##plt.xticks(ticks, dates[ticks], rotation='vertical')
##
#plt.figure(figsize=[7,7])
#plt.plot([0,126,156], [flarejmag[0],flarejmag[-2],flarejmag[-1]], 'bs')#, label='J semester flux')
#plt.plot([0,126,156], [flarehmag[0],flarehmag[-2],flarehmag[-1]], 'ks')#, label='H semester flux')
#plt.plot([0,126,156], [flarekmag[0],flarekmag[-2],flarekmag[-1]], 'rs')#, label='K semester flux')
#plt.plot(Jxdata, flarejmag[1:-2], 'bo', label='J flare mag')
#plt.plot(Hxdata, flarehmag[1:-2], 'ko', label='H flare mag')
#plt.plot(Kxdata, flarekmag[1:-2], 'ro', label='K flare mag')
##plt.legend()
#plt.xticks(ticks, dates[ticks], rotation='vertical')
#plt.ylabel('Magnitude of flare in 1.4" aperture')
#plt.xlabel('Date')
#plt.gca().invert_yaxis()
#plt.tight_layout()
#
#plt.figure(figsize=[7,7])
##plt.errorbar([0,126,156], [sejmag[0],sejmag[-2],sejmag[-1]],
##             [sejmagerr[0],sejmagerr[-2],sejmagerr[-1]], 'fmt=bs')#, label='J semester flux')
#plt.errorbar(0, sejmag[0], sejmagerr[0], fmt='bs')#, label='J semester flux')
##plt.plot([0,126,156], [hmag[0],hmag[-2],hmag[-1]], 'ks')#, label='H semester flux')
##plt.plot([0,126,156], [kmag[0],kmag[-2],kmag[-1]], 'rs')#, label='K semester flux')
#plt.errorbar([0,126,156], [sekmag[0],sekmag[-2],sekmag[-1]],
#             [sekmagerr[0],sekmagerr[-2],sekmagerr[-1]], fmt='rs')#, label='J semester flux')
#plt.errorbar(Jxdata, sejmag[1:-2], sejmagerr[1:-2], fmt='bo', label='J object mag')
#plt.errorbar(Hxdata, sehmag[1:-2], sehmagerr[1:-2], fmt='ko', label='H object mag')
#plt.errorbar(Kxdata, sekmag[1:-2], sekmagerr[1:-2], fmt='ro', label='K object mag')
#plt.legend()
#plt.xticks(ticks, dates[ticks], rotation='vertical')
#plt.ylabel('Magnitude of object in 3" aperture')
#plt.xlabel('Date')
#plt.gca().invert_yaxis()
#plt.tight_layout()
#
#plt.figure(figsize=[7,7])
#plt.plot([0,126,156], [flarejmag[0],flarejmag[-2],flarejmag[-1]], 'bs')#, label='J semester flux')
#plt.plot(Jxdata, flarejmag[1:-2], 'bo', label='J flare mag')
#plt.legend()
#plt.xticks(ticks, dates[ticks], rotation='vertical')
#axes = plt.gca()
#ylims = axes.get_ylim()
#ymid = (ylims[1]+ylims[0])/2
#plt.ylim(ymin=ymid-1.5, ymax=ymid+1.5)
#plt.ylabel('J magnitude of flare in 1.4" aperture')
#plt.xlabel('Date')
#plt.gca().invert_yaxis()
#plt.tight_layout()
#
#plt.figure(figsize=[7,7])
#plt.plot([0,126,156], [flarehmag[0],flarehmag[-2],flarehmag[-1]], 'ks')#, label='H semester flux')
#plt.plot(Hxdata, flarehmag[1:-2], 'ko', label='H flare mag')
#plt.legend()
#plt.xticks(ticks, dates[ticks], rotation='vertical')
#axes = plt.gca()
#ylims = axes.get_ylim()
#ymid = (ylims[1]+ylims[0])/2
#plt.ylim(ymin=ymid-1.5, ymax=ymid+1.5)
#plt.ylabel('H magnitude of flare in 1.4" aperture')
#plt.xlabel('Date')
#plt.gca().invert_yaxis()
#plt.tight_layout()
#
#plt.figure(figsize=[7,7])
#plt.plot([0,126,156], [flarekmag[0],flarekmag[-2],flarekmag[-1]], 'rs')#, label='K semester flux')
#plt.plot(Kxdata, flarekmag[1:-2], 'ro', label='K flare mag')
#plt.legend()
#plt.xticks(ticks, dates[ticks], rotation='vertical')
#axes = plt.gca()
#ylims = axes.get_ylim()
#ymid = (ylims[1]+ylims[0])/2
#plt.ylim(ymin=ymid-1.5, ymax=ymid+1.5)
#plt.ylabel('K magnitude of flare in 1.4" aperture')
#plt.xlabel('Date')
#plt.gca().invert_yaxis()
#plt.tight_layout()

#%% calculate rest frame mags
from astropy.cosmology import FlatLambdaCDM
from astropy import units as u

### Define cosmology ###
cosmo = FlatLambdaCDM(H0=70, Om0=0.3)

z = 1.51 #approximately
DL = cosmo.luminosity_distance(z)
DL = DL.to(u.pc)

M_g = sejmag - 5*(np.log10(DL.value)-1) - 2.5*np.log10(1+z)
M_r = sehmag - 5*(np.log10(DL.value)-1) - 2.5*np.log10(1+z)
M_z = sekmag - 5*(np.log10(DL.value)-1) - 2.5*np.log10(1+z)

M_g_err = M_g * (sejmagerr/sejmag)
M_r_err = M_r * (sehmagerr/sehmag)
M_z_err = M_z * (sekmagerr/sekmag)

### plot new curves ####
plt.figure(figsize=[7,7])
#plt.plot([0,126,156], [M_g[0],M_g[-2],M_g[-1]], 'bs')#, label='J semester flux')
#plt.plot([0,126,156], [M_r[0],M_r[-2],M_r[-1]], 'ks')#, label='H semester flux')
#plt.plot([0,126,156], [M_z[0],M_z[-2],M_z[-1]], 'rs')#, label='K semester flux')
#plt.plot([0,126], [M_g[0],M_g[-1]], 'bs', mfc='None')#, label='J semester flux')
#plt.plot([0,126], [M_r[0],M_r[-1]], 'ks', mfc='None')#, label='H semester flux')
#plt.plot([0,126], [M_z[0],M_z[-1]], 'rs', mfc='None')#, label='K semester flux')
plt.errorbar(Jxdata, M_g[1:-2], M_g_err[1:-2], fmt='bo', label='M_g object mag')
plt.errorbar(Hxdata, M_r[1:-2], M_r_err[1:-2], fmt='ko', label='M_r object mag')
plt.errorbar(Kxdata, M_z[1:-2], M_z_err[1:-2], fmt='ro', label='M_z object mag')
plt.legend()
plt.xticks(ticks, dates[ticks], rotation='vertical')
plt.ylabel('Restframe magnitude of flare in 3" aperture')
plt.xlabel('Date')
plt.gca().invert_yaxis()
plt.tight_layout()


'''
Script for transforming our jpg images dataset into a npz file usable by yolo.

Written by Luca Derumier.
Version 1.0 - May 2020.
'''
import argparse
import numpy as np
import os
import pickle
import PIL.Image
from skimage.color import gray2rgb
from utils import load_annotation

#########################################################
################### Parsing arguments ###################
#########################################################

argparser = argparse.ArgumentParser(
    description="Transforms an image dataset into a npz file usable by yolo.")

argparser.add_argument(
    '-d',
    '--data_path',
    help="path to the folder containing training and validation data folders.",
    default=os.path.join('pelvis_scan','data'))

########################################################
######################### Main #########################
########################################################

def _main(args):
    # Raw arguments from parser
    data_path = args.data_path

    # dictionary of the organs and their index in the classes.txt file
    label_dict = {'bladder' : 0, 'rectum' : 1, 'prostate' : 2}

    # Environment variables
    sets = [x for x in ['train','val','test'] if x in os.listdir(data_path)]
    if len(sets) == 0:
        raise ValueError('No set to extract the data from.')

    for set in sets:
        dataset_dir = os.path.join(data_path,set)
        annotation = 'annotations_'+set+'.p'
        filename = 'pelvis_data_'+set+'.npz'
        dict = load_annotation(os.path.join(dataset_dir,annotation))
        yolo_dataset(dict,dataset_dir,filename,label_dict)

#############################################
################### Utils ###################
#############################################

def yolo_dataset(dict,path,npz_filename,label_dict,shuffle = False):
    '''Transform our dictionary and images to npz data file that yolo can use.

    Inputs:
        dict: dictionary with the bounding box annotations
        path: the paht to the folder that has the images.
        npz_filename: the name we want for our output file.
        label_dict: dictionary that contains the organs as keys and their correspong class values as values.
        shuffle: wether to shuffle the dataset or not.'''

    dir_list = sorted(os.listdir(path))
    images = []
    size = len([f for f in dir_list if(f.endswith('.jpg'))])
    image_labels = []

    # Loads images and stores bounding boxes
    for filename in dir_list:
        if(filename.endswith('.jpg')):

            # Loads the image and
            image = PIL.Image.open(os.path.join(path,filename))

            # Convert the image to RGB foramt (needed for yolo)
            rgbimg = PIL.Image.new("RGB", image.size)
            rgbimg.paste(image)
            img = np.array(rgbimg, dtype=np.uint8)

            images.append(img)

            all_boxes = []

            for organ,index in label_dict.items():
                if(organ in dict[filename]['bb'].keys() and dict[filename]['bb'][organ] is not None):
                    print('Extracting {} box for {}'.format(organ,filename))
                    box = dict[filename]['bb'][organ]
                    organ_index = label_dict[organ]
                    xA = box[1]
                    yA = box[0]
                    xB = box[3]
                    yB = box[2]

                    if(xA > 0 or yA > 0 or xB < image.width-1 or yB != image.height-1):
                        new_box = np.array([organ_index,xA,yA,xB,yB])
                        all_boxes.append(new_box)

            print(all_boxes)
            image_labels.append(np.array(all_boxes))

    # Convert list to numpy array for saving
    images = np.array(images, dtype=np.uint8)
    image_labels = np.array(image_labels)

    # Format checking
    print('image shape : {}'.format(images.shape))
    print('labels shape: {}'.format(image_labels.shape))
    print('image labels [0] shape: {}'.format(image_labels[0].shape))

    # Shuffle dataset
    if shuffle:
        np.random.seed(13)
        indices = np.arange(len(images))
        np.random.shuffle(indices)
        images, image_labels = images[indices], image_labels[indices]

    # Save int npz file usable by yolo
    np.savez(os.path.join(path,npz_filename), images=images, boxes=image_labels)
    print('Data saved into {}'.format(os.path.join(path,npz_filename)))

def patients_full(path):
    ''' Returns the list of all patients that are in the path folder (full images).

    Input:
        path: path to the full images folder

    Returns:
        patients: list of the patients numbers
    '''

    dir_list = sorted(os.listdir(path))

    patients = []
    for folder in dir_list:
        if(folder.startswith('charleroi_')):
            patients.append(int(folder[10:]))

    return patients

def patients(path):
    ''' Returns the list of all patients that are used int the path folder (image slices).

    Input:
        path: path to the data folder

    Returns:
        patients: list of the patients numbers
    '''

    dir_list = [f for f in sorted(os.listdir(path)) if f.endswith('jpg')]
    patients = []

    for file in dir_list:
        string_list = file.split('-')
        num = str(string_list[2])
        if num not in patients:
            patients.append(num)

    return patients

########################################################
######################### Main #########################
########################################################

if __name__ == '__main__':
    args = argparser.parse_args()
    _main(args)


import ujson
from pathlib import Path
from typing import List

import rdflib
from rdflib.namespace import RDF, RDFS, OWL
from whoosh.fields import Schema, TEXT, ID, analysis
from whoosh.index import create_in, open_dir
from whoosh.qparser import QueryParser

from api.config import HOME_DIR

class OntologyService:
    """TODO: doesn't support multi-users yet"""
    instance = None

    def __init__(self):
        self.schema = Schema(uri=ID(stored=True), label=TEXT(
            analyzer=analysis.NgramWordAnalyzer(minsize=2, maxsize=10), stored=True, spelling=True))
        self.ont_classes = {}
        self.ont_predicates = {}
        self.prefixes = {}
        self.namespaces = {}

        self.index_dir: Path = (HOME_DIR / "index_dir").absolute()
        if not self.index_dir.exists():
            self.index_dir.mkdir()
            (self.index_dir / "ont_class").mkdir()
            (self.index_dir / "ont_predicate").mkdir()
            self.ix_ont_class = create_in(
                str(self.index_dir / "ont_class"), self.schema)
            self.ix_ont_predicate = create_in(
                str(self.index_dir / "ont_predicate"), self.schema)
        else:
            self.ix_ont_class = open_dir(str(self.index_dir / "ont_class"))
            self.ix_ont_predicate = open_dir(
                str(self.index_dir / "ont_predicate"))

        if (self.index_dir / "ontologies.json").exists():
            with open(str(self.index_dir / "ontologies.json"), "r") as f:
                o = ujson.load(f)
                self.ont_classes = o['ont_classes']
                self.ont_predicates = o['ont_predicates']
                self.prefixes = o['prefixes']
                self.namespaces = o['namespaces']

    @staticmethod
    def get_instance():
        if OntologyService.instance is None:
            OntologyService.instance = OntologyService()
        return OntologyService.instance

    def has_ontology(self, namespace: str):
        return namespace in self.namespaces

    def add_ontology(self, raw_ont_str: str, namespace: str, prefix: str, format: str):
        g = rdflib.Graph()
        g.parse(data=raw_ont_str, format=format)

        cprefix = prefix + ":"
        new_classes, new_predicates = [], []

        for u in set(g.subjects(RDF.type, RDFS.Class)).union(
                set(g.subjects(RDF.type, OWL.Class))):
            u = str(u)
            if u.startswith(namespace) and u not in self.ont_classes:
                self.ont_classes[u] = {
                    "URI": u,
                    "namespace": namespace,
                    "prefix": prefix,
                    "shortURI": u.replace(namespace, cprefix)
                }
                new_classes.append(u)

        for u in set(g.subjects(RDF.type, RDF.Property)):
            u = str(u)
            if u.startswith(namespace) and u not in self.ont_predicates:
                self.ont_predicates[u] = {
                    "URI": u,
                    "namespace": namespace,
                    "prefix": prefix,
                    "shortURI": u.replace(namespace, cprefix)
                }
                new_predicates.append(u)

        if len(new_classes) + len(new_predicates) > 0:
            self.prefixes[prefix] = namespace
            self.namespaces[namespace] = prefix

            if not (self.index_dir / f"{prefix}.log").exists():
                # index the ontology
                writer = self.ix_ont_class.writer()
                for u in new_classes:
                    writer.add_document(
                        uri=u, label=self.ont_classes[u]['shortURI'])
                writer.commit()
                writer = self.ix_ont_predicate.writer()
                for u in new_predicates:
                    writer.add_document(
                        uri=u, label=self.ont_predicates[u]['shortURI'])
                writer.commit()
                open(str(self.index_dir / f"{prefix}.log"), 'a').close()
                with open(str(self.index_dir / "ontologies.json"), "w") as f:
                    ujson.dump({
                        "prefixes": self.prefixes,
                        "namespaces": self.namespaces,
                        "ont_classes": self.ont_classes,
                        "ont_predicates": self.ont_predicates
                    }, f)

        return len(new_classes), len(new_predicates)

    def search_ont_class(self, query: str) -> List[dict]:
        results = []
        with self.ix_ont_class.searcher() as searcher:
            parser = QueryParser("label", self.schema)
            for res in searcher.search(parser.parse(query)):
                results.append(self.ont_classes[res['uri']])
        return results

    def search_ont_predicate(self, query: str) -> List[dict]:
        results = []
        with self.ix_ont_predicate.searcher() as searcher:
            parser = QueryParser("label", self.schema)
            for res in searcher.search(parser.parse(query)):
                results.append(self.ont_predicates[res['uri']])
        return results


import pygame, sys
from tileC import Tile

def interaction(screen,survivor):

	Mpos = pygame.mouse.get_pos() # returns [x, y]
	Mx = Mpos[0] / Tile.width #0=x
	My = Mpos[1] / Tile.height #1=y

	"""*****************TEST MOUSE******************"""

	mouseX = Mpos[0]
	mouseY = Mpos[1]
	pygame.draw.rect(screen, [50,120,120], (mouseX,mouseY,40,40), 20)
	"""*****************TEST MOUSE******************"""

	for event in pygame.event.get():

		if event.type == pygame.QUIT:
			pygame.quit()
			sys.exit()

		if event.type == pygame.MOUSEBUTTONDOWN:

			for tile in Tile.List:
				#if tile.x == (Mx * Tile.width+640) and tile.y == (My * Tile.height+400):
				if tile.collidepoint(event.pos):
					tile.type = 'solid'
					tile.walkable = False
					break #Why? Becasue it cant be turned back?

		if event.type == pygame.KEYDOWN:

			if event.key == pygame.K_w: # North
				future_tile_number = survivor.get_number() - Tile.V
				if future_tile_number in range(1, Tile.total_tiles + 1): # Check if tile exists
					if Tile.get_tile(future_tile_number).walkable:
						survivor.y -=survivor.height

			if event.key == pygame.K_s: # South
				future_tile_number = survivor.get_number() + Tile.V
				if Tile.get_tile(future_tile_number).walkable:
					survivor.y +=survivor.height

			if event.key == pygame.K_a: # West
				future_tile_number = survivor.get_number() - Tile.H
				if future_tile_number in range(1, Tile.total_tiles + 1): # Check if tile exists

					if Tile.get_tile(future_tile_number).walkable:
						survivor.x -=survivor.width

			if event.key == pygame.K_d: # East
				future_tile_number = survivor.get_number() + Tile.H
				if future_tile_number in range(1, Tile.total_tiles + 1): # Check if tile exists
					if Tile.get_tile(future_tile_number).walkable:
						survivor.x +=survivor.width


import pygame, sys, math, os

def rotate2d(pos,rad):
	x,y=pos
	s,c=math.sin(rad),math.cos(rad)
	return x*c-y*s,y*c+x*s

class Cam(object):
	def __init__(self, pos=(0,0,0), rot=(0,0)):
		self.pos = list(pos)
		self.rot = list(rot)

	def events(self,event):
		if event.type == pygame.MOUSEMOTION:
			x, y = event.rel
			x/=500.0
			y/=500.0
			self.rot[0]+=y
			self.rot[1]+=x

	def update(self, key):
		s = 1

		if key[pygame.K_q]: self.pos[1]+=s
		if key[pygame.K_e]: self.pos[1]-=s

		x,y = s*math.sin(self.rot[1]),s*math.cos(self.rot[1])

		if key[pygame.K_w]:
			self.pos[0]+=x
			self.pos[2]+=y

		if key[pygame.K_s]:
			self.pos[0]-=x
			self.pos[2]-=y

		if key[pygame.K_a]:
			self.pos[0]-=y
			self.pos[2]+=x
		if key[pygame.K_d]:
			self.pos[0]+=y
			self.pos[2]-=x

class Cube(object):
	vertices = (-1,-1,-1),(1,-1,-1),(1,1,-1),(-1,1,-1),(-1,-1,1),(1,-1,1),(1,1,1),(-1,1,1)
	edges = (0,1),(1,2),(2,3),(3,0),(4,5),(5,6),(6,7),(7,4),(0,4),(1,5),(2,6),(3,7)
	faces = (0,1,2,3),(4,5,6,7),(0,1,5,4),(2,3,7,6),(0,3,7,4),(1,2,6,5)
	colors = (0,255,255),(255,0,255),(255,0,0),(0,255,0),(0,0,255),(255,255,0)
	center = [0,0,0]

	def __init__(self,pos=(0,0,0)):
		x, y, z = pos
		self.verts = [(x+X/2.0,y+Y/2.0,z+Z/2.0) for X,Y,Z in self.vertices]

		self.initialvert = self.verts

		self.cent = []

		for i in self.center:
			self.cent.append(i+pos[i]/2.0)

	def update(self,key):

		vert_list = []
		vert_list += [list(vert) for vert in self.verts]
		rad = 0.1

		if key[pygame.K_LEFT]:
			self.cent[0] -= 1
			for vert in vert_list:
				vert[0] -= 1

		if key[pygame.K_RIGHT]:
			self.cent[0] += 1
			for vert in vert_list:
				vert[0] += 1

		if key[pygame.K_UP]:
			self.cent[1] -= 1
			for vert in vert_list:
				vert[1] -= 1

		if key[pygame.K_DOWN]:
			self.cent[1] += 1
			for vert in vert_list:
				vert[1] += 1

		if key[pygame.K_m]:

			Xdiff = self.center[0] - self.cent[0]
			Ydiff = self.center[1] - self.cent[1]

			for vert in vert_list:

				x = vert[0] + Xdiff
				y = vert[1] + Ydiff

				vert[0] = self.center[0] + (x-self.center[0])*math.cos(rad) - (y-self.center[1])*math.sin(rad)
				vert[1] = self.center[1] + (x-self.center[0])*math.sin(rad) + (y-self.center[1])*math.cos(rad)

				vert[0] -= Xdiff
				vert[1] -= Ydiff

		self.verts = []
		self.verts += [tuple(vert) for vert in vert_list]

pygame.init()
w, h = 800, 600
cx, cy = w/2.0, h/2.0
os.environ['SDL_VIDEO_CENTERED'] = '1'
fov = min(w,h)
screen = pygame.display.set_mode((w,h))
clock = pygame.time.Clock()
FPS = 30

cubes = [Cube((0,0,0)), Cube((2,0,0)), Cube((-2,0,0))]

cam = Cam((0,0,-5))

pygame.event.get()
pygame.mouse.get_rel()
pygame.mouse.set_visible(False)
pygame.event.set_grab(True)

while True:

	for event in pygame.event.get():
		if event.type == pygame.QUIT or (event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE):
			pygame.quit()
			sys.exit()
		cam.events(event)

	screen.fill((0,0,0))

	face_list = []
	face_color = []
	depth = []

	for obj in cubes:

		vert_list = []
		screen_coords = []
		for x,y,z in obj.verts:
			x-=cam.pos[0]
			y-=cam.pos[1]
			z-=cam.pos[2]

			x,z = rotate2d((x,z),cam.rot[1])
			y,z = rotate2d((y,z),cam.rot[0])
			vert_list += [(x,y,z)]

			#perspectiva
			f = fov/float(z)
			x, y = x*f, y*f
			screen_coords+=[(cx+int(x), cy+int(y))]

		for f in range(len(obj.faces)):
			face = obj.faces[f]

			on_screen = False
			for i in face:
				x, y = screen_coords[i]
				if vert_list[i][2]>0 and x>0 and x+w and y>0 and h>y:
					on_screen = True
					break

			if on_screen:
				coords = [screen_coords[i] for i in face]
				face_list += [coords]
				face_color += [obj.colors[f]]

				depth += [sum(sum(vert_list[j][i] for j in face)**2 for i in range(3))]

	order = sorted(range(len(face_list)), key=lambda i: depth[i],reverse=True)

	for i in order:
		try:
			pygame.draw.polygon(screen,face_color[i],face_list[i])
		except:
			pass

	pygame.display.flip()
	clock.tick(FPS)

	key = pygame.key.get_pressed()
	cam.update(key)
	cubes[0].update(key)


import redis
from wallace import DoesNotExist, ValidationError

from bowser.queues.fifo import FIFOQueue
from bowser.queues.scheduled import ScheduledQueue

QUEUE_TYPES = ('fifo', 'scheduled',)

class QueueService(object):

    db = redis.Redis()

    _lookup_key = 'worker_type_registry'

    @classmethod
    def get_queue(cls, name):
        qtype = cls.db.hget(cls._lookup_key, name)
        if not qtype:
            raise DoesNotExist('missing queue `%s`' % name)

        return cls._build(name, qtype)

    @classmethod
    def register(cls, name, queue_type):
        q = cls._build(name, queue_type)
        cls.db.hset(cls._lookup_key, name, queue_type)
        return q

    @classmethod
    def _build(cls, name, qtype):
        qcls = cls._switch(qtype)
        return qcls(name)

    @staticmethod
    def _switch(qtype):
        if qtype == 'fifo':
            return FIFOQueue
        if qtype == 'scheduled':
            return ScheduledQueue
        raise ValidationError('must be one of %s' % ', '.join(QUEUE_TYPES))

    @classmethod
    def last_heartbeat_for_queue(cls, queue_name):
        queue = cls.get_queue(queue_name)
        return queue.last_heartbeat()

    @classmethod
    def last_heartbeat_for_all_queues(cls):
        return dict(
            (q.queue_name, q.last_heartbeat(),)
            for q in cls.all_queues()
        )

    @classmethod
    def active_worker_counts(cls):
        return dict(
            (q.queue_name, q.num_active_workers(),)
            for q in cls.all_queues()
        )

    @classmethod
    def all_queues(cls):
        queue_type_map = cls.db.hgetall(cls._lookup_key)
        return [
            cls._build(qname, qtype)
            for qname, qtype in queue_type_map.iteritems()
        ]

    @classmethod
    def pause_all(cls):
        for q in cls.all_queues():
            q.pause()

    @classmethod
    def resume_all(cls):
        for q in cls.all_queues():
            q.resume()


"""rest URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url,include
from django.contrib import admin
from blog.views import get_articles,change_article,ArticleViewSet,CategoryViewSet
from rest_framework.routers import DefaultRouter
from django.views.generic import TemplateView

router = DefaultRouter()
router.register(r'articles',ArticleViewSet)
router.register(r'categories',CategoryViewSet)
urlpatterns = router.urls

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^articles/$', get_articles, name = 'articles'),
    url(r'^article/(?P<pk>\d+)/$', change_article, name = 'change_article'),

    url(r'^$', TemplateView.as_view(template_name='base.html')),
]
urlpatterns += router.urls

from django.conf import settings
from django.views.static import serve

if settings.DEBUG:
    urlpatterns += [
        url(r'^media/(?P<path>.*)$', serve, {
            'document_root': settings.MEDIA_ROOT,
        }),
    ]


import os

from twisted.application import internet, service
from twisted.web import resource, server, static
from twisted.python import usage

from acuity import cmdresources, views

PORT = 8000

class Options(usage.Options):
    optParameters = [
        ]

def makeService(config):
    root = resource.Resource()

    root.putChild("", cmdresources.Root())

    # Servce Django media files off of /media:
    staticrsrc = static.File(os.path.join(os.path.dirname(__file__), "media"))
    root.putChild("static", staticrsrc)
    root.putChild("media", staticrsrc)

    root.putChild("browse", views.DirList())

    # The cool part! Add in pure Twisted Web Resouce in the mix
    # This 'pure twisted' code could be using twisted's XMPP functionality, etc:
    root.putChild("perform", cmdresources.ShellResource())

    # Serve it up:
    main_site = server.Site(root)
    return internet.TCPServer(PORT, main_site)


import torch

def gradient(outputs, inputs, grad_outputs=None, retain_graph=None, create_graph=False):
    '''
    Compute the gradient of `outputs` with respect to `inputs`
    gradient(x.sum(), x)
    gradient((x * y).sum(), [x, y])
    '''
    if torch.is_tensor(inputs):
        inputs = [inputs]
    else:
        inputs = list(inputs)
    grads = torch.autograd.grad(outputs, inputs, grad_outputs,
                                allow_unused=True,
                                retain_graph=retain_graph,
                                create_graph=create_graph)
    grads = [x if x is not None else torch.zeros_like(y) for x, y in zip(grads, inputs)]
    return torch.cat([x.contiguous().view(-1) for x in grads])

def jacobian(outputs, inputs, create_graph=False):
    '''
    Compute the Jacobian of `outputs` with respect to `inputs`
    jacobian(x, x)
    jacobian(x * y, [x, y])
    jacobian([x * y, x.sqrt()], [x, y])
    '''
    if torch.is_tensor(outputs):
        outputs = [outputs]
    else:
        outputs = list(outputs)

    if torch.is_tensor(inputs):
        inputs = [inputs]
    else:
        inputs = list(inputs)

    jac = []
    for output in outputs:
        output_flat = output.view(-1)
        output_grad = torch.zeros_like(output_flat)
        for i in range(len(output_flat)):
            output_grad[i] = 1
            jac += [gradient(output_flat, inputs, output_grad, True, create_graph)]
            output_grad[i] = 0
    return torch.stack(jac)

def hessian(output, inputs, out=None, allow_unused=False, create_graph=False):
    '''
    Compute the Hessian of `output` with respect to `inputs`
    hessian((x * y).sum(), [x, y])
    '''
    assert output.ndimension() == 0

    if torch.is_tensor(inputs):
        inputs = [inputs]
    else:
        inputs = list(inputs)

    n = sum(p.numel() for p in inputs)
    if out is None:
        out = output.new_zeros(n, n)

    ai = 0
    for i, inp in enumerate(inputs):
        [grad] = torch.autograd.grad(output, inp, create_graph=True, allow_unused=allow_unused)
        grad = torch.zeros_like(inp) if grad is None else grad
        grad = grad.contiguous().view(-1)

        for j in range(inp.numel()):
            if grad[j].requires_grad:
                row = gradient(grad[j], inputs[i:], retain_graph=True, create_graph=create_graph)[j:]
            else:
                row = grad[j].new_zeros(sum(x.numel() for x in inputs[i:]) - j)

            out[ai, ai:].add_(row.type_as(out))  # ai's row
            if ai + 1 < n:
                out[ai + 1:, ai].add_(row[1:].type_as(out))  # ai's column
            del row
            ai += 1
        del grad

    return out

def eval_hessian(loss_grad, model):
    cnt = 0
    for g in loss_grad:
        g_vector = g.contiguous().view(-1) if cnt == 0 else torch.cat([g_vector, g.contiguous().view(-1)])
        cnt = 1
    l = g_vector.size(0)
    hessian = torch.zeros(l, l)
    for idx in range(l):
        grad2rd = autograd.grad(g_vector[idx], model.parameters(), create_graph=True)
        cnt = 0
        for g in grad2rd:
            g2 = g.contiguous().view(-1) if cnt == 0 else torch.cat([g2, g.contiguous().view(-1)])
            cnt = 1
        hessian[idx] = g2
    return hessian.cpu().data.numpy()

def KL_diag_gaussian(mu_1,diag_1,mu_2,diag_2):
    ratio=diag_1/diag_2
    return torch.sum(0.5*(mu_1-mu_2)**2/diag_2)+0.5*torch.sum(ratio)-0.5*torch.sum(torch.log(ratio))-mu_1.size(0)/2

def low_rank_cov_inverse(L,sigma):
    # L is D*R
    dim=L.size(0)
    rank=L.size(1)
    var=sigma**2
    inverse_var=1.0/var
    inner_inverse=torch.inverse(torch.diag(torch.ones([rank]))+inverse_var*(L.t())@L)
    return inverse_var*torch.diag(torch.ones([dim]))-inverse_var**2*L@inner_inverse@L.t()

def low_rank_gaussian_logdet(L,sigma):
    dim=L.size(0)
    rank=L.size(1)
    var=sigma**2
    inverse_var=1.0/var
    return torch.logdet(torch.diag(torch.ones([rank]))+inverse_var*(L.t())@L)+dim*tf.log(var)

def KL_low_rank_gaussian_with_diag_gaussian(mu_1,L_1,sigma_1,mu_2,sigma_2,cuda=True):
    dim_1=L_1.size(0)
    rank_1=L_1.size(1)
    var_1=sigma_1**2
    inverse_var_1=1.0/var_1
    if cuda:
        logdet_1=torch.logdet(torch.diag(torch.ones([rank_1]).cuda())+inverse_var_1*(L_1.t())@L_1)+dim_1*torch.log(var_1)
        cov_1=L_1@L_1.t()+torch.diag(torch.ones([dim_1]).cuda())*var_1
    else:
        logdet_1=torch.logdet(torch.diag(torch.ones([rank_1]))+inverse_var_1*(L_1.t())@L_1)+dim_1*torch.log(var_1)
        cov_1=L_1@L_1.t()+torch.diag(torch.ones([dim_1]))*var_1
    mu_diff=(mu_1-mu_2).view(-1,1)
    var_2=sigma_2**2
    return -0.5*(logdet_1-dim_1*torch.log(var_2)+dim_1-(1/var_2)*torch.trace(cov_1)-(1/var_2)*mu_diff.t()@mu_diff)

def KL_low_rank_gaussian_with_low_rank_gaussian_cuda(mu_1,L_1,sigma_1,mu_2,L_2,sigma_2):
    dim_1=L_1.size(0)
    rank_1=L_1.size(1)
    var_1=sigma_1**2
    inverse_var_1=1.0/var_1
    logdet_1=torch.logdet(torch.diag(torch.ones([rank_1]).cuda())+inverse_var_1*(L_1.t())@L_1)+dim_1*torch.log(var_1)
    cov_1=L_1@L_1.t()+torch.diag(torch.ones([dim_1]).cuda())*var_1

    dim_2=L_2.size(0)
    rank_2=L_2.size(1)
    var_2=sigma_2**2
    inverse_var_2=1.0/var_2
    logdet_2=torch.logdet(torch.diag(torch.ones([rank_2]).cuda())+inverse_var_2*(L_2.t())@L_2)+dim_1*torch.log(var_2)

    inner_inverse_2=torch.inverse(torch.diag(torch.ones([rank_2]).cuda())+inverse_var_2*(L_2.t())@L_2)
    cov_inverse_2=inverse_var_2*torch.diag(torch.ones([dim_2]).cuda())-inverse_var_2**2*L_2@inner_inverse_2@L_2.t()

    mu_diff=(mu_1-mu_2).view(-1,1)
    return -0.5*(logdet_1-logdet_2+dim_1-torch.trace(cov_1@cov_inverse_2)-mu_diff.t()@ cov_inverse_2@mu_diff)

def KL_low_rank_gaussian_with_low_rank_gaussian(mu_1,L_1,sigma_1,mu_2,L_2,sigma_2):
    dim_1=L_1.size(0)
    rank_1=L_1.size(1)
    var_1=sigma_1**2
    inverse_var_1=1.0/var_1
    logdet_1=torch.logdet(torch.diag(torch.ones([rank_1]))+inverse_var_1*(L_1.t())@L_1)+dim_1*torch.log(var_1)
    cov_1=L_1@L_1.t()+torch.diag(torch.ones([dim_1]))*var_1

    dim_2=L_2.size(0)
    rank_2=L_2.size(1)
    var_2=sigma_2**2
    inverse_var_2=1.0/var_2
    logdet_2=torch.logdet(torch.diag(torch.ones([rank_2]))+inverse_var_2*(L_2.t())@L_2)+dim_1*torch.log(var_2)

    inner_inverse_2=torch.inverse(torch.diag(torch.ones([rank_2]))+inverse_var_2*(L_2.t())@L_2)
    cov_inverse_2=inverse_var_2*torch.diag(torch.ones([dim_2]))-inverse_var_2**2*L_2@inner_inverse_2@L_2.t()

    mu_diff=(mu_1-mu_2).view(-1,1)
    return -0.5*(logdet_1-logdet_2+dim_1-torch.trace(cov_1@cov_inverse_2)-mu_diff.t()@ cov_inverse_2@mu_diff)

def general_kl_divergence(mu_1,cov_1,mu_2,cov_2):
    mu_diff=(mu_1-mu_2).view(-1,1)
    cov_2_inverse=torch.inverse(cov_2)
    return -0.5*(torch.logdet(cov_1)-torch.logdet(cov_2)+mu_1.size(0)-torch.trace(cov_1@cov_2_inverse)-mu_diff.t()@cov_2_inverse@mu_diff)

def low_rank_gaussian_one_sample(mu,L,sigma,cuda=True):
    # L is D*R
    dim=L.size(0)
    rank=L.size(1)
    if cuda:
        eps_z=torch.randn([rank]).cuda()
        eps=torch.randn([dim]).cuda()
    else:
        eps_z=torch.randn([rank])
        eps=torch.randn([dim])

    return eps_z@L.t()+eps*sigma+mu

def low_rank_gaussian_sample(mu,L,sigma,amount,cuda=True):
    # L is D*R
    dim=L.size(0)
    rank=L.size(1)
    if cuda:
        eps_z=torch.randn([amount,rank]).cuda()
        eps=torch.randn([amount,dim]).cuda()
    else:
        eps_z=torch.randn([amount,rank])
        eps=torch.randn([amount,dim])

    return eps_z@L.t()+eps*sigma+mu

def sample_from_batch_categorical(batch_logits,cuda=True):
    ### shape batch*dim
    ### gumbel max trick
    if cuda:
        noise = torch.rand(batch_logits.size()).cuda()
    else:
        noise = torch.rand(batch_logits.size())
    return torch.argmax(batch_logits - torch.log(-torch.log(noise)), dim=-1)

def sample_from_batch_categorical_multiple(batch_logits,sample_num,cuda=True):
    ### shape batch*dim
    ### gumbel max trick
    shape=list(batch_logits.size())
    shape.insert(-1, sample_num)
    if cuda:
        noise = torch.rand(shape).cuda()
    else:
        noise = torch.rand(shape)
    batch_logits_multiple=batch_logits.repeat(1,1,1,sample_num).view(shape)
    return torch.argmax(batch_logits_multiple - torch.log(-torch.log(noise)), dim=-1)

def one_hot_embedding(labels, num_classes,cuda=True):
    if cuda:
        y = torch.eye(num_classes).cuda()
    else:
        y = torch.eye(num_classes)
    return y[labels]

def kroneck(a,b):
    c=a.unsqueeze(-2).unsqueeze(-1).mul(b.unsqueeze(-3).unsqueeze(-2))
    return c.view(-1,c.size()[1]*c.size()[2],c.size(3)*c.size(4))


# -*- coding: utf-8 -*-

from flask import Flask, jsonify, abort

import simplemetrics, auth

app = Flask(__name__)

@app.route("/")
def hello_world():
    if not auth.validate():
        abort(401)

    return "I want coffee."

@app.route("/metrics/cpu")
def metrics_cpu():
    if not auth.validate():
        abort(401)

    cpu = simplemetrics.cpu()
    return jsonify(cpu)

@app.route("/metrics/ram")
def metrics_ram():
    if not auth.validate():
        abort(401)

    memory = simplemetrics.memory()
    return jsonify(memory)

@app.route("/metrics/disk")
def metrics_disk():
    if not auth.validate():
        abort(401)

    disks = simplemetrics.disks()
    return jsonify(disks)

@app.route("/metrics/network")
def metrics_network():
    if not auth.validate():
        abort(401)

    network = simplemetrics.network()
    return jsonify(network)

@app.route("/metrics/services")
def metrics_services():
    if not auth.validate():
        abort(401)

    process_list = simplemetrics.process()
    return jsonify(process_list)

if __name__ == '__main__':
    import os

    if not simplemetrics.hasRequirements():
        print("Exiting...")
        exit(1)

    # Development server
    # if 'DEBUG' env exist
    if os.environ.get("DEBUG"):
        if str(os.environ.get("DEBUG")).lower() == "true":
            app.run(debug=True, host='0.0.0.0', port=5000)
        else:
            app.run(debug=False, host='0.0.0.0', port=5000)

    # Production server
    else:
        from waitress import serve
        serve(app, port=5000)


import pygame
from settings import Settings
import game_functions as gf
from ship import Ship
from pygame.sprite import Group
from game_stats import GameStats
from button import Button
from scoreboard import Scoreboard

def run_game():
    """ This is the main game function."""
    # Initiate pygame, create a screen and caption/title.
    filename = "Highscore.json"

    pygame.init()
    ai_settings = Settings()
    screen = pygame.display.set_mode((ai_settings.screen_width, ai_settings.screen_height))
    pygame.display.set_caption("Bull's eye")

    # create an instance of ship.
    ship = Ship(ai_settings, screen)

    # create groups of bullets.
    bullets = Group()
    targets = Group()

    # create target.
    gf.create_target(targets, ai_settings, screen)

    # Instantiate statistics, play button and scoreboard.
    stats = GameStats(ai_settings, filename)
    button = Button(screen, 'Play')
    sb = Scoreboard(ai_settings, screen, stats)

    while True:
        """Main loop/runs the game and updates everything."""
        gf.check_events(ship, bullets, ai_settings, screen, stats, button, targets, sb)
        if stats.game_active:
            gf.update_bullet(bullets, targets, ai_settings, screen, stats, ship, sb, filename)
            gf.update_target(targets, ai_settings)
            ship.update()
        gf.update_screen(ai_settings, screen, ship, bullets, targets, button, stats, sb)

run_game()


from r import r
import logging

filename = 'history.log'

pubsub = r.pubsub()
pubsub.subscribe('file.new')
pubsub.subscribe('file.changed')

logging.basicConfig(filename=filename, level=logging.DEBUG)

def process_event(event):
    """
    Event has keys: pattern, channel, type, data
    """
    data = event['data']
    if type(data) == bytes:
        logging.info(event['data'].decode())

while True:
    for event in pubsub.listen():
        process_event(event)


#coding=utf-8
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from Env3DSolo import *
import mpl_toolkits.mplot3d.axes3d as p3

R = 2.

def play3D(agent = None):

    fig = plt.figure()
    ax = p3.Axes3D(fig)

    n_points = 10000

    g = Game3DSolo(2., 9.8,0.5, 7.,3.)
    state = g.reset()

    positions = np.zeros((n_points,3))
    positions1 = np.zeros((n_points,3))

    def animate(i):
        if agent == None:
            action1 = np.array([np.random.rand()*2 - 1,np.random.rand()*2 - 1])
        else:
            action1 = agent.forward(state)
        #action2 = np.array([np.random.rand(),np.random.rand()])
        if g.step(action1)[2] == False:
            if i < n_points:
                positions[i,:] = g.p_ball
                positions1[i,:] = g.P1.R*np.array([np.cos(g.P1.phi)*np.cos(g.P1.theta), np.cos(g.P1.phi)*np.sin(g.P1.theta),np.sin(g.P1.phi)])
                #positions2[i,:] = g.P2.R*np.array([np.cos(g.P2.phi)*np.cos(g.P2.theta), np.cos(g.P2.phi)*np.sin(g.P2.theta),np.sin(g.P2.phi)])
            last = int(np.min([i+1,n_points - 1]))
            line.set_data([positions[:last,0],positions[:last,1]]) # update the data
            line.set_3d_properties(positions[:last,2])
            line1.set_data([positions1[:last,0],positions1[:last,1]])
            line1.set_3d_properties(positions1[:last,2])
            #line2.set_data([positions2[:last,0],positions2[:last,1]])
            #line2.set_3d_properties(positions2[:last,2])
        return line,

    # Make data
    u = np.linspace(0, np.pi/2, 100)
    v = np.linspace(0, np.pi/2, 100)
    x = R * np.outer(np.cos(u), np.sin(v))
    y = R * np.outer(np.sin(u), np.sin(v))
    z = R * np.outer(np.ones(np.size(u)), np.cos(v))
    ax.plot_surface(x, y, z, color=(0.4,0.4 , 0.9, 0.1),shade=True)

    positions[0,:] = g.p_ball
    positions1[0,:] = g.P1.R*np.array([np.cos(g.P1.phi)*np.cos(g.P1.theta), np.cos(g.P1.phi)*np.sin(g.P1.theta),np.sin(g.P1.phi)])
    #positions2[0,:] = g.P2.R*np.array([np.cos(g.P2.phi)*np.cos(g.P2.theta), np.cos(g.P2.phi)*np.sin(g.P2.theta),np.sin(g.P2.phi)])

    line = ax.plot(positions[0:1,0], positions[0:1,1],positions[0:1,2])[0]
    line1 = ax.plot(positions1[0:1,0], positions1[0:1,1],positions1[0:1,2])[0]
    #line2 = ax.plot(positions2[0:1,0], positions2[0:1,1],positions2[0:1,2])[0]

    ani = animation.FuncAnimation(fig, animate, np.arange(1, 200),interval=500)
    plt.show()


import redis
import atexit
from service import Service

def mainMenu():
    print('0>: exit')
    print('1>: registration')
    print('2>: authorization')

    return int(input('>: '))

def menuForLoggedUser():
    print('0>: exit from user account and return to previous menu')
    print('1>: send message')
    print('2>: all messages')
    print('3>: get result messages')
    return int(input('>: '))

def main():
    connect = redis.Redis(charset='UTF-8', decode_responses=True)
    service = Service(connect)
    currentId = -1

    def endH():
        service.logout(currentId)

    atexit.register(endH)
    menu = mainMenu
    while 1:
        switch = menu()
        if switch == 1:
            login = input('Enter login: ')
            service.registration(login)
        elif switch == 2:
            login = input('Enter login: ')
            currentId = service.login(login)
            if currentId != -1:
                connect.publish('users', f'User {login} connected')
                while 1:
                    switch = menuForLoggedUser()
                    if switch == 1:
                        message = input('message: ')
                        recipient = input('recipient login: ')
                        service.sendMessage(message, currentId, recipient)

                    elif switch == 2:
                        mssList = service.connection.smembers(f'sentto:{currentId}')
                        for mssId in mssList:
                            message = service.connection.hmget(f'message:{mssId}',
                                                               ['messageFromId', 'text', 'status'])
                            messageFromId = message[0]
                            getValueFrom = service.connection.hmget(f'user:{messageFromId}', ['login'])[0]
                            print(f'Message by: {getValueFrom} - {message[1]} ')
                            if message[2] != 'deliver':
                                connectPipeline = service.connection.pipeline(True)
                                connectPipeline.hset(f'message:{mssId}', 'status', 'deliver')
                                connectPipeline.hincrby(f'user:{messageFromId}', 'sent', -1)
                                connectPipeline.hincrby(f'user:{messageFromId}', 'deliver', 1)
                                connectPipeline.execute()
                    elif switch == 3:
                        loggedUser = connect.hmget(f'user:{currentId}',['queue', 'check', 'block', 'sent', 'deliver'])
                        print('QUEUE message: {} || CHECK message: {} ||BLOCK message: {} ||SENT message: {} '
                              '||DELIVER message: {} '
                                .format(*tuple(loggedUser)))
                    elif switch == 0:
                        service.logout(currentId)
                        connect.publish('users', f'User signed out')
                        return
        elif switch == 0:
            return

if __name__ == '__main__':
    main()


import xml.etree.cElementTree as ET
from collections import defaultdict
import re
import pprint

OSMFILE = "vancouver.osm"
street_type_re = re.compile(r'\b\S+\.?$', re.IGNORECASE)

expected = ["Street", "Avenue", "Boulevard", "Drive", "Court", "Place", "Square", "Lane", "Road", "Trail", "Parkway", "Commons",
            "Kingsway", "Greenway", "Walk", "North", "South", "West", "East", "Scantlings", "Highway", "Diversion", "Mall", "Crescent",
            "Hill", "Way", "Sawcut", "Broadway", "Seawall"]

mapping = { "St": "Street",
            "street": "Street",
            "St.": "Street",
            "Ave": "Avenue",
            "Ave.": "Avenue",
            "Blvd": "Boulevard",
            "Blvd.": "Boulevard",
            "Dr": "Drive",
            "Dr.": "Drive",
            "Rd": "Road",
            "Rd.": "Road" }

def audit_street_type(street_types, street_name):
    m = street_type_re.search(street_name)
    if m:
        street_type = m.group()
        if street_type not in expected:
            street_types[street_type].add(street_name)

def is_street_name(elem):
    return (elem.attrib['k'] == "addr:street")

def audit(osmfile):
    osm_file = open(osmfile, "r")
    street_types = defaultdict(set)
    for event, elem in ET.iterparse(osm_file, events=("start",)):
        if elem.tag == "node" or elem.tag == "way":
            for tag in elem.iter("tag"):
                if is_street_name(tag):
                    audit_street_type(street_types, tag.attrib['v'])
    osm_file.close()
    return street_types

def update_name(street_type, mapping):
    m = street_type_re.search(street_type)
    if m:
        better_street_type = mapping[m.group()]
        better_street_name = street_type_re.sub(better_street_type, street_type)
    return better_street_name

def start():
    st_types = audit(OSMFILE)
    for st_types, ways in st_types.iteritems():
        for name in ways:
            better_name = update_name(name, mapping)
            print("updating " + name + " --> " + better_name)
            name = better_name
    print("auditing complete!")

if __name__ == '__main__':
    start()


import pygame
from sys import exit
import numpy as np
import time

def nsum(a):
    b = (np.roll(a,-1,0) +
         np.roll(a,-1,1) +
         np.roll(np.roll(a,-1,0),-1,1) +
         np.roll(np.roll(a,1,0),-1,1) +
         np.roll(np.roll(a,-1,0),1,1) +
         np.roll(np.roll(a,1,0),1,1) +
         np.roll(a,1,1) +
         np.roll(a,1,0))
    return b

pygame.init()
clock = pygame.time.Clock()

width,height = 1000,1000
nx,ny = 1000,1000
dx,dy = int(width/nx),int(width/ny)
screen = pygame.display.set_mode((width,height))

bg = 25,25,25
screen.fill(bg)

ones = np.ones((nx,ny),dtype='B')
gameState = np.random.randint(0,2,(nx,ny),dtype='B')

def mksurf(gs):
    Z = 255*np.kron(gs,np.ones((dx,dy),dtype='B'))
    return pygame.surfarray.make_surface(Z)

pauseExect = False
n = 0

while n<1000:

    screen.blit(mksurf(gameState),(0,0))

    if not pauseExect:
        n_neigh = nsum(gameState)
        newGameState = ((n_neigh*(gameState==1)==3) + (n_neigh*(gameState==1)==2) + (n_neigh*(gameState==0)==3))*ones

    pygame.display.flip()

    for event in pygame.event.get():
        if event.type == pygame.KEYDOWN:
            pauseExect = not pauseExect
        click = pygame.mouse.get_pressed()
        if sum(click)>0:
            px,py = pygame.mouse.get_pos()
            cx,cy = int(np.floor(px/dx)),int(np.floor(py/dy))
            newGameState[cx,cy] = not click[2]
        if event.type == pygame.QUIT:
            pygame.quit()
            print(time.time()-T,n)
            exit()

    gameState = np.copy(newGameState)

    clock.tick()
    fps = clock.get_fps()
    print(f'{int(fps)}')

    n += 1