from __future__ import print_function, division
import scipy
from keras import metrics
# from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization
from keras.layers import Input, Dense, Reshape, Flatten, Dropout, Embedding, LSTM, Lambda
from keras.layers import BatchNormalization, Activation, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.layers.recurrent import SimpleRNN
from keras.models import Sequential, Model, load_model
from keras.optimizers import Adam
from keras import backend as K
from keras.preprocessing import sequence
import datetime
import matplotlib.pyplot as plt
import sys
import numpy as np
import os
import pandas as pd
# from data_loader import DataLoader
from keras.utils import np_utils
from keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping, CSVLogger
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import train_test_split
import tensorflow as tf
import random

gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

class Universal():
    # A=R  B=T
    def __init__(self):
        # Input shape
        self.maccs_shape = (None, 166)  # (length, group)
        self.smile_shape = (None, 50)  # (length, char)
        self.x_shape = (None, 200)
        # Load data
        self.maccs_data = self.load_maccs_data()
        self.smile_data = self.load_smile_data()
        self.smile_data_3d = self.load_smile_data_3d()

        # set optimizer
        optimizer = Adam(1e-7, 0.5)  # (lr,beta)

        # Build the Dis_S
        self.Dis_S = self.build_Dis_S()
        self.Dis_S.compile(loss='binary_crossentropy',
                           optimizer=optimizer,
                           metrics=['accuracy'])

        # Build and compile for Decoder
        self.D_X2M = self.build_D_X2M()
        self.D_X2S = self.build_D_X2S()

        # Build the Encoder
        self.E_M2X = self.build_E_M2X()
        self.E_S2X = self.build_E_S2X()
        # Build the Error_X
        self.X_error = self.build_X_error()
        # Build a coder for noise
        self.X_M_noise = self.build_X_M_noise()

        # Input images from both domains
        Maccs = Input(batch_shape=self.maccs_shape)
        Smile = Input(batch_shape=self.smile_shape)

        # Translate images to the other domain
        X_from_M = self.E_M2X(Maccs)
        X_from_S = self.E_S2X(Smile)
        X_from_M_with_noise = self.X_M_noise(self.E_M2X(Maccs))

        # Translate images back to original domain
        reconstr_M = self.D_X2M(X_from_M)
        generate_S = self.D_X2S(X_from_M_with_noise)
        reconstr_S = self.D_X2S(X_from_S)
        generate_M = self.D_X2M(X_from_S)
        X_error = self.X_error([X_from_M, X_from_S])

        # For the combined model we will only train the generators
        self.Dis_S.trainable = False

        # Discriminators determines validity of translated images
        discrmin_S = self.Dis_S(generate_S)

        # Combined model trains generators to fool discriminators
        self.AE_combined = Model(inputs=[Maccs, Smile],
                                 outputs=[reconstr_S,
                                          generate_M,
                                          reconstr_M,
                                          discrmin_S,
                                          X_error])
        self.AE_combined.compile(loss=['categorical_crossentropy', 'binary_crossentropy',
                                       'binary_crossentropy', 'binary_crossentropy',
                                       'mse'],
                                 loss_weights=[5, 1,
                                               1, 1,
                                               3],
                                 optimizer=optimizer,
                                 metrics=['accuracy'
                                          # metrics.binary_accuracy,
                                          # metrics.categorical_accuracy
                                          ]
                                 )

        self.Dis_S.summary()
        self.AE_combined.summary()
        self.AE_combined.save('./model_save/universal.h5')
        # self.save_generator()

    @staticmethod
    def load_maccs_data():
        a = pd.read_csv('./DataPool/maccsdata.csv', header=None).values
        return a

    @staticmethod
    def load_smile_data():
        b = pd.read_csv('./DataPool/smile.csv', header=None).values
        return b

    @staticmethod
    def load_smile_data_3d():
        smile_d = pd.read_csv('./DataPool/smile.csv', header=None).values
        target_texts = smile_d[:, :50]
        decoder_input_data = np.zeros((1303, 50, 161), dtype='float32')
        for i in range(1303):
            for k in range(50):
                decoder_input_data[i, k, int(target_texts[i, k])] = 1.
        return decoder_input_data

    def build_E_M2X(self):
        encoder_inputs = Input(batch_shape=self.maccs_shape)
        d0 = Dense(200, activation='relu')(encoder_inputs)
        output = Dense(200, activation='relu')(d0)
        return Model(encoder_inputs, output, name='E_M2X')

    def build_X_M_noise(self, latent_dim=200, epsilon_std=3.0):
        # noise part
        encoder_h = Input(batch_shape=self.x_shape)
        r0 = Reshape((200,))(encoder_h)
        # mean vector
        z_mean = Dense(latent_dim, activation='relu')(r0)
        # standard deviation vector
        z_log_var = Dense(latent_dim, activation='relu')(r0)

        def sampling(args):
            # Reparameterization trick
            z_mean, z_log_var = args
            # get epsilon from standard normal distribution
            epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim), mean=0., stddev=epsilon_std)
            return z_mean + K.exp(z_log_var / 2) * epsilon

        # latent vector z
        z = Lambda(sampling, output_shape=(None, latent_dim))([z_mean, z_log_var])
        return Model(encoder_h, z, name='X_M_noise')

    def build_D_X2S(self):
        # z  = X_M+noise = X_S
        z = Input(batch_shape=self.x_shape)
        z1 = Reshape((50, 4))(z)
        z1 = Dense(50, activation='relu')(z1)
        z1 = LSTM(161, return_sequences=True, return_state=False, activation='tanh')(z1)
        output = Dense(161, activation='softmax', name='output')(z1)
        return Model(z, output, name='D_X2S')

    def build_E_S2X(self):
        input_shape = Input(batch_shape=self.smile_shape)
        embedding_layer = Embedding(input_dim=160, input_length=50, output_dim=50, mask_zero=True)(input_shape)       # mask zero= true
        d0 = Flatten()(embedding_layer)
        output = Dense(units=200, activation='relu', name='E_S2X')(d0)
        return Model(input_shape, output, name='E_S2X')

    def build_D_X2M(self):
        # z  = X_M = X_S
        z = Input(batch_shape=self.x_shape)
        z1 = Reshape((200, -1))(z)
        L0 = LSTM(1000, return_sequences=False, return_state=False, activation='tanh')(z1)
        output = Dense(units=166, activation='hard_sigmoid', name='D_X2M')(L0)
        return Model(z, output, name='D_X2M')

    def build_Dis_S(self):
        d0 = Input(batch_shape=(None, 50, 161))
        d3 = Dense(128, activation='relu')(d0)
        d4 = Dense(64, activation='relu')(d3)
        d4 = Flatten()(d4)
        output = Dense(1, activation='hard_sigmoid', name='Dis_0_1_layer')(d4)
        return Model(d0, output, name='Dis_S')

    def build_X_error(self):
        x = Input(batch_shape=self.x_shape, name='x_input')
        x_ = Input(batch_shape=self.x_shape, name='x__input')
        output = Lambda(lambda e: K.square(e[0] - e[1]))([x, x_])
        return Model([x, x_], output, name='x_error')

    def data_loader(self, batch_size):
        dataA = self.load_maccs_data()
        dataB = self.load_smile_data()
        dataC = self.load_smile_data_3d()
        self.n_batch = int(min(len(dataA), len(dataB)) / batch_size)   # self.n_batch=20 = 1303 / 64
        total_samples = self.n_batch * batch_size
        for i in range(self.n_batch):
            batchA = dataA[i * batch_size:(i + 1) * batch_size, :]
            batchB = dataB[i * batch_size:(i + 1) * batch_size, :]
            batchC = dataC[i * batch_size:(i + 1) * batch_size, :]
            yield batchA, batchB, batchC

# this part don't work
    @staticmethod
    def check_fake_smile(gen_smile):
        smiles = pd.read_csv('./DataPool/smile.csv', header=None).values
        for i in range(len(smiles)):
            if gen_smile == smiles[i]:
                return None
            else:
                return gen_smile

    def train(self, epochs, batch_size=1):
        valid = np.ones((batch_size,))
        fake = np.zeros((batch_size,))

        # Train the generators
        for epoch in range(epochs):
            self.R = epoch
            self.T = epochs
            for batch_i, (self.maccs_data, self.smile_data, self.smile_data_3d) in enumerate(
                    self.data_loader(batch_size=batch_size)):
                # ----------------------
                #  Train Discriminators
                # ----------------------

                # discriminator training part
                fake_smile = self.check_fake_smile(self.D_X2S.predict(self.E_M2X.predict(self.maccs_data)))
                d_loss_real = self.Dis_S.train_on_batch(self.smile_data_3d, valid)  # valid 1
                d_loss_fake = self.Dis_S.train_on_batch(fake_smile, fake)  # fake 0
                d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)

                maccs_train, maccs_test, smile_train, smile_test, smile_train_3d, smile_test_3d = train_test_split \
                    (self.maccs_data, self.smile_data, self.smile_data_3d, test_size=0.2, random_state=33)

                # ------------------
                #  Train Generators
                # ------------------

                # generator training part
                '''        self.AE_combined = Model(inputs=[Maccs, Smile],
                                 outputs=[reconstr_S, generate_M,
                                          reconstr_M, discrmin_S,
                                          X_error])'''
                g_loss = self.AE_combined.train_on_batch([maccs_train, smile_train],
                                                         [smile_train_3d,
                                                          maccs_train,
                                                          maccs_train,
                                                          np.ones((len(maccs_train), 1), dtype='float32'),
                                                          np.zeros((len(maccs_train), 200), dtype='float32')])

                print(
                    "[Epoch %d/%d] [Batch %d/%d] "
                    "[D loss: %f, acc: %3d%%,%3d%%,%3d%% ] "
                    "[G loss: %05f, "
                    "rec_S: %05f,%05f, "
                    "gen_M: %05f,%05f, "
                    "rec_M: %05f,%05f,  "
                    "dis_S: %05f,%05f,  "
                    "X_error: %05f, ]"
                    \
                    % (epoch + 1, epochs, batch_i + 1, self.n_batch,
                       d_loss[0], 100 * d_loss[1], 100 * d_loss_real[1], 100 * d_loss_fake[1],
                       g_loss[0],
                       g_loss[1], g_loss[6],  # categorical
                       g_loss[2], g_loss[7],  # binary
                       g_loss[3], g_loss[8],  # binary
                       g_loss[4], g_loss[9],  # binary
                       g_loss[5],  # loss
                       ))

        # Save models
        self.save_models()

    def save_models(self):
        os.makedirs('h5/G/G_%sin%s' % (self.R, self.T), exist_ok=True)
        self.AE_combined.save('./h5/G/G_%sin%s/g_%sin%s.h5' % (self.R, self.T, self.R, self.T))

    def load_model(self):

        self.AE_combined = load_model('./h5/G/G_%s%s/g_%s%s.h5' % (self.R, self.T, self.R, self.T))

    def M2S_test(self):
        return 0

    def M2M_test(self):
        return 0

    def S2M_test(self):
        return 0

    def S2S_test(self):
        return 0

    def X_error_test(self):
        return 0

# 如果在其他的py檔想執行universal 那以下單元測試也會被執行 可以把if判斷改成false就不會執行(how?)
if __name__ == '__main__':
    uni = Universal()  # load data build model
    uni.train(epochs=300, batch_size=64)
    # uni.S2M_test()
    # uni.S2S_test()


# -*- coding: utf-8 -*-
"""
Created on Tue May 11 18:13:50 2021

@author: modak_ng8awn0
"""

"""How to Display Text on Screen """

""" Import Lib"""
import pygame

""" Initialize Pygame"""
pygame.init()

""" Set the Dimension of the Screen"""
Width = 1000
Height = 560

""" Set the Screen"""

screen = pygame.display.set_mode((Width,Height))

""" Set The Title of Screen"""

pygame.display.set_caption("Display Text")

""" Load the Image"""
screenImg = pygame.image.load("Images/Background.png")

""" Display Image at Specific Co-Ordinate"""

screen.blit(screenImg,(0,0))

"""To Dispaly Text"""

font = pygame.font.SysFont("Eras Bold ITC",50)
text=font.render("My First Text",True,(255,0,0),(255,255,0))
screen.blit(text,(100,200))

""" Update the Display Continuously"""

EventStatus="None"

while True:

    pygame.display.update()

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            EventStatus="Quit"
            break

    if EventStatus == "Quit":
        break

print("Closing")


import os
import numpy as np
from keras.optimizers import Adam, Optimizer
from keras.models import Model
from keras.layers import Input, Dense
from keras.initializers import RandomNormal
from keras.utils import plot_model
from keras.engine.network import Network
import keras.backend as K
from PIL import Image
from cv2 import cv2 as cv
import random
import time
from colorama import Fore
from collections import deque
from typing import Union
import json
from statistics import mean
import imagesize
from multiprocessing.pool import ThreadPool

from ..utils.batch_maker import BatchMaker
from ..models import discriminator_models_spreadsheet, generator_models_spreadsheet
from ..keras_extensions.custom_tensorboard import TensorBoardCustom
from ..utils.helpers import time_to_format, get_paths_of_files_from_path

class DCGAN:
  CONTROL_THRESHOLD = 100_000 # Threshold when after whitch we will be testing training process
  AGREGATE_STAT_INTERVAL = 1_000  # Interval of saving data
  GRADIENT_CHECK_INTERVAL = 10_000  # Interval of checking norm gradient value of combined model
  CHECKPOINT_SAVE_INTERVAL = 1_000  # Interval of saving checkpoint

  def __init__(self, dataset_path:str,
               gen_mod_name: str, disc_mod_name: str,
               latent_dim:int,
               training_progress_save_path:str,
               testing_dataset_path:str=None,
               generator_optimizer:Optimizer=Adam(0.0002, 0.5), discriminator_optimizer:Optimizer=Adam(0.0002, 0.5),
               discriminator_label_noise:float=None, discriminator_label_noise_decay:float=None, discriminator_label_noise_min:float=0.001,
               batch_size: int = 32, buffered_batches:int=20,
               generator_weights:Union[str, None]=None, discriminator_weights:Union[str, None]=None,
               start_episode:int=0, load_from_checkpoint:bool=False,
               check_dataset:bool=True, num_of_loading_workers:int=8):

    self.disc_mod_name = disc_mod_name
    self.gen_mod_name = gen_mod_name
    self.generator_optimizer = generator_optimizer

    self.latent_dim = latent_dim
    assert self.latent_dim > 0, Fore.RED + "Invalid latent dim" + Fore.RESET

    self.batch_size = batch_size
    assert self.batch_size > 0, Fore.RED + "Invalid batch size" + Fore.RESET

    self.discriminator_label_noise = discriminator_label_noise
    self.discriminator_label_noise_decay = discriminator_label_noise_decay
    self.discriminator_label_noise_min = discriminator_label_noise_min

    self.progress_image_dim = (16, 9)

    if start_episode < 0: start_episode = 0
    self.episode_counter = start_episode

    # Initialize training data folder and logging
    self.training_progress_save_path = training_progress_save_path
    self.training_progress_save_path = os.path.join(self.training_progress_save_path, f"{self.gen_mod_name}__{self.disc_mod_name}")
    self.tensorboard = TensorBoardCustom(log_dir=os.path.join(self.training_progress_save_path, "logs"))

    # Create array of input image paths
    self.train_data = get_paths_of_files_from_path(dataset_path, only_files=True)
    assert self.train_data, Fore.RED + "Training dataset is not loaded" + Fore.RESET

    self.testing_data = None
    if testing_dataset_path:
      self.testing_data = get_paths_of_files_from_path(testing_dataset_path)
      assert self.testing_data, Fore.RED + "Testing dataset is not loaded" + Fore.RESET

    # Load one image to get shape of it
    tmp_image = cv.imread(self.train_data[0])
    self.image_shape = tmp_image.shape
    self.image_channels = self.image_shape[2]

    # Check image size validity
    if self.image_shape[0] < 4 or self.image_shape[1] < 4: raise Exception("Images too small, min size (4, 4)")

    # Check validity of whole datasets
    if check_dataset:
      self.validate_dataset()

    # Define static vars
    if os.path.exists(f"{self.training_progress_save_path}/static_noise.npy"):
      self.static_noise = np.load(f"{self.training_progress_save_path}/static_noise.npy")
      if self.static_noise.shape[0] != (self.progress_image_dim[0] * self.progress_image_dim[1]):
        print(Fore.YELLOW + "Progress image dim changed, restarting static noise!" + Fore.RESET)
        os.remove(f"{self.training_progress_save_path}/static_noise.npy")
        self.static_noise = np.random.normal(0.0, 1.0, size=(self.progress_image_dim[0] * self.progress_image_dim[1], self.latent_dim))
    else:
      self.static_noise = np.random.normal(0.0, 1.0, size=(self.progress_image_dim[0] * self.progress_image_dim[1], self.latent_dim))
    self.kernel_initializer = RandomNormal(stddev=0.02)

    # Load checkpoint
    self.initiated = False
    loaded_gen_weights_path = None
    loaded_disc_weights_path = None
    if load_from_checkpoint:
      loaded_gen_weights_path, loaded_disc_weights_path = self.load_checkpoint()

    # Create batchmaker and start it
    self.batch_maker = BatchMaker(self.train_data, self.batch_size, buffered_batches=buffered_batches, num_of_loading_workers=num_of_loading_workers)

    self.testing_batchmaker = None
    if self.testing_data:
      self.testing_batchmaker = BatchMaker(self.testing_data, self.batch_size, buffered_batches=buffered_batches, num_of_loading_workers=num_of_loading_workers)
      self.testing_batchmaker.start()

    #################################
    ###   Create discriminator    ###
    #################################
    self.discriminator = self.build_discriminator(disc_mod_name)
    self.discriminator.compile(loss="binary_crossentropy", optimizer=discriminator_optimizer)

    #################################
    ###     Create generator      ###
    #################################
    self.generator = self.build_generator(gen_mod_name)
    if self.generator.output_shape[1:] != self.image_shape: raise Exception("Invalid image input size for this generator model")

    #################################
    ### Create combined generator ###
    #################################
    noise_input = Input(shape=(self.latent_dim,), name="noise_input")
    gen_images = self.generator(noise_input)

    # Create frozen version of discriminator
    frozen_discriminator = Network(self.discriminator.inputs, self.discriminator.outputs, name="frozen_discriminator")
    frozen_discriminator.trainable = False

    # Discriminator takes images and determinates validity
    valid = frozen_discriminator(gen_images)

    # Combine models
    # Train generator to fool discriminator
    self.combined_generator_model = Model(noise_input, valid, name="dcgan_model")
    self.combined_generator_model.compile(loss="binary_crossentropy", optimizer=self.generator_optimizer)

    # Print all summaries
    print("\nDiscriminator Summary:")
    self.discriminator.summary()
    print("\nGenerator Summary:")
    self.generator.summary()
    print("\nGAN Summary")
    self.combined_generator_model.summary()

    # Load weights from checkpoint
    try:
      if loaded_gen_weights_path: self.generator.load_weights(loaded_gen_weights_path)
    except:
      print(Fore.YELLOW + "Failed to load generator weights from checkpoint" + Fore.RESET)

    try:
      if loaded_disc_weights_path: self.discriminator.load_weights(loaded_disc_weights_path)
    except:
      print(Fore.YELLOW + "Failed to load discriminator weights from checkpoint" + Fore.RESET)

    # Load weights from param and override checkpoint weights
    if generator_weights: self.generator.load_weights(generator_weights)
    if discriminator_weights: self.discriminator.load_weights(discriminator_weights)

  # Function for creating gradient generator
  def gradient_norm_generator(self):
    grads = K.gradients(self.combined_generator_model.total_loss, self.combined_generator_model.trainable_weights)
    summed_squares = [K.sum(K.square(g)) for g in grads]
    norm = K.sqrt(sum(summed_squares))
    inputs = self.combined_generator_model._feed_inputs + self.combined_generator_model._feed_targets + self.combined_generator_model._feed_sample_weights
    func = K.function(inputs, [norm])
    return func

  # Check if datasets have consistent shapes
  def validate_dataset(self):
    def check_image(image_path):
      im_shape = imagesize.get(image_path)
      if im_shape[0] != self.image_shape[0] or im_shape[1] != self.image_shape[1]:
        return False
      return True

    print(Fore.BLUE + "Checking dataset validity" + Fore.RESET)
    with ThreadPool(processes=8) as p:
      res = p.map(check_image, self.train_data)
      if not all(res): raise Exception("Inconsistent training dataset")

      if self.testing_data:
        res = p.map(check_image, self.testing_data)
        if not all(res): raise Exception("Inconsistent testing dataset")

    print(Fore.BLUE + "Dataset valid" + Fore.RESET)

  # Create generator based on template selected by name
  def build_generator(self, model_name:str):
    noise = Input(shape=(self.latent_dim,))

    try:
      m = getattr(generator_models_spreadsheet, model_name)(noise, self.image_shape, self.image_channels, self.kernel_initializer)
    except Exception as e:
      raise Exception(f"Generator model not found!\n{e}")

    return Model(noise, m, name="generator_model")

  # Create discriminator based on teplate selected by name
  def build_discriminator(self, model_name:str, classification:bool=True):
    img = Input(shape=self.image_shape)

    try:
      m = getattr(discriminator_models_spreadsheet, model_name)(img, self.kernel_initializer)
    except Exception as e:
      raise Exception(f"Discriminator model not found!\n{e}")

    if classification:
      m = Dense(1, activation="sigmoid")(m)

    return Model(img, m, name="discriminator_model")

  def train(self, target_episode:int,
            feed_prev_gen_batch:bool=False, feed_old_perc_amount:float=0.2,
            progress_images_save_interval:int=None, save_raw_progress_images:bool=True, weights_save_interval:int=None,
            discriminator_smooth_real_labels:bool=False, discriminator_smooth_fake_labels:bool=False,
            generator_smooth_labels:bool=False):

    # Function for adding random noise to labels (flipping them)
    def noising_labels(labels: np.ndarray, noise_ammount:float=0.01):
      array = np.zeros(labels.shape)
      for idx in range(labels.shape[0]):
        if random.random() < noise_ammount:
          array[idx] = 1 - labels[idx]
          if array[idx] < 0: array[idx] = -array[idx]
        else:
          array[idx] = labels[idx]
      return labels

    # Function for replacing new generated images with old generated images
    def replace_random_images(orig_images: np.ndarray, repl_images: deque, perc_ammount:float=0.20):
      repl_images = np.array(repl_images)
      for idx in range(orig_images.shape[0]):
        if random.random() < perc_ammount:
          orig_images[idx] = repl_images[random.randint(0, repl_images.shape[0] - 1)]
      return orig_images

    # Check arguments and input data
    assert target_episode > 0, Fore.RED + "Invalid number of epochs" + Fore.RESET
    if progress_images_save_interval:
      assert progress_images_save_interval <= target_episode, Fore.RED + "Invalid progress save interval" + Fore.RESET
    if weights_save_interval:
      assert weights_save_interval <= target_episode, Fore.RED + "Invalid weights save interval" + Fore.RESET

    # Calculate epochs to go
    end_episode = target_episode
    target_episode = target_episode - self.episode_counter
    assert target_episode > 0, Fore.CYAN + "Training is already finished" + Fore.RESET

    # Save noise for progress consistency
    if progress_images_save_interval is not None:
      if not os.path.exists(self.training_progress_save_path): os.makedirs(self.training_progress_save_path)
      np.save(f"{self.training_progress_save_path}/static_noise.npy", self.static_noise)

    # Training variables
    prev_gen_images = deque(maxlen=3*self.batch_size)
    get_gradients = self.gradient_norm_generator()

    epochs_time_history = deque(maxlen=self.AGREGATE_STAT_INTERVAL * 50)

    # Save starting kernels and biases
    if not self.initiated:
      self.__save_imgs(save_raw_progress_images)
      self.tensorboard.log_kernels_and_biases(self.generator)
      self.save_checkpoint()

    print(Fore.GREEN + f"Starting training on episode {self.episode_counter} for {target_episode} episodes" + Fore.RESET)
    for _ in range(target_episode):
      ep_start = time.time()

      ### Train Discriminator ###
      # Select batch of valid images
      imgs = self.batch_maker.get_batch()

      # Sample noise and generate new images
      gen_imgs = self.generator.predict(np.random.normal(0.0, 1.0, (self.batch_size, self.latent_dim)))

      # Train discriminator (real as ones and fake as zeros)
      if discriminator_smooth_real_labels:
        disc_real_labels = np.random.uniform(0.8, 1.0, size=(self.batch_size, 1))
      else:
        disc_real_labels = np.ones(shape=(self.batch_size, 1))

      if discriminator_smooth_fake_labels:
        disc_fake_labels = np.random.uniform(0, 0.2, size=(self.batch_size, 1))
      else:
        disc_fake_labels = np.zeros(shape=(self.batch_size, 1))

      if feed_prev_gen_batch:
        if len(prev_gen_images) > 0:
          tmp_imgs = replace_random_images(gen_imgs, prev_gen_images, feed_old_perc_amount)
          prev_gen_images += deque(gen_imgs)
          gen_imgs = tmp_imgs
        else:
          prev_gen_images += deque(gen_imgs)

      # Adding random noise to discriminator labels
      if self.discriminator_label_noise and self.discriminator_label_noise > 0:
        disc_real_labels = noising_labels(disc_real_labels, self.discriminator_label_noise / 2)
        disc_fake_labels = noising_labels(disc_fake_labels, self.discriminator_label_noise / 2)

      self.discriminator.trainable = True
      disc_real_loss = self.discriminator.train_on_batch(imgs, disc_real_labels)
      disc_fake_loss = self.discriminator.train_on_batch(gen_imgs, disc_fake_labels)

      ### Train Generator ###
      # Train generator (wants discriminator to recognize fake images as valid)
      if generator_smooth_labels:
        gen_labels = np.random.uniform(0.8, 1.0, size=(self.batch_size, 1))
      else:
        gen_labels = np.ones(shape=(self.batch_size, 1))
      self.discriminator.trainable = False
      gan_loss = self.combined_generator_model.train_on_batch(np.random.normal(0.0, 1.0, (self.batch_size, self.latent_dim)), gen_labels)

      self.episode_counter += 1
      self.tensorboard.step = self.episode_counter
      self.tensorboard.update_stats(disc_real_loss=disc_real_loss, disc_fake_loss=disc_fake_loss, gan_loss=gan_loss, disc_label_noise=self.discriminator_label_noise if self.discriminator_label_noise else 0)

      # Decay label noise
      if self.discriminator_label_noise and self.discriminator_label_noise_decay:
        self.discriminator_label_noise = max([self.discriminator_label_noise_min, (self.discriminator_label_noise * self.discriminator_label_noise_decay)])

        if (self.discriminator_label_noise_min == 0) and (self.discriminator_label_noise != 0) and (self.discriminator_label_noise < 0.001):
          self.discriminator_label_noise = 0

      # Seve stats and print them to console
      if self.episode_counter % self.AGREGATE_STAT_INTERVAL == 0:
        self.tensorboard.log_kernels_and_biases(self.generator)

        # Change color of log according to state of training
        print(Fore.GREEN + f"{self.episode_counter}/{end_episode}, Remaining: {time_to_format(mean(epochs_time_history) * (end_episode - self.episode_counter))}\t\t[D-R loss: {round(float(disc_real_loss), 5)}, D-F loss: {round(float(disc_fake_loss), 5)}] [G loss: {round(float(gan_loss), 5)}] - Epsilon: {round(self.discriminator_label_noise, 4) if self.discriminator_label_noise else 0}" + Fore.RESET)

      # Save progress
      if self.training_progress_save_path is not None and progress_images_save_interval is not None and self.episode_counter % progress_images_save_interval == 0:
        self.__save_imgs(save_raw_progress_images)

      # Save weights of models
      if weights_save_interval is not None and self.episode_counter % weights_save_interval == 0:
        self.__save_weights()

      # Save checkpoint
      if self.episode_counter % self.CHECKPOINT_SAVE_INTERVAL == 0:
        self.save_checkpoint()
        print(Fore.BLUE + "Checkpoint created" + Fore.RESET)

      if self.episode_counter % self.GRADIENT_CHECK_INTERVAL == 0:
        # Generate evaluation noise and labels
        eval_noise = np.random.normal(0.0, 1.0, (self.batch_size, self.latent_dim))
        eval_labels = np.ones(shape=(self.batch_size, 1))

        # Create gradient function and evaluate based on eval noise and labels
        norm_gradient = get_gradients([eval_noise, eval_labels, np.ones(len(eval_labels))])[0]

        # Check norm gradient
        if norm_gradient > 100 and self.episode_counter > self.CONTROL_THRESHOLD:
          print(Fore.RED + f"Current generator norm gradient: {norm_gradient}")
          print("Gradient too high!" + Fore.RESET)
          if input("Do you want exit training?\n") == "y": return
        elif norm_gradient < 0.2 and self.episode_counter > self.CONTROL_THRESHOLD:
          print(Fore.RED + f"Current generator norm gradient: {norm_gradient}")
          print("Gradient vanished!" + Fore.RESET)
          if input("Do you want exit training?\n") == "y": return
        else:
          print(Fore.BLUE + f"Current generator norm gradient: {norm_gradient}" + Fore.RESET)

        # Change seed
        np.random.seed(None)
        random.seed()

      epochs_time_history.append(time.time() - ep_start)

    # Shutdown helper threads
    print(Fore.GREEN + "Training Complete - Waiting for other threads to finish" + Fore.RESET)
    if self.testing_batchmaker: self.testing_batchmaker.__terminate = True
    self.batch_maker.terminate()
    self.save_checkpoint()
    self.__save_weights()
    self.batch_maker.join()
    if self.testing_batchmaker: self.testing_batchmaker.join()
    print(Fore.GREEN + "All threads finished" + Fore.RESET)

  # Function for saving progress images
  def __save_imgs(self, save_raw_progress_images:bool=True):
    if not os.path.exists(self.training_progress_save_path + "/progress_images"): os.makedirs(self.training_progress_save_path + "/progress_images")
    gen_imgs = self.generator.predict(self.static_noise)

    # Rescale images 0 to 255
    gen_imgs = (0.5 * gen_imgs + 0.5) * 255

    final_image = np.zeros(shape=(self.image_shape[0] * self.progress_image_dim[1], self.image_shape[1] * self.progress_image_dim[0], self.image_channels)).astype(np.float32)

    cnt = 0
    for i in range(self.progress_image_dim[1]):
      for j in range(self.progress_image_dim[0]):
        if self.image_channels == 3:
          final_image[self.image_shape[0] * i:self.image_shape[0] * (i + 1), self.image_shape[1] * j:self.image_shape[1] * (j + 1), :] = gen_imgs[cnt]
        else:
          final_image[self.image_shape[0] * i:self.image_shape[0] * (i + 1), self.image_shape[1] * j:self.image_shape[1] * (j + 1), 0] = gen_imgs[cnt, :, :, 0]
        cnt += 1
    final_image = cv.cvtColor(final_image, cv.COLOR_RGB2BGR)

    if save_raw_progress_images:
      cv.imwrite(f"{self.training_progress_save_path}/progress_images/{self.episode_counter}.png", final_image)
    self.tensorboard.write_image(np.reshape(cv.cvtColor(final_image, cv.COLOR_BGR2RGB) / 255, (-1, final_image.shape[0], final_image.shape[1], final_image.shape[2])).astype(np.float32))

  def save_models_structure_images(self):
    save_path = self.training_progress_save_path + "/model_structures"
    if not os.path.exists(save_path): os.makedirs(save_path)
    plot_model(self.combined_generator_model, os.path.join(save_path, "combined.png"), expand_nested=True, show_shapes=True)
    plot_model(self.generator, os.path.join(save_path, "generator.png"), expand_nested=True, show_shapes=True)
    plot_model(self.discriminator, os.path.join(save_path, "discriminator.png"), expand_nested=True, show_shapes=True)

  def load_checkpoint(self):
    checkpoint_base_path = os.path.join(self.training_progress_save_path, "checkpoint")
    if not os.path.exists(os.path.join(checkpoint_base_path, "checkpoint_data.json")): return None, None

    with open(os.path.join(checkpoint_base_path, "checkpoint_data.json"), "rb") as f:
      data = json.load(f)

      if data:
        self.episode_counter = int(data["episode"])
        if data["disc_label_noise"]:
          self.discriminator_label_noise = float(data["disc_label_noise"])
        self.initiated = True
        return data["gen_path"], data["disc_path"]
      return None, None

  def save_checkpoint(self):
    checkpoint_base_path = os.path.join(self.training_progress_save_path, "checkpoint")
    if not os.path.exists(checkpoint_base_path): os.makedirs(checkpoint_base_path)

    gen_path = f"{checkpoint_base_path}/generator_{self.gen_mod_name}.h5"
    disc_path = f"{checkpoint_base_path}/discriminator_{self.disc_mod_name}.h5"

    if os.path.exists(gen_path): os.rename(gen_path, f"{checkpoint_base_path}/generator_{self.gen_mod_name}.h5.lock")
    if os.path.exists(disc_path): os.rename(disc_path, f"{checkpoint_base_path}/discriminator_{self.disc_mod_name}.h5.lock")

    self.generator.save_weights(gen_path)
    self.discriminator.save_weights(disc_path)

    if os.path.exists(f"{checkpoint_base_path}/generator_{self.gen_mod_name}.h5.lock"): os.remove(f"{checkpoint_base_path}/generator_{self.gen_mod_name}.h5.lock")
    if os.path.exists(f"{checkpoint_base_path}/discriminator_{self.disc_mod_name}.h5.lock"): os.remove(f"{checkpoint_base_path}/discriminator_{self.disc_mod_name}.h5.lock")

    data = {
      "episode": self.episode_counter,
      "gen_path": gen_path,
      "disc_path": disc_path,
      "disc_label_noise": self.discriminator_label_noise
    }

    with open(os.path.join(checkpoint_base_path, "checkpoint_data.json"), "w", encoding='utf-8') as f:
      json.dump(data, f)

  def __save_weights(self):
    save_dir = self.training_progress_save_path + "/weights/" + str(self.episode_counter)
    if not os.path.exists(save_dir): os.makedirs(save_dir)
    self.generator.save_weights(f"{save_dir}/generator_{self.gen_mod_name}.h5")
    self.discriminator.save_weights(f"{save_dir}/discriminator_{self.disc_mod_name}.h5")

  def make_progress_gif(self, frame_duration:int=16):
    if not os.path.exists(self.training_progress_save_path): os.makedirs(self.training_progress_save_path)
    if not os.path.exists(self.training_progress_save_path + "/progress_images"): return

    frames = []
    img_file_names = os.listdir(self.training_progress_save_path + "/progress_images")

    for im_file in img_file_names:
      if os.path.isfile(self.training_progress_save_path + "/progress_images/" + im_file):
        frames.append(Image.open(self.training_progress_save_path + "/progress_images/" + im_file))

    if len(frames) > 2:
      frames[0].save(f"{self.training_progress_save_path}/progress_gif.gif", format="GIF", append_images=frames[1:], save_all=True, optimize=False, duration=frame_duration, loop=0)


""".alive Plugin for @UniBorg"""
import asyncio
from telethon import events
from telethon.tl.types import ChannelParticipantsAdmins
from uniborg.util import admin_cmd

@borg.on(admin_cmd(pattern="alive"))
async def _(event):
    if event.fwd_from:
        return
    mentions = "`AƁƐƳ SAALƐ!╰(°ㅂ°)╯Ƶιη∂α нυ мαι! Mααƒ кαяηα gυѕѕє мє ι∂нαя υ∂нαя ηιкαℓ נαтα нυ..\n\nƬєℓєтнση νєяѕιση:69.69\nƤутнση:6.9\nƤєяυ Uѕєя:`[█▬█ █ ▀█▀](tg://user?id=705627922) \n`Ɓσт Ƈяєαтσя:`@CallMe_HIT\n\n`Sєχвαѕє Sтαтυѕ:Ƭєℓєgяαм ѕєχвαѕєѕ ƒυηcтισηιηg ησямαℓℓу!\n\nI am ꔠ༏ⲧᏰ❍ⲧ swagat toh karo hamara...`\n\n**Ƈαℓℓ Jσнηηу Sιηѕ тσ ∂єρℓσу тнιѕ υѕєявσт ησω:**+916969696969"
    chat = await event.get_input_chat()
    async for x in borg.iter_participants(chat, filter=ChannelParticipantsAdmins):
        mentions += f""
    reply_message = None
    if event.reply_to_msg_id:
        reply_message = await event.get_reply_message()
        await reply_message.reply(mentions)
    else:
        await event.reply(mentions)
    await event.delete()


from flask import Flask, render_template, request, redirect
import sqlite3
from flask.helpers import url_for
import folium
from flask_paginate import Pagination, get_page_parameter

app = Flask(__name__)

@app.route('/')
def base():
    conn = sqlite3.connect('quotes.db')
    conn.row_factory = sqlite3.Row
    cur = conn.cursor()
    cur.execute("select * from webdata")
    datas = cur.fetchall();
    conn.close()
    return render_template('index.html', datas = datas)

@app.route('/folium')
def maps():
    start_coords = (42, 43)
    folium_map = folium.Map(location=start_coords, zoom_start=8)
    folium_map.save('templates/folmap.html')
    return render_template('maps.html')

@app.route('/', methods=['GET', 'POST'])
def search():
    a = request.form['search-input']
    conn = sqlite3.connect('quotes.db')
    conn.row_factory = sqlite3.Row
    cur = conn.cursor()
    cur.execute("select * from webdata where content=? or name=?", (a,a))
    datas = cur.fetchall();
    conn.close()
    return render_template("search.html", datas = datas)

"""@app.route('/', methods=['GET', 'POST'])
def query_search():
    a = request.form['search-input']
    return redirect(url_for("search"), a=a)"""

if __name__== '__main__':
    app.run(debug=True)


import numpy as np
import cv2
import pickle
import pprint

import os
import sys

this_path = os.path.dirname(__file__)

def outputter(file_name, val):
    path = this_path+'/temp/'+file_name
    with open(path, 'wb') as f:
        pickle.dump(val, f, protocol=2)
    print('outputted file to ' + path)

def opener(file_path, message):
    try:
        with open(this_path+'/temp/'+file_path, 'rb') as f:
            data = pickle.load(f)
            pprint.pprint(data)
    except IOError:
        print(message)
        exit()
    return data

def getCameraMatrix():
    return opener('camera_matrix', 'Camera matrix not fonud. Please run `calibration/realtimeCalibrate.py` or `realtimeCalibrate_wide.py` before.')

def getStereoMatrix():
    return opener('stereo_matrices', 'stereo_matrices not found. Please run `calibration/stereoCalibrate_wide.py` before.')

def getStereoRect():
    return opener('stereo_rect', 'oops!')

def getStereoMap():
    return opener('stereo_map', 'ooops!')

def getUndistortedImage(image, cam_mtx, mode=0):
    h, w = image.shape[:2]
    newcameramtx, roi = cv2.getOptimalNewCameraMatrix(cam_mtx['mtx'], cam_mtx['dist'], (w,h), mode, (w,h))
    mapx, mapy = cv2.initUndistortRectifyMap(cam_mtx['mtx'], cam_mtx['dist'], None, newcameramtx, (w,h), 5)
    dst = cv2.remap(image, mapx, mapy, cv2.INTER_LINEAR)

    x, y, w, h = roi
    dst = dst[y:y+h, x:x+w]
    return dst

def getUndistortedImage_wide(image, cam_mtx):
    h, w = image.shape[:2]
    map1, map2 = cv2.fisheye.initUndistortRectifyMap(cam_mtx['mtx'], cam_mtx['dist'], np.eye(3), cam_mtx['mtx'], (w,h), cv2.CV_16SC2)
    undistorted_img = cv2.remap(image, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
    return undistorted_img

# restration removed pixels by remap
def getUndistortedImage_wide_fixed(image, cam_mtx):
    balance = 0.8 # 1 is full
    h, w = image.shape[:2]
    K = cam_mtx['mtx']
    D = cam_mtx['dist']
    dim1 = (w, h)
    assert dim1[0]/dim1[1]

    dim2 = dim1
    dim3 = dim1

    scaled_K = K * dim1[0] / h
    scaled_K[2][2] = 1.0 # Except that K[2][2] is always 1.0

    new_K = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify(scaled_K, D, dim2, np.eye(3), balance=balance)
    map1, map2 = cv2.fisheye.initUndistortRectifyMap(scaled_K, D, np.eye(3), new_K, dim3, cv2.CV_16SC2)
    undistorted_img = cv2.remap(image, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)

    return undistorted_img


"""
Program that smooths boundary data
Author: Noah Gaeta
"""
import re
import numpy
from pykml import parser
import matplotlib.pyplot as plt

REGION_FILE_PATH = './region.kml'
SAVE_PATH = './region_no_outliers.kml'  # lets not over write the file just in case

def main():
    coordinates = get_coordinates_kml_file(REGION_FILE_PATH)
    plot_coordinates(coordinates)  # plot original values
    coordinates = remove_outliers(coordinates)
    plot_coordinates(coordinates)  # plot values after removal of outliers
    save_coordinates_kml_file(REGION_FILE_PATH, SAVE_PATH, coordinates)
    print("New kml file saved to: ", SAVE_PATH)
    return 0

def get_kml_etree(file_path):
    """ Generate element tree from kml file """
    with open(file_path) as file:
        return parser.parse(file)

def get_coordinates_kml_file(file_path):
    """ Gets coordinates from kml file """
    root = get_kml_etree(file_path).getroot()
    coordinates = str(root.Document.Placemark.Polygon.outerBoundaryIs.LinearRing.coordinates)  # get coordinates from kml
    coordinates = re.sub(r',0', '', coordinates)  # get rid 0's we don't need them
    coordinates = coordinates.split()  # split string by space
    # split each string in coordinates list by ',' then convert each element in said list to floating point number
    return numpy.array(list(map(lambda cords: list(map(lambda cord: float(cord), cords.split(','))), coordinates)))

def save_coordinates_kml_file(load_path, save_path, coordinates):
    """  Save modified kml file """
    etree = get_kml_etree(load_path)
    root = etree.getroot()
    coords_str = format_coords_into_string(coordinates)
    root.Document.Placemark.Polygon.outerBoundaryIs.LinearRing.coordinates = coords_str
    etree.write(save_path, pretty_print=True)

def format_coords_into_string(coordinates):
    """ Formats coordinates list into original string format """
    coords_str = ''
    for coord in coordinates:
        for i in range(0, 2):
            coords_str += str(coord[i])
            if i == 1:
                coords_str += ',0 '
            else:
                coords_str += ','
    return coords_str

def plot_coordinates(coordinates):
    x, y = coordinates.T  # get x, y coordinates
    plt.scatter(x, y)
    plt.show()

def remove_outliers(coordinates):
    """ Removes outliers from coordinate list """
    x, y = coordinates.T  # split coordinates into their own lists
    outliers_x = find_outliers(x)  # find x outliers
    outliers_y = find_outliers(y)  # find y outliers
    no_outliers = []
    for coordinate in coordinates:
        # append to list only if coordinate does not contain outlier
        if coordinate[0] not in outliers_x and coordinate[1] not in outliers_y:
            no_outliers.append(coordinate)
    return numpy.array(no_outliers)

def find_outliers(flat_list):
    """ Find outliers """
    sd = numpy.std(flat_list)  # get standard deviation of the elements in the list
    mean = numpy.mean(flat_list, axis=0)  # get the mean of the elements in the list
    final_list = []
    for cord in flat_list:
        if mean + 2 * sd < cord > mean - 2 * sd:  # check if outliers
            final_list.append(cord)
    return final_list

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django.contrib import messages
from django.http import HttpResponseRedirect
from django.contrib.auth.models import User
from login.forms import LoginForm, SignupForm
from login.models import Employee
# Create your views here.

#for the login button on index page
def index(request):
	if request.user.is_authenticated():
		request.session['username']=request.user.user
		u=request.user
		e=Employee.objects.get(user=request.user)
		return render(request, 'login/profile.html', {"employee": e, "user": u})
	else:

		idpwd = LoginForm()
		return render(request, 'login/index.html', {'form': idpwd})

# for the logout button anywhere
def log_out(request):
	logout(request)
	return HttpResponseRedirect('login:index')

#for the signup button on signup page
def signup(request):
	if request.user.is_authenticated():
		request.session['username'] = request.user.username
		return HttpResponseRedirect('login:index')
	elif request.method == 'POST':
		details = SignupForm()
		if details.is_valid():
			u=User.objects.create(**UserForm.cleaned_data)
			e=Employee.objects.create(user=u, **SignupForm.cleaned_data)
			e.save()
			HttpResponseRedirect('login:index')
		else:
			return render(request, 'login/signup.html', {'form': details})

#for the profile page
def profile(request):
	if request.user.is_authenticated():
		request.session['username'] = request.user.username
		return HttpResponseRedirect('login:index')
	elif request.method == 'POST':
		e=Employee.objects.get(user=request.user)
		return render(request, 'login/profile.html', {'emp':e})


import os
import cv2

fps = 15

img_arrays = {"cloudy2" : [], "cloudy": [], "night": [], "rainy": [], "sunny": []}

model = "haar"

for frame in os.listdir(f"{model}-images"):

    name = frame[:-9]

    img = cv2.imread(f"{model}-images/{frame}")

    height, width, layers = img.shape
    size = (width, height)

    img_arrays[name].append(img)

for name in img_arrays:

    img_array = img_arrays[name]
    h, w, _ = img_array[0].shape
    out = cv2.VideoWriter(f"{model}-videos/{name}.avi", cv2.VideoWriter_fourcc(*'DIVX'), fps, (w,h))

    for i in range(len(img_array)):
        img = img_array[i][0:h,0:w]
        out.write(img)

    out.release()


"""Example script for setting up and solving a flexible load optimal operation problem."""

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pyomo.environ as pyo

import fledge.config
import fledge.database_interface
import fledge.der_models
import fledge.electric_grid_models

def main():

    # Settings.
    scenario_name = 'singapore_6node'
    plots = True  # If True, script may produce plots.

    # Recreate / overwrite database, to incorporate changes in the CSV files.
    fledge.database_interface.recreate_database()

    # Obtain data.
    scenario_data = fledge.database_interface.ScenarioData(scenario_name)
    der_data = fledge.database_interface.DERData(scenario_name)
    price_data = fledge.database_interface.PriceData(scenario_name)

    # Obtain price timeseries.
    price_name = 'energy'
    price_timeseries = price_data.price_timeseries_dict[price_name]

    # Obtain model.
    der_name = der_data.flexible_loads['der_name'][0]  # Pick first `der_name`.
    flexible_load_model = fledge.der_models.FlexibleLoadModel(der_data, der_name)

    # Instantiate optimization problem.
    optimization_problem = pyo.ConcreteModel()

    # Define variables.
    flexible_load_model.define_optimization_variables(optimization_problem)

    # Define constraints.
    flexible_load_model.define_optimization_constraints(optimization_problem)

    # Define objective.
    flexible_load_model.define_optimization_objective(optimization_problem, price_timeseries)

    # Solve optimization problem.
    optimization_solver = pyo.SolverFactory(fledge.config.solver_name)
    optimization_result = optimization_solver.solve(optimization_problem, tee=fledge.config.solver_output)
    try:
        assert optimization_result.solver.termination_condition is pyo.TerminationCondition.optimal
    except AssertionError:
        raise AssertionError(f"Solver termination condition: {optimization_result.solver.termination_condition}")
    # optimization_problem.display()

    # Obtain results.
    (
        state_vector,
        control_vector,
        output_vector
    ) = flexible_load_model.get_optimization_results(
        optimization_problem
    )

    # Print results.
    print(f"state_name = \n{state_vector.to_string()}")
    print(f"control_name = \n{control_vector.to_string()}")
    print(f"output_name = \n{output_vector.to_string()}")

    # Plot results.
    if plots:

        for output_name in flexible_load_model.output_names:
            plt.plot(flexible_load_model.output_maximum_timeseries[output_name], label="Maximum", drawstyle='steps-post')
            plt.plot(flexible_load_model.output_minimum_timeseries[output_name], label="Minimum", drawstyle='steps-post')
            plt.plot(output_vector[output_name], label="Optimal", drawstyle='steps-post')
            plt.legend()
            plt.title(f"Output: {output_name}")
            plt.show()
            plt.close()

        plt.plot(price_timeseries['price_value'], drawstyle='steps-post')
        plt.title(f"Price: {price_name}")
        plt.show()
        plt.close()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
#
# - Main role of this file is to remove polling in tasktracker.
# - should be running when the tasktracker(mdserv.py) is launched.
# - recommend to use daemon mode.
# 		> nohup ./[this file's name] &
#
# @modified: June 17th, 2012
# @author: Yurong Jiang, Xing Xu, Moo-Ryong Ra
#
import time, os, copy, sys, random, threading, socket, select
import mdlib
import BaseHTTPServer, urlparse, cgi
from xml.etree import ElementTree as ET

HOSTNAME = 'localhost'
NPORT = 7667
CONN = {}

class MedTTCbHandler(BaseHTTPServer.BaseHTTPRequestHandler):
	#
	def do_HEAD(s):
		s.send_response(200)
		s.send_header("Content-type", "text/html")
		s.end_headers()
	#
	def do_POST(self):
		ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))

		if ctype == 'multipart/form-data':
			postvars = cgi.parse_multipart(self.rfile, pdict)
		elif ctype == 'application/x-www-form-urlencoded':
			length = int(self.headers.getheader('content-length'))
			postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
			process_file(postvars)
		else:
			postvars = {}

		self.send_response(200)
		self.send_header("Content-type", "text/html")
		self.end_headers()

class MedTTEventRelay(threading.Thread):
	#
	def run(self):
		global CONN

		soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		soc.bind(('localhost', 20968))
		soc.listen(5)

		while True:
			soc_input = [soc]
			for (k, v) in CONN.iteritems():
				soc_input.append(v)

			_in, _out, _exc = select.select(soc_input, [], [])

			for s in _in:
				if s == soc:
					(c, a) = soc.accept()
					pid = mdlib.msg_recv(c, 1024)
					CONN[str(pid)] = c

					mdlib.log("* pid registered: "+ pid)
				else:
					temp = s.recv(1024)
					if len(temp) == 0:
						mdlib.log('* closing socket')
						s.close()

					for (k, v) in CONN.iteritems():
						if s == v:
							mdlib.log('* ' + str(k) + ' closed.')
							del CONN[k]
							break

def run_medscript(prognames):
	#
	global CONN, soc

	for i in prognames:
		newpid = os.fork()

		if newpid == 0:
			cmd = "python mdserv.py %s" % i
			os.system(cmd)
			os.remove(i)
			os._exit(1)
		else:
			pids = (os.getpid(), newpid)
			mdlib.log("* parent: %d, child: %d" % pids)

def generate_medscript(filename):
	#
	ori_xml = ET.parse(filename)
	input = ori_xml.findall('app/input')
	root = ori_xml.find('app')

	for i in input:
		root.remove(i)

	ret_val = []
	x = 0

	for i in input:
		x = x + 1
		tree = copy.deepcopy(ori_xml)
		test = tree.find('app')
		for j in range(len(i)):
			test.insert(0, i[len(i) - j - 1])
		fname ="program/"+ "%s_%s.xml" % (str(int(time.time())), str(int(random.random()*10000)))
		tree.write(fname)
		ret_val.append(fname)

	os.system("rm %s" % filename)

	return ret_val

def process_file(in_file):
	#
	global CONN, soc
	pair = in_file.popitem()

	mdlib.log("* pair: " + pair[1][0] + ", " + pair[0])

	if pair[0] == "medusa_rss":
		info = pair[1][0].split(',')
		pid = info[1]
		qid = info[2]

		mdlib.log("* pid=%s qid=%s" % (str(pid),str(qid)))

		if CONN.has_key(pid):
			mdlib.msg_send(CONN[(pid)], pair[1][0])
	else:
		t_filename = "%s%s" % (str(int(random.random()*10000)), pair[0])
		prognames = generate_medscript(t_filename)
		run_medscript(prognames)

if __name__ == '__main__':
	# initialization
	t = MedTTEventRelay()
	t.start()

	server_class = BaseHTTPServer.HTTPServer
	httpd = server_class((HOSTNAME, NPORT), MedTTCbHandler)

	mdlib.log("* starting medscript_acceptor - %s:%s" % (HOSTNAME, NPORT))

	# event-loop.
	httpd.serve_forever()

	# interrupted.
	for i in CONN:
		CONN[i].close()

	httpd.server_close()
	mdlib.log("* stopping medscript_acceptor - %s:%s" % (HOSTNAME, NPORT))


import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

cor = []
cen = []
equ = []
for i in range(325):
    print("Counting %d.csv ..." % i)
    cor += [len(open("Correct/%d.csv" % i, 'r').readlines()) / 1000]
    cen += [len(open("Centralized/%d.csv" % i, 'r').readlines()) / 1000]
    equ += [len(open("Equalized/%d.csv" % i, 'r').readlines()) / 1000]

print("Plotting...")
figure = plt.figure(figsize=(1800/300, 600/300), dpi=300)
plt.subplot(131)
plt.plot(range(325), cor, 'r', label="Correct", lw=0.3)
plt.xlabel("Records(K)")
plt.legend(fontsize="xx-small", loc="upper left")
plt.subplot(132)
plt.plot(range(325), cen, 'b', label="Centralized", lw=0.3)
plt.xlabel("Records(K)")
plt.legend(fontsize="xx-small", loc="upper left")
plt.subplot(133)
plt.plot(range(325), equ, 'g', label="Equalized", lw=0.3)
plt.xlabel("Records(K)")
plt.legend(fontsize="xx-small", loc="upper left")
plt.savefig("Count.png")
print("Done.")


from backend.settings import BASE_DIR
import math, os, json

def art_calculator(parameters):

    # some parameters require scores
    # import json file from data/calculations
    path_to_json = os.path.join(BASE_DIR, 'data/calculations/art.json')
    f = open(path_to_json, 'r')
    scores = f.read()
    f.close()
    scores = json.loads(scores)

    form = parameters['physical_form']
    # E: intrinsic property
    # liquid

    if form == 'liquid':
        vp = float(parameters['vp'])
        mf = float(parameters['mf'])

        # vapours
        if vp > 10:
            E = vp * mf / 30000
        # mists
        else:
            E = scores['viscosity'][parameters['viscosity']]

    # activity class - subclass - underlying determinants
    d1 = 1
    d2 = 1
    d3 = 1
    d4 = 1

    if 'd1' in parameters:
        if parameters['d1'] != False:
            d1 = scores['d1'][ parameters['d1'] ]

    if 'd2' in parameters:
        if parameters['d2'] != False:
            d2 = scores['d2'][ parameters['d2'] ]

    if 'd3' in parameters:
        if parameters['d3'] != False:
            d3 = scores['d3'][ parameters['d3'] ]

    if 'd4' in parameters:
        if parameters['d4'] != False:
            d4 = scores['d4'][ parameters['d4'] ]

    H = d1 * d2 * d3 * d4

    # Local controls
    Lc1 = 1
    Lc2 = 1
    if 'lc1' in parameters:
        if parameters['lc1'] == 'vrs':
            Lc1 = 0.2
        elif parameters['lc1'] != 'no':
            Lc1 = scores['lc'][ parameters['lc1_tech'] ]

        # secondary LC
        if parameters['lc1'] != 'no':
            if parameters['lc2'] == 'vrs':
                Lc2 = 0.2
            else:
                Lc2 = scores['lc'][ parameters['lc2_tech'] ]

    Lc = Lc1 * Lc2

    # calculated MF above used to evalueate Score
    # and score is quantified to calculate exposure
    score = E * H * Lc

    # set input quantification parameter - alpha
    alpha = 10.56
    if form == 'liquid':
        if vp <= 10:
            apha = 10.23

    elif form == 'powder':
        alpha = 3.01
    elif form == 'solid':
        alpha = 0.48
    elif form == 'dissolved':
        apha = 10.23
    elif form == 'paste':
        apha = 3.01

    gm = math.exp(math.log(score) + alpha)
    gm = round(gm)

    exposure = {
        "p50": gm,
        "p75": gm*1.5,
        "p90": gm*2,
        "p95": gm*3,
        "p99": gm*3.8,
        "p50_ci75": [0.05, 0.15],
        "p50_ci80": [0.05, 0.15],
        "p50_ci90": [0.05, 0.15],
        "p50_ci95": [0.05, 0.15],
        "p75_ci75": [0.05, 0.15],
        "p75_ci80": [0.05, 0.15],
        "p75_ci90": [0.05, 0.15],
        "p75_ci95": [0.05, 0.15],
        "p90_ci75": [0.05, 0.15],
        "p90_ci80": [0.05, 0.15],
        "p90_ci90": [0.05, 0.15],
        "p90_ci95": [0.05, 0.15],
        "p95_ci75": [0.05, 0.15],
        "p95_ci80": [0.05, 0.15],
        "p95_ci90": [0.05, 0.15],
        "p95_ci95": [0.05, 0.15],
        "p99_ci75": [0.05, 0.15],
        "p99_ci80": [0.05, 0.15],
        "p99_ci90": [0.05, 0.15],
        "p99_ci95": [0.05, 0.15]
    }
    return exposure

def sm_calculator(parameters):
    gm = 3
    exposure = {
        "p50": gm,
        "p75": gm*1.5,
        "p90": gm*2,
        "p95": gm*3
    }
    return exposure

def tra_calculator(parameters):
    exposure = {"p75": 2}
    return exposure


#!/usr/env python
"""
.. module:: convert_kids450photometry_h5py
:synopsis: Script to get the KIDS 450 photometry from the shear catalog and save as a h5 file.
.. moduleauthor:: Maria Elidaiana <mariaeli@brandeis.edu>
"""

import numpy as np
#from astropy.io import fits
from astropy.table import Table, vstack
import h5py as h5

kidsdir = '/global/projecta/projectdirs/lsst/groups/WL/projects/wl-txpipe-hack/KIDS/KiDS450_DR3/'

def getdata(filename):
    kids_shear = Table.read(kidsdir + filename, memmap=True)
    return kids_shear

kidsfiles = ['KiDS_DR3.1_G9_ugri_shear.fits', 'KiDS_DR3.1_G12_ugri_shear.fits',
             'KiDS_DR3.1_G15_ugri_shear.fits', 'KiDS_DR3.1_G23_ugri_shear.fits',
             'KiDS_DR3.1_GS_ugri_shear.fits']
tables = []
for i in range(len(kidsfiles)):
    d = getdata(kidsfiles[i])
    tables.append(d)

#Joining the tables
kids_shearall = vstack(tables)

#Sorting by ID
kids_shearall.sort('ID')

#Getting the photometry columns
tilename  = kids_shearall['KIDS_TILE']
dec       = kids_shearall['DECJ2000']
g_mag     = kids_shearall['MAG_g']
g_mag_err = kids_shearall['MAGERR_g']
i_mag     = kids_shearall['MAG_i']
i_mag_err = kids_shearall['MAGERR_i']
objectId  = kids_shearall['ID']
r_mag     = kids_shearall['MAG_r']
r_mag_err = kids_shearall['MAGERR_r']
ra        = kids_shearall['RAJ2000']
snr_g     = 1.086/g_mag_err
snr_i     = 1.086/i_mag_err
snr_r     = 1.086/r_mag_err
u_mag     = kids_shearall['MAG_u']
u_mag_err = kids_shearall['MAGERR_u']
snr_u     = 1.086/u_mag_err
y_mag     = kids_shearall['MAG_i']    #Placeholder
y_mag_err = kids_shearall['MAGERR_i'] #Placeholder
snr_y     = 1.086/y_mag_err           #Placeholder
z_mag     = kids_shearall['MAG_i']    #Placeholder
z_mag_err = kids_shearall['MAGERR_i'] #Placeholder
snr_z     = 1.086/z_mag_err           #Placeholder

#Dealing with unicode, string needs to be S12
tilename = np.array([a.encode('utf8') for a in tilename])
objectId = np.array([a.encode('utf8') for a in objectId])

data   = [tilename, dec, g_mag, g_mag_err, i_mag, i_mag_err, objectId, r_mag, r_mag_err, ra, snr_g, snr_i, snr_r, u_mag, u_mag_err, snr_u, y_mag, y_mag_err, snr_y, z_mag, z_mag_err, snr_z]
dnames = ['tilename', 'dec', 'g_mag', 'g_mag_err', 'i_mag', 'i_mag_err', 'objectId', 'r_mag', 'r_mag_err', 'ra', 'snr_g', 'snr_i', 'snr_r', 'u_mag', 'u_mag_err', 'snr_u', 'y_mag', 'y_mag_err', 'snr_y', 'z_mag', 'z_mag_err', 'snr_z']

outputdir = '/global/cscratch1/sd/elp25/txpipe-reanalysis/data/kids/'
#Saving the h5 file...
f = h5.File(outputdir + 'photometry_catalog_kids450.h5', 'w')
g = f.create_group('photometry')
for i in range(len(data)):
    g.create_dataset(dnames[i], data=data[i], dtype=data[i].dtype)
f.close()


import pytest
import voluptuous as vol

import homeassistant.helpers.config_validation as cv

def test_latitude():
    """Test latitude validation."""
    schema = vol.Schema(cv.latitude)

    for value in ('invalid', None, -91, 91, '-91', '91', '123.01A'):
        with pytest.raises(vol.MultipleInvalid):
            schema(value)

    for value in ('-89', 89, '12.34'):
        schema(value)

def test_longitude():
    """Test longitude validation."""
    schema = vol.Schema(cv.longitude)

    for value in ('invalid', None, -181, 181, '-181', '181', '123.01A'):
        with pytest.raises(vol.MultipleInvalid):
            schema(value)

    for value in ('-179', 179, '12.34'):
        schema(value)

def test_icon():
    """Test icon validation."""
    schema = vol.Schema(cv.icon)

    for value in (False, 'work', 'icon:work'):
        with pytest.raises(vol.MultipleInvalid):
            schema(value)

    schema('mdi:work')

def test_platform_config():
    """Test platform config validation."""
    for value in (
        {'platform': 1},
        {},
        {'hello': 'world'},
    ):
        with pytest.raises(vol.MultipleInvalid):
            cv.PLATFORM_SCHEMA(value)

    for value in (
        {'platform': 'mqtt'},
        {'platform': 'mqtt', 'beer': 'yes'},
    ):
        cv.PLATFORM_SCHEMA(value)

def test_entity_id():
    """Test entity ID validation."""
    schema = vol.Schema(cv.entity_id)

    with pytest.raises(vol.MultipleInvalid):
        schema('invalid_entity')

    schema('sensor.light')

def test_entity_ids():
    """Test entity ID validation."""
    schema = vol.Schema(cv.entity_ids)

    for value in (
        'invalid_entity',
        'sensor.light,sensor_invalid',
        ['invalid_entity'],
        ['sensor.light', 'sensor_invalid'],
        ['sensor.light,sensor_invalid'],
    ):
        with pytest.raises(vol.MultipleInvalid):
            schema(value)

    for value in (
        [],
        ['sensor.light'],
        'sensor.light'
    ):
        schema(value)

    assert schema('sensor.light, light.kitchen ') == [
        'sensor.light', 'light.kitchen'
    ]

def test_temperature_unit():
    """Test temperature unit validation."""
    schema = vol.Schema(cv.temperature_unit)

    with pytest.raises(vol.MultipleInvalid):
        schema('K')

    schema('C')
    schema('F')

def test_time_zone():
    """Test time zone validation."""
    schema = vol.Schema(cv.time_zone)

    with pytest.raises(vol.MultipleInvalid):
        schema('America/Do_Not_Exist')

    schema('America/Los_Angeles')
    schema('UTC')


import pickle
import asyncio
import json
import pprint

from indy import pool, ledger, wallet, did, anoncreds
from indy.error import ErrorCode, IndyError
from write_did_functions import print_log

async def schema_request(pool_,submitter):
    # 9.
    print_log('\n9. Build the SCHEMA request to add new schema to the ledger as a Steward\n')

    # Define Schema:
    schema = {
        'data': {
            'name': 'gvt',
            'version': '1.0',
            'ver': '1.0',
            'attributes': ["age", "sex", "height", "name"]
        }
    }

    # This function call creates the schema and assumes the issuer of credentials using the schema to be the
    # submitter of the schema request:

    schema['id'],schema['json'] = await anoncreds.issuer_create_schema(issuer_did=submitter['did'],
                                                                        name=schema['data']['name'],
                                                                        version=schema['data']['version'],
                                                                        attrs=json.dumps(schema['data']['attributes']))

    print_log('Schema data: ')
    pprint.pprint(schema['data'])
    print_log('Schema json: ')
    pprint.pprint(json.loads(schema['json']))
    print_log('Schema: ')
    pprint.pprint(schema)

    # Create schema for an issuer: return issuer_schema_id, issuer_schema_json:

    schema_request = await ledger.build_schema_request(submitter['did'],schema['json'])
    print_log('Schema request: ') # form of a json string
    pprint.pprint(json.loads(schema_request))

    # 10.
    print_log('\n10. Sending the SCHEMA request to the ledger\n')

    schema_response = await ledger.sign_and_submit_request(pool_handle=pool_['handle'],
                                                            wallet_handle=submitter['wallet'],
                                                            submitter_did=submitter['did'],
                                                            request_json=schema_request)

    print_log('Schema response:')
    pprint.pprint(json.loads(schema_response))
    schema_response_dict = json.loads(schema_response)

    if schema_response_dict['op'] == 'REJECT':
        print_log('\nSending GET SCHEMA request to the ledger for existing Version Number\n')
        schema_response = await get_schema_request(pool_,submitter,schema)

    print_log('Schema response:')
    pprint.pprint(json.loads(schema_response))

    return(schema)

async def get_schema_request(pool_,submitter,schema):

    schema_request = await ledger.build_get_schema_request(submitter['did'],schema['id'])
    schema_response = await ledger.sign_and_submit_request(pool_handle=pool_['handle'],
                                                            wallet_handle=submitter['wallet'],
                                                            submitter_did=submitter['did'],
                                                            request_json=schema_request)
    return(schema_response)

async def credential_definition(pool_,issuer,schema):

    # Define a credential definition:
    # Cred['def'] : - id
    #               - json
    #               - tag
    #               - type
    #               - config

    # 11.
    print_log('\n11. Creating and storing CRED DEFINITION using anoncreds as Trust Anchor, for the given Schema\n')

    cred_def = {'Issuer':issuer['name']}
    cred_def['tag'] = 'cred_def_tag'
    cred_def['type'] = 'CL'
    cred_def['config'] = json.dumps({"support_revocation": False})

    (cred_def['id'], cred_def['json']) = await anoncreds.issuer_create_and_store_credential_def(wallet_handle=issuer['wallet'],
                                                                                issuer_did=issuer['did'],
                                                                                schema_json=schema['json'],
                                                                                tag=cred_def['tag'],
                                                                                signature_type=cred_def['type'],
                                                                                config_json=cred_def['config'])
    print_log('Credential definition: ')
    pprint.pprint(json.loads(cred_def['json']))

    return(cred_def)


"""
Parse a Haka metadata file and print out attribute-filter for it.
Short term manual hack for IdP 2 as Haka stopped releasing attribute-filter file.

Usage help: ./manage.py cleandb -h
"""
from django.core.management.base import BaseCommand
from lxml import etree

from rr.models.attribute import Attribute

def haka_attribute_parser(filename):
    """
    Using CamelCase instead of regular underscore attribute names in element tree.
    """
    parser = etree.XMLParser(
        ns_clean=True, remove_comments=True, remove_blank_text=True, resolve_entities=False, no_network=True
    )
    tree = etree.parse(filename, parser)
    root = tree.getroot()
    attribute_filter_policy_group = etree.Element(
        "AttributeFilterPolicyGroup", id="urn:mace:funet.fi:haka", nsmap={"xmlns": "urn:mace:shibboleth:2.0:afp"}
    )
    attribute_filter_policy_group.attrib["{urn:mace:shibboleth:2.0:afp}basic"] = "urn:mace:shibboleth:2.0:afp:mf:basic"
    attribute_filter_policy_group.attrib["{urn:mace:shibboleth:2.0:afp}saml"] = "urn:mace:shibboleth:2.0:afp:mf:saml"
    attribute_filter_policy_group.attrib["{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"] = (
        "urn:mace:shibboleth:2.0:afp classpath:/schema/shibboleth-2.0-afp.xsd "
        "urn:mace:shibboleth:2.0:afp:mf:basic "
        "classpath:/schema/shibboleth-2.0-afp-mf-basic.xsd "
        "urn:mace:shibboleth:2.0:afp:mf:saml "
        "classpath:/schema/shibboleth-2.0-afp-mf-saml.xsd"
    )
    for a in root:
        entity_id = a.get("entityID")
        if entity_id:
            for b in a:
                if etree.QName(b.tag).localname == "SPSSODescriptor":
                    attributes = []
                    for c in b:
                        if etree.QName(c.tag).localname == "AttributeConsumingService":
                            for d in c:
                                if etree.QName(d.tag).localname == "RequestedAttribute":
                                    friendlyname = d.get("FriendlyName")
                                    name = d.get("Name")
                                    if friendlyname:
                                        attribute = Attribute.objects.filter(name=name).first()
                                        if not attribute:
                                            attribute = Attribute.objects.filter(friendlyname=friendlyname).first()
                                        if attribute:
                                            attributes.append(attribute)
                                        else:
                                            print(
                                                "Could not add attribute "
                                                + friendlyname
                                                + ", "
                                                + name
                                                + " for "
                                                + entity_id
                                            )
                    if attributes:
                        attribute_filter_policy = etree.SubElement(
                            attribute_filter_policy_group, "AttributeFilterPolicy", id="haka-default-" + entity_id
                        )
                        policy_requirement_rule = etree.SubElement(
                            attribute_filter_policy, "PolicyRequirementRule", value=entity_id
                        )
                        policy_requirement_rule.attrib[
                            "{http://www.w3.org/2001/XMLSchema-instance}type"
                        ] = "basic:AttributeRequesterString"
                        for attribute in attributes:
                            attribute_rule = etree.SubElement(
                                attribute_filter_policy, "AttributeRule", attributeID=attribute.attributeid
                            )
                            permit_value_rule = etree.SubElement(attribute_rule, "PermitValueRule")
                            permit_value_rule.attrib["{http://www.w3.org/2001/XMLSchema-instance}type"] = "basic:ANY"
    return etree.tostring(attribute_filter_policy_group, pretty_print=True, encoding="UTF-8")

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument("-i", type=str, action="store", dest="input", help="Metadata input file name")
        parser.add_argument("-o", type=str, action="store", dest="output", help="Attribute-filter output file name")

    def handle(self, *args, **options):
        metadata_input = options["input"]
        attribute_output = options["output"]
        if metadata_input and attribute_output:
            data = haka_attribute_parser(metadata_input)
            with open(attribute_output, "wb") as f:
                f.write('<?xml version="1.0" encoding="UTF-8"?>\n'.encode("utf-8"))
                # Hack for correcting namespace definition by removing prefix.
                f.write(data.replace(b"xmlns:xmlns", b"xmlns"))
        else:
            self.stdout.write("Please give both input and output files")