import argparse
import asyncio
import json
from datetime import datetime

from sqlalchemy.orm import Session

from pptx_clarifier.db import engine
from pptx_clarifier.db.models import Upload
from pptx_clarifier.pptx_explainer import initial_prompt, explainer_logger as logger
from pptx_clarifier.pptx_explainer.openai_interactor import interact
from pptx_clarifier.pptx_explainer.presentation_parser import open_presentation, parse_slide

def create_slide_prompt(index, slide):
    """
    Prepares a slide prompt for the AI.

    Args:
        index (int): Index of the slide.
        slide (slide): Slide to be parsed.

    Returns:
        str: The prompt for the AI.
    """
    # Prepare the message input for the current slide
    parsed_slide = parse_slide(slide)
    slide_prompt = json.dumps({f"Slide-{index}": parsed_slide})

    # Return the message to the AI
    return slide_prompt

async def process_presentation(upload, session):
    """
    Opens a presentation and parses its slides to OpenAI chat prompts.
    Each prompt is sent to the AI asynchronously.

    Args:
        upload: Path to the presentation file.
        session: The database session.

    Returns:
        list: List of all responses from the AI.
    """
    messages = [{"role": "system", "content": initial_prompt}]

    # Open the presentation
    presentation = open_presentation(upload.get_upload_path())
    responses = {}
    upload.status = "pending"
    session.commit()

    # Parse each slide to prompt and send it to the AI asynchronously
    for index, slide in enumerate(presentation.slides, start=1):
        try:
            prompt = create_slide_prompt(index, slide)

            if prompt is None:
                responses[f"Slide {index}"] = "No text to clarify"

            else:
                res = await interact(prompt, messages)
                responses[f"Slide {index}"] = res

        except ConnectionError as connection_error:
            logger.error(connection_error)
            responses[f"Slide {index}"] = f"Could not clarify slide {index}"

    return responses

async def clarify(upload, session):
    """
    Generates explanations for a PowerPoint presentation and saves it to a JSON file.

    Args:
        upload : The uploaded presentation file object.
        session: The database session.

    Raises:
        PathError: If the path does not exist.
        ValueError: If the path is not a pptx file.
        Exception: If an error occurs while parsing the presentation.
    """
    logger.info(f'Clarifying {upload.filename}')
    explanations = await process_presentation(upload, session)

    with open(str(upload.get_output_path()), "w") as file:
        json.dump(explanations, file)

    logger.info(f'Finished clarifying {upload.filename}')

if __name__ == "__main__":
    # Create an argument parser
    parser = argparse.ArgumentParser(description='Script to clarify pptx files.')

    # Add an argument for the file path
    parser.add_argument('file_path', nargs=1, type=str, help='Path to the pptx file')

    # Parse the command-line arguments
    args = parser.parse_args()

    # Extract the file path from the arguments
    file_path = args.file_path[0]

    # Check if the file path argument is missing
    if not file_path:
        parser.error('Please provide a path to the pptx file.')

    try:
        upload_file = Upload(upload_time=datetime.now(), filename=file_path, status="processing")
        # Call the function to process the file
        asyncio.run(clarify(upload_file))
    except Exception as e:
        print(e)


# Add the functions in this file
import json
import math
def load_journal(filename):
    f = open(filename)
    s=f.read()
    content = json.loads(s)
    f.close()
    return content

def compute_phi(filename,event):
    d=load_journal(filename)
    a=[0,0,0,0,0,0,0,0]
    for i in d:
        if(event in i['events']):
            a[4]+=1
            if(i['squirrel']):
                a[6]+=1
                a[0]+=1
            else :
                a[7]+=1
                a[2]+=1
        else:
            a[5]+=1
            if (i['squirrel']):
                a[6]+=1
                a[3]+=1
            else:
                a[7]+=1
                a[1]+=1
    val=(a[0] * a[1] - a[2] * a[3]) / math.sqrt(a[4] * a[5] * a[6] * a[7])
    return val

def compute_correlations(filename):
    data={}
    d=load_journal(filename)
    for i in d:
        for j in i['events']:
            if j not in data:
                data[j]=compute_phi(filename,j)
    return data
def diagnose(filename):
    data=compute_correlations(filename)
    maxval = max(data, key = data.get)
    minval = min(data, key = data.get)
    return maxval,minval


#!/usr/bin/env python

# Listing 6
# cacluate similarity between pairs of SMILES files and report
# the most similar training set molecule for each test set molecule

import sys,string
from rdkit import Chem
from rdkit import DataStructs
from rdkit.Chem.Fingerprints import FingerprintMols

# build a list of fingerprints from an input file
def buildFingerprintList(fileName):
    suppl = Chem.SmilesMolSupplier(fileName,titleLine=False)
    fpList = []
    for idx,mol in enumerate(suppl):
        print >> sys.stderr,"\rGenerating fingerprints for %s " % (fileName),idx+1,
        sys.stderr.flush()
        fpList.append([mol.GetProp("_Name"),FingerprintMols.FingerprintMol(mol)])
    print >> sys.stderr,"\rGenerating fingerprints for %s " % (fileName),idx+1
    return fpList

# for each molecule in queryFpLIst, find and report the most similar molecule in refFpList
def findMostSimilar(refFpList,queryFpList):
    outList = []
    for idx,[name,fp] in enumerate(queryFpList):
        print >> sys.stderr,"\rCalculating similarity ",idx,
        sys.stderr.flush()
        simList = [[x[0],DataStructs.FingerprintSimilarity(fp,x[1])] for x in refFpList]
        simList.sort(lambda a,b: cmp(b[1],a[1]))
        outList.append([name,simList[0]])
    print >> sys.stderr,"\rCalculating similarity ",idx+1
    return outList

# setup training and test sets
trainingFiles = ["huuskonen_train.smi"]
testFiles = ["huuskonen_test.smi","jcim.smi","pubchem.smi"]
trainDict = {}
testDict = {}

# open the output file
ofs = open("similarity.txt","w")

# generate fingerprints
for fileName in trainingFiles:
    trainDict[fileName] = buildFingerprintList(fileName)
for fileName in testFiles:
    testDict[fileName] = buildFingerprintList(fileName)

# write the output
print >> ofs,"Query Reference Similarity Dataset"
for trainFileName,trainFpList in trainDict.iteritems():
    for testFileName,testFpList in testDict.iteritems():
        print >> sys.stderr,"Processing %s" % testFileName
        simList = findMostSimilar(trainFpList,testFpList)
        for query,[ref,sim] in simList:
            print >> ofs,query,ref,"%.2f" % (sim),testFileName.split(".")[0].upper()

print >> sys.stderr,"Results have been written to similarity.txt"


import requests, os, datetime, pandas as pd

def downLoadAndCopy(inputLink, outputLink):
    r = requests.get(inputLink)
    with open(outputLink, 'wb') as f:
        f.write(r.content)

'''

The following code downloads the bhavcopy for derivatives and equity
as well as the list of banned securities from NSE server.
Required inputs
1) Date as string input in 'ddmmyyyy' format,
2) holidays and no holidays dates stored in csv files under the column header 'Date'
3) The path to the directory where output from the program will be stored.

Please see below block to check user input variables.

'''

############################# User inputs here ###################################
# Enter the date for download in 'ddmmyyyy' format
date = '19122019'

# Enter the path for output
outputLink = r'D:\Work\NSEDownloader\Output'

# Path to the 'holidays file'
holidayListPath = r"D:\Work\NSEDownloader\Holidaylist_NSE.csv"

# Path to the 'no holidays file'
noHolidayListPath = r"D:\Work\NSEDownloader\NoHoliday_NSE.csv"

##################################################################################
# Assertions about files
assert os.path.isfile(holidayListPath), "The following path for the holidays list file is incorrect.\n" + holidayListPath
assert os.path.isfile(noHolidayListPath), "The following path for the no holidays list file is incorrect.\n" + noHolidayListPath

# Convert string to datetime object
dateMod = datetime.datetime.strptime(date, "%d%m%Y")

# Read the 'holidays file' and 'no holidays file' and convert them to required string format
holidays = pd.read_csv(holidayListPath, encoding='ISO-8859-1')['Date'].dropna()
noHolidays = pd.read_csv(noHolidayListPath, encoding='ISO-8859-1')['Date'].dropna()

holidays = holidays.map(lambda x: datetime.datetime.strptime(x, "%Y-%m-%d"))
noHolidays = noHolidays.map(lambda x: datetime.datetime.strptime(x, "%Y-%m-%d"))

holidays = holidays.map(lambda x: datetime.datetime.strftime(x, "%d%m%Y"))
noHolidays = noHolidays.map(lambda x: datetime.datetime.strftime(x, "%d%m%Y"))

# Check if date is in list of holidays
if date in list(holidays):
    raise Exception("The given date corresponds to a holiday.")

# Check if date is a weekend and date is a 'no holiday day' for the markets
if (datetime.datetime.strftime(dateMod, "%w") in ['0','6']):
    if date not in list(noHolidays):
        raise Exception("The given date corresponds to a weekend.")

# Downloading for banned securities
foBanFileName = 'fo_secban_' + date + '.csv'
foBanLink = 'https://www.nseindia.com/archives/fo/sec_ban/'

downLoadAndCopy(foBanLink + foBanFileName, os.path.join(outputLink, foBanFileName))

# Downloading historical data for equities
equityFileName = 'cm' + datetime.datetime.strftime(dateMod, '%d%b%Y').upper() + 'bhav.csv.zip'
equityFileLink = 'https://www.nseindia.com/content/historical/EQUITIES/' + datetime.datetime.strftime(dateMod, "%Y") + '/' + datetime.datetime.strftime(dateMod, "%b").upper()  + '/' + equityFileName

downLoadAndCopy(equityFileLink, os.path.join(outputLink, equityFileName))

# Downloading historical data for derivatives
derivFileName = 'fo' + datetime.datetime.strftime(dateMod, '%d%b%Y').upper() + 'bhav.csv.zip'
derivFileLink = 'https://www.nseindia.com/content/historical/DERIVATIVES/' + datetime.datetime.strftime(dateMod, "%Y") + '/' + datetime.datetime.strftime(dateMod, "%b").upper()  + '/' + derivFileName

downLoadAndCopy(derivFileLink, os.path.join(outputLink, derivFileName))


# (c) Nelen & Schuurmans.  GPL licensed, see LICENSE.rst.
from django.template.defaultfilters import slugify

import factory

from lizard_maptree.models import Category

class CategoryFactory(factory.DjangoModelFactory):
    """Factory for creating Category instances"""
    class Meta:
        model = Category

    @classmethod
    def _prepare(cls, create, **kwargs):
        name = kwargs.get('name', None)
        category = super(CategoryFactory, cls)._prepare(create, **kwargs)
        if name:
            category.slug = slugify(name)
            if create:
                category.save()
        return category


import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from tensorflow.contrib.learn.python.learn.datasets import mnist
from tensorflow.python.framework import dtypes
import os
import numpy as np
import matplotlib.pyplot as plt
from load_mnist import MNIST
from sklearn import preprocessing

#train_x_all, train_y_all = MNIST(path="./",return_type="numpy",mode="vanilla").load_training()
#test_x, test_y = MNIST(path="./",return_type="numpy",mode="vanilla").load_testing()
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True, dtype=dtypes.uint8, validation_size=0)

log_path = "./graphs"
model_path = './model'
infer = False

train_x_all = mnist.train.images
train_y_all = mnist.train.labels
test_x = mnist.test.images
test_y = mnist.test.labels

#Normalize from [0:255] => [0.0:1.0]
# train_x_all = np.multiply(train_x_all, 1.0 / 255.0)
# train_x_all_size = train_x_all.shape[1]
# train_x_all_width = train_x_all_height = np.ceil(np.sqrt(train_x_all_size)).astype(np.uint8)

for data in [test_x,train_x_all]:
    print("shape", data.shape)
    print("mean", data.mean())
    print("max",data.max())
    print("min", data.min())
    # var=np.square(data-data.mean)
    # var /= 10000
    # data=(data - data.mean)/var
def normalize1(input_data):
    average = input_data.mean(axis=0)
    total = input_data.sum(axis=0)
    variance = input_data.var(axis=0)
    normalized = (input_data - average) / variance
    return normalized

def normalize(input_data):
    minimum = input_data.min(axis=0)
    maximum = input_data.max(axis=0)
    normalized = (input_data - minimum) / ( maximum - minimum )
    #normalized = preprocessing.normalize(input_data, norm='l2')
    return normalized
# train_x_all = normalize1(train_x_all)
# train_x_all = normalize(train_x_all)

training_epochs = 4
batch_size = 60
learning_rate = 0.01
training_step = 0
n_input = 28*28
n_hidden_1 = 10
n_classes = 10

hyper_str = "hidden{}_lr{}".format(n_hidden_1, learning_rate)
total_batch = int(mnist.train.num_examples/batch_size)
# 2. BUILD MODEL
with tf.name_scope('Input'):
    x = tf.placeholder(tf.float32, [None, n_input], name = 'image')
    y = tf.placeholder(tf.float32, [None, n_classes], )

with tf.device('/cpu:0'):
    with tf.name_scope('Layers'):  #tf.zeros o baghye ghesmatha!!!

        hidden_1 = tf.layers.dense(inputs=x,
                                   units=n_hidden_1,
                                   bias_initializer=tf.zeros_initializer(),
                                   kernel_initializer= tf.random_normal_initializer(0., 0.2),
                                   activation=tf.nn.sigmoid,
                                   name = 'dense1')
        pred = tf.layers.dense(inputs=hidden_1,
                               bias_initializer=tf.zeros_initializer(),
                               kernel_initializer= tf.random_normal_initializer(0., 0.2),
                               units=n_classes,
                               name = 'dense2')

    # 2.2 Define Loss and Optimizer
    with tf.name_scope('Loss_Optimizer'):
        loss = tf.losses.softmax_cross_entropy(logits=pred,
                                               onehot_labels=y)
        optimizer = tf.train.\
            GradientDescentOptimizer(learning_rate=learning_rate)\
            .minimize(loss)

    # 2.3 Define Accuracy
    with tf.name_scope('Accuracy'):
        correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# 2.4 Create a summary for accuracy & loss:
tf.summary.scalar('loss',loss)
tf.summary.scalar('accuracy',accuracy)
summeries  = tf.summary.merge_all()
if not infer:
    writer = tf.summary.FileWriter(log_path+'/'+hyper_str)

# 2.5 Add op for saving the model
save_path = model_path + '/mlp_' + hyper_str + '.cpkl'
if not os.path.exists(save_path):
    os.makedirs(save_path)
saver = tf.train.Saver()
# 3. RUN THE MODEL
with tf.Session() as sess:
    # 3.0 Initialize all variables
    sess.run(tf.global_variables_initializer())
    if not infer:
        # 3.1 Use TensorBoard
        writer.add_graph(sess.graph)
        # 3.2 Training Loop
        for epoch in range(training_epochs):
            epoch_accr = 0
            # 3.3 Loop over batches
            training_step = 0
            for i in range(total_batch):
                training_step = training_step + 1
                batch_x, batch_y = mnist.train.next_batch(batch_size)
                batch_loss , batch_accr = \
                    sess.run([optimizer, accuracy],
                             feed_dict={x: batch_x,y: batch_y})
                epoch_accr += batch_accr/total_batch

            # Test

            test_accuracy, summ = \
                sess.run([accuracy,summeries],
                         feed_dict={x: test_x,
                                    y: test_y})
            print("Epoch:", epoch, "Train Accuracy",
                  epoch_accr,"Test Accuracy", test_accuracy)
            writer.add_summary(summ, global_step = epoch+1)

        # Save the model
        saver = tf.train.Saver()
        saver.save(sess,model_path + '/mlp_' + hyper_str + '.cpkl')
        # Close TensorBoard Writer
        writer.flush()
        writer.close()
    else:
        saver.restore(sess,model_path + '/mlp_' + hyper_str + '.cpkl')
        rand_num = np.random.randint(len(test_x))
        image = test_x[rand_num].reshape([1, 784])
        target = test_y[rand_num]
        pred = sess.run([pred], feed_dict={x: image})
        print('True label:',np.argmax(target), 'Predicted label:',np.argmax(pred))
        imgplot = plt.imshow(image.reshape([28,28]))
        plt.show()


#!/usr/bin/python3
import logging as out
import socket
import ssl
import sys
import time
from urllib.parse import urlparse

out.basicConfig(level=out.INFO, format="[%(levelname)s] %(message)s")

class SmartClient:
    """SmartClient that returns the index page or indicated page."""
    # Shared socket, I call it session.
    __session = None
    # Combine these index names below will generate the default index page path
    # Only use if path is empty, and no 301/302
    __index_suffix_list = [".do", ".php", ".htm", ".html", ".jsp"]
    __index_name_list = ["index", "default"]
    # Raw header text list
    __header_text_list = []
    # Raw body text list
    __body_text_list = []
    # Raw body text
    __body_text = ""
    # Protocol priority
    __protocol_priority = ["https", "http"]
    # Default port
    __default_port = {"https": 443, "http": 80}
    # Protocol
    __protocol = ["h2", "http/1.1", "http/1.0"]
    # Support highest protocol
    __highest_protocol = ""
    # Support protocol
    __support_protocol = {"https": False, "http": False}
    # URL
    __url = None
    # Http code
    __status = 0
    # Protected by password
    __password_protection = False
    # Cookies
    __cookies = []
    # Javascript cookies
    __js_cookies = []

    def __init__(self, url: str):
        """Constructor that try to figure out the protocol server use.
        When url has critical error, or any connection error, it will raise error.

        :param url: URL you want to get.
        """
        # prevent incomplete url like "google.com"
        try:
            if "//" not in url[:8]:
                self.__url = urlparse("https://{}".format(url))
            else:
                self.__url = urlparse(url)
            out.info("Input URL is {}".format(url))
            # this will check port range
            __test_port_var = self.__url.port
        except ValueError as err:
            raise SyntaxError("URL is critical. {}".format(err))
        # first try https because http2 is basically over https
        # if user prefer http, then use http first
        if self.__url.scheme == "http":
            self.__protocol_priority = ["http", "https"]
        out.warning("Detecting protocol, there are some requests will be send.")
        self.__try_protocol()
        if self.__highest_protocol == "None":
            raise ConnectionError("This server does not support any http protocol.")
        out.warning("Try get cookies by fetching the given path {}. If there's 404, I will try some default path."
                    .format(self.__url.path if self.__url.path != '' else "/"))
        self.__fetch()
        # Now I have the latest page, I can get cookies from this page.
        self.__generate_cookies()
        out.warning("Summary of input URL {}".format(url))
        out.warning("=" * 80)
        out.warning("This server support highest protocol is: {}".format(self.__highest_protocol))
        out.warning("Does it support h2? {}".format("Yes" if self.__highest_protocol == "h2" else "No"))
        out.warning("Does it support https? {}".format("Yes" if self.__support_protocol["https"] else "No"))
        out.warning(
            "Does it support pure http (http/1.1)? {}".format("Yes" if self.__support_protocol["http"] else "No"))
        out.warning("Password-protected: {}".format("Yes" if self.__password_protection else "No"))
        out.warning("=" * 80)
        if self.__cookies:
            out.warning("Cookies in http header:")
            out.debug(self.__cookies)
            for c in self.__analyze_cookies():
                out.warning(c)
        else:
            out.warning("There's no cookie in header.")
        out.warning("=" * 80)
        if self.__js_cookies:
            out.warning("These javascript statements below could set cookies:")
            for j in self.__js_cookies:
                out.warning(j)
        out.warning("End")

    def __analyze_cookies(self) -> list:
        """Convert cookie list to human-readable list.

        :return: Cookie details texts in list.
        """
        __return_text_list = []
        for d in self.__cookies:
            __temp_text = "Cookie name: "
            __temp_text += list(d.keys())[0]
            __temp_text += ", value: "
            __temp_text += d[list(d.keys())[0]]
            if "expires" in d.keys():
                __temp_text += ', expire time: ' + d['expires']
            if "domain" in d.keys():
                __temp_text += ', domain: ' + d['domain'] if d['domain'] is not None else ''
            __return_text_list.append(__temp_text)
        return __return_text_list

    def __generate_cookies(self):
        """Split cookies into __header_text_list."""
        for h in self.__header_text_list:
            if "set-cookie" in h.lower():
                __temp_cookie_string = h.split(": ")[1]
                __temp_cookie_dictionary = {}
                for __temp_cookie_param in __temp_cookie_string.split("; "):
                    if "=" in __temp_cookie_param:
                        __key, __value = __temp_cookie_param.split("=", 1)
                        __temp_cookie_dictionary[__key] = __value
                    else:
                        # no need to store these cookies that not include "="
                        continue
                self.__cookies.append(__temp_cookie_dictionary)

        # These code below have not been tested because I can not find a website generate cookie from javascript
        __start_position = -1
        __start_position = self.__body_text.find("document.cookie")
        while __start_position != -1:
            __end_position = self.__body_text.find(";", __start_position)
            __temp_cookie_string = self.__body_text[__start_position: __end_position]
            __start_position = self.__body_text.find("document.cookie", __start_position + 1)
            self.__js_cookies.append(__temp_cookie_string)

    def __fetch(self):
        """Fetch URL to figure out the last page's cookies."""
        out.debug(self.__highest_protocol)
        out.debug(self.__support_protocol)
        __now_protocol = "https" if self.__support_protocol["https"] else "http"
        self.__get(__now_protocol)
        __count = 0
        while self.__status != 200:
            if __count >= 10:
                # Jump too much, maybe wrong configuration
                raise ConnectionError("Too much redirection, abort.")
            for h in self.__header_text_list:
                if "WWW-Authenticate" in h:
                    self.__password_protection = True
            if self.__status == 301 or self.__status == 302:
                # Redirect
                # first get location
                __new_location = ""
                for h in self.__header_text_list:
                    if "location" in h.lower():
                        __new_location = h.split(" ")[1]
                        break
                if __new_location == "":
                    raise ConnectionError("Server response 301/302 but no new location was gave.")
                # assume that this url from server is reliable
                out.warning("Meeting {} , Fetching new location: {}".format(self.__status, __new_location))
                if "//" not in __new_location[:8]:
                    # default http, not influence protocol that __try_protocol detected
                    self.__url = urlparse("http://{}".format(__new_location))
                else:
                    self.__url = urlparse(__new_location)
                if self.__url.scheme:
                    self.__get(self.__url.scheme)
                    self.__support_protocol[self.__url.scheme] = True
            if self.__status == 404:
                # this path not found, check if count is 0.
                if __count == 0:
                    # try those default path
                    for __filename in self.__index_name_list:
                        for __suffix in self.__index_suffix_list:
                            self.__get(__now_protocol, "/{}{}".format(__filename, __suffix))
                            out.debug(self.__status)
                            if self.__status != 404:
                                self.__url = urlparse("{}://{}/{}{}"
                                                      .format(self.__url.scheme, self.__url.netloc,
                                                              __filename, __suffix))
                                __count += 1
                else:
                    # it is just server configuration error, ignore
                    pass
            if self.__status == 401:
                # No need to continue
                self.__password_protection = True
                break
            __count += 1

    def __new_session(self, protocol: str):
        """Create new http/https socket.

        :param protocol: "http" or "https", socket will generate according this.
        """
        if self.__session:
            self.__session.close()
        if protocol == "https":
            context = ssl.create_default_context()
            # just priority, so add h2 in it, once it supports h2, it will use
            context.set_alpn_protocols(self.__protocol)
            self.__session = context.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM),
                                                 server_hostname=self.__url.hostname)
        else:
            self.__session = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            self.__session.settimeout(5)
            if self.__url.port:
                self.__session.connect((self.__url.hostname, self.__url.port))
            else:
                self.__session.connect((self.__url.hostname, self.__default_port[protocol]))
            self.__session.settimeout(None)
        except (socket.error, ssl.CertificateError, ssl.SSLError, socket.timeout):
            raise ConnectionError("Cannot open socket, maybe network issue.")

    def __get(self, protocol: str, path=""):
        """Send get header and receive, which means full steps of http get.

        :param protocol: "http" or "https", will use this to create socket.
        :param path: Optional, if exists then use that path, not self.__url.path.
        """
        try:
            self.__new_session(protocol)
        except Exception as err:
            raise err
        __header = []
        if path == "":
            if self.__url.path == "" or self.__url.path == "/":
                # try / first
                __path = "/"
            else:
                __path = self.__url.path
        else:
            __path = path
        out.debug("{}://{}/{}".format(protocol, self.__url.hostname, path))
        # Package header
        __header.append("GET {} HTTP/1.1".format(__path))
        __header.append("Host: {}".format(self.__url.hostname))
        __header.append("Accept: */*")
        __header.append("User-agent: SmartClient/0.0.1")
        if protocol == "http":
            __header.append("Connection: close")
            __header.append("Upgrade: h2c")
        else:
            __header.append("Connection: close")
        self.__send(__header)
        self.__receive()

    def __send(self, requests: list):
        """Send request line by line in request list.

        :param requests: Request list include headers, actually can include body.
        """
        out.info("Sending Requests:")
        out.info("---Request Begin---")
        for request in requests:
            out.info(request)
            self.__session.sendall(request.encode())
            self.__session.sendall("\r\n".encode())
        self.__session.sendall("\r\n".encode())
        out.info("---Request End---")

    def __receive(self):
        """Receive lines and split with "\r\n"."""
        try:
            __response = ""
            __timeout_start = None
            while True:
                __temp_recv = self.__session.recv(8192)
                if __temp_recv:
                    __response += __temp_recv.decode(errors='ignore')
                else:
                    if not __timeout_start:
                        __timeout_start = time.time()
                if __timeout_start:
                    if __temp_recv:
                        if time.time() - __timeout_start >= 1:
                            continue
                    else:
                        if time.time() - __timeout_start >= 2:
                            break
            out.info('---Respond Header Begin---')
            self.__header_text_list = __response.split("\r\n\r\n")[0].split('\r\n')
            for h in self.__header_text_list:
                out.info(h)
            out.info('---Respond Header End---')
            if len(__response.split("\r\n\r\n")) > 1:
                out.info('---Respond Body Begin---')
                for bs in __response.split("\r\n\r\n")[1:]:
                    self.__body_text += bs
                    for b in bs.split("\n"):
                        self.__body_text_list.append(b)
                        out.info(b)
                out.info('---Respond Body End---')
            else:
                out.warning("There's no respond body.")
            # maybe cause error in converting to int
            self.__status = int(self.__header_text_list[0].split(" ")[1])
            out.debug(self.__status)
        except Exception as err:
            raise err

    def __try_protocol(self):
        """Test protocol."""
        for p in self.__protocol_priority:
            try:
                self.__get(p)
            except Exception as err:
                out.warning("Protocol {} not support, reason: {}".format(p, err))
                self.__support_protocol[p] = False
            else:
                self.__support_protocol[p] = True
        if self.__support_protocol["https"]:
            self.__new_session("https")
            if self.__session.selected_alpn_protocol() == "h2":
                self.__highest_protocol = "h2"
            else:
                self.__highest_protocol = "https"
        elif self.__support_protocol["http"]:
            self.__highest_protocol = "http"
        else:
            self.__highest_protocol = "None"

    def get_protocol_support(self) -> dict:
        """Get variable __support_protocol, this var is a dictionary of protocol that server support.

        :return: A dictionary of protocol that server support, like {"http": True. "https": True}
        """
        return self.__support_protocol

    def get_highest_protocol(self) -> str:
        """Get variable __highest_protocol, this var is a string of the highest protocol server support.

        :return: A string of the highest protocol server support, maybe "h2", "https", "http" or "None".
        """
        return self.__highest_protocol

if __name__ == "__main__":
    try:
        SmartClient(sys.argv[1])
    except SyntaxError as e:
        out.error("URL mistake: {}".format(e))
    except ConnectionError as e:
        out.error(e)
    except IndexError as e:
        out.error("Command wrong. [{}] Usage: python3 SmartClient.py [URL]".format(e))
    else:
        exit(0)
    exit(-1)


from selenium import webdriver
import time

from xlsxwriter import Workbook

from src.scrapy.journeyinlife.journey.journey.locator import bluesprint_locator
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
from src.scrapy.journeyinlife.journey.journey.items import JourneyItem
import xlsxwriter
import hashlib

time_loading = 2

class JourneyParse:
    page_total = 5
    driver = None
    page_number = 1
    count = 1
    workbook = None
    worksheet = None

    def __init__(self):
        options = webdriver.ChromeOptions()
        options.add_argument('--ignore-certificate-errors')
        options.add_argument('--incognito')
        options.add_argument('--headless')
        self.driver = webdriver.Chrome('./chromedriver', chrome_options=options)
        self.workbook = xlsxwriter.Workbook(f'journey_in_life_{self.page_total}.xlsx')

        self.worksheet = self.workbook.add_worksheet()
        self.worksheet.write('A1', 'STT')
        self.worksheet.write('B1', 'Key')
        self.worksheet.write('C1', 'Title')
        self.worksheet.write('D1', 'Meaning')
        self.worksheet.write('E1', 'Link')
        self.worksheet.write('F1', 'Image_Link')
        self.worksheet.write('G1', 'thumb_Link')

    def parse_list_soup(self):
        self.list_item = []
        url = f'https://www.journeyinlife.net/search/label/phrase?v=full&page={self.page_number}'
        self.driver.get(url)
        print(f"url: {url}")
        time.sleep(time_loading)
        page_source = self.driver.page_source
        soup = BeautifulSoup(page_source, 'lxml')

        row = soup.find('div', class_='row')
        list_phrase = row.find_all('div', class_='col-md-4')

        for i in list_phrase:
            item_journey = JourneyItem()
            a = i.find('a')
            link = a.get('href')
            item_journey.link = link
            # print(link)
            img = a.find('img').get('data-src')
            item_journey.thumb = img
            # print(img)
            title = i.find('div', class_='title').text
            item_journey.key = hashlib.md5(title.encode()).hexdigest()
            item_journey.title = title
            self.list_item.append(item_journey)

            print(item_journey.title)

        for detail in self.list_item:
            self.parse_detail(detail)

        self.writer_csv(list_item=self.list_item)

        if self.page_number < self.page_total:
            self.page_number += 1
            self.parse_list_soup()

    # def tag_content(self, tag):
    #     return tag['class'] == 'col-md-4' and tag.parent['class'] == 'row'

    def parse_detail(self, item_journey):
        self.driver.get(item_journey.link)
        print(f"url: {item_journey.link}")
        time.sleep(time_loading)
        page_source = self.driver.page_source
        soup = BeautifulSoup(page_source, 'lxml')
        page = soup.find('div', class_='entry-body')
        item_journey.image = page.find('img').get('src')

        texts = page.findAll('div', {'style': 'text-align: justify;'})
        for text in texts:
            item_journey.str_content = f'{item_journey.str_content} \n ' \
                                       f'{text.text}'
            # item_journey.content.append(text.text)

        print(item_journey.title)

    def close(self):
        self.workbook.close()
        self.driver.close()
        self.driver.quit()

    def writer_csv(self, list_item):

        print('writer excel')

        for i in range(len(list_item)):
            row_number = self.count + 1
            self.worksheet.write(f'A{row_number}', f'{self.count}')
            self.worksheet.write(f'B{row_number}', list_item[i].key)
            self.worksheet.write(f'C{row_number}', list_item[i].title)
            self.worksheet.write(f'D{row_number}', list_item[i].str_content)
            self.worksheet.write(f'E{row_number}', list_item[i].link)
            self.worksheet.write(f'F{row_number}', list_item[i].image)
            self.worksheet.write(f'G{row_number}', list_item[i].thumb)
            self.count += 1

if __name__ == '__main__':
    parse = JourneyParse()
    try:
        parse.parse_list_soup()
    except:
        parse.close()
    parse.close()


"""
Demonstrates how to use the `ChatInterface` and `PanelCallbackHandler` to create a
chatbot to talk to your Pandas DataFrame. This is heavily inspired by the
[LangChain `chat_pandas_df` Reference Example](https://github.com/langchain-ai/streamlit-agent/blob/main/streamlit_agent/chat_pandas_df.py).
"""
from __future__ import annotations

from pathlib import Path
from textwrap import dedent

import pandas as pd
import panel as pn
import param
import requests
from langchain.agents import AgentType
from langchain.chat_models import ChatOpenAI
from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent

from panel_chat_examples import EnvironmentWidgetBase

pn.extension("perspective")

PENGUINS_URL = (
    "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/penguins.csv"
)
PENGUINS_PATH = Path(__file__).parent / "penguins.csv"
if not PENGUINS_PATH.exists():
    response = requests.get(PENGUINS_URL)
    PENGUINS_PATH.write_text(response.text)

FILE_DOWNLOAD_STYLE = """
.bk-btn a {
    padding: 0px;
}
.bk-btn-group > button, .bk-input-group > button {
    font-size: small;
}
"""

class Environment(EnvironmentWidgetBase):
    """Will be asking the user for the API Key if not set as environment variable"""

    OPENAI_API_KEY = param.String()

class AgentConfig(param.Parameterized):
    """Configuration used for the Pandas Agent"""

    user = param.String("Pandas Agent")
    avatar = param.String("🐼")

    show_chain_of_thought = param.Boolean(default=False)

    def _get_agent_message(self, message: str) -> pn.chat.ChatMessage:
        return pn.chat.ChatMessage(message, user=self.user, avatar=self.avatar)

class AppState(param.Parameterized):
    data = param.DataFrame()

    llm = param.Parameter(constant=True)
    pandas_df_agent = param.Parameter(constant=True)

    config: AgentConfig = param.ClassSelector(class_=AgentConfig)
    environ: Environment = param.ClassSelector(class_=Environment, constant=True)

    def __init__(
        self, config: AgentConfig | None = None, environ: Environment | None = None
    ):
        if not config:
            config = AgentConfig()
        if not environ:
            environ = Environment()
        super().__init__(config=config, environ=environ)

    @param.depends("environ.OPENAI_API_KEY", on_init=True, watch=True)
    def _reset_llm(self):
        with param.edit_constant(self):
            if self.environ.OPENAI_API_KEY:
                self.llm = ChatOpenAI(
                    temperature=0,
                    model="gpt-3.5-turbo-0613",
                    api_key=self.environ.OPENAI_API_KEY,
                    streaming=True,
                )
            else:
                self.llm = None

    @param.depends("llm", "data", on_init=True, watch=True)
    def _reset_pandas_df_agent(self):
        with param.edit_constant(self):
            if not self.error_message:
                self.pandas_df_agent = create_pandas_dataframe_agent(
                    self.llm,
                    self.data,
                    verbose=True,
                    agent_type=AgentType.OPENAI_FUNCTIONS,
                    handle_parsing_errors=True,
                )
            else:
                self.pandas_df_agent = None

    @property
    def error_message(self):
        if not self.llm and self.data is None:
            return dedent(
                """\
                Please provide your `OPENAI_API_KEY`, **upload a `.csv` file**
                and click the **send** button."""
            )
        if not self.llm:
            return "Please provide your `OPENAI_API_KEY`."
        if self.data is None:
            return "Please **upload a `.csv` file** and click the **send** button."
        return ""

    @property
    def welcome_message(self):
        return dedent(
            f"""
            I'm your LangChain Pandas DataFrame Agent.

            I execute LLM generated Python code under the hood - this can be bad if
            the `llm` generated Python code is harmful. Use cautiously!

            {self.error_message}"""
        ).strip()

    async def callback(self, contents, user, instance):
        if isinstance(contents, pd.DataFrame):
            self.data = contents
            instance.active = 1
            message = self.config._get_agent_message(
                dedent(
                    """You can ask me anything about the data. For example 'how many
                    species are there?'"""
                )
            )
            return message

        if self.error_message:
            message = self.config._get_agent_message(self.error_message)
            return message

        if self.config.show_chain_of_thought:
            langchain_callbacks = [
                pn.chat.langchain.PanelCallbackHandler(instance=instance)
            ]
        else:
            langchain_callbacks = []

        response = await self.pandas_df_agent.arun(
            contents, callbacks=langchain_callbacks
        )
        message = self.config._get_agent_message(response)
        return message

state = AppState()

chat_interface = pn.chat.ChatInterface(
    widgets=[
        pn.widgets.FileInput(name="Upload"),
        pn.widgets.TextInput(name="Message", placeholder="Send a message"),
    ],
    renderers=pn.pane.Perspective,
    callback=state.callback,
    callback_exception="verbose",
    show_rerun=False,
    show_undo=False,
    show_clear=False,
    min_height=400,
)
chat_interface.send(
    state.welcome_message,
    user=state.config.user,
    avatar=state.config.avatar,
    respond=False,
)

download_button = pn.widgets.FileDownload(
    PENGUINS_PATH,
    button_type="primary",
    button_style="outline",
    height=30,
    width=335,
    stylesheets=[FILE_DOWNLOAD_STYLE],
)

layout = pn.template.MaterialTemplate(
    title="🦜 LangChain - Chat with Pandas DataFrame",
    main=[chat_interface],
    sidebar=[
        download_button,
        "#### Agent Settings",
        state.config.param.show_chain_of_thought,
    ],
)

if state.environ.variables_not_set:
    layout.sidebar.append(state.environ)

layout.servable()


import csv
import sys
import diffevo
import bayesopt
import ga
import random_search
import grid_search
from tqdm import tqdm

def run_n_times(algorithm, segments, n, iters):
    """
    Perform a meta-experiment. The chosen algorithm is run n times for a given
    number of gradient segments. The CRF score per iteration and the cumulative
    runtime per iteration are written to csv files. Filepaths need to be specified
    manually and should indicate which sample was used in read_data.py. This has
    to be done manually because optimization algorithm packages don't allow
    for extra arguments in the objective function, other than the parameters
    to be optimized.

    :algorithm: Optimization algorithm. Choose from:
                BayesOpt/DiffEvo/GenAlgo/GridSearch/RandomSearch
    :segments: Number of gradient segments in the gradient profile.
    """

    filename = algorithm + "_" + str(segments) + "segments_sample1.csv"
    filename_runtime = algorithm + "_" + str(segments) + "segments" + "_runtime_sample1" + ".csv"
    filepath = "results/" + str(segments) + "segments/" + filename
    filepath_runtime = "results/" + str(segments) + "segments/" + filename_runtime

    if(algorithm == "BayesOpt"):

        for nth_experiment in tqdm(range(n)):
            # n = number of meta experiments
            return_list = bayesopt.bayesopt(iters, segments)
            func_vals = return_list[4]
            runtime_per_iteration = return_list[5]
            # write the data from the list to a (csv?) file as a single line
            f = open(filepath, 'a', newline ='\n')

            # writing the data into the file
            with f:
                writer = csv.writer(f)
                writer.writerow(func_vals)

            f = open(filepath_runtime, 'a', newline ='\n')

            # writing the data into the file
            with f:
                writer = csv.writer(f)
                writer.writerow(runtime_per_iteration)

    elif(algorithm == "DiffEvo"):

        for nth_experiment in tqdm(range(n)):
            # n = number of meta experiments
            return_list = diffevo.diffevo(iters, segments)

            func_vals = return_list[4]
            runtime_per_iteration = return_list[5]
            f = open(filepath, 'a', newline ='\n')

            with f:
                writer = csv.writer(f)
                writer.writerow(func_vals)

            f = open(filepath_runtime, 'a', newline ='\n')

            with f:
                writer = csv.writer(f)
                writer.writerow(runtime_per_iteration)

    elif(algorithm == "GenAlgo"):

        for nth_experiment in tqdm(range(n)):
            return_list = ga.ga(iters, segments)
            func_vals = return_list[4]
            runtime_per_iteration = return_list[5]
            f = open(filepath, 'a', newline ='\n')

            with f:
                writer = csv.writer(f)
                writer.writerow(func_vals)

            f = open(filepath_runtime, 'a', newline ='\n')

            with f:
                writer = csv.writer(f)
                writer.writerow(runtime_per_iteration)

    elif(algorithm == "RandomSearch"):

        for nth_experiment in tqdm(range(n)):
            # n = number of meta experiments
            return_list = random_search.run_rs(iters, segments)
            func_vals = return_list[4]
            runtime_per_iteration = return_list[5]
            f = open(filepath, 'a', newline ='\n')

            with f:
                writer = csv.writer(f)
                writer.writerow(func_vals)

            f = open(filepath_runtime, 'a', newline ='\n')

            with f:
                writer = csv.writer(f)
                writer.writerow(runtime_per_iteration)

    elif(algorithm == "GridSearch"):

        return_list = grid_search.grid_search(iters, segments)
        func_val = return_list[2]
        runtime = return_list[1]
        f = open(filepath, 'a', newline ='\n')

        with f:
            writer = csv.writer(f)
            writer.writerow([func_val])

        f = open(filepath_runtime, 'a', newline ='\n')

        with f:
            writer = csv.writer(f)
            writer.writerow([runtime])

def main():
    if len(sys.argv) > 5:
        print('You have specified too many arguments.')
        sys.exit()

    if len(sys.argv) < 5:
        print('Please specify the following parameters in order:')
        print("- Choose an optimization algorithm (BayesOpt/DiffEvo/GenAlgo/GridSearch/RandomSearch)")
        print("- Number of segments in the gradient profile")
        print("- Number of sub-experiments the meta-experiment should consist of")
        print("- Number of iterations. Note that if the chosen algorithm is grid search, this is the number of grid points per dimension.")
        sys.exit()

    algorithm = sys.argv[1]
    number_of_segments = int(sys.argv[2])
    sub_experiments = int(sys.argv[3])
    iterations = int(sys.argv[4])

    run_n_times(algorithm, number_of_segments, number_of_segments, iterations)

if __name__ == '__main__':
    main()


# -*- encoding: utf-8 -*-

import os
import logging
from gs_pipe.pipeline import PipeLine
from redis import Redis
from rq import Queue

LOG = logging.getLogger(__name__)

task_queue = Queue(connection=Redis())

class Task(object):

    def __init__(self, id, status, result=None, created_at=None, description=None):
        self.id = id
        self.status = status
        self.result = result
        self.created_at = created_at
        self.desrciption = description

    @classmethod
    def from_job(cls, job):
        return cls(
            id=job.id,
            status=job.status,
            created_at=job.enqueued_at,
            description=job.description,
            result = job.result
        )

    def as_dict(self):
        return dict(
            id=self.id,
            status=self.status,
            result=self.result,
            created_at=str(self.created_at),
            description=self.desrciption
        )

def create_task(input_argument, pipeline):
    # queue.enque
    job = task_queue.enqueue(evaluate_pipeline, input_argument, pipeline)

    return Task.from_job(job).as_dict()

def get_task(task_id):
    job = task_queue.fetch_job(task_id)
    return Task.from_job(job).as_dict()

def list_mod():
    mods_dir = os.path.join(os.path.dirname(__file__), "mods")
    # todo ....
    # iterate mods directory, find all funcs ..
    # def find_funcs(fpath):
    #     with open(fpath, "r") as f :
    #         lines = f.read()
    # for dirpath, _ , files in os.walk(mods_dir):
    #     fpaths = [os.path.join(dirpath,f) for f in files]
    # todo

    return [
        'gs_pipe.mods.example.square',
        'gs_pipe.mods.example.minus_one'
    ]

def evaluate_pipeline(input_argument, funcs):
    '''
        input_argument: 10
        funcs: [
            'mods.example.square',
            'mods.example.minus_one'
        ]
        result: minus_one(square(10)) -> 99

    '''
    import importlib

    def find_func(func_str):
        func_name = func_str.split('.')[-1]
        mod_str = '.'.join(func_str.split('.')[:-1])
        # import pdb
        # pdb.set_trace()
        mod = importlib.import_module(mod_str)
        func = getattr(mod, func_name, None)
        if not callable(func):
            return None
        return func

    # invalid mod check
    # find mod func, remove none func
    # create pipeline
    # evaluate

    funcs = [find_func(func_str) for func_str in funcs]
    pipeline = PipeLine(*funcs)
    result = pipeline.evaluate(input_argument)
    return list(result)

# if __name__ == '__main__':
#     print(evaluate_pipeline(
#         10, ['mods.example.square', 'mods.example.minus_one']))

#     print(evaluate_pipeline(
#         [10, 11, 12, 13, 18, 100, 1112], [
#             'mods.example.square', 'mods.example.minus_one']
#     ))


import numpy as np
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.layers import RNN
from keras.utils import np_utils
from keras.callbacks import ModelCheckpoint

text = (open("dataset/dataset2.txt").read())
text=text.lower()

text=text.split('\n')
lenght=len(text)
sentences=[]
for  i  in range(lenght):
    temp=text[i].split("\t")
    if len(temp)>1:

        sentences.append(temp[1])
sentences=sentences[0:3000]
input_data_set=[]
words=[]

for i in sentences:
    temp=[]
    i=i[:-1]
    temp=i.split(" ")
    temp[-1]="."
    input_data_set.append(temp)
lengt2=len(input_data_set)
for i in range(lengt2):
    for j in range(len(input_data_set[i])):
        words.append(input_data_set[i][j])

words_set= list(set(words))
words_set.sort()
words_set.insert(0,' ')

num2word = {n:char for n, char in enumerate(words_set)}

word2num = {char:n for n, char in enumerate(words_set)}

X=[]
Y=[]
length=len(temp)
sequence_length=5

for sentence in input_data_set:
    length4=len(sentence)

    for i in range(length4):

        displacement=abs(sequence_length-(1))

        j=0
        for k in range(sequence_length):
            input_train=[]

            if not (abs(displacement)<0):

                for j in range(abs(displacement)):
                    input_train.append(word2num[' '])
            j=0

            while (len(input_train))!=sequence_length:

                if i+j >=length4:
                    j+=1
                    break
                input_train.append(word2num[sentence[i+j]])
                j+=1

            displacement-=1

            if len(input_train)==sequence_length:

                X.append(input_train)

                if (i+j)>=length4:
                    X.pop()
                else:
                    Y.append(word2num[sentence[i+j]])

for i in range(length-sequence_length):
    sequence=temp[i:i+sequence_length]
    label=temp[i+sequence_length]
    X.append([word2num[word] for word in sequence])
    Y.append(word2num[label])

X_modified = np.reshape(X, (len(X), sequence_length, 1))
X_modified = X_modified / float(len(words_set))
Y_modified = np_utils.to_categorical(Y)

model = Sequential()
model.add(LSTM(400, input_shape=(X_modified.shape[1], X_modified.shape[2]), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(400, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(400))
model.add(Dropout(0.2))
model.add(Dense(Y_modified.shape[1], activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam')

#filepath="checkpoint_model_{epoch:02d}.hdf5"
#checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=False, mode='auto',period=5)
#callbacks_list = [checkpoint]

#model.fit(X_modified, Y_modified, validation_split=0.33, epochs=100, batch_size=64, callbacks=callbacks_list, verbose=1)
#model.fit(X_modified, Y_modified, epochs=100, batch_size=32)

#model.save_weights('flicker.h5')

model.load_weights('checkpoints/checkpoint_model_100.hdf5')

string_mapped = X[5]
while True:
    strings=input().split(' ')

    while (len(strings)<sequence_length):
        strings.insert(0,' ')
    if len(strings)>sequence_length:
        strings=strings[-4:]
    full_string=strings
    string_mapped=[]

    for i in range(sequence_length):
        string_mapped.append(word2num[strings[i]])

    # generating characters
    for i in range(20):
        x = np.reshape(string_mapped,(1,len(string_mapped), 1))
        x = x / float(len(words_set))
        pred_index = np.argmax(model.predict(x, verbose=0))
        seq = [num2word[value] for value in string_mapped]
        full_string.append(num2word[pred_index])

        string_mapped.append(pred_index)
        string_mapped = string_mapped[1:len(string_mapped)]
    txt=""
    for char in full_string:
        txt = txt+" "+ char
        if char==".":
            break
    print("Output :")
    print(txt)


from django.contrib.auth.decorators import login_required, user_passes_test
from django.shortcuts import render
from django.http import HttpResponse
from ADAUGDB.resources import BTCoursesResource
from ADAUGDB.models import BTCycleCoordinator, BTRegistrationStatus, BTCourseStructure, BTMarksDistribution,BTCourses
from BThod.models import BTCoordinator
from BTco_ordinator.models import BTSubjects_Staging, BTSubjects
from BTco_ordinator.forms import MDACoursesUploadForm, SubjectFinalizeEventForm
from ADAUGDB.user_access_test import mda_subject_access
from import_export.formats.base_formats import XLSX
from tablib import Dataset
from django.db import transaction

@login_required(login_url="/login/")
@user_passes_test(mda_subject_access)
def add_mda_courses(request):
    user = request.user
    groups = user.groups.all().values_list('name', flat=True)
    regIDs = []
    if 'Cycle-Co-ordinator' in groups:
        coordinator = BTCycleCoordinator.objects.filter(User=user, RevokeDate__isnull=True).first()
        valid_course_structures = BTCourseStructure.objects.filter(Category__in=['MDC', 'MOE'], BYear=1, Dept=coordinator.Cycle)
        regIDs = BTRegistrationStatus.objects.filter(Status=1, RegistrationStatus=1, BYear=1, Dept=coordinator.Cycle, Mode='R').\
            filter(Dept__in=valid_course_structures.values_list('Dept', flat=True), BSem__in=valid_course_structures.values_list('BSem', flat=True))
    elif 'Co-ordinator' in groups:
        coordinator = BTCoordinator.objects.filter(User=user, RevokeDate__isnull=True).first()
        valid_course_structures = BTCourseStructure.objects.filter(Category__in=['MDC', 'MOE'], BYear=coordinator.BYear, Dept=coordinator.Dept)
        regIDs = BTRegistrationStatus.objects.filter(Status=1, RegistrationStatus=1, BYear=coordinator.BYear, Dept=coordinator.Dept, Mode='R', \
            BSem__in=valid_course_structures.values_list('BSem', flat=True))
    if request.method == 'POST':
        form = MDACoursesUploadForm(regIDs, request.POST, request.FILES)
        if form.is_valid():
            if request.POST.get('Submit'):
                file = form.cleaned_data.get('file')
                data = bytes()
                for chunk in file.chunks():
                    data += chunk
                dataset = XLSX().create_dataset(data)
                newDataset= Dataset()
                invalidData = []
                newDataset.headers = ['SubCode', 'SubName', 'OfferedBy', 'CourseStructure', 'lectures', 'tutorials', 'practicals',\
                            'DistributionRatio', 'MarkDistribution']
                event = regIDs.filter(id=form.cleaned_data.get('regID')).first()
                for row in dataset:
                    if (row[2], row[3], row[4], row[6]) != (event.BYear, event.BSem, event.Dept, event.Regulation):
                        invalidData.append(row)
                        continue
                    course_struct_obj = BTCourseStructure.objects.filter(Category=row[10].upper(), Type=row[9].upper(), Creditable=row[7], Credits__lte=row[8],\
                        Regulation=row[6], BYear=row[2], BSem=row[3], Dept=row[4], Rigid=False).first()
                    mark_dis = row[15].split(';')
                    mark_distribution = BTMarksDistribution.objects.filter(Regulation=event.Regulation, Distribution=mark_dis[0], \
                        DistributionNames=mark_dis[-1], PromoteThreshold=mark_dis[1]).first()
                    newRow = (row[0], row[1], row[5], course_struct_obj.id, row[11], row[12], row[13], row[14], mark_distribution.id)

                    newDataset.append(newRow)
                    courses_resource = BTCoursesResource()
                    result = courses_resource.import_data(newDataset, dry_run=True)
                    if not result.has_errors():
                        courses_resource.import_data(newDataset, dry_run=False)
                        with transaction.atomic():
                            course_structure = BTCourseStructure.objects.filter(Regulation=event.Regulation, BYear=event.BYear, BSem=event.BSem, Dept=event.Dept, Category__in=['MDC', 'MOE'])
                            courses = BTCourses.objects.filter(CourseStructure_id__in=course_structure.values_list('id', flat=True))
                            for course in courses:
                                if not BTSubjects_Staging.objects.filter(RegEventId_id=event.id, course_id=course.id).exists():
                                    subject_row = BTSubjects_Staging(RegEventId_id=event.id, course_id=course.id)
                                    subject_row.save()
                        msg = 'Courses Uploaded successfully'
                        return render(request, 'BTco_ordinator/MDAcoursesUpload.html', {'form':form, 'invalidData':invalidData, 'msg': msg})
                    else:
                        errors = result.row_errors()
                        print(errors[0][1][0].__dict__)
                        # print(errors.error())
                        indices = set([i for i in range(len(newDataset))])
                        errorIndices = set([i[0]-1 for i in errors])
                        cleanIndices = indices.difference(errorIndices)
                        cleanDataset = Dataset()
                        for i in list(cleanIndices):
                            cleanDataset.append(newDataset[i])
                        cleanDataset.headers = newDataset.headers

                        result1 = courses_resource.import_data(cleanDataset, dry_run=True)
                        if not result1.has_errors():
                            courses_resource.import_data(cleanDataset, dry_run=False)
                            with transaction.atomic():
                                course_structure = BTCourseStructure.objects.filter(Regulation=event.Regulation, BYear=event.BYear, BSem=event.BSem, Dept=event.Dept, Category__in=['MDC', 'MOE'])
                                courses = BTCourses.objects.filter(CourseStructure_id__in=course_structure.values_list('id', flat=True))
                                for course in courses:
                                    if not BTSubjects_Staging.objects.filter(RegEventId_id=event.id, course_id=course.id).exists():
                                        subject_row = BTSubjects_Staging(RegEventId_id=event.id, course_id=course.id)
                                        subject_row.save()
                        errorRows = []
                        for i in errorIndices:
                            errorRows.append(dataset[i])
                        if errorRows:
                            return render(request, 'BTco_ordinator/MDAcoursesUpload.html', {'form':form, 'errorRows': errorRows, 'invalidData':invalidData})
                        msg = 'Courses Uploaded successfully'
                        return render(request, 'BTco_ordinator/MDAcoursesUpload.htmll', {'form':form, 'invalidData':invalidData, 'msg': msg})
            elif request.POST.get('download'):
                from ADAUGDB.utils import CoursesTemplateExcelFile
                event = regIDs.filter(id=form.cleaned_data.get('regID')).first()
                response = HttpResponse(content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',)
                response['Content-Disposition'] = 'attachment; filename={event_str}.xlsx'.format(event_str=event.__str__())
                BookGenerator = CoursesTemplateExcelFile(event.Regulation, event.BYear)
                workbook = BookGenerator.generate_workbook()
                workbook.save(response)
                return response
        pass
    else:
        form = MDACoursesUploadForm(regIDs)
    return render(request, 'BTco_ordinator/MDAcoursesUpload.html', {'form':form})

@transaction.atomic
@login_required(login_url="/login/")
@user_passes_test(mda_subject_access)
def mda_subject_finalize(request):
    user = request.user
    groups = user.groups.all().values_list('name', flat=True)
    regIDs = None
    msg = ''
    if 'Cycle-Co-ordinator' in groups:
        coordinator = BTCycleCoordinator.objects.filter(User=user, RevokeDate__isnull=True).first()
        valid_course_structures = BTCourseStructure.objects.filter(Category__in=['MDC', 'MOE'], BYear=1, Dept=coordinator.Cycle)
        regIDs = BTRegistrationStatus.objects.filter(Status=1, RegistrationStatus=1, BYear=1, Dept=coordinator.Cycle, Mode='R').\
            filter(Dept__in=valid_course_structures.values_list('Dept', flat=True), BSem__in=valid_course_structures.values_list('BSem', flat=True))
    elif 'Co-ordinator' in groups:
        coordinator = BTCoordinator.objects.filter(User=user, RevokeDate__isnull=True).first()
        valid_course_structures = BTCourseStructure.objects.filter(Category__in=['MDC', 'MOE'], BYear=coordinator.BYear, Dept=coordinator.Dept)
        regIDs = BTRegistrationStatus.objects.filter(Status=1, RegistrationStatus=1, BYear=coordinator.BYear, Dept=coordinator.Dept, Mode='R', \
            BSem__in=valid_course_structures.values_list('BSem', flat=True))
    if(request.method=='POST'):
        form = SubjectFinalizeEventForm(regIDs, request.POST)
        if(form.is_valid()):
            subjects = []
            subjects = BTSubjects_Staging.objects.filter(RegEventId_id=form.cleaned_data.get('regID'), course__CourseStructure__Category__in=['MDC', 'MOE'])
            for sub in subjects:
                s=BTSubjects(RegEventId_id=sub.RegEventId_id, course_id=sub.course_id)
                s.save()
            msg = 'Subjects finalized successfully.'
    else:
        form = SubjectFinalizeEventForm(regIDs)
    return render(request, 'BTco_ordinator/BTSubjectFinalize.html',{'form':form, 'msg':msg})


from sqlalchemy import create_engine, Column, INTEGER, String, DATETIME, BOOLEAN, FLOAT, ForeignKey
from sqlalchemy.orm import scoped_session, sessionmaker, relationship
from sqlalchemy.ext.declarative import declarative_base

engine = create_engine('sqlite:///bookstore.db', convert_unicode=True)
db_session = scoped_session(sessionmaker(autocommit=False, bind=engine))

Base = declarative_base()
Base.query = db_session.query_property()

class Book(Base):
    __tablename__ = 'book'
    id = Column(INTEGER, primary_key=True)
    title = Column(String(256), index=True)
    status = Column(String(20))
    reserved = Column(BOOLEAN)
    fine = Column(FLOAT)
    dateRent = Column(DATETIME)

    def __repr__(self):
        return '<Book {}>'.format(self.id)

    def save(self):
        db_session.add(self)
        db_session.commit()

    def delete(self):
        db_session.delete(self)
        db_session.commit()

class Client(Base):
    __tablename__ = 'client'
    id = Column(INTEGER, primary_key=True)
    name = Column(String(256), index=True)

    def save(self):
        db_session.add(self)
        db_session.commit()

    def delete(self):
        db_session.delete(self)
        db_session.commit()

class Rent(Base):
    __tablename__ = 'rent'
    id = Column(INTEGER, primary_key=True)
    client_id = Column(INTEGER)
    book_id = Column(INTEGER, ForeignKey('book.id'))
    book = relationship('Book')
    dateReservation = Column(DATETIME)

    def __repr__(self):
        return '<Rent {}>'.format(self.book_id, self.client_id)

    def save(self):
        db_session.add(self)
        db_session.commit()

    def delete(self):
        db_session.delete(self)
        db_session.commit()

def init_db():
    Base.metadata.create_all(bind=engine)

if __name__ == '__main__':
    init_db()


"""
Main script to fit and predict the label (classes) of the blobs
"""
import warnings
from pathlib import Path

import pytorch_lightning as pl

from zero_to_hero.config_reader import read_config
from zero_to_hero.data.blobs import BlobsDataModule
from zero_to_hero.models.blobs_classifier import BlobsClassifierModel
from zero_to_hero.trainer.blobs_with_callbacks import get_blobs_trainer_with_callbacks

warnings.filterwarnings("ignore")

def main() -> None:
    """
    Main function that runs the fit (training) of the blobs' classifier and its predictions
    :return:
    """
    pl.seed_everything(
        seed=1,
        workers=True,
    )

    config = read_config(path=Path("configs/blobs.yml"))

    datamodule = BlobsDataModule(config=config)
    model = BlobsClassifierModel(config=config)

    trainer = get_blobs_trainer_with_callbacks(config=config)

    trainer.fit(
        model=model,
        datamodule=datamodule,
    )
    print("Best checkpoint path:", trainer.checkpoint_callback.best_model_path)

    trainer.test(
        model=model,
        datamodule=datamodule,
        ckpt_path="best",
        verbose=True,
    )

    predictions_probabilities = trainer.predict(
        model=model,
        datamodule=datamodule,
        return_predictions=True,
        ckpt_path="best",
    )
    print(predictions_probabilities)

if __name__ == "__main__":
    main()


# -*- encoding: utf-8 -*-
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
"""
Utils for testing the API service.
"""

import datetime
import hashlib
import json

from ironic.api.controllers.v1 import allocation as al_controller
from ironic.api.controllers.v1 import chassis as chassis_controller
from ironic.api.controllers.v1 import deploy_template as dt_controller
from ironic.api.controllers.v1 import node as node_controller
from ironic.api.controllers.v1 import port as port_controller
from ironic.api.controllers.v1 import portgroup as portgroup_controller
from ironic.api.controllers.v1 import utils as api_utils
from ironic.api.controllers.v1 import volume_connector as vc_controller
from ironic.api.controllers.v1 import volume_target as vt_controller
from ironic.tests.unit.db import utils as db_utils

ADMIN_TOKEN = '4562138218392831'
MEMBER_TOKEN = '4562138218392832'

ADMIN_BODY = {
    'access': {
        'token': {'id': ADMIN_TOKEN,
                  'expires': '2100-09-11T00:00:00'},
        'user': {'id': 'user_id1',
                 'name': 'user_name1',
                 'tenantId': '123i2910',
                 'tenantName': 'mytenant',
                 'roles': [{'name': 'admin'}]},
    }
}

MEMBER_BODY = {
    'access': {
        'token': {'id': MEMBER_TOKEN,
                  'expires': '2100-09-11T00:00:00'},
        'user': {'id': 'user_id2',
                 'name': 'user-good',
                 'tenantId': 'project-good',
                 'tenantName': 'goodies',
                 'roles': [{'name': 'Member'}]},
    }
}

DEFAULT_CACHE_VALUES = {
    ADMIN_TOKEN: ADMIN_BODY,
    MEMBER_TOKEN: MEMBER_BODY
}

class FakeMemcache(object):
    """Fake cache that is used for keystone tokens lookup."""

    def __init__(self, cache_values=None):
        if not cache_values:
            cache_values = DEFAULT_CACHE_VALUES
        self._cache = {}
        for k, v in cache_values.items():
            self._cache['tokens/%s' % k] = v
            # NOTE(lucasagomes): keystonemiddleware >= 2.0.0 the token cache
            # keys are sha256 hashes of the token key. This was introduced in
            # https://review.opendev.org/#/c/186971
            self._cache['tokens/%s' %
                        hashlib.sha256(k.encode()).hexdigest()] = v
        self.set_key = None
        self.set_value = None
        self.token_expiration = None

    def get(self, key):
        dt = datetime.datetime.utcnow() + datetime.timedelta(minutes=5)
        return json.dumps((self._cache.get(key), dt.isoformat()))

    def set(self, key, value, time=0, min_compress_len=0):
        self.set_value = value
        self.set_key = key

def remove_internal(values, internal):
    # NOTE(yuriyz): internal attributes should not be posted, except uuid
    int_attr = [attr.lstrip('/') for attr in internal if attr != '/uuid']
    return {k: v for (k, v) in values.items() if k not in int_attr}

def remove_other_fields(values, allowed_fields):
    return {k: v for (k, v) in values.items() if k in allowed_fields}

def node_post_data(**kw):
    node = db_utils.get_test_node(**kw)

    # NOTE(jroll): pop out fields that were introduced in later API versions,
    # unless explicitly requested. Otherwise, these will cause tests using
    # older API versions to fail.
    for field in api_utils.VERSIONED_FIELDS:
        if field not in kw:
            node.pop(field, None)

    return remove_other_fields(
        node, node_controller.node_schema()['properties'])

def port_post_data(**kw):
    port = db_utils.get_test_port(**kw)
    return remove_other_fields(port,
                               port_controller.PORT_SCHEMA['properties'])

def volume_connector_post_data(**kw):
    connector = db_utils.get_test_volume_connector(**kw)
    return remove_other_fields(connector,
                               vc_controller.CONNECTOR_SCHEMA['properties'])

def volume_target_post_data(**kw):
    target = db_utils.get_test_volume_target(**kw)
    return remove_other_fields(target,
                               vt_controller.TARGET_SCHEMA['properties'])

def chassis_post_data(**kw):
    chassis = db_utils.get_test_chassis(**kw)
    return remove_other_fields(
        chassis, chassis_controller.CHASSIS_SCHEMA['properties'])

def post_get_test_node(**kw):
    # NOTE(lucasagomes): When creating a node via API (POST)
    #                    we have to use chassis_uuid
    node = node_post_data(**kw)
    chassis = db_utils.get_test_chassis()
    node['chassis_uuid'] = kw.get('chassis_uuid', chassis['uuid'])
    return node

def portgroup_post_data(**kw):
    """Return a Portgroup object without internal attributes."""
    portgroup = db_utils.get_test_portgroup(**kw)

    # These values are not part of the API object
    portgroup.pop('version')
    portgroup.pop('node_id')

    # NOTE(jroll): pop out fields that were introduced in later API versions,
    # unless explicitly requested. Otherwise, these will cause tests using
    # older API versions to fail.
    new_api_ver_arguments = ['mode', 'properties']
    for arg in new_api_ver_arguments:
        if arg not in kw:
            portgroup.pop(arg)

    return remove_other_fields(
        portgroup, portgroup_controller.PORTGROUP_SCHEMA['properties'])

def post_get_test_portgroup(**kw):
    """Return a Portgroup object with appropriate attributes."""
    portgroup = portgroup_post_data(**kw)
    node = db_utils.get_test_node()
    portgroup['node_uuid'] = kw.get('node_uuid', node['uuid'])
    return portgroup

def allocation_post_data(node=None, **kw):
    """Return an Allocation object without internal attributes."""
    allocation = db_utils.get_test_allocation(**kw)
    if node:
        # This is not a database field, so it has to be handled explicitly
        allocation['node'] = node
    return remove_other_fields(
        allocation, al_controller.ALLOCATION_SCHEMA['properties'])

def deploy_template_post_data(**kw):
    """Return a DeployTemplate object without internal attributes."""
    template = db_utils.get_test_deploy_template(**kw)
    # These values are not part of the API object
    template.pop('version')
    # Remove internal attributes from each step.
    step_internal = api_utils.DEPLOY_STEP_SCHEMA['properties']
    template['steps'] = [remove_other_fields(step, step_internal)
                         for step in template['steps']]
    # Remove internal attributes from the template.
    return remove_other_fields(
        template, dt_controller.TEMPLATE_SCHEMA['properties'])

def post_get_test_deploy_template(**kw):
    """Return a DeployTemplate object with appropriate attributes."""
    return deploy_template_post_data(**kw)


from models import resnet_lc as resnet

import torch
import torch.optim as optim
from torchsummary import summary

import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable, grad
from torch import autograd
import numpy as np
import matplotlib.pyplot as plt

import os, time
from datetime import datetime
import logging

# Model Run Code
run_code = 'wgan-gp_LC_64'

# Define Optimizer
lr = 1e-4
beta1=.5
beta2=.9
batch_size = 64

num_epochs = 20

# noise length
nz = 128

n_critic = 5

image_size=64
num_classes=10
# Number of workers for dataloader
workers = 4

# Number of GPUs available. Use 0 for CPU mode.
ngpu = 1

# Print Model Structure.
print_model = False

# Logging..
log_dir='../logs/' + run_code + '/'
model_dir='../logs/' + run_code + '/model/'

# for Log
if not os.path.isdir(log_dir): os.makedirs(log_dir)
logging.basicConfig(filename=log_dir+'logging.log', level=logging.INFO)

# for Model Save
if not os.path.isdir(model_dir): os.makedirs(model_dir)

G_save_path = model_dir + 'generator.pth'
D_save_path = model_dir + 'critic.pth'

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
use_cuda = torch.cuda.is_available()

# onehot converter
onehot = torch.zeros(num_classes, num_classes)
onehot = onehot.scatter_(1,
                         torch.LongTensor(list(range(num_classes)))
                         .view(num_classes,1),1)\
                            .view(num_classes, num_classes,1,1)

# random
noise_label = (torch.rand(batch_size, 1)*num_classes).type(torch.LongTensor).squeeze()
noise_label_onehot = onehot[noise_label]
noise_label_onehot = noise_label_onehot.to(device)

# Layer Conditional
netG = resnet.Generator64(resnet.ResBlock, num_classes=num_classes, nz_dim=nz, in_channels=64,
                                        labels=noise_label_onehot).to(device)
netD = resnet.Discriminator64(resnet.ResBlock, num_classes=num_classes, in_channels=64,
                                            labels=noise_label_onehot).to(device)

optimizerD = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, beta2))
optimizerG = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, beta2))

if print_model:
    summary(netG, input_size=[(128, 1, 1), (num_classes,1,1)])
    summary(netD, input_size=[(3, image_size, image_size), (num_classes,1,1)])

dataroot = "../data/"
# We can use an image folder dataset the way we have it setup.
# Create the dataset
dataset = dset.ImageFolder(root=dataroot,
                           transform=transforms.Compose([
                               transforms.Resize(image_size),
                               transforms.CenterCrop(image_size),
                               transforms.ToTensor(),
                               transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                           ]))
# Create the dataloader
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
                                         shuffle=True, num_workers=workers)

# Decide which device we want to run on
device = torch.device("cuda:0" if (torch.cuda.is_available() and ngpu > 0) else "cpu")

# Gradient Penalty..
def calc_gradient_penalty(netD, real_data, fake_data, labels=None):
    alpha = torch.rand(batch_size, 1)
    alpha = alpha.expand(batch_size, int(real_data.nelement()/batch_size)).contiguous()
    alpha = alpha.view(batch_size, 3, 64, 64)
    alpha = alpha.to(device)

    fake_data = fake_data.view(batch_size, 3, 64, 64)
    interpolates = alpha * real_data.detach() + ((1 - alpha) * fake_data.detach())

    interpolates = interpolates.to(device)
    interpolates.requires_grad_(True)

    interp_label = labels

    disc_interpolates = netD(interpolates, interp_label)

    gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,
                              grad_outputs=torch.ones(disc_interpolates.size()).to(device),
                              create_graph=True, retain_graph=True, only_inputs=True)[0]

    gradients = gradients.view(gradients.size(0), -1)
    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * 10
    return gradient_penalty

# fixed noise for test
fixed_noise = torch.randn(batch_size, nz, 1, 1, device=device)

# Training Loop
# Lists to keep track of progress
img_list = []
G_losses = []
D_losses = []
GP = []

iters = 0

one = torch.FloatTensor([1])
mone = one * -1
one = one.cuda()
mone = mone.cuda()

print("Starting Training Loop...")
logging.debug('Starting Training Loop...')

# For each epoch
start_time = time.time()
for iters in range(num_epochs * len(dataloader)):
    # For each batch in the dataloader
    ############################
    # Train D
    ###########################
    for p in netD.parameters():  # reset requires_grad
        p.requires_grad = True  # they are set to False below in netG update

    for i in range(n_critic):
        real_data, real_label = next(iter(dataloader))
        netD.zero_grad()

        b_size = real_data.size(0)
        real_label = torch.tensor(np.array(real_label), dtype=torch.long)
        real_label_onehot = torch.tensor(np.eye(num_classes)[real_label], dtype=torch.long)
        real_label_onehot = real_label_onehot.view(real_label_onehot.shape[0], real_label_onehot.shape[1],1,1)
        if use_cuda:
            real_label_onehotv = Variable(real_label_onehot.type(torch.FloatTensor).cuda())

        # train with real
        if use_cuda:
            real_data = real_data.cuda()
        real_data_v = Variable(real_data)

        D_real = netD(real_data_v)
        D_real = D_real.mean()
        D_real= netD(real_data_v, real_label_onehotv)
        D_real.backward(mone)

        noise = torch.randn(b_size, nz, 1, 1, device=device)

        with torch.no_grad():
            noisev = Variable(noise)  # totally freeze netG

        noise_label = (torch.rand(batch_size, 1)*num_classes).type(torch.LongTensor).squeeze()
        noise_label_onehot = onehot[noise_label]
        if use_cuda:
            noise_label_onehot = noise_label_onehot.cuda()
        with torch.no_grad():
            noise_label_onehotv = Variable(noise_label_onehot.type(torch.FloatTensor).cuda())

        fake = Variable(netG(noisev, noise_label_onehotv).data)
        D_fake = netD(fake, noise_label_onehotv)
        D_fake = D_fake.mean()
        D_fake.backward(one)

        # train with gradient penalty
        gradient_penalty = calc_gradient_penalty(netD, real_data_v.data, fake.data, real_label_onehotv)
        GP_cost = gradient_penalty

        D_cost = D_fake - D_real + GP_cost
        GP_cost.backward()

        optimizerD.step()

        D_losses.append(D_cost.cpu().data.numpy())
        GP.append(GP_cost.cpu().data.numpy())

    ############################
    # Train G
    ###########################
    for p in netD.parameters():
        p.requires_grad = False  # to avoid computation
    netG.zero_grad()

    noise = torch.randn(batch_size, nz, 1, 1, device=device)
    noisev = autograd.Variable(noise)
    noise_label = (torch.rand(batch_size, 1)*num_classes).type(torch.LongTensor).squeeze()
    noise_label_onehot = onehot[noise_label]
    if use_cuda:
        noise_label_onehot = noise_label_onehot.cuda()
        noise_label_onehotv = Variable(noise_label_onehot.type(torch.FloatTensor).cuda())

    fake = netG(noisev, noise_label_onehotv)
    G= netD(fake, noise_label_onehotv)
    G = G.mean()
    G.backward(mone)

    G_cost = -G
    optimizerG.step()

    G_losses.append(G_cost.cpu().data.numpy())

    # Output training stats
    if iters % 10 == 0:
        s = datetime.now().strftime('[%Y-%m-%d %H:%M:%S]')
        iter_time = time.time() - start_time  # 현재시각 - 시작시간 = 실행 시간
        start_time = time.time()
        print(s + ' [%d/%d]\t GP: %.4f\t D: %.4f\t G: %.4f\t %.4f sec'
              % (iters, num_epochs * len(dataloader), GP[-1], D_losses[-1], G_losses[-1], iter_time ))
        logging.info(s + ' [%d/%d]\t GP: %.4f\t D: %.4f\t G: %.4f\t %.4f sec'
              % (iters, num_epochs * len(dataloader), GP[-1], D_losses[-1], G_losses[-1], iter_time ))

        torch.save(netG.state_dict(), G_save_path)
        torch.save(netD.state_dict(), D_save_path)
    if iters % 10 == 0:
        vutils.save_image(fake.reshape([64,3,64,64]), log_dir + 'fake_curren.png', nrow=8, padding=2, normalize=True)
    iters += 1


from apps.users.models import EmailUser
from django.forms import ModelForm

from .models import Hit

class HitCreateForm(ModelForm):
    class Meta:
        model = Hit
        fields = ('target', 'description',  'hitmen_by')

    def __init__(self, *args, manager=None, **kwargs):
        super().__init__(*args, **kwargs)
        if manager:
            self.fields['hitmen_by'].choices = [
               (x.id, x) for x in EmailUser.objects.filter(managed_by=manager)]

class HitUpdateForm(ModelForm):
    class Meta:
        model = Hit
        fields = ('status',)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['status'].choices = [
            ('failed', 'Failed'),
            ('completed', 'Completed')]


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import os

import tensorflow as tf
import tqdm
from absl import app
from absl import flags

from data.common import MODEL_DIR, get_pretrained_word_embedding
from data.mscoco import load_mscoco_data, DATA_DIR
from membership.utils import compute_adversarial_advantage
from text_encoder import ImageTextEncoder

flags.DEFINE_integer('print_every', 100, 'Print loss every N iterations')
flags.DEFINE_integer('batch_size', 512, 'Batch size')
flags.DEFINE_integer('encoder_dim', 512, 'encoder dim')
flags.DEFINE_integer('epoch', 100, 'Epochs of training')
flags.DEFINE_boolean('contrastive', False, 'Use vse++ contrastive loss')
flags.DEFINE_float('train_size', 0.1, 'Temp for sharpening')
flags.DEFINE_string('metric', 'cosine', 'Metric to use for membership')
flags.DEFINE_string('model', 'linear', 'Attack model')
flags.DEFINE_string('encoder_type', 'rnn', 'Text encoder type')
flags.DEFINE_string('image_model_name', 'resnet',
                    'Image feature extractor name')
flags.DEFINE_string('save_dir', os.path.join(MODEL_DIR, 'imgtxt'),
                    'Model directory for embedding model')
FLAGS = flags.FLAGS

def collect_metrics(img_embs, text_embs, img_id_text_ids):
  metrics = []
  for img_id in tqdm.tqdm(img_id_text_ids):
    text_ids = img_id_text_ids[img_id]
    i_emb = img_embs[img_id]
    t_embs = text_embs[text_ids]
    if FLAGS.metric == 'l2':
      scores = [-np.linalg.norm(i_emb - t_emb) for t_emb in t_embs]
    elif FLAGS.metric == 'cosine':
      scores = [np.dot(i_emb, t_emb) for t_emb in t_embs]
    else:
      raise ValueError(FLAGS.metrics)

    metrics.append(scores)
  return metrics

def encode_image_text_data():
  train_data, val_data, vocab = load_mscoco_data(FLAGS.image_model_name)
  word_embedding = get_pretrained_word_embedding(
      vocab, glove=False, vocab_include_pad=True,
      save_path=DATA_DIR + 'w2v.npz')
  encoder = ImageTextEncoder(word_embedding, FLAGS.encoder_dim,
                             FLAGS.encoder_type)

  model_name = 'coco_e{}_b{}_{}'.format(
    FLAGS.epoch, FLAGS.batch_size, FLAGS.encoder_type)
  if FLAGS.contrastive:
    model_name += '_vse'

  save_path = os.path.join(FLAGS.save_dir, model_name)

  encoder.load_weights(save_path)
  train_img_embs, train_text_embs, train_img_id_text_ids = \
      encoder.encode(train_data)
  test_img_embs, test_text_embs, test_img_id_text_ids = \
      encoder.encode(val_data)

  train_data = train_img_embs, train_text_embs, train_img_id_text_ids
  test_data = test_img_embs, test_text_embs, test_img_id_text_ids
  tf.keras.backend.clear_session()
  return train_data, test_data

def main(_):
  train_data, test_data = encode_image_text_data()
  train_metrics = collect_metrics(*train_data)
  test_metrics = collect_metrics(*test_data)
  compute_adversarial_advantage([np.mean(m) for m in train_metrics],
                                [np.mean(m) for m in test_metrics])

if __name__ == '__main__':
  app.run(main)


import os.path
import os
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from datetime import datetime

# If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/drive']

def main():
    """Shows basic usage of the Drive v3 API.
    Prints the names and ids of the first 10 files the user has access to.
    """

    creds = None
    # The file token.json stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPES)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            print("Refresh Creds")
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                'client_secrets.json', SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open('token.json', 'w') as token:
            token.write(creds.to_json())

    service = build('drive', 'v3', credentials=creds)

    # Call the Drive v3 API
    results = service.files().list(
        q="mimeType = 'application/vnd.google-apps.folder' and '0ALNhV0hP-QYDUk9PVA' in parents",
        pageSize=100, fields="nextPageToken, files(id, name, parents)").execute()
    items = results.get('files', [])

    pic_id = ''

    if not items:
        print('No files found.')
    else:
        print('1st Files:')
        for item in items:
            if item['name']=='KIOSK Picture':
                pic_id = item['id']
            print(u'{0} ({1}) - {2}'.format(item['name'], item['id'], item['parents']))
    #print(pic_id)

    # Check Machine ID
    q_str = "mimeType = 'application/vnd.google-apps.folder' and '" + str(pic_id) +"' in parents"
    #print(q_str)
    results = service.files().list(
        q=q_str, #"mimeType = 'application/vnd.google-apps.folder' and '" + str(pic_id) +"' in parents",
        pageSize=10, fields="nextPageToken, files(id, name, parents)").execute()
    items = results.get('files', [])

    bHasBaseFolder = False
    sMachineID = 'Test_MachineID'
    sMachineID_ID = ''
    if not items:
        print('No files found.')
    else:
        print('2nd Files:')
        for item in items:
            if item['name']==sMachineID:
                bHasBaseFolder = True
                sMachineID_ID = item['id']
            print(u'{0} ({1}) - {2}'.format(item['name'], item['id'], item['parents']))

    if bHasBaseFolder == False:
        file_metadata = {
            'name': sMachineID,
            'mimeType': 'application/vnd.google-apps.folder',
            'parents': [str(pic_id)]
        }
        file = service.files().create(body=file_metadata,
                                    fields='id').execute()
        sMachineID_ID = str(file.get('id'))
        print('Folder ID: %s' % file.get('id'))

    #print(sMachineID_ID)

    # Check Date Folder
    sTodayDateString = datetime.now().strftime("%Y%m%d")
    sTodayDate_ID = ''
    bHasBaseFolder = False
    q_str = "mimeType = 'application/vnd.google-apps.folder' and '" + str(sMachineID_ID) +"' in parents"
    results = service.files().list(
        q=q_str,
        pageSize=10, fields="nextPageToken, files(id, name, parents)").execute()
    items = results.get('files', [])
    if not items:
        print('No files found.')
    else:
        print('3nd Files:')
        for item in items:
            if item['name']==sTodayDateString:
                bHasBaseFolder = True
                sTodayDate_ID = item['id']
            print(u'{0} ({1}) - {2}'.format(item['name'], item['id'], item['parents']))
    if bHasBaseFolder == False:
        file_metadata = {
            'name': sTodayDateString,
            'mimeType': 'application/vnd.google-apps.folder',
            'parents': [str(sMachineID_ID)]
        }
        file = service.files().create(body=file_metadata,
                                    fields='id').execute()
        sTodayDate_ID = str(file.get('id'))
        print('Folder ID: %s' % file.get('id'))

    #Check Test Location
    sTestLocation='我是測試考場(真的是測試用)'
    sTestLocation_ID = ''
    bHasBaseFolder = False
    q_str = "mimeType = 'application/vnd.google-apps.folder' and '" + str(sTodayDate_ID) +"' in parents"
    results = service.files().list(
        q=q_str,
        pageSize=10, fields="nextPageToken, files(id, name, parents)").execute()
    items = results.get('files', [])
    if not items:
        print('No files found.')
    else:
        print('4nd Files:')
        for item in items:
            if item['name']==sTestLocation:
                bHasBaseFolder = True
                sTestLocation_ID = item['id']
            print(u'{0} ({1}) - {2}'.format(item['name'], item['id'], item['parents']))
    if bHasBaseFolder == False:
        file_metadata = {
            'name': sTestLocation,
            'mimeType': 'application/vnd.google-apps.folder',
            'parents': [str(sTodayDate_ID)]
        }
        file = service.files().create(body=file_metadata,
                                    fields='id').execute()
        sTestLocation_ID = str(file.get('id'))
        print('Folder ID: %s' % file.get('id'))

    sTestLocation_ID = CreateGoogleDriveFolder(service, sTestLocation, sTodayDate_ID)
    print('Check Function')
    print(sTestLocation_ID)

def CreateGoogleDriveFolder(service, titlestring, folderid):
    returnfolderid=''

    bHasFolder = False
    q_str = "mimeType = 'application/vnd.google-apps.folder' and '" + str(folderid) +"' in parents"
    results = service.files().list(
        q=q_str,
        pageSize=10, fields="nextPageToken, files(id, name, parents)").execute()
    items = results.get('files', [])
    if not items:
        print('No files found.')
    else:
        for item in items:
            if item['name']==titlestring:
                bHasFolder = True
                returnfolderid = item['id']
            print(u'{0} ({1}) - {2}'.format(item['name'], item['id'], item['parents']))
    if bHasFolder == False:
        try:
            file_metadata = {
                'name': titlestring,
                'mimeType': 'application/vnd.google-apps.folder',
                'parents': [str(folderid)]
            }
            file = service.files().create(body=file_metadata,
                                        fields='id').execute()
            returnfolderid = str(file.get('id'))
            print('Folder ID: %s' % file.get('id'))
        except Exception as ex:
            print(ex)

    return returnfolderid

if __name__ == '__main__':
    main()