import sqlite3 as lite
import paho.mqtt.client as mqtt

def on_connect(client, data, rc):
    client.subscribe([(temperatureTopic, 0)])

def on_message(client, userdata, msg):
    msg_data = msg.payload.split("#")
    cur.execute("INSERT INTO temperatures (timestamp, temp, sent) VALUES(?, ?, ?)", (msg_data[0], msg_data[1], False))
    con.commit()

try:
    con = lite.connect('fermentation.db')
    cur = con.cursor()

    temperatureTopic = "smarthomebrew/sensor/temperatura"

    client = mqtt.Client(client_id='DATABASE',
                         protocol=mqtt.MQTTv31)

    client.on_connect = on_connect
    client.on_message = on_message

    client.connect("127.0.0.1", 1883)
    client.loop_forever()

except KeyboardInterrupt:
    print("Pressed CTRL+C! :(")
finally:
    if con:
        con.close()


import numpy as np

import torch
from typing import Dict

from labml import lab, monit
from labml.utils import download

class CoraDataset:
    '''
    # Cora Dataset

    2,708건의 Scientific Publication을 포함하는 데이터셋.
    7개의 class로 분류되며, 각각의 publication을 노드로 하여 인용 네트워크를 이룬다.
    해당 인용 네트워크는 5,429개의 link(edge)를 가지는 Directed Graph이다.

    label prediction과 link prediction 등에서 Benchmarks Dataset로 사용된다.
    '''
    labels: torch.Tensor
    classes: Dict[str, int]
    features: torch.Tensor
    adj_mat: torch.Tensor

    # Download the Dataset
    @staticmethod
    def _download():
        if not (lab.get_data_path() / 'cora').exists():
            download.download_file('https://linqs-data.soe.ucsc.edu/public/lbc/cora.tgz',
                                   lab.get_data_path() / 'cora.tgz')
            download.extract_tar(lab.get_data_path() /
                                 'cora.tgz', lab.get_data_path())

    def __init__(self, include_edges: bool = True):
        self.include_edges = include_edges
        self._download()

        with monit.section('Read Content'):
            content = np.genfromtxt(
                str(lab.get_data_path() / 'cora/cora.content'), dtype=np.dtype(str))

        with monit.section('Read Citations'):
            citations = np.genfromtxt(
                str(lab.get_data_path() / 'cora/cora.cites'), dtype=np.int32)

        # Get the feature vectors
        features = torch.tensor(np.array(content[:, 1:-1], dtype=np.float32))

        # Normalize
        self.features = features / features.sum(dim=1, keepdim=True)

        # Get class names and assign integer tags
        self.classes = {s: i for i, s in enumerate(set(content[:, -1]))}
        # to labels
        self.labels = torch.tensor([self.classes[i]
                                   for i in content[:, -1]], dtype=torch.long)

        # Get paper ids
        paper_ids = np.array(content[:, 0], dtype=np.int32)
        # Map id to idx
        ids_to_idx = {id_: i for i, id_ in enumerate(paper_ids)}

        # Empty adj_mat
        self.adj_mat = torch.eye(len(self.labels), dtype=torch.bool)

        # Mark the citations in adj_mat
        if self.include_edges:
            for e in citations:
                e1, e2 = ids_to_idx[e[0]], ids_to_idx[e[1]]

                self.adj_mat[e1][e2] = True
                self.adj_mat[e2][e1] = True


import unittest
import os
from config import Config
from src.store_transactions import TransactionRecord

test_config = Config()

class TestApp(unittest.TestCase):
    # set up some test parameters to be used
    test_store_id1 = "YR4226"
    test_product_id = 46988220
    test_unavailable_product_id = 123
    test_csv_file_path = test_config.csv_filepath
    test_output_folder = test_config.output_folder
    test_VIP_WEIGHT = test_config.VIP_WEIGHT

    # Create a valid object with known output for testing
    test_transaction = TransactionRecord(
        test_csv_file_path,
        test_output_folder,
        test_product_id,
        test_VIP_WEIGHT
    )
    response = test_transaction.get_product_transaction()

    # Create a non-valid object with known output for testing
    test_unavailable_transaction = TransactionRecord(
        test_csv_file_path,
        test_output_folder,
        test_unavailable_product_id,
        test_VIP_WEIGHT
    )

    def test_config(self):
        """
        This function test if config parameters parsed correctely
        """
        self.assertEqual(
            self.test_transaction.csv_filepath,
            self.test_csv_file_path
        )
        self.assertEqual(
            self.test_transaction.output_folder,
            self.test_output_folder
        )
        self.assertEqual(
            self.test_transaction.VIP_WEIGHT,
            self.test_VIP_WEIGHT
        )

    def test_read_csv(self):
        """
        This function test if data from csv file read correctely
        """
        self.assertEqual(
            self.test_transaction.df.iloc[0, :]["PRODUCT_ID"],
            12254943
        )

    def test_error_hanling_for_unavilable_id(self):
        """
        This function test error handling if request product_id does not
        exists in the data
        """
        response = self.test_unavailable_transaction.get_product_transaction()
        self.assertEqual(
            response["status"],
            "Failed"
        )

    def test_output_file_generation(self):
        """
        This function test if output file generates correctely for valid product id
        """

        self.assertEqual(
            self.response["status"],
            "Success"
        )
        self.assertTrue(os.path.exists(self.test_transaction.output_filepath))

    def test_classification_value(self):
        """
        This function test if correct CLASSIFICATION value added based on MRP
        """
        transactions = self.response["content"]
        vip_transaction = transactions[transactions["MRP"] >= self.test_transaction.VIP_WEIGHT].iloc[0, :]

        self.assertEqual(
             vip_transaction["CLASSIFICATION"],
             "VIP"
        )

    def test_output_file_exist_functionality(self):
        """
        This function test error handling for case when product id requested
        is already requested before and output file exists in output folder
        This functionality need to remove when working with live data
        """
        response = self.test_transaction.get_product_transaction()
        self.assertEqual(
            response["status"],
            "File exists"
        )
        self.assertEqual(
            response["content"],
            f"{self.test_transaction.output_file} file has alredy been generated"
        )

    def test_remove_output_file_generated(self):
        """
        This function remove previousely generated output files for test case,
        so that it's ready for next run
        """
        os.remove(self.test_transaction.output_filepath)
        self.assertFalse(os.path.exists(self.test_transaction.output_filepath))

if __name__ == "__main__":
    unittest.main()


import json
import os

class Task:

  def __init__(self, content, completed=False) -> None:
    self.completed = completed
    self.content = content
    self.position = 0

class TaskEncoder(json.JSONEncoder):

  def default(self, obj):
    if isinstance(obj, Task):
      return {
          '_type': 'Task',
          'content': obj.content,
          'completed': obj.completed,
          'position': obj.position
      }
    return super().default(obj)

def task_decoder(obj):
  if '_type' in obj and obj['_type'] == 'Task':
    return Task(obj['content'], obj['completed'])
  return obj

class Note:

  def __init__(self) -> None:
    self.tasks = []
    self.id = 1
    self.load_tasks()

  def add_task(self, text):
    task = Task(text)
    task.position = self.id
    self.tasks.append({"id": self.id, "task": task})
    self.id += 1
    self.save_tasks()
    print("\nTask added successfully!")

  def set_complete(self):
    incompleted_tasks = [
        task['task'] for task in self.tasks if not task['task'].completed
    ]

    for i, task in enumerate(incompleted_tasks):
      print(f"{i+1}- {task.content}")

    task_index = int(input("\nChoose task to complete: ")) - 1

    if 0 <= task_index < len(self.tasks):
      incompleted_tasks[task_index].completed = True
      self.save_tasks()
      print("\nThe selected task marked as completed!")
    else:
      print("Invalid task index!")

  def view_tasks(self):
    if self.tasks:
      for task in self.tasks:
        print(f"\nTask {task['id']}: {task['task'].content}")
        print(
            f"Status: {'Completed' if task['task'].completed else 'Not completed yet'}"
        )
    else:
      print("< There are no tasks yet >")

  def load_tasks(self):
    if os.path.exists('tasks.json'):
      with open('tasks.json', 'r') as file:
        data = json.load(file, object_hook=task_decoder)
        self.tasks = data['tasks']
        self.id = data['id']

  def save_tasks(self):
    with open('tasks.json', 'w') as file:
      data = {'tasks': self.tasks, 'id': self.id}
      json.dump(data, file, cls=TaskEncoder)

  def clear_tasks(self):
    confirm = input("Are you sure you want to clear all tasks? (yes/no): ").strip().lower()
    if confirm == "yes":
      self.tasks = []
      self.id = 1
      self.save_tasks()
      print("All tasks have been cleared.")
    else:
      print("Task clearing canceled.")


"""pbm.py
"""

__license__ = "Apache License, Version 2.0"
__author__  = "Roland Kwitt, Kitware Inc., 2013"
__email__   = "E-Mail: roland.kwitt@kitware.com"
__status__  = "Development"

import os
import sys
import json
import numpy as np
import SimpleITK as sitk
from scipy.spatial.distance import cdist
from sklearn.decomposition import DictionaryLearning
from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.decomposition import sparse_encode
from optparse import OptionParser
from core import pbmutils
from core import regtools

def usage():
    """Print usage information"""
    print("""
Pattern based morphometry implementation using sklearn's dictionary learning
code. We learn a sparse dictionary for difference volumes (images), built by
subtracting an image of population A from its K closest neighbors (in the
Euclidean sense) in population B. The dictionary elements can then be visualized
and hopefully highlight characteristic differences between populations A and B.

    USAGE:
        {0} [OPTIONS]
        {0} -h

    OPTIONS (Overview):

        -k NUM
        -D NUM
        -r NUM
        -s NUM
        -r NUM
        -c FILE
        -d FILE
        -a FILE
        -x FILE

    OPTIONS (Detailed):

        -c FILE

        FILE is a JSON file that contains the absolute paths to a collection
        of binaries that are used by core.regtools.

        -i FILE

        FILE is a JSON file that contains information about all the image files
        that need to be processed, as well as the group information.

        -k NUM (default: 5)

        NUM specifies the number of neighbors that are used to build the
        difference images that are later used to compute the dictionary.

        -D NUM (default: 5)

        NUM is an integer value that specifies the size of the learned
        dictionary.

        -r NUM (optional)

        NUM is a float value that specifies the resizing factor of the input
        images. Resampling is a good way to achieve lower computation times
        in the dictionary learning stage.

        -s NUM (optional)

        First, this flag indicates that we only want to run on image slices.
        NUM specifies exactly the slice (in the AP direction) to use. CAUTION:
        Make sure that the slice is within the image!.

        -d FILE (optional)

        If -x is specified, FILE specifies the output file to which the
        difference image data is written (as float32).

        -a FILE (optional)

        If -a is given, FILE specifies the output dictionary file that will be
        written upon completion of the dictionary learning stage (as float32).

        -x FILE (optional)

        If -x is given, FILE specifies the output file to which the raw image
        data is written (as float32).

AUTHOR: Roland Kwitt, Kitware Inc., 2013
        roland.kwitt@kitware.com
""".format(sys.argv[0]))

def main(argv=None):
    if argv is None:
        argv=sys.argv

    parser = OptionParser(add_help_option=False)
    parser.add_option("-i", dest="imgJSON")
    parser.add_option("-c", dest="cfgJSON")
    parser.add_option("-a", dest="outAtomFile")
    parser.add_option("-d", dest="outDiffFile")
    parser.add_option("-x", dest="outImagFile")
    parser.add_option("-s", dest="imSlice", type="int")
    parser.add_option("-r", dest="imScale", type="float")
    parser.add_option("-D", dest="dictSiz", type="int", default=5)
    parser.add_option("-k", dest="nearest", type="int", default=5)
    parser.add_option("-h", dest="doHelp", action="store_true", default=False)
    options, _ = parser.parse_args()

    if options.doHelp:
        usage()
        sys.exit(-1)

    imgJSON = options.imgJSON
    cfgJSON = options.cfgJSON

    outAtomFile = options.outAtomFile
    outImagFile = options.outImagFile
    outDiffFile = options.outDiffFile

    imSlice = options.imSlice
    dictSiz = options.dictSiz
    imScale = options.imScale
    nearest = options.nearest

    imData = json.load(open(imgJSON))
    helper = regtools.regtools(cfgJSON)

    groupSet = set()
    groupMap = dict()
    groupLab = []

    # generate numeric labels for each image
    for entry in imData["Data"]: groupSet.add(entry["Group"])
    for cnt, group in enumerate(groupSet): groupMap[group] = cnt
    for entry in imData["Data"]: groupLab.append(groupMap[entry["Group"]])

    imgFiles = []
    [imgFiles.append(str(e["Source"])) for e in imData["Data"]]

    dataList = []
    for i, imFile in enumerate(imgFiles):
        im0 = sitk.ReadImage(imFile)
        im1 = pbmutils.imResize(im0, imScale)
        imSz = sitk.GetArrayFromImage(im1).shape
        helper.infoMsg("Image size : (%d,%d,%d)" % imSz)
        if not imSlice is None:
            sl0 = pbmutils.imSlice(im1, [0, 0, imSlice])
            dataList.append(sl0.ravel())
        else:
            dataList.append(sitk.GetArrayFromImage(im1).ravel())
        helper.infoMsg("Done with image %d!" % i)

    # write raw image data
    if not outImagFile is None:
        tfid = open(outImgFile, 'w')
        np.reshape(np.asmatrix(dataList).T,-1).astype('float32').tofile(tfid)
        tfid.close()

    # build difference images
    diffIm = pbmutils.groupDiff(np.asmatrix(dataList).T, groupLab, nearest)
    helper.infoMsg("Difference image matrix (%d x %d)" % diffIm.shape)

    # write raw difference data
    if not outDiffFile is None:
        outFid = open(outDiffFile, 'w')
        np.reshape(diffIm, -1).ravel().astype('float32').tofile(outFid)
        outFid.close()

    # create the dictionary learner and run (alpha=1)
    lrnObj = MiniBatchDictionaryLearning(dictSiz, 1, verbose=True)
    lrnRes = lrnObj.fit(np.asmatrix(diffIm).T).components_

    # write dictionary atoms
    if not outAtomFile is None:
        outFid = open(outAtomFile, 'w')
        np.reshape(lrnRes.T, -1).ravel().astype('float32').tofile(outFid)
        outFid.close()

if __name__ == "__main__":
    sys.exit(main())


import pickle
import os,sys,glob
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import cv2
from scipy.spatial.distance import cdist
from scipy.interpolate import interp1d

def find_nearest(pnt1, arr, num=1):
    dist = (np.sum((pnt1.reshape(1,-1)-arr[:,0:4])**2,axis=1))**0.5
    return np.argsort(dist)[0:num]

def predict2(pnt1, matched_arr, width1, height1, width2, height2):
    vertices = np.array([[0,0,0,0],\
                [width1,0,width2,0],\
                [0,height1,0,height2],\
                [width1,height1,width2,height2]])
    matched = matched_arr

    nn_ls = []

    matched_1 = matched[:,0:2]
    field1 = np.where(np.logical_and(matched_1[:,0]<=pnt1[0], matched_1[:,1]<=pnt1[1]))[0]
    if len(field1)>0:
        nn1 = field1[find_nearest(pnt1, matched_1[field1], 1)]
        nn_ls = np.concatenate((nn_ls,nn1))

    field2 = np.where(np.logical_and(matched_1[:,0]>=pnt1[0], matched_1[:,1]<=pnt1[1]))[0]
    if len(field2)>0:
        nn2 = field2[find_nearest(pnt1, matched_1[field2], 1)]
        nn_ls = np.concatenate((nn_ls,nn2))

    field3 = np.where(np.logical_and(matched_1[:,0]<=pnt1[0], matched_1[:,1]>=pnt1[1]))[0]
    if len(field3)>0:
        nn3 = field3[find_nearest(pnt1, matched_1[field3], 1)]
        nn_ls = np.concatenate((nn_ls,nn3))

    field4 = np.where(np.logical_and(matched_1[:,0]>=pnt1[0], matched_1[:,1]>=pnt1[1]))[0]
    if len(field4)>0:
        nn4 = field4[find_nearest(pnt1, matched_1[field4], 1)]
        nn_ls = np.concatenate((nn_ls,nn4))

    matched_nearest = matched[nn_ls.astype(int)]
    weights = 1.0/((np.sum((matched_nearest[:,0:2] - pnt1)**2, axis=1))**0.5)

    transfer = np.average(matched_nearest[:,2:4] - matched_nearest[:,0:2], axis=0, weights = weights)
    mscore = np.average(matched_nearest[:,4], weights = weights)

    pnt2 = pnt1 + transfer

    return pnt2,mscore

def predict(pnt1, matched_arr, width1, height1, width2, height2):
    vertices = np.array([[0,0,0,0],\
                [width1,0,width2,0],\
                [0,height1,0,height2],\
                [width1,height1,width2,height2]])
    matched = matched_arr

    nn_ls = []

    matched_1 = matched[:,0:2]
    matched_1_dist = np.sum((matched_1-pnt1.reshape(1,-1))**2,axis=1)
    nn_ls = np.argsort(matched_1_dist)[0:4]

    matched_nearest = matched[nn_ls.astype(int)]
    weights = 1.0/((np.sum((matched_nearest[:,0:2] - pnt1)**2, axis=1))**0.5)

    transfer = np.average(matched_nearest[:,2:4] - matched_nearest[:,0:2], axis=0, weights = weights)
    mscore = np.average(matched_nearest[:,4], weights = weights)

    pnt2 = pnt1 + transfer

    return pnt2,mscore

def predict_inter(pnt1, matched_arr, width1, height1, width2, height2):
    # pnt1 = [x1, y1]
    vertices = np.array([[0,0,0,0],\
                [width1,0,width2,0],\
                [0,height1,0,height2],\
                [width1,height1,width2,height2]])

    matched = matched_arr

    nn_ls = []

    matched_1 = matched[:,0:2]
    matched_1_dist = np.sum((matched_1-pnt1.reshape(1,-1))**2,axis=1)
    nn_ls = np.argsort(matched_1_dist)[0:10]

    matched_nearest = matched[nn_ls.astype(int)]
    sidx = np.argsort(matched_nearest[:,0])
    try:
        fx = interp1d(matched_nearest[sidx,0], matched_nearest[sidx,2], kind='linear', fill_value="extrapolate")
    except:
        print(matched_nearest[sidx,0], matched_nearest[sidx,2])
    sidx = np.argsort(matched_nearest[:,1])
    fy = interp1d(matched_nearest[sidx,1], matched_nearest[sidx,3], kind='linear', fill_value="extrapolate")
    pnt2 = np.array([fx(pnt1[0]), fy(pnt1[1])])

    weights = 1.0/((np.sum((matched_nearest[:,0:2] - pnt1)**2, axis=1))**0.5)
    mscore = np.average(matched_nearest[:,4], weights = weights)

    return pnt2,mscore

def draw_img(img1,img2,pnt_img1,pnt_img2,savefile,edge=0):
    plt.close()
    fig,ax = plt.subplots(1,2,figsize=(20,5))
    ax[0].imshow(img1[:,:,::-1])
    ax[1].imshow(img2[:,:,::-1])

    for pp in pnt_img1:
        circ = patches.Circle(pp,3)
        ax[0].add_patch(circ)
        if edge > 0:
            edge_half = int(edge/2)
            rect = patches.Rectangle((pp[0]-edge_half,pp[1]-edge_half),edge,edge,linewidth=1,edgecolor='r',facecolor='none')
            ax[0].add_patch(rect)

    for pp in pnt_img2:
        circ = patches.Circle(pp,3)
        ax[1].add_patch(circ)
        if edge > 0:
            edge_half = int(edge/2)
            rect = patches.Rectangle((pp[0]-edge_half,pp[1]-edge_half),edge,edge,linewidth=1,edgecolor='r',facecolor='none')
            ax[1].add_patch(rect)

    plt.savefig(savefile)

root_dir = '/home/yutong/SPMatch/vp_test_sedan/'
dir_ls = glob.glob(os.path.join(root_dir,'a*e*'))
total_spnum=39
draw_p=True
for img_dir in dir_ls:
    if not 'a20e0' in img_dir:
        continue

    draw=False

    print(img_dir)
    # file list
    filelist = os.path.join(img_dir, 'file_list.txt')
    with open(filelist, 'r') as fh:
        contents = fh.readlines()

    img_list = [cc.strip() for cc in contents]
    syn_n = 3

    for img_idx2 in range(len(img_list)-syn_n, len(img_list)):
        img_file2 = img_list[img_idx2]
        img2 = cv2.imread(img_file2)
        rratio = 224.0/np.min(img2.shape[0:2])
        img2 = cv2.resize(img2, (0,0), fx=rratio, fy=rratio)

        img2_name = img_file2.split('/')[-1].split('.')[0]

        info_file = os.path.join(img_dir, 'sp_info_{}.pickle'.format(img2_name))
        sp_info = pickle.load(open(info_file, 'rb'))

        img2_pool4_file = '.'.join(img_file2.split('.')[0:-1])+'_pool4.pickle'
        img2_pool4 = pickle.load(open(img2_pool4_file, 'rb'))
        sp_features = []
        for isp in range(total_spnum):
            sp_features.append([])

            isp_ls = [spp[1] for spp in sp_info if spp[0]==isp]
            if len(isp_ls)==0:
                continue

            for sppos in isp_ls:
                pnt2 = np.array([min(img2.shape[1],max(0,sppos[0])),\
                                 min(img2.shape[0],max(0,sppos[1]))])
                pnt2_pool4 = (pnt2//16).astype(int)
                sp_features[-1].append(img2_pool4[min(img2_pool4.shape[0]-1,pnt2_pool4[1]), \
                                                  min(img2_pool4.shape[1]-1,pnt2_pool4[0])])

        for img_idx1 in range(len(img_list)-syn_n):
            img_file1 = img_list[img_idx1]
            img_file_name = img_file1.split('/')[-1]
            img_file_name = img_file_name.split('.')[0]

            img1_pool4_file = '.'.join(img_file1.split('.')[0:-1])+'_pool4.pickle'
            img1_pool4 = pickle.load(open(img1_pool4_file, 'rb'))

            # read in images
            img1 = cv2.imread(img_file1)
            matched_file = os.path.join(img_dir, 'img{}VSimg{}_matched.pickle'.format(img_idx1, img_idx2))
            with open(matched_file, 'rb') as fh:
                matched = pickle.load(fh)

            matched_arr = np.array(matched[1])
            # print(matched_arr.shape)

            matched_arr_rv = np.zeros_like(matched_arr)
            matched_arr_rv[:,0:2] = matched_arr[:,2:4]
            matched_arr_rv[:,2:4] = matched_arr[:,0:2]
            matched_arr_rv[:,4] = matched_arr[:,4]

            save_dir = os.path.join(img_dir, 'img{}VSimg{}_transferSP_nearest'.format(img_idx1, img_idx2))
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)

            predict_sp_info = []
            for isp in range(total_spnum):
                isp_ls = [spp[1] for spp in sp_info if spp[0]==isp]

                if len(isp_ls)==0:
                    continue

                save_file = os.path.join(save_dir, 'transfer_sp{}.png'.format(isp))
                pnt_ls1 = []
                pnt_ls2 = []
                for spii,sppos in enumerate(isp_ls):
                    pnt2 = np.array([min(img2.shape[1],max(0,sppos[0])),\
                                     min(img2.shape[0],max(0,sppos[1]))])

                    # pnt1 = np.array([(sppos[0]+sppos[2])/2, (sppos[1]+sppos[3])/2])
                    pnt1,_ = predict(pnt2,matched_arr_rv,img2.shape[1],img2.shape[0],img1.shape[1],img1.shape[0] )
                    pnt1 = np.array([min(img1.shape[1],max(0,pnt1[0])),\
                                     min(img1.shape[0],max(0,pnt1[1]))])

                    pnt1_pool4 = (pnt1//16).astype(int)
                    pnt1_pool4_f = img1_pool4[min(img1_pool4.shape[0]-1,pnt1_pool4[1]), \
                                              min(img1_pool4.shape[1]-1,pnt1_pool4[0])]
                    mscore = 1-cdist(pnt1_pool4_f.reshape(-1,512), np.array(sp_features[isp][spii:spii+1]), 'cosine')[0,0]

                    img1_pool4_h, img1_pool4_w = img1_pool4.shape[0:2]
                    img1_pool4_dist = cdist(img1_pool4.reshape(-1,512), np.array(sp_features[isp][spii:spii+1]), 'cosine')

                    img1_pool4_dist = img1_pool4_dist.reshape(img1_pool4_h*img1_pool4_w,)

                    to_select = int(img1_pool4_h*img1_pool4_w*0.03) # top 3% close features
                    img1_pool4_cans = np.argsort(img1_pool4_dist)[0:to_select]

                    img1_cans = []
                    for p4c in img1_pool4_cans:
                        prow,pcol = np.unravel_index(p4c, (img1_pool4_h, img1_pool4_w))
                        img1_cans.append(np.array([pcol,prow])*16-42+50)

                    img1_nearest_idx = np.argmin(cdist(np.array(img1_cans), pnt1.reshape(1,2)).reshape(-1,))
                    img1_nearest = img1_cans[img1_nearest_idx]
                    img1_nearest_score = 1-img1_pool4_dist[img1_pool4_cans[img1_nearest_idx]]

                    pnt_ls1.append(pnt1)
                    pnt_ls2.append(pnt2)
                    predict_sp_info.append([img_file_name,isp,pnt1,mscore])

                if draw:
                    draw_img(img1,img2,pnt_ls1,pnt_ls2,save_file,100)

            sp_file = os.path.join(img_dir, 'img{}VSimg{}_transSP.pickle'.format(img_idx1, img_idx2))
            pickle.dump(predict_sp_info, open(sp_file, 'wb'))


import numpy as np
import gensim
import os
import sys
from scipy import sparse
import pickle
import json
from gensim.parsing.preprocessing import STOPWORDS
import logging
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)
logging.root.level = logging.INFO

if len(sys.argv) != 2:
    print("Usage:\npython process_tmn.py <input_data_file>")
    exit(0)
data_file = sys.argv[1]
data_dir = os.path.dirname(data_file)
with open(os.path.join(data_file), 'U') as fin:
    text = gensim.utils.to_unicode(fin.read(), 'latin1').strip()

news_lst = text.split("\n")
print(news_lst[:5])
msgs = []
labels = []
label_dict = {}

for n_i, line in enumerate(news_lst):
    msg, label = line.strip().split("######")
    msg = list(gensim.utils.tokenize(msg, lower=True))
    msgs.append(msg)
    if label not in label_dict:
        label_dict[label] = len(label_dict)
    labels.append(label_dict[label])

# build dictionary
dictionary = gensim.corpora.Dictionary(msgs)

import copy
bow_dictionary = copy.deepcopy(dictionary)
bow_dictionary.filter_tokens(list(map(bow_dictionary.token2id.get, STOPWORDS)))
len_1_words = list(filter(lambda w: len(w) == 1, bow_dictionary.values()))
bow_dictionary.filter_tokens(list(map(bow_dictionary.token2id.get, len_1_words)))
bow_dictionary.filter_extremes(no_below=3, keep_n=None)
bow_dictionary.compactify()

def get_wids(text_doc, seq_dictionary, bow_dictionary, ori_labels):
    seq_doc = []
    # build bow
    row = []
    col = []
    value = []
    row_id = 0
    m_labels = []

    for d_i, doc in enumerate(text_doc):
        if len(bow_dictionary.doc2bow(doc)) < 3:    # filter too short
            continue
        for i, j in bow_dictionary.doc2bow(doc):
            row.append(row_id)
            col.append(i)
            value.append(j)
        row_id += 1

        wids = list(map(seq_dictionary.token2id.get, doc))
        wids = np.array(list(filter(lambda x: x is not None, wids))) + 1
        m_labels.append(ori_labels[d_i])
        seq_doc.append(wids)
    lens = list(map(len, seq_doc))
    bow_doc = sparse.coo_matrix((value, (row, col)), shape=(row_id, len(bow_dictionary)))
    logging.info("get %d docs, avg len: %d, max len: %d" % (len(seq_doc), np.mean(lens), np.max(lens)))
    return seq_doc, bow_doc, m_labels

seq_title, bow_title, label_title = get_wids(msgs, dictionary, bow_dictionary, labels)

# split data
indices = np.arange(len(seq_title))
np.random.shuffle(indices)
nb_test_samples = int(0.2 * len(seq_title))
seq_title = np.array(seq_title)[indices]
seq_title_train = seq_title[:-nb_test_samples]
seq_title_test = seq_title[-nb_test_samples:]
bow_title = bow_title.tocsr()
bow_title = bow_title[indices]
bow_title_train = bow_title[:-nb_test_samples]
bow_title_test = bow_title[-nb_test_samples:]
label_title = np.array(label_title)[indices]
label_title_train = label_title[:-nb_test_samples]
label_title_test = label_title[-nb_test_samples:]

# save
logging.info("save data...")
pickle.dump(seq_title, open(os.path.join(data_dir, "dataMsg"), "wb"))
pickle.dump(seq_title_train, open(os.path.join(data_dir, "dataMsgTrain"), "wb"))
pickle.dump(seq_title_test, open(os.path.join(data_dir, "dataMsgTest"), "wb"))
pickle.dump(bow_title, open(os.path.join(data_dir, "dataMsgBow"), "wb"))
pickle.dump(bow_title_train, open(os.path.join(data_dir, "dataMsgBowTrain"), "wb"))
pickle.dump(bow_title_test, open(os.path.join(data_dir, "dataMsgBowTest"), "wb"))
pickle.dump(label_title, open(os.path.join(data_dir, "dataMsgLabel"), "wb"))
pickle.dump(label_title_train, open(os.path.join(data_dir, "dataMsgLabelTrain"), "wb"))
pickle.dump(label_title_test, open(os.path.join(data_dir, "dataMsgLabelTest"), "wb"))
dictionary.save(os.path.join(data_dir, "dataDictSeq"))
bow_dictionary.save(os.path.join(data_dir, "dataDictBow"))
json.dump(label_dict, open(os.path.join(data_dir, "labelDict.json"), "w"), indent=4)
logging.info("done!")


from typing import Callable, List, Optional
from functools import wraps
import logging

from .threshold_type import ThresholdType
from .error.failure import PromptTestSetupException
from .runner.runner import run_prompttest

TESTS_TO_RUN = []

def prompttest(
    metric_name: str,
    eval_fn: Callable,
    prompts: List[str],
    threshold: float = 1.0,
    threshold_type: ThresholdType = ThresholdType.MINIMUM,
    expected: Optional[List[str]] = None,
):
    r"""
    Creates a decorator for prompt tests, which can annotate evaluation functions.
    This enables developers to create a prompt test suite from their evaluations.
    """

    def prompttest_decorator(completion_fn: Callable):
        @wraps(completion_fn)
        def runs_test():
            results = [completion_fn(prompt) for prompt in prompts]
            return run_prompttest(
                metric_name,
                eval_fn,
                threshold,
                threshold_type,
                prompts,
                results,
                expected=expected,
            )

        TESTS_TO_RUN.append(runs_test)
        return runs_test

    return prompttest_decorator

def main():
    logging.getLogger().setLevel(logging.WARNING)
    print("Running " + str(len(TESTS_TO_RUN)) + " test(s)")
    failures = int(sum([test() for test in TESTS_TO_RUN]))
    if failures == 0:
        print("All " + str(len(TESTS_TO_RUN)) + " test(s) passed!")
        exit(0)
    else:
        print("Tests failed: " + str(failures))
        exit(1)


import pygame
import random
import math

minMovement = 0.5
maxSpeed = 5

class Person:
    colors = {"healthy": "white", "sick": "red", "recovered": "blue"}

    # status - "healthy" , "sick" or "recovered"
    def __init__(self, x, y, status, socialDistancing):
        self.x = x
        self.y = y
        self.status = status
        self.socialDistancing = socialDistancing
        self.radius = 6
        self.vx = 0
        self.vy = 0
        self.turnsSick = 0
        self.recoveryTime = random.randint(100, 150)

        if not self.socialDistancing:
            while -minMovement < self.vx < minMovement and -minMovement < self.vy < minMovement:
                self.vx = random.uniform(-maxSpeed, maxSpeed)
                self.vy = random.uniform(-maxSpeed, maxSpeed)

    # screen - the main surface
    def draw(self, screen):
        pygame.draw.circle(screen, pygame.Color(self.colors[self.status]), (round(self.x), round(self.y)), self.radius)

    # execute once per frame
    def update(self, screen, people):
        self.move()
        if self.status == "sick":
            self.turnsSick += 1
            if self.turnsSick == self.recoveryTime:
                self.status = "recovered"
        self.checkCollidingWithWall(screen)
        for other in people:
            if self != other:  # ensure you don't check for collision with yourself
                if self.checkCollidingWithOther(other):
                    self.updateCollisionVelocities(other)
                    # update status
                    if self.status == "sick" and other.status == "healthy":
                        other.status = "sick"
                    elif other.status == "sick" and self.status == "healthy":
                        self.status = "sick"

    def move(self, ):
        if not self.socialDistancing:
            self.x += self.vx
            self.y += self.vy

    # check for collisions with walls and update velocities
    def checkCollidingWithWall(self, screen):
        if self.x + self.radius >= screen.get_width() and self.vx > 0:  # self.vx > 0 is to prevent it from getting
            # stuck to the wall
            self.vx *= -1
        elif self.x - self.radius <= 0 and self.vx < 0:
            self.vx *= -1
        if self.y + self.radius >= screen.get_height() and self.vy > 0:
            self.vy *= -1
        elif self.y - self.radius <= 0 and self.vy < 0:
            self.vy *= -1

    # return True if the self is colliding with the other, False otherwise
    def checkCollidingWithOther(self, other):
        distance = math.sqrt(math.pow(self.x - other.x, 2) + math.pow(self.y - other.y, 2))
        if distance <= self.radius + other.radius:
            return True
        return False

    # update velocities on collision
    def updateCollisionVelocities(self, other):
        # type 1 collision - both objects are moving; neither is social distancing

        if not self.socialDistancing and not other.socialDistancing:
            tempVX = self.vx
            tempVY = self.vy
            self.vx = other.vx
            self.vy = other.vy
            other.vx = tempVX
            other.vy = tempVY

        # type 2 collision : one object that is social distancing and one that is not

        elif other.socialDistancing:
            # works okay, not great
            # self.vx *= -1
            # self.vy *= -1
            #
            # # works okay
            # tempVX = self.vx
            # self.vx = self.vy
            # self.vy = tempVX

            magV = math.sqrt(math.pow(self.vx, 2) + math.pow(self.vy, 2))
            tempVector = (self.vx + (self.x - other.x), self.vy + (self.y - other.y))
            magTempVector = math.sqrt(math.pow(tempVector[0], 2) + math.pow(tempVector[1], 2))
            normTempVector = (tempVector[0] / magTempVector, tempVector[1] / magTempVector)
            self.vx = normTempVector[0] * magV
            self.vy = normTempVector[1] * magV


from typing import Any, AsyncGenerator, AsyncIterator, Generator, Iterator

import pytest

# Import of Annotated not available from typing until 3.9
from typing_extensions import Annotated

from flexdi._util import determine_return_type
from flexdi.errors import UntypedError

class SimpleClass:
    pass

def sync_func() -> int:
    return 1

async def async_func() -> int:
    return 1

def sync_iter() -> Iterator[int]:
    yield 1

async def async_iter() -> AsyncIterator[int]:
    yield 1

def sync_generator() -> Generator[int, Any, Any]:
    yield 1

async def async_generator() -> AsyncGenerator[int, Any]:
    yield 1

@pytest.mark.parametrize(
    "test_input,expected_output",
    [
        pytest.param(SimpleClass, SimpleClass, id="SimpleClass"),
        pytest.param(sync_func, int, id="SyncFunction"),
        pytest.param(async_func, int, id="AsyncFunction"),
        pytest.param(sync_iter, int, id="SyncIterator"),
        pytest.param(async_func, int, id="AsyncIterator"),
        pytest.param(sync_generator, int, id="SyncGenerator"),
        pytest.param(async_generator, int, id="AsyncGenerator"),
    ],
)
def test_determine_return_type(test_input: Any, expected_output: Any) -> None:
    actual_output = determine_return_type(test_input)
    assert actual_output is expected_output

def invalid_typed_func():  # type: ignore
    return 1

@pytest.mark.parametrize(
    "test_input",
    [
        pytest.param(Annotated[int, "i"], id="NonInstantiableClass"),
        pytest.param(invalid_typed_func, id="MissingReturnStatement"),
    ],
)
def test_determine_invalid_return_type(test_input: Any) -> None:
    with pytest.raises(UntypedError):
        determine_return_type(invalid_typed_func)


# -*- encoding: utf8 -*-
import requests
import time
import sys
import re as regx
import MetaInfos as meta
from pygame import mixer
from bs4 import BeautifulSoup as bs

######################################################################
#########################    SETTINGS    #############################
######################################################################

check_time_term = 2        #3초에 한번 확인
id = "id"           #아이디 입력
pw = "pw"            #패스워드 입력
reserve_date = '20190113'   #예약 날짜 입력"
time_min = '1950'           #예약 희망 시간 최저
time_max = '2053'           #예약 희망 시간 최대
depart_station = '동대구'
arrive_station = '수서'

########################    STAION LIST    ###########################
'공주', '광주송정', '김천구미', '나주', '대전', '동대구', '동탄', '목포', '부산', '수서', '신경주', '오송', '울산', '익산', '정읍', '지제', '천안아산'
######################################################################

######################################################################
######################    REQUEST META INFO     ######################
######################################################################

######################################################################
##########################    LOGICS    ##############################
######################################################################

def login(id, pw):
    header = dict(meta.common_header)
    header["Referer"] = meta.login_referer
    header['Content-Type'] = 'application/x-www-form-urlencoded'

    # 더 확실한 키워드를 찾아보자.
    login_success_keyword = 'location.replace(\'/main.do\')'

    response = requests.post(meta.login_request_url,
                             headers=header,
                             params=login_param(id, pw))

    if response.cookies and response.cookies['JSESSIONID_ETK']:
        print ("get cookie from login response:"+response.cookies['JSESSIONID_ETK'])
        meta.my_cookie["JSESSIONID_ETK"] = response.cookies['JSESSIONID_ETK']

    if response.status_code != 200:
        return False
    elif ('오류' in response.text) or ('실패' in response.text) or ('존재' in response.text):
        return False
    elif login_success_keyword in response.text:
        return True

def login_param(id, pw):
    return {
        'rsvTpCd': None,
        'goUrl': None,
        'from': None,
        'srchDvCd': '1',
        'srchDvNm': id,
        'hmpgPwdCphd': pw
    }

def reserve(reserv_targets):
    header = dict(meta.common_header)
    header['Cookie'] = "SR_MB_CD_NO="+str(id) +"; JSESSIONID_ETK="+meta.my_cookie['JSESSIONID_ETK']
    header['Referer'] = meta.check_seat_url

    for tr in reserv_targets:
        response = requests.post(meta.reservation_url,
                                 headers=header,
                                 params=reservation_param(tr))

        if not (response.status_code == 200 and "location.replace('/hpg/hra/02/requestReservationInfo.do?pageId=TK0101030000')" in response.text) :
            continue

        response = requests.get(meta.reservation_url2, headers = header)
        if not (response.status_code == 200 and "location.replace('confirmReservationInfo.do?pageId=TK0101030000')" in response.text):
            continue

        response = requests.get(meta.reservation_confirm_url, headers = header)
        if not (response.status_code == 200 and "10분 내에 결제하지 않으면 예약이 취소됩니다" in response.text):
            continue

        printPretty("예약 성공")
        reserve_id = bs(response.text,'html.parser').find('input', attrs={"name":"pnrNo"})
        print(reserve_id)

        return reserve_id
        #예약 성공하면 반환값 리턴하면서 종료
    return False

def reservation_param(tr):
    param = dict(meta.reserve_param)

    train_info_list = bs(str(tr), 'html.parser').select("td.trnNo > input")
    train_info_dict = { bs(str(info), 'html.parser').find()['name'].split('[')[0]: bs(str(info),'html.parser').find()['value'] for info in train_info_list }

    param['dptDt1'] = train_info_dict['dptDt']
    param['runDt1'] = train_info_dict['runDt']
    param['arvStnConsOrdr1'] = train_info_dict['arvStnConsOrdr']
    param['arvStnRunOrdr1'] = train_info_dict['arvStnRunOrdr']
    param['arvRsStnCd1'] = train_info_dict['arvRsStnCd']
    param['dirSeatAttCd1'] = '000'
    param['dptRsStnCd1'] = train_info_dict['dptRsStnCd']
    param['dptStnConsOrdr1'] = train_info_dict['dptStnConsOrdr']
    param['dptTm1'] = train_info_dict['dptTm']
    param['jrnySqno1'] = train_info_dict['jrnySqno']
    param['locSeatAttCd1'] = "000"
    param['reqTime'] = int(time.time()*1000) #현재시간
    param['rqSeatAttCd1'] = train_info_dict['seatAttCd']
    param['stlbTrnClsfCd1'] = train_info_dict['stlbTrnClsfCd']
    param['trnGpCd1'] = train_info_dict['trnGpCd']
    param['trnNo1'] = train_info_dict['trnNo']
    param['trnOrdrNo1'] = train_info_dict['trnOrdrNo'] #화면에서 몇번째 라인에 있던 열차인지

    return param

#빈 좌석이 있는지 확인
def find_empty_seats(start, dest, date, time_min ='000000', time_max ='220000'):
    header = dict(meta.common_header)
    header["Referer"] = "https://etk.srail.co.kr/main.do"
    header['Content-Type'] = 'application/x-www-form-urlencoded'

    print("좌석 정보를 조회합니다..")
    response = requests.post(meta.check_seat_url,
                             headers=header,
                             params=finding_seats_param(start, dest, date))

    #열차 정보
    trains = bs(response.text, 'html.parser').select("tbody > tr")

    #매진이라도 열차가 하나라도 없다면 종료되도록
    if not trains:
        printPretty("원하는 시간대에 배차가 없습니다. 입력하신 시간을 수정해주세요")
        shutdown()

    reserv_targets = []
    for tr in trains:
        depart_time = bs(str(tr), "html.parser").find('input', attrs={"name": regx.compile("dptTm*")})['value'] #regular expression
        if int(time_min.ljust(6,'0')) > int(depart_time) or int(depart_time) > int(time_max.ljust(6,'0')):
            print("depart_time:"+depart_time+"은 예약대상이 아닙니다..")
            continue
        td_list = bs(str(tr), "html.parser").select('td')
        if "매진" not in str(td_list[6]):
            print("예약가능: {}, {}".format(td_list[3], td_list[4]))
            reserv_targets.append(tr)
    return reserv_targets

def finding_seats_param(start, dest, date):
    return {
        'chtnDvCd': '1',
        'isRequest': 'Y',
        'psgInfoPerPrnb1': '1',
        'psgInfoPerPrnb2': '0',
        'psgInfoPerPrnb3': '0',
        'psgInfoPerPrnb4': '0',
        'psgInfoPerPrnb5': '0',
        'psgNum': '1',
        'seatAttCd': '015',
        'stlbTrnClsfCd': '05',
        'trnGpCd': '300',  # 300으로 하면 srt만 나옴, 109는 상관없이 다.
        'arvRsStnCdNm': dest,
        'arvRsStnCd': meta.station_meta_info[dest],
        'dptRsStnCdNm': start,
        'dptRsStnCd': meta.station_meta_info[start],
        'dptDt': date,
        'dptTm': time_min + '0000'
    }

def validate_setting_info():
    stations = meta.station_meta_info.keys()
    if (depart_station not in stations) or (arrive_station not in stations):
        printPretty("역 정보가 올바르지 않습니다. 다시 입력하세요")
        sys.exit(1)
    if int(time_min) > int(time_max):
        printPretty("예약 희망 시간대가 비정상적입니다. 다시 입력하세요")
        sys.exit(1)

def printPretty(msg):
    print("************ [ {} ] ************".format(msg))

def shutdown():
    sys.exit(1)

def announce_success():
    mixer.init()
    mixer.music.load('gunshot.mp3')
    # wait for load
    time.sleep(1)
    mixer.music.play()
    while mixer.music.get_busy():
        time.sleep(1)
    print("done")

######################################################################
#########################    MAIN LOGIC    ###########################
######################################################################

validate_setting_info()

if not login(id, pw):
    printPretty("login fail")
    shutdown()

printPretty("login success")

isRight = input("출발역 = %s, 도착역 = %s, 예약하고자 하는 날짜 = %s, 희망시간대는 %s ~ %s 맞나요?(y/n):"%(depart_station, arrive_station, reserve_date, time_min, time_max))
if (isRight.lower() != 'y') and (isRight.upper() != 'Y'):
    shutdown()

while True:
    try:
        reserv_targets = find_empty_seats(depart_station, arrive_station, reserve_date, time_min, time_max)
        if len(reserv_targets) > 0 and reserve(reserv_targets):
            break
    except Exception as error:
        print(error)
        if not login(id, pw):
            printPretty("login fail")
            shutdown()
    time.sleep(check_time_term)

printPretty("end")
announce_success()


from app.models import db, Review, environment, SCHEMA
from sqlalchemy.sql import text

def seed_reviews():
    burnout_review_1 = Review(
        recomended=False,
        game_id=1,
        user_id=3,
        review_text='Unrealistic premise. People can work more than 60 hours a week without issue. Refunded.'
    )
    africanus_review_1 = Review(
        recomended=True,
        game_id=2,
        user_id=2,
        review_text='The gameplay is solid, if not spectacular, but the setting itself really sets it apart from the other grand strategy games out there.'
    )
    spacefield_review_1 = Review(
        recomended=True,
        game_id=3,
        user_id=1,
        review_text="It wasn't the industry-defining standard we were hoping for, but it was certainly fun."
    )
    db.session.add(burnout_review_1)
    db.session.add(africanus_review_1)
    db.session.add(spacefield_review_1)
    db.session.commit()

def undo_reviews():
    if environment == "production":
        db.session.execute(f"TRUNCATE table {SCHEMA}.reviews RESTART IDENTITY CASCADE;")
    else:
        db.session.execute(text("DELETE FROM reviews"))

    db.session.commit()


from datetime import datetime
from airflow.models import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.python_operator import PythonOperator

default_args = {
    'owner': 'homer_j_simpsom',
    'start_date': datetime(2020, 5, 20, 11, 0, 0)
}

def write_title():
    print("El uso de Airflow en la universidad de Springfield.")

def write_intro():
    print("El otro día mi hija me dijo que Airflow no se utilizaba en la universidad de Springfield, y yo le dije: qué no Lisa? qué no?")

def write_body():
    for i in range(50):
        print("Púdrete Flanders")

with DAG('dag_homer_essay',
         default_args=default_args,
         schedule_interval='@daily') as dag:

    start = DummyOperator(task_id='start')
    wr_title = PythonOperator(task_id='wr_title',python_callable=write_title)
    wr_intro = PythonOperator(task_id='wr_intro',python_callable=write_intro)
    wr_body = PythonOperator(task_id='wr_body',python_callable=write_body)
    end = DummyOperator(task_id='end')

start >> wr_title >> wr_intro >> wr_body >> end


import datetime
import logging

from django.core.management import BaseCommand, CommandError
from django.db.models import Q

from stats.models import Challenge, GlobalStats

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    help = "Calcule les statistiques globales affichées sur la page Statistiques et les stats des challenge en cours"

    def handle(self, *args, **options):
        try:
            GlobalStats.objects.get().refresh_stats()
            logger.info("STATS updated succesfully")
        except KeyboardInterrupt:
            raise CommandError("Interrompu.")

        # Compute challenges stats
        today = datetime.datetime.today()

        for challenge in Challenge.objects.filter(
            Q(
                start_date__gt=today,
            )
            | Q(start_date__lte=today, end_date__gt=today)
        ):
            try:
                challenge.refresh_stats()
                logger.info(f"STATS for challenge {challenge} updated")
            except KeyboardInterrupt:
                raise CommandError("Interrompu.")


from __future__ import print_function

import argparse
import random
import numpy as np
import time
import logging
import warnings
from tqdm import tqdm
import torch
from torch import nn
from scipy.special import softmax
from torch.optim.lr_scheduler import StepLR

import data_utils
from models import KHANModel
import gc
gc.collect()
torch.cuda.empty_cache()

warnings.simplefilter("ignore", UserWarning)
warnings.simplefilter("ignore", FutureWarning)

# for Reproducibility
def set_random_seeds(random_seed=0):
    torch.manual_seed(random_seed)
    torch.cuda.manual_seed(random_seed)
    torch.cuda.manual_seed_all(random_seed) # if use multi-GPU
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    np.random.seed(random_seed)
    random.seed(random_seed)
    #  pass

parser = argparse.ArgumentParser(description='Parsing parameters')
parser.add_argument("--gpu_index", type=int, help="GPU index. Necessary for specifying the CUDA device.")
# models & datasets
parser.add_argument('--model', type=str, default='KHAN', help='Name of Model.')
parser.add_argument("--embed_size", type=int, default=128, help="Word/Sentence Embedding size.")
parser.add_argument('--max_sentence', type=int, default=40, help='Maximum Number of Sentences in each document.')
parser.add_argument('--num_layer', type=int, default=1, help='Number of Transformer Encoder Layers.')
parser.add_argument('--num_head', type=int, default=4, help='Number of Multihead Attentions.')
parser.add_argument('--d_hid', type=int, default=256, help='Dimension of a hidden layer.')
parser.add_argument('--dropout', type=float, default=0.3, help='Dropout probability.')
parser.add_argument('--alpha', type=float, default=0.5, help='Weight of common knowledge.')
parser.add_argument('--beta', type=float, default=0.5, help='Weight of political knowledge.')

parser.add_argument('--dataset', type=str, default='SEMEVAL', help='Name of dataset (SEMEVAL, ALLSIDES).')
parser.add_argument('--data_path', type=str, default='./data', help='Data path.')
parser.add_argument('--save_model', action='store_false', default=False, help='For Saving the current Model')
parser.add_argument('--model_dir', type=str, default='../trained_models', help='Path for saving the trained model')

# user-defined hyperparameters
parser.add_argument("--num_epochs", type=int, default=50, help="Number of training epochs.")
parser.add_argument("--batch_size", type=int, default=16, help="Training batch size.")
parser.add_argument("--eval_batch_size", type=int, default=16, help="Evaluation and test batch size.")
parser.add_argument("--learning_rate", type=float, default=0.001, help="Learning rate.")
parser.add_argument('--seed', type=int, default=1, metavar='S', help='Random seed (default: 1)') # for reproducibility

args = parser.parse_args()

# Get the dataloaders and model
device = torch.device("cuda:{}".format(args.gpu_index))
print('Count of using GPUs:', torch.cuda.device_count())

def evaluate(model, device, dataloader) -> float:
    model.eval() # turn on eval mode
    total_acc = 0
    total_count = 0
    with torch.no_grad():
        for idx, (labels, titles, sentences) in enumerate(dataloader):
            predicted_label = model(sentences, titles)
            total_acc += (predicted_label.argmax(1) == labels).sum().item()
            total_count += labels.size(0)
    return total_acc/total_count

def train_each_fold(train_iter, test_iter, vocab, num_class, knowledge_list, fold_idx, k_folds, sampler):

    train_iter, test_iter, vocab, num_class, knowledge_list, fold_idx, k_folds, sampler = train_iter, test_iter, vocab, num_class, knowledge_list, fold_idx, k_folds, sampler

    train_data, val_data, test_data = data_utils.get_dataloaders(train_iter, test_iter, vocab, args.batch_size, args.max_sentence, sampler, device)

    nhead = args.num_head
    d_hid = args.d_hid
    dropout = args.dropout
    nlayers = args.num_layer
    alpha = args.alpha
    beta = args.beta

    model = KHANModel(len(vocab), args.embed_size, nhead, d_hid, nlayers, dropout, num_class, knowledge_list, alpha, beta)
    params = list(model.parameters())
    print(len(params))
    print(params[0].size())

    model = model.to(device) # model to GPU
    total = sum([param.nelement() for param in model.parameters()])
    print("Total params: %.2fM" % (total/1e6))

    criterion = nn.CrossEntropyLoss() # loss function
    optimizer = torch.optim.AdamW(model.parameters(), lr=args.learning_rate, weight_decay=0.05) # optimizer
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, factor=0.5, verbose=1) # learning rate scheduling

    start_time = time.time()
    total_start_time = start_time
    max_accuracy = 0
    num_batches = 0

    for epoch in range(args.num_epochs):
        epoch_start_time = time.time()
        model.train()  # turn on train mode
        train_correct, train_count = 0, 0
        start_time = time.time()
        total_loss = 0.

        # ------------------------ Epoch Start ------------------------ #
        for idx, (labels, titles, sentences) in enumerate(train_data):
            optimizer.zero_grad()
            predicted_labels = model(sentences, titles)
            loss = criterion(predicted_labels, labels)
            loss.backward()

            # Gradient Clipping
            max_grad_norm = 3
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)
            optimizer.step()

            train_correct += (predicted_labels.argmax(1) == labels).sum().item()
            train_count += labels.size(0)
            total_loss += loss.item()

        epoch_loss = total_loss / len(train_data)
        scheduler.step(epoch_loss)
        epoch_time = time.time() - epoch_start_time
        # ------------------------ Epoch End ------------------------ #

        # --------------------------------- EVALUATION --------------------------------- #
        # ----- at the end of every epoch, evaluating the current model a ccuracy ------ #
        # ------------------------------------------------------------------------------ #
        train_accuracy = train_correct / train_count
        val_accuracy = evaluate(model, device, test_data)

        print('Fold: {:3d} | Epoch: {:3d} | Loss: {:6.4f} | TrainAcc: {:6.4f} | ValAcc: {:6.4f} | Time: {:5.2f}'.format(
            fold_idx,
            (epoch+1),
            total_loss/ len(train_data),
            train_accuracy,
            val_accuracy,
            epoch_time)
            )

        # ------------------------- Save Best Model ------------------------- #
        if val_accuracy > max_accuracy:
            max_accuracy = val_accuracy

        if args.save_model:
            torch.save(model.state_dict(), args.model_dir)

    # ----------------------------------------------------------------------------#
    # ----------------------------- End of Training ------------------------------#
    # ----------------------------------------------------------------------------#

    fold_train_time = time.time() - total_start_time

    print('')
    print('================================== FOLD - {:3d} =================================='.format(fold_idx))
    print('============= Test Accuracy: {:.4f}, Training time: {:.2f} (sec.) ================'.format(max_accuracy, fold_train_time))
    print('================================================================================')

    return max_accuracy, fold_train_time

def main():

    # ----------------------------------------------------------------------------#
    # ---------------------------- Main Training Loop ----------------------------#
    # ----------------------------------------------------------------------------#
    print('')
    print('==================================== Training Start ====================================')

    data_utils.train_datasets(args.dataset, args.data_path)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)

    # Set random seeds for reproducibility
    set_random_seeds(args.seed)

    # Summary of training information
    print('====================================TRAIN INFO START====================================')
    print('  - TRAINING MODEL = %s' % (args.model))
    print('     - Embedding Size = %s' % (args.embed_size))
    print('     - Maximum Length = %s' % (args.max_sentence))
    print('     - Number of Transformer Encoder Layers = %s' % (args.num_layer))
    print('     - Number of Multi-head Attentions = %s' % (args.num_head))
    print('     - Hidden Layer Dimension = %s' % (args.d_hid))
    print('     - Dropout Probability = %s' % (args.dropout))
    print('     - Alpha = %s' % (args.alpha))
    print('     - Beta = %s' % (args.beta))
    print('  - DATASET = %s' % (args.dataset))
    print('  - BATCH SIZE = ' + str(args.batch_size))
    print('  - NUM EPOCHS = ' + str(args.num_epochs))
    print('  - LEARNING RATE = ' + str(args.learning_rate))
    print('   ')
    # add more if needed

    main()


from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect

from .models import Question, Choice
from django.utils import timezone

# Create your views here.
def get_question(question_id):
    return Question.objects.get(id = question_id)

def index(request):
    latest_question_list = Question.objects.order_by('-pub_date')
    data = {'latest_question_list': latest_question_list}

    return render(request, 'polls_app/index.html', data)

def detail(request, question_id):
    question = get_question(question_id)

    # data = {'question': question}
    return  render(request, 'polls_app/detail.html', {'question': question})

def results(request, question_id):
    question = get_question(question_id)

    return render(request, 'polls_app/results.html', {'question': question})

def vote(request, question_id):
    question = get_question(question_id)

    try:
        selected_choice = question.choice_set.get(pk=request.POST['choice'])

    except (KeyError, Choice.DoesNotExist):
        data = {'question': question,
                'error_message': "You didn't select a choice.",}
        return render(request, 'polls_app/detail.html', data)

    selected_choice.votes += 1
    selected_choice.save()

    return HttpResponseRedirect(f'/polls/{question_id}/results')

# for demo
def add_question(request, question_text):
    question = Question(question_text = question_text, pub_date= timezone.now())
    question.save()

    question.choice_set.create(choice_text='Not much')
    question.choice_set.create(choice_text='The sky', votes= 0)
    question.choice_set.create(choice_text='nothing at all')

    choices = question.choice_set.all()
    return HttpResponse(choices)


import argparse
import os
import sys

def convert(infile, outfile, num_avg, num_stdev):
    with open(infile, 'r') as in_table, open(outfile, 'w') as out_table:
        first = in_table.readline().strip("\n").split("\t")[3:]
        total = len(first)+3
        out_table.write("scaffold")
        for item in first:
            if item[-4:] == '-var':
                out_table.write("\tstdev_" + str(item.rsplit("-",1)[0]))
            else:
                out_table.write("\tavg_" + str(item))
        for line in in_table:
            line = line.strip("\n").split("\t")
            out_table.write("\n" + line[0])
            for n in range(3,total,2):
                #out_table.write("\t" + line[n])
                out_table.write("\t" + str(float(line[n])*float(num_avg)))
                sd = float(line[n+1])**0.5
                sd = sd*float(num_stdev)
                out_table.write("\t" + str(sd))

if __name__ == '__main__':

    vRhyme = argparse.ArgumentParser(description="Convert a MetaBat2 'jgi_summarize_bam_contig_depths' generated coverage table into vRhyme format for -c input.")
    vRhyme.add_argument('-i', metavar='', type=str, nargs=1, required=True, help="input coverage table in MetaBat2 format")
    vRhyme.add_argument('-o', metavar='', type=str, nargs=1, required=True, help='output coverage table in vRhyme format')
    vRhyme.add_argument('-multiplyAvg', metavar='', type=str, nargs=1, required=True, help='-multiplyAvg = num, multiply avg MetaBat for num')
    vRhyme.add_argument('-multiplyStdev', metavar='', type=str, nargs=1, required=True, help='-multiplyStdev = num, multiply stdev MetaBat for num')
    #
    args = vRhyme.parse_args()
    #
    #
    if os.path.exists(str(args.o[0])):
        sys.stderr.write("\nError: output table (-o) already exists. Exiting.\n\n")
        exit()

    convert(args.i[0], args.o[0], args.multiplyAvg[0], args.multiplyStdev[0])

#
#
#


from . import tasks
from datetime import timedelta

class RetryTask(tasks.Task):
    '''
    Retry a task up to `tries` times until it succeeds. If it fails `tries`
    times, run the task returned by `retry_failed`. Wait `interval` time
    between tries, with `backoff` additional delay each time.
    '''

    required_attrs = tasks.Task.required_attrs + ['tries', 'interval', 'backoff', 'retry_failed']

    backoff = timedelta(0)

    def __init__(self, tries=None, next_delay=None, **kwargs):
        if tries is not None:
            self.tries = tries
        if next_delay is None:
            next_delay = self.interval
        self.next_delay = next_delay
        super(RetryTask, self).__init__(**kwargs)

    def failed(self):
        if self.tries <= 1:
            return self.retry_failed()

        return self.copy(tries=self.tries - 1, delay=self.next_delay, next_delay=self.next_delay + self.backoff)


'''
I wrote this to make tasks for the tasks.org app with an etebase sync server.

It's not feature comprehensive but it's helpful for bulk task making.

The CLI is not great, it's easier to do python -i pytasks.py and code up
whatever you want.

https://github.com/tasks/tasks
https://github.com/etesync
'''
import argparse
import datetime
import etebase
import sys
import tasks_credentials
import time

from voussoirkit import betterhelp
from voussoirkit import passwordy
from voussoirkit import pipeable
from voussoirkit import timetools
from voussoirkit import vlogging

log = vlogging.getLogger(__name__, '')

PRODID = 'voussoir/pytasks'

client = etebase.Client(tasks_credentials.CLIENT_NAME, tasks_credentials.SERVER)
account = etebase.Account.login(client, tasks_credentials.USERNAME, tasks_credentials.PASSWORD)
collection_manager = account.get_collection_manager()
collections = collection_manager.list('etebase.vtodo', etebase.FetchOptions().limit(500000))
collections = [c for c in collections.data if not c.deleted]

todo = collections[0]
item_manager = collection_manager.get_item_manager(todo)

def make_vcal(text, *, categories=[], due=None):
    now = timetools.now().strftime('%Y%m%dT%H%M%SZ')

    if due is None:
        due = ''
    elif isinstance(due, datetime.date):
        due = 'DUE;VALUE=DATE:' + due.strftime('%Y%m%d')
    elif isinstance(due, datetime.datetime):
        due = 'DUE:' + due.strftime('%Y%m%dT%H%M%S')
    else:
        raise TypeError(due)

    lines = [
        'BEGIN:VCALENDAR',
        'VERSION:2.0',
        'PRODID:' + PRODID,
        'BEGIN:VTODO',
        'DTSTAMP:' + now,
        'UID:' + passwordy.random_digits(32),
        due,
        'CREATED:' + now,
        'LAST-MODIFIED:' + now,
        'SUMMARY:' + text,
        ('CATEGORIES:' + ','.join(categories)) if len(categories) > 0 else '',
        'END:VTODO',
        'END:VCALENDAR',
    ]
    lines = [l for l in lines if l]
    vcal = '\r\n'.join(lines) + '\r\n'
    return vcal.encode('utf-8')

def make_item(*args, **kwargs):
    item = item_manager.create({}, make_vcal(*args, **kwargs))
    return item

def publish_items(items):
    if isinstance(items, etebase.Item):
        item_manager.batch([items])
    else:
        item_manager.batch(list(items))

def export_argparse(args):
    items = item_manager.list(etebase.FetchOptions().limit(50000000))
    items = list(items.data)
    for item in items:
        pipeable.stdout(item.content.decode('utf-8'))
        pipeable.stdout()
    return 0

def make_tasks_argparse(args):
    titles = pipeable.input_many(args.titles, strip=True, skip_blank=True)
    items = []
    for title in titles:
        log.info('Creating task %s.', title)
        item = make_item(title)
        items.append(item)
    publish_items(items)
    return 0

@vlogging.main_decorator
def main(argv):
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    ################################################################################################

    p_make_tasks = subparsers.add_parser('make_tasks', aliases=['make-tasks', 'maketasks', 'maketask'])
    p_make_tasks.add_argument('titles', nargs='+')
    p_make_tasks.set_defaults(func=make_tasks_argparse)

    ################################################################################################

    p_export = subparsers.add_parser('export')
    p_export.set_defaults(func=export_argparse)

    return betterhelp.go(parser, argv)

if __name__ == '__main__':
    raise SystemExit(main(sys.argv[1:]))


from xmlrpc.server import SimpleXMLRPCServer
from xmlrpc.server import SimpleXMLRPCRequestHandler
import xml.etree.ElementTree as ET
import requests

tree = ET.parse('db.xml')
root = tree.getroot()

print("Server started")

# Restrict to a particular path.
class RequestHandler(SimpleXMLRPCRequestHandler):
    rpc_paths = ('/RPC2',)

# Create server
with SimpleXMLRPCServer(('localhost', 8000), requestHandler=RequestHandler) as server:
    server.register_introspection_functions()

    def check_topic_exists(topic):
        """Checks if a topic exists in the database"""

        exists = False
        for child in root:
            if (child.attrib['name'] == topic):
                exists = True

        return exists

    server.register_function(check_topic_exists, "check_topic_exists")

    def list_topics():
        """Returns a list of all the existing topics"""

        topics = []
        for child in root:
            topics.append(child.attrib)
        return topics

    server.register_function(list_topics, "list_topics")

    def get_content_of_topic(topic):
        """Returns the content of a topic"""

        topic_content = []

        for child in root:
            if (child.attrib['name'] == topic):

                # Getting the content of the topic inside the note tag
                topic_content.append(child.find("note").find("text").text)
                topic_content.append(child.find("note").find("timestamp").text)

                return topic_content

    server.register_function(get_content_of_topic, "get_content_of_topic")

    def add_data_to_existing_topic(topic, text, timestamp):
        """If topic that user wants exists, appends text to the existing topic"""

        for child in root:
            # Finding the topic to which data needs to be added
            if (child.attrib['name'] == topic):

                note = child.find("note")

                # Finding the existing text and appending the new text
                existing_text = note.find("text").text
                new_text = f"{existing_text}\n{text}"
                note.find("text").text = new_text

                # Updating the timestamp
                note.find("timestamp").text = timestamp

                tree.write('db.xml')
                # Returning True if data was added successfully
                return True

        # Returning False if data was not added
        return False

    server.register_function(add_data_to_existing_topic, "add_data_to_existing_topic")

    def add_new_topic(topic, text, timestamp):
        """If topic that user wants does not exist, creates a new topic"""

        # Adding new topic to the root
        new_topic = ET.SubElement(root, 'topic', name=topic)
        new_note = ET.SubElement(new_topic, 'note')
        new_text = ET.SubElement(new_note, 'text')
        new_timestamp = ET.SubElement(new_note, 'timestamp')

        new_text.text = text
        new_timestamp.text = timestamp

        tree.write('db.xml')
        return True

    server.register_function(add_new_topic, "add_new_topic")

    def wikipedia_data_search(topic):
        """Function to search for a topic on Wikipedia"""

        S = requests.Session()

        URL = "https://en.wikipedia.org/w/api.php"

        SEARCHPAGE = topic

        PARAMS = {
            "action": "opensearch",
            "namespace": "0",
            "search": SEARCHPAGE,
            "limit": "1",
            "format": "json"
        }

        R = S.get(url=URL, params=PARAMS)
        DATA = R.json()

        print(DATA)

        return DATA

    server.register_function(wikipedia_data_search, "wikipedia_data_search")

    # Run the server's main loop
    server.serve_forever()


from mpi4py import MPI

import pytest

from dolfinx.fem import extract_function_spaces, form, functionspace
from dolfinx.mesh import create_unit_square
from ufl import TestFunction, TrialFunction, dx, inner

def test_extract_forms():
    """Test extraction on unique function spaces for rows and columns of
    a block system"""
    mesh = create_unit_square(MPI.COMM_WORLD, 32, 31)
    V0 = functionspace(mesh, ("Lagrange", 1))
    V1 = functionspace(mesh, ("Lagrange", 2))
    V2 = V0.clone()
    V3 = V1.clone()

    v0, u0 = TestFunction(V0), TrialFunction(V0)
    v1, u1 = TestFunction(V1), TrialFunction(V1)
    v2, u2 = TestFunction(V2), TrialFunction(V2)
    v3, u3 = TestFunction(V3), TrialFunction(V3)

    a = form([[inner(u0, v0) * dx, inner(u1, v1) * dx],
              [inner(u2, v2) * dx, inner(u3, v3) * dx]])
    with pytest.raises(AssertionError):
        extract_function_spaces(a, 0)
    with pytest.raises(AssertionError):
        extract_function_spaces(a, 1)

    a = form([[inner(u0, v0) * dx, inner(u2, v1) * dx],
              [inner(u0, v2) * dx, inner(u2, v2) * dx]])
    with pytest.raises(AssertionError):
        extract_function_spaces(a, 0)
    Vc = extract_function_spaces(a, 1)
    assert Vc[0] is V0._cpp_object
    assert Vc[1] is V2._cpp_object

    a = form([[inner(u0, v0) * dx, inner(u1, v0) * dx],
              [inner(u2, v1) * dx, inner(u3, v1) * dx]])
    Vr = extract_function_spaces(a, 0)
    assert Vr[0] is V0._cpp_object
    assert Vr[1] is V1._cpp_object
    with pytest.raises(AssertionError):
        extract_function_spaces(a, 1)


from typing import List, Optional, Sequence, Union

from draftjs_exporter.constants import BLOCK_TYPES
from draftjs_exporter.dom import DOM
from draftjs_exporter.options import Options, OptionsMap
from draftjs_exporter.types import Block, Element, Props, RenderableType

class Wrapper:
    """
    A wrapper is an element that wraps other nodes. It gets created
    when the depth of a block is different than 0, so the DOM elements
    have the appropriate amount of nesting.
    """

    __slots__ = ("depth", "last_child", "type", "props", "elt")

    def __init__(self, depth: int, options: Optional[Options] = None) -> None:
        self.depth = depth
        self.last_child = None

        if options:
            self.type = options.wrapper
            self.props = options.wrapper_props

            wrapper_props = dict(self.props) if self.props else {}
            wrapper_props["block"] = {"type": options.type, "depth": depth}

            self.elt = DOM.create_element(self.type, wrapper_props)
        else:
            self.type = None
            self.props = None
            self.elt = DOM.create_element()

    def is_different(
        self, depth: int, type_: RenderableType, props: Optional[Props]
    ) -> bool:
        return depth > self.depth or type_ != self.type or props != self.props

class WrapperStack:
    """
    Stack data structure for element wrappers.
    The bottom of the stack contains the elements closest to the page body.
    The top of the stack contains the most nested nodes.
    """

    __slots__ = "stack"

    def __init__(self) -> None:
        self.stack: List[Wrapper] = []

    def __str__(self) -> str:
        return str(self.stack)

    def length(self) -> int:
        return len(self.stack)

    def append(self, wrapper: Wrapper) -> None:
        return self.stack.append(wrapper)

    def get(self, index: int) -> Wrapper:
        return self.stack[index]

    def slice(self, length: int) -> None:
        self.stack = self.stack[:length]

    def head(self) -> Wrapper:
        if self.length() > 0:
            wrapper = self.stack[-1]
        else:
            wrapper = Wrapper(-1)

        return wrapper

    def tail(self) -> Wrapper:
        return self.stack[0]

class WrapperState:
    """
    This class does the initial node building for the tree.
    It sets elements with the right tag, text content, and props.
    It adds a wrapper element around elements, if required.
    """

    __slots__ = ("block_options", "blocks", "stack")

    def __init__(
        self, block_options: OptionsMap, blocks: Sequence[Block]
    ) -> None:
        self.block_options = block_options
        self.blocks = blocks
        self.stack = WrapperStack()

    def __str__(self) -> str:
        return f"<WrapperState: {self.stack}>"

    def element_for(
        self, block: Block, block_content: Union[Element, Sequence[Element]]
    ) -> Element:
        type_ = block["type"] if "type" in block else "unstyled"
        depth = block["depth"] if "depth" in block else 0
        options = Options.get(self.block_options, type_, BLOCK_TYPES.FALLBACK)
        props = dict(options.props)
        props["block"] = block
        props["blocks"] = self.blocks

        # Make an element from the options specified in the block map.
        elt = DOM.create_element(options.element, props, block_content)

        parent = self.parent_for(options, depth, elt)

        return parent

    def parent_for(self, options: Options, depth: int, elt: Element) -> Element:
        if options.wrapper:
            parent = self.get_wrapper_elt(options, depth)
            DOM.append_child(parent, elt)
            self.stack.stack[-1].last_child = elt
        else:
            # Reset the stack if there is no wrapper.
            if self.stack.length() > 0:
                self.stack = WrapperStack()
            parent = elt

        return parent

    def get_wrapper_elt(self, options: Options, depth: int) -> Element:
        head = self.stack.head()
        if head.is_different(depth, options.wrapper, options.wrapper_props):
            self.update_stack(options, depth)

        # If depth is lower than the maximum, we cut the stack.
        if depth < head.depth:
            self.stack.slice(depth + 1)

        return self.stack.get(depth).elt

    def update_stack(self, options: Options, depth: int) -> None:
        if depth >= self.stack.length():
            # If the depth is gte the stack length, we need more wrappers.
            depth_levels = range(self.stack.length(), depth + 1)

            for level in depth_levels:
                new_wrapper = Wrapper(level, options)

                # Determine where to append the new wrapper.
                if self.stack.head().last_child is None:
                    # If there is no content in the current wrapper, we need
                    # to add an intermediary node.
                    props = dict(options.props)
                    props["block"] = {
                        "type": options.type,
                        "depth": depth,
                        "data": {},
                    }
                    props["blocks"] = self.blocks

                    wrapper_parent = DOM.create_element(options.element, props)
                    DOM.append_child(self.stack.head().elt, wrapper_parent)
                else:
                    # Otherwise we can append at the end of the last child.
                    wrapper_parent = self.stack.head().last_child

                DOM.append_child(wrapper_parent, new_wrapper.elt)

                self.stack.append(new_wrapper)
        else:
            # Cut the stack to where it now stops, and add new wrapper.
            self.stack.slice(depth)
            self.stack.append(Wrapper(depth, options))


from rest_framework import serializers
from django.contrib.auth.models import User
from taggit.models import Tag, TaggedItem

from .models import UserProfile
from matching.models import HelpRequest

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ('id', 'first_name', 'last_name', 'email')

class UserProfileSerializer(serializers.ModelSerializer):
    user = UserSerializer()

    class Meta:
        model = UserProfile

class TagSerializer(serializers.ModelSerializer):
    class Meta:
        model = Tag

class TaggedItemSerializer(serializers.ModelSerializer):
    tag = TagSerializer()

    class Meta:
        model = TaggedItem

class HelpRequestSerializer(serializers.ModelSerializer):
    requester = UserSerializer()
    provider = UserSerializer()

    class Meta:
        model = HelpRequest

    def update(self, instance, validated_data):
        try:
            update_user = User.objects.get(email=validated_data['provider']['email'])
            instance.provider = update_user
            instance.save()
        except User.DoesNotExist:
            return instance
        return instance


import argparse
import itertools
import os
import random

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import DataLoader
from tqdm import tqdm

from dataset import bair_robot_pushing_dataset
from models.lstm import gaussian_lstm, lstm
from models.vgg_64 import vgg_decoder, vgg_encoder
from utils import init_weights, kl_criterion, plot_pred, finn_eval_seq, pred
import matplotlib.pyplot as plt
import pickle

torch.backends.cudnn.benchmark = True

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--lr', default=0.002, type=float, help='learning rate')
    parser.add_argument('--beta1', default=0.9, type=float, help='momentum term for adam')
    parser.add_argument('--batch_size', default=12, type=int, help='batch size')
    parser.add_argument('--log_dir', default='./logs/fp', help='base directory to save logs')
    parser.add_argument('--model_dir', default='', help='base directory to save logs')
    parser.add_argument('--data_root', default='./sample_code/data', help='root directory for data')
    parser.add_argument('--optimizer', default='adam', help='optimizer to train with')
    parser.add_argument('--niter', type=int, default=300, help='number of epochs to train for')
    parser.add_argument('--epoch_size', type=int, default=600, help='epoch size')
    parser.add_argument('--tfr', type=float, default=1.0, help='teacher forcing ratio (0 ~ 1)')
    parser.add_argument('--tfr_start_decay_epoch', type=int, default=100, help='The epoch that teacher forcing ratio become decreasing')
    parser.add_argument('--tfr_decay_step', type=float, default=0, help='The decay step size of teacher forcing ratio (0 ~ 1)')
    parser.add_argument('--tfr_lower_bound', type=float, default=0, help='The lower bound of teacher forcing ratio for scheduling teacher forcing ratio (0 ~ 1)')
    parser.add_argument('--kl_anneal_cyclical', default=True, action='store_true', help='use cyclical mode')
    parser.add_argument('--kl_anneal_ratio', type=float, default=1, help='The decay ratio of kl annealing')
    parser.add_argument('--kl_anneal_cycle', type=int, default=4, help='The number of cycle for kl annealing during training (if use cyclical mode)')
    parser.add_argument('--seed', default=1, type=int, help='manual seed')
    parser.add_argument('--n_past', type=int, default=2, help='number of frames to condition on')
    parser.add_argument('--n_future', type=int, default=10, help='number of frames to predict')
    parser.add_argument('--n_eval', type=int, default=30, help='number of frames to predict at eval time')
    parser.add_argument('--rnn_size', type=int, default=256, help='dimensionality of hidden layer')
    parser.add_argument('--posterior_rnn_layers', type=int, default=1, help='number of layers')
    parser.add_argument('--predictor_rnn_layers', type=int, default=2, help='number of layers')
    parser.add_argument('--z_dim', type=int, default=64, help='dimensionality of z_t')
    parser.add_argument('--p_dim', type=int, default=7, help='dimensionality of pos')
    parser.add_argument('--g_dim', type=int, default=128, help='dimensionality of encoder output vector and decoder input vector')
    parser.add_argument('--beta', type=float, default=0.0001, help='weighting on KL to prior')
    parser.add_argument('--num_workers', type=int, default=4, help='number of data loading threads')
    parser.add_argument('--last_frame_skip', action='store_true', help='if true, skip connections go between frame t and frame t+t rather than last ground truth frame')
    parser.add_argument('--cuda', default=True, action='store_true')

    args = parser.parse_args()
    return args

def train(x, cond, modules, optimizer, kl_anneal, args):
    modules['frame_predictor'].zero_grad()
    modules['posterior'].zero_grad()
    modules['encoder'].zero_grad()
    modules['decoder'].zero_grad()

    # initialize the hidden state.
    modules['frame_predictor'].hidden = modules['frame_predictor'].init_hidden()
    modules['posterior'].hidden = modules['posterior'].init_hidden()
    mse = 0
    kld = 0
    # print(x.shape,cond.shape)
    # torch.Size([12, 30, 3, 64, 64]) torch.Size([12, 30, 7])
    # epoch, time, channel, height, width
    use_teacher_forcing = True if random.random() < args.tfr else False
    x = x.permute(1, 0, 2, 3 ,4)
    cond = cond.permute(1, 0, 2)
    x = x.to(torch.device('cuda'))
    cond = cond.to(torch.device('cuda'))
    #print(x.shape,cond.shape)
    pred_list = [x[0]]
    # teacher forcing
    # reconstruct loss + KL loss
    for i in range(1, args.n_past + args.n_future):
        #raise NotImplementedError
        if use_teacher_forcing:
            prev_x = x[i-1]
        else:
            prev_x = pred_list[i-1]

        h_t, _ = modules['encoder'](x[i])
        z_t, mu, logvar = modules['posterior'](h_t)

        if i < args.n_past:
            h_prev, skip = modules['encoder'](prev_x)
        else:
            h_prev, _= modules['encoder'](prev_x)
        # concat cond, h_prev, z_t
        # print(cond[i].shape,h_prev.shape,z_t.shape)
        g_t = modules['frame_predictor'](torch.cat([cond[i],h_prev,z_t],1))
        x_pred = modules['decoder']([g_t,skip])
        pred_list.append(x_pred)
        mse += modules['mseloss'](x_pred,x[i])
        kld += kl_criterion(mu, logvar,args)
    #print("here",len(x[args.n_past:args.n_future]),len(pred_list[args.n_past:args.n_future]))
    _,_,psnr = finn_eval_seq(x[args.n_past:args.n_future],pred_list[args.n_past:args.n_future])
    beta = kl_anneal.get_beta()
    loss = mse + kld * beta
    loss.backward()
    optimizer.step()

    return loss.detach().cpu().numpy() / (args.n_past + args.n_future), mse.detach().cpu().numpy() / (args.n_past + args.n_future), kld.detach().cpu().numpy() / (args.n_future + args.n_past)

class kl_annealing():
    def __init__(self, args):
        super().__init__()
        self.beta = 0
        self.cyclical = args.kl_anneal_cyclical
        self.ratio = args.kl_anneal_ratio
        self.cycle = args.kl_anneal_cycle
        self.iters = args.niter * args.epoch_size
        self.idx = -1

        if self.cyclical:
            iter_cycle = self.iters/self.cycle
            tmp1 = np.linspace(0, 1, num = int(iter_cycle * self.ratio))
            tmp2 = np.ones(int(iter_cycle - int(iter_cycle * self.ratio)))
            tmp3 = np.concatenate((tmp1,tmp2))
            print("cyclical",tmp1.shape,tmp2.shape)
            for i in range(self.cycle - 1):
                tmp3 = np.concatenate((tmp3,tmp1,tmp2))
        else:
            tmp1 = np.linspace(0, 1, num = int(self.iters * self.ratio))
            tmp2 = np.ones(int(self.iters - int(self.iters * self.ratio)))
            print("monotonic",tmp1.shape,tmp2.shape)
            tmp3 = np.concatenate((tmp1,tmp2))

        self.Betas = tmp3
        print("Beta shape",self.Betas.shape)
        plt.plot(self.Betas)
        plt.grid()
        plt.savefig("a.png")

    def update(self):
        self.idx = self.idx + 1

    def get_beta(self):
        self.update()
        return self.Betas[self.idx]

def main():
    args = parse_args()
    if args.cuda:
        assert torch.cuda.is_available(), 'CUDA is not available.'
        device = 'cuda'
    else:
        device = 'cpu'

    assert args.n_past + args.n_future <= 30 and args.n_eval <= 30
    assert 0 <= args.tfr and args.tfr <= 1
    assert 0 <= args.tfr_start_decay_epoch
    assert 0 <= args.tfr_decay_step and args.tfr_decay_step <= 1

    if args.model_dir != '':
        # load model and continue training from checkpoint
        saved_model = torch.load('%s/model.pth' % args.model_dir)
        optimizer = args.optimizer
        model_dir = args.model_dir
        niter = args.niter
        args = saved_model['args']
        args.optimizer = optimizer
        args.model_dir = model_dir
        args.log_dir = '%s/continued' % args.log_dir
        start_epoch = saved_model['last_epoch']
    else:
        name = 'rnn_size=%d-predictor-posterior-rnn_layers=%d-%d-n_past=%d-n_future=%d-lr=%.4f-g_dim=%d-z_dim=%d-last_frame_skip=%s-beta=%.7f'\
            % (args.rnn_size, args.predictor_rnn_layers, args.posterior_rnn_layers, args.n_past, args.n_future, args.lr, args.g_dim, args.z_dim, args.last_frame_skip, args.beta)

        args.log_dir = '%s/%s' % (args.log_dir, name)
        niter = args.niter
        start_epoch = 0

    os.makedirs(args.log_dir, exist_ok=True)
    os.makedirs('%s/gen/' % args.log_dir, exist_ok=True)

    print("Random Seed: ", args.seed)
    random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)

    if os.path.exists('./{}/train_record.txt'.format(args.log_dir)):
        os.remove('./{}/train_record.txt'.format(args.log_dir))

    print(args)

    with open('./{}/train_record.txt'.format(args.log_dir), 'a') as train_record:
        train_record.write('args: {}\n'.format(args))

    # ------------ build the models  --------------

    if args.model_dir != '':
        frame_predictor = saved_model['frame_predictor']
        posterior = saved_model['posterior']
    else:
        frame_predictor = lstm(args.g_dim+args.z_dim+args.p_dim, args.g_dim, args.rnn_size, args.predictor_rnn_layers, args.batch_size, device)
        print(args.g_dim+args.z_dim, args.g_dim, args.rnn_size, args.predictor_rnn_layers, args.batch_size, device)
        posterior = gaussian_lstm(args.g_dim, args.z_dim, args.rnn_size, args.posterior_rnn_layers, args.batch_size, device)
        frame_predictor.apply(init_weights)
        posterior.apply(init_weights)

    if args.model_dir != '':
        decoder = saved_model['decoder']
        encoder = saved_model['encoder']
        mse_criterion = saved_model['mseloss']
    else:
        encoder = vgg_encoder(args.g_dim)
        decoder = vgg_decoder(args.g_dim)
        encoder.apply(init_weights)
        decoder.apply(init_weights)
        mse_criterion = nn.MSELoss()

    # --------- transfer to device ------------------------------------
    frame_predictor.to(device)
    posterior.to(device)
    encoder.to(device)
    decoder.to(device)
    mse_criterion.to(device)

    # --------- load a dataset ------------------------------------
    train_data = bair_robot_pushing_dataset(args, 'train')
    validate_data = bair_robot_pushing_dataset(args, 'validate')
    train_loader = DataLoader(train_data,
                            num_workers=args.num_workers,
                            batch_size=args.batch_size,
                            shuffle=True,
                            drop_last=True,
                            pin_memory=True)
    train_iterator = iter(train_loader)

    validate_loader = DataLoader(validate_data,
                            num_workers=args.num_workers,
                            batch_size=args.batch_size,
                            shuffle=True,
                            drop_last=True,
                            pin_memory=True)

    validate_iterator = iter(validate_loader)

    # ---------------- optimizers ----------------
    if args.optimizer == 'adam':
        args.optimizer = optim.Adam
    elif args.optimizer == 'rmsprop':
        args.optimizer = optim.RMSprop
    elif args.optimizer == 'sgd':
        args.optimizer = optim.SGD
    else:
        raise ValueError('Unknown optimizer: %s' % args.optimizer)

    params = list(frame_predictor.parameters()) + list(posterior.parameters()) + list(encoder.parameters()) + list(decoder.parameters())
    optimizer = args.optimizer(params, lr=args.lr, betas=(args.beta1, 0.999))
    kl_anneal = kl_annealing(args)

    modules = {
        'frame_predictor': frame_predictor,
        'posterior': posterior,
        'encoder': encoder,
        'decoder': decoder,
        'mseloss':mse_criterion
    }
    # --------- training loop ------------------------------------

    progress = tqdm(total=args.niter)
    best_val_psnr = 0
    for epoch in range(start_epoch, start_epoch + niter):
        frame_predictor.train()
        posterior.train()
        encoder.train()
        decoder.train()

        epoch_loss = 0
        epoch_mse = 0
        epoch_kld = 0

        for _ in range(args.epoch_size):
            try:
                seq, cond = next(train_iterator)
            except StopIteration:
                train_iterator = iter(train_loader)
                seq, cond = next(train_iterator)

            loss, mse, kld = train(seq, cond, modules, optimizer, kl_anneal, args)
            epoch_loss += loss
            epoch_mse += mse
            epoch_kld += kld

        if epoch >= args.tfr_start_decay_epoch:
            ### Update teacher forcing ratio ###
            if args.tfr_decay_step == 0:
                args.tfr = max(0, 1-(epoch-args.tfr_start_decay_epoch)/(args.niter-args.tfr_start_decay_epoch))
            else:
                args.tfr = max(0, args.tfr - args.tfr_decay_step)

        progress.update(1)
        with open('./{}/train_record.txt'.format(args.log_dir), 'a') as train_record:
            train_record.write(('[epoch: %02d] loss: %.5f | mse loss: %.5f | kld loss: %.5f\n' % (epoch, epoch_loss  / args.epoch_size, epoch_mse / args.epoch_size, epoch_kld / args.epoch_size)))
        with open('./{}/KL.txt'.format(args.log_dir), 'a') as train_record:
            train_record.write(('{}\n'.format(epoch_kld / args.epoch_size)))

        frame_predictor.eval()
        encoder.eval()
        decoder.eval()
        posterior.eval()

        if epoch % 5 == 0:
            psnr_list = []
            for _ in range(len(validate_data) // args.batch_size):
                try:
                    validate_seq, validate_cond = next(validate_iterator)
                except StopIteration:
                    validate_iterator = iter(validate_loader)
                    validate_seq, validate_cond = next(validate_iterator)
                validate_seq = validate_seq.permute(1, 0, 2, 3 ,4)
                validate_cond = validate_cond.permute(1, 0, 2)
                pred_seq = pred(validate_seq, validate_cond, modules, args, device)
                plot_pred(validate_seq, pred_seq, epoch, args)
                _, _, psnr = finn_eval_seq(validate_seq[args.n_past:args.n_future], pred_seq[args.n_past:args.n_future])
                psnr_list.append(psnr)

            ave_psnr = np.mean(np.concatenate(psnr))

            with open('./{}/train_record.txt'.format(args.log_dir), 'a') as train_record:
                train_record.write(('====================== validate psnr = {:.5f} ========================\n'.format(ave_psnr)))
            with open('./{}/ave_psnr.txt'.format(args.log_dir), 'a') as train_record:
                train_record.write(('{:.5f}\n'.format(ave_psnr)))

            if ave_psnr > best_val_psnr:
                best_val_psnr = ave_psnr
                # save the model
                torch.save({
                    'encoder': encoder,
                    'decoder': decoder,
                    'frame_predictor': frame_predictor,
                    'posterior': posterior,
                    'args': args,
                    'mseloss':mse_criterion,
                    'last_epoch': epoch},
                    '%s/model.pth' % args.log_dir)

        if epoch % 20 == 0:
            try:
                validate_seq, validate_cond = next(validate_iterator)
            except StopIteration:
                validate_iterator = iter(validate_loader)
                validate_seq, validate_cond = next(validate_iterator)

if __name__ == '__main__':
    main()


import random
from io import TextIOWrapper
from math import ceil
from typing import List
from src.Utils.LatinNGrams import LatinNGrams

def keyCharOrdering(key: str) -> [int]:

    if not key.isalpha():
        raise ValueError(f"non-alpha key cannot be converted to unordered numeric form: {key}")

    key = key.upper()
    s = sorted(key)
    m: dict = {c: v for c, v in enumerate(s)}
    ordering = [-1] * len(key)   # (sorted key index, unsorted key index)

    for keyInd, keyChr in m.items():
        for index, order in enumerate(ordering):
            if key[index] == keyChr and order == -1:
                ordering[index] = keyInd
                break

    return ordering

def columnTransposition(msg: str, key: str) -> str:
    transposed = ""

    columns = [(orderedInd, unorderedInd) for unorderedInd, orderedInd in enumerate(keyCharOrdering(key))]

    # sort the columns based on assigned numerical value (sort a-z)
    columns = sorted(columns, key=lambda pair: pair[0])

    for col in columns:
        msgInd = col[1]
        while msgInd < len(msg):
            transposed += msg[msgInd]
            msgInd += len(key)

    return transposed

def invertedColumnTransposition(cipherText: str, key) -> str:
    columns = key

    if isinstance(columns, str):
        columns = keyCharOrdering(key)
    if not isinstance(columns, type([int])):
        raise TypeError("Incorrect columns type for invertedColumnTransposition()")

    invColTrans: str = ""

    rowCount = ceil(float(len(cipherText)) / len(columns))
    fillCol = len(cipherText) % len(columns)        # index in columns of first unfilled col

    shortColumns = columns[fillCol:] if fillCol > 0 else []     # unfilled column (last row)
    leftShiftAmounts = {c: sum(sc < c for sc in shortColumns) for ind, c in enumerate(columns)}

    for row in range(rowCount):
        for ind, col in enumerate(columns):
            msgInd = row + col*rowCount - leftShiftAmounts[col]
            if fillCol and ind >= fillCol and row == rowCount-1:
                continue
            invColTrans += cipherText[msgInd]

    return invColTrans

"""
    Each character in the latin alphabet is assigned a value of O-25.
    To sum two characters, do
        ( val(a) + val(b) ) % 26
    where the value wraps around as to stay in the 0-25 range.
    Keeps case of a.
"""
def sumChars(a: chr, b: chr):
    lower = ord(a) >= 0x61
    a = a.upper()
    b = b.upper()
    c = chr(((ord(a) + ord(b)) % 26) + 0x41)
    return c.lower() if lower else c.upper()

"""
    Each character in the latin alphabet is assigned a value of O-25.
    To subtract two characters, do
        ( val(a) - val(b) ) % 26
    where the value wraps around as to stay in the 0-25 range.
    Keeps case of a.
"""
def diffChars(a: chr, b: chr):
    lower = ord(a) >= 0x61
    a = a.upper()
    b = b.upper()
    c = chr(((ord(a) - ord(b)) % 26) + 0x41)
    return c.lower() if lower else c.upper()

def writePairList(pList: List, of, sort=False, reverse=False):

    opened = False
    if isinstance(of, str):
        opened = True
        of = open(of, 'w')

    if not isinstance(of, TextIOWrapper):
        raise ValueError("The of parameter of writePairList accepts a string or a TextIOWrapper type")

    if sort:
        pList.sort(key=lambda p: p[1], reverse=reverse)

    for item in pList:
        of.write(str(item[0]) + " " + str(item[1]) + ("\n" if item != pList[-1] else ""))

    if opened:
        of.close()

def exchange2letters(lst: List[str]):
    i = random.randint(0, 24)
    j = random.randint(0, 24)

    temp = lst[i]
    lst[i] = lst[j]
    lst[j] = temp

def swap2rows(lst: List[str]):
        i = random.randint(0, 4) * 5
        j = random.randint(0, 4) * 5

        for k in range(5):
            temp = lst[i + k]
            lst[i + k] = lst[j + k]
            lst[j + k] = temp

def swap2cols(lst: List[str]):
    i = random.randint(0, 4)
    j = random.randint(0, 4)

    for k in range(5):
        k *= 5
        temp = lst[k + i]
        lst[k + i] = lst[k + j]
        lst[k + j] = temp

def copyList(lst: List[str], toCopy: List[str], reverse=False):
    if reverse:
        endIndex = len(lst) - 1
        for i in range(len(lst)):
            lst[i] = toCopy[endIndex - i]
    else:
        for i in range(len(lst)):
            lst[i] = toCopy[i]

def generateMonoFiles(cipherText: str, countFreq: str,
                      storedPath:str, measuredPath: str, combinedPath: str):

    monoCounts = {}
    for c in cipherText:
        monoCounts.setdefault(c, 0)
        monoCounts[c] += 1
    writePairList(list(monoCounts.items()), measuredPath, True, True)

    storedMono = LatinNGrams(iPath=storedPath, statType=countFreq, toLogFreq=False).getFrequenciesList()
    measuredMono = LatinNGrams(iPath=measuredPath, statType=countFreq, toLogFreq=False).getFrequenciesList()

    of = open(combinedPath, 'w')
    of.write("stored mono freq\n")
    writePairList(storedMono, of)
    of.write("\n\n")
    of.write("measured mono freq\n")
    writePairList(measuredMono, of)
    of.close()

def generateBiFiles(cipherText: str, countFreq: str,
                    storedPath: str, measuredPath: str, combinedPath: str):

    biCounts = {}
    for index in range(len(cipherText)-1):
        bigram = cipherText[index] + cipherText[index+1]
        biCounts.setdefault(bigram, 0)
        biCounts[bigram] += 1
    writePairList(list(biCounts.items()), measuredPath, True, True)

    storedMono = LatinNGrams(iPath=storedPath, statType=countFreq, toLogFreq=False).getFrequenciesList()
    measuredMono = LatinNGrams(iPath=measuredPath, statType=countFreq, toLogFreq=False).getFrequenciesList()

    of = open(combinedPath, 'w')
    of.write("stored bi freq\n")
    writePairList(storedMono, of)
    of.write("\n\n")
    of.write("measured bi freq\n")
    writePairList(measuredMono, of)
    of.close()