import sys
sys.path.append('/data/cmatthe/python/wobble_aux')#this  path only works on lx39
import combine_results as cr
from parameters import Parameters
#end local imports
import os
from time import time
import wobble
import h5py
import numpy as np
import shutil
import yaml

if __name__ == "__main__":
    queue = True
    #################### parameters
    if queue:
        parameters = Parameters(filename = "yaml_temp/optimize_parameters.yaml")

        starname = parameters.dictionary["starname"]
        K_star = parameters.dictionary["K_star"]
        K_t = parameters.dictionary["K_t"]
        niter = parameters.dictionary["niter"]
        start = parameters.dictionary["start"]
        end = parameters.dictionary["end"]
        chunk_size = parameters.dictionary["chunk_size"]
        defaultQ = parameters.dictionary["defaultQ"]
        output_suffix = parameters.dictionary["output_suffix"]
        if defaultQ:
            default_str = "_def"
        else:
            default_str = ""

    else:
        starname = 'GJ876'
        K_star = 0
        K_t = 3
        start = 0 #first order #reg seems to break if not startet at 0, TODO and at any rate wobble would not use the file propperly in that case
        end = 61 # first order that is *not* included
        chunk_size = 42 #use at least 2 so median is not out of bounds, make sure last chunk is not accidentaly 1 long

    data_file= '../data/' + starname+ '_vis'+'_e2ds.hdf5'

    reg_file_base = '../wobble/regularization/{0}_star_K{1}'.format(starname, K_star)# NOTE this is defined separately in top and chunk -> could cause issues
    reg_t_file_base = '../wobble/regularization/{0}_t_K{1}'.format(starname, K_t)

    star_filename_final = reg_file_base + '_orders[{0},{1})_'.format(start, end)+ output_suffix + '.hdf5'
    tellurics_filename_final = reg_t_file_base + '_orders[{0},{1})_'.format(start, end)+ output_suffix + '.hdf5'

    ####################

    start_time = time()

    #make list of chunks

    chunks = cr.chunk_list(start, end, chunk_size)

    # HACK global epoch cutting: use epoch (and order) cutting only once on all data, then pass epochs to chunks
    orders = np.arange(start, end)
    data = wobble.Data(data_file, orders = orders, min_flux=10**-5)
    epochs = np.asarray(data.epochs)
    epochs_list = data.epochs.tolist()
    epochs_list_str = str(list(epochs))

    for i in range(len(chunks)):
        start_chunk = int(chunks[i, 0])
        end_chunk = int(chunks[i, 1])

        parameters.dictionary.update({
        "epochs_list" : epochs_list,
        "start_chunk" : start_chunk,
        "end_chunk" : end_chunk
        })

        #Write into YAML to pass to chunk script
        parameters.write("yaml_temp/optimize_parameters_chunk.yaml")

        os.system("python3 regularization_chunk_1.py")
        print("time elapsed total: {0:.2f} min".format((time() - start_time)/60.0))

    print("all chunks optimized: writing combined file")

    ####Star combine
    # initialize with first chunk (save first chunk as stitched file)
    #using the first file as the base will mean assuming (hoping) they all have the same number of epochs
    start_order = chunks[0, 0]
    end_order = chunks[0, 1]
    filename_chunk = cr.reg_file_chunk(start_order, end_order, reg_file_base)
    shutil.copy(filename_chunk, star_filename_final)

    if not len(chunks) == 1:
        with h5py.File(star_filename_final,'r+') as f:

            #append orders from other chunks
            for i in range(1, len(chunks)):
            # load files and combine them
                start_order = chunks[i, 0]
                end_order = chunks[i, 1]
                filename_chunk = cr.reg_file_chunk(start_order, end_order, reg_file_base)
                with h5py.File(filename_chunk,'r') as g:
                    for key in list(g.keys()):
                        # append new regularization parameters
                        temp = f[key][()]
                        del f[key] # HACK but works
                        f[key] = np.append(temp, g[key][()])

        ####tellurics combine
        # initialize with first chunk (save first chunk as stitched file)
        #using the first file as the base will mean assuming (hoping) they all have the same number of epochs
        start_order = chunks[0, 0]
        end_order = chunks[0, 1]
        filename_chunk = cr.reg_t_file_chunk(start_order, end_order, reg_t_file_base)
        shutil.copy(filename_chunk, tellurics_filename_final)

        with h5py.File(tellurics_filename_final,'r+') as f:

            #append orders from other chunks
            for i in range(1, len(chunks)):
            # load files and combine them
                start_order = chunks[i, 0]
                end_order = chunks[i, 1]
                filename_chunk = cr.reg_t_file_chunk(start_order, end_order, reg_t_file_base)
                with h5py.File(filename_chunk,'r') as g:
                    for key in list(g.keys()):
                        # append new regularization parameters
                        temp = f[key][()]
                        del f[key] # HACK but works
                        f[key] = np.append(temp, g[key][()])

    print("regulization files finished")
    print("time elapsed total: {0:.2f} min".format((time() - start_time)/60.0))


import cv2

# Load a photo of Donald Trump
trump = cv2.imread('images/trump.jpg', cv2.IMREAD_COLOR)
# Resize Trump
trump = cv2.resize(trump,(800,450))
# Show the image of Trump
cv2.imshow("tash", trump)
# Keep the image open until a key is pressed
cv2.waitKey(0)

# Challenge: Use the openCV documentation to draw red eyes on Donald Trump
# https://docs.opencv.org/3.1.0/dc/da5/tutorial_py_drawing_functions.html


###Não é necessárioimport numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from shutil import copyfile
from datetime import date
import cv2 as cv
import os

dataset_directory = "./../../datasets"
o_dataset_name = "demonstrate_market_dataset_y_out_classes"
network = "vgg16"

dim = (224, 224)

resize = True#Must be True always, False is exception
repeted = True
round_sample = -1
dataset_name = "demonstrate_market_dataset_y_out_classes"
dataset_name_case = "demonstrate_market_dataset_y_out_classes"

if not os.path.exists(os.path.join(dataset_directory, network, o_dataset_name)):
    os.makedirs(os.path.join(dataset_directory, network, o_dataset_name))
    print("Path", os.path.join(dataset_directory, network, o_dataset_name), "created.")
else:
    print("Directory already exist.")

#Load dataset
data = pd.read_csv(os.path.join(dataset_directory, o_dataset_name + ".txt"), sep=" ", engine="python", encoding="ISO-8859-1", header=None).values

min_max_array = []
def getMinMax(array, col):
    return [data[:,col].min(), data[:,col].max()]

min_max_array.append(getMinMax(data, 1))#acc_y. Muste be 2, or others

#Normalization
#data[:,1] = (data[:,1]-data[:,1].min())/(data[:,1].max()-data[:,1].min())#Nomalize de 0 a 1
##data[:,1] = data[:,1].astype(float)
##data[:,2] = (data[:,2] - np.mean(data[:,2]))/np.std(data[:,2])#normalize (x - media) / desvio padrao
##data[:,2] = data[:,2].astype(float)
##data[:,3] = (data[:,3]-data[:,3].min())/(data[:,3].max()-data[:,3].min())#Nomalize

#trainingSet, testSet = train_test_split(data, test_size=0.3)
testSet = data.copy()
trainingSet = pd.DataFrame().values

src = os.path.join(dataset_directory, o_dataset_name)
dst = os.path.join(dataset_directory, network, o_dataset_name)

folders = ["train", "val"]

#trainingSet = trainingSet[trainingSet[:,0].argsort()]
testSet = testSet[testSet[:,0].argsort()]

sets = [trainingSet, testSet]

#Legenda: nmp = noamalização média desvio padrão
dst_dataset_name = os.path.join(dst, dataset_name)

dataset_name_flag = True

print("Started")

for folder in folders:
    if not os.path.exists(os.path.join(dst_dataset_name, folder, dataset_name_case, "images")):
        os.makedirs(os.path.join(dst_dataset_name, folder, dataset_name_case, "images"))
        dataset_name_flag = True

trainFile = open(os.path.join(dst_dataset_name, "train", dataset_name_case, "gyro.txt"), "w")
valFile = open(os.path.join(dst_dataset_name, "val", dataset_name_case, "gyro.txt"), "w")

##trainFileGP = open(dst_dataset_name + "/" + "train" + "/" + dataset_name_case + "/" + "gyro_gp.txt", "w")
##valFileGP = open(dst_dataset_name + "/" + "val" + "/" + dataset_name_case + "/" + "gyro_gp.txt", "w")

def check(list, elem):
    for i in list:
        if(elem == i):
            return True
    return False

def elemEqualsSet(name, dataset, lis):
    today = date.today()
    log = open(name + "_" + str(today) + ".txt", "w")
    for i in lis:
        print("-----------")
        for j in range(len(dataset)):
            if(dataset[j][1] == i):
                print(dataset[j][0], j, i)
                log.write(dataset[j][0] + " " + str(j) + " " + str(i) + " " + "\n")
    for col in ["gyro"]:
        for sample in min_max_array:
                log.write(col + ":" + str(sample))

##      log.write("min pedo" + " " + str(data[:,2].min()))
##      log.write("max pedo" + " " + str(data[:,2].max()))
    log.close()

#print(trainingSet)
print(testSet)

programPause = input("Press the <ENTER> key to continue save train and test...")
for folder in zip(folders, sets):
    elem = []
    alreadyExist = []
    if(dataset_name_flag):
        print(folder[0])
        for sample in folder[1]:
            if(not check(elem, sample[1]) or repeted):
                img = cv.imread(os.path.join(src, sample[0][:]), cv.IMREAD_UNCHANGED)
                if(resize):
                    print(folder[0], sample[0])
                    resized = cv.resize(img, dim, interpolation = cv.INTER_AREA)
                    cv.imwrite(os.path.join(dst_dataset_name, folder[0], dataset_name_case, "images/", sample[0]), resized)
                else:
                    cv.imwrite(os.path.join(dst_dataset_name, folder[0], dataset_name_case, "images/", sample[0]), img)
#                               print(dst_dataset_name + "/" + folder[0] + "/" + sample[0])
                if(folder[0] == "train"):
                    if(round_sample == -1):
                        trainFile.write(folder[0] + "/" + sample[0][:] + " " + str(sample[3]) + "\n")
##                                              trainFileGP.write(folder[0] + "/" + sample[0][1:] + " " + str(sample[1]) + " " + str(sample[2]) + "\n")
                    else:
                        trainFile.write(folder[0] + "/" + sample[0][:] + " " + str(round(sample[3], round_sample)) + "\n")
##                                              trainFileGP.write(folder[0] + "/" + sample[0][1:] + " " + str(round(sample[1], round_sample)) + " " + str(round(sample[2], round_sample)) + "\n")
                    elem.append(sample[1])
                if(folder[0] == "val"):
                    if(round_sample == -1):
                        valFile.write(folder[0] + "/" + sample[0][:] + " " + str(sample[3]) + "\n")
##                                              valFileGP.write(folder[0] + "/" + sample[0][1:] + " " + str(sample[1]) + " " + str(sample[2]) + "\n")
                    else:
                        valFile.write(folder[0] + "/" + sample[0][:] + " " + str(round(sample[3], round_sample)) + "\n")
##                                              valFileGP.write(folder[0] + "/" + sample[0][1:] + " " + str(round(sample[1], round_sample)) + " " + str(round(sample[2], round_sample)) + "\n")
                    elem.append(sample[1])
            else:
                alreadyExist.append(sample[1])
        print("Qtd samples", len(elem), "| All samples", len(folder[1]))
        print("Already exist in " + folder[0] + " : ", alreadyExist)
        elemEqualsSet(dst_dataset_name + "/log_" + folder[0], folder[1], alreadyExist)

trainFile.close()
valFile.close()


"""
Parse one of KLEEs "info" files
"""

import re
import logging
from datetime import datetime, timedelta
from ..exceptions import InputError

_logger = logging.getLogger(__name__)

class Info:
    """
    Contains the information from a KLEE "info" file
    """
    __re_pid = re.compile(r"PID: (\d+)\r?\n")
    __re_start = re.compile(r"Started: (\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\r?\n")
    __re_finish = re.compile(r"Finished: (\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\r?\n")
    __re_elapsed = re.compile(r"Elapsed: (\d{2}):(\d{2}):(\d{2})\r?\n")
    __re_explored = re.compile(r"KLEE: done: explored paths = (\d+)\r?\n")
    __re_constructs_per_query = re.compile(r"KLEE: done: avg\. constructs per query = (\d+)\r?\n")
    __re_queries = re.compile(r"KLEE: done: total queries = (\d+)\r?\n")
    __re_queries_sat = re.compile(r"KLEE: done: valid queries = (\d+)\r?\n")
    __re_queries_nsat = re.compile(r"KLEE: done: invalid queries = (\d+)\r?\n")
    __re_queries_cex = re.compile(r"KLEE: done: query cex = (\d+)\r?\n")
    __re_instructions = re.compile(r"KLEE: done: total instructions = (\d+)\r?\n")
    __re_paths = re.compile(r"KLEE: done: completed paths = (\d+)\r?\n")
    __re_tests = re.compile(r"KLEE: done: generated tests = (\d+)\r?\n")

    @staticmethod
    def __force_match(regex, line, message, path):
        match = regex.fullmatch(line)
        if not match:
            raise InputError(message.format(path))
        return match

    def __parse_searcher(self, infofile, path):
        """Parse the searcher description."""
        line = infofile.readline().rstrip()
        if line != r"BEGIN searcher description":
            raise InputError('Info file "{}" does not contain a valid begin searcher tag in line 4'.format(path))
        self.searcher = []
        while True:
            line = infofile.readline()
            if line == "":
                raise InputError('Info file "{}" missing end searcher tag'.format(path))
            line = line.rstrip()
            if line == r"END searcher description":
                break
            self.searcher.append(line)

    def __init__(self, path: "Path to a KLEE info file."):
        """Open a KLEE "info" file."""
        _logger.debug('Creating Info from "{}"'.format(path))
        with open(path) as infofile:
            line = infofile.readline()
            if len(line) == 0:
                self.empty = True
                return
            self.empty = False
            self.command = line.rstrip()
            if len(self.command) == 0:
                raise InputError('Info file "{}" has empty command'.format(path))

            match = self.__force_match(self.__re_pid, infofile.readline(), 'Info file "{}" does not contain a valid PID entry in line 2', path)
            self.pid = int(match.group(1))

            match = self.__force_match(self.__re_start, infofile.readline(), 'Info file "{}" does not contain a valid started entry in line 3', path)
            self.start = datetime(*[int(match.group(x)) for x in range(1, 7)])

            self.__parse_searcher(infofile, path)

            match = self.__force_match(self.__re_finish, infofile.readline(), 'Info file "{}" does not contain a valid finished entry 1 line after the end searcher tag', path)
            self.finish = datetime(*[int(match.group(x)) for x in range(1, 7)])

            match = self.__force_match(self.__re_elapsed, infofile.readline(), 'Info file "{}" does not contain a valid elapsed entry 2 lines after the end searcher tag', path)
            self.elapsed = timedelta(0, int(match.group(1)) * 3600 + int(match.group(2)) * 60 + int(match.group(3)))

            match = self.__force_match(self.__re_explored, infofile.readline(), 'Info file "{}" does not contain a valid explored paths entry 3 lines after the end searcher tag', path)
            self.explored_paths = int(match.group(1))

            nextLine = infofile.readline()
            try:
              match = self.__force_match(self.__re_constructs_per_query,
                  nextLine,
                  'Info file "{}" does not contain a valid avg. constructs per query entry 4 lines after the end searcher tag',
                  path)
              self.constructs_per_query = int(match.group(1))
              nextLine = infofile.readline() # Get the next line for the next __force_match
            except InputError:
              # This line might not be emitted if there are no queries
              self.constructs_per_query = 0

            match = self.__force_match(self.__re_queries, nextLine, 'Info file "{}" does not contain a valid total queries entry 5 lines after the end searcher tag', path)
            self.queries = int(match.group(1))

            match = self.__force_match(self.__re_queries_sat, infofile.readline(), 'Info file "{}" does not contain a valid valid queries entry 6 lines after the end searcher tag', path)
            self.satisfiable_queries = int(match.group(1))

            match = self.__force_match(self.__re_queries_nsat, infofile.readline(), 'Info file "{}" does not contain a valid invalid queries entry 7 lines after the end searcher tag', path)
            self.unsatisfiable_queries = int(match.group(1))

            match = self.__force_match(self.__re_queries_cex, infofile.readline(), 'Info file "{}" does not contain a valid query cex entry 8 lines after the end searcher tag', path)
            self.cex = int(match.group(1))

            line = infofile.readline().rstrip()
            if len(line) > 0:
                raise InputError('Info file "{}" does not contain an empty line 9 lines after the end searcher tag'.format(path))

            match = self.__force_match(self.__re_instructions, infofile.readline(), 'Info file "{}" does not contain a valid total instruction entry 10 lines after the end searcher tag', path)
            self.instructions = int(match.group(1))

            match = self.__force_match(self.__re_paths, infofile.readline(), 'Info file "{}" does not contain a valid a completed paths entry 11 lines after the end searcher tag', path)
            self.completed_paths = int(match.group(1))

            match = self.__force_match(self.__re_tests, infofile.readline(), 'Info file "{}" does not contain a valid a generated tests entry 12 lines after the end searcher tag', path)
            self.tests = int(match.group(1))

            line = infofile.readline()
            if len(line) > 0:
                raise InputError('Info file "{}" did not end as expected.'.format(path))


# -*- coding: utf-8 -*-
"""
Created on Sun Mar 24 15:46:12 2019

@author: durgesh singh
"""

from torch.utils.data import Dataset
import nltk
from nltk.corpus import treebank
from nltk.corpus import brown
from collections import defaultdict
import torch

class CustomDataset(Dataset):

    def __init__(self,dname='brown'):
        super().__init__()

        data = None
        #selecting the datset
        if dname =='treebank':
            if len(treebank.words()) == 0:
                nltk.download('treebank')
            data = treebank.tagged_sents(tagset='universal')

        elif dname == 'brown':
            if len(brown.words()) == 0:
                nltk.download('brown')
            data = brown.tagged_sents(tagset='universal')

        self.data=data
        #print(data[0:1])
        vocab,tags =self._build_vocab()
        max_sent_len = max(map(len, data))
        self.max_sent_len = max_sent_len
        self.word_to_idx = defaultdict(lambda:0, {word:idx for idx,word in enumerate(vocab)})
        self.idx_to_word = {idx:word for word,idx in self.word_to_idx.items()}
        self.tag_to_idx = {tag:idx for idx,tag in enumerate(tags)}
        self.idx_to_tag = {idx:tag for tag,idx in self.tag_to_idx.items()}
        self.sen_list,self.tag_list = self._convert_to_num()

    def get_target_size(self):
        return len(self.tag_to_idx)

    def get_vocab_size(self):
        return len(self.word_to_idx)

    def _convert_to_num(self):
        data = self.data
        max_sent_len = self.max_sent_len
        sent_list=[]
        taggs_list=[]

        for sen in data:
            num_row = [self.word_to_idx[word.lower()] for word,tag in sen]
            tag_row = [self.tag_to_idx[tag] for word,tag in sen]
            num_row = num_row +[0]*(max_sent_len-len(num_row))
            tag_row = tag_row +[0]*(max_sent_len-len(tag_row))
            num_row = torch.tensor(num_row)
            tag_row = torch.tensor(tag_row)
            sent_list.append(num_row)
            taggs_list.append(tag_row)

        return sent_list,taggs_list

    def _build_vocab(self):
        data=self.data
        vocabset=set()
        tagset = set()
        all_sents_tags = [[(word.lower(), tag) for word, tag in sentence] for sentence in data]
        for sent_tag in all_sents_tags:
            sent,tags = zip(*sent_tag)
            vocabset.update(sent)
            tagset.update(tags)
        return ['<UNK>','<EOS>']+list(vocabset),list(tagset)

    def __len__(self):
        return len(self.sen_list)

    def __getitem__(self,idx):
        return self.sen_list[idx],self.tag_list[idx]

cdataset = CustomDataset(dname='brown')


# -*- coding: utf-8 -*-

"""Plot a snapshot of particles"""

# -----------------------------------
# plotcloud.py
#
# Bjørn Ådlandsvik, <bjorn@imr.no>
# Institute of Marine Research
# 2009-11-01
# -----------------------------------

import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset

# ----------------------
# User settings
# ----------------------

# Particle file name
fname = "FW.nc"
#fname = "BW.nc"

# Time frame
tframe = 1

# Geographical extent
lon0, lat0 = -10, 60
lon1, lat1 =  60, 80

# ------------------------
# Read data
# ------------------------

# Particle data

f = Dataset(fname)

X = f.variables['X'][tframe,:]
Y = f.variables['Y'][tframe,:]
life = f.variables['life'][tframe,:]

X = X[life==1]
Y = Y[life==1]

f.close()

# Get coast data
nc = Dataset('kyst.nc')
Xcoast = nc.variables['X'][:]
Ycoast = nc.variables['Y'][:]
S = nc.variables['S'][:]
nc.close()

# Get topography data
i0 = (lon0+180)*60
j0 = (lat0+90)*60
imax = (lon1-lon0)*60 + 1
jmax = (lat1-lat0)*60 + 1
f = Dataset('/home/bjorn/Data/etopo1/ETOPO1_Bed_g_gmt4.grd')
H = -f.variables['z'][j0:j0+jmax, i0:i0+imax]
lon = f.variables['lon'][i0:i0+imax]
lat = f.variables['lat'][j0:j0+jmax]
f.close()

# ----------------------
# Plot
# ---------------------

# Plot coast data
for i in xrange(len(S)-1):
    plt.fill(Xcoast[S[i]:S[i+1]], Ycoast[S[i]:S[i+1]], facecolor=(0.2,0.8,0.2))

# Plot bottom contours
plt.contour(lon, lat, H, levels=(100,200,400), colors='r')

# Plot cloud of particle tracks
plt.plot(X, Y, 'b.')

# Limit plot area
plt.axis((lon0, lon1, lat0, lat1))

plt.show()


import urllib.request
import simplejson

# The request also includes the userip parameter which provides the end
# user's IP address. Doing so will help distinguish this legitimate
# server-side traffic from traffic which doesn't come from an end-user.
url = ('https://ajax.googleapis.com/ajax/services/search/web'
       '?v=1.0&q=python&userip=121.133.116.69')

url2 ='https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=python&userip=121.133.116.69'

def get_search_result(keyword, user_ip):
    url = ('https://ajax.googleapis.com/ajax/services/search/web'
           '?v=1.0&q='
           +keyword
           +'&userid='
           +user_ip)
    return url

for i in range(1,2) :
    url = get_search_result('python', '121,13,116,69')

#    request = urllib.request.Request(url, None, {'Referer': '1.1.1.12:8888'})
    request = urllib.request.Request(url)

    response = urllib.request.urlopen(request)

    print(response)

    print (i)
    print ('\n###########\n')

    # Process the JSON string.
    results = simplejson.load(response)
    decoded_data = simplejson.load(response)
    # now have some fun with the results...
    print (results)

    result_len = len(decoded_data['responseData']['results'])

    for i in range(0, result_len):
        print (i)
        print (decoded_data['responseData']['results'][i]['content'])


import datetime as dt
import pandas_datareader as web
import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings('ignore')

class Portfolio:
	def __init__(self, basketPortfolio, startDate, endDate, benchmarkTicker):
		"""
		Args:
		    basketPortfolio (dict): stocks in the portfolio and their corresponding number of shares
		    startDate (datetime object): start date of the lifetime of the portfolio
		    endDate (datetime object): end date of the lifetime of the portfolio
		    benchmarkTicker (str): ticker of the portfolio's benchmark index
		"""
		self.basketPortfolio = basketPortfolio

		self.startDate = startDate
		self.endDate = endDate

		self.benchmarkTicker = benchmarkTicker
		self.symbols = list(self.basketPortfolio.keys())

		# Get the historical adjusted close stock prices for the stocks in the portfolio
		self.priceData = web.DataReader(self.symbols, "yahoo" , self.startDate, self.endDate)['Adj Close']

		self.priceData['Portfolio Value'] = None
		self.priceData['Daily Returns'] = None

		totalValue = 0

		# Calculate daily portfolio value and daily returns
		for row in range(0, len(self.priceData)): # iterate through rows
			for i in self.basketPortfolio: # iterate through tickers
				totalValue += self.priceData[i].iloc[row] * self.basketPortfolio[i]
			self.priceData['Portfolio Value'].iloc[row] = totalValue
			if row != 0:
				self.priceData['Daily Returns'] = self.priceData['Portfolio Value'].pct_change(1)
			totalValue = 0

	def averageDailyReturn(self):
		"""Used CAGR to calculate the average daily return
		CAGR is a good measure to determine what an investment yields on a daily compounded basis

		CAGR = (final portfolio value/initial portfolio value) ^ (1/business days in between the start date and the end date) - 1

		Returns:
		    float: the average daily percent return of the portfolio
		"""
		# The length of the dataframe is the number of trading days in the lifespan of the portfolio
		averageDailyGain = (((self.priceData['Portfolio Value'].iloc[len(self.priceData) - 1])/self.priceData['Portfolio Value'].iloc[0])**(1/len(self.priceData)) - 1)
		return averageDailyGain

	def volatility(self):
		"""Calculate the standard deviation of the daily returns of portfolio value
		Made the variable that is returned a private class variable, so that other methods in the class can access the portfolio volatility value

		Returns:
		    float: the portfolio's volatility over the lifespan of the portfolio
		"""
		self.__portfolioVolatility = self.priceData['Daily Returns'].std()
		return self.__portfolioVolatility

	def riskRatio(self):
		"""
		Returns:
		    float: volatility ratio of the portfolio compared to the volatility of the benchmark over the lifetime of the portfolio
		"""
		# Get the historical adjusted close stock prices for the portfolio's benchmark
		self.benchmarkData = pd.DataFrame(web.DataReader(self.benchmarkTicker, "yahoo" , self.startDate, self.endDate)['Adj Close'])
		self.benchmarkData = self.benchmarkData.assign(DailyReturns = "None")

		# Calculate daily returns of the benchmark index
		for row in range(0, len(self.benchmarkData)):
			self.benchmarkData['DailyReturns'] = self.benchmarkData['Adj Close'].pct_change(1)

		# Calculate standard deviation of the benchmark's daily returns
		self.__benchmarkVolatility = self.benchmarkData['DailyReturns'].std()

		# Risk Ratio = Portfolio Volatility/Benchmark Volatility
		return self.__portfolioVolatility/self.__benchmarkVolatility

	def marginalVolatility(self, ticker, shares):
		"""
		Args:
		    ticker (str): ticker of the stock to be added to the portfolio
		    shares (int): number of shares of the stock to be added to the portfolio

		Returns:
		    float: the difference in volatility of the portfolio if the specified number of shares of the ticker was added to the
		    portfolio
		"""
		marginalPortfolio = self.basketPortfolio
		# If the stock is already in the portfolio, update the share count
		if ticker in marginalPortfolio:
			marginalPortfolio[ticker] += shares
		# Otherwise add a new key/value pair to the dictionary with the new stock ticker and number of shares
		else:
			marginalPortfolio[ticker] = shares

		marginalSymbols = list(marginalPortfolio.keys())

		# Get the historical adjusted close stock prices for the stocks in the portfolio with the included stock and the number of shares
		marginalData = web.DataReader(marginalSymbols, "yahoo", self.startDate, self.endDate)['Adj Close']

		marginalData['Portfolio Value'] = None
		marginalData['Daily Returns'] = None
		totalValue = 0

		# Calculate daily portfolio value and daily returns
		for row in range(0, len(marginalData)): # iterate through rows
			for i in marginalPortfolio: # iterate through tickers
				totalValue += marginalData[i].iloc[row] * marginalPortfolio[i]
			marginalData['Portfolio Value'].iloc[row] = totalValue
			if row != 0:
				marginalData['Daily Returns'] = marginalData['Portfolio Value'].pct_change(1)
			totalValue = 0

		# Calculate standard deviation of the portfolio's daily returns with the included stock and the number of shares
		self.__marginalPortfolioVolatility = marginalData['Daily Returns'].std()
		return self.__marginalPortfolioVolatility - self.__portfolioVolatility

	def maxDrawDown(self):
		"""
		Returns:
		    float: maximum drawdown of the portfolio (largest drop from peak to trough in the lifespan of the portfolio)
		"""
		# Initialize values to 0
		peak = 0
		trough = 0
		DDCurrent = 0
		DDTotal = 0

		# Iterate through the daily portfolio values
		for value in self.priceData['Portfolio Value']:
			# If the current value is greater than the current peak, set the peak equal to the current value
			# Reset trough value
			# Reset current dropdown value
			if value > peak:
				peak = value
				trough = 0
				DDCurrent = 0

			# If trough has not been initialized and the current value is less than the peak, set the trough equal to the current value
			elif value < peak and trough == 0:
				trough = value

			# If the current value is less than the trough, set the trough equal to the current value
			else:
				trough = value

			# When both the peak and trough are initialized, calculate current dropdown value
			if peak != 0 and trough != 0:
				DDCurrent = peak - trough

			# If the current dropdown value is greater than the previous dropdown value, update with the overall max dropdown value
			if DDCurrent > DDTotal:
				DDTotal = DDCurrent
		self.__maxDrawDownValue = DDTotal/peak
		return self.__maxDrawDownValue

basket = {"AAPL": 50, "GME": 150, "TSLA": 5, "AAL": 200, "AMZN": 1}
start = dt.date(2019, 1, 1)
end = dt.date(2021, 3, 1)
benchmark = "VOO"

portfolio = Portfolio(basket, start, end, benchmark)

print("Average Daily Return: " + str(round(portfolio.averageDailyReturn()*100, 3)) + "%")
print("Volatility: " + str(round(portfolio.volatility()*100, 3)) + "%")
print("Risk Ratio: " + str(round(portfolio.riskRatio(), 3)))
print("Marginal Volatility: " + str(round(portfolio.marginalVolatility("GME", 300)*100, 3)) + "%")
print("Max Draw Down: " + str(round(portfolio.maxDrawDown()*100, 3)) + "%")


from __future__ import print_function
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt

class MultiClassLogisticRegressorPurePython(object):
    def __init__(self, lr, n_iter):
        self.lr = lr
        self.n_iter = n_iter

    def fit(self, X, y):
        self.n_classes_ = len(np.unique(y))
        self.m_ = X.shape[0]
        # self.w_ = np.zeros((X.shape[1], self.n_classes_))
        self.w_ = np.random.normal(1, 0.001, ((X.shape[1], self.n_classes_)))
        self.cost_ = []

        for i in range(self.n_iter):
            z = self.net_input(X)
            assert not np.isnan(np.sum(z))
            p_y = self.softmax_fn(z)
            y_onehot = self.onehot_fn(y)
            error = (y_onehot - p_y)
            grad = (-1 / self.m_) * X.T.dot(error)
            self.w_ = self.w_ - (self.lr * grad)

            cost = (-1 / self.m_) * np.sum(y_onehot * np.log(p_y))
            self.cost_.append(cost)

        return self

    def onehot_fn(self, y):
        onehot = np.eye(self.n_classes_)[y]
        return onehot

    def net_input(self, X):
        return np.dot(X, self.w_)

    def predict(self, X):
        z = self.net_input(X)
        return np.argmax(self.softmax_fn(z), axis=1)

    def softmax_fn(self, z):
        z -= np.max(z)
        softmax = (np.exp(z).T / np.sum(np.exp(z), axis=1)).T
        return softmax

# get data
all_charts = pd.read_csv('BillboardLyricData.txt', sep='\t', encoding='utf-8')
all_charts = all_charts.dropna()
print(all_charts.shape)

# countvecotrize data
n_features_to_extract_from_text = 500
vectorizer = CountVectorizer(max_df=0.5, min_df=0.0,max_features=n_features_to_extract_from_text,stop_words='english')
X = np.asarray(vectorizer.fit_transform(all_charts.lyrics).todense())
print(X.shape)

# y to ints
class_mapping = {label:idx for idx,label in enumerate(np.unique(all_charts.chart))}
y = all_charts.chart.map(class_mapping)

# split into train and test
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

# scale
std_scaler = StandardScaler()
X_train_std = std_scaler.fit_transform(X_train)
X_test_std = std_scaler.transform(X_test)
print('X_train_std mean', X_train_std.mean())
print('X_test_std mean', X_test_std.mean())

# instantiate and fit model1
model1 = LogisticRegression(penalty='l2', C=0.3)
model1 = model1.fit(X_train_std, y_train)
# evaluate model1
train_acc = model1.score(X_train_std, y_train)
test_acc = model1.score(X_test_std, y_test)
print('Model 1')
print('Train accuracy: {}'.format(train_acc))
print('Test accuracy: {}'.format(test_acc))

# instantiate and fit model2
model2 = MultiClassLogisticRegressorPurePython(n_iter=50, lr=0.001)
model2.fit(X_train_std, y_train)
plt.plot(model2.cost_)
# plt.show()
# evaluate model2
train_ps = model2.predict(X_train_std)
train_ps = model2.predict(X_train_std)
train_acc = np.sum(train_ps == y_train) / float(len(X_train))
test_acc = np.sum(model2.predict(X_test_std) == y_test) / float(len(X_test))
print('Model 2')
print('Train accuracy: {}'.format(train_acc))
print('Test accuracy: {}'.format(test_acc))


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
# pylint: disable=E1137,E1136,E0110
import sys
import os
import time
import datetime
import collections
import argparse

import numpy as np

import logging
LOGGER = logging.getLogger("server")

import rpyc
from rpyc.utils.server import OneShotServer, ThreadedServer

from web_tool.ModelSessionKerasExample import KerasDenseFineTune
from web_tool.ModelSessionPyTorchExample import TorchFineTuning
from web_tool.Utils import setup_logging, serialize, deserialize

class MyService(rpyc.Service):

    def __init__(self, model):
        self.model = model

    def on_connect(self, conn):
        pass

    def on_disconnect(self, conn):
        pass

    def exposed_last_tile(self):
        return serialize(self.model.last_tile)

    def exposed_run(self, tile, inference_mode=False):
        tile = deserialize(tile) # need to serialize/deserialize numpy arrays
        output = self.model.run(tile, inference_mode)
        return serialize(output) # need to serialize/deserialize numpy arrays

    def exposed_retrain(self):
        return self.model.retrain()

    def exposed_add_sample_point(self, row, col, class_idx):
        return self.model.add_sample_point(row, col, class_idx)

    def exposed_undo(self):
        return self.model.undo()

    def exposed_reset(self):
        return self.model.reset()

    def exposed_save_state_to(self, directory):
        return self.model.save_state_to(directory)

    def exposed_load_state_from(self, directory):
        return self.model.load_state_from(directory)

def main():
    global MODEL
    parser = argparse.ArgumentParser(description="AI for Earth Land Cover Worker")

    parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose debugging", default=False)

    parser.add_argument("--port", action="store", type=int, help="Port we are listenning on", default=0)
    parser.add_argument("--model", action="store", dest="model",
        choices=[
            "keras_dense",
            "pytorch"
        ],
        help="Model to use", required=True
    )
    parser.add_argument("--model_fn", action="store", dest="model_fn", type=str, help="Model fn to use", default=None)
    parser.add_argument("--fine_tune_layer", action="store", dest="fine_tune_layer", type=int, help="Layer of model to fine tune", default=-2)

    parser.add_argument("--gpu", action="store", dest="gpuid", type=int, help="GPU to use", required=False)

    args = parser.parse_args(sys.argv[1:])

    # Setup logging
    log_path = os.path.join(os.getcwd(), "tmp/logs/")
    setup_logging(log_path, "worker")

    # Setup model
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = "" if args.gpuid is None else str(args.gpuid)
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    if args.model == "keras_dense":
        model = KerasDenseFineTune(args.model_fn, args.gpuid, args.fine_tune_layer)
    elif args.model == "pytorch":
        model = TorchFineTuning(args.model_fn, args.gpuid, args.fine_tune_layer)
    else:
        raise NotImplementedError("The given model type is not implemented yet.")

    t = OneShotServer(MyService(model), port=args.port)
    t.start()

if __name__ == "__main__":
    main()


from torch.utils.data import Dataset
from transformers import BertTokenizer
import torch
import logging

class WOSDataset(Dataset):
    logger = logging.getLogger(__name__)

    def __init__(self,
                 file_path: str = './data/WOS11967/',
                 tokenizer: str = 'bert-base-cased',
                 split: str = 'train'):
        super().__init__()
        assert split in {'training', 'validation', 'test'}
        self.file_path = file_path + split

        self.x = []
        self.y = []

        try:
            with open(self.file_path+'/X.txt', 'r') as feature_data_file:
                for line in feature_data_file:
                    self.x.append(line.strip())
        except FileNotFoundError:
            logging.error(
                f'### Failed to find data file: {self.file_path}X.txt')
            raise FileNotFoundError

        tag_set = set()
        try:
            with open(self.file_path+'/Y.txt', 'r') as feature_data_file:
                for line in feature_data_file:
                    self.y.append(int(line.strip()))
                    tag_set.add(int(line.strip()))
        except FileNotFoundError:
            logging.error(
                f'### Failed to find data file: {self.file_path}YL1.txt')
            raise FileNotFoundError
        self.num_labels = len(tag_set)
        assert len(self.x) == len(self.y)

    def __len__(self):
        return len(self.x)

    def __getitem__(self, idx):
        sample = {}
        sample['text'] = self.x[idx]
        sample['label'] = torch.LongTensor([int(self.y[idx])])
        return sample

    def __iter__(self):
        return iter(range(self.__len__()))


from os import getenv
from json import loads
import flask
from rockset import Client, Q

app = flask.Flask(__name__, template_folder='js')
rs = Client(api_key=getenv('RS2_TOKEN'), api_server='api.rs2.usw2.rockset.com')

def redirect(link):
    return '<script>window.location.replace("{}")</script>'.format(link)

selects = {
    'treasure': 'drops',
    'monsters': 'drops',
    'materials': 'cooking_effect, hearts_recovered',
    'equipment': 'attack, defense'
}
creatures_selects = {
    'food': 'hearts_recovered, cooking_effect',
    'others': 'drops'
}

def creatures_category():
    others = list(
        rs.sql(
            Q(
                'select id, name, description, {} from "botw-api".creatures where cooking_effect is null'.format(creatures_selects['others'])
            )
        )
    )
    foods = list(
        rs.sql(
            Q(
                'select id, name, description, {} from "botw-api".creatures where cooking_effect is not null'.format(creatures_selects['food'])
            )
        )
    )
    return {'non-food': others, 'food': foods}

def single_category(category):
    query = 'select id, name, description, common_locations, {} from "botw-api".{}'.format(selects[category], category)
    return list(
        rs.sql(Q(query))
    )

def id_name_query(target, where):

    for category in list(selects.keys()):
        print('select id, name, description, {} from "botw-api".{} where {}=\'{}\''.format(selects[category], category, where, target))
        res = list(rs.sql(Q('select id, name, description, {} from "botw-api".{} where {}=\'{}\''.format(selects[category], category, where, target.replace('\'', '\'\'')))))
        if res != []:
            return category, res[0]

    res = list(rs.sql(Q('select id, name, description, hearts_recovered, cooking_effect from "botw-api".creatures where {}=\'{}\''.format(where, target))))
    if res != []:
        if res[0]['cooking_effect'] == None:
            res = list(rs.sql(Q('select id, name, description, drops from "botw-api".creatures where {}=\'{}\''.format(where, target))))
        return 'creatures', res[0]
    return None

@app.route('/api/v1')
def all():
    category_metadata = {}
    for category in selects.keys():
        category_metadata[category] = single_category(category)
    category_metadata['creatures'] = creatures_category()
    return {'data': category_metadata}

@app.route('/api/v1/entry/<inp>')
def entry(inp):
    try:
        try:
            int(inp)
            cat, query_res = id_name_query(inp, '_id')
            query_res['category'] = cat
            return {'data': query_res}
        except ValueError:
            cat, query_res = id_name_query(inp.lower().replace('_', ' '), 'name')
            query_res['category'] = cat
            return {'data': query_res}
    except TypeError:
        return {'data': {}, 'message': 'no results'}

@app.route('/api/v1/category/treasure')
def treasure():
    return {'data': single_category('treasure')}

@app.route('/api/v1/category/monsters')
def monsters():
    return {'data': single_category('monsters')}

@app.route('/api/v1/category/materials')
def materials():
    return {'data': single_category('materials')}

@app.route('/api/v1/category/equipment')
def equipment():
    return {'data': single_category('equipment')}

@app.route('/api/v1/category/creatures')
def creatures():
    return {'data': creatures_category()}

@app.route('/issues')
@app.route('/bugs')
@app.route('/suggestions')
def issues():
    return redirect('https://github.com/gadhagod/Hyrule-Compendium-API/issues')

@app.route('/')
@app.route('/api')
def home():
    return redirect('https://github.com/gadhagod/Hyrule-Compendium-API')


# -*- coding: utf-8 -*-
# @Author: Vincent Xu
# @E-mail: wenhong0815@qq.com
# For my Graduation Design about RS
import requests
from functiontool.constant_cookies import COOKIES_V4 as cookies
XHR_HEADER_WZ = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
    'Host': 'www.zhihu.com',
    'Access-Control-Request-Method': 'GET',
    'Connection': 'keep-alive'
}

XHR_HEADER_API = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
    'Host': 'api.zhihu.com',
    'Access-Control-Request-Method': 'GET',
    'Connection': 'keep-alive'
}

def getRequest(havecookie=True):
    '''
    返回一个request.session对象
    该函数存在的意义是设置cookie
    '''
    req = requests.Session()
    if havecookie is False:
        return req
    for cookie in cookies:
        req.cookies.set(cookie['name'], cookie['value'])
    return req

tags = [101, 102, 103, 104, 105, 106, 107, 108,109, 201, 202, 203, 301, 302, 303, 304, 305]


import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup

url = input("Enter: ")
html = urllib.request.urlopen(url).read()
soup = BeautifulSoup(html,"html.parser")

span_tags = soup('span')
comment_total = []
for span in span_tags:
    comment_total.append(span.string)

print(comment_total)


import utm

class calcularCoordsFromPoint():
    def __init__(self, corner = (10.448436368184836, -75.39254156781966), lenghtSide = 300):
        self.corner = corner
        self.side = float(lenghtSide)
        self.coordGPSTwo = ()
        self.coordGPSOne = ()
        self.coordGPSThree = ()
        self.coordGPSCenter = ()

    def calculate(self):
        coordUTM = utm.from_latlon(self.corner[0], self.corner[1])
        UTMZone = coordUTM[2:]
        # print(coordUTM[0], coordUTM[1])
        coordOne = (coordUTM[0]+self.side, coordUTM[1])
        coordTwo = (coordUTM[0]+self.side, coordUTM[1]-self.side)
        coordThree = (coordUTM[0], coordUTM[1]-self.side)
        center = (coordUTM[0]+float(self.side)/2, coordUTM[1]-float(self.side)/2)
        # coordFour = (coordUTM[0]-self.side, coordUTM[1]-self.side)
        self.coordGPSOne = self.UTM2GPS(coordOne,UTMZone)
        self.coordGPSTwo = self.UTM2GPS(coordTwo,UTMZone)
        self.coordGPSThree = self.UTM2GPS(coordThree,UTMZone)
        self.coordGPSCenter = self.UTM2GPS(center,UTMZone)
        print(self.coordGPSOne[0], self.coordGPSOne[1])
        print(self.coordGPSTwo[0], self.coordGPSTwo[1])
        print(self.coordGPSThree[0], self.coordGPSThree[1])
        print(self.coordGPSCenter[0], self.coordGPSCenter[1])

    def UTM2GPS(self, cord, zone):
        return utm.to_latlon(*cord, *zone)

def main():
    calculos = calcularCoordsFromPoint()
    calculos.calculate()

if __name__ == "__main__":
    main()


# Author:     Birthday Boy
# File:       stats_analysis.py
# Project:    Pokemon Team Builder

'''
stats_analysis.py: This file performs an elementary statistical analysis on all
                   Pokemon stats in the pokedex.json file.  It finds information
                   such as stat averages and standard deviation.
'''

import json
import os
import os.path as osp
import numpy as np
from numpy.linalg import norm
from math import sqrt, exp
import matplotlib.pyplot as plt

def make_stats_matrix(pokedex):
    # creates an mx6 matrix of Pokemon stats where m is the number of Pokemon in
    # the pokedex and counts the number of mons
    num_mons = len(pokedex)
    stats_matrix = []
    for mon in pokedex:
        # make the first 5 indices the normal stats and set the sixth to
        # the speed rank, because speed doesn't matter but rank does
        stat_list = pokedex[mon]['stats'][:5]
        # TODO: change this now that num_mons and speed rank max are different
        stat_list.append((pokedex[mon]['speed_rank']) / (num_mons - 1))
        stats_matrix.append(stat_list)
    stats_matrix = np.asarray(stats_matrix)

    # fixes the stats matrix to more accurately reflect the in-game stats of
    # Pokemon at level 100 (see the PDF for a deeper explanation)
    for stat_index, row in enumerate(stats_matrix.T):
        # Change base HP to (HP * 2 + 141)
        if stat_index == 0:
            stats_matrix[:, stat_index] = row * 2 + 141
        # Change every stat except HP and Speed to (stat * 2 + 36)
        elif stat_index < 5:
            stats_matrix[:, stat_index] = row * 2 + 36

    return stats_matrix, num_mons

def get_role_data(stats_matrix):
    # this function takes in a corrected mx6 stats matrix and uses it to find
    # each Pokemon's strengths, i.e. physical/special durability,
    # physical/special offensiveness, etc.
    num_mons = max(stats_matrix.shape)
    phys_dur = stats_matrix[:, 0] * stats_matrix[:, 2]
    spec_dur = stats_matrix[:, 0] * stats_matrix[:, 4]
    phys_off = stats_matrix[:, 1] * ((stats_matrix[:, 1] * \
               sigmoid(stats_matrix[:, 5]) + 450) / (stats_matrix[:, 1] * \
               (1 - sigmoid(stats_matrix[:, 5])) + 450))
    spec_off = stats_matrix[:, 3] * ((stats_matrix[:, 3] * \
               sigmoid(stats_matrix[:, 5]) + 450) / (stats_matrix[:, 3] * \
               (1 - sigmoid(stats_matrix[:, 5])) + 450))

    phys_dur = np.reshape(phys_dur, (num_mons, 1))
    spec_dur = np.reshape(spec_dur, (num_mons, 1))
    phys_off = np.reshape(phys_off, (num_mons, 1))
    spec_off = np.reshape(spec_off, (num_mons, 1))

    # | PD | SD | PO | SO |
    stat_rankings = np.concatenate((phys_dur, spec_dur, phys_off, spec_off), \
                                    axis=1)

    # standardize the stats so the later calculations are more consistent
    stat_rankings = standard_score(stat_rankings, new_std=25)

    # find the overall rankings and then normalize them
    overall_bst_ranking = np.asarray([sum(row) for row in stat_rankings])
    overall_bst_ranking = np.reshape(overall_bst_ranking, (num_mons, 1))
    overall_bst_ranking = min_max_data(overall_bst_ranking, b=num_mons)

    # create a list for the biases
    biases = []
    for i, row in enumerate(stat_rankings):
        # subtract the max offensiveness stat from the max durability stat. A
        # more negative score means a pokemon is more offensively biased and a
        # more positive score means it is defensively biased
        off_dur_bias = (max(row[:2]) - max(row[2:]))

        # find the difference between the special stats and the physical stats.
        # A more negative score means a pokemon is specially biased and a more
        # positive score means it is physicall biased
        phys_spec_bias = row[0] + row[2] - (row[1] + row[3])
        biases.append([off_dur_bias, phys_spec_bias])

    # normalize the biases to have a mean of 0 and standard deviation of 5
    biases = standard_score(np.asarray(biases), new_std=5)

    # concatenate the biases and overall to the rankings
    stat_rankings = np.concatenate((stat_rankings, biases, \
                                    overall_bst_ranking), axis=1)

    return stat_rankings

def get_mean_std(data_matrix, samples):
    # this function takes in a matrix of mxn size and computes the standard
    # deviation and mean of the data.

    # transposes the data_matrix so that the rows are the stats and then sums
    # them and divides by the number of samples to find the average
    mean_vec = np.asarray([sum(row) / samples for row in data_matrix.T])

    # does the same thing as above but finds standard deviation instead
    std_vec = np.asarray([sqrt(norm(row - mean_vec[i])**2 / samples) for \
                         i, row in enumerate(data_matrix.T)])

    return mean_vec, std_vec

def min_max_data(data_matrix, a=1, b=100):
    # this function takes in a matrix and normalizes the data by using the
    # min-max method
    data_matrix = data_matrix.T
    for i, row in enumerate(data_matrix):
        data_matrix[i, :] = a + ((row - min(row)) * (b - a)) / \
                            (max(row) - min(row))

    return data_matrix.T

def standard_score(data_matrix, new_mean=0, new_std=1):
    # this function takes in a stats matrix and standardizes it based on
    # (X - mu) / sigma then returns the mxn matrix

    # get the mean and standard deviation
    mean_vec, std_vec = get_mean_std(data_matrix, max(data_matrix.shape))

    standard_matrix = np.asarray([(new_mean + new_std * ((row - mean_vec[i]) / \
                                  std_vec[i])) for i, row in \
                                  enumerate(data_matrix.T)])
    return standard_matrix.T

def sigmoid(x, shift=0.45, stretch=5.5):
    # this function is a shifted sigmoid function that is modified to fit into
    # [0, 1] so that it can be used on the speed ranking of each mon.
    for i in range(max(x.shape)):
        x[i] = 1 / (1 + exp(-(stretch * (x[i] - shift))))
    return x

def plot_stats(stats_matrix):
    # creates a distribution plot for all stats
    stat_names = ['HP', 'Attack', 'Defense', 'Sp. Attack', 'Sp. Defense', \
                  'Speed']

    num_bins = round(sqrt(max(stats_matrix.shape)))
    for i, stat_name in enumerate(stat_names):

        # set up the subplot
        cur = plt.subplot(3, 2, i + 1)

        # grab the current stat vector from the stats matrix
        stat_list = list(stats_matrix[:, i])

        # find the stat range for the plot
        stat_range = max(stat_list) - min(stat_list)

        # create matplotlib histogram
        cur.hist(stat_list, color = 'blue', edgecolor = 'black',
                 bins = num_bins)

        # add labels
        cur.set_title('Distribution of Pokemon {} stats'.format(stat_name))
        cur.set_xlabel('Stat range (standard deviations away from the mean)')
        cur.set_ylabel('Number of mons in range')

    plt.tight_layout()
    plt.show()

def save_results(path_string, pokedex, data_matrix):
    # takes the results from the analysis and stores it as a simple text file
    # for human readability.  Not meant to store data for later use.

    # find the longest pokemon name, used for prettier output
    max_len = 0
    for key in pokedex.keys():
        if len(key) > max_len:
            max_len = len(key)

    # open the user-specified file and write the scores output
    with open(path_string, 'w') as f_out:
        f_out.write('-' * (max_len + 31))
        f_out.write('\n')
        for i, pokemon in enumerate(pokedex):
            f_out.write('|{}\t|'.format(i + 1))
            # ddd more spaces to make the output prettier
            spaces = ' ' * (max_len - len(pokemon))
            f_out.write(' {}|\n'.format(pokemon + spaces))
            # write the scores to 6 sig figs
            f_out.write('| PD = {:.5f}\t|'.format(data_matrix[i, 0]))
            f_out.write(' SD = {:.5f}\t|'.format(data_matrix[i, 1]))
            f_out.write(' PO = {:.5f}\t|'.format(data_matrix[i, 2]))
            f_out.write(' SO = {:.5f}\t|'.format(data_matrix[i, 3]))
            f_out.write(' ODB = {:.5f}\t|'.format(data_matrix[i, 4]))
            f_out.write(' PSB = {:.5f}\t|'.format(data_matrix[i, 5]))
            f_out.write(' BSR = {:.5f}\t|\n'.format(data_matrix[i, 6]))
        f_out.write('-' * (max_len + 31))
    return

def main(do_plot=False):
    with open(osp.join(os.pardir, 'data', 'pokedex.json')) as f_in:
        pokedex = json.load(f_in)
    stats_matrix, num_mons = make_stats_matrix(pokedex)
    mean_vec, std_vec = get_mean_std(stats_matrix, num_mons)
    roles = get_role_data(stats_matrix)
    save_results('../dumps/roles.txt', pokedex, roles)
    if do_plot:
        standard_matrix = standard_score(stats_matrix)
        plot_stats(standard_matrix)

if __name__ == '__main__':
    main()


import numpy as np
import cv2

def view(video_file, name):
    cap = cv2.VideoCapture(video_file)

    if not cap.isOpened():
        cap.open(video_file)

    last_red_frame = None
    last_green_frame = None

    while True:
        ret, frame = cap.read()
        if ret:
            hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

            lower_red = np.array([0, 85, 110])
            upper_red = np.array([15, 255, 255])
            lower_violet = np.array([165, 85, 110])
            upper_violet = np.array([180, 255, 255])
            lower_green = np.array([40, 85, 110])
            upper_green = np.array([91, 255, 255])

            red_mask_orange = cv2.inRange(hsv, lower_red, upper_red)
            red_mask_violet = cv2.inRange(hsv, lower_violet, upper_violet)
            green_mask = cv2.inRange(hsv, lower_green, upper_green)
            red_mask_full = red_mask_orange + red_mask_violet

            if last_red_frame is None:
                last_red_frame = red_mask_full
            if last_green_frame is None:
                last_green_frame = green_mask

            red_dif_frame = (last_red_frame - red_mask_full)
            green_dif_frame = (green_mask - last_green_frame)

            ret, red_dif_frame = cv2.threshold(red_dif_frame, 127, 255, cv2.THRESH_BINARY)
            ret, green_dif_frame = cv2.threshold(green_dif_frame, 127, 255, cv2.THRESH_BINARY)

            last_green_frame = green_mask
            last_red_frame = red_mask_full

            res_red = cv2.bitwise_and(frame, frame, mask=red_mask_full)
            res_green = cv2.bitwise_and(frame, frame, mask=green_mask)
            res_all = res_green + res_red

            frame_rs = cv2.resize(frame, (640, 360))
            green_mask_rs = cv2.resize(green_mask, (640, 360))
            red_mask_full_rs = cv2.resize(red_mask_full, (640, 360))
            res_all_rs = cv2.resize(res_all, (640, 360))
            red_dif_frame_rs = cv2.resize(red_dif_frame, (640, 360))
            green_dif_frame_rs = cv2.resize(green_dif_frame, (640, 360))

            cv2.imshow(f'{name} original', frame_rs)
            cv2.imshow(f'{name} red color mask diff', red_dif_frame_rs)
            cv2.imshow(f'{name} green color mask diff', green_dif_frame_rs)
            # cv2.imshow(f'{name} mask_red', red_mask_full_rs)
            # cv2.imshow(f'{name} mask_green', green_mask_rs)
            cv2.imshow(f'{name} res', res_all_rs)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
        else:
            break

    cap.release()
    cv2.destroyAllWindows()


from django.db import models
from django.contrib.auth.models import *

from . import choices as app_choices

class UserPicture(models.Model):
	user = models.ForeignKey(
		User,
		on_delete=models.CASCADE
	)
	picture = models.ImageField(
		default="default_user_picture.png",
		null=True,
		blank=True,
	)

# POST #
class Post(models.Model):
	title = models.CharField(
		null=False,
		max_length=256,
		verbose_name='Titulo'
	)
	text = models.CharField(
		null=False,
		max_length=256,
		verbose_name='Texto'
	)
	status = models.CharField(
		default="DRAFT",
		choices=app_choices.post_status_choices,
		null=False,
		max_length=256,
		verbose_name="Estatus"
	)
	record_create = {}
	record_edit = {}

	comments = []

	def __str__(self):
		return str(self.title)

# POST HISTORY #
class PostRecord(models.Model):
	post = models.ForeignKey(
		Post,
		on_delete=models.CASCADE
	)
	user = models.ForeignKey(
		User,
		on_delete=models.CASCADE
	)
	date = models.DateTimeField(
		auto_now=True
	)
	action = models.CharField(
		choices=app_choices.post_actions_choices,
		null=False,
		max_length=256
	)

	def __str__(self):
		return str(self.action) + ": " + str(self.post) + " by " + str(self.user)

# RATE OF POST #
class RateOfPost(models.Model):
	post = models.ForeignKey(
		Post,
		on_delete=models.CASCADE
	)
	rate = models.IntegerField(
		null=False
	)

	def __str__(self):
		return str(self.rate)

# TAG #
class Tag(models.Model):
	name = models.CharField(
		null=False,
		max_length=256
	)

	def __str__(self):
		return str(self.name)

# TAGS OF POST #
class TagOfPost(models.Model):
	post = models.ForeignKey(
		Post,
		on_delete=models.CASCADE
	)
	tag = models.ForeignKey(
		Tag,
		on_delete=models.CASCADE
	)

	def __str__(self):
		return str(self.post) + " is of " + str(self.tag)

# COMMENT #
class PostComment(models.Model):
	post = models.ForeignKey(
		Post,
		on_delete=models.CASCADE
	)
	text = models.CharField(
		null=False,
		max_length=256,
		verbose_name='Texto'
	)
	record_create = {}
	record_edit = {}

	def __str__(self):
		return "Comment of " + str(self.post)

# COMMENT RECORD #
class PostCommentRecord(models.Model):
	post_comment = models.ForeignKey(

		PostComment,
		on_delete=models.CASCADE
	)
	user = models.ForeignKey(
		User,
		on_delete=models.CASCADE
	)
	date = models.DateTimeField(
		auto_now=True
	)
	action = models.CharField(
		choices=app_choices.comment_actions_choices,
		null=False,
		max_length=256
	)

	def __str__(self):
		return str(self.action) + ": " + str(self.post_comment) + " by " + str(self.user)


#/usr/bin/env python
#-*- encoding:utf8 -*-
# File Name:task1.py
# Author:ulin
# Time:2018年05月22日 星期二 22时41分20秒

from twisted.internet import task, defer

def logTimeout(result, timeout):
    print('Got {}, but actually timed out after {} seconds'.format(
        result, timeout))
    return result + ' (timed out)'

def main(reactor):

    d = defer.Deferred(lambda c: c.callback("Everything's OK!"))
    d.addTimeout(2, reactor, onTimeoutCancel=logTimeout)
    #d.addBoth(print)
    return d

task.react(main)


import os
from setuptools import find_packages, setup

here = os.path.abspath(os.path.dirname(__file__))

# Get the long description from the README file
with open(os.path.join(here, 'README.md')) as f:
    long_description = f.read()

setup(
    name='teams',
    version='0.1.0',
    author='Mikey Waites, Jack Saunders, Kris Winiarski',
    packages=find_packages(where='.', exclude=['contrib', 'docs', 'tests']),
)


# coding: utf-8

# In[1]:

import pygame
import sys
from pygame.locals import *

pygame.init()

WHITE = (255,255,255)
BLACK = (0,0,0)
GREEN = (0,255,0)

points = [(200,75),(300,25),(400,75),(450,25),(450, 125),(475,75),(300,125)]

size = width,height = 600,400

clock = pygame.time.Clock()
screen = pygame.display.set_mode(size)
pygame.display.set_caption('First pygame')

while True:
    for event in pygame.event.get():
        if event.type == QUIT:
            sys.exit()

    screen.fill(WHITE)

    pygame.draw.polygon(screen,GREEN, points, 0)

    pygame.display.flip()

    clock.tick(20)


from peewee import *
import datetime
from datetime import date

db = SqliteDatabase('todo.db')
db.connect()

class todo_db(Model):
    task = CharField(max_length=255)
    timestamp = DateTimeField(default=datetime.datetime.now)
    done = BooleanField(default=False)
    protected = BooleanField(default=False)

    class Meta:
        database = db

class todo_controller:

    def __init__(self):

        self.table = todo_db()
        todo_db.create_table()
        self.selected = 0

    def getall(self):
        all = self.table.select()
        # for item in all:
        #     print(item.task, " time: ", item.timestamp)

        for ind, entry in enumerate(all):

            timestamp = entry.timestamp.strftime('%d/%B/%Y')
            # if timestamp != prev_timestamp:  # same timestamps get printed only once
            print('\n')
            print(timestamp)

            print('=' * len(timestamp))
            if self.selected == ind:
                entry.save()
                print("-"*len(entry.task))
                print(">",entry.task)
                print("-"*len(entry.task))

                self.selectedTimestamp = entry.timestamp
            else:
                entry.save()
                print(entry.task)
            print("Done: ",entry.done)
            print("Protected: ",entry.protected)
        return all
            # prev_timestamp = timestamp

    def previous(self):
        all = self.table.select()
        if self.selected==0:
            self.selected = len(all) -1
        else:
            self.selected-=1
    def next(self):
        all = self.table.select()
        if self.selected>=len(all) - 1:
            self.selected = 0
        else:
            self.selected+=1
    def add_task(self):
        text = input("Enter task: ")
        protect = input("Protect task? (y/n): ")
        if protect == "y":
            self.table.create(task = text, protected = True)
        elif protect == "n":
            self.table.create(task = text, protected = False)
        else:
            print("Invalid Input")

    def modify_menu(self):
        all = self.table.select()
        for ind, entry in enumerate(all):
            if ind == self.selected:
                return True
        return False

    def cleanup(self):

        all = self.table.select()

        for task in all:

            today = datetime.date.today()
            date_now = today.today()
            date_created = date(task.timestamp.year, task.timestamp.month, task.timestamp.day)

            if task.done == True and task.protected == False and (date_now - date_created).days>=7:
                task.delete_instance()

    def quit(self):
        exit()

#second menu
    def modify_task(self):
        all = self.table.select()
        for ind, entry in enumerate(all):
            if ind == self.selected:
                print(entry.task)
                new = input("Enter modified task: ")
                entry.task = new
                entry.save()

    def toggle_done(self):
        all = self.table.select()
        for ind, entry in enumerate(all):
            if ind == self.selected:
                if entry.done == False:
                    entry.done = True
                else:
                    entry.done = False
                entry.save()

    def toggle_protected(self):
        all = self.table.select()
        for ind, entry in enumerate(all):
            if ind == self.selected:
                if entry.protected == False:
                    entry.protected = True
                else:
                    entry.protected = False
                entry.save()
    def erase(self):
        all = self.table.select()
        for ind, entry in enumerate(all):
            if ind == self.selected:
                entry.delete_instance()

    def backtomain(self):
        pass

class todo_menu:
    def __init__(self):
        self.current = None
        self.controller = todo_controller()

    def menu(self):
        while True:

            self.controller.getall()

            print("Previous/Next: p/n \n")
            print("a) Add a new task")
            print("m) Modify selected entry")
            print("c) Cleanup: delete completed, non-protected entries older than a week")
            print("q) Quit")

            self.current  = input("Action: ")

            if self.current == "a":
                self.controller.add_task()

            elif self.current == "p":
                self.controller.previous()

            elif self.current == "n":
                self.controller.next()

            elif self.current == "c":
                self.controller.cleanup()

            elif self.current == "q":
                self.controller.quit()

            elif self.current == "m":
                if self.controller.modify_menu():
                    print("m) Modify task")
                    print("d) Toggle 'DONE'")
                    print("p) Toggle 'protected'")
                    print("e) Erase entry")
                    print("q) Back to Main")

                    self.current = input("Action: ")

                    if self.current == "m":
                        self.controller.modify_task()    #change task to new text

                    elif self.current == "d":
                        self.controller.toggle_done()#toggle done

                    elif self.current == "p":
                        self.controller.toggle_protected()#toggle protected

                    elif self.current == "e":
                        self.controller.erase()#erase entry

                    elif self.current == "q":
                        self.controller.backtomain()#back to main
                else:
                    print("You have to add a task first!")

if __name__ == '__main__':
    menus = todo_menu()
    menus.menu()


from django.urls import path
from . import views

app_name = 'example_login'
urlpatterns = [
    path('login/', views.login, name='login'),
    path('login_session/', views.login_session, name='login_session'),
    path('logout_session/', views.logout_session, name='logout_session'),
    path('', views.index, name='index'),
]


import tensorflow as tf

from tensorflow.python.platform import tf_logging as logging
import DenseNet.nets.densenet as densenet
import research.slim.nets.mobilenet.mobilenet_v2 as mobilenet_v2
import research.slim.nets.inception_resnet_v2 as inception
from utils.gen_tfrec import load_batch, get_dataset, load_batch_dense, load_batch_estimator

import os
import sys
import time
import datetime

slim = tf.contrib.slim

flags = tf.app.flags
flags.DEFINE_float('gpu_p', 1.0, 'Float: allow gpu growth value to pass in config proto')
flags.DEFINE_string('dataset_dir', '/data','String: Your dataset directory')
flags.DEFINE_string('train_dir', '/artefacts', 'String: Your train directory')
flags.DEFINE_boolean('log_device_placement', True, "Whether to log device placement.")
flags.DEFINE_string('ckpt','/weights','String: Your dataset directory')
FLAGS = flags.FLAGS

#=======Dataset Informations=======#
#==================================#
dataset_dir = FLAGS.dataset_dir
train_dir = FLAGS.train_dir
summary_dir = os.path.join(train_dir , "summary")
gpu_p = FLAGS.gpu_p
#Emplacement du checkpoint file
checkpoint_dir= FLAGS.ckpt
checkpoint_file = os.path.join(checkpoint_dir, "mobilenet_v2_1.4_224.ckpt")
ckpt_state = tf.train.get_checkpoint_state(train_dir)
image_size = 224
#Nombre de classes à prédire
file_pattern = "catvsdog_%s_*.tfrecord"
file_pattern_for_counting = "catvsdog"
#Chest-X ray num_samples
#num_samples = 100908
#Fruit num_samples=29184:
#MURA num_samples = 36807
num_samples = 22500
#Création d'un dictionnaire pour reférer à chaque label
#MURA Labels
labels_to_name = {
    'negative':0,
    'positive':1
}

#==================================#
#=======Training Informations======#
#Nombre d'époques pour l'entraînement
num_epochs = 100
#State your batch size
batch_size = 16
#Learning rate information and configuration (Up to you to experiment)
initial_learning_rate = 1e-4
#Decay factor
learning_rate_decay_factor = 0.95
num_epochs_before_decay = 1
#Calculus of batches/epoch, number of steps after decay learning rate
num_batches_per_epoch = int(num_samples / batch_size)
#num_batches = num_steps for one epcoh
decay_steps = int(num_epochs_before_decay * num_batches_per_epoch)
#==================================#
#Create log_dir:
if not os.path.exists(train_dir):
    os.mkdir(os.path.join(os.getcwd(),train_dir))
if not os.path.exists(summary_dir):
    os.mkdir(os.path.join(os.getcwd(),summary_dir))
#===================================================================== Training ===========================================================================#
#Adding the graph:
#Set the verbosity to INFO level
tf.reset_default_graph()
tf.logging.set_verbosity(tf.logging.INFO)

def input_fn(mode, dataset_dir,file_pattern, file_pattern_for_counting, labels_to_name, batch_size, image_size):
    train_mode = mode==tf.estimator.ModeKeys.TRAIN
    with tf.name_scope("dataset"):
        dataset = get_dataset("train" if train_mode else "validation",
                                        dataset_dir, file_pattern=file_pattern,
                                        file_pattern_for_counting=file_pattern_for_counting,
                                        labels_to_name=labels_to_name)
    with tf.name_scope("load_data"):
        dataset = load_batch_estimator(dataset, batch_size, image_size, image_size,
                                                        shuffle=train_mode, is_training=train_mode)
    return dataset

def model_fn(features, mode):
    train_mode = mode==tf.estimator.ModeKeys.TRAIN
    tf.summary.image("images",features['image/encoded'])
    #Create the model inference
    with slim.arg_scope(mobilenet_v2.training_scope(is_training=train_mode, weight_decay=0.0005, stddev=1., bn_decay=0.99)):
            #TODO: Check mobilenet_v1 module, var "excluding
            logits, _ = mobilenet_v2.mobilenet(features['image/encoded'],depth_multiplier=1.4, num_classes = len(labels_to_name))
    excluding = ['MobilenetV2/Logits']
    variables_to_restore = slim.get_variables_to_restore(exclude=excluding)
    if (not ckpt_state) and checkpoint_file:
        variables_to_restore = variables_to_restore[1:]
        tf.train.init_from_checkpoint(checkpoint_file,
                            {v.name.split(':')[0]: v for v in variables_to_restore})

    predicted_classes = tf.argmax(logits, axis=1)
    labels = tf.cast(features["image/class/id"], tf.int32)

        #Defining losses and regulization ops:
    with tf.name_scope("loss_op"):
        total_loss = tf.losses.sparse_softmax_cross_entropy(labels = labels, logits = logits)
        #total_loss = tf.losses.get_total_loss() #obtain the regularization losses as well
    #FIXME: Replace classifier function (sigmoid / softmax)
    if mode != tf.estimator.ModeKeys.PREDICT:
        metrics = {
        'Accuracy': tf.metrics.accuracy(labels, predicted_classes, name="acc_op"),
        'Precision': tf.metrics.precision(labels, predicted_classes, name="precision_op"),
        'Recall': tf.metrics.recall(labels, predicted_classes, name="recall_op")
        }
        for name, value in metrics.items():
            tf.summary.scalar(name, value[1])

    if mode == tf.estimator.ModeKeys.TRAIN:
        #Create the global step for monitoring the learning_rate and training:
        global_step = tf.train.get_or_create_global_step()
        with tf.name_scope("learning_rate"):
            lr = tf.train.exponential_decay(learning_rate=initial_learning_rate,
                                    global_step=global_step,
                                    decay_steps=decay_steps,
                                    decay_rate = learning_rate_decay_factor,
                                    staircase=True)
            tf.summary.scalar('learning_rate', lr)
        #Define Optimizer with decay learning rate:
        with tf.name_scope("optimizer"):
            optimizer = tf.train.AdamOptimizer(learning_rate = lr)
            train_op = slim.learning.create_train_op(total_loss,optimizer,
                                                    update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS))
        return tf.estimator.EstimatorSpec(mode, loss=total_loss, train_op=train_op)

    #For Evaluation Mode
    if mode == tf.estimator.ModeKeys.EVAL:
        return tf.estimator.EstimatorSpec(mode, loss=total_loss, eval_metric_ops=metrics)

    #For Predict/Inference Mode:
    if mode == tf.estimator.ModeKeys.PREDICT:
        predictions = {
            'classes':predicted_classes,
            'probabilities': tf.nn.softmax(logits, name="Softmax")
            }
        export_outputs = {
        'prediction': tf.estimator.export.PredictOutput(predictions)
        }

        return tf.estimator.EstimatorSpec(mode,predictions=predictions,
                                            export_outputs=export_outputs)
def main():
    #Define the checkpoint state to determine initialization: from pre-trained weigths or recovery

    #Define max steps:
    max_step = num_epochs*num_batches_per_epoch
    #Define configuration non-distributed work:
    run_config = tf.estimator.RunConfig(model_dir=train_dir, save_checkpoints_steps=num_batches_per_epoch,keep_checkpoint_max=num_epochs)
    train_spec = tf.estimator.TrainSpec(input_fn=lambda:input_fn(tf.estimator.ModeKeys.TRAIN,
                                                dataset_dir,file_pattern,
                                                file_pattern_for_counting, labels_to_name,
                                                batch_size, image_size),max_steps=max_step)
    eval_spec = tf.estimator.EvalSpec(input_fn=lambda:input_fn(tf.estimator.ModeKeys.EVAL,
                                                    dataset_dir, file_pattern,
                                                    file_pattern_for_counting, labels_to_name,
                                                    batch_size,image_size))
    work = tf.estimator.Estimator(model_fn = model_fn,
                                    model_dir=train_dir,
                                    config=run_config)

    tf.estimator.train_and_evaluate(work, train_spec, eval_spec)
if __name__ == '__main__':
    main()


from django.http.response import HttpResponse
from django.shortcuts import render
from npb_stats.forms import SelectForm
from npb_stats.pitcher import pitcherSearch
from npb_stats.hitter import hitterSearch

def index(request):
    f= SelectForm()
    playerChoice = request.GET.get('playerChoice')
    teamChoice = request.GET.get('teamChoice')
    hitterTargetChoice = request.GET.get('hitterTargetChoice')
    pitcherTargetChoice = request.GET.get('pitcherTargetChoice')
    hitterDataChoice = request.GET.get('hitterDataChoice')
    pitcherDataChoice = request.GET.get('pitcherDataChoice')
    number = request.GET.get('number')

    if playerChoice==None:
        return render(request, 'form.html', {'form1': f})
    elif playerChoice=="1":
        isFarm = ""
        result = hitterSearch(isFarm, teamChoice, hitterTargetChoice, hitterDataChoice, number, request)
    elif playerChoice=="2":
        isFarm = "-farm"
        result = hitterSearch(isFarm, teamChoice, hitterTargetChoice, hitterDataChoice, number, request)
    elif playerChoice=="3":
        isFarm = ""
        result = pitcherSearch(isFarm, teamChoice, pitcherTargetChoice, pitcherDataChoice, number, request)
    elif playerChoice=="4":
        isFarm = "-farm"
        result = pitcherSearch(isFarm, teamChoice, pitcherTargetChoice, pitcherDataChoice, number, request)
    else:
        return render(request, 'error.html')

    return render(request, 'result.html', {'result':result})


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import django.utils.timezone

class Migration(migrations.Migration):

    dependencies = [
        ('api_dataservices', '0026_auto_20160205_1900'),
    ]

    operations = [

        migrations.DeleteModel(
            name='Media_Comment_clap',
        ),
        migrations.CreateModel(
            name='MediaCommentClap',
            fields=[
                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
                ('clap_type', models.IntegerField(default=0, null=True)),
                ('created_at', models.DateTimeField(default=django.utils.timezone.now)),
                ('comment', models.ForeignKey(to='api_dataservices.MediaComment')),
                ('media', models.ForeignKey(to='api_dataservices.Media')),
            ],
            options={
                'db_table': 'media_comment_clap',
            },
        ),

        migrations.AddField(
            model_name='mediacommentreply',
            name='media',
            field=models.ForeignKey(to='api_dataservices.Media', default=None),
            preserve_default=False,
        ),

        migrations.AddField(
            model_name='mediacommentclap',
            name='reply',
            field=models.ForeignKey(to='api_dataservices.MediaCommentReply', null=True),
        ),
        migrations.AddField(
            model_name='mediacommentclap',
            name='user',
            field=models.ForeignKey(to='api_dataservices.User'),
        ),
    ]


import re
import json

class ContainerModel(dict):

    def run(self, command):
        cmd_exec = self['docker_client'].exec_create(
            self['config']['name'], cmd=command)
        output = self['docker_client'].exec_start(cmd_exec['Id'])
        return output

    def get_suse_release(self):
        info = dict()
        content = self.run('cat /etc/SuSE-release')
        for line in content.split('\n'):
            match = re.match('([a-zA-Z]+)\s*=\s*(\d+)', line)
            if match:
                info.update([[match.group(1), int(match.group(2))]])
        return info

    def get_os_release(self):
        content = self.run('cat /etc/os-release')
        return dict(
            filter(
                lambda it: len(it) == 2,
                [it.replace('"', '').strip().split('=') for it in content.split('\n')]
            )
        )

class MasterModel(dict):

    def salt_key_raw(self, *args):
        command = ['salt-key']
        command.extend(args)
        command.append('--output=json')
        return self['container'].run(' '.join(command))

    def salt_key(self, *args):
        return json.loads(self.salt_key_raw(*args))

    def salt_key_accept(self, minion_id):
        return self.salt_key_raw('-a', minion_id, '-y')

    def salt(self, minion_id, salt_command, *args):
        docker_command = "salt {0} {1} --output=json -l quiet".format(
            minion_id, salt_command, ' '.join(args))
        return json.loads(self['container'].run(docker_command))

class MinionModel(dict):

    def salt_call(self, salt_command, *args):
        docker_command = "salt-call {0} {1} --output=json -l quiet".format(
            salt_command, ' '.join(args)
        )
        raw = self['container'].run(docker_command)
        try:
            out = json.loads(raw)
        except ValueError:
            raise Exception(raw)
        return out['local']


#!/usr/bin/env python
# coding: utf-8

import sys
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import scipy as scp
#import nltk
#import imageio
import sklearn

#from sklearn.model_selection import train_test_split
#from sklearn.linear_model import LinearRegression
#from sklearn.feature_selection import RFE
#from sklearn.svm import SVR
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
from sklearn.manifold import TSNE
from sklearn import metrics
from sklearn.metrics import pairwise_distances

#from timeit import default_timer as timer
#import json as js

args = sys.argv
n_args = len(args[1:])

assert (n_args==3), "Please give `$(pwd)`, data folder name, and the number from `CollectAndSortFiles.sh`."

project_folder = args[1]
theName = args[2]

project_folder = re.sub('PythonScripts$', '', project_folder)

# The `.csv`-file will be in path:
path2csv = project_folder + "{}/Data_Collection.csv".format(theName)
path2out = project_folder + "{}/outputs/".format(theName)

n_perm = int(args[3])            # Number of ways in which to combine the protonation states of the single molecules forming a neutral cluster

# # How to Cluster
print("""

Welcome to Clusterin'Clusters!
Please enjoy your coffee while we take care of your clusters...

You will find the results in {}

"We're a lighthouse, your call."

""".format(path2out))

def init_DF(csvfile):
    "Reads a xyz-file and initializes the dataframe."

    global n_atoms, atomnames

    filescsv = pd.read_csv(csvfile)                    # Read tthe .csv-file to a DataFrame
    xyz_temp = filescsv.iloc[0,2]                      # Get one .xyz-file as a reference

    xyz_test = read_xyz(xyz_temp,flatten=False)        # Read coordinates to a DF (no flattening)

    #filenames = filescsv.iloc[:,0]                    # Get the names for files as a Series (This one not used here)
    n_atoms = xyz_test.index.size                      # Number of atoms from the reference .xyz-file
    atomnames = np.array(xyz_test.index)               # Names of all the atoms as ndarray
    #np.save('{}{}/outputs/atomnames'.format(project_folder,theName),atomnames)                     # Save the atomnames for later use

    list_atomnames = []
    for i,j in enumerate(atomnames):                   # Give numbers for the atoms
        list_atomnames.append("{}_{}".format(j,i))
    arr_atomnames = np.array(list_atomnames)           # Enumerated atomnames as an ndarray

    col_names = makeMultiIndexs(arr_atomnames)         # Make column names (multi-index)

    arr_clusters,filenames = init_xyz_array(filescsv,n_atoms) # Read the actual coordinate data

    coord_df = pd.DataFrame(arr_clusters,index=filenames,columns=col_names) # Save the data in nice DF-fornat

    return coord_df

def init_xyz_array(filecsv, n_atoms):
    """Reads .xyz-file paths from .csv-file and returns
    an array with all xyz-coords in rows, and the filenames for DF indexes"""

    arr_clusters = np.zeros((1,3*n_atoms+2))             # cols of coord array should be of shape [clusters,atoms*[x,y,z]]
    filenames = np.array(filecsv.Filename)               # Get the names of the clusters from the .csv-file

    for index, row in filecsv.iterrows():                # Iterate through clusters and save their attributes
        dipole = row.Dipole                              # find the dipole value
        energy = row.Energy                              # find the energy value
        metarr = np.array([energy,dipole])               # put dipole and energy to an array
        path = row.XYZPath                               # find the path for .xyz-file
        xyzs = read_xyz(path)                            # Read coordinates to an ndarray with flatten=True (default)

        arrr = np.array(list(metarr)+list(xyzs))         # concatenate one cluster info
        arr_clusters = np.vstack((arr_clusters,arrr))    # concatenate clusters to one array
    return arr_clusters[1:,:], filenames

def read_xyz(xyzfile,flatten=True):
    """
    usage: read_xyz(xyzfile,flatten(Default=True))

    Reads a xyz-file and returns the coordinates as DF
    or flattened to a Numpy-array
    """
    xyz_temp = pd.read_csv(xyzfile,                        sep='\s+',                        skiprows=2,                        header=None,                        index_col=0,                        names=['x','y','z']                      ).sort_index(ascending=False)       # Reads coordinates from an xyz-file

    atoms3 = xyz_temp.shape[0]*3                          # multiply n of atoms with 3 dimensions

    if flatten==True:
        xyz_flat = xyz_temp.values.flatten().reshape(1,atoms3)#.astype(np.float)
        return xyz_flat[0]                                # returns coordinates as ndarray of shape [1,atoms*3]
    else:
        return xyz_temp                                   # returns coordinates as a DF

def makeMultiIndexs(arr):
    """
    Gets names of atoms as array and makes a pandas multi-indexing structure
    with x-, y- and z- columns for each atom.
    """
    top_arr = np.empty((arr.size * 3 +2)).astype(str)
    top_arr[0] = "Properties"
    top_arr[1] = "Properties"
    top_arr[2::3] = arr
    top_arr[3::3] = arr
    top_arr[4::3] = arr
    butt_arr = np.empty((arr.size * 3 +2)).astype(str)
    butt_arr[0] = "Energy"
    butt_arr[1] = "Dipole"
    butt_arr[2::3] = "x"
    butt_arr[3::3] = "y"
    butt_arr[4::3] = "z"
    col_arr = np.vstack((top_arr,butt_arr))
    #col_arr.transpose()
    col_arr_tuples = list(zip(*col_arr))
    col_names = pd.MultiIndex.from_tuples(col_arr_tuples, names=["Atom","Coord"])
    return col_names

# This one executes all functions above

clusters_df = init_DF(path2csv)

#from multi-index to one index
clusters_df.columns = ['_'.join(col) for col in clusters_df.columns.values]

energy=clusters_df['Properties_Energy']                                # %1 should take only the decimals
coord=clusters_df.drop(['Properties_Energy','Properties_Dipole'],1)    # dipoles and energy out

print("""
____________________________________________________
Finished reading the data.
You may see some pictures in outputs-folder...
""")

# ## Data exploration / EDA

# Having a look on the distributions of the energy variable, it is asymmetric, with a tail on the right.

# Plot energy distribution

clusters_df['Properties_Energy'].hist()
plt.title("Energy")
plt.axvline(np.mean(clusters_df['Properties_Energy']), color='k', linestyle='dashed', linewidth=2)
plt.savefig("{}Energy_hist.png".format(path2out))

sns.catplot( y="Properties_Energy", kind="box", data=clusters_df)
plt.savefig("{}Energy_cat.png".format(path2out))

# ## Principal Component Analysis
# The PCA is a tool used to reduce the dimensionality of the data.
# It tries to preserve as much information as possible.
# These two goals in PCA are pursued by means of a transformation of the original variables into new variables, called Principal Components (PCs).
# They consists of a linear combination of the original variables.
# PCs are uncorrelated and arranged in order of decreasing variance, so that the first PCs account for most of the variation in the sample.

# Next apply PCA to our **coordinates** variable, in order to select new variables, that explain at least 80% of the variability of our data.

pca = PCA(n_components=0.8,svd_solver='full')   # from docstring: this should include PCs so that 80% of variance is explained
PCs_coord = pca.fit_transform(coord)            # fit_transform on eri kun pelkkä transrofm
PCs_df = pd.DataFrame(data = PCs_coord, columns=["PC{}".format(i) for i in range(PCs_coord.shape[1])])
#print('Explained variance :', np.sum(pca.explained_variance_ratio_)*100, '%')    # It seems that it is 80% :D

print("""
____________________________________________________
Finished the Principal Component Analysis.
""")

#2D graph of first two PCs
#print('Explained variance PC1 :', pca.explained_variance_ratio_[0]*100, '%')
#print('Explained variance PC2 :', pca.explained_variance_ratio_[1]*100, '%')

#sns.scatterplot(x=PCs_df['PC1'], y=PCs_df['PC2'], hue=energy,  alpha=0.3)
#plt.title("2 first PCs of PCA")
#plt.ylabel('PC2')
#plt.xlabel('PC1')
#plt.show()

# > Here $\uparrow$ could be some clustering seen but the representation as cartesian coordinates makes it challenging for machine to learn

# ## K-means / Clustering
# In order to reduce the variety of our observations we are clustering our data, by using a k-mean algorith.
# This algorithm is a non-hierarchical methods of clustering, i.e. we assume the number $ k $ of groups is assumed to be fixed.
# Algorithm introduced by MacQueen (1967): each statistical unit is assigned to the cluster whose centroid (i.e. vector of means) is the closest one. The metric used to measure the distance among groups is typically the Euclidean one.

# #### Clustering with Coordinates as PCs

#clustering using PCS as variables
kmeans_PCs = KMeans(n_clusters=n_perm)
kmeans_PCs.fit(PCs_df)
PCs_kmeans = kmeans_PCs.predict(PCs_df)

plt.scatter(PCs_df['PC1'], PCs_df['PC2'], c=PCs_kmeans,s=3, cmap='viridis', alpha=0.2)
centers_PCs = kmeans_PCs.cluster_centers_
plt.scatter(centers_PCs[:, 0], centers_PCs[:, 1], c='black', s=200, alpha=0.5)
plt.ylabel('PC2')
plt.xlabel('PC1')
plt.savefig("{}clustered_PCs.png".format(path2out),dpi=300)
#plt.show()

# #### Clustering with coordinates as PCs and Energy

PCsE_df = pd.concat([PCs_df, clusters_df[['Properties_Energy']]], axis = 1)

#clustering using PCS+E as variables
kmeans_PCE = KMeans(n_clusters=n_perm)
kmeans_PCE.fit(PCsE_df)
PCE_kmeans = kmeans_PCE.predict(PCsE_df)

plt.scatter(PCsE_df['PC1'], PCsE_df['PC2'], c=PCE_kmeans,s=3, cmap='viridis', alpha=0.2)
centers_PCE = kmeans_PCE.cluster_centers_
plt.scatter(centers_PCE[:, 0], centers_PCE[:, 1], c='black', s=200, alpha=0.5)
plt.ylabel('PC2')
plt.xlabel('PC1')
plt.savefig("{}clustered_PCsE.png".format(path2out),dpi=300)
#plt.show()

# #### Reversing the PCA will produce "average clusters" which correspond to the centroids of K-means -clusters

mu = np.mean(coord,axis=0)                      # calculate the average of the xyz coordinates
pca_rev = np.dot(centers_PCs,pca.components_)   # calculate a dot product between those things
for i in range(pca_rev.shape[0]):
    pca_rev[i,:] += mu                          # add the average
                                                # this is then the coordinates of the centers

stat_clusts_DF = pd.DataFrame(data=pca_rev,columns=clusters_df.columns[2:])

# Here the average molecules are saved as `.xyz`-files to the output-folder

for index,row in stat_clusts_DF.iterrows():
    row_arr = np.array(row).reshape((n_atoms,3))
    #np.save('{}{}/outputs/coord{}'.format(project_folder,theName,index),row_arr)   # Save the average coordinates for later use
    row_arr = np.column_stack((row_arr,atomnames))
    newperm = [3,0,1,2]
    avg_xyz = pd.DataFrame(row_arr[:,newperm],columns=['atom','x','y','z'])
    avg_xyz.to_csv("{}coord{}.xyz".format(path2out,index),sep='\t',index=False,header=[str(n_atoms),'','','\n'],quotechar=' ')

print("""
____________________________________________________
Finished clustering!
There's more pictures in outputs-folder :)

Also there's new .xyz-files!!!
Those represent the Clusters
at the center of the clusters

Please remain seated
and wait for the code to run t_SNE

Actually, you may go and grab another coffee..
""")

# ## t-SNE for visualizing the clusters better

# The t-SNE produces clearer pictures of clustered material.

tsne_PCE = TSNE(n_components=2).fit_transform(PCsE_df)    # make t-SNE magic happen

tsne_PCE_cluster = pd.concat((pd.DataFrame(tsne_PCE,columns=['tsne1','tsne2']), pd.DataFrame(PCE_kmeans,columns=['groupID'])), axis = 1)
# combine t-SNE info with the info of the cluster

centers_tSNE = tsne_PCE_cluster.groupby(by='groupID').mean()
# calculate the centers as means

plt.scatter(tsne_PCE[:,0],tsne_PCE[:,1], c=PCE_kmeans,s=3, cmap='viridis', alpha=0.3)

plt.scatter(centers_tSNE.iloc[:, 0], centers_tSNE.iloc[:, 1], c='black', s=200, alpha=0.5)
plt.title("Clusters' Cluster")
plt.ylabel('t-SNE component 2')
plt.xlabel('t-SNE component 1')
plt.savefig("{}clustered_tsne.png".format(path2out),dpi=300)
#plt.show()

print("""
____________________________________________________
Finished t-SNE!
There's one more picture in outputs-folder :o

Put away your coffee and start doing work!
""")

# ***

# #### Variables in this notebook
#
# | Variable             | Usage                     | Data type   |
# | :------------------- | :-----------------------  | :-----------|
# | `project_folder`     | get the absolute path     | str         |
# | `the_name`           | name of the data folder   | str         |
# | `path2csv`           | to find the `.csv`-file   | str         |
# | `path2out`           | to put the outputs        | str         |
# | `n_perm`             | number of permutations    | int         |
# | `n_atoms`            | atoms in the clusters     | int         |
# | `atomnames`          | atomnames from coords     | DF          |
# | `clusters_df`        | Whole data                | DF          |
# | `energy`             | energies col              | Series      |
# | `coord`              | coordinate data           | DF          |
# | `coord_train`        | x for training            | array       |
# | `coord_test`         | x for testing             | array       |
# | `energy_train`       | y for training            | array       |
# | `energy_test`        | y for testing             | array       |
# | `model`              | Linear Reg                | ***         |
# | `r_sq`               | $ R^2 $-score             | float       |
# | `energy_pred`        | predicted energies        | array       |
# | `predictions`        | pred comparison           | DF          |
# | `pca`                | PCA analysis obejct       | pca         |
# | `PCs_coord`          | PCs of coordinates        | array?      |
# | `PCs_df`             | PCs of coordinates        | DF          |
# | `PCsE_df`            | PCs and energies          | DF          |
# | `kmeans_ED`          | K-means object E+dipole   | kmeans      |
# | `ED_kmeans_res`      | K-means prediction        | array       |
# | `centers_ED`         | E+dipol centers for plot  | array       |
# | `kmeans_PCs`         | K-means object PCA        | kmeans      |
# | `PCs_kmeans`         | K-means pred from PCs     | array       |
# | `centers_PCs`        | PCs centers for plot      | array       |
# | `kmeans_PCE`         | K-means object PCs+E      | kmeans      |
# | `PCE_kmeans`         | K-means pred from PCs+E   | array       |
# | `centers_PCE`        | E+PCA centers for plot    | array       |
# | `mu`                 | mean for PCA rev          | float       |
# | `pca_rev`            | avg coords from PCA rev   | array       |
# | `stat_clusts_DF`     | avg coords as DF          | DF          |
# | `row_arr`            | one coord in loop         | array       |
# | `avg_xyz`            | `row_arr` ordered to save | DF          |
# | `k_values`           | look for optimal n_clusts | array       |
# | `ch_values`          | calinski-harabasz scores  | array       |
# | `k_best`             | the best amount of clusts | int         |
# | `kbest_PCE`          | K-means PCs+E /w k_best   | kmeans      |
# | `PCE_kbest`          | pred from PCs+E (k_best)  | array       |
# | `centers_kbest`      | E+PCA centers /w k_best   | array       |
# | `tsne_PCE`           | t_SNE for PCs+E           | array       |
# | `tsne_PCE_cluster`   | t_SNE + groupID           | array       |
# | `centers_tSNE`       | E+PCA centers for t-SNE   | array       |
#


from threading import Thread
from socket import socket, gethostname, error
from time import sleep
from sqlite3 import connect

global ports, queue
ports = {}

# Elements of the queue that need to be executed and dequeued
class QueueObject(object):
	def __init__(self, conn, cur, query, data):
		self.conn = conn
		self.cur = cur
		self.query = query
		self.data = data

# The actual queue
class Queue(object):
	def __init__(self):
		self.queue = []

	def enqueue(self, qobj):
		self.queue.append(qobj)

	def dequeue(self):
		global queue
		temp = self.queue[0]
		temp.cur.execute(temp.query, temp.data)
		temp.conn.commit()
		del self.queue[0]

queue = Queue()

# One thread per port
class ListenerThread(Thread):
	def __init__(self, port):
		Thread.__init__(self)
		self.port = port
		self.socket = socket()
		self.host = gethostname()
		self.socket.bind((self.host, self.port))
		self.conn = connect("status.db")
		self.cur = self.conn.cursor()

	def update(self, status):
		global queue
		query = "update status set status = ? where port = ?"
		data = (status, self.port)
		qobj = QueueObject(self.conn, self.cur, query, data)
		queue.enqueue(qobj)

	def run(self):
		global ports
		text = ""
		self.socket.listen(1)
		while text == "":
			self.update(0)
			c, addr = self.socket.accept()
			text = c.recv(1024)
		ports[self.port] = 1
		self.update(1)