#!/usr/bin/python3
'''  the “base” of all other classes in this project '''
import json
import csv
import os

class Base:
    '''  the “base” of all other classes in this project '''

    ''' private class attribute __nb_objects (id) '''
    __nb_objects = 0

    def __init__(self, id=None):
        """Initializes the data.
        Args:
        id: assign the public instance attribute
        """
        if id is not None:
            self.id = id
        else:
            Base.__nb_objects += 1
            self.id = Base.__nb_objects

    @staticmethod
    def to_json_string(list_dictionaries):
        ''' returns the JSON string representation of list_dictionaries '''
        if list_dictionaries is None or list_dictionaries == "[]":
            return "[]"
        else:
            return json.dumps(list_dictionaries)

    @classmethod
    def save_to_file(cls, list_objs):
        '''  writes the JSON string representation of list_objs to a file '''
        filename = cls. __name__ + ".json"
        string = ""
        with open(filename, 'w') as f:
            if list_objs is None:
                f.write("[]")
            else:
                to_dict = [i.to_dictionary() for i in list_objs]
                string = cls.to_json_string(to_dict)
                f.write(string)

    @staticmethod
    def from_json_string(json_string):
        ''' returns the list of the JSON string representation json_string '''
        if json_string is None or json_string == "[]":
            return "[]"
        else:
            return json.loads(json_string)

    @classmethod
    def create(cls, **dictionary):
        ''' returns an instance with all attributes already set '''
        if dictionary is not None:
            if cls.__name__ == 'Rectangle':
                inst = cls(1, 1)
            if cls.__name__ == 'Square':
                inst = cls(1)
            inst.update(**dictionary)
        return inst

    @classmethod
    def load_from_file(cls):
        ''' returns a list of instances '''
        filename = cls. __name__ + ".json"
        with open(filename, 'r') as f:
            if not os.path.exists(filename):
                return []
            else:
                string = f.read()
                lst = cls.from_json_string(string)
                return [cls.create(**i) for i in lst]

    @classmethod
    def save_to_file_csv(cls, list_objs):
        '''  writes the CSV string representation of list_objs '''
        filename = cls. __name__ + ".csv"
        fields = ["id", "width", "height", "x", "y"]
        with open(filename, mode='w', newline="") as f:
            if list_objs is None or list_objs == "[]":
                f.write("[]")
            if cls.__name__ == 'Rectangle':
                string = csv.DictWriter(f, fieldnames=fields)
            else:
                string = csv.DictWriter(f, fieldnames=["id", "size", "x", "y"])
            for row in list_objs:
                string.writerow(row.to_dictionary())

    @classmethod
    def load_from_file_csv(cls):
        ''' returns a list of instances from csv '''
        filename = cls. __name__ + ".csv"
        string = []
        fields = ["id", "width", "height", "x", "y"]
        with open(filename, mode='r') as f:
            if not os.path.exists(filename):
                return []
            if cls.__name__ == 'Rectangle':
                string = csv.DictReader(f, fieldnames=fields)
            else:
                string = csv.DictReader(f, fieldnames=["id", "size", "x", "y"])
            lst = [dict([k, int(v)] for k, v in d.items()) for d in string]
            return [cls.create(**i) for i in lst]


# -*- coding: utf-8 -*-
"""
Created on Wed May 12 16:22:58 2021

@author: alankar
"""

import numpy as np
import h5py as hdf
import matplotlib.pyplot as plt
from matplotlib import colors
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from pycse import regress
#from scipy import interpolate
from decimal import Decimal
import random
import statsmodels.api as sm
lowess = sm.nonparametric.lowess

def fexp(number):
    (sign, digits, exponent) = Decimal(number).as_tuple()
    return len(digits) + exponent - 1

def fman(number):
    return Decimal(number).scaleb(-fexp(number)).normalize()

def RegisterCustomColormaps():
    """Adds the 'RdGn' colormap to matplotlib's database
    """
    import matplotlib as mpl
    LUTSIZE = mpl.rcParams['image.lut']

    # RdGn colormap
    RdGn =  [ [0.40392156862745099,  0.0                ,  0.12156862745098039],
              [0.69803921568627447,  0.09411764705882353,  0.16862745098039217],
              [0.83921568627450982,  0.37647058823529411,  0.30196078431372547],
              [0.95686274509803926,  0.6470588235294118 ,  0.50980392156862742],
              [0.99215686274509807,  0.85882352941176465,  0.7803921568627451 ],
              [1.0                ,  1.0                ,  1.0                ],
              #[0.96862745098039216,  0.96862745098039216,  0.96862745098039216],
              [0.85098039215686272,  0.94117647058823528,  0.82745098039215681],
              [0.65098039215686276,  0.85882352941176465,  0.62745098039215685],
              [0.35294117647058826,  0.68235294117647061,  0.38039215686274508],
              [0.10588235294117647,  0.47058823529411764,  0.21568627450980393],
              [0.0                ,  0.26666666666666666,  0.10588235294117647] ]

    plt.register_cmap(cmap=colors.LinearSegmentedColormap.from_list('RdGn', RdGn, LUTSIZE) )

RegisterCustomColormaps()

redshift = 0.5
a = 1/(1+redshift)
mp = 1.6726e-24
kB = 1.3807e-16
Msun = 2e33
yr = 365*24*60**2
kpc = 3.086e21
ckpc = kpc*a
h = 0.67
XH = 0.76

data = hdf.File('../data-gasandcloud.h5', 'r')
print(list(data.keys()))

SFR = np.array(data['SFR'])
#exclude ISM --> SFR == 0
condition = np.logical_or(SFR==0, SFR!=0)
density = np.array(data['density'])[condition]
Pmag = np.array(data['Pmag'])[condition]
temperature = np.array(data['temperature'])[condition]
data.close()

tmp = density.shape[0] #restrict the number of points to analyze for fast prototyping

color_min, color_max = 0.3, 1.8 #np.min(temperature[:tmp]), np.max(temperature[:tmp])
bincount = 80
bins = np.log10(np.logspace( np.min(np.log10(density/mp)[:tmp]), np.max(np.log10(density/mp)[:tmp]),
                            bincount))
print('Width: %.1f dex'%(bins[1]-bins[0]))

print('Plotting gamma_m')
fig, ax = plt.subplots(1, 1, figsize=(13,10))

for i in range(1,len(bins)):
    #print('Bin #: ',i)
    choose = None
    if i != (len(bins)-1):
        choose = np.logical_and(np.log10(density/mp)[:tmp]>=bins[i-1], np.log10(density/mp)[:tmp]<bins[i])
    else:
        choose = np.logical_and(np.log10(density/mp)[:tmp]>=bins[i-1], np.log10(density/mp)[:tmp]<=bins[i])

    density_supset     = (np.log10(density/mp)[:tmp])[choose]
    pressure_supset    = (np.log10(Pmag/kB)[:tmp])[choose]
    temperature_supset = (temperature[:tmp])[choose]

    #choose at max 1000 points
    points_no = min(1000, pressure_supset.shape[0])
    subsample = np.column_stack((density_supset, pressure_supset, temperature_supset))
    subsample = subsample[np.random.choice(subsample.shape[0], points_no, replace=False), :]
    density_subsample     = subsample[:,0]
    pressure_subsample    = subsample[:,1]
    temperature_subsample = subsample[:,2]

    temperature_subsample /= np.percentile(temperature_subsample, 50)
    #smooth the colors
    temperature_subsample = lowess(temperature_subsample, density_subsample , frac=0.2)[:,1]

    scat = ax.scatter(density_subsample, pressure_subsample,
                          c=temperature_subsample, cmap = 'RdGn',
                          norm=colors.Normalize(vmin=color_min, vmax=color_max), zorder=-10)
ax.set_rasterization_zorder(0)
ax.set_xlabel(r'Density [$\rm log\ m_p\ cm^{-3}$]',size=32)
ax.set_ylabel(r'Magnetic Pressure [$\rm log\ k_B\ K\ cm^{-3}$]',size=32)
ax.tick_params(axis='both', which='major', labelsize=28, direction="out", pad=5)
ax.tick_params(axis='both', which='minor', labelsize=28, direction="out", pad=5)
ax.grid()

dens_line = np.linspace(np.min(np.log10(density/mp)[:tmp]),
                        np.max(np.log10(density/mp)[:tmp]), 50)
pfit, stats = np.polynomial.Polynomial.fit(np.log10(density/mp)[:tmp],
                                           np.log10(Pmag/kB)[:tmp], 1,
                                           full=True,
                                           window=(np.min(np.log10(density/mp)[:tmp]), np.max(np.log10(density/mp)[:tmp])),
                                           domain=(np.min(np.log10(density/mp)[:tmp]), np.max(np.log10(density/mp)[:tmp])))
C, gamma_m = pfit
print(pfit)

x = np.log10(density/mp)[:tmp]
y = np.log10(Pmag/kB)[:tmp]
A = np.column_stack([x**0, x])

p, pint, se = regress(A, y, alpha=0.05)

N = 500
B = np.random.normal(p[0], se[0], N)
M = np.random.normal(p[1], se[1], N)
for b,m in zip(B, M):
    ax.plot(dens_line, m*dens_line + b, '-', color='gray', alpha=0.01)

txt = r"$\rm \gamma _m \approx %.2f \pm %.1f \times 10^{%d}$"%( p[1], fman(se[1].item()), fexp(se[1].item()) )
ax.plot(dens_line, p[0]+p[1]*dens_line, color='tab:gray', linewidth=5, linestyle='-',
        label=r'$\rm p_{mag} \propto \rho ^ {\gamma_m}$'+'\n'+txt)

cbaxes = inset_axes(ax, width="45%", height="5%", loc='lower right',
                    bbox_to_anchor=(0.08,0.07,0.9,0.8), bbox_transform=ax.transAxes)
cb = plt.colorbar(scat, fraction=0.025, orientation = 'horizontal', cax=cbaxes)#, #
                  #ticks=[1e4,1e5,1e6,1e7,1e8])
cb.set_label(r'$\rm Delta$ Temperature', rotation=0, size=26, labelpad=-86)
cb.ax.tick_params(labelsize=25, pad=5)

ax.legend(loc='upper left', prop={'size': 26}, ncol=1, shadow=False, fancybox=True, framealpha=0.4)

plt.savefig('pres-mag-subsample.png', transparent=True, bbox_inches='tight')
plt.show()


from . import setup_test
import unittest
import database_connector
import database_manager
import product_manager
import helper
import image_manager

class TestDatabaseManager(unittest.TestCase):
    @classmethod
    def setUpClass(self):
        self.database_connector = database_connector.DatabaseConnector()
        self.item_manager = database_manager.ItemManager(
            self.database_connector)
        self.tag_manager = database_manager.TagManager(
            self.database_connector)

    def setUp(self):
        setup_test.create_test_database_setup(self.database_connector)

    def tearDown(self):
        setup_test.tear_down_database_setup(self.database_connector)

    def test_add_item(self):
        # arrange
        item_id = helper.get_id()

        # act
        self.item_manager.add_item(item_id, item_id, item_id, setup_test.IMG_BRIGHTNESS)

        # assert
        self.assertEqual(setup_test.count_rows(
            self.database_connector, 'Items'), 1)

    def test_delete_item(self):
        # arrange
        item_ids = setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        self.item_manager.delete_item(item_ids[0])

        # assert
        self.assertEqual(setup_test.count_rows(
            self.database_connector, 'Items'), 3)

    def test_get_item_details(self):
        # arrange
        item_id = helper.get_id()
        self.item_manager.add_item(item_id, item_id, item_id, setup_test.IMG_BRIGHTNESS)

        # act
        item_details = self.item_manager.get_item_details(item_id)

        # assert
        self.assertEqual(len(item_details), 4)
        self.assertIn(item_id, item_details['VectorPath'])
        self.assertIn(item_id, item_details['PngPath'])
        self.assertEqual(setup_test.IMG_BRIGHTNESS, item_details['ImageBrightness'])

    def test_get_all_items(self):
        # arrange
        setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        all_items = self.item_manager.get_all_items()

        # assert
        self.assertEqual(len(all_items), 4)

    def test_get_random_items(self):
        # arrange
        setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        random_items = self.item_manager.get_random_items(2)

        # assert
        self.assertEqual(len(random_items), 2)

    def test_add_tags(self):
        # arrange
        tags = ('hello', 'hello', 'josie')

        # act
        self.tag_manager.add_tags(tags, setup_test.ITEM_ID_1)

        # assert
        self.assertEqual(setup_test.count_rows(
            self.database_connector, 'Tags'), 2)

    def test_get_unique_matches(self):
        # arrange
        setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        matches = self.tag_manager.get_unique_matches("h")
        find_unique_id = self.tag_manager.get_unique_matches(setup_test.ITEM_ID_1)

        # assert
        self.assertEqual(len(matches), 2)
        self.assertEqual(len(find_unique_id), 1)
        self.assertEqual(find_unique_id[0]['ItemId'], setup_test.ITEM_ID_1)

    def test_get_item_tags(self):
        # arrange
        item_ids = setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        item_tags = self.tag_manager.get_item_tags(item_ids[3])

        # assert
        self.assertIn("h", item_tags)
        self.assertIn("hiya", item_tags)
        self.assertIn("hello", item_tags)

    def test__collate_tags(self):
        # arrange
        item_tags = [{'Tag': 'h'}, {'Tag': 'hiya'}, {'Tag': 'hello'}]

        # act
        tags = self.tag_manager._collate_tags(item_tags)

        # assert
        self.assertTupleEqual(tags, ('h', 'hiya', 'hello'))

    def test_get_unique_tags_list(self):
        # arrange
        setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        unique_tags = self.tag_manager.get_unique_tags_list()

        # assert
        self.assertEqual(len(unique_tags), 5)

    def test_delete_all_tags_for_item(self):
        # arrange
        setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        self.tag_manager.delete_all_tags_for_item(setup_test.ITEM_ID_4)

        # assert
        self.assertEqual(setup_test.count_rows(
            self.database_connector, 'Tags'), 4)

    def test_delete_tags_for_item(self):
        # arrange
        setup_test.test_items_tags_setup(
            self.item_manager, self.tag_manager)

        # act
        self.tag_manager.delete_tags_for_item(
            ('h', 'hiya'), setup_test.ITEM_ID_4)

        # assert
        self.assertEqual(setup_test.count_rows(
            self.database_connector, 'Tags'), 5)

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/python
# Author: Siddhartha Gairola (t-sigai at microsoft dot com))

import numpy as np
import os
import io
import math
import random
import pandas as pd

import matplotlib.pyplot as plt
import librosa
import librosa.display
import argparse
import cv2
from torch.utils import data
import cmapy

import nlpaug
import nlpaug.augmenter.audio as naa

import torch
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torch.optim import Adam, lr_scheduler

import torchvision
from torchvision.transforms import Compose, Normalize, ToTensor

# load external modules
from utils import *
from image_dataloader import *
from nets.network_cnn import *

print ("Train import done successfully")

# input argmuments
parser = argparse.ArgumentParser(description='Lung Sound Classification')
parser.add_argument('--wav_file', default="", type=str, help='lung sound wav file')
parser.add_argument('--sample_rate', default=6000, type=int, help="wav file's sample rate")
parser.add_argument('--model_path', default="./model/cnn/ckpt_best_200_-1.pkl", type=str, help="saved model path")

args = parser.parse_args()

class image_loader2(Dataset):
    def __init__(self, data_dir, sample_rate = 6000, input_transform = None):

        # getting device-wise information
        self.file_to_device = {}
        device_id = 0

        #extracting the audiofilenames and the data for breathing cycle and it's label

        self.audio_data = []
        self.labels = "none"
        self.data_dir = os.path.dirname(data_dir)
        self.file_name = os.path.basename(data_dir)
        self.input_transform = input_transform

        mean, std = [0.5091, 0.1739, 0.4363], [0.2288, 0.1285, 0.0743]
        self.input_transform = Compose([ToTensor(), Normalize(mean, std)])

        # parameters for spectrograms
        self.sample_rate = sample_rate
        self.desired_length = 8
        self.n_mels = 64
        self.nfft = 256
        self.hop = self.nfft//2
        self.f_max = 2000
        self.train_flag = False

        self.dump_images = False
        self.filenames_with_labels = []

        print("Extracting Cycles...")
        self.cycle_list = []
        data = get_sound_samples_for_predict(self.file_name, self.data_dir, self.sample_rate)

        cycles_with_labels = [(d[0], d[3], self.file_name, cycle_idx, 0) for cycle_idx, d in enumerate(data[1:])]
        # cycles_with_labels = [ (잘린 오디오 파일1, 라벨1, filename, 0, 0), (잘린 오디오 파일2, 라벨2, filename, 1, 0) ]
        self.cycle_list.extend(cycles_with_labels)

        for idx, sample in enumerate(self.cycle_list) :
            output = split_and_pad(sample, self.desired_length, self.sample_rate, types = 1)
            self.audio_data.extend(output)

    def __getitem__(self, index):

        audio = self.audio_data[index][0]

        aug_prob = random.random()
        if self.train_flag and aug_prob > 0.5:
            # apply augmentation to audio
            audio = gen_augmented(audio, self.sample_rate)

            # pad incase smaller than desired length
            audio = split_and_pad([audio, 0,0,0,0], self.desired_length, self.sample_rate, types=1)[0][0]

        # roll audio sample
        roll_prob = random.random()
        if self.train_flag and roll_prob > 0.5:
            audio = rollAudio(audio)

        # convert audio signal to spectrogram
        # spectrograms resized to 3x of original size
        audio_image = cv2.cvtColor(create_mel_raw(audio, self.sample_rate, f_max=self.f_max,
            n_mels=self.n_mels, nfft=self.nfft, hop=self.hop, resz=3), cv2.COLOR_BGR2RGB)

        # blank region clipping
        audio_raw_gray = cv2.cvtColor(create_mel_raw(audio, self.sample_rate, f_max=self.f_max,
            n_mels=self.n_mels, nfft=self.nfft, hop=self.hop), cv2.COLOR_BGR2GRAY)

        audio_raw_gray[audio_raw_gray < 10] = 0
        for row in range(audio_raw_gray.shape[0]):
            black_percent = len(np.where(audio_raw_gray[row,:]==0)[0])/len(audio_raw_gray[row,:])
            if black_percent < 0.80:
                break

        if (row+1)*3 < audio_image.shape[0]:
            audio_image = audio_image[(row+1)*3:, :, :]
        audio_image = cv2.resize(audio_image, (audio_image.shape[1], self.n_mels*3), interpolation=cv2.INTER_LINEAR)

        if self.dump_images:
            save_images((audio_image, self.audio_data[index][2], self.audio_data[index][3],
                self.audio_data[index][5], self.audio_data[index][1]), self.train_flag)

        # label
        label = self.audio_data[index][1]

        # apply image transform
        if self.input_transform is not None:
            audio_image = self.input_transform(audio_image)

        print("***********************")
        print(audio_image.shape)

        return audio_image, label

    def __len__(self):
        return len(self.audio_data)

def predict() :
    dataset = image_loader2(args.wav_file, args.sample_rate)

    net = model(num_classes=4).cuda()
    checkpoint = torch.load(args.model_path)
    net.load_state_dict(checkpoint)
    net.fine_tune(block_layer=5)
    print("Pre-trained Model Loaded:", args.model_path)

    net = nn.DataParallel(net, device_ids=[0])
    net.eval()

    data_loader = DataLoader(dataset, num_workers=4, batch_size=1, shuffle=False)

    for i, (image, label) in tqdm(enumerate(data_loader)):
        image = image.cuda()
        output = net(image)
        score = output.cpu().detach().numpy()
        normal_score = score[0][0]
        rale_Score = score[0][1]
        rhonchi_score = score[0][2]
        wheezing_score = score[0][3]

        print()
        print(" [ SCORE ]")
        print("normal score : " + str(normal_score))
        print("rale score : " + str(rale_Score))
        print("rhonchi score : " + str(rhonchi_score))
        print("wheezing score : " + str(wheezing_score))
        print("-------------------------------")
        print()

        _, preds = torch.max(output, 1)

        print(" [ RESULT ]")
        if preds == 0 :
            print("normal")
        elif preds == 1 :
            print("rale")
        elif preds == 2 :
            print("rhonchi")
        elif preds == 3 :
            print("wheezing")
        else :
            print("error!")
        print()
        print()

if __name__ == "__main__":
    predict()


import random
import shapefile
import pngcanvas

def pip(x,y,poly):
    n = len(poly)
    inside = False
    p1x,p1y = poly[0]
    for i in range(n+1):
        p2x,p2y = poly[i % n]
        if y > min(p1y,p2y):
            if y <= max(p1y,p2y):
                if x <= max(p1x,p2x):
                    if p1y != p2y:
                        xinters = (y-p1y)*(p2x-p1x)/(p2y-p1y)+p1x
                    if p1x == p2x or x <= xinters:
                        inside = not inside
        p1x,p1y = p2x,p2y
    return inside

# Source shapefile
r = shapefile.Reader("GIS_CensusTract_poly.shp")

# pixel to coordinate info
xdist = r.bbox[2] - r.bbox[0]
ydist = r.bbox[3] - r.bbox[1]
iwidth = 600
iheight = 500
xratio = iwidth/xdist
yratio = iheight/ydist

c = pngcanvas.PNGCanvas(iwidth,iheight,color=[255,255,255,0xff])

# background color
c.filledRectangle(0,0,iwidth,iheight)

# Pen color
c.color = [139,137,137,0xff]

# Draw the polygons
for shape in r.shapes():
  pixels = []
  for x,y in shape.points:
    px = int(iwidth - ((r.bbox[2] - x) * xratio))
    py = int((r.bbox[3] - y) * yratio)
    pixels.append([px,py])
  c.polyline(pixels)

rnum = 0
trnum = len(r.shapeRecords())
for sr in r.shapeRecords():
  rnum += 1
  #print rnum, " of ", trnum
  density = sr.record[20]
  total = int(density / 50)
  count = 0
  minx, miny, maxx, maxy = sr.shape.bbox
  while count < total:
    x = random.uniform(minx,maxx)
    y = random.uniform(miny,maxy)
    if pip(x,y,sr.shape.points):
      count += 1
      #print " ", count, " of ", total
      px = int(iwidth - ((r.bbox[2] - x) * xratio))
      py = int((r.bbox[3] - y) * yratio)
      c.point(px,py,color=[255,0,0,0xff])

f = file("density_pure.png", "wb")
f.write(c.dump())
f.close()


# Write your crossmatch function here.
import numpy as np
import time
from astropy.coordinates import SkyCoord
from astropy import units as u

def crossmatch(cat1, cat2, max_dist):
  start = time.perf_counter()
  cat1 = SkyCoord(cat1*u.degree, frame='icrs')
  cat2 = SkyCoord(cat2*u.degree, frame='icrs')

  idx = 0
  match = []
  nomatch = []

  closest_ids, closest_dists, closest_dists3d = cat1.match_to_catalog_sky(cat2)
  closest_dists_array = closest_dists.value

  for i in closest_dists_array:
    if i < max_dist:
      match.append( (idx, closest_ids[idx], i) )

    else:
      nomatch.append(idx)

    idx += 1

  timing = time.perf_counter() - start

  return (match, nomatch, timing)

# You can use this to test your function.
# Any code inside this `if` statement will be ignored by the automarker.
if __name__ == '__main__':
  # The example in the question
  cat1 = np.array([[180, 30], [45, 10], [300, -45]])
  cat2 = np.array([[180, 32], [55, 10], [302, -44]])
  matches, no_matches, time_taken = crossmatch(cat1, cat2, 5)
  print('matches:', matches)
  print('unmatched:', no_matches)
  print('time taken:', time_taken)

  # A function to create a random catalogue of size n
  def create_cat(n):
    ras = np.random.uniform(0, 360, size=(n, 1))
    decs = np.random.uniform(-90, 90, size=(n, 1))
    return np.hstack((ras, decs))

  # Test your function on random inputs
  np.random.seed(0)
  cat1 = create_cat(10)
  cat2 = create_cat(20)
  matches, no_matches, time_taken = crossmatch(cat1, cat2, 5)
  print('matches:', matches)
  print('unmatched:', no_matches)
  print('time taken:', time_taken)


# -*- coding:UTF-8 -*-

import tushare as ts
import numpy as np
import pandas as pd
import os
import time
import datetime
import sys
import math
sys.path.append("..")
from common import base_common
from common import np_common
import tushare_data
import feature

def CreateDSFa3DSplitMTFunc(param, msg):
    param.CreateDSFa3DSplitStock(msg)

class DSFa3DDataset():
    def __init__(self,
                 o_data_source,
                 o_feature):
        self.data_source = o_data_source
        self.feature = o_feature
        self.date_list = self.data_source.date_list
        self.code_list = self.data_source.code_list
        self.date_index_map = self.data_source.date_index_map
        self.code_index_map = self.data_source.code_index_map

    def FileNameDSFa3DDataset(self):
        temp_file_name = './data/dataset/DSFa3D_%s_%s.npy' %(\
            self.data_source.setting_name, \
            self.feature.setting_name)
        return temp_file_name

    def FileNameDSFa3DSplit(self, ts_code):
        temp_file_name = './data/DSFa3D/%s_%s_%s.npy' %(\
            self.data_source.setting_name_stock, \
            self.feature.setting_name, \
            ts_code)
        return temp_file_name

    def CreateDSFa3DSplitStock(self, ts_code):
        data_unit_date_index = self.feature.index_date
        pp_data = self.data_source.LoadStockPPData(ts_code)
        if len(pp_data) == 0:
            return
        dateset_index2 = self.code_index_map[ts_code]
        for day_loop in range(0, len(pp_data)):
            data_unit = self.feature.GetDataUnit(pp_data, day_loop)
            if len(data_unit) == 0:
                continue
            temp_date = int(data_unit[data_unit_date_index])
            if temp_date > self.data_source.EndDate():
                continue
            if temp_date < self.data_source.start_date:
                break
            dateset_index1 = self.date_index_map[temp_date]
            self.dataset[dateset_index1][dateset_index2] = data_unit
        sys.stdout.write("DSFa - %-4d : %s 100%%\n" % (self.code_index_map[ts_code], ts_code))

    def CreateDSFa3DDataset(self):
        dataset_file_name = self.FileNameDSFa3DDataset()
        self.dataset = np.zeros((len(self.date_list), len(self.code_list), self.feature.unit_size))
        # base_common.ListMultiThread(CreateDSFa3DSplitMTFunc, self, 1, self.code_list)
        for ts_code in self.code_list:
            self.CreateDSFa3DSplitStock(ts_code)
        base_common.MKFileDirs(dataset_file_name)
        np.save(dataset_file_name, self.dataset)
        self.dataset = None

    def GetDSFa3DDataset(self):
        dataset_file_name = self.FileNameDSFa3DDataset()
        if not os.path.exists(dataset_file_name):
            self.CreateDSFa3DDataset()
        return np.load(dataset_file_name)

if __name__ == "__main__":
    data_source = tushare_data.DataSource(20000101, '', '', 1, 20000101, 20200106, False, False, True)
    data_source.ShowStockCodes()

    o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False)

    o_dataset = DSFa3DDataset(data_source, o_feature)
    temp_dataset = o_dataset.GetDSFa3DDataset()
    print("dataset: {}".format(temp_dataset.shape))


from django.conf import settings

def mesads_settings(request):
    """Expose settings starting with MESADS_ to templates."""
    return {
        key: getattr(settings, key)
        for key in dir(settings)
        if key.startswith("MESADS_")
    }


import unittest
import torch
import time

from src.models.computer_vision.cv_modules import BlurPool2d, Convolution1, Convolution2, Convolution3, Convolution4, Convolution5, AttentionPooling

class ResnetModulesUnitTest(unittest.TestCase):
    def test_blur_pooling_2d(self):
        x = torch.rand(64, 3, 56, 56)
        model = BlurPool2d(n_channels=3, kernel_size=3)

        start = time.time()
        out = model(x)
        end = time.time()

        message = f"BlurPool forward time: {end - start} seconds"
        print(message)

        self.assertEqual(out.shape, (64, 3, 28, 28), msg=f"Failed, out size {out.shape} should be (64, 3, 28, 28)")

    def test_attention_pooling(self):
        x = torch.rand(64, 512, 7, 7)
        model = AttentionPooling(in_size=(7,7), dim_attention=32)

        start = time.time()
        out = model(x)
        end = time.time()

        message = f"AttentionPooling forward time: {end - start} seconds"
        print(message)

        self.assertEqual(out.shape, (64, 512, 32))

    def test_residual_1(self):
        x = torch.rand(32, 3, 224, 224)
        model = Convolution1()

        start = time.time()
        out = model(x)
        end = time.time()

        message = f"Convolution1 forward time: {end - start} seconds"
        print(message)

        self.assertEqual(out.shape, (32, 64, 56, 56), msg=f"Failed, out size {out.shape} should be (32, 64, 56, 56)")

    def test_residual_2(self):
        x = torch.rand(32, 3, 56, 56)
        model = Convolution2(n_channels=3)

        start = time.time()
        out = model(x)
        end = time.time()

        message = f"Convolution2 forward time: {end - start} seconds"
        print(message)

        self.assertEqual(out.shape, (32, 3, 56, 56), msg=f"Failed, out size {out.shape} should be (32, 3, 56, 56)")

    def test_residual_3(self):
        x = torch.rand(32, 3, 56, 56)
        model = Convolution3(in_channels=3, out_channels=64)

        start = time.time()
        out = model(x)
        end = time.time()

        message = f"Convolution3 forward time: {end - start} seconds"
        print(message)

        self.assertEqual(out.shape, (32, 64, 28, 28), msg=f"Failed, out size {out.shape} should be (32, 64, 28, 28)")

    def test_residual_4(self):
        x = torch.rand(32, 3, 28, 28)
        model = Convolution4(in_channels=3, out_channels=64)

        start = time.time()
        out = model(x)
        end = time.time()

        message = f"Convolution4 forward time: {end - start} seconds"
        print(message)

        self.assertEqual(out.shape, (32, 64, 14, 14), msg=f"Failed, out size {out.shape} should be (32, 64, 14, 14)")

    def test_residual_5(self):
        x = torch.rand(32, 3, 14, 14)
        model = Convolution5(in_channels=3, out_channels=64)

        start = time.time()
        out = model(x)
        end = time.time()

        message = f"Convolution5 forward time: {end - start} seconds"
        print(message)

        self.assertEqual(out.shape, (32, 64, 7, 7), msg=f"Failed, out size {out.shape} should be (32, 64, 7, 7)")


import pkgutil

import tensorflow as tf

from tensorflow.python import tf2
from tensorflow.python.platform import test

class ModuleTest(test.TestCase):

  def testCanLoadWithPkgutil(self):
    out = pkgutil.find_loader('tensorflow')
    self.assertIsNotNone(out)

  def testDocString(self):
    self.assertIn('TensorFlow', tf.__doc__)
    self.assertNotIn('Wrapper', tf.__doc__)

  def testDict(self):
    # Check that a few modules are in __dict__.
    # pylint: disable=pointless-statement
    tf.nn
    tf.keras
    tf.image
    # pylint: enable=pointless-statement
    self.assertIn('nn', tf.__dict__)
    self.assertIn('keras', tf.__dict__)
    self.assertIn('image', tf.__dict__)

  def testName(self):
    self.assertEqual('tensorflow', tf.__name__)

  def testBuiltInName(self):
    # range is a built-in name in Python. Just checking that
    # tf.range works fine.
    if tf2.enabled():
      self.assertEqual(
          'tf.Tensor([1 2 3 4 5 6 7 8 9], shape=(9,), dtype=int32)',
          str(tf.range(1, 10)))
    else:
      self.assertEqual('Tensor("range:0", shape=(9,), dtype=int32)',
                       str(tf.range(1, 10)))

  def testCompatV2HasCompatV1(self):
    # pylint: disable=pointless-statement
    tf.compat.v2.compat.v1.keras
    # pylint: enable=pointless-statement

  def testSummaryMerged(self):
    # pylint: disable=pointless-statement
    tf.summary.image
    # If we use v2 API, check for create_file_writer,
    # otherwise check for FileWriter.
    if hasattr(tf, '_major_api_version') and tf._major_api_version == 2:
      tf.summary.create_file_writer
    else:
      tf.compat.v1.summary.FileWriter
    # pylint: enable=pointless-statement

  def testPythonModuleIsHidden(self):
    self.assertNotIn('python', dir(tf))

if __name__ == '__main__':
  test.main()


# -*- coding: utf-8 -*-

import xlrd

# excelFile = xlrd.open_workbook(r'bike.xlsx')

def read03Excel(path):
    workbook = xlrd.open_workbook(path)
    sheets = workbook.sheet_names()
    worksheet = workbook.sheet_by_name(sheets[1])
    for i in range(0, worksheet.nrows):
        row = worksheet.row(i)
        for j in range(0, len(row)):
            print(str(worksheet.cell_value(i, j)) + "|", end=' ')

        print()

read03Excel("bike.xlsx")

# 原因：xlrd升级后不支持xlsx文件读取了，通过降低版本
# pip3 uninstall xlrd
# pip3 install xlrd==1.2.0


# -*- coding: utf-8 -*-

# TODO MAKE ME A PACKAGE!

from datetime import datetime as dt
from datetime import timedelta as td
from copy import deepcopy

import airflow
from airflow import DAG
from airflow.operators.subdag_operator import SubDagOperator
from airflow.operators.dummy_operator import DummyOperator
from airflow.executors.celery_executor import CeleryExecutor

from airflow.contrib.operators.bigquery_to_gcs import BigQueryToCloudStorageOperator
from airflow.contrib.operators.dataproc_operator import DataprocClusterCreateOperator
from airflow.contrib.operators.dataproc_operator import DataProcPySparkOperator

from debussy.operators.basic import StartOperator, FinishOperator
from debussy.operators.bigquery import BigQueryTableFlushOperator, BigQueryRawToClean
from debussy.operators.datastore import DatastoreGetObjectOperator
from debussy.operators.datastore import DatastorePutObjectOperator
from debussy.operators.extractors import DatastoreExtractorTemplateOperator
from debussy.operators.extractors import JDBCExtractorTemplateCustomQueryOperator
from debussy.operators.extractors import JDBCExtractorTemplateOperator
from debussy.operators.notification import SlackOperator
from debussy.operators.python import MonthlyBranchPython

# INTERNALS BEGIN

def _create_subdag(subdag_func, parent_dag, task_id, phase, default_args,
    trigger_rule='all_success'):
    """Creates a subdag, initiated by a StartOperator and ended by a FinishOperator.
    Also, uses an internal function, passed as argument, to create the other subdag
    operators in between.
    """
    subdag = DAG(
        dag_id='{}.{}'.format(parent_dag.dag_id, task_id),
        schedule_interval=None,
        catchup=False,
        default_args=default_args
    )

    begin_task = StartOperator(phase, trigger_rule=trigger_rule, **default_args)
    subdag >> begin_task
    end_task = FinishOperator(phase, trigger_rule=trigger_rule, **default_args)
    subdag >> end_task
    subdag_func(begin_task, end_task)

    return SubDagOperator(
        subdag=subdag,
        task_id=task_id,
        dag=parent_dag,
        executor=CeleryExecutor()
    )

def _create_bigquery_simple_mirror_subdag(parent_dag, project_params, extractor_class,
    task_id_format, default_args):
    """Creates a subdag for simple mirroring, without "raw" and "clean" stages.
    """
    project_id = project_params['project_id']
    config = project_params['config']

    metadata_params_get = dict(project_params['metadata_params_get'])
    metadata_params_put = dict(project_params['metadata_params_put'])

    extractor_params = dict(project_params['extract'])
    extractor_params['project'] = project_id
    extractor_params['config'] = config
    extractor_params.update(default_args)

    table = extractor_params['table']

    def _internal(begin_task, end_task):
        datastore_get_task = DatastoreGetObjectOperator(**metadata_params_get)
        extract_task = extractor_class(**extractor_params)
        datastore_put_task = DatastorePutObjectOperator(**metadata_params_put)

        begin_task >> datastore_get_task >> extract_task >> datastore_put_task
        datastore_put_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        task_id_format.format(table),
        table,
        default_args
    )

def _create_bigquery_mirror_subdag(parent_dag, project_params, extractor_class,
    conversor_wrapper, task_id_format, default_args):
    """Creates a subdag to mirror a table from a given source to BigQuery.
    """
    project_id = project_params['project_id']
    config = project_params['config']

    metadata_params_get = dict(project_params['metadata_params_get'])
    metadata_params_put = dict(project_params['metadata_params_put'])

    extractor_params = dict(project_params['extract'])
    extractor_params['project'] = project_id
    extractor_params['config'] = config
    extractor_params.update(default_args)

    bigquery_params = dict(project_params['bigquery'])
    bigquery_pool = bigquery_params['pool']
    bigquery_table = bigquery_params['table']
    raw_table_name = bigquery_params['raw_table_name']
    clean_table_name = bigquery_params['clean_table_name']

    def _internal(begin_task, end_task):
        datastore_get_task = DatastoreGetObjectOperator(**metadata_params_get)

        bq_flush_raw_task = BigQueryTableFlushOperator(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=raw_table_name,
            pool=bigquery_pool,
            task_id='flush-raw-table-{}'.format(bigquery_table),
            **default_args
        )

        extract_task = extractor_class(**extractor_params)

        raw2clean_task = BigQueryRawToClean(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=clean_table_name,
            source_table_path=raw_table_name,
            conversor_wrapper=conversor_wrapper,
            pool=bigquery_pool,
            **default_args
        )

        datastore_put_task = DatastorePutObjectOperator(**metadata_params_put)

        begin_task >> datastore_get_task >> bq_flush_raw_task >> extract_task
        extract_task >> raw2clean_task >> datastore_put_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        task_id_format.format(bigquery_table),
        bigquery_table,
        default_args
    )

def _create_full_load_to_bigquery(parent_dag, project_params, extractor_class,
    conversor_wrapper, task_id_format, default_args):
    """Creates a subdag to perform a full load of a table from a given source to BigQuery.
    """
    project_id = project_params['project_id']
    config = project_params['config']

    extractor_params = dict(project_params['extract'])
    extractor_params['project'] = project_id
    extractor_params['config'] = config
    extractor_params.update(default_args)

    bigquery_params = dict(project_params['bigquery'])
    bigquery_pool = bigquery_params['pool']
    bigquery_table = bigquery_params['table']
    raw_table_name = bigquery_params['raw_table_name']
    clean_table_name = bigquery_params['clean_table_name']

    def _internal(begin_task, end_task):
        bq_flush_raw_task = BigQueryTableFlushOperator(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=raw_table_name,
            pool=bigquery_pool,
            task_id='flush-raw-table-{}'.format(bigquery_table),
            **default_args
        )

        extract_task = extractor_class(**extractor_params)

        bq_flush_clean_task = BigQueryTableFlushOperator(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=clean_table_name,
            task_id='flush-clean-table-{}'.format(bigquery_table),
            pool=bigquery_pool,
            **default_args
        )

        raw2clean_task = BigQueryRawToClean(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=clean_table_name,
            source_table_path=raw_table_name,
            conversor_wrapper=conversor_wrapper,
            pool=bigquery_pool,
            **default_args
        )

        begin_task >> bq_flush_raw_task >> extract_task >> bq_flush_clean_task
        bq_flush_clean_task >> raw2clean_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        task_id_format.format(bigquery_table),
        bigquery_table,
        default_args
    )

# INTERNALS END

# EXTERNALS BEGIN

def create_notification_subdag(parent_dag, env_level, phase, message, default_args):
    """Creates a subdag that notifies a given message in a slack channel.
    """
    def _internal(begin_task, end_task):
        notification_task = SlackOperator(phase, message, **default_args)
        begin_task >> notification_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        '{}_notification_subdag'.format(phase),
        phase,
        default_args
    )

def create_jdbc_bigquery_simple_mirror_subdag(parent_dag, project_params, default_args):
    return _create_bigquery_simple_mirror_subdag(
        parent_dag,
        project_params,
        JDBCExtractorTemplateCustomQueryOperator,
        'table_{}_simple_mirror_subdag',
        default_args
    )

def create_sqlserver_bigquery_mirror_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    """DEPRECATED

    Creates a subdag for incremental mirroring of a SQL Server table into BigQuery.
    """
    return _create_bigquery_mirror_subdag(
        parent_dag,
        project_params,
        JDBCExtractorTemplateOperator,
        conversor_wrapper,
        'table_{}_mirror_subdag',
        default_args
    )

def create_jdbc_bigquery_mirror_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    """Creates a subdag for incremental mirroring of a SQL Server table into BigQuery.
    """
    return _create_bigquery_mirror_subdag(
        parent_dag,
        project_params,
        JDBCExtractorTemplateOperator,
        conversor_wrapper,
        'table_{}_mirror_subdag',
        default_args
    )

def create_sqlserver_bigquery_full_load_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    """Creates a subdag for full load of a SQL Server table into BigQuery.
    """
    return _create_full_load_to_bigquery(
        parent_dag,
        project_params,
        JDBCExtractorTemplateOperator,
        conversor_wrapper,
        'table_{}_full_load_subdag',
        default_args
    )

def create_datastore_bigquery_mirror_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    return _create_bigquery_mirror_subdag(
        parent_dag,
        project_params,
        DatastoreExtractorTemplateOperator,
        conversor_wrapper,
        '{}_mirror_subdag',
        default_args
    )

def create_simple_pyspark_subdag(parent_dag, project_params, zone, region,
    storage_bucket, subnetwork_uri, main_file, extra_files, job_name, arguments,
    default_args):
    """Creates a simplified Dataproc cluster for processing a PySpark script.
    To make things as simple as possible, the cluster is fixed as a standard-2 master (no HA),
    4 standard-4 workers (2 of which preemptible), 200gb fixed disk size and GCS/BQ
    connectors (no other packages can be installed). The cluster auto-deletes with a 10min
    idle timeout, and only internal IPs. This subdag aims for a single, pure PySpark process,
    without much personalization and dependencies.
    :param parent_dag: the parent dag that this subdag will be associated
    :type parent_dag: DAG
    :param project_params: dict with project-wide configurations, such as project_id, dataset_id, etc
    :type project_params: dict
    :param zone: the GCP zone (e.g. us-central1-b) where the cluster will be created
    :type zone: str
    :param region: the GCP region (e.g. us-central1) where the cluster will be created
    :type region: str
    :param storage_bucket: the bucket where temporary data for the cluster will be stored
    :type storage_bucket: str
    :param subnetwork_uri: the full URI of the subnetwork where the Dataproc cluster will execute
        (e.g.: projects/project-id/regions/us-central1/subnetworks/subnet-id)
    :type subnetwork_uri: str
    :param main_file: the path of the file to be executed (should be in a GCS bucket that the cluster will have access to)
    :type main_file: str
    :param extra_files: the list of support files to be passed to the cluster (must be a Python file like .py or .egg)
    :type extra_files: list|str
    :param job_name: the name of the job that will be displayed in the cluster's jobs list
    :type job_name: str
    :param arguments: the list of arguments to be passed to the main file
    :type arguments: list|str
    """
    project_id = project_params['project_id']

    def _internal(begin_task, end_task):
        task_id = 'cluster-{}'.format(job_name)
        cluster_name = '{}-cluster'.format(job_name)

        cluster = DataprocClusterCreateOperator(
            task_id=task_id,
            project_id=project_id,
            zone=zone,
            cluster_name=cluster_name,
            num_workers=2,
            num_preemptible_workers=2,
            storage_bucket=storage_bucket,
            master_machine_type='n1-standard-2',
            master_disk_size=200,
            worker_machine_type='n1-standard-4',
            worker_disk_size=200,
            init_actions_uris=['gs://dataproc-initialization-actions/connectors/connectors.sh'],
            metadata={
                'gcs-connector-version': '1.9.16',
                'bigquery-connector-version': '0.13.16'
            },
            subnetwork_uri=subnetwork_uri,
            internal_ip_only=True,
            region=region,
            idle_delete_ttl=600,
            **default_args
        )

        job = DataProcPySparkOperator(
            task_id=job_name,
            main=main_file,
            arguments=arguments,
            pyfiles=extra_files,
            job_name=job_name,
            cluster_name=cluster_name,
            region=region,
            **default_args
        )

        begin_task >> cluster >> job >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        'pyspark-{}'.format(job_name),
        job_name,
        default_args,
        trigger_rule='all_done'
    )

# EXTERNALS END


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os

def graf_freq_relativa(bc, metrica, lim_eixo):
    # ajustes no processamento da técnica IDMAP (preenchimento do valor zero para rank vazio)
    bc['1'] = bc['1'].apply(np.floor)
    idmap = bc['1'].value_counts(normalize=True).sort_index() * 100
    if metrica in ['mq4']:
        idmap[7.0] = 0
        idmap[8.0] = 0
    idmap = idmap.sort_index()
    df_idmap = pd.DataFrame(idmap)
    df_idmap.rename(columns={'1': 'qtde'}, inplace = True)
    df_idmap['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])
    df_idmap['tecnica'] = np.full(8,'IDMAP')

    # ajustes no processamento da técnica IPCA (preenchimento do valor zero para rank vazio)
    bc['2'] = bc['2'].apply(np.floor)
    ipca = bc['2'].value_counts(normalize=True).sort_index() * 100
    if metrica in ['mq1','mq2','mq4']:
        ipca[1.0] = 0
    if metrica in ['mq4']:
        ipca[2.0] = 0
        ipca[4.0] = 0
    ipca = ipca.sort_index()
    df_ipca = pd.DataFrame(ipca)
    df_ipca.rename(columns={'2': 'qtde'}, inplace = True)
    df_ipca['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])#np.full(8, np.arange(1,9))
    df_ipca['tecnica'] = np.full(8,'IPCA')

    # ajustes no processamento da técnica LAMP (preenchimento do valor zero para rank vazio)
    bc['3'] = bc['3'].apply(np.floor)
    lamp = bc['3'].value_counts(normalize=True).sort_index() * 100
    if metrica in ['mq2','mq4']:
        lamp[1.0] = 0
    if metrica in ['mq4']:
        lamp[2.0] = 0
    lamp = lamp.sort_index()
    df_lamp = pd.DataFrame(lamp)
    df_lamp.rename(columns={'3': 'qtde'}, inplace = True)
    df_lamp['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])#np.full(8, np.arange(1,9))
    df_lamp['tecnica'] = np.full(8,'LAMP')

    # ajustes no processamento da técnica LMDS (preenchimento do valor zero para rank vazio)
    bc['4'] = bc['4'].apply(np.floor)
    lmds = bc['4'].value_counts(normalize=True).sort_index() * 100
    if metrica in ['mq1','mq2','mq4','rm']:
        lmds[1.0] = 0
    if metrica in ['mq4']:
        lmds[2.0] = 0
        lmds[3.0] = 0
        lmds[5.0] = 0
    lmds = lmds.sort_index()
    df_lmds = pd.DataFrame(lmds)
    df_lmds.rename(columns={'4': 'qtde'}, inplace = True)
    df_lmds['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])#np.full(8, np.arange(1,9))
    df_lmds['tecnica'] = np.full(8,'LMDS')

    # ajustes no processamento da técnica MDS (preenchimento do valor zero para rank vazio)
    bc['5'] = bc['5'].apply(np.floor)
    mds = bc['5'].value_counts(normalize=True).sort_index() * 100
    if metrica in ['mq1','mq2','mq3','mq4']:
        mds[1.0] = 0
    if metrica in ['mq4']:
        mds[7.0] = 0
        mds[8.0] = 0
    mds = mds.sort_index()
    df_mds = pd.DataFrame(mds)
    df_mds.rename(columns={'5': 'qtde'}, inplace = True)
    df_mds['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])#np.full(8, np.arange(1,9))
    df_mds['tecnica'] = np.full(8,'MDS')

    # ajustes no processamento da técnica PBC (preenchimento do valor zero para rank vazio)
    bc['6'] = bc['6'].apply(np.floor)
    pbc = bc['6'].value_counts(normalize=True).sort_index() * 100
    if metrica in ['mq1','mq3','rm']:
        pbc[1.0] = 0
    if metrica in ['mq5']:
        pbc[2.0] = 0
    pbc = pbc.sort_index()
    df_pbc = pd.DataFrame(pbc)
    df_pbc.rename(columns={'6': 'qtde'}, inplace = True)
    df_pbc['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])#np.full(8, np.arange(1,9))
    df_pbc['tecnica'] = np.full(8,'PCB')

    # ajustes no processamento da técnica TSNE (preenchimento do valor zero para rank vazio)
    bc['7'] = bc['7'].apply(np.floor)
    tsne = bc['7'].value_counts(normalize=True).sort_index() * 100
    if metrica in ['mq2','mq4']:
        tsne[8.0] = 0
    if metrica in ['mq4']:
        tsne[4.0] = 0
        tsne[5.0] = 0
        tsne[7.0] = 0
    if metrica in ['rm']:
        tsne[6.0] = 0
        tsne[7.0] = 0
    tsne = tsne.sort_index()
    df_tsne = pd.DataFrame(tsne)
    df_tsne.rename(columns={'7': 'qtde'}, inplace = True)
    df_tsne['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])#np.full(8, np.arange(1,9))
    df_tsne['tecnica'] = np.full(8,'TSNE')

    # ajustes no processamento da técnica UMAP (preenchimento do valor zero para rank vazio)
    bc['8'] = bc['8'].apply(np.floor)
    umap = bc['8'].value_counts(normalize=True).sort_index() * 100
    if metrica == 'mq1':
        umap[5.0] = 0
        umap[7.0] = 0
    umap = umap.sort_index()
    df_umap = pd.DataFrame(umap)
    df_umap.rename(columns={'8': 'qtde'}, inplace = True)
    df_umap['rank'] = np.full(8, ['1 °','2 °','3 °','4 °','5 °','6 °','7 °','8 °'])#np.full(8, np.arange(1,9))
    df_umap['tecnica'] = np.full(8,'UMAP')

    # Organizando o dataframe para utilização no grafico de barras
    df = pd.merge(df_idmap, df_ipca, how = 'outer')
    df = pd.merge(df, df_lamp, how = 'outer')
    df = pd.merge(df, df_lmds, how = 'outer')
    df = pd.merge(df, df_mds, how = 'outer')
    df = pd.merge(df, df_pbc, how = 'outer')
    df = pd.merge(df, df_tsne, how = 'outer')
    df = pd.merge(df, df_umap, how = 'outer')

    df.to_csv('%s/%s_df_freq_relativa_mqs.csv' %(os.getcwd(), metrica),index=None, header=None)

    #sns.set_context('paper')
    #sns.set_color_codes('dark')
    graf = sns.barplot(x ='tecnica', y = 'qtde', data = df,
                hue = "rank" )
    plt.legend(title = "Ranking", loc = 2, bbox_to_anchor = (1,1), frameon = False)
    plt.xlabel(metrica)
    plt.ylabel('%')
    plt.ylim(0, lim_eixo)
    graf.figure.set_size_inches(10, 6)
    plt.savefig('resulting_images/freq_relativa_%s.png' %(metrica))
    plt.show()


from yt_dlp import YoutubeDL
from src.player.observers import DownloaderObservable
from src.player.media_metadata import MediaMetadata

class SearchVideos(DownloaderObservable):
    """Searches for a specified number of videos on YouTube.
    """
    _SITE_MAPPING = {
        "YouTube":'ytsearch',
        "Bilibili":'bilisearch'
    }

    def __init__(self, chosen_site = "YouTube", limit = 5):
        super().__init__()
        self._search_key = self._SITE_MAPPING[chosen_site]
        self._limit = limit

    def search(self, query):
        ydl = YoutubeDL({
            'format': 'bestaudio',
            'ignoreerrors': 'only_download'
        })
        search_res = ydl.extract_info(f"{self._search_key}{self._limit}:{query}", download=False)['entries']
        self.notify_observers()
        return [MediaMetadata(i) for i in search_res]


import pandas as pd
import numpy as np
import scipy.io as scp
import scipy.optimize as scy
import matplotlib.pyplot as plt
import copy
import math
import seaborn as sns
import json
import csv
from unidecode import unidecode # acentos

def CorrectString(palabra):
    "Quita acentos, mayusculas, etc"
    palabra = palabra.lower() # quitar mayusculas
    palabra = palabra.replace("á", "a")
    palabra = palabra.replace("é", "e")
    palabra = palabra.replace("í", "i")
    palabra = palabra.replace("ó", "o")
    palabra = palabra.replace("ú", "u")
    palabra = palabra.replace("ñ", "n")
    palabra = palabra.replace("ç", "c")
    palabra = palabra.replace(" ", "")
    palabra = palabra.strip() # quitar espacios
    return palabra

# PARTE 1:==============================================================================
def Firebase(dic, data, variableNames):
    # Guardamos el nombre de las variables
    for j in data['User']["Este es mi usuario"]:
        if(j != "userName" and j != "zzlastGameSaved" and j != "email"):
            dic[j] = []
            variableNames = np.concatenate((variableNames, [j]))

    # Guardamos los datos
    for i in data['User']:
        # NO USAMOS "Este es mi usuario" o Jugadores con 0 partidas jugadas
        if i != "Este es mi usuario" and data['User'][i]['gamesPlayed'] != 0:
            # Recorremos atributos del user
            for j in data['User'][i]:
                # Guardamos los atributos que nos interesan
                if(j != "userName" and j != "zzlastGameSaved" and j != "email"):
                    dic[j] = np.concatenate((dic[j], [data['User'][i][j]]))

    return dic, variableNames

# PARTE 2:==============================================================================
def BigFive(dic, user_names, file_name, variableNames): #'a.csv'
    "PROCESAMOS EL CSV DE BIG FIVE"

    # Guardamos el nombre de las variables
    variableNames = np.concatenate((variableNames, ["Extraversion", "Amabilidad", "Responsabilidad", "Apertura", "Neuroticismo"]))
    dic["Extraversion"] = []
    dic["Amabilidad"] = []
    dic["Responsabilidad"] = []
    dic["Apertura"] = []
    dic["Neuroticismo"] = []

    with open(file_name, newline='', encoding='utf-8') as archivo_csv: #sin el utf-8 petaba
        lector_csv = csv.reader(archivo_csv)
        next(lector_csv)

        # RECORREMOS LOS USARIOS DE FIREBASE
        for user_name in user_names:
            encontrado = False

            # BUSCAR FILA EN CSV
            for fila in lector_csv:
                nombre = fila[1]
                nombre = CorrectString(nombre)

                if nombre == user_name:
                    encontrado = True
                    ex = fila[5]
                    ex2=fila[7]
                    ex3=fila[9]
                    ex4=fila[11]
                    ex5=fila[13]
                    #print(nombre, ex, ex2, ex3, ex4, ex5)
                    res=int(ex)
                    res2=int(ex2)
                    res3=int(ex3)
                    res4=int(ex4)
                    res5=int(ex5)

                    extraversion = res/12*100
                    amabilidad = (res2/12)*100
                    responsabilidad = (res3/12) * 100
                    apertura = (res4/12) * 100
                    neuroticismo = (res5/12)*100

                    dic["Extraversion"] = np.concatenate((dic["Extraversion"], [extraversion]))
                    dic["Amabilidad"] = np.concatenate((dic["Amabilidad"], [amabilidad]))
                    dic["Responsabilidad"] =  np.concatenate((dic["Responsabilidad"], [responsabilidad]))
                    dic["Apertura"] = np.concatenate((dic["Apertura"], [apertura]))
                    dic["Neuroticismo"] = np.concatenate((dic["Neuroticismo"], [neuroticismo]))
                    archivo_csv.seek(0) # Volvemos al principio del archivo
                    break

            if encontrado == False:
                print("ERROR: Nombre " + user_name + " no existe en Big Five.")

            archivo_csv.seek(0) # Volvemos al principio del archivo
    return dic, variableNames

# PARTE 3:==============================================================================
def Satisfaccion(dic, user_names, file_name, variableNames):
    "PROCESAMOS EL CSV DE BIG FIVE"
    # Guardamos el nombre de las variables
    variableNames = np.concatenate((variableNames, ["1ª vez jugando", "Familiaridad", "Objetivo juego", "Disfrute", "Mejor q team", "Mejor q rivals"]))
    dic["1ª vez jugando"] = []
    dic["Familiaridad"] = []
    dic["Objetivo juego"] = []
    dic["Disfrute"] = []
    dic["Mejor q team"] = []
    dic["Mejor q rivals"] = []

    with open(file_name, newline='', encoding='utf-8') as archivo_csv: #sin el utf-8 petaba
        lector_csv = csv.reader(archivo_csv)
        next(lector_csv)

        # RECORREMOS LOS USARIOS DE FIREBASE
        for user_name in user_names:
            encontrado = False

            # BUSCAR FILA EN CSV
            for fila in lector_csv:
                nombre = fila[2]
                nombre = CorrectString(nombre)

                if nombre == user_name:
                    encontrado = True
                    first_time = fila[5] #Sí, No
                    familiar_control=fila[6] # Sí, No
                    understands_goal=fila[7] # Sí, No

                    res0 = 0
                    res1 = 0
                    res2 = 0

                    desfase = 0
                    if first_time == 'Sí':
                        res0 = 1
                    else:
                        desfase = 3
                    if familiar_control != "No":
                        res1 = 1
                    if understands_goal != "No":
                        res2 = 1

                    fun=fila[8 + desfase] # 1, 2, 3, 4, 5
                    best_in_team=fila[9 + desfase] # 1, 2, 3, 4, 5
                    better_team=fila[10 + desfase] # 1, 2, 3, 4, 5

                    res3 = int(fun)
                    res4 = int(best_in_team)
                    res5 = int(better_team)

                    dic["1ª vez jugando"] = np.concatenate((dic["1ª vez jugando"], [res0]))
                    dic["Familiaridad"] = np.concatenate((dic["Familiaridad"], [res1]))
                    dic["Objetivo juego"] = np.concatenate((dic["Objetivo juego"], [res2]))
                    dic["Disfrute"] = np.concatenate((dic["Disfrute"], [res3]))
                    dic["Mejor q team"] = np.concatenate((dic["Mejor q team"], [res4]))
                    dic["Mejor q rivals"] = np.concatenate((dic["Mejor q rivals"], [res5]))

                    archivo_csv.seek(0) # Volvemos al principio del archivo
                    break

            if encontrado == False:
                print("ERROR: Nombre " + user_name + " no existe en Satisfacción.")

            archivo_csv.seek(0) # Volvemos al principio del archivo
    return dic, variableNames

# MAIN==================================================================================
def main():
    firebase_file = 'Firebase/PruebasMM3.json'
    big_five_file = 'BigFivePrueba1.csv'
    satisfaccion_file = 'Satisfaccion.csv'
    #-------------RECOGIDA DE DATOS----------------
    # df = {
    # 'x': [45, 37, 42, 35, 39],
    # 'y': [38, 31, 26, 28, 33],
    # 'z': [10, 15, 17, 21, 12]
    # }

    with open(firebase_file, encoding="utf-8") as user_files:
        file_contents = user_files.read()

    data = json.loads(file_contents)

    dic = { } # cada fila es un usuario, cada columna una variable dic[usario] = [deaths, kills, ...]
    #GUARDAR NOMBRES DE USERS===============================================0
    user_names = []

    for i in data['User']:
        # NO USAMOS "Este es mi usuario" o Jugadores con 0 partidas jugadas
        if i != "Este es mi usuario" and data['User'][i]['gamesPlayed'] != 0:
            new_name = data['User'][i]["userName"]
            new_name = CorrectString(new_name)
            user_names = np.concatenate((user_names, [new_name]))

    variableNames = [] # Guardamos los nombres de las variables de la tabla (RESPETAR EL ORDEN!!!!)

    # PARTE 1:==============================================================================
    #Añadir datos del Firebase al diccionario
    dic, variableNames = Firebase(dic, data, variableNames)

    # PARTE 2:==============================================================================
    # Añadir los datos de Big Five al dicionario
    dic, variableNames = BigFive(dic, user_names, big_five_file, variableNames)

    # PARTE 3:==============================================================================
    # Añadir los datos de satisfaccion
    dic, variableNames = Satisfaccion(dic, user_names, satisfaccion_file, variableNames)

    # form dataframe
    df = pd.DataFrame(dic, columns=variableNames)

    #-------------MATRIZ DE CORRELACION----------------
    matrix = df.corr(method = 'pearson').round(2)  # The method of correlation

    #.............VISUALIZACION-----------------------
    sns.heatmap(matrix, annot=True, vmax=1, vmin=-1, center=0, cmap='vlag')
    plt.show()

main()


import pandas
import requests

# Loads URLs into a Pandas Dataframe

dataframe= pandas.read_csv('urls.csv')
counter = 0
for url in dataframe['urls']:

    try:

        response = requests.get(url, allow_redirects=False)

        status = response.status_code

    except:

        status = "0"

    dataframe.at[counter, 'Status'] = status

    counter = counter + 1

dataframe['Status'] = dataframe['Status'].astype(int)

# Prints out terminal

print(dataframe)

#Saves in CSV

dataframe.to_csv('Statuscodes.csv')


from django.shortcuts import render
from .forms import walletForm,userForm
from .models import DataUser,walletData
from django.shortcuts import redirect

# Create your views here.
def index(request):
    uform = userForm()

    context ={
      'judul':'.demo',
      'content':'Wellcome to my web.',
      'webname': 'demo',
      'uform': uform,
    }

    if request.method == 'POST':
      nik = request.POST['nik']
      hp_number = request.POST['hp_number']
      alamat = request.POST['alamat']
      ttl = request.POST['ttl']
      pekerjaan = request.POST['pekerjaan']
      user_signature = request.POST['user_signature']

      DataUser.objects.create(
        nik = nik,
        user_name = request.user.username,
        alamat = alamat,
        ttl = ttl,
        pekerjaan = pekerjaan,
        user_signature = user_signature,
        hp_number = hp_number,
      )
      return redirect("home")

    return render(request,'userdata/index.html', context)


import discord
from discord.ext import commands
from itertools import cycle
import json
global to_delete
to_delete = []

class help_channel():

    all_channels = []

    def __init__(self, channel_id, guild_id):
        self.channel_id = channel_id
        self.Is_occupated = False
        self.all_channels.append(self)
        self.user = []
        self.guild_id = guild_id

with open(r"C:\Users\ariso\Desktop\Snake projects\aribot\cogs\data.json", "r") as f:
    data = json.load(f)
for x in data["serwers"]:
    for y in x:
        for helps in x[y]["help_channels"]:
            help_channel(helps, y)

def get_channel_for_guild(guild_id, wanted_channel):
    for x in data["serwers"]:
        for y in x:
            if int(y) == guild_id:
                return int(x[y][wanted_channel])

class coggs(commands.Cog):
    def __init__(self, client):
        self.client = client

    @commands.command(brief="brief description of your problem")
    @commands.cooldown(1, 30, commands.BucketType.user)
    async def get_help(self, ctx, *, info):
        await ctx.channel.purge(limit=1)
        await ctx.channel.send(f"{ctx.author} your request has been send")
        guild_id = ctx.guild.id
        for x in help_channel.all_channels:

            if x.Is_occupated == False and int(x.guild_id) == ctx.guild.id:
                print("Check")
                x.Is_occupated = True
                print(x.Is_occupated)
                channel = self.client.get_channel(int(x.channel_id))
                channel2 = self.client.get_channel(
                    get_channel_for_guild(guild_id, "helpers_channel"))
                await channel.set_permissions(ctx.author, send_messages=True)
                await channel.send(f"hi, {ctx.author} please describe your problem here\nSoon one of the helpers will come here to solve your problem  ")
                x.user.append(ctx.author)
                emb = discord.Embed(title=f"{ctx.author} is looking for help with:",
                                    description=f"{info}\n\n*Reat with ✅ to take request*\n*react with ❌ to close this request* ", color=0xFF0000)
                await channel2.send(embed=emb)
                break

    @commands.command(brief="mods only don't try it")
    @commands.has_permissions(manage_messages=True)
    async def close(self, ctx):
        for x in help_channel.all_channels:
            if self.client.get_channel(int(x.channel_id)) == ctx.channel:
                await ctx.channel.purge(limit=1000)
                await ctx.channel.set_permissions(x.user[0], send_messages=False)
                break

    @commands.Cog.listener()
    async def on_raw_reaction_add(self, payload):

        channel = self.client.get_channel(payload.channel_id)

        if channel == self.client.get_channel(get_channel_for_guild(payload.guild_id, "helpers_channel")):
            print("check0")
            msg = await channel.fetch_message(payload.message_id)
            msge = msg.embeds
            title = msge[0].title

            msg_splited = title.split(" ")
            for x in help_channel.all_channels:

                print(type(payload.guild_id))
                print(type(int(x.guild_id)))

                if len(x.user) != 0 and int(x.guild_id) == payload.guild_id:
                    print("check 1")
                    if str(x.user[0]) == msg_splited[0]:
                        print("check 2")
                        channel2 = self.client.get_channel(int(x.channel_id))
                        break

        if str(payload.emoji.name) == "❌":

            if channel == self.client.get_channel(get_channel_for_guild(payload.guild_id, "helpers_channel")):
                print(len(to_delete))
                for u in to_delete:
                    await channel2.set_permissions(u, send_messages=False)

                msg = await channel.fetch_message(payload.message_id)
                await msg.delete()
        elif str(payload.emoji.name) == "✅":
            if channel == self.client.get_channel(get_channel_for_guild(payload.guild_id, "helpers_channel")):
                user = await self.client.fetch_user(payload.user_id)
                print(user)
                print(type(user))
                to_delete.append(user)

                await channel2.set_permissions(user, send_messages=True)
                await channel2.send(f"It's your lucky day {user} is here to help you")

def setup(client):
    client.add_cog(coggs(client))


import logging
import os
import socket
import sys

def get_host_ip():
    try:
        ip = os.popen(
            "ifconfig | grep \"inet addr:10\" | awk '{print $2}' | awk -F \":\" '{print $2}'"
        ).read()
        ip = ip.strip("\n").strip(" ")
        if not ip:
            raise Exception()
    except Exception:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        try:
            s.connect(("9.0.0.0", 80))
        except socket.error:
            return None
        ip, port = s.getsockname()
        s.close()
    except Exception:
        host_name = socket.gethostname()
        ip = socket.gethostbyname(host_name)
    return ip

def log_to_stdout(level=logging.DEBUG):
    root = logging.getLogger()
    root.setLevel(level)

    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    )
    handler.setFormatter(formatter)
    root.addHandler(handler)


import torch
import torchvision
from torchvision.datasets import MNIST
from torch.utils.data import DataLoader
from torch_geometric.datasets import Planetoid
from torch_geometric.transforms import NormalizeFeatures
import dgl.data

def load_mnist_data(root='data', flatten=True, batch_size=32):
    if flatten:
        transform = torchvision.transforms.Compose(
            [torchvision.transforms.ToTensor(),
             torchvision.transforms.Lambda(lambda x: torch.flatten(x))]
        )
    else:
        transform = torchvision.transforms.ToTensor(),

    train_dataset = MNIST(root=root, download=True, transform=transform)
    test_dataset = MNIST(root=root, train=False,
                         download=True, transform=transform)

    train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)
    test_dataloader = DataLoader(test_dataset, batch_size=32, shuffle=False)

    return train_dataloader, test_dataloader

def load_cora():
    dataset = Planetoid(root='data/Planetoid', name='Cora', transform=NormalizeFeatures())

    print()
    print(f'Dataset: {dataset}:')
    print('======================')
    print(f'Number of graphs: {len(dataset)}')
    print(f'Number of features: {dataset.num_features}')
    print(f'Number of classes: {dataset.num_classes}')

    data = dataset[0]  # Get the first graph object.
    return data, dataset

def load_cora_dgl():
    dataset = dgl.data.CoraGraphDataset()
    g = dataset[0]
    return g, dataset


import sagemaker
import boto3

from sagemaker.huggingface.model import HuggingFaceModel

sess = sagemaker.Session()

sagemaker_session_bucket = "sentence-transformers-sagemaker-hf-150523"

try:
    role = sagemaker.get_execution_role()
except ValueError:
    iam = boto3.client("iam")
    role = iam.get_role(RoleName="AmazonSageMaker-ExecutionRole-20220224T111993")[
        "Role"
    ]["Arn"]

sess = sagemaker.Session(default_bucket=sagemaker_session_bucket)

print(f"sagemaker role arn: {role}")
print(f"sagemaker bucket: {sess.default_bucket()}")
print(f"sagemaker session region: {sess.boto_region_name}")

repository = "sentence-transformers/all-mpnet-base-v2"

model_id = repository.split("/")[-1]
s3_location = f"s3://{sess.default_bucket()}/custom_inference/{model_id}/model.tar.gz"

# create Hugging Face Model Class
huggingface_model = HuggingFaceModel(
    model_data=s3_location,
    role=role,
    transformers_version="4.28",
    pytorch_version="2.0",
    py_version="py310",
)

# deploy the endpoint endpoint
predictor = huggingface_model.deploy(
    initial_instance_count=1,
    instance_type="ml.g4dn.xlarge",
    endpoint_name="sentence-transformers-all-mpnet-base-v2",
)

# "ml.g4dn.xlarge"
# ml.g5.2xlarge


##
# This file contains functions related to working with histograms
##

import cv2

# convert to HSV get histogram on hue
def getHueHistogram(image):
    hsvImage = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    hueHist = cv2.calcHist([hsvImage], [0], None, [180], [0,180])
    return hueHist

# convert to HSV get 2d histogram on hue and saturation
def getHueSatHistogram(image):
    hsvImage = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    hueSatHist = cv2.calcHist([hsvImage], [0,1], None, [180,256], [0,180, 0,256])

    # remove low saturation which corresponds to white
    hueSatHist[:,0:50] = 0
    return hueSatHist


import random
from re import I, X

import numpy as np

from Appliance import Appliance
from Appliance_Type import appliance_type
from scipy.optimize import linprog
from Household import Household
from PriceGenerators import RTP_Price

NUMBER_OF_HOUSES = 30
HOURS_IN_DAY = 24
EV_MAX_HOURLY = 3

def get_houses_with_ev(num_houses):
    houses_with_EVs = []
    for _ in range(num_houses):
        houses_with_EVs.append(random.randint(0, 1))
    return houses_with_EVs

def main():
    houses_with_EVs = get_houses_with_ev(NUMBER_OF_HOUSES)
    number_of_appliances = sum(houses_with_EVs)

    left_eq = []
    households = []
    for i in range(NUMBER_OF_HOUSES):
        house = Household("House " + str(i))
        households.append(house)
        if (houses_with_EVs[i] == 0):
            left_eq.append([0] * HOURS_IN_DAY * NUMBER_OF_HOUSES)
            continue

        ev = Appliance("EV", 9.9, 0, EV_MAX_HOURLY, appliance_type.shiftable)
        house.add_appliance(ev)
        left_eq.append(ev.set_operation_time(
            20, 8, HOURS_IN_DAY, NUMBER_OF_HOUSES))

    # Sets daily usage of household to daily usage of it's appliances
    rhs_eq = []
    for house in households:
        appliances = house.get_appliances()
        if len(appliances) == 0:
            rhs_eq.append(0)
        else:
            rhs_eq.append(appliances[0].daily_usage)

    ineq_matrix = np.identity(HOURS_IN_DAY * NUMBER_OF_HOUSES)

    rhs_ineq = [EV_MAX_HOURLY] * HOURS_IN_DAY * NUMBER_OF_HOUSES

    rtp = RTP_Price(HOURS_IN_DAY, NUMBER_OF_HOUSES)
    objective = rtp.get_rtp_prices()

    opt = linprog(c=objective, A_ub=ineq_matrix,
                  b_ub=rhs_ineq, A_eq=left_eq, b_eq=rhs_eq, method="revised simplex")

    print('Optimized consumption for the EV each hour House # 1:', np.round(opt.x[:25], 2),
          '\nOptimized consumption for the EV each hour House # 2', np.round(
              opt.x[25:49], 2),
          '\nOptimized consumption for the EV each hour House # 3', np.round(
              opt.x[48:72], 2),
          '\nTotal Optamized cost for the nebougherhood in NOK', np.round(opt.fun, 2))

if __name__ == '__main__':
    main()


import multiprocessing

def worker(d, l):
    d[1] = '1'
    d['2'] = 2
    d[0.25] = None
    l.reverse()

if __name__ == '__main__':
    with multiprocessing.Manager() as manager:
        d = manager.dict()
        l = manager.list(range(10))

        p = multiprocessing.Process(target=worker, args=(d, l))
        p.start()
        p.join()

        print(d)
        print(l)


from pathlib import Path
import yaml
#from ruamel.yaml import YAML as RUAMEL_YAML
#import ruamel.yaml
import src.common.processing_vars as var
def write_role_file(self, d:dict):
    yml_path = 'snowflake/account/roles/' + d['ROLE_NAME_SANS_ENV'] + '.yml'

    data = self.create_parent_load_data(yml_path)

    data['OWNER'] = self.choose_value_string(data, 'OWNER', d,'OWNER', var.EMPTY_STRING)
    data['COMMENT'] = self.choose_value_string(data, 'COMMENT', d, 'COMMENT', var.EMPTY_STRING)
    #data['PARENT_ROLES'] = self.choose_value_string(data, 'PARENT_ROLES', d, 'PARENT_ROLES', var.EMPTY_STRING)
    data['CHILD_ROLES'] = self.choose_value_list(data, 'CHILD_ROLES', d, 'CHILD_ROLES', var.EMPTY_LIST)
    #print('#### RAW ####')
    #print(d['CHILD_ROLES'])
    #print('#### MODIFIED ####')
    #print(data['CHILD_ROLES'])
    if 'TAGS' in d and d['TAGS'] != []:
        if 'TAGS' in data:
            data['TAGS'] = self.choose_list_objects(d['TAGS'], data['TAGS'])
        else:
            data['TAGS'] = d['TAGS']
    else:
        data['TAGS']=var.EMPTY_STRING

    #ryaml = ruamel.yaml.YAML(typ=['rt', 'string'])
    #ryaml = RUAMEL_YAML(typ=['rt', 'string'])
    #yaml_string = ryaml.dump_to_string(data)
    yaml_string = yaml.dump(data, sort_keys=False)
    #yaml_string_converted = self.replace_jinja_ref_string(yaml_string)
    #yaml_string_converted = yaml_string_converted.replace("'"+var.EMPTY_STRING+"'",'')
    yaml_string_converted = self.convert_special_characters_back_in_file(yaml_string)
    with open(yml_path, "w+") as f:
        f.write(yaml_string_converted)


import mongomock
import pytest
from fastapi.testclient import TestClient
from main import app, get_db

@pytest.fixture()
def mock_mongo():
    mongo_client = mongomock.MongoClient()
    db = mongo_client["government_catnip"]

    def fake_db():
        return db

    app.dependency_overrides[get_db] = fake_db

    yield db

@pytest.fixture()
def db_location(mock_mongo):
    yield mock_mongo["location_information"]

@pytest.fixture()
def mock_location(db_location):
    db_location.insert_many([
        {
            "locationID": 1,
            "location": "Amphawa",
            "population": 10000,
            "numberOfVoters": 9995
        },
        {
            "locationID": 2,
            "location": "Bang Len",
            "population": 20000,
            "numberOfVoters": 18995
        }
    ])

@pytest.fixture()
def db_population(mock_mongo):
    yield mock_mongo["personal_information"]

@pytest.fixture()
def mock_population(db_population):
    db_population.insert_many([
        {
            "citizenID": 4569871354123,
            "title": "Mr.",
            "firstName": "Anuman",
            "lastName": "Saengthong",
            "sex": "Male",
            "locationID": 557,
            "rightToVote": True,
            "blacklist": False
        },
        {
            "citizenID": 1234567898765,
            "title": "Ms.",
            "firstName": "Achara",
            "lastName": "Sukkasem",
            "sex": "Female",
            "locationID": 231,
            "rightToVote": True,
            "blacklist": False
        }])

@pytest.fixture()
def db_election_result(mock_mongo):
    yield mock_mongo["election_result"]

@pytest.fixture()
def mock_election_result(db_election_result):
    db_election_result.insert_many([
        {
            "locationID": 1,
            "location": "Amphawa",
            "numberOfVoters": 9900,
            "nameOfParliament": "Jakarin Chujan",
            "nameOfParty": "Catnip"

        },
        {
            "locationID": 2,
            "location": "Bang Len",
            "numberOfVoters": 9900,
            "nameOfParliament": "Chananya Photan",
            "nameOfParty": "Catnip"
        }
    ])

@pytest.fixture()
def db_district(mock_mongo):
    yield mock_mongo["district"]

@pytest.fixture()
def mock_district(db_district):
    db_district.insert_many([
        {
            "districtID": 1,
            "district": "Amphawa",
            "districtTH": "อัมพวา",
            "province": "Samut Songkhram",
            "provinceTH": "สมุทรสงคราม",
            "region": "Centre"
        },
        {
            "districtID": 2,
            "district": "Bang Len",
            "districtTH": "บางเลน",
            "province": "Nakhon Pathom",
            "provinceTH": "นครปฐม",
            "region": "Centre"
        }
    ])

@pytest.fixture()
def db_personal_cvv(mock_mongo):
    yield mock_mongo["personal_cvv"]

@pytest.fixture()
def mock_person_cvv(db_personal_cvv):
    db_personal_cvv.insert_many([
        {
            "citizenID": 440556794906,
            "citizenCVV": "$2b$12$xFOUTVJjht350Ik7D1eQhOB1yB0/yog4etgEbn4aDk3Jf3CT65rm."
        },
        {
            "citizenID": 173518749711,
            "citizenCVV": "$2b$12$UHsLQtJ5hit1XZyEYHo9IO.61aDeJV/rdc1/mp35OwJKcISe.hN0W"
        }
    ])

@pytest.fixture()
def client():
    yield TestClient(app)


import asyncio
from pycreate2 import Create2
import time

class Roomba_Instance:
	def __init__(self, ):
		self.keepDancing = 4

	async def Dock(self):
		exit()

	async def Forward(self):
		if self.keepDancing == 0:
			await self.Dock()
		if self.keepDancing != 0:
			await self.Right()

	async def Right(self):
		self.keepDancing = keepDancing -1
		await self.Forward()

	async def Backward(self):
		await self.Right()

	async def Ready(self):

		global roomba
		sensor = roomba.get_sensors()
		print("global roomba {sensor.battery_charge} / {sensor.battery_capacity} charged")

		await self.Backward()

	async def Connect(self):

		global roomba
		port = '/dev/ttyUSB0'
		print('Port set to \'/dev/ttyUSB0\'.')
		roomba = Create2(port)
		print('global roomba object created.')
		roomba.start()
		roomba.safe()
		print('global roomba set to safe mode.')

		await self.Ready()

loop = asyncio.get_event_loop()
Process = Roomba_Instance()
loop.run_until_complete(Process.Connect())


from typing import List

class Solution:
    def productExceptSelf(self, nums: List[int]) -> List[int]:
        if not nums:
            return []

        numsLeft, numsRight, output = [], [], []

        sum = 1
        numsLeft.append(sum)
        for num in nums:
            sum *= num
            numsLeft.append(sum)
        numsLeft.append(1)

        sum = 1
        numsRight.append(sum)
        for num in reversed(nums):
            sum *= num
            numsRight.append(sum)
        numsRight.append(1)
        numsRight.reverse()

        for i in range(0, len(nums)):
            output.append(numsLeft[i - 1] * numsRight[i + 1])

        return output

sln = Solution()
print(sln.productExceptSelf([1,2,3,4]))


# This script computes the halo bispectrum in real- and redshift-space. It takes as input
# the first and last number of the wanted realizations, the cosmology and the snapnum
# In redshift-space it computes the bispectrum along the 3 different axes.
import argparse
from mpi4py import MPI
import numpy as np
import sys,os
import readgadget,readfof
import redshift_space_library as RSL
import Pk_library as PKL
import MAS_library as MASL

###### MPI DEFINITIONS ######
comm   = MPI.COMM_WORLD
nprocs = comm.Get_size()
myrank = comm.Get_rank()

# read the first and last realization to identify voids
parser = argparse.ArgumentParser(description="This script computes the bispectrum")
parser.add_argument("first",   help="first realization number", type=int)
parser.add_argument("last",    help="last  realization number", type=int)
parser.add_argument("cosmo",   help="folder with the realizations")
parser.add_argument("snapnum", help="snapshot number",          type=int)
args = parser.parse_args()
first, last, cosmo, snapnum = args.first, args.last, args.cosmo, args.snapnum

# This routine computes and saves the bispectrum
def find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
            fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles):

    if os.path.exists(fpk):  return 0

    # read header
    head     = readgadget.header(snapdir)
    BoxSize  = head.boxsize/1e3  #Mpc/h
    Omega_m  = head.omega_m
    Omega_l  = head.omega_l
    redshift = head.redshift
    Hubble   = 100.0*np.sqrt(Omega_m*(1.0+redshift)**3+Omega_l)#km/s/(Mpc/h)
    h        = head.hubble

    # read halo catalogue
    FoF   = readfof.FoF_catalog(folder, snapnum, long_ids=False,
                                swap=False, SFR=False, read_IDs=False)
    pos_h = FoF.GroupPos/1e3            #Mpc/h
    mass  = FoF.GroupMass*1e10          #Msun/h
    vel_h = FoF.GroupVel*(1.0+redshift) #km/s
    if fixed_Mmin:
        indexes = np.where(mass>Mmin)[0]
        pos_h = pos_h[indexes];  vel_h = vel_h[indexes];  del indexes
    else:
        indexes = np.argsort(mass)[-Nhalos:] #take the Nhalos most massive halos
        pos_h = pos_h[indexes];  vel_h = vel_h[indexes];  del indexes

    # move halos to redshift-space
    if do_RSD:  RSL.pos_redshift_space(pos_h, vel_h, BoxSize, Hubble, redshift, axis)

    # calculate Pk
    delta = np.zeros((grid,grid,grid), dtype=np.float32)
    MASL.MA(pos_h, delta, BoxSize, MAS)
    delta /= np.mean(delta, dtype=np.float64);  delta -= 1.0
    Pk = PKL.Pk(delta, BoxSize, axis, MAS, threads)

    # save results to file
    hdr = ('Nhalos=%i BoxSize=%.3f'%(pos_h.shape[0],BoxSize))
    if save_multipoles:
        np.savetxt(fpk, np.transpose([Pk.k3D, Pk.Pk[:,0], Pk.Pk[:,1], Pk.Pk[:,2]]),
                   delimiter='\t', header=hdr)
    else:
        np.savetxt(fpk, np.transpose([Pk.k3D, Pk.Pk[:,0]]),
                   delimiter='\t', header=hdr)

root = '/simons/scratch/fvillaescusa/pdf_information'
##################################### INPUT #########################################
# Pk parameters
grid    = 1024
MAS     = 'CIC'
threads = 2

# halo parameters
fixed_Mmin = True   #whether fix Mmin or nbar
Mmin       = 3.2e13 #Msun/h; fixed Mmin
Nhalos     = 150000 #also consider the Nhalos more massive of the sim (nbar fixed)

# output folder name
root_out = '/simons/scratch/fvillaescusa/pdf_information/Pk/halos_Mmin_3.2e13/'
#####################################################################################

# find the redshift
z = {4:0, 3:0.5, 2:1, 1:2, 0:3}[snapnum]

# create output folder if it does not exist
if myrank==0:
    if not(os.path.exists(root_out+cosmo)):  os.system('mkdir %s/%s/'%(root_out,cosmo))

# get the realizations each cpu works on
numbers = np.where(np.arange(args.first, args.last)%nprocs==myrank)[0]
numbers = np.arange(args.first, args.last)[numbers]

######## standard simulations #########
for i in numbers:

    folder  = '%s/Halos/%s/%d'%(root,cosmo,i) #folder with the halo catalogue
    if not(os.path.exists(folder)):  continue
    snapdir = '%s/%s/%d/snapdir_%03d/snap_%03d'%(root,cosmo,i,snapnum,snapnum)

    # real-space
    fpk = '%s/%s/Pk_%d_z=%s.txt'%(root_out,cosmo,i,z)
    do_RSD, axis, save_multipoles = False, 0, False
    find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
            fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)

    # redshift-space
    for axis in [0,1,2]:
        fpk = '%s/%s/Pk_RS%d_%d_z=%s.txt'%(root_out,cosmo,axis,i,z)
        do_RSD, save_multipoles = True, True
        find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
                fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)

###### paired fixed realizations ######
for i in numbers:

    for pair in [0,1]:
        folder  = '%s/Halos/%s/NCV_%d_%d'%(root,cosmo,pair,i) #halo catalogue folder
        if not(os.path.exists(folder)):  continue
        snapdir = '%s/%s/NCV_%d_%d/snapdir_%03d/snap_%03d'\
                  %(root,cosmo,pair,i,snapnum,snapnum)

        # real-space
        fpk = '%s/%s/Pk_NCV_%d_%d_z=%s.txt'%(root_out,cosmo,pair,i,z)
        do_RSD, axis, save_multipoles = False, 0, False
        find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
                fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)

        # redshift-space
        for axis in [0,1,2]:
            fpk = '%s/%s/Pk_RS%d_NCV_%d_%d_z=%s.txt'%(root_out,cosmo,axis,pair,i,z)
            do_RSD, save_multipoles = True, True
            find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
                    fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)


import os
import sys
import argparse

dirname = os.path.dirname(__file__)
path = os.path.join(dirname, './../')
sys.path.append(path)

from Prepare_Data import pickle_load, prepare_CIFAR10, prepare_CIFAR100, prepare_SVHN
from Prepare_Model import prepare_GBP_End2End, prepare_Resnet

import numpy as np
import tensorflow as tf
import keras

from cleverhans.model import CallableModelWrapper
from cleverhans.utils_tf import model_eval
from cleverhans.attacks import\
    FastGradientMethod,\
    CarliniWagnerL2,\
    DeepFool,\
    ElasticNetMethod,\
    FastFeatureAdversaries,\
    LBFGS,\
    MadryEtAl,\
    MomentumIterativeMethod,\
    VirtualAdversarialMethod,\
    SaliencyMapMethod,\
    SPSA,\
    vatm

def main(type="Resnet", dataset="CIFAR10", attack_type="FGM"):

    size = 256
    eval_params = {'batch_size': 128}

    ############################################# Prepare the Data #####################################################

    if dataset == 'CIFAR10':
        (_, _), (x_test, y_test) = prepare_CIFAR10()
        num_classes = 10
        input_dim = 32
    elif dataset == 'CIFAR100':
        (_, _), (x_test, y_test) = prepare_CIFAR100()
        num_classes = 100
        input_dim = 32
    else:
        (_, _), (x_test, y_test) = prepare_SVHN("./Data/")
        num_classes = 10
        input_dim = 32

    x_test = x_test / 255.
    y_test = keras.utils.to_categorical(y_test, num_classes)

    ############################################# Prepare the Data #####################################################

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    with tf.Session(config=config) as sess:

        # prepare the placeholders
        x = tf.placeholder(tf.float32, [None, input_dim, input_dim, 3])
        y = tf.placeholder(tf.float32, [None, num_classes])

        input_output = []
        def modelBuilder(x, num_classes, dataset, type, sess, input_output):

            if len(input_output) == 0:

                reuse = False

                # Model/Graph
                if type == 'End2End':
                    _, tf_model = \
                        prepare_GBP_End2End(num_classes,
                                            inputT=x, sess=sess,
                                            checkpoint_dir='./{}_{}/'.format(dataset, type), reuse=reuse)
                else:

                    _, tf_model = \
                        prepare_Resnet(num_classes,
                                       inputT=x, sess=sess,
                                       checkpoint_dir='./{}_{}/'.format(dataset, type), reuse=reuse)

                input_output.append(x)
                input_output.append(tf_model.logits)

            else:

                reuse = True

                # Model/Graph
                if type == 'End2End':
                    _, tf_model = \
                        prepare_GBP_End2End(num_classes, inputT=x, reuse=reuse)
                else:
                    _, tf_model = \
                        prepare_Resnet(num_classes, inputT=x, reuse=reuse)

                input_output.append(x)
                input_output.append(tf_model.logits)

            return tf_model.logits

        # create an attackable model for the cleverhans
        model = CallableModelWrapper(lambda placeholder: modelBuilder(placeholder, num_classes, dataset, type, sess, input_output), 'logits')

        # TODO: check the configurations
        if attack_type == "FGM": # pass
            attack = FastGradientMethod(model, back='tf', sess=sess)
            params = {
                'eps' : 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "CWL2": # pass
            attack = CarliniWagnerL2(model, back='tf', sess=sess)
            params = {
                'confidence': 0.9,
                'batch_size': 128,
                'learning_rate': 0.005,
            }
        elif attack_type == "DF": # pass
            attack = DeepFool(model, back='tf', sess=sess)
            params = {
            }
        elif attack_type == "ENM": # configurations checked, quickly tested
            attack = ElasticNetMethod(model, back='tf', sess=sess)
            params = {
                'confidence': 0.9,
                'batch_size': 128,
                'learning_rate': 0.005,
            }
        elif attack_type == "FFA": # configuration checked
            attack = FastFeatureAdversaries(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'eps_iter': 0.005,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "LBFGS":
            attack = LBFGS(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "MEA":
            attack = MadryEtAl(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "MIM":
            attack = MomentumIterativeMethod(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "SMM":
            attack = SaliencyMapMethod(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "SPSA":
            attack = SPSA(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "VATM":
            attack = vatm(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        elif attack_type == "VAM":
            attack = VirtualAdversarialMethod(model, back='tf', sess=sess)
            params = {
                'eps': 0.06,
                'clip_min': 0.,
                'clip_max': 1.
            }
        else:
            raise Exception("I don't recognize {} this attack type. I will use FGM instead.".format(attack_type))

        # tf operation
        adv_x = attack.generate(x, **params)

        # generate the adversarial examples
        adv_vals = sess.run(adv_x, feed_dict={x: x_test[:size]})

        # notice that "adv_vals" may contain NANs because of the failure of the attack
        # also the input may not be perturbed at all because of the failure of the attack
        to_delete = []
        for idx, adv in enumerate(adv_vals):
            # for nan
            if np.isnan(adv).any():
                to_delete.append(idx)
            # for no perturbation
            if np.array_equiv(adv, x_test[idx]):
                to_delete.append(idx)

        # cleanings
        adv_vals_cleaned = np.delete(adv_vals, to_delete, axis=0)
        ori_cleaned = np.delete(x_test[:size], to_delete, axis=0)
        y_cleaned = np.delete(y_test[:size], to_delete, axis=0)

        if len(adv_vals_cleaned) == 0:
            print("No adversarial example is generated!")
            return

        print("{} out of {} adversarial examples are generated.".format(len(adv_vals_cleaned), size))

        print("The average L_inf distortion is {}".format(
            np.mean([np.max(np.abs(adv - ori_cleaned[idx])) for idx, adv in enumerate(adv_vals_cleaned)])))

        # TODO: visualize the adv_vals

        accuracy = model_eval(sess, input_output[0], y, tf.nn.softmax(input_output[1]), x_test[:size], y_test[:size],
                              args=eval_params)
        print('Test accuracy on normal examples: %0.4f' % accuracy)

        accuracy = model_eval(sess, input_output[0], y, tf.nn.softmax(input_output[1]), adv_vals_cleaned, y_cleaned,
                              args=eval_params)
        print('Test accuracy on adversarial examples: %0.4f' % accuracy)

if __name__ == '__main__':

    os.environ['CUDA_VISIBLE_DEVICES'] = '0'

    ap = argparse.ArgumentParser()
    ap.add_argument("-t", "--type", required=True, help="Resnet or End2End")
    ap.add_argument("-d", "--dataset", required=True, help="CIFAR10, CIFAR100 or SVHN")
    ap.add_argument("-a", "--attack", required=True, help="Attack Type")
    args = vars(ap.parse_args())

    main(type=args["type"], dataset=args["dataset"], attack_type=args["attack"])


from django.conf.urls import patterns, url

from calendar_mchp import views

urlpatterns = patterns('',
    url(r'^create/', views.calendar_create, name='calendar_create'),
    url(r'^update/', views.calendar_update, name='calendar_update'),
    url(r'^delete/', views.calendar_delete, name='calendar_delete'),
    url(r'^unsubscribe/', views.calendar_unsubscribe, name='calendar_unsubscribe'),

    url(r'^events/add/', views.event_add, name='event_add'),
    url(r'^events/update/', views.event_update, name='event_update'),
    url(r'^events/delete/', views.event_delete, name='event_delete'),
    url(r'^events/(?P<pk>\d+)/$', views.event_detail, name='event_detail'),

    url(r'^preview/(?P<pk>[^/]+)', views.calendar_preview, name='calendar_preview'),

    url(r'^feed/$', views.calendar_feed, name='calendar_feed'),

    url(r'^list/$', views.calendar_list, name='calendar_list'),

    url(r'^$', views.calendar, name='calendar'),
)