import json
import random
import string
import traceback

from ceph.ceph import CommandFailed
from tests.cephfs.cephfs_utilsV1 import FsUtils
from utility.log import Log

log = Log(__name__)

def run(ceph_cluster, **kw):
    """
    Test Cases Covered :
    CEPH-83573522   Verify the retained snapshot details with "ceph fs info" command

    Pre-requisites :
    1. We need atleast one client node to execute this test case
    1. creats fs volume create cephfs if the volume is not there
    2. ceph fs subvolumegroup create <vol_name> <group_name> --pool_layout <data_pool_name>
        Ex : ceph fs subvolumegroup create cephfs subvolgroup_info_retain
    3. ceph fs subvolume create <vol_name> <subvol_name> [--size <size_in_bytes>] [--group_name <subvol_group_name>]
       [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>]  [--namespace-isolated]
       Ex: ceph fs subvolume create cephfs subvol_2 --size 5368706371 --group_name subvolgroup_
    4. Create Data on the subvolume
        Ex:  python3 /home/cephuser/smallfile/smallfile_cli.py --operation create --threads 10 --file-size 400 --files
            100 --files-per-dir 10 --dirs-per-dir 2 --top /mnt/cephfs_fuse1baxgbpaia_1/
    5. Create snapshot of the subvolume
        Ex: ceph fs subvolume snapshot create cephfs subvol_2 snap_1 --group_name subvolgroup_info_retain

    Retain the snapshots nad verify the data after cloning:
    1. Check the state od subvolume it should be in complete.
    2. ceph fs snapshot rm <vol_name> <subvol_name> snap_name [--group_name <subvol_group_name>
        --force --retain-snapshots]
    3. Remove the sub volume.
    4. Check the state of subvolume it should be in snapshot-retained.

    Clean Up:
    1. Del all the snapshots created
    2. Del Subvolumes
    3. Del SubvolumeGroups
    """
    try:
        fs_util = FsUtils(ceph_cluster)
        config = kw.get("config")
        clients = ceph_cluster.get_ceph_objects("client")
        build = config.get("build", config.get("rhbuild"))
        fs_util.prepare_clients(clients, build)
        fs_util.auth_list(clients)
        log.info("checking Pre-requisites")
        if len(clients) < 1:
            log.info(
                f"This test requires minimum 1 client nodes.This has only {len(clients)} clients"
            )
            return 1
        default_fs = "cephfs"
        mounting_dir = "".join(
            random.choice(string.ascii_lowercase + string.digits)
            for _ in list(range(10))
        )
        client1 = clients[0]
        fs_details = fs_util.get_fs_info(client1)
        if not fs_details:
            fs_util.create_fs(client1, "cephfs")
        subvolumegroup_list = [
            {"vol_name": default_fs, "group_name": "subvolgroup_info_retain"},
        ]
        for subvolumegroup in subvolumegroup_list:
            fs_util.create_subvolumegroup(client1, **subvolumegroup)
        subvolume = {
            "vol_name": default_fs,
            "subvol_name": "subvol_retain_info",
            "group_name": "subvolgroup_info_retain",
            "size": "5368706371",
        }
        fs_util.create_subvolume(client1, **subvolume)
        log.info("Get the path of sub volume")
        subvol_path, rc = client1.exec_command(
            sudo=True,
            cmd=f"ceph fs subvolume getpath {default_fs} subvol_retain_info subvolgroup_info_retain",
        )
        kernel_mounting_dir_1 = f"/mnt/cephfs_kernel{mounting_dir}_1/"
        mon_node_ips = fs_util.get_mon_node_ips()
        fs_util.kernel_mount(
            [clients[0]],
            kernel_mounting_dir_1,
            ",".join(mon_node_ips),
            sub_dir=f"{subvol_path.strip()}",
        )
        client1.exec_command(
            sudo=True,
            cmd=f"python3 /home/cephuser/smallfile/smallfile_cli.py --operation create --threads 10 --file-size 400 "
            f"--files 100 --files-per-dir 10 --dirs-per-dir 2 --top "
            f"{kernel_mounting_dir_1}",
            long_running=True,
        )
        snapshot = {
            "vol_name": default_fs,
            "subvol_name": "subvol_retain_info",
            "snap_name": "snap_1",
            "group_name": "subvolgroup_info_retain",
        }
        fs_util.create_snapshot(client1, **snapshot)
        client1.exec_command(sudo=True, cmd=f"mkdir -p /tmp/{mounting_dir}")
        client1.exec_command(
            sudo=True, cmd=f"cp -r {kernel_mounting_dir_1}/* /tmp/{mounting_dir}"
        )
        subvol_info, rc = client1.exec_command(
            sudo=True,
            cmd=f"ceph fs subvolume info {default_fs} subvol_retain_info subvolgroup_info_retain --format json",
            check_ec=False,
        )
        subvol_info_state = json.loads(subvol_info)
        log.info(
            f"subvol state before removing the volume with --retain-snapshots {subvol_info_state['state']}"
        )
        if subvol_info_state["state"] != "complete":
            raise CommandFailed(
                f"subvol state should be in complete state "
                f"but current state is {subvol_info_state['state']}"
            )
        fs_util.remove_subvolume(
            client1, **subvolume, retain_snapshots=True, force=True, validate=False
        )
        log.info(
            "Verifying Get the path of sub volume as subvolume will still be listed in filesystem"
        )
        subvol_path, rc = client1.exec_command(
            sudo=True,
            cmd=f"ceph fs subvolume getpath {default_fs} subvol_retain_info subvolgroup_info_retain",
            check_ec=False,
        )
        if rc == 0:
            raise CommandFailed(
                "Remove subvolume with --retainSnapshots has not succeeded."
                "We are still able to fetch path of subvolume after deletion"
            )
            return 1
        subvol_info, rc = client1.exec_command(
            sudo=True,
            cmd=f"ceph fs subvolume info {default_fs} subvol_retain_info subvolgroup_info_retain --format json",
            check_ec=False,
        )
        subvol_info_state = json.loads(subvol_info)
        if subvol_info_state["state"] != "snapshot-retained":
            raise CommandFailed(
                f"subvol state should be in snapshot-retained state "
                f"but current state is {subvol_info_state['state']}"
            )
        log.info(
            f"subvol state after removing the volume with --retain-snapshots {subvol_info_state['state']}"
        )
        return 0
    except Exception as e:
        log.info(e)
        log.info(traceback.format_exc())
        return 1
    finally:
        log.info("Clean Up in progess")
        rmclone_list = [
            {"vol_name": default_fs, "subvol_name": "subvol_retain_info"},
        ]
        for clone_vol in rmclone_list:
            fs_util.remove_subvolume(
                client1, **clone_vol, validate=False, force=True, check_ec=False
            )
        fs_util.remove_snapshot(client1, **snapshot, validate=False, force=True)
        for subvolumegroup in subvolumegroup_list:
            fs_util.remove_subvolumegroup(
                client1, **subvolumegroup, force=True, check_ec=False, validate=False
            )


from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications import MobileNetV2
from tensorflow.keras.layers import AveragePooling2D, Dropout, Flatten, Dense, Input
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
from tensorflow.keras.preprocessing.image import img_to_array, load_img
from tensorflow.keras.utils import to_categorical
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from imutils import paths
import matplotlib.pyplot as plt
import numpy as np
import os
import confusion_matrix

# inizializzazione learning rate, n.epoche, batch size
INIT_LR = 1e-4
EPOCHS = 20
BS = 32

# caricamento percorso dataset
print("[INFO] loading images...")
imagePaths = list(paths.list_images("./cropped_eyes_emotions"))
data = []
labels = []

# loop in imagepaths
for imagePath in imagePaths:
	# estrazione label dal nome file
	label = imagePath.split(os.path.sep)[-2]

	# caricamento immagine (dimensione: 224x224) e preprocessing
	image = load_img(imagePath, target_size=(224, 224))
	image = img_to_array(image)
	image = preprocess_input(image)

	# aggiornamento data e labels
	data.append(image)
	labels.append(label)

# conversione data e labels in due vettori numpy
data = np.array(data, dtype="float32")
labels = np.array(labels)

# Binarizzare le etichette
lb = LabelBinarizer()
labels = lb.fit_transform(labels)
labels = to_categorical(labels)

# partizionamento del dataset in 80% train, 10% validation, 10%test
(trainX, testX, trainY, testY) = train_test_split(data, labels, test_size=0.1, stratify=labels, random_state=42)
(trainX, validX, trainY, validY) = train_test_split(trainX, trainY, test_size=0.1, random_state=41)

# utilizzo della classe ImageDataGenerator per aumentare la numerosità del set, considerando un'inclinazione della
# immagine di 20°
aug = ImageDataGenerator(
	rotation_range=20,
	fill_mode="nearest")

#caricamento della rete MobileNetV2, con i pesi ImageNet pre addestrati  e i layer del top non inclusi
baseModel = MobileNetV2(weights="imagenet", include_top=False,
	input_tensor=Input(shape=(224, 224, 3)))

# costruzione headModel, il quale sarà posizionato sul top del baseModel
headModel = baseModel.output
headModel = AveragePooling2D(pool_size=(7, 7))(headModel)
headModel = Flatten(name="flatten")(headModel)
headModel = Dense(128, activation="relu")(headModel)
headModel = Dropout(0.5)(headModel)
headModel = Dense(2, activation="softmax")(headModel)

# posizionare l'headModel sul baseModel, model sarà il modello finale
model = Model(inputs=baseModel.input, outputs=headModel)

#congelare i pesi di tutti i layer del basemodel
for layer in baseModel.layers:
	layer.trainable = False

# compile model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
model.compile(loss="binary_crossentropy", optimizer=opt,
	metrics=["accuracy"])

# train della head della rete
print("[INFO] training head...")
H = model.fit(
	aug.flow(trainX, trainY, batch_size=BS),
	steps_per_epoch=len(trainX) // BS,
	validation_data=(validX, validY),
	validation_steps=len(testX) // BS,
	epochs=EPOCHS)

#predizine test set
print("[INFO] evaluating network...")
predIdxs = model.predict(testX, batch_size=BS)

#predizione con probabilità più alta
predIdxs = np.argmax(predIdxs, axis=1)

#mostrare classification_report
print(classification_report(testY.argmax(axis=1), predIdxs,
	target_names=lb.classes_))

#serializzazione modello in locale
print("[INFO] saving mask detector model...")
model.save("model.h5")

# plot training accuracy
N = EPOCHS
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, N), H.history["accuracy"],'b', label="train_acc")
plt.plot(np.arange(0, N), H.history["val_accuracy"],'r',label="valid_acc")
plt.title("Training and Validation accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Accuracy")
plt.legend(loc="lower left")
plt.savefig("plot")

# plot training loss
N = EPOCHS
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, N), H.history["loss"],'b', label="train_loss")
plt.plot(np.arange(0, N), H.history["val_loss"],'r', label="valid_loss")
plt.title("Training and Validation loss")
plt.xlabel("Epoch #")
plt.ylabel("Loss")
plt.legend(loc="lower left")
plt.savefig("plot1")

# creazione della matrice di confusione
confusion_matrix.show_confusion_matrix(testY.argmax(axis=1), predIdxs)


"""
This is ninth task from Python Course - Basic (Day 13 of the course)
Done by DirtySiwy12

Task 9.4.3: Create a Pong (game).
"""

import pygame
from pygame.locals import *

RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLUE = (0, 0, 255)
DARK_BLUE = (0, 0, 128)
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
PINK = (255, 200, 200)
YELLOW = (255, 255, 0)

TOP_MARGIN = 5
BOTTOM_MARGIN = 470

# Creating class for bars
class Bar:
    def __init__(self, dimensions, _color, x, y):
        _bar = pygame.Surface(dimensions)
        self.bar = pygame.Surface.convert(_bar)
        self.bar.fill(_color)
        self.rect = self.bar.get_rect()
        self.rect.top = y
        self.rect.left = x
        self.move_y = 0
        self.score = 0

    def blit(self, _screen):
        _screen.blit(self.bar, self.rect)

    # Bars movement method
    def update_position(self):
        self.rect.y += self.move_y

        if self.rect.top <= TOP_MARGIN:
            self.rect.top = TOP_MARGIN + 2
        elif self.rect.bottom > BOTTOM_MARGIN:
            self.rect.bottom = BOTTOM_MARGIN

pygame.init()

screen = pygame.display.set_mode((640, 480), 0, 32)
pygame.display.set_caption('Pong!')

back = pygame.Surface((640, 480), 0, 32)
bg = pygame.Surface.convert(back)
bg.fill(BLACK)

# Create bars
bar1 = Bar(dimensions=(10, 50), _color=BLUE, x=10, y=215)
bar2 = Bar(dimensions=(10, 50), _color=RED, x=620, y=215)

# Create ball
radius = 7
ball_surface = pygame.Surface((15, 15))
pygame.draw.circle(ball_surface, GREEN, (radius, radius), radius)
ball = pygame.Surface.convert(ball_surface)
ball.set_colorkey(BLACK)

ball_x, ball_y = 307, 232
ball_speed_x, ball_speed_y = 250, 250

# Clock and font
clock = pygame.time.Clock()
font = pygame.font.SysFont('calibri', 40)

# Main game loop
loop_end = False

while not loop_end:

    # Field
    screen.blit(bg, (0, 0))
    frame = pygame.draw.rect(screen, WHITE, Rect((5, 5), (630, 470)), 2)
    middle_line = pygame.draw.aaline(screen, WHITE, (330, 5), (330, 475))

    # Bars
    bar1.blit(screen)
    bar2.blit(screen)

    # Ball
    screen.blit(ball, (ball_x, ball_y))

    # Score
    score1 = font.render(str(bar1.score), True, WHITE)
    score2 = font.render(str(bar2.score), True, WHITE)
    screen.blit(score1, (250, 210))
    screen.blit(score2, (380, 210))

    # Update bars movement
    bar1.update_position()
    bar2.update_position()

    # Ball movement
    time_passed = clock.tick(30)
    time_sec = time_passed / 1000.0
    ball_x += ball_speed_x * time_sec
    ball_y += ball_speed_y * time_sec
    ai_speed = abs(ball_speed_x) * time_sec

    # Bar deflection
    if ball_x <= bar1.rect.right:
        if bar1.rect.top <= ball_y + radius <= bar1.rect.bottom:
            ball_x = 20
            ball_speed_x = -ball_speed_x

    elif ball_x + 2 * radius >= bar2.rect.left:
        if bar2.rect.top <= ball_y + radius <= bar2.rect.bottom:
            ball_x = 605
            ball_speed_x = -ball_speed_x

    # Get score
    if ball_x < 5:
        bar2.score += 1
        ball_x, ball_y = 320, 232
        bar1.rect.top, bar2.rect.top = 215, 215

    elif ball_x > 620:
        bar1.score += 1
        ball_x, ball_y = 307, 232
        bar1.rect.top, bar2.rect.top = 215, 215

    # Top and bottom deflection
    if ball_y + radius <= TOP_MARGIN:
        ball_speed_y = -ball_speed_y
        ball_y = TOP_MARGIN

    elif ball_y - radius >= BOTTOM_MARGIN:
        ball_speed_y = -ball_speed_y
        ball_y = BOTTOM_MARGIN

    # Game
    for event in pygame.event.get():
        if event.type == QUIT:
            loop_end = True

        # Movement for bars
        if event.type == KEYDOWN:
            if event.key == K_UP:
                bar1.move_y = -ai_speed
            elif event.key == K_DOWN:
                bar1.move_y = ai_speed
            elif event.key == K_w:
                bar2.move_y = -ai_speed
            elif event.key == K_s:
                bar2.move_y = ai_speed
            elif event.key == K_q:
                loop_end = True
                break

        elif event.type == KEYUP:
            if event.key == K_UP or event.key == K_DOWN:
                bar1.move_y = 0
            if event.key == K_w or event.key == K_s:
                bar2.move_y = 0

    pygame.display.update()

pygame.quit()


from rq import get_current_job

from netpalm.backend.core.models.task import Response

def write_meta_error(data):
    """custom exception handler for within an rpc job"""
    job = get_current_job()
    job.meta["result"] = "failed"
    if type(data) == list:
        job.meta["errors"].append(data.split('\n'))
    else:
        job.meta["errors"].append(data)
    job.save_meta()

def render_netpalm_payload(job_result={}):
    """in band rpc job result renderer"""
    try:
        job = get_current_job()
        resultdata = Response(status="success",
                              data={"task_id": job.id,
                                    "created_on": job.created_at.strftime("%Y-%m-%d %H:%M:%S.%f"),
                                    "task_queue": job.description,
                                    "task_status": "finished",
                                    "task_result": job_result,
                                    "task_errors": job.meta["errors"]
                                    }).dict()
        return resultdata

    except Exception as e:
        return e


__author__ = 'tinglev@kth.se'

import os
import unittest
import root_path
from test import mock_test_data # pylint: disable=C0411
from modules.util import data_defs, ci_status

class TestUtilCiStatus(unittest.TestCase):

    def test_create_post_json(self):
        data = mock_test_data.get_pipeline_data()
        result = ci_status.create_post_json(data, 'TEST_STEP', 'OK', 0, 'Test description')
        self.assertEqual(result['reportingService'], 'aspen')
        self.assertEqual(result['description'], 'Test description')
        result = ci_status.create_post_json({}, 'TEST_STEP', 'OK', 0, None)
        self.assertIsNone(result)

    def test_add_message_to_queue(self):
        data = {}
        ci_status.add_message_to_queue(data, 'TEST_STEP', 'OK', 0, None)
        # Should fail because data does not contain anything to
        # create a post json from
        self.assertEqual(0, len(data[data_defs.CI_STATUS_QUEUE]))
        data = mock_test_data.get_pipeline_data()
        ci_status.add_message_to_queue(data, 'TEST_STEP', 'OK', 0, None)
        self.assertEqual(1, len(data[data_defs.CI_STATUS_QUEUE]))


import math
import numpy as np

import ray
from ray import tune
from ray.tune.suggest.bayesopt import BayesOptSearch
from ray.tune.suggest import ConcurrencyLimiter
import unittest

def loss(config, reporter):
    x = config.get("x")
    reporter(loss=x**2)  # A simple function to optimize

class ConvergenceTest(unittest.TestCase):
    """Test convergence in gaussian process."""

    def shutDown(self):
        ray.shutdown()

    def test_convergence_gaussian_process(self):
        np.random.seed(0)
        ray.init(local_mode=True, num_cpus=1, num_gpus=1)

        # This is the space of parameters to explore
        space = {"x": tune.uniform(0, 20)}

        resources_per_trial = {"cpu": 1, "gpu": 0}

        # Following bayesian optimization
        gp = BayesOptSearch(random_search_steps=10)
        gp.repeat_float_precision = 5
        gp = ConcurrencyLimiter(gp, 1)

        # Execution of the BO.
        analysis = tune.run(
            loss,
            metric="loss",
            mode="min",
            # stop=EarlyStopping("loss", mode="min", patience=5),
            search_alg=gp,
            config=space,
            num_samples=100,  # Number of iterations
            resources_per_trial=resources_per_trial,
            raise_on_failed_trial=False,
            fail_fast=True,
            verbose=1)
        assert len(analysis.trials) in {13, 40, 43}  # it is 43 on the cluster?
        assert math.isclose(analysis.best_config["x"], 0, abs_tol=1e-5)

if __name__ == "__main__":
    import pytest
    import sys
    sys.exit(pytest.main(["-v", __file__]))


"""
This project was created and being maintained by Shobhit Agarwal. The model created in this program CAN NOT be used in transfer learning.

Thank you and enjoy reverse engineering this code :)

PERMISSION NEEDED TO USE AND DISTRIBUTE CODE
PROPERTY OF APPARATUS DIAGNOSING
For permission, email: shobhitagarwal122@gmail.com or dev.shobhitagarwal@gmail.com

WORK IN PROGRESS...

"""

from glob import glob
from cv2 import cv2
from matplotlib import pyplot as plt
from PIL import Image
from sklearn.model_selection import train_test_split
import fnmatch
from progressbar import printProgressBar
import time
import numpy as np

imagePatches = glob('/mnt/c/Users/shobh/Documents/breast-histopathology-images/IDC_regular_ps50_idx5/**/*.png', recursive=True)

pix_array = []

# for filename in imagePatches:
#     #print(filename)
#     im_array = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
#     plt.imshow(im_array, cmap="gray")
#     plt.show()
#     break

""""OPTIMIZTION PROBLEM: FIGURE OUT HOW TO EFFECTIVELY CYCLE THROUGH ALL IMAGES"""

trainingData = []

CATEGORIES = ["BENIGN", "MALIGNANT"]

#Initialize crazy function
def create_training_data():
    patternZero = '*class0.png'
    patternOne = '*class1.png'
    classZero = fnmatch.filter(imagePatches, patternZero)
    l1 = len(classZero[0:1000])
    classOne = fnmatch.filter(imagePatches, patternOne)
    l2 = len(classOne[0:1000])
    # Update: module needs to have two lengths. These categories are for the different images (Benign and Malignant)
    for i, filename in enumerate(classZero[0:1000]):
        #print(filename)
        try:

            im_array = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
            new_array = cv2.resize(im_array, (50, 50))
            trainingData.append([new_array, 0])
            time.sleep(0.1)
            printProgressBar(i + 1, l1, prefix = 'Progress:', suffix = 'Complete', length = 50)
        except Exception as e:
            print(e)
            pass
    for i, filename in enumerate(classOne[0:1000]):
        try:
            im_array = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
            new_array = cv2.resize(im_array, (50, 50))
            trainingData.append([new_array, 1])
            time.sleep(0.1)
            printProgressBar(i + 1, l2, prefix = 'Progress:', suffix = 'Complete', length = 50)
        except Exception as e:
            print(e)
            pass

# This will take an inordinate amount of time - 277524 images, will be easier to use a cloud computer (global accessible neural net - possible idea)
# New Idea: We can take the first 1000 images and use them for training, the latter option would take very long
create_training_data()

print(trainingData)
#print("\n" + len(trainingData))

import random

random.shuffle(trainingData)

for sample in trainingData:
    print(sample[1])

X = []
y = []

for features, label in trainingData:
    print(features)
    X.append(features)
    y.append(label)

X = np.array(X).reshape(-1, 50, 50, 1)

import pickle
pickle_out = open("X.pickle", "wb")
pickle.dump(X, pickle_out)
pickle_out.close()

pickle_out = open("y.pickle", "wb")
pickle.dump(y, pickle_out)
pickle_out.close()

# image_name = imagePatches[0]

#
#
# print(classZero)
# (train_images, train_labels) = imagePatches

# # Normalize pixel values to be between 0 and 1
# train_images = imagePatches / 255.0
# print(train_images)

# plt.figure(figsize=(10,10))
# for i in range(25):
#     plt.subplot(5,5,i+1)
#     plt.xticks([])
#     plt.yticks([])
#     plt.grid(False)
#     plt.imshow(train_images[i])
#     # The CIFAR labels happen to be arrays,
#     # which is why you need the extra index
# plt.show()

# def plotImage(image_location):
#     image = cv2.imread(image_name)
#     image = cv2.resize(image, (50, 50))
#     plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
#     plt.axis('off')
#     plt.show()

# plotImage(image_name)


"""
Tools
-----------------

Additional functions to modify tfs files.

:author: Jaime
:module: tools

"""
import logging

import numpy as np

from tfs.handler import TfsFormatError, read_tfs, write_tfs

LOG = logging.getLogger(__name__)

def significant_digits(value: float, error: float) -> (str, str):
    """Computes value and its error properly rounded with respect to the size of the error"""
    if error == 0:
        raise ValueError("Input error of 0. Cannot compute significant digits.")
    digits = -int(np.floor(np.log10(error)))
    if np.floor(error * 10 ** digits) == 1:
        digits = digits + 1
    return f"{round(value,digits):.{max(digits, 0)}f}", f"{round(error, digits):.{max(digits, 0)}f}"

def remove_nan_from_files(list_of_files: list, replace: bool = False):
    """ Remove NAN-Entries from files in list_of_files.

    If replace=False a new file with .dropna in it's name is created, otherwise the file is
    overwritten.
    """
    for filepath in list_of_files:
        try:
            df = read_tfs(filepath)
            LOG.info(f"Read file {filepath:s}")
        except (IOError, TfsFormatError):
            LOG.info(f"Skipped file {filepath:s}")
        else:
            df = df.dropna(axis='index')
            if not replace:
                filepath += ".dropna"
            write_tfs(filepath, df)

def remove_header_comments_from_files(list_of_files: list):
    """ Check the files in list for invalid headers (no type defined) and removes them. """
    for filepath in list_of_files:
        LOG.info(f"Checking file: {filepath:s}")
        with open(filepath, "r") as f:
            f_lines = f.readlines()

        del_idcs = []
        for idx, line in enumerate(f_lines):
            if line.startswith("*"):
                break
            if line.startswith("@") and len(line.split("%")) == 1:
                del_idcs.append(idx)

        if len(del_idcs) > 0:
            LOG.info(f"    Found {len(del_idcs):d} lines to delete.")
            for idx in reversed(del_idcs):
                deleted_line = f_lines.pop(idx)
                LOG.info(f"    Deleted line: {deleted_line.strip():s}")

            with open(filepath, "w") as f:
                f.writelines(f_lines)

class DotDict(dict):
    """ Make dict fields accessible by . """
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        for key in self:
            if isinstance(self[key], dict):
                self[key] = DotDict(self[key])

    # __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

    def __getattr__(self, key: object) -> object:
        """ Needed to raise the correct exceptions """
        try:
            return super().__getitem__(key)
        except KeyError as e:
            raise AttributeError(e).with_traceback(e.__traceback__) from e


# -*- coding: utf-8 -*-
#
# AUTOR: Douglas Medeiros Nehme
#
# CONTACT: medeiros.douglas3@gmail.com
#
# CRIATION: fev/2017
#
# LAST MODIFICATION: mar/2017
#
# OBJECTIVE: Manipulating Antartica wind data

import os
import math
import numpy as np
import pandas as pd
import xarray as xr

##################################################################################################################################
#### CONFIG PARAMETERS AND GLOBAL VARIABLES ######################################################################################
##################################################################################################################################
# pep-8 conventions suggest upper case for global variables

ROOTDIR = '/home/douglasnehme/Desktop/bia'
DATADIR = os.path.join(ROOTDIR, 'arquivos')

##################################################################################################################################
#### FUNCTIONS ###################################################################################################################
##################################################################################################################################

def pol2cart_wind(wspd, wdir, axes_rotation = 0, magnetic_declination = 0):
    """
    Converts meteorological stations' wind measure, speed and direction (in degrees), into meridional and zonal components.

    Parameters
    ----------
    wspd = wind speed

    wdir = wind direction (in degrees)

    axes_rotation = permits axes rotation with posite values doing a clockwise moviment and negative values an anticlockwise. It
    allows a better fit between wind components and local wind regime

    magnetic_declination = permits correction between true and magnetic norths

    Reference
    ---------
    MIRANDA, L. B.; CASTRO, B. M.; KJERFVE, B. Redução e Análise de Dados Experimentais: Fluxo e Transporte de Propriedades. In:
    Princípios de Oceanografia Física em Estuários. 2 ed. São Paulo: Editora da Universidade de São Paulo, 2012. Cap. 5, p. 153-192.
    ISBN: 978-85-314-0675-1.

    Steps
    -----
    1 - Decimal places standardization (Input data)

    2 - Transform angles from meteorological (wdir) to cartesian referential (phi), where trigonometric equations are valid

        METEOROLOGICAL REFERENTIAL       CARTESIAN REFERENTIAL
                 360°/0°                         90°
                    |                             |
                    |                             |
         270° ______|______ 90°        180° ______|______ 360°/0°
                    |                             |
                    |                             |
                    |                             |
                   180°                          270°

    3 - Fix all phi values between 0° and 360°

    4 - Transform angles from degrees to radians and calculating wind components

    5 - Decimal places standardization (Output data)
    """
    import math

    # Step 1
    wdir = math.ceil(wdir) # always rounds the float number to next integer, but returns a float number
    wspd = round(wspd, 1)  # round a number to a given precision in decimal digits considering round laws

    # Step 2
    phi = 90. - (wdir + magnetic_declination) + axes_rotation

    # Step 3
    if phi < 0.:

        phi = phi + 360.

    # Step 4
    phi = math.radians(phi)

    u = wspd * math.cos(phi)
    v = wspd * math.sin(phi)

    # Step 5
    u = round(u, 2)
    v = round(v, 2)

    return(u, v)

def cart2pol_wind(u, v, axes_rotation = 0, magnetic_declination = 0):
    """
    Converts meridional and zonal wind components into speed and direction (in degrees) values.

    Parameters
    ----------
    u = zonal component

    v = meridional component

    axes_rotation = undo axes rotation with posite values doing a clockwise moviment and negative values an anticlockwise. It
    allows a better fit between wind components and local wind regime

    magnetic_declination = Put here for extension of pol2cart_wind function, but doesn't have usage now, possible in future

    Reference
    ---------
    MIRANDA, L. B.; CASTRO, B. M.; KJERFVE, B. Redução e Análise de Dados Experimentais: Fluxo e Transporte de Propriedades. In:
    Princípios de Oceanografia Física em Estuários. 2 ed. São Paulo: Editora da Universidade de São Paulo, 2012. Cap. 5, p. 153-192.
    ISBN: 978-85-314-0675-1.

    Steps
    -----
    1 - Decimal places standardization (Input data)

    2 - Calculate wind speed and cartesian referential angle (phi) from wind componentes and convert phi from radians to degrees

    3 - Transform angles from cartesian (phi) to meteorological referential (wdir)

        METEOROLOGICAL REFERENTIAL       CARTESIAN REFERENTIAL
                 360°/0°                         90°
                    |                             |
                    |                             |
         270° ______|______ 90°        180° ______|______ 360°/0°
                    |                             |
                    |                             |
                    |                             |
                   180°                          270°

    4 - Fix all wdir values between 0° and 360°

    5 - Decimal places standardization (Output data)
    """
    import math

    # Step 1
    u = round(u, 2)
    v = round(v, 2)

    # Step 2
    wspd = math.sqrt(u**2 + v**2)
    phi  = math.atan2(v, u)

    phi = math.degrees(phi)

    # Step 3
    wdir = 90. - (phi + magnetic_declination) + axes_rotation

    # Step 4
    if wdir < 0.:

        wdir = wdir + 360.

    # Step 5
    wdir = math.ceil(wdir) # always rounds the float number to next integer, but returns a float number
    wspd = round(wspd, 1) # round a number to a given precision in decimal digits considering round laws

    return(wspd, wdir)

##################################################################################################################################
#### IMPORTING AND MANIPULATING ALL TIME SERIES ##################################################################################
##################################################################################################################################

era = xr.open_dataset(os.path.join(DATADIR, 'era.nc'))

era.longitude.data = era.longitude.data - 360.

wspd, wdir = [np.nan] * len(era.time), [np.nan] * len(era.time)

for i in xrange(len(era.time)):
    wspd[i], wdir[i] = cart2pol_wind(era.u10[ :, 5, 29].data[i], era.v10[ :, 5, 29].data[i])

df = pd.DataFrame (index = era.time.data, columns = ['u', 'v', 'spd'],
                   data = {'u': era.u10[ :, 5, 29].data, 'v': era.v10[ :, 5, 29].data, 'spd': wspd} )

df = df.resample('A').mean()

df = df.to_period('A')

df.to_excel(os.path.join(ROOTDIR, 'reanalise', 'era_interim2.xlsx'))


from django.contrib import admin
# from accounts import views
from community import views
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static

app_name ="community"

urlpatterns = [
    path('home/', views.home, name='home'),
    path('bill_main/', views.bill_main, name="bill_main"),
    path('bill/<str:bill_id>', views.bill_detail, name="bill_detail"),
    path('bill_write/', views.bill_write, name="bill_write"),
    path('bill_create/', views.bill_create, name="bill_create"),
    path('bill_delete/<str:bill_id>', views.bill_delete, name="bill_delete"),
    path('debate/<str:debate_id>', views.debate_detail, name="debate_detail"),
    path('debate_main/', views.debate_main, name="debate_main"),
    path('mypage/',views.mypage,name="mypage"),
    path('bill/<str:bill_id>/comment/', views.comment_to_bill, name="comment_to_bill"),
    path('debate/<str:debate_id>/comment/', views.comment_to_debate, name="comment_to_debate"),
    path('forbidden/',views.forbidden,name="forbidden"),

] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


## input
#N, M = 4, 5
#matrix = [[0,0,1,1,0],
#          [0,0,0,1,1],
#          [1,1,1,1,1],
#          [0,0,0,0,0]]
'''
def dfs(x,y):
    if x < 0 or x >= N or y < 0 or y >= M:
        return False
    if matrix[x][y] == 0:
        matrix[x][y] = 1
        dfs(x-1,y)
        dfs(x,y-1)
        dfs(x+1,y)
        dfs(x,y+1)
        return True
    return False

count = 0
for i in range(N):
    for j in range(M):
        if dfs(i,j) == True:
            count += 1
'''
## answer is 3
#print(count)

from collections import deque

N, M = 5, 6
maze = [[1,0,1,0,1,0],
        [1,1,1,1,1,1],
        [0,0,0,0,0,1],
        [1,1,1,1,1,1],
        [1,1,1,1,1,1]]

moves = [[1,0],[-1,0],[0,1],[0,-1]]

def bfs(x,y):
    queue = deque()
    queue.append((x,y))
    while queue:
        x, y = queue.popleft()
        for move in moves:
            nx = x + move[0]
            ny = y + move[1]
            if nx < 0 or nx >= N or ny < 0 or ny >= M:
                continue
            if maze[nx][ny] == 0:
                continue
            if maze[nx][ny] == 1:
                maze[nx][ny] = maze[x][y] + 1
                queue.append((nx,ny))

    return maze[N-1][M-1]

print(bfs(0,0))


import keras
import keras.backend as K

def random_normal():
    return keras.initializers.RandomNormal(stddev=0.02)

def batchnorm(h):
    return keras.layers.normalization.BatchNormalization(
        momentum=0.9,
        epsilon=2e-05,
    )(h)

def leaky_relu(h):
    return keras.layers.advanced_activations.LeakyReLU(
        alpha=0.2,
    )(h)

def weight_decay():
    return keras.regularizers.l2(0.00001)

def down_cbr(h, filters):
    h = keras.layers.Conv2D(
        filters=filters,
        kernel_size=4,
        strides=2,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
    )(h)
    h = batchnorm(h)
    h = leaky_relu(h)
    return h

def up_cbr(h, filters, dropout=False, use_resize_conv=False):
    if use_resize_conv:
        h = keras.layers.UpSampling2D(
            size=(2, 2),
        )(h)
        h = keras.layers.Conv2D(
            filters=filters,
            kernel_size=3,
            strides=1,
            padding='same',
            kernel_initializer=random_normal(),
            kernel_regularizer=weight_decay(),
        )(h)
    else:
        h = keras.layers.Conv2DTranspose(
            filters=filters,
            kernel_size=4,
            strides=2,
            padding='same',
            kernel_initializer=random_normal(),
            kernel_regularizer=weight_decay(),
        )(h)
    h = batchnorm(h)
    if dropout:
        h = keras.layers.Dropout(0.5)(h)
    h = keras.layers.core.Activation('relu')(h)
    return h

def generator(w, in_ch, out_ch, base_ch, use_resize_conv=False):
    x = keras.layers.Input(shape=(w, w, in_ch))

    h = keras.layers.Conv2D(
        filters=base_ch,
        kernel_size=5,
        strides=1,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
    )(x)

    h0 = leaky_relu(h)
    h = keras.layers.Conv2D(
        filters=base_ch * 2,
        kernel_size=3,
        strides=1,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
    )(h0)
    h = batchnorm(h)
    h1 = leaky_relu(h)

    h2 = down_cbr(h1, base_ch * 4)
    h3 = down_cbr(h2, base_ch * 8)
    h4 = down_cbr(h3, base_ch * 8)
    h5 = down_cbr(h4, base_ch * 8)
    h6 = down_cbr(h5, base_ch * 8)
    h7 = down_cbr(h6, base_ch * 8)

    h = up_cbr(h7, base_ch * 8, dropout=True, use_resize_conv=use_resize_conv)

    h = keras.layers.concatenate([h, h6])
    h = up_cbr(h, base_ch * 8, dropout=True, use_resize_conv=use_resize_conv)

    h = keras.layers.concatenate([h, h5])
    h = up_cbr(h, base_ch * 8, dropout=True, use_resize_conv=use_resize_conv)

    h = keras.layers.concatenate([h, h4])
    h = up_cbr(h, base_ch * 8, dropout=False, use_resize_conv=use_resize_conv)

    h = keras.layers.concatenate([h, h3])
    h = up_cbr(h, base_ch * 4, dropout=False, use_resize_conv=use_resize_conv)

    h = keras.layers.concatenate([h, h2])
    h = up_cbr(h, base_ch * 2, dropout=False, use_resize_conv=use_resize_conv)

    h = keras.layers.concatenate([h, h1])
    h = keras.layers.Conv2D(
        filters=base_ch,
        kernel_size=3,
        strides=1,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
    )(h)
    h = batchnorm(h)
    h = keras.layers.Activation('relu')(h)

    h = keras.layers.concatenate([h, h0])
    h = keras.layers.Conv2D(
        filters=out_ch,
        kernel_size=5,
        strides=1,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
        name='output_gen',
    )(h)
    return x, h

def discriminator(w, ch0, ch1, base_ch):
    assert base_ch % 2 == 0

    x0 = keras.layers.Input(shape=(w, w, ch0))
    x1 = keras.layers.Input(shape=(w, w, ch1))
    h0 = keras.layers.Conv2D(
        filters=base_ch // 2,
        kernel_size=5,
        strides=1,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
    )(x0)
    h0 = batchnorm(h0)
    h0 = leaky_relu(h0)

    h1 = keras.layers.Conv2D(
        filters=base_ch // 2,
        kernel_size=5,
        strides=1,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
    )(x1)
    h1 = batchnorm(h1)
    h1 = leaky_relu(h1)

    h = down_cbr(keras.layers.concatenate([h0, h1]), base_ch * 2)
    h = down_cbr(h, base_ch * 4)
    h = down_cbr(h, base_ch * 8)
    h = keras.layers.Conv2D(
        filters=1,
        kernel_size=3,
        strides=1,
        padding='same',
        kernel_initializer=random_normal(),
        kernel_regularizer=weight_decay(),
    )(h)
    return x0, x1, h

def pix2pix(w, in_ch, out_ch, base_ch, use_resize_conv=False):
    gen_in, gen_out = generator(w, in_ch, out_ch, base_ch, use_resize_conv)
    dis_in_0, dis_in_1, dis_out = discriminator(w, in_ch, out_ch, base_ch)

    gen = keras.models.Model(gen_in, gen_out, 'Generator')
    gen_frozen = keras.models.Model(gen_in, gen_out, 'Generator-Frozen')
    gen_frozen.trainable = False

    dis = keras.models.Model([dis_in_0, dis_in_1], dis_out, 'Discriminator')
    dis_frozen = keras.models.Model([dis_in_0, dis_in_1], dis_out, 'Discriminator-Frozen')
    dis_frozen.trainable = False

    x_in = keras.layers.Input(shape=(w, w, in_ch))
    x_real = keras.layers.Input(shape=(w, w, out_ch))

    x_fake_gen = gen(x_in)
    y_fake_gen = dis_frozen([x_in, x_fake_gen])

    x_fake = gen_frozen(x_in)
    y_real_dis = dis([x_in, x_real])
    y_fake_dis = dis([x_in, x_fake])

    gen_trainer = keras.models.Model([x_in, x_real], [x_fake_gen, y_fake_gen])
    dis_trainer = keras.models.Model([x_in, x_real], [y_real_dis, y_fake_dis])

    return gen, dis, gen_trainer, dis_trainer

lam1 = 100
lam2 = 1/8

def gen_loss_l1(y_true, y_pred):
    return lam1 * keras.losses.mean_absolute_error(y_true, y_pred)

def gen_loss_adv(_, y_pred):
    return lam2 * K.mean(K.softplus(y_pred), axis=-1)

def dis_loss_real(_, y_pred):
    return K.mean(K.softplus(y_pred), axis=-1)

def dis_loss_fake(_, y_pred):
    return K.mean(K.softplus(-y_pred), axis=-1)


import discord
from discord.ext import commands
from Commands.General import Help
from Commands import Pokemon

'''
KEY THINGS TO PUT:
- DISCORD BOT TOKEN
- GUILD ID
- CHANNEL ID - specify Pokemon.py file
- SPECIFY INTENTS IN DISCORD DEVELOPER APPLICATION
'''

# SPECIFY YOUR INTENTS HERE: BOT MAY NOT WORK IF CERTAIN INTENTS NOT ALLOWED, SUCH AS MEMBERS
# I HAVE SPECIFIED ALL, SO YOU MUST ALLOW ALL PRIVILEGED GATEWAY INTENTS IN YOUR DISCORD APP IF
# YOU ARE NOT SPECIFYING YOUR OWN INTENTS
# See https://discordpy.readthedocs.io/en/latest/api.html#intents for more
intents = discord.Intents.all()
bot = commands.Bot(command_prefix='!', intents=intents)

@bot.event
async def on_ready():
    print('Logged in as {}, {}'.format(bot.user.name, bot.user.id))
    await bot.change_presence(activity=discord.Game(name="Pokemon!"))
    my_guild = bot.get_guild('''YOUR GUILD ID''')  # YOUR GUILD ID HERE
    bot.add_cog(Pokemon.Pokemon(bot, my_guild))

bot.add_cog(Pokemon.Pokedex(bot))
bot.add_cog(Help.Greetings(bot))
bot.run('YOUR DISCORD BOT TOKEN')  # YOUR BOT TOKEN HERE


from rest_framework import serializers

from .models import OrderProduct
from carts.models import Cart
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.shortcuts import get_object_or_404

class OrderProductSerializer(serializers.ModelSerializer):

    class Meta:
        model = OrderProduct
        cart_id = serializers.ReadOnlyField(source='cart')
        fields = ['id', 'product', 'quantity', 'cart_id']

    def create(self, data):

        if self.context['request'].user:
            user_id = self.context['request'].user.id
        else:
            user_id = None

        user = get_object_or_404(User, pk=user_id)

        try:
            cart = Cart.objects.get(created_by=user, status='in_cart')
        except Cart.DoesNotExist:
            cart = Cart.objects.create(created_by=user, status='in_cart')
            cart.save()

        data['cart'] = cart
        data['created_by'] = user

        try:
            order_product = OrderProduct.objects.get(product=data.get('product'), cart=data.get('cart'),)
            order_product.quantity += data.get('quantity')
            order_product.save()
        except ObjectDoesNotExist:
            order_product = OrderProduct.objects.create(**data)

        return order_product


# ******************************************************************************
# Copyright 2017-2019 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
import numpy as np
import pytest

import ngraph as ng
from test.ngraph.util import get_runtime, run_op_numeric_data

@pytest.mark.skip_on_gpu
def test_concat():
    a = np.array([[1, 2], [3, 4]])
    b = np.array([[5, 6]])
    axis = 0
    expected = np.concatenate((a, b), axis=0)

    runtime = get_runtime()
    parameter_a = ng.parameter(list(a.shape), name='A', dtype=np.float32)
    parameter_b = ng.parameter(list(b.shape), name='B', dtype=np.float32)
    node = ng.concat([parameter_a, parameter_b], axis)
    computation = runtime.computation(node, parameter_a, parameter_b)
    result = computation(a, b)
    assert np.allclose(result, expected)

@pytest.mark.parametrize('val_type, value', [
    (bool, False),
    (bool, np.empty((2, 2), dtype=bool)),
])
@pytest.mark.skip_on_gpu  # under investigation, runtime error is: function failed to compile
def test_constant_from_bool(val_type, value):
    expected = np.array(value, dtype=val_type)
    result = run_op_numeric_data(value, ng.constant, val_type)
    assert np.allclose(result, expected)

@pytest.mark.parametrize('val_type, value', [
    (np.float32, np.float32(0.1234)),
    (np.float64, np.float64(0.1234)),
    (np.int8, np.int8(-63)),
    (np.int16, np.int16(-12345)),
    (np.int32, np.int32(-123456)),
    (np.int64, np.int64(-1234567)),
    (np.uint8, np.uint8(63)),
    (np.uint16, np.uint16(12345)),
    (np.uint32, np.uint32(123456)),
    (np.uint64, np.uint64(1234567)),
])
@pytest.mark.skip_on_gpu  # under investigation, runtime error is: function failed to compile
def test_constant_from_scalar(val_type, value):
    expected = np.array(value, dtype=val_type)
    result = run_op_numeric_data(value, ng.constant, val_type)
    assert np.allclose(result, expected)

@pytest.mark.parametrize('val_type', [
    np.float32,
    np.float64,
])
@pytest.mark.skip_on_gpu  # under investigation, runtime error is: function failed to compile
def test_constant_from_float_array(val_type):
    np.random.seed(133391)
    input_data = np.array(-1 + np.random.rand(2, 3, 4) * 2, dtype=val_type)
    result = run_op_numeric_data(input_data, ng.constant, val_type)
    assert np.allclose(result, input_data)

@pytest.mark.parametrize('val_type, range_start, range_end', [
    (np.int8, -8, 8),
    (np.int16, -64, 64),
    (np.int32, -1024, 1024),
    (np.int64, -16383, 16383),
    (np.uint8, 0, 8),
    (np.uint16, 0, 64),
    (np.uint32, 0, 1024),
    (np.uint64, 0, 16383),
])
@pytest.mark.skip_on_gpu  # under investigation, runtime error is: function failed to compile
def test_constant_from_integer_array(val_type, range_start, range_end):
    np.random.seed(133391)
    input_data = np.array(np.random.randint(range_start, range_end, size=(2, 2)), dtype=val_type)
    result = run_op_numeric_data(input_data, ng.constant, val_type)
    assert np.allclose(result, input_data)


# Copyright 2020 The Kraken Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import logging

from .bg import jobs as bg_jobs
from .bg.clry import app as clryapp
from . import consts

log = logging.getLogger(__name__)

def _is_in_celery_queue(func, args):
    with clryapp.pool.acquire(block=True) as conn:
        tasks = conn.default_channel.client.lrange('celery', 0, -1)

    args = repr(args)

    for task in tasks:
        j = json.loads(task)
        hdrs = j['headers']
        f = hdrs['task']
        a = hdrs['argsrepr']
        if f == func and a == args:
            return True

    return False

def trigger_run(stage_id, flow_kind=consts.FLOW_KIND_CI, reason=None):
    logging.basicConfig(format=consts.LOG_FMT, level=logging.INFO)

    args = (stage_id, flow_kind, reason)

    if _is_in_celery_queue('kraken.server.bg.jobs.trigger_run', args):
        log.info('skipped trigger run for stage %s as it is already in celery queue', stage_id)
        return

    log.info('trigger run for stage %s', stage_id)
    t = bg_jobs.trigger_run.delay(*args)
    log.info('triggering run for stage %s, bg processing: %s', stage_id, t)

def refresh_schema_repo(stage_id):
    logging.basicConfig(format=consts.LOG_FMT, level=logging.INFO)

    args = (stage_id,)

    if _is_in_celery_queue('kraken.server.bg.jobs.refresh_schema_repo', args):
        log.info('skipped refresh stage %s schema from repo as it is already in celery queue', stage_id)
        return

    log.info('refresh stage %s schema from repo', stage_id)
    t = bg_jobs.refresh_schema_repo.delay(stage_id)
    log.info('refreshing stage %s schema from repo, bg processing: %s', stage_id, t)


"""
I am version {} of the J.A.R.V.I.S. natural language interface for Slack,
configured to perform a multitude of functions. The following modules have been
loaded:
"""
import contextlib
import logging
import json
import time

import slackclient

from .api import build_user
from .db import conn
from .db import initialize_database
from .error import SlackError
from .plugins import get_plugins

__version__ = '1.2.3'

logger = logging.getLogger(__name__)

class Jarvis(object):
    def __init__(self, token, init=False):
        self.last_ping = 0

        self.slack = slackclient.SlackClient(token)
        self.slack.rtm_connect()

        if init:
            self.init()

        self.plugins = get_plugins(self.slack)

    def init(self):
        initialize_database()

        users = json.loads(self.slack.api_call('users.list'))
        for user in users['members']:
            if user['deleted']:
                continue

            if user['is_bot'] or user['id'] == 'USLACKBOT':
                continue

            user_fields = build_user(self.slack, user)
            with contextlib.closing(conn.cursor()) as cur:
                cur.execute(""" INSERT INTO user
                                (uuid, first_name, last_name, email, username,
                                 is_admin, channel)
                                VALUES (?, ?, ?, ?, ?, ?, ?)
                            """, user_fields)
                conn.commit()

    def handle_message(self, channel, text, user):
        with contextlib.closing(conn.cursor()) as cur:
            dm = cur.execute(""" SELECT 1 FROM user WHERE channel = ? """,
                             [channel]).fetchone()

        # TODO: consider changing how Jarvis pays attention to public channels
        if not dm and not text.startswith('jarvis'):
            return

        ch = self.slack.server.channels.find(channel)
        if not ch:
            raise SlackError('Could not look up channel {}.'.format(channel))

        if 'help' in text:
            message = [__doc__.format(__version__).replace('\n', ' ')]
            for plugin in self.plugins:
                message.append('- {}'.format(plugin.name))
                if plugin.name in text:
                    plugin.help(ch=ch)
                    break
            else:
                ch.send_message('\n'.join(message))
            return

        for plugin in self.plugins:
            plugin.respond(ch=ch, user=user, msg=text)

    def input(self, data):
        kind = data.get('type')
        if kind in ('pong', 'presence_change', 'reconnect_url', 'user_typing'):
            return

        channel = data.get('channel')
        text = data.get('text', '').lower()
        user = data.get('user')

        if kind == 'message':
            self.handle_message(channel, text, user)
            return

        # TODO: handle new and changing users differently?
        if kind in ('team_join', 'user_change'):
            user_fields = build_user(self.slack, user)
            with contextlib.closing(conn.cursor()) as cur:
                cur.execute(""" INSERT OR REPLACE INTO user
                                (uuid, first_name, last_name, email, username,
                                 is_admin, channel)
                                VALUES (?, ?, ?, ?, ?, ?, ?)
                            """, user_fields)
                conn.commit()
            return

        logger.debug('Did not respond to event %s', data)

    def keepalive(self):
        now = int(time.time())
        if now > self.last_ping + 3:
            self.slack.server.ping()
            self.last_ping = now

    def run(self):
        # TODO: interrupt > poll
        while True:
            for message in self.slack.rtm_read():
                self.input(message)

            self.keepalive()
            time.sleep(.1)


# encoding:utf-8
import re
import json
from datetime import date
from colleges.models import *
from django.utils.translation import ugettext as _
from django import forms

#import yougor
from const.choice import *
from utils.html import sanitize_html
from utils import html2text
from forms.base import *

class Character_Field(forms.ChoiceField):

    def __init__(self, *args, **kwargs):
        super(Character_Field, self).__init__(*args, **kwargs)
        self.required = False
        self.choices = CHARACTER_CHOICES
        self.label = '院校性质'
        self.help_text = '请选择院校的性质'
        self.initial = ''

class Header_Basic_Field(forms.CharField):

    def __init__(self, *args, **kwargs):
        super(Header_Basic_Field, self).__init__(*args, **kwargs)
        self.required = False
        self.widget = forms.TextInput(
            attrs={'size': 30, 'autocomplete': 'off'})
        self.max_length = 245
        self.label = ''
        self.help_text = ''

class Institution_cn_Field(forms.CharField):

    def __init__(self, *args, **kwargs):
        super(Institution_cn_Field, self).__init__(*args, **kwargs)
        self.required = True
        self.widget = forms.TextInput(
            attrs={'size': 70, 'autocomplete': 'off'})
        self.max_length = 255
        self.label = '中文名'
        self.help_text = ''
        self.initial = ''

class Position_Field(forms.CharField):

    def __init__(self, *args, **kwargs):
        super(Position_Field, self).__init__(*args, **kwargs)
        self.required = False
        self.widget = forms.TextInput(
            attrs={'size': 30, 'autocomplete': 'off'})
        self.max_length = 30
        self.label = '位置信息'
        self.help_text = '例如: 22.337857,114.1819618'
        self.initial = ''

    def clean(self, value):
        if len(value.strip()) >= 30:
        #example: 22.337857,114.1819618
            raise forms.ValidationError("请不要超过30个字符")
        return value

class Yougoer_Rank_Field(forms.CharField):

    def __init__(self, *args, **kwargs):
        super(Yougoer_Rank_Field, self).__init__(*args, **kwargs)
        self.required = False
        self.widget = forms.TextInput(
            attrs={'size': 10})
        self.max_length = 4
        self.label = '官方评分'
        self.help_text = '例如: 8.2'
        self.initial = ''

    def clean(self, value):
        try:
            if not value:
                return 0.0
            float(value) #[DEBUG] print "@@@@@@@@@@@@@@@%s" % len(value.strip())
            if len(value.strip()) > 4:
                raise forms.ValidationError("请输入正确评分")
            else:
                return value
        except:
           raise forms.ValidationError("请输入正确评分")

class Institution_en_Field(forms.CharField):

    def __init__(self, *args, **kwargs):
        super(Institution_en_Field, self).__init__(*args, **kwargs)
        self.required = True
        self.widget = forms.TextInput(
            attrs={'size': 70, 'autocomplete': 'off'})
        self.max_length = 255
        self.label = '英文名'
        self.help_text = ''
        self.initial = ''

class Institution_others_Field(forms.CharField):

    def __init__(self, *args, **kwargs):
        super(Institution_others_Field, self).__init__(*args, **kwargs)
        self.required = False
        self.widget = forms.TextInput(
            attrs={'size': 70, 'autocomplete': 'off'})
        self.max_length = 255
        self.label = '别称'
        self.help_text = "例如 东京大学:  东大 とうきょうだいがく とうだい (用空格 隔开)"
        self.initial = ''

class Country_Field(forms.ChoiceField):

    def __init__(self, *args, **kwargs):
        super(Country_Field, self).__init__(*args, **kwargs)
        self.required = True
        self.choices = COUNTRY_CHOICES
        self.label = '国家'
        self.help_text = '请选择院校所属国家'
        self.initial = ''

class LOGO_Field(forms.ImageField):

    def __init__(self, *args, **kwargs):
        super(LOGO_Field, self).__init__(*args, **kwargs)
        self.required = True
        self.label = '上传校徽'
        self.help_text = '上传校徽'
        self.widget = forms.ClearableFileInput(
            attrs={'class': 'required', 'accept': 'image/*'})
        self.initial = ''

class Images_Field(forms.ImageField):

    def __init__(self, *args, **kwargs):
        super(Images_Field, self).__init__(*args, **kwargs)
        self.required = True
        self.initial = ''
        self.label = '上传图片'
        self.help_text = '上传图片'
        self.widget = forms.ClearableFileInput(
            attrs={'class': 'required upload_img', 'accept': 'image/*'})
        self.initial = ''

class TagNamesField(forms.CharField):

    def __init__(self, *args, **kwargs):
        super(TagNamesField, self).__init__(*args, **kwargs)
        self.required = False
        self.widget = forms.TextInput(
            attrs={'size': 100, 'autocomplete': 'off'})
        self.max_length = 255
        self.label = "标签"
        self.help_text = "例如: 医学 商科 人文 政法 (用空格分开)"
        self.initial = ''

    def clean(self, value):
        value = super(TagNamesField, self).clean(value)
        data = value.strip()
        list = data.split(' ')
        list_temp = []
        if len(list) >= 18:
            raise forms.ValidationError("请不要添加超过18个标签")
        for tag in list:
            if len(tag) >= 20:
                raise forms.ValidationError("每个标签字数在10个字以内")
            # take tag regex from settings
            #tagname_re = re.compile(r'[a-z0-9]+|[\u4e00-\u9fa5]+')
            # if not tagname_re.match(tag):
            # raise forms.ValidationError(_('please use following characters in tags: letters, chinese, numbers, and characters \'.-_#\''))
            # only keep one same tag
            if tag not in list_temp and len(tag.strip()) > 0:
                list_temp.append(tag)
        return u' '.join(list_temp)

class NewCollegeForm(forms.Form):
    # college
    institution_en = Institution_en_Field()
    institution_cn = Institution_cn_Field()
    institution_others = Institution_others_Field()
    country = Country_Field()
    # college_wiki_article
    text = EditorField()
    summary = SummayField()
    tags = TagNamesField()

    times_rank = forms.IntegerField(required=False, )
    qs_rank = forms.IntegerField(required=False, )
    arwu_rank = forms.IntegerField(required=False, )

    user = forms.CharField(
        required=False, max_length=255, widget=forms.TextInput(attrs={'size': 35}))
    email = forms.CharField(
        required=False, max_length=255, widget=forms.TextInput(attrs={'size': 35}))

class EditCollegeForm(forms.Form):
    # college
    institution_en = Institution_en_Field()
    institution_cn = Institution_cn_Field()
    institution_others = Institution_others_Field()
    tags = TagNamesField()
    # college_wiki_article
    summary = SummayField()
    yougoer_rank = Yougoer_Rank_Field()

    def __init__(self, college_wiki_article, college, *args, **kwargs):
        super(EditCollegeForm, self).__init__(*args, **kwargs)
        self.fields['institution_en'].initial = college.institution_en
        self.fields['institution_cn'].initial = college.institution_cn
        self.fields['institution_others'].initial = college.institution_others

        self.fields['summary'].initial = college_wiki_article.summary
        self.fields['tags'].initial = college.tagnames
        self.fields['yougoer_rank'].initial = college.yougoer_rank

class EditCollegeTextForm(forms.Form):
    # college_wiki_article
    text = EditorField()

    def __init__(self, college_wiki_article, *args, **kwargs):
        super(EditCollegeTextForm, self).__init__(*args, **kwargs)
        self.fields['text'].initial = html2text.html2text(college_wiki_article.content)

class EditCollegePositionForm(forms.Form):
    position = Position_Field()

    def __init__(self, college, *args, **kwargs):
        super(EditCollegePositionForm, self).__init__(*args, **kwargs)
        if college.latitude and college.latitude:
            self.fields['position'].initial = college.latitude + ',' + college.longitude

class CollegeLogoForm(forms.Form):
    logo_img = LOGO_Field()

class CollegeImgForm(forms.Form):
    img = Images_Field()

class CollegeCommentForm(forms.Form):
    comment = TextField()

class HeaderBasicForm(forms.Form):
    character = Character_Field()   # 学校性质: 大学 社区学院 高中
    basic01 = Header_Basic_Field()  # 雅思成绩
    basic02 = Header_Basic_Field()  # 托福成绩
    basic03 = Header_Basic_Field()  # 平均录取率
    basic04 = Header_Basic_Field()  # 学校主页
    basic05 = Header_Basic_Field()  # 申请页面
    basic06 = Header_Basic_Field()  # 学费
    basic07 = Header_Basic_Field()  # 学生数
    basic08 = Header_Basic_Field()  # 地理位置
    basic09 = Header_Basic_Field()  # 建校年份
    basic10 = Header_Basic_Field()  # 就业率
    # basic11 = Header_Basic_Field()

    def __init__(self, basic_data, *args, **kwargs):
        super(HeaderBasicForm, self).__init__(*args, **kwargs)
        if basic_data:
            self.fields['character'].initial = basic_data['character'] if basic_data.has_key("character") else None
            self.fields['basic01'].initial = basic_data['IELTS'] if basic_data.has_key("IELTS") else None
            self.fields['basic02'].initial = basic_data['TOEFL'] if basic_data.has_key("TOEFL") else None
            self.fields['basic03'].initial = basic_data['accept_rate'] if basic_data.has_key("accept_rate") else None
            self.fields['basic04'].initial = basic_data['official_website'] if basic_data.has_key("official_website") else None
            self.fields['basic05'].initial = basic_data['application_website'] if basic_data.has_key("application_website") else None
            self.fields['basic06'].initial = basic_data['tuition'] if basic_data.has_key("tuition") else None
            self.fields['basic07'].initial = basic_data['student_number'] if basic_data.has_key("student_number") else None
            self.fields['basic08'].initial = basic_data['position'] if basic_data.has_key("position") else None
            self.fields['basic09'].initial = basic_data['found'] if basic_data.has_key("found") else None
            self.fields['basic10'].initial = basic_data['employment_rate'] if basic_data.has_key("employment_rate") else None


#!/usr/bin/python
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import random
from collections import defaultdict

import torch
from torch.utils.data import Dataset
import torchvision.transforms as T

import numpy as np
import h5py
import PIL

from .utils import imagenet_preprocess, Resize

class VgEvalDataset(Dataset):
  def __init__(self, ground_truth_dir, model_out_dir, image_size=(64, 64), name_fn=None):
    super(VgEvalDataset, self).__init__()

    self.ground_truth_dir = ground_truth_dir
    self.model_out_dir = model_out_dir
    self.image_size = image_size
    self.model_out_imgs = os.listdir(model_out_dir)
    self.ground_truth_imgs = os.listdir(ground_truth_dir)

    transform = [Resize(image_size), T.ToTensor()]
    self.transform = T.Compose(transform)
    self.name_fn = name_fn

  def __len__(self):
    return len(self.model_out_imgs)

  def __getitem__(self, index):
    """
    Returns a tuple of:
    - image: FloatTensor of shape (C, H, W)
    """
    filename = self.model_out_imgs[index]
    model_out_img = os.path.join(self.model_out_dir, filename)
    if self.name_fn is not None:
        filename = self.name_fn(filename)
    ground_truth_img = os.path.join(self.ground_truth_dir, filename)

    with open(ground_truth_img, 'rb') as f:
        with PIL.Image.open(f) as image:
            ground_truth_img = self.transform(image.convert('RGB'))

    with open(model_out_img, 'rb') as f:
        with PIL.Image.open(f) as image:
            model_out_img = self.transform(image.convert('RGB'))

    return ground_truth_img, model_out_img

def vg_collate_fn(batch):
  """
  Collate function to be used when wrapping a VgSceneGraphDataset in a
  DataLoader. Returns a tuple of the following:

  - imgs: FloatTensor of shape (N, C, H, W)
  """
  # batch is a list, and each element is (image, objs, boxes, triples)
  all_ground_truth_imgs, all_model_out_imgs = [], []
  for i, (ground_truth_img, model_out_img) in enumerate(batch):
    all_ground_truth_imgs.append(ground_truth_img[None])
    all_model_out_imgs.append(model_out_img[None])

  all_ground_truth_imgs = torch.cat(all_ground_truth_imgs)
  all_model_out_imgs = torch.cat(all_model_out_imgs)

  out = (all_ground_truth_imgs, all_model_out_imgs)
  return out

def vg_uncollate_fn(batch):
  """
  Inverse operation to the above.
  """
  all_ground_truth_imgs, all_model_out_imgs = batch
  out = []
  for i in range(all_ground_truth_imgs.size(0)):
    cur_ground_truth_img = all_ground_truth_imgs[i]
    cur_model_out_img = all_model_out_imgs[i]
    out.append((cur_ground_truth_img, cur_model_out_img))
  return out


import sys,os
import MySQLdb
import urllib
import StringIO
import traceback
import time
import json
import ConfigParser
import multiprocessing

def filelog(filename):
    f = open("/tmp/%s" %(filename),"aw")
    f.write(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime()))
    f.write("\n")
    fp = StringIO.StringIO()
    traceback.print_exc(file=fp)
    errmsg = fp.getvalue()
    f.write(errmsg)
    f.write("\n")

def hisdown(stype):
    stype1 = "cdownget"
    stype2 = "cdowndel"

    url_his = "http://%s/interface/status/stat/%s/?biz=%s&work_type=DTS" % (ip,stype1,stype)

    while 1:
        a = 120
        try:
            conn = MySQLdb.connect(host="%s" %(ip_db),user="root",passwd="%s" % (passwds),db="%s" % (dbs),charset="utf8")
        except:
            filelog(fname)
        else:
            cur = conn.cursor()
            try:
                res1 = urllib.urlopen(url_his)
            except:
                filelog(fname)
            else:
                res2 = json.loads(res1.read())
                lens = len(res2)
                if lens == 20:
                    a = 1
                if res2:
                    for i in res2:
                        orgn_code = i.get("orgn_code")
                        status_type = i.get("status_type")
                        status_time = i.get("status_time")
                        status = i.get("status")
                        dts_orgn_code = i.get("dts_orgn_code")
                        sys_type = i.get("sys_type")
                        id = i.get("_id")
                        sql = "insert into %s_t_trans_his_center(orgn_code,status_type,status_time,status,dts_orgn_code,sys_type) values('%s','%s','%s','%s','%s','%s')" % (stype,orgn_code,status_type,status_time,status,dts_orgn_code,sys_type)
                        try:
                            cur.execute(sql)
                        except:
                            f = open("/tmp/%s" % (fname),"aw")
                            f.write(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime()))
                            f.write("\n")
                            fp = StringIO.StringIO()
                            traceback.print_exc(file=fp)
                            errmsg = fp.getvalue()
                            f.write(errmsg)
                            f.write("\n")
                            f.close()
                            res4 = errmsg.find("Duplicate")
                            if res4 != -1:
                                try:
                                    url_his1 = "http://%s/interface/status/stat/%s/?biz=%s&work_type=DTS&rows=%s" % (ip,stype2,stype,id)
                                    urllib.urlopen(url_his1)
                                except:
                                    filelog(fname)
                        else:
                            conn.commit()
                            try:
                                url_his2 = "http://%s/interface/status/stat/%s/?biz=%s&work_type=DTS&rows=%s" % (ip,stype2,stype,id)
                                urllib.urlopen(url_his2)
                            except:
                                filelog(fname)
                conn.close()
                time.sleep(a)

if __name__ == '__main__':
    filename = sys.argv[0][sys.argv[0].rfind(os.sep)+1:]
    # filename = fname[7:16]
    filename1 = "dts" + filename[7:16]
    fname = filename[:-3]
    confPath = sys.path[0]+'/configall.ini'
    cf = ConfigParser.ConfigParser()
    cfg = cf.read(confPath)
    ip = cf.get("%s" %(filename1),"ip")
    ip_db = cf.get("pubinfo","citymysql_ip")
    passwds = cf.get("pubinfo","citymysql_pass")
    dbs = cf.get("pubinfo","citymysql_elaw")

    thread1 = multiprocessing.Process(target=hisdown,args=('ss',))
    thread2 = multiprocessing.Process(target=hisdown,args=('ps',))
    thread3 = multiprocessing.Process(target=hisdown,args=('om',))
    thread1.start()
    thread2.start()
    thread3.start()


"""
Copyright 2021 Tsinghua University
Apache 2.0.
Author: Zheng Huahuan (zhh20@mails.tsinghua.edu.cn)

Directly execute: (in working directory)
    python3 ctc-crf/monitor.py <path to my exp>
"""

import sys
import os
import pandas as pd
import matplotlib.pyplot as plt

def plot_monitor(task: str, interactive_show=False):

    train_log = f'exp/{task}/ckpt/log_train.csv'
    dev_log = f'exp/{task}/ckpt/log_eval.csv'

    if not os.path.isfile(train_log):
        raise FileNotFoundError(f"'{train_log}' doesn't exist!")
    if not os.path.isfile(dev_log):
        raise FileNotFoundError(f"'{dev_log}' doesn't exist!")

    df_train = pd.read_csv(train_log)
    df_eval = pd.read_csv(dev_log)

    _, axes = plt.subplots(2, 2)

    # Time
    ax = axes[0][0]
    batch_per_epoch = len(df_train)//len(df_eval)
    accum_time = df_train['time'].values
    for i in range(1, len(accum_time)):
        accum_time[i] += accum_time[i-1]
        if (i + 1) % batch_per_epoch == 0:
            accum_time[i] += df_eval['time'].values[(i+1)//batch_per_epoch-1]
    accum_time = [x/3600 for x in accum_time]
    ax.plot(accum_time)
    props = dict(boxstyle='round', facecolor='wheat', alpha=0.8)
    speed = accum_time[-1]/len(df_eval)
    ax.text(0.05, 0.95, "{:.2f}h/epoch".format(speed), transform=ax.transAxes,
            fontsize=8, verticalalignment='top', bbox=props)

    ax.ticklabel_format(axis="x", style="sci", scilimits=(0, 0))
    ax.grid(ls='--')
    ax.set_ylabel('Total time / h')
    del ax

    # Learning rate
    ax = axes[0][1]
    lrs = df_train['net_lr'].values
    sim_lrs = [0]
    for i in range(1, len(lrs)):
        if lrs[i] != lrs[i-1]:
            sim_lrs += [i-1, i]
    if sim_lrs[-1] < len(lrs) - 1:
        sim_lrs.append(len(lrs)-1)

    if len(sim_lrs) > 1000 or len(sim_lrs) == 1:
        ax.semilogy(lrs)
    else:
        ax.set_yscale('log')
        for i in range(len(sim_lrs)-1):
            _xs = [sim_lrs[i], sim_lrs[i+1]]
            _ys = [lrs[sim_lrs[i]], lrs[sim_lrs[i+1]]]
            if _ys[0] == _ys[1]:
                ax.plot(_xs, _ys, color="C0")
            else:
                ax.plot(_xs, _ys, ls='--', color='black', alpha=0.5)
        del _xs
        del _ys
    del sim_lrs
    del lrs
    ax.ticklabel_format(axis="x", style="sci", scilimits=(0, 0))
    ax.grid(ls='--')
    ax.set_ylabel('learning rate')
    del ax

    # Training loss and moving average
    ax = axes[1][0]
    train_loss = df_train['loss_real'].values
    running_mean = [train_loss[0]]
    for i in range(1, len(train_loss)):
        running_mean.append(running_mean[i-1]*0.9+0.1*train_loss[i])
    min_loss = min(train_loss)
    if min_loss <= 0.:
        # ax.set_yscale('symlog')
        ax.plot(train_loss, alpha=0.3)
        ax.plot(running_mean, color='orangered')
    else:
        ax.semilogy(train_loss, alpha=0.3)
        ax.semilogy(running_mean, color='orangered')

    del train_loss
    del running_mean
    ax.ticklabel_format(axis="x", style="sci", scilimits=(0, 0))
    ax.grid(True, which="both", ls='--')
    ax.set_ylabel('Train set loss')
    ax.set_xlabel("Step")
    del ax

    # Dev loss
    ax = axes[1][1]
    min_loss = min(df_eval['loss_real'])
    if min_loss <= 0.:
        # ax.set_yscale('symlog')
        ax.plot([i+1 for i in range(len(df_eval))],
                df_eval['loss_real'].values)
    else:
        ax.semilogy([i+1 for i in range(len(df_eval))],
                    df_eval['loss_real'].values)

    ax.axhline(y=min_loss, ls='--', color='black', alpha=0.5)
    props = dict(boxstyle='round', facecolor='wheat', alpha=0.8)
    textstr = '\n'.join([
        "min={:.2f}".format(min_loss),
        f"{len(df_eval)} epoch"
    ])
    speed = accum_time[-1]/len(df_eval)
    ax.text(0.95, 0.95, textstr, transform=ax.transAxes,
            fontsize=8, verticalalignment='top', horizontalalignment='right', bbox=props)
    ax.grid(True, which="both", ls='--')
    ax.set_ylabel('Dev set loss')
    ax.set_xlabel('Epoch')
    del ax

    # Global settings
    titles = [
        task.replace('dev_', '')
    ]
    plt.suptitle('\n'.join(titles))
    plt.tight_layout()
    plt.savefig(f'exp/{task}/monitor.png', dpi=300)
    print(f'> Saved at exp/{task}/monitor.png')
    if interactive_show:
        plt.show()
    else:
        print("Current lr: {:.2e} | Speed: {:.2f} hour / epoch.".format(
            df_train['net_lr'].values[-1], speed))
    plt.close()
    return None

if __name__ == "__main__":
    if len(sys.argv) < 2:
        raise ValueError("Please input the experiment name.")

    task = sys.argv[1].strip('/').split('/')[-1]
    assert os.path.isdir(
        f'exp/{task}'), f"\'exp/{task}\' is not a valid directory!"

    plot_monitor(task)


import numpy as np
from parcels import FieldSet, ParticleSet, ScipyParticle, JITParticle, ErrorCode, AdvectionRK4, Variable,ParticleFile, Field, VectorField
from datetime import timedelta
from datetime import datetime
from timeit import default_timer as timer
# from plot_functions import plot_gif
import glob
import os

def periodicBC(particle, fieldset, time):
    """
    Kernel for periodic boundaries in longitude, not sure if this is correct
    """
    if particle.lon < 0.:
        particle.lon += 360.
    elif particle.lon > 360.:
        particle.lon -= 360.

def DeleteParticle(particle, fieldset, time):
    """Kernel for deleting particles if they are out of bounds."""
    particle.delete()

def Sample_sit(particle, fieldset, time):  # Custom function that samples fieldset.P at particle location
    """
    Kernel for attributing sea ice thickness to a particle
    """
    particle.sit = fieldset.sit[time, particle.depth, particle.lat, particle.lon]
    particle.sip = fieldset.sip[time, particle.depth, particle.lat, particle.lon]

def AdvectionRK4_ice(particle, fieldset, time):
    """Advection of particles using fourth-order Runge-Kutta integration.
    Function needs to be converted to Kernel object before execution in ice or in water"""

    if particle.sit > 1e-16:
        (u1, v1) = fieldset.UV[time, particle.depth, particle.lat, particle.lon]
        lon1, lat1 = (particle.lon + u1*.5*particle.dt, particle.lat + v1*.5*particle.dt)
        (u2, v2) = fieldset.UV[time + .5 * particle.dt, particle.depth, lat1, lon1]
        lon2, lat2 = (particle.lon + u2*.5*particle.dt, particle.lat + v2*.5*particle.dt)
        (u3, v3) = fieldset.UV[time + .5 * particle.dt, particle.depth, lat2, lon2]
        lon3, lat3 = (particle.lon + u3*particle.dt, particle.lat + v3*particle.dt)
        (u4, v4) = fieldset.UV[time + particle.dt, particle.depth, lat3, lon3]
        particle.lon += (u1 + 2*u2 + 2*u3 + u4) / 6. * particle.dt
        particle.lat += (v1 + 2*v2 + 2*v3 + v4) / 6. * particle.dt

    else:
        (u1, v1) = fieldset.UVocean[time, particle.depth, particle.lat, particle.lon]
        lon1, lat1 = (particle.lon + u1*.5*particle.dt, particle.lat + v1*.5*particle.dt)
        (u2, v2) = fieldset.UVocean[time + .5 * particle.dt, particle.depth, lat1, lon1]
        lon2, lat2 = (particle.lon + u2*.5*particle.dt, particle.lat + v2*.5*particle.dt)
        (u3, v3) = fieldset.UVocean[time + .5 * particle.dt, particle.depth, lat2, lon2]
        lon3, lat3 = (particle.lon + u3*particle.dt, particle.lat + v3*particle.dt)
        (u4, v4) = fieldset.UVocean[time + particle.dt, particle.depth, lat3, lon3]
        particle.lon += (u1 + 2*u2 + 2*u3 + u4) / 6. * particle.dt
        particle.lat += (v1 + 2*v2 + 2*v3 + v4) / 6. * particle.dt

def AdvectionRK4_ocean(particle, fieldset, time):
    """Advection of particles using fourth-order Runge-Kutta integration.
    Function needs to be converted to Kernel object before execution"""
    (u1, v1) = fieldset.UVocean[time, particle.depth, particle.lat, particle.lon]
    lon1, lat1 = (particle.lon + u1*.5*particle.dt, particle.lat + v1*.5*particle.dt)
    (u2, v2) = fieldset.UVocean[time + .5 * particle.dt, particle.depth, lat1, lon1]
    lon2, lat2 = (particle.lon + u2*.5*particle.dt, particle.lat + v2*.5*particle.dt)
    (u3, v3) = fieldset.UVocean[time + .5 * particle.dt, particle.depth, lat2, lon2]
    lon3, lat3 = (particle.lon + u3*particle.dt, particle.lat + v3*particle.dt)
    (u4, v4) = fieldset.UVocean[time + particle.dt, particle.depth, lat3, lon3]
    particle.lon += (u1 + 2*u2 + 2*u3 + u4) / 6. * particle.dt
    particle.lat += (v1 + 2*v2 + 2*v3 + v4) / 6. * particle.dt

def run_experiment(simdays = 2000, lat_npart = 30, lon_npart=50, ocean_currents = True, custom_kernel = "AdvectionRK4_ice" ):
    start = timer()
    Peeken = False

    res       = "0083"
    data_dir  = '/data2/imau/oceanparcels/hydrodynamic_data/NEMO-MEDUSA/ORCA%s-N006/means/' %res #Directory for nemo data
    outputdir = '/scratch/AnnekeV/output/' #Directory for output files
#     outputdir = '/home/students/6252699/thesis/parcels2/output/'

    if (simdays < 305):
        ifiles    = sorted(glob.glob(data_dir+'ORCA%s-N06_2001????d05I.nc' %res))
        ufiles    = sorted(glob.glob(data_dir+'ORCA%s-N06_2001????d05U.nc' %res))
        vfiles    = sorted(glob.glob(data_dir+'ORCA%s-N06_2001????d05V.nc' %res))

    else:
        ifiles    = sorted(glob.glob(data_dir+'ORCA%s-N06_200?????d05I.nc' %res))
        ufiles    = sorted(glob.glob(data_dir+'ORCA%s-N06_200?????d05U.nc' %res))
        vfiles    = sorted(glob.glob(data_dir+'ORCA%s-N06_200?????d05V.nc' %res))

    mesh_mask = data_dir + "ORCA%s-N06_20090813d05U.nc" %res
    filenames = {'U':   {'lon': mesh_mask, 'lat': mesh_mask, 'data': ifiles},
                 'V':   {'lon': mesh_mask, 'lat': mesh_mask, 'data': ifiles},
                 'sit': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ifiles}, # sea ice thickness
                 'sip': {'lon': mesh_mask, 'lat': mesh_mask, 'data': ifiles}} # sea ice presence

    variables = {'U': 'uice_ipa',
                 'V': 'vice_ipa',
                 'sit': 'sit'  ,
                 'sip':'sip'
                }

    dimensions = {'lat'  : 'nav_lat',
                  'lon'  : 'nav_lon',
                  'time' : 'time_counter'
                  }

    fieldset = FieldSet.from_nemo(filenames, variables, dimensions, allow_time_extrapolation=False)

    if ocean_currents:
        dimensionsU = {'data': 'uo', 'lon': 'nav_lon', 'lat': 'nav_lat', 'time': 'time_counter'}
        dimensionsV = {'data': 'vo', 'lon': 'nav_lon', 'lat': 'nav_lat', 'time': 'time_counter'}
        Uocean = Field.from_netcdf(ufiles, 'Uocean', dimensionsU, fieldtype='U', allow_time_extrapolation=False)
        Vocean = Field.from_netcdf(vfiles, 'Vocean', dimensionsV, fieldtype='V', allow_time_extrapolation=False)
        fieldset.add_field(Uocean)
        fieldset.add_field(Vocean)
        uv_ocean = VectorField('UVocean', fieldset.Uocean, fieldset.Vocean)
        fieldset.add_vector_field(uv_ocean)

    lat  = np.linspace(66,66.5,lat_npart)
    lon  = np.linspace(360-169.5,360-168,lon_npart)
    x,y  = np.meshgrid(lat,lon)

    npart = lat_npart *lon_npart
    time = np.repeat(np.datetime64('2001-02-01'),npart)

    if Peeken:
        '''Latitude and longitude from peeken'''
        path = '/home/students/6252699/thesis/data/'
        coordinates = np.loadtxt(path + "peeken_table1.txt", skiprows = 1)
        lat_peeken  = coordinates[:,0]
        lon_peeken  = coordinates[:,1]
        '''Pick last three since they were measured in 2005, hence in a time slot that we can calculate stuff with'''
        lat_peeken    = lat_peeken[-3:]
        lon_peeken    = lon_peeken[-3:]
        time = [np.datetime64('2016-08-29'), np.datetime64('2016-08-18'), np.datetime64('2016-09-19')]

    class Sea_Ice_Particle(JITParticle):         # Define a new particle class
        sit = Variable('sit', initial=np.nan)  # Variable 'p' initialised by sampling the pressure
        sip = Variable('sip', initial=np.nan)

    pset = ParticleSet.from_list(fieldset,
                                 pclass = Sea_Ice_Particle,
                                 time=time,
                                 lon=y,
                                 lat=x)

    kernels    = pset.Kernel(periodicBC) + pset.Kernel(Sample_sit) +  pset.Kernel(eval(custom_kernel))

    output_name = outputdir   + "run_02_09_bering_npart_{}_start_{}_lat_{}_{}_simdays_{}_".format(npart, time[0],lat[0],lat[-1],simdays) + custom_kernel
    output_file =  pset.ParticleFile(
                    name=output_name,
                    outputdt=timedelta(days=5))

    print("No. particles is {}".format(npart))
    print("Start time is ", time[0])
    print("Output name is " + output_name)

    pset.execute(kernels,
                 runtime     = timedelta(days=simdays),
                 dt          = timedelta(minutes=10),
                 output_file = output_file,
                 recovery    = {ErrorCode.ErrorOutOfBounds: DeleteParticle})

    end = timer()

    print("Time elapsed = {:.1f} s ".format(end-start))
    return output_name

if __name__ == "__main__":
    output_name = run_experiment()
    plot_gif(output_name, dt_days = 5, extra_title = "bering2000winter", with_background = True)


import torch, os
import torchvision
import torchvision.transforms as transforms
from tqdm import tqdm
import numpy as np

import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models

import pandas as pd
import csv

read = '~/Data'
write = 'Results/2CNN'

models = 'Models/2CNN/model-'
ext = '.pth'

trainName = write + '/train.csv'
classTrainName = write + '/classTrain.csv'

pd.DataFrame(columns = ['epoch', 'accuracy']).to_csv(trainName, index=False)
pd.DataFrame(columns = ['epoch', 'aeroplane', 'cat', 'deer', 'dog', 'frog']).to_csv(classTrainName, index=False)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=5)
        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)
        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=5)
        self.fc1 = nn.Linear(in_features=128, out_features=64)
        self.fc2 = nn.Linear(in_features=64, out_features=32)
        self.fc3 = nn.Linear(in_features=32, out_features=5)      # change out_features according to number of classes

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = F.avg_pool2d(x, kernel_size=x.shape[2:])
        x = x.view(x.shape[0], -1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

num_epochs = 15

########################################################################
# The output of torchvision datasets are PILImage images of range [0, 1].

# Apply necessary image transfromations here

transform = transforms.Compose([ #torchvision.transforms.RandomHorizontalFlip(p=0.5),
                                #torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(0.1, 0.1), scale=(0.8, 1.2)),
                                transforms.ToTensor(),
                                transforms.Normalize(mean=[0.5,0.5,0.5], std=[0.5, 0.5, 0.5])])
print(transform)

batch_size =  4

train_data_dir = read + '/train' # put path of training dataset
val_data_dir = read + '/val' # put path of validation dataset
test_data_dir = read + '/test' # put path of test dataset

trainset = torchvision.datasets.ImageFolder(root= train_data_dir, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,
                                          shuffle=True, num_workers=2)

valset = torchvision.datasets.ImageFolder(root= val_data_dir, transform=transform)
valloader = torch.utils.data.DataLoader(valset, batch_size=batch_size,
                                         shuffle=False, num_workers=2)

testset = torchvision.datasets.ImageFolder(root= test_data_dir, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,
                                         shuffle=False, num_workers=2)

def eval(loader, evalNet):
  correct = 0
  total = 0
  with torch.no_grad():
      for data in tqdm(loader):
          images, labels = data
          if torch.cuda.is_available():
              images, labels = images.cuda(), labels.cuda()
          outputs = evalNet(images)
          _, predicted = torch.max(outputs.data, 1)
          total += labels.size(0)
          correct += (predicted == labels).sum().item()
  return (100 * correct / total)

def find_classes():
    classes = ['aeroplane', 'cat', 'deer', 'dog', 'frog']
    class_to_idx = {classes[i]: i for i in range(len(classes))}
    return classes, class_to_idx

def classwise_test(loader, net):
########################################################################
# class-wise accuracy

    classes, _ = find_classes()
    n_class = len(classes) # number of classes

    class_correct = list(0. for i in range(n_class))
    class_total = list(0. for i in range(n_class))
    with torch.no_grad():
        for data in tqdm(loader):
            images, labels = data
            if torch.cuda.is_available():
                images, labels = images.cuda(), labels.cuda()
            outputs = net(images)
            _, predicted = torch.max(outputs, 1)
            c = (predicted == labels).squeeze()
            for i in range(4):
                label = labels[i].item()
                class_correct[label] += c[i].item()
                class_total[label] += 1

    for i in range(n_class):
        print('Accuracy of %10s : %2d %%' % (
            classes[i], 100 * class_correct[i] / class_total[i]))

    return (100 * class_correct[0] / class_total[0]), (100 * class_correct[1] / class_total[1]), (100 * class_correct[2] / class_total[2]), (100 * class_correct[3] / class_total[3]), (100 * class_correct[4] / class_total[4])

def test(isTest, epoch):

    net = Net()

    # transfer the model to GPU
    if torch.cuda.is_available():
        net = net.cuda()

    net.load_state_dict(torch.load( models + str(epoch) + ext ))

    perc = eval(trainloader, net)
    with open(trainName, 'a') as newFile:
        newFileWriter = csv.writer(newFile)
        newFileWriter.writerow([epoch+1, perc])
        print('Accuracy of the network on the train images: ' + str(perc))
    with open(classTrainName, 'a') as newFile:
        class0, class1, class2, class3, class4 = classwise_test(trainloader, net)
        newFileWriter = csv.writer(newFile)
        newFileWriter.writerow([epoch+1, class0, class1, class2, class3, class4])

for epoch in range(num_epochs):  # loop over the dataset multiple times
    print('epoch ', epoch + 1)
    test(False, epoch)

print('Finished')


import cv2
import numpy as np
import matplotlib.pyplot as plt
import datetime

im1 = cv2.imread('../output/000045.png')

template = cv2.imread('../output/template.png')

template = cv2.cvtColor(template, cv2.COLOR_BGR2RGB)

winSize = (64,64)
blockSize = (16,16)
blockStride = (8,8)
cellSize = (8,8)
nbins = 9
derivAperture = 1
winSigma = 4
histogramNormType = 0
L2HysThreshold = 2.0000000000000001e-01
gammaCorrection = 0
nlevels = 64

hog = cv2.HOGDescriptor(winSize,blockSize,blockStride,cellSize,nbins,derivAperture,winSigma,
                        histogramNormType,L2HysThreshold,gammaCorrection,nlevels)

#img = hog.compute(im1)
#tmp = hog.compute(im1[::2,::2,:])
#print(img.shape,tmp.shape)

winStride = (4,4)
padding = (8,8)
locations = ((20,20),)

#hist = hog.compute(im1[::2,::2,:],winStride,padding,locations)

meanShift = True
patch = (350,850)
# padding = (100,100)
template = im1[290:450,750:950,:]
print(template.shape)
locations = (patch,)
#hist = hog.compute(im1,winStride,padding,locations)
print(1)
# print(cv2.HOGDescriptor_getDefaultPeopleDetector().shape)
hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
r,c,_ = template.shape
#im1 = im1[800:1000,250:350,:]
sse = np.zeros(im1.shape[:2])
print(1.5)
start = datetime.datetime.now()
(rects, weights) = hog.detectMultiScale(im1, winStride=winStride,padding=padding,scale=1.5, useMeanshiftGrouping=meanShift)
print(2)
print(rects)
for (x,y,w,h) in rects:
    cv2.rectangle(im1,(x,y), (x+w, y+h), (0,255,0), 2)

# print
# print(im1.shape)
# w= 800
# h = 250
# for i in range(0,200,10):
#     for j in range(0,200,10):
#         locations = ((h+i,w+j),)
#         desc = hog.compute(im1,winStride,padding,locations)
#         sse[i,j] = np.sqrt(np.square(desc - hist).sum(axis=(0,1))).sum()

# print(np.unravel_index(np.argmin(sse),template.shape[:2]))
print((datetime.datetime.now()-start).total_seconds())
a = plt.figure()
a.add_subplot(121)
plt.imshow(im1)
a.add_subplot(122)
plt.imshow(template,cmap='gray')
plt.show()


#!/usr/bin/python3

import serial
import time

serialHandle = serial.Serial("/dev/ttyAMA0", 115200) # Initialize Port

command = {"MOVE_WRITE":1, "POS_READ":28, "LOAD_UNLOAD_WRITE": 31}

##
## Sending Package
##
def servoWriteCmd(id, cmd, par1 = None, par2 = None):
    buf = bytearray(b'\x55\x55')
    try:
        len = 3   # Set Default Package Length
        buf1 = bytearray(b'')
        ## Edit Package
        if par1 is not None:
            len += 2  #Increase Package Length
            par1 = 0xffff & par1
            buf1.extend([(0xff & par1), (0xff & (par1 >> 8))])
        if par2 is not None:
            len += 2
            par2 = 0xffff & par2
            buf1.extend([(0xff & par2), (0xff & (par2 >> 8))])

        buf.extend([(0xff & id), (0xff & len), (0xff & cmd)]) #Add id, package length
        buf.extend(buf1) #Add Package

        ##Checksum Package
        sum = 0x00
        for b in buf:
            sum += b
        sum = sum - 0x55 - 0x55
        sum = ~sum
        buf.append(0xff & sum)

        serialHandle.write(buf) #Send

    except Exception as e:
        print(e)

##
## Read Position
##
def readPosition(id):
    serialHandle.flushInput() # Flush Buffer
    servoWriteCmd(id, command["POS_READ"]) # Set Port to Read
    time.sleep(0.00034) # Delay for sending
    time.sleep(0.005)  # Delay for receiving
    count = serialHandle.inWaiting() # Count reading package length
    print("inWaiting = ", serialHandle.inWaiting)
    pos = None
    print("count =", count)
    if count != 0:
        recv_data = serialHandle.read(count)
        if count == 8:
            if recv_data[0] == 0x55 and recv_data[1] == 0x55 and recv_data[4] == 0x1C :
                 pos= 0xffff & (recv_data[5] | (0xff00 & (recv_data[6] << 8)))
    return pos

servoWriteCmd(1, command["LOAD_UNLOAD_WRITE"],0)  #Decouple Servo to enable manual control
while True:
    try:
        pos = readPosition(1) # Read Servo1 Postion
        print(pos)
        time.sleep(1)
    except Exception as e:
        print(e)
        break


from fastapi import FastAPI, File, UploadFile

# Create Consumer to read Text Corpus Values from the Kafka Cluster

# Crate Producer to Pass Created Audio Files to the Kafka Cluster

# List for holding fetched text values
fetched_data = []

suppose_values = [
    {'id': 1, 'text': 'I am working on the server'},
    {'id': 2, 'text': 'I am flabergased'},
    {'id': 3, 'text': 'I am milky'},
    {'id': 4, 'text': 'I am laughing'},
    {'id': 5, 'text': 'I am joking'},
    {'id': 6, 'text': 'I am playing'},
    {'id': 7, 'text': 'I am shoveling'}
]

async def getvalues():
    return suppose_values

app = FastAPI()

@app.get('/fetch-text')
async def fetch_text():
    # Check if we have previously fetched data with more than 20 data objects
    if(len(fetched_data) <= 5):
        print('Items lower than 5, fetching data')
        # Fetch Data
        data = await getvalues()
        fetched_data.extend(data)

    # Pop a Data Value from fetched_data list
    return_data = fetched_data.pop(0)
    print("\t-> Returning:", return_data)

    # Return Data Value (id and text)
    return return_data

@app.post('/upload-audio')
async def handle_upload_audio(file: UploadFile = File(...)):
    try:
        # filename give you myimage.jpg
        # content_type gives image/jpeg
        # file gives us a spooledtemporaryfile(file like object)

        # Upload file to S3 Bucket

        # Send Data to Kafka the text id and reference link from S3 using the producer

        # return Success or Failure
        # return {'filename': file.filename, 'content_type': file.content_type}
        return {'status': 'success'}

    except Exception as e:
        print(e)
        return {'status': 'failed', 'error-message': str(e)}