from chalice import Chalice
import validators
import re
import gzip
import urllib.request
import urllib.parse
import ImageFile
from bs4 import BeautifulSoup
from urllib.request import (
    Request,
    urlopen,
)

USER_AGENT = "Workspace Preview Bot"
IMAGE_LINK_MAX_WIDTH = 768
url = "https://docbao.vn/video/video-lao-xe-qua-cong-sat-roi-dam-vao-nha-dan-nu-tai-xe-khien-nguoi-than-gao-thet-tintuc688635"
# url='https://www.now.vn/ho-chi-minh/2-chi-em-com-ga-nha-trang'
parsed_uri = urllib.parse.urlsplit(url)
q = Request(url)
q.add_header('User-Agent', USER_AGENT)
html = urlopen(q, timeout=10)

encoding = html.getheader("Content-Encoding")

content = html.read()

if encoding == "gzip":
    content = gzip.decompress(content)

soup = BeautifulSoup(
    content.decode("utf-8", "ignore"),
    "html.parser"
)

title = str(soup.title.string)

# titles = re.compile("[-–|:•]+").split(title)

# title = titles[0].strip()

# Get the desc from whatever we can find
desc_elems = [soup.findAll(
    attrs={attr: re.compile(r"Desc", re.I)}
) for attr in ["name", "property"]]

for i in range(1):
    if len(desc_elems[i]) > 0:
        desc = desc_elems[i][0]["content"]
        break
    else:
        desc = ""

if len(desc.split()) > 30:
    desc = " ".join(desc.split()[0:29]).strip()

    desc = desc.strip("…")
    desc = desc.strip(".")
    desc += "..."

icon_link = soup.find("link", rel=re.compile(r"shortcut icon"))

if icon_link is None:
    icon_link = soup.find("link", rel=re.compile(r"icon"))

if icon_link is not None:
    # Check if icon link is global or relative
    icon_href = icon_link["href"]

    if icon_href.find("http") != -1:
        icon = icon_href
    elif icon_href.find("//") != -1:
        icon = 'http:' + icon_href
    else:
        icon = parsed_uri.hostname + icon_href

# Fetch Open Graph Image
image = soup.find("meta", property="og:image")

if image is None:
    # Use favicon if no image is specified
    image = icon

if image is not None:
    # Check if image link is global or relative
    image_link = image["content"]

    if image_link.find("http") != -1:
        image = image_link
    elif image_link.find("//") != -1:
        image = 'http:' + image_link
    else:
        image = parsed_uri.hostname + image_link

print('title:', title)
print('description:', desc)
print('image:', image)
print('icon:', icon)
print("=========================")

# from linkpreview import link_preview
# preview = link_preview(url)
# print("title:", preview.title)
# print("description:", preview.description)
# print("image:", preview.image)
# print("=========================")

# from link_preview import link_preview
# dict_elem = link_preview.generate_dict(url)
# print("title:", dict_elem['title'])
# print("description:", dict_elem['description'])
# print("image:", dict_elem['image'])
# print("=========================")


"""Tests the controller and database collection objects methods
directly. These tests rely on the `./tests/Pd` directory, which has the model
outputs that temporary directories will be compared to.
"""
import numpy as np
from os import path
import pytest
from matdb.utility import reporoot, relpath
from matdb.atoms import AtomsList
import six

def _mimic_vasp(folder, xroot, prefix="W.1"):
    """Copies a `vasprun.xml` and `OUTCAR ` output files from the given folder into
    the execution directory to mimic what VASP would have done.

    Args:
        folder (str): path to the folder where the model files are stored.
        xroot (str): path to the root folder where the config steps are stored.
    """
    from matdb.utility import chdir
    from glob import glob
    from os import path
    from matdb.utility import symlink

    files = ["vasprun.xml", "OUTCAR"]

    with chdir(folder):
        for vaspfile in files:
            pattern = vaspfile + "__*"
            for dft in glob(pattern):
                name, config = dft.split("__")
                xpath = path.join(xroot, path.join(*config.split("_")), prefix)
                #We want to make some testing assertions to ensure that the
                #stubs ran correctly.
                #Bypass checking the DynMatrix subfolder
                if "DynMatrix" in xpath:
                    continue
                assert path.isfile(path.join(xpath, "CONTCAR"))
                assert path.isfile(path.join(xpath, ".matdb.module"))
                target = path.join(xpath, name)
                symlink(target, path.join(folder, dft))

@pytest.fixture()
def Pd(tmpdir):
    from matdb.utility import relpath, copyonce
    from matdb.database import Controller
    from os import mkdir, symlink, remove, path

    target = relpath("./tests/Pd/matdb.yml")
    dbdir = str(tmpdir.join("pd_db"))
    mkdir(dbdir)
    copyonce(target, path.join(dbdir, "matdb.yml"))
    target = path.join(dbdir,"matdb")

    #We need to copy the POSCAR over from the testing directory to the temporary
    #one.
    from shutil import copy
    POSCAR = relpath("./tests/Pd/POSCAR")
    mkdir(path.join(dbdir,"seed"))
    copy(POSCAR, path.join(dbdir,"seed","Pd"))
    # the source file `matdb.yml` linked to might be gone, that left `matdb.yml` not an valid "file"
    # we need to get rid if it anyway
    try:
        remove("matdb.yml")
    except:
       pass
    symlink("{}.yml".format(target),"matdb.yml")

    result = Controller("matdb", dbdir)
    remove("matdb.yml")
    result = Controller(target, dbdir)
    return result

@pytest.fixture()
def Pd_copy(tmpdir):
    from matdb.utility import relpath, copyonce
    from matdb.database import Controller
    from os import path, remove, mkdir

    target = relpath("./tests/Pd/matdb_copy.yml")
    dbdir = str(tmpdir.join("pd_db_copy"))
    mkdir(dbdir)
    copyonce(target, path.join(dbdir, "matdb.yml"))
    target = path.join(dbdir,"matdb")

    from shutil import copy
    POSCAR = relpath("./tests/Pd/POSCAR")

    if path.isfile("matdb_copy.yml"):
        remove("matdb_copy.yml")

    result = Controller(target, dbdir)

    mkdir(path.join(dbdir,"seed"))
    copy(POSCAR, path.join(dbdir, "seed", "Pd"))
    result = Controller(target, dbdir)
    return result

@pytest.fixture()
def Pd_split(tmpdir):
    from matdb.utility import relpath, copyonce
    from matdb.database import Controller
    from os import mkdir, path

    target = relpath("./tests/Pd/matdb_split.yml")
    dbdir = str(tmpdir.join("pd_db_splits"))
    mkdir(dbdir)
    copyonce(target, path.join(dbdir, "matdb.yml"))
    target = path.join(dbdir,"matdb")

    from shutil import copy
    POSCAR = relpath("./tests/Pd/POSCAR")
    mkdir(path.join(dbdir,"seed"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-1"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-2"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-3"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-4"))
    copy(POSCAR, path.join(dbdir,"seed","Pd-5"))

    result = Controller(target, dbdir)
    return result

def test_Pd_setup(Pd, Pd_copy):
    """Makes sure the initial folders were setup according to the spec.
    """
    #raise Exception("RAWR")
    Pd.setup()
    modelroot = path.join(Pd.root, "Manual","phonon.manual","Pd")
    #import pdb; pdb.set_trace()
    assert Pd["Manual/phonon/Pd/"].root == modelroot

    #The matdb.yml file specifies the following database:
    dbs = ["Manual/phonon.manual/Pd/"]
    #Each one should have a folder for: ["hessian", "modulations"]
    #On the first go, the modulations folder will be empty because the DFT
    #calculations haven't been performed yet. However, hessian should have DFT
    #folders ready to go.
    folders = {
        "__files__": ["compute.pkl","jobfile.sh"],
        "S1.1": {
            "__files__": ["INCAR", "POSCAR", "POTCAR", "PRECALC", "KPOINTS"]
        }
    }

    from matdb.utility import compare_tree
    for db in dbs:
        dbfolder = path.join(Pd.root, db)
        compare_tree(dbfolder, folders)

    #import pdb; pdb.set_trace()
    #Now we will test some of the border cases of the database __init__ method
    Pd_copy.setup()

    db = "Manual/phonon.manual/Pd"
    dbfolder = path.join(Pd_copy.root, db)
    compare_tree(dbfolder, folders)

def test_steps(Pd):
    """Tests compilation of all steps in the database.
    """
    assert Pd.steps() == ['manual/phonon']
    Pd.setup()
    steps = sorted(['manual/phonon/Pd'])
    assert Pd.steps() == steps

    seqs = sorted(['Pd'])
    assert Pd.sequences() == seqs

def test_find(Pd):
    """Tests the find function and the __getitem__ method with pattern matching.
    """
    Pd.setup()
    steps = Pd.find("manual/phonon")
    model = ['phonon']
    assert model == [s.parent.name for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual')]
    assert sorted(model) == sorted([s.root for s in steps])

    steps = Pd.find("*/phonon")
    model = ['phonon']
    assert model == [s.parent.name for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual')]
    assert model == [s.root for s in steps]

    steps = Pd.find("manual/phonon/Pd")
    model = ['manual']
    assert model == [s.parent.name for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual/Pd')]
    assert model == [s.root for s in steps]

    steps = Pd.find("phonon")
    model = ['phonon']
    assert model == [s.name for s in steps]
    model = [Pd.root]
    assert model == [s.root for s in steps]

    steps = Pd.find('*')
    model = ['phonon']
    assert model == [s.name for s in steps]
    model = [Pd.root]
    assert model == [s.root for s in steps]

    steps = Pd.find("manual/phonon/Pd/S1.1")
    model = [('phonon','manual')]
    assert model == [(s.parent.parent.name,s.name) for s in steps]
    model = [path.join(Pd.root,'Manual/phonon.manual/Pd')]
    assert model == [s.root for s in steps]

    # test uuid finding.
    assert all([Pd.find(s.uuid)==s for s in steps])

    # test the __getitem__ method
    model = 'phonon'
    modelroot = path.join(Pd.root,'Manual/phonon.manual')
    group = Pd["manual/phonon"]
    assert group.parent.name == model
    assert group.root == modelroot

    model = 'manual'
    modelroot = path.join(Pd.root,'Manual/phonon.manual/Pd')
    group = Pd["manual/phonon/Pd"]
    assert group.parent.name == model
    assert group.root == modelroot

    group = Pd["manual/phonon/Pd/S1.1"]
    assert group.parent.name == model
    assert group.root == modelroot

    group = Pd["enumeration/phonon"]
    assert group == None

def test_execute(Pd, capsys):
    """Tests the execute and extract methods
    """
    from os import path
    from matdb.utility import relpath, chdir
    from matdb.msg import verbosity

    verbosity = 2

    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 0/0; finished executing 0/0;"
    assert status in output.out

    # test to ensure execute prints error message if ran before setup
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    output = capsys.readouterr()
    status = "Group phonon.manual is not ready to execute yet, or is already executing. Done."
    assert status in output.out

    # Run setup and status to make sure the staus output is correct
    Pd.setup()
    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 1/1; finished executing 0/1;"
    assert status in output.out

    # Execute the jobfile and test an incomplete OUTCAR to check the status
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot, "tests", "data", "Pd", "manual_recover")
    _mimic_vasp(folder, Pd.root,"S1.1")

    Pd.status(True)
    busy_status = "Pd./Manual/phonon.manual/Pd/S1.1"
    output = capsys.readouterr()
    assert busy_status in output.out

    #now use a complete OUTCAR and test the status again
    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")

    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 1/1; finished executing 1/1;"
    assert status in output.out

    # Run exctract and test to see if the status is correct
    Pd.extract()
    Pd.status()
    output = capsys.readouterr()
    status = "ready to execute 1/1; finished executing 1/1;"
    assert status in output.out

    # Run extract again to make sure the atoms.h5 files are no rewritten
    Pd.extract()

def test_recovery(Pd):
    """Tests the rerun on unfinshed jobs
    """
    from os import path
    from matdb.utility import symlink, chdir
    from glob import glob

    Pd.setup()
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})

    files = ["vasprun.xml", "OUTCAR"]
    folder = path.join(reporoot, "tests", "data", "Pd", "manual_recover")
    with chdir(folder):
        for vaspfile in files:
            pattern = vaspfile + "__*"
            for dft in glob(pattern):
                name, config = dft.split("__")
                xpath = path.join(Pd.root, path.join(*config.split("_")), "S1.1")
                target = path.join(xpath, name)
                symlink(target, path.join(folder, dft))

    Pd.extract()

    Pd.recover(True)
    assert path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","recovery.sh"))
    assert path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","failures"))

    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")
    Pd.recover(True)
    assert not path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","recovery.sh"))
    assert not path.isfile(path.join(Pd.root,"Manual","phonon.manual","Pd","failures"))

def test_hash(Pd):
    """Tests the hash_dbs and verify_hash methods
    """
    from os import path, chdir
    Pd.setup()
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")
    db_hash = Pd.hash_dbs()
    assert Pd.verify_hash(db_hash)

def test_finalize(Pd):
    """ Test the finalize function in the controller module
    """
    from os import path
    Pd.setup()
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot,"tests","data","Pd","manual")
    _mimic_vasp(folder,Pd.root,"S1.1")

    Pd.extract()
    Pd.split()
    Pd.finalize()

    from matdb.io import load_dict_from_h5
    from matdb import __version__
    import h5py

    str_ver = []
    for item in __version__:
        str_ver.append(str(item))
    mdb_ver = ".".join(str_ver)
    target = path.join(Pd.root, "final_{}.h5".format(mdb_ver))
    with h5py.File(target, "r") as hf:
        loaded_final = load_dict_from_h5(hf)
    assert path.isfile(target)

def test_split(Pd_split):
    """ Test the split function in the controller object
    """
    from os import path
    Pd_split.setup()
    Pd_split.execute(env_vars={"SLURM_ARRAY_TASK_ID":"1"})
    folder = path.join(reporoot,"tests","data","Pd","manual_split")
    _mimic_vasp(folder,Pd_split.root,"S1.1")

    Pd_split.extract()
    Pd_split.split()

    for dbname, db in Pd_split.collections.items():
        for s, p in db.splits.items():
            tfile = path.join(db.train_file(s).format(s))
            hfile = path.join(db.holdout_file(s).format(s))
            sfile = path.join(db.super_file(s).format(s))

            tal = AtomsList(tfile)
            hal = AtomsList(hfile)
            sal = AtomsList(sfile)

            assert len(tal) == int(np.ceil(5*p))
            assert len(hal) == int(np.ceil((5-len(tal))*p))
            assert len(sal) == 5-len(tal)-len(hal)

def test_Pd_hessian(Pd):
    """Tests the `niterations` functionality and some of the standard
    methods of the class on simple Pd.

    """
    from os import remove
    Pd.setup()

    #Test the status, we should have some folder ready to execute.
    Pd.status()
    Pd.status(busy=True)

    #Test execution command; this uses stubs for all of the commands that would
    #be executed.
    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID": "1"})
    folder = path.join(reporoot, "tests", "data", "Pd", "recover")
    _mimic_vasp(folder, Pd.root)

    #Now that we have vasp files, we can queue recovery. The recover
    #`vasprun.xml` files that we linked to are not complete for all the
    #structures (on purpose).
    Pd.recover()
    recoveries = ["Manual/phonon/Pd"]

    for rkey in recoveries:
        assert path.isfile(path.join(Pd[rkey].root, "recovery.sh"))

    Pd.execute(env_vars={"SLURM_ARRAY_TASK_ID": "1"}, recovery=True)
    folder = path.join(reporoot, "tests", "data", "Pd", "manual")
    _mimic_vasp(folder,Pd.root,"S1.1")
    remove(path.join(Pd[rkey].root, "recovery.sh"))

    #Now that we have recovered vasp files, we can queue a *second*
    #recovery. All but one of the `vasprun.xml` files that we linked to are
    #complete for all the structures. This test that we can recover and execute
    #multiple times in order.
    Pd.recover()
    okay = ["Manual/phonon/Pd"]
    for rkey in okay:
        assert not path.isfile(path.join(Pd[rkey].root, "recovery.sh"))


import cv2
import numpy as np

# بارگیری مدل پیش آموزش دیده برای تشخیص علف‌های هرز
model = cv2.dnn.readNetFromCaffe("path/to/caffe/deploy.prototxt", "path/to/caffe/model.caffemodel")

# تنظیم تنظیمات ورودی و خروجی مدل
input_size = (300, 300)  # اندازه ورودی مدل
scale_factor = 1.0       # فاکتور مقیاس‌بندی تصویر ورودی

# بارگیری ویدئو
video = cv2.VideoCapture("path/to/video.mp4")

while True:
    ret, frame = video.read()
    if not ret:
        break

    # تغییر اندازه تصویر ورودی
    resized_frame = cv2.resize(frame, input_size)

    # ساخت بلوب از تصویر ورودی
    blob = cv2.dnn.blobFromImage(resized_frame, scale_factor, input_size, (104, 177, 123))

    # اجرای تشخیص علف‌های هرز بر روی بلوب
    model.setInput(blob)
    detections = model.forward()

    # نمایش نتایج تشخیص
    for i in range(detections.shape[2]):
        confidence = detections[0, 0, i, 2]
        if confidence > 0.5:  # تعیین آستانه اطمینان برای تشخیص
            # محاسبه مستطیل محدود کننده علف هرز
            box = detections[0, 0, i, 3:7] * np.array([input_size[0], input_size[1], input_size[0], input_size[1]])
            (startX, startY, endX, endY) = box.astype("int")

            # نمایش مستطیل و علف هرز تشخیص داده شده در تصویر اصلی
            cv2.rectangle(frame, (startX, startY), (endX, endY), (0, 255, 0), 2)

    # نمایش ویدئو با تشخیص علف‌های هرز
    cv2.imshow("Weed Detection", frame)
    if cv2.waitKey(1) == ord('q'):
        break

# آزاد کردن منابع
video.release()
cv2.destroyAllWindows()


from django.urls import path
from . import views

urlpatterns = [
    path('novo_pet/', views.new_pet, name="new_pet"),
    path('seus_pets/', views.your_pets, name="your_pets"),
    path('remover_pet/<int:id>', views.remove_pet, name="remove_pet"),
    path('ver_pet/<int:id>', views.see_pet, name="see_pet"),
    path('ver_pedido_adocao/', views.see_adopt_application, name="see_adopt_application"),
    path('dashboard/', views.dashboard, name="dashboard"),
    path('api_adocoes_por_raca/', views.adoption_by_breed_api, name="adoption_by_breed_api"),
]


import seaborn as sns
import matplotlib.pyplot as plt

from hrpe.models.eval import score_model
from hrpe.data.load import load_minute_data
from hrpe.features.time import make_datetime_features

data = load_minute_data("staplegrove")
data = make_datetime_features(data)

# First exploration: how many periods have bad data points?
badpp = data.groupby("period_time").agg(
    {"quality": lambda x: x.isin(["Bad", "Bad Ip"]).sum()}
)
print(f"There are {sum(badpp.quality > 0)} periods with bad data out of {len(badpp)}")
print(f"There are {sum(badpp.quality == 30)} periods that are entirely bad")
# 94 of 30672 periods have some bad data, is 0.3%
# 51 periods have entirely bad data data (all 30 mins are bad)
# 0.3% is not much to drop so just drop it
badpp["is_bad"] = badpp["quality"] > 0
data = data.merge(badpp[["is_bad"]], on="period_time")
data = data[~data["is_bad"]]

# Calculate the half-hourly features data from the minute data
hh_data = data.groupby("period_time").agg({"value": ["max", "min", "mean"]})
hh_data.columns = ["_".join(col) for col in hh_data.columns.to_flat_index()]
hh_data["time"] = hh_data.index
hh_data = make_datetime_features(hh_data).reset_index(drop=True)

hh_data["delta_max"] = hh_data["value_max"] - hh_data["value_mean"]
hh_data["delta_min"] = hh_data["value_mean"] - hh_data["value_min"]
hh_data["is_weekday"] = hh_data["is_weekday"].astype(int)

# plot delta_max over time
g = sns.FacetGrid(hh_data, col="period", col_wrap=8)
g.map(sns.lineplot, "time", "delta_max")
plt.show()
# Plot shows that the larger deltas tend to occur at 6am to 6pm which makes sense

g = sns.FacetGrid(hh_data, col="period", col_wrap=8)
g.map(sns.lineplot, "time", "delta_min")
plt.show()
# Fairly similar patterns for delta_min

# doens't seem to be much in day of week
g = sns.FacetGrid(hh_data[hh_data["period"] == 24], row="day_of_week")
g.map(sns.lineplot, "time", "delta_max")
plt.show()

# Value mean relationship is kind of weird, I can't quite explain it -
# as value_mean increases (i.e. 30min avg demand increases) the delta_max decreases;
# but is more variable at lower value_mean?
sns.relplot(data=hh_data, x="value_mean", y="delta_max", col="period", col_wrap=8)
sns.relplot(data=hh_data, x="value_mean", y="delta_max")
sns.relplot(data=hh_data, x="value_mean", y="delta_min")
plt.show()

# %%
# Test eval function
preds = hh_data[["time", "value_max", "value_min", "value_mean"]]
truths = preds.copy()

bench = preds[["time", "value_mean"]]
bench["value_max"] = preds["value_mean"]
bench["value_min"] = preds["value_mean"]

score_model(preds, truths)  # should be 0
score_model(bench, truths)  # should be 1


#!/usr/bin/env python3
import numpy as np
import cv2
from skimage import segmentation, color
from skimage.future import graph
from matplotlib import pyplot as plt
from skimage.exposure import rescale_intensity

np.seterr(divide='ignore', invalid='ignore')

def convolution(image, kernel):
    """
    Implementation of convolution operator.
    """
    (iH, iW) = image.shape  # image height and width
    (kH, kW) = kernel.shape  # kernel height and width
    pad = kH // 2  # tiling window
    # output image with image borders processing
    image = cv2.copyMakeBorder(image, pad, pad, pad, pad,
                               cv2.BORDER_REPLICATE)
    image_padded = np.zeros(shape=(iH + kH + pad, iW + kW + pad))
    image_padded[pad:-pad, pad:-pad] = image
    out = np.zeros(shape=image.shape)  # create output image as original image

    # sliding kernel over the image and compute convolution by element-wise multiplication
    for y in np.arange(pad, iH + pad):
        for x in np.arange(pad, iW + pad):
            for i in range(kH):
                for j in range(kW):
                    out[y, x] += image_padded[y + i, x + j] * kernel[i, j]

    # rescale the output image to be in the range [0, 255]
    out = rescale_intensity(out, in_range=(0, 255))
    out = (out * 255).astype("uint8")
    return out

def sobel_filter(data):
    """
    Implementation of the convolution of a Sobel filter 3x3 with the final goal of obtaining edges in images.
    """
    img = cv2.imread(data)
    grey_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    sobelX = np.array(([-1, 0, 1],
                       [-2, 0, 2],
                       [-1, 0, 1]), dtype="int")
    sobelY = np.array(([-1, -2, -1],
                       [0, 0, 0],
                       [1, 2, 1]), dtype="int")
    """
    # shifted kernel matrix to highlight edges in the opposite way
    sobelX = np.array(([+1, 0, -1],
                       [+2, 0, -2],
                       [+1, 0, -1]), dtype="int")
    sobelY = np.array(([+1, +2, +1],
                       [0, 0, 0],
                       [-1, -2, -1]), dtype="int")
    """

    Gx = convolution(grey_img, sobelX)  # horizontal gradient
    Gy = convolution(grey_img, sobelY)  # vertical gradient

    # OpenCV equivalent computations
    Gx_cv2 = cv2.Sobel(grey_img, cv2.CV_8U, 1, 0, ksize=3)
    Gy_cv2 = cv2.Sobel(grey_img, cv2.CV_8U, 0, 1, ksize=3)

    return grey_img, Gx, Gy, Gx_cv2, Gy_cv2

def normalized_cut(data):
    """
    Image segmentation through normalized cut.
    """
    img = cv2.imread(data)
    labels1 = segmentation.slic(img, compactness=30, n_segments=400)
    segmented_image = color.label2rgb(labels1, img, kind='avg')

    g = graph.rag_mean_color(img, labels1, mode='similarity')
    labels2 = graph.cut_normalized(labels1, g)
    cut_normalized_image = color.label2rgb(labels2, img, kind='avg')

    fig, ax = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(6, 8))

    ax[0].imshow(segmented_image)
    ax[1].imshow(cut_normalized_image)
    return fig

def sift_descriptor(data):
    """
    Create the SIFT descriptor of an image.
    """
    image = cv2.imread(data)
    grey_img = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    sift = cv2.xfeatures2d.SIFT_create()
    kp, des = sift.detectAndCompute(grey_img, None)
    return kp, des


import datetime
from dateutil.parser import parse as parse_datetime
from collections import OrderedDict
from typing import Dict, List, Union

class Tick(object):
    """
    Encapsulate scraped data into it's own object

    @param timestamp: The timestamp of the tick as an int epoch, datetime object, or string
    @param open: The open price of the tick
    @param high: The high price of the tick
    @param low: The low price of the tick
    @param close: The close price of the tick
    @param volume: The volume of the tick
    """

    def __init__(
        self,
        timestamp: Union[datetime.datetime, int, str],
        open: float,
        high: float,
        low: float,
        close: float,
        volume: int
    ):
        if not isinstance(timestamp, datetime.datetime):
            if isinstance(timestamp, (int, float)):
                timestamp = datetime.datetime.fromtimestamp(timestamp)
            elif isinstance(timestamp, str):
                try:
                    timestamp = datetime.datetime.fromtimestamp(float(timestamp))
                except:
                    timestamp = parse_datetime(timestamp)

        self.timestamp = timestamp
        self.open = float(open) if open is not None else None
        self.high = float(high) if open is not None else None
        self.low = float(low) if low is not None else None
        self.close = float(close) if close is not None else None
        self.volume = int(volume) if volume is not None else None

    @property
    def as_list(self) -> List:
        """
        Retrieve the tick data as a list of the timestamp, open price, high price,
        low price, close price, and volume

        @return: [timestamp, open price, high price, low price, close price, volume]
        """

        return [self.timestamp, self.open, self.high, self.low, self.close, self.volume]

    @property
    def as_dict(self) -> Dict:
        """
        Retrieve the tick data as a dictionary

        @return: {timestamp, open, high, low, close, volume}
        """

        return OrderedDict(
            timestamp=self.timestamp,
            open=self.open,
            high=self.high,
            low=self.low,
            close=self.close,
            volume=self.volume
        )


# -*- coding: utf-8 -*-
import argparse
import datetime
import glob
import multiprocessing
import platform

import pandas as pd

from utils.utils import get_json_tasks, get_client_tasks, json_wrapper, client_wrapper

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Process some integers.')
    parser.add_argument('-pid', type=str, help='Product ID IMG or MOD')
    parser.add_argument('-mode', type=str, help='Mode to be selected between roi based or csv based')
    parser.add_argument('-y', type=str, help='Year if mode is selected as csv')
    parser.add_argument('-dn', type=str, help='Daynight')
    parser.add_argument('-roi', type=str, help='lon_min,lat_min,lon_max,lat_max')
    parser.add_argument('-sd', type=str, help='Start Date if mode is selected as roi')
    parser.add_argument('-ed', type=str, help='Start Date if mode is selected as roi')
    parser.add_argument('-rs', type=bool, help='Resume downloading', default=False)
    args = parser.parse_args()
    mode = args.mode
    product_id = args.pid
    rs = args.rs
    dn = []
    for i in range(len(args.dn)):
        dn.append(args.dn[i])
    dates = ['2018-07-03', '2018-07-04', '2018-07-05', '2018-07-06', '2018-07-07']
    if platform.system()=='Windows':
        dir_json = 'data\\VNPL1'
        dir_nc = 'G:\\viirs\\VNPNC'
    elif platform.system()=='Darwin':
        dir_json = 'data\\VNPL1'
        dir_nc = 'G:\\viirs\\VNPNC'
    else:
        dir_json = 'data/VNPL1'
        dir_nc = 'data/VNPNC'
    num_processes = 8

    collection_id = '5200' # 5110 for VNP series
    products_id = ['VNP02'+product_id, 'VNP03'+product_id] #['VJ102IMG', 'VJ103IMG'] ['VNP02MOD', 'VNP03MOD'], ['VNP02IMG', 'VNP03IMG'], ['VJ102MOD', 'VJ103MOD']

    if mode == 'csv':
        year = args.y
        filename = 'roi/us_fire_' + year + '_out_new.csv'
        df = pd.read_csv(filename)
        df = df.sort_values(by=['Id'])
        ids, start_dates, end_dates, lons, lats = df['Id'].values.astype(str), df['start_date'].values.astype(str), df['end_date'].values.astype(str), df['lon'].values.astype(float), df['lat'].values.astype(float)
        for i, id in enumerate(ids):
            if rs:
                log_list = glob.glob('log/'+'nc_check*.log')
                log_list.sort()
                log_file = log_list[-1]
                with open(log_file) as f:
                    file = f.read()
                    missing_list = file.split('\n')
                    missing_list = missing_list[:-1]
                    missing_dates = [i.split('/')[3] for i in missing_list]
            lon, lat, start_date, end_date = lons[i], lats[i], start_dates[i], end_dates[i]
            roi_size = 1
            roi = [lon - roi_size, lat - roi_size, lon + roi_size, lat + roi_size]
            duration = datetime.datetime.strptime(end_date, '%Y-%m-%d')-datetime.datetime.strptime(start_date, '%Y-%m-%d')
            area_of_interest = 'W'+str(roi[0])+' '+'N'+str(roi[3])+' '+'E'+str(roi[2])+' '+'S'+str(roi[1])
            print('Currently processing id {}'.format(id))
            tasks = get_json_tasks(id, start_date, duration, area_of_interest, products_id, dn, dir_json, collection_id)
            tasks2 = get_client_tasks(id, start_date, duration, products_id, dn, dir_json, dir_nc, collection_id)
            with multiprocessing.Pool(processes=4) as pool:
                results = list(pool.imap_unordered(json_wrapper, tasks))
            print(results)
            with multiprocessing.Pool(processes=4) as pool:
                results = list(pool.imap_unordered(client_wrapper, tasks2))
            print(results)
    elif mode == 'error':
        year = args.y
        filename = 'roi/us_fire_' + year + '_out_new.csv'
        df = pd.read_csv(filename)
        df = df.sort_values(by=['Id'])
        with open('log/error', 'r') as f:
            file = f.read().split('\n')

        def get_id(dir_str):
            return dir_str.split('/')[1]
        target_ids = list(map(get_id, file))

        def get_date(dir_str):
            return dir_str.split('/')[-1][:10]
        target_dates = list(map(get_date, file))

        for i, target_id in enumerate(target_ids):
            df_row = df[df.Id==int(target_id)]
            lon, lat, start_date = df_row['lon'].values.astype(float), df_row['lat'].values.astype(float), target_dates[i]
            roi_size = 1
            roi = [lon - roi_size, lat - roi_size, lon + roi_size, lat + roi_size]
            area_of_interest = 'W'+str(roi[0])+' '+'N'+str(roi[3])+' '+'E'+str(roi[2])+' '+'S'+str(roi[1])
            print('Currently processing id {}'.format(target_id))
            duration = datetime.timedelta(days=1)
            tasks = get_json_tasks(target_id, start_date, duration, area_of_interest, products_id, dn, dir_json, collection_id)
            tasks2 = get_client_tasks(target_id, start_date, duration, products_id, dn, dir_json, dir_nc, collection_id)
            with multiprocessing.Pool(processes=4) as pool:
                results = list(pool.imap_unordered(json_wrapper, tasks))
            print(results)
            with multiprocessing.Pool(processes=4) as pool:
                results = list(pool.imap_unordered(client_wrapper, tasks2))
            print(results)
    elif mode == 'roi':
        roi_arg = args.roi
        start_date = args.sd
        end_date = args.ed
        roi = [float(roi_arg.split(',')[0]), float(roi_arg.split(',')[1]), float(roi_arg.split(',')[2]), float(roi_arg.split(',')[3])]
        duration = datetime.datetime.strptime(end_date, '%Y-%m-%d') - datetime.datetime.strptime(start_date, '%Y-%m-%d')
        area_of_interest = 'W' + str(roi[0]) + ' ' + 'N' + str(roi[3]) + ' ' + 'E' + str(roi[2]) + ' ' + 'S' + str(
            roi[1])
        id = 'Donnie_Creek_Fire'
        print('Currently processing id {}'.format(id))
        tasks = get_json_tasks(id, start_date, duration, area_of_interest, products_id, dn, dir_json, collection_id)
        tasks2 = get_client_tasks(id, start_date, duration, products_id, dn, dir_json, dir_nc, collection_id)
        with multiprocessing.Pool(processes=4) as pool:
            results = list(pool.imap_unordered(json_wrapper, tasks))
        with multiprocessing.Pool(processes=4) as pool:
            results = list(pool.imap_unordered(client_wrapper, tasks2))
    elif mode == 'roi_from_csv':
        fire_name = args.y
        if fire_name not in ['quebec_fire', 'alberta_fire']:
            raise 'For roi_from_csv mode quebec_fire or alberta_fire is expected from year arguments'
        filename = 'roi/'+fire_name+'.csv'
        df = pd.read_csv(filename)
        ids, start_dates, end_dates, bottom_lefts, top_rights = df['system:index'].values.astype(str), df['start_date'].values.astype(str), df['end_date'].values.astype(str), df['bottom_left'].values.astype(str), df['top_right'].values.astype(str)
        for i, id in enumerate(ids):
            if i<8:
                continue
            roi = (bottom_lefts[i] + top_rights[i]).replace('[', '').split(']')[0].split(', ')+(bottom_lefts[i] + top_rights[i]).replace('[', '').split(']')[1].split(', ')
            start_date = start_dates[i].split('T')[0]
            end_date = end_dates[i].split('T')[0]
            duration = datetime.datetime.strptime(end_date, '%Y-%m-%d') - datetime.datetime.strptime(start_date, '%Y-%m-%d')
            area_of_interest = 'W' + str(roi[0]) + ' ' + 'N' + str(roi[3]) + ' ' + 'E' + str(roi[2]) + ' ' + 'S' + str(roi[1])
            id = fire_name+'_'+id
            print('Currently processing id {}'.format(id))
            tasks = get_json_tasks(id, start_date, duration, area_of_interest, products_id, dn, dir_json, collection_id)
            tasks2 = get_client_tasks(id, start_date, duration, products_id, dn, dir_json, dir_nc, collection_id)
            with multiprocessing.Pool(processes=4) as pool:
                results = list(pool.imap_unordered(json_wrapper, tasks))
            with multiprocessing.Pool(processes=4) as pool:
                results = list(pool.imap_unordered(client_wrapper, tasks2))
    else:
        raise('No Support Mode')


import logging
import os.path
import pathlib
import sys

from citygame.src.constants.game_constants import SAVE_FOLDER_NAME

LOG = logging.getLogger("PathsUtil")

def get_save_file_directory() -> str:
    home_directory = str(pathlib.Path.home().absolute())

    # Default to relative path
    save_file_directory = os.path.join(os.getcwd(), "saves")

    if sys.platform.startswith("win32"):
        save_file_directory = os.path.join(home_directory, f"AppData/Local/Saved Games/{SAVE_FOLDER_NAME}")
    elif sys.platform.startswith("linux"):
        save_file_directory = os.path.join(home_directory, f"saves/{SAVE_FOLDER_NAME}")
    elif sys.platform.startswith("darwin"):
        save_file_directory = os.path.join(home_directory, f"Documents/SavedGames/{SAVE_FOLDER_NAME}")
    else:
        LOG.warning("Could not determine OS!")

    return save_file_directory


import argparse
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['text.usetex'] = True
plt.rcParams['text.latex.preamble'] = [r'\usepackage[cm]{sfmath}']
plt.rcParams['font.family'] = 'sans-serif'
plt.rcParams['font.sans-serif'] = 'cm'
plt.rcParams['axes.facecolor']='white'
plt.rcParams['savefig.facecolor']='white'

parser = argparse.ArgumentParser()
parser.add_argument('--show', action='store_true')
args = parser.parse_args()

# read the data from the files
[pre_iters, pre_res, pre_grad, pre_feas, _, _, _, _] = \
    np.loadtxt('./kona_krylov_pre.dat', unpack=True)
[nopre_iters, nopre_res, nopre_grad, nopre_feas, _, _, _, _] = \
    np.loadtxt('./kona_krylov.dat', unpack=True)

# set some formating parameters
axis_fs = 8 # axis title font size
label_fs = 8 # axis labels' font size

# plot cconvergence
fig = plt.figure(figsize=(4, 3), dpi=300)
ax = fig.add_subplot(111)

ms = 4.0 # marker size
mew = 0.75 # marker edge width
lw = 0.75 # line width
ax.semilogy(pre_iters, pre_res, '-^k', ms=ms, mfc='w', mew=mew,
            linewidth=lw, label='Preconditioned')
ax.semilogy(nopre_iters, nopre_res, ':ok', ms=ms, mfc='w', mew=mew,
            linewidth=lw, label='Not Preconditioned')

# plot formatting
ax.axis([0, 16, 10**-4, 10**1])
ax.set_xlabel('Krylov iterations', fontsize=axis_fs, weight='bold')
ax.set_ylabel('Relative residual norm', fontsize=axis_fs, weight='bold', labelpad=6)
ax.xaxis.tick_bottom() # use ticks on bottom only
ax.yaxis.tick_left()
for label in ax.xaxis.get_ticklabels():
    label.set_fontsize(label_fs)
for label in ax.yaxis.get_ticklabels():
    label.set_fontsize(label_fs)

leg = ax.legend(loc='upper right', fontsize=label_fs, labelspacing=0.75, borderpad=0.75,
                numpoints=1, handlelength=3, fancybox=False, framealpha=1.0, edgecolor='k')

save_dir = '/Users/denera/Documents/RPI/Optimal Design Lab/IDF-RSNK-journal'
plt.savefig('%s/nozzle_krylov.eps'%save_dir, format='eps', dpi=300,
            bbox_inches='tight')
plt.savefig('%s/nozzle_krylov.png'%save_dir, format='png',
            bbox_inches='tight')
if args.show:
    plt.show()
plt.close()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from collections import OrderedDict
import io
import os
from editolido.constants import PINS, GOOGLE_ICONS, PIN_NONE

try:
    # noinspection PyUnresolvedReferences
    integer_types = (int, long,)
except NameError:
    integer_types = (int,)

default_linestyle = """
    <Style id='{0}'>
        <LineStyle>
            <width>6</width>
            <color>{{{0}_color}}</color>
        </LineStyle>
    </Style>
"""
default_iconstyle = """
    <Style id='{0}'>
        <IconStyle>
            <Icon>
                <href><![CDATA[{1}]]></href>
            </Icon>
            <hotSpot x="0.5"  y="0.0" xunits="fraction" yunits="fraction"/>
        </IconStyle>
    </Style>
"""

class KMLGenerator(object):
    def __init__(self, template=None, point_template=None, line_template=None,
                 folder_template=None, style_template=default_linestyle,
                 icon_template=default_iconstyle, segment_template=None,
                 icons=GOOGLE_ICONS):
        datadir = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), 'data')

        def read_template(name):
            with io.open(os.path.join(datadir, name),
                         encoding='utf-8') as f:
                return f.read()

        self.template = (template
                         or read_template('mapsme_template.kml'))
        self.point_template = (point_template
                               or read_template('mapsme_point_tpl.kml'))
        self.line_template = (line_template
                              or read_template('mapsme_line_tpl.kml'))
        self.folder_template = (folder_template
                                or read_template('mapsme_folder_tpl.kml'))
        self.segment_template = (segment_template
                                 or read_template('mapsme_segment_tpl.kml'))
        self.style_template = style_template
        self.icon_template = icon_template
        self.folders = OrderedDict()
        self.styles = OrderedDict()
        self.folder_styles = OrderedDict()
        self.icons = icons
        self.pin_styles = []  # pin style register to avoid duplicates

    @staticmethod
    def escape(text):
        text = text.replace("<", "<")
        text = text.replace(">", ">")
        text = text.replace("&", "&")
        text = text.replace("\"", """)
        return text

    def add_folder(self, name, pin=PIN_NONE):
        """
        Add a folder and a style to the kml
        If there is no pin you must specify style for points added into
        that folder
        :param name: the folder name
        :param pin: a default pin for points in the folder
        """
        self.folders[name] = []
        style = self.style_template.format(name)
        if pin != PIN_NONE and PINS[pin] not in self.pin_styles:
            style += self.icon_template.format(PINS[pin][1:], self.icons[pin])
            self.pin_styles.append(PINS[pin])
        self.styles[name] = style
        self.folder_styles[name] = PINS[pin]

    def add_folders(self, *values):
        for value in values:
            try:
                name, pin = value
            except ValueError:
                name = value
                pin = PIN_NONE
            self.add_folder(name, pin)

    @staticmethod
    def _update_kwargs(folder, kwargs):
        """
        Apply style value based on context
        - if style is not set, uses folder name
        - if style is integer, replace with the style's name
        :param folder: str
        :param kwargs: list
        """
        style = kwargs.get('style', None)
        if style is None:
            kwargs['style'] = '#' + folder
        elif isinstance(style, integer_types):
            kwargs['style'] = PINS[style]

    def add_line(self, folder, route, **kwargs):
        """
        Add a route as a LineString in the .kml
        :param folder: folder name
        :param route: Route
        :param kwargs: optional args passed to the renderer
        """
        self._update_kwargs(folder, kwargs)
        coordinates = []
        for p in route:
            coordinates.append(
                "{lng:.6f},{lat:.6f}".format(lat=p.latitude, lng=p.longitude))
        variables = dict(
            name=route.name or '',
            description=route.description or '')
        variables.update(kwargs)
        self.folders[folder].append(self.line_template.format(
            coordinates=' '.join(coordinates),
            **variables))

    def add_points(self, folder, route, excluded=None, **kwargs):
        """
        Add a route as a Points in the .kml
        :param folder: folder name
        :param route: Route
        :param excluded: list of GeoPoint to exclude from rendering
        :param kwargs: optional args passed to the renderer

        If folder does not have a pin set, points must define style kwargs
        otherwise nothing is added
        """
        self._update_kwargs(folder, kwargs)
        excluded = excluded or []
        for geopoint in route:
            if geopoint not in excluded:
                self.add_point(folder, geopoint, **kwargs)

    def add_point(self, folder, geopoint, **kwargs):
        """
        Add a GeoPoint in the .kml
        :param folder: folder name
        :param geopoint: GeoPoint
        :param kwargs: optional args passed to the renderer

        If folder does not have a pin set, point must define style kwargs
        otherwise nothing is added
        """
        kwargs.setdefault('style', self.folder_styles[folder])
        self._update_kwargs(folder, kwargs)
        if kwargs['style'] != PINS[PIN_NONE]:
            coordinates = "{lng:.6f},{lat:.6f}".format(
                lat=geopoint.latitude, lng=geopoint.longitude)
            variables = dict(
                name=geopoint.name or geopoint.dm,
                description=geopoint.description or '')
            variables.update(kwargs)
            self.folders[folder].append(self.point_template.format(
                coordinates=coordinates,
                **variables))

    def add_segments(self, folder, route, **kwargs):
        """
        Add a route as LineString segments in the .kml
        :param folder: folder name
        :param route: Route
        :param kwargs: optional args passed to the renderer
        """
        self._update_kwargs(folder, kwargs)
        for p1, p2 in route.segments:
            coordinates = "{p1.longitude:.6f},{p1.latitude:.6f} " \
                          "{p2.longitude:.6f},{p2.latitude:.6f}".format(
                              p1=p1, p2=p2)
            variables = dict(
                name="{label}: {p1}->{p2}".format(
                    label=route.name or folder,
                    p1=p1.name or p1.dm,
                    p2=p2.name or p2.dm))
            variables.update(kwargs)
            self.folders[folder].append(self.segment_template.format(
                coordinates=coordinates,
                **variables))

    def render(self, **kwargs):
        """
        Render the full .kml
        :param kwargs:
        :return: unicode
        """
        return self.template.format(
            styles=''.join(self.styles.values()).format(**kwargs),
            folders=self.render_folders(),
            **kwargs
        )

    def render_folder(self, folder):
        """
        Render a folder content
        :param folder:
        :return: str
        """
        return self.folder_template.format(
            name=folder,
            open=1,
            content='\n'.join(self.folders[folder]),
        )

    def render_folders(self):
        """
        Render all folders
        :return: str
        """
        return '\n'.join(
            [self.render_folder(folder) for folder in self.folders])


import unittest
import numpy as np
from utils import Utils

class TestUtils(unittest.TestCase):

    def setUp(self):
        self.util = Utils()
        self.window_size = 3
        self.sentence = "Er sieht.\nIvana gibt.\nStefan mag.\nStefan am 3. Mai"
        self.data_set_char_gold = [(1.0, "eht. Iv"), (1.0, "ibt. St"),
                                   (1.0, "mag. St"), (0.0, 'm 3. Ma')]

        self.char_2_id_dict_gold = \
            {'e': 1, 'h': 2, 't': 3, '.': 4, ' ': 5, 'I': 6, 'v': 7, 'i': 8,
             'b': 9, 'S': 10, 'm': 11, 'a': 12, 'g': 13, '3': 14, 'M': 15}

        self.data_set_gold = \
            [(1.0, np.array([float(i) for i in [1, 2, 3, 4, 5, 6, 7]])),
             (1.0, np.array([float(i) for i in [8, 9, 3, 4, 5, 10, 3]])),
                (1.0, np.array([float(i) for i in [11, 12, 13, 4, 5, 10, 3]])),
                (0.0, np.array([float(i) for i in [11, 5, 14, 4, 5, 15, 12]]))
             ]

        self.test_sentence = "Er sagt: ›Das ist so?‹ Wir machen “weiter”! Oki haben! – Aber dann\n"
        self.potential_eos_list_gold = [(7, "agt: ›D"), (20, " so?‹ W"),
                                        (42, "er”! Ok"),
                                        (53, "ben! – ")]

    def test_build_char_2_id_dict(self):
        char_2_id_dict_cur = \
            self.util.build_char_2_id_dict(self.data_set_char_gold, 1)

        self.assertEqual(
            len(char_2_id_dict_cur), len(
                self.char_2_id_dict_gold))
        self.assertDictEqual(char_2_id_dict_cur, self.char_2_id_dict_gold)

    def test_build_data_set(self):
        data_set_cur = self.util.build_data_set(self.data_set_char_gold,
                                                self.char_2_id_dict_gold,
                                                self.window_size)

        self.assertEqual(len(data_set_cur), len(self.data_set_gold))

        for i in range(0, len(self.data_set_gold)):
            feature_vector_equals = np.array_equal(data_set_cur[i][1],
                                                   self.data_set_gold[i][1])
            self.assertEqual(data_set_cur[i][0], self.data_set_gold[i][0])
            self.assertEqual(feature_vector_equals, True)

    def test_build_data_set_char(self):
        data_set_char_cur = self.util.build_data_set_char(self.sentence,
                                                          self.window_size)

        self.assertEqual(data_set_char_cur, self.data_set_char_gold)

    def test_build_potential_eos_list(self):
        potential_eos_list_cur = self.util.build_potential_eos_list(
            self.test_sentence, self.window_size)

        self.assertEqual(len(potential_eos_list_cur),
                         len(self.potential_eos_list_gold))

        for i in range(0, len(self.potential_eos_list_gold)):
            self.assertEqual(potential_eos_list_cur[i],
                             self.potential_eos_list_gold[i])

if __name__ == '__main__':
    unittest.main()


import argparse
from operator import itemgetter

# note: nargs ---> number of arguments

def parse_args():
    parser = argparse.ArgumentParser(description='Sort file text')

    parser.add_argument('files', type=str, nargs='+')

    parser.add_argument('-n', action='store_true', help='Sort by numerical value')

    parser.add_argument('-k', type=int, nargs='+', help='Sort by a specified key(s). Numbers will sort by column(start indices by 0)')

    parser.add_argument('-r', action='store_true', help='Sort in reverse')

    parser.add_argument('-f', action='store_true', help='Ignore case')

    return parser.parse_args()

def sort_by_key(txt, args):
    if args['n']:
        txt = sort_by_num(txt)
    if args['k']:
        column = args['k']
        txt = sort_by_column(txt, column)
    return txt

def sort_by_column(txt, column):
    column_key = itemgetter(*column)
    try:
        # Sometimes we don't know if the column is all integers, characters, or a mix
        # Defaulting to sorting by string representation seems safer(so far)
        try:
            return sorted(txt, key=lambda s: int(column_key(s.split())))
        except (ValueError, TypeError):
            input("Attempt to sort numerically failed, sorting done as if string")
            return sorted(txt, key=lambda s: column_key(s.split()))
    except:
        input("Unable to sort by column.")

def sort_by_num(txt):
    try:
        # sorted implicitly sorts numerics before other characters besides whitespace
        return sorted(txt, key=lambda s: sorted(s.strip().split()))
    except:
        input("Unable to sort numerically.")

if __name__ == '__main__':
    args = vars(parse_args())
    files = args.pop('files')
    for file in files:
        with open(file, 'r') as cur:
            txt = sorted(cur.readlines())
            if args['k'] or args['n']:
                txt = sort_by_key(txt, args)
            if args['f']:
                txt = sorted(txt, key=lambda s: s.casefold())
            if args['r']:
                txt = reversed(txt)
            print('\n'.join(txt))


import torch.nn
import numpy as np
from torchesn.nn import ESN
from torchesn import utils
import time
from sklearn.preprocessing import MinMaxScaler
from scipy.io import loadmat
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split #to make a % of the dataset for quicker testing

myData = loadmat('POF60m_PAMExp_2PAM_DR600Mbps(single column).mat')

Rx = myData['PAMsymRx']#.reshape((1,-1))
Tx = myData['PAMsymTx']#.reshape((1,-1))

datasetRx = Rx[0:5000]#1004020] #set to 1004040 for full dataset
datasetTx = Tx[0:5000]#1004020]

#scalar = MinMaxScaler(feature_range=(0,1))

#datasetRx = scalar.fit_transform(datasetRx)
#datasetTx = scalar.fit_transform(datasetTx)

datasetRx = datasetRx.reshape(-1, 1, 1)
datasetTx = datasetTx.reshape(-1, 1, 1)

X_data = torch.from_numpy(datasetRx)
Y_data = torch.from_numpy(datasetTx)

(trainX, testX, trainY, testY) = train_test_split(
    X_data, Y_data, test_size = 0.2, random_state=42
) #75% for training 25% for testing

washout = [500]
input_size = output_size = 1
hidden_size = 1000#

loss_fcn = torch.nn.MSELoss()

if __name__ == "__main__":
    start = time.time()

    #Training
    trainY_flat = utils.prepare_target(trainY.clone(), [trainX.size(0)], washout)

    model = ESN(input_size, hidden_size, output_size)
    model(trainX, washout, None, trainY_flat)
    model.fit()

    output, hidden = model(trainX, washout)

    print("Training error:", loss_fcn(output, trainY[washout[0]:]).item())

    # Test
    output, hidden = model(testX, [0], hidden)
    print("Test error:", loss_fcn(output, testY).item())

    np_out      = output.detach().numpy().flatten()
    np_in       = testX.detach().numpy().flatten()
    np_ideal    = testY.detach().numpy().flatten()

    unmod_MSE = (np.square(np_ideal - np_in )).mean(axis=None)
    print("Untampered MSE: ", unmod_MSE)
    print("Ended in", time.time() - start, "seconds.")

    plt.plot(np_out,'g-o',markersize=4, label = "output")#linestyle = '-')
    plt.plot(np_ideal,'b-o',markersize=4, label = "ideal")#linestyle = '-')
    plt.plot(np_in, 'r-o',markersize=4, label = "input", linestyle = '--', linewidth = '1')
    plt.xlabel('Number of bits')
    plt.ylabel('Bit status')
    plt.grid(True)
    plt.xlim(0,50)
    #plt.ylim(-1,2)
    plt.legend(loc='upper right')
    plt.show()


#conver a picture to list
from PIL import Image
def imageprepare(argv):
    raw_im = Image.open(argv)
    img = raw_im.convert('L')
    xsize, ysize = raw_im.size
    if xsize != 28 or ysize != 28:
        img = img.resize((28, 28), Image.ANTIALIAS)
    arr = []
    for i in range(28):
        for j in range(28):
            pixel = float(1.0 - float(img.getpixel((j, i))) / 255.0)
            arr.append(pixel)
    return arr