#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""eq

Usage:
    eq [--density=<n>]
    eq (-h | --help)
    eq --version

Options:
    --density=<n>  PDF to PNG conversion image density [default: 300].
    -h --help      Show this screen.
    --version      Show version.
"""

import shutil
import os
import tempfile
import platform
import subprocess as sp
from docopt import docopt
import eq.template
from eq.version import __version__
from eq.log import log, warn, error

def main():
    arguments = docopt(__doc__, version=f'eq v{__version__}')
    system = platform.system()

    # Determine presence of and default executables

    latexmk = shutil.which('latexmk')
    if latexmk is None:
        error('The latexmk executable cannot be found.\n'
              'eq needs latexmk to compile your equation, '
              'please make sure you have a LaTeX distribution installed and '
              'available to run, and then install latexmk.')
        exit(1)

    if shutil.which('magick') is None:
        # convert could not be found; imagemagick may still exist, as newer
        # versions rely on the "magick" command.
        if system != 'Windows' and shutil.which('convert') is None:
            error('The imagemagick executable (magick or convert) could not '
                  'be found.\n'
                  'eq needs imagemagick to process your equation, '
                  'please make sure you have imagemagick installed and '
                  'available to run.')
            exit(1)
        # else:
        # The `convert` command was found.
        magick = ['convert']
    else:
        # `magick` was found.
        magick = ['magick', 'convert']

    first = lambda x: next((y for y in x if y is not None), None)
    editor = first([shutil.which('editor'),
                    os.environ.get('VISUAL'),
                    os.environ.get('EDITOR'),
                    shutil.which('vim'),
                    shutil.which('nano')])
    if editor is None:
        error('Could not find an editor to edit the equation with.\n'
              'Tried `editor`, $VISUAL, $EDITOR, `vim`, `nano`, in that order. '
              'Please ensure one of these exists, or set $VISUAL/$EDITOR to '
              'your preferred editor.')
        exit(1)

    if system == 'Windows':
        nircmd = shutil.which('nircmd')
        if nircmd is None:
            error('The nircmdc executable could not be found.\n'
                  'eq needs nircmd to copy the equation image into your '
                  'clipboard. Please ensure it is installed and available.')

        def copy(png_name):
            png_name = os.path.realpath(png_name)
            return sp.run([nircmd, 'clipboard', 'copyimage', png_name])
    else:
        if system != 'Linux':
            warn('Treating system as Linux')
        xclip = shutil.which('xclip')
        if xclip is None:
            error('The xclip executable could not be found.\n'
                  'eq needs xclip to copy the equation image into your '
                  'clipboard. Please ensure it is installed and available.')
            exit(1)

        def copy(png_name):
            return sp.run([xclip, '-in', '-selection', 'clipboard',
                            '-target', 'image/png', png_name])

    # Create a temporary LaTeX project.
    with tempfile.TemporaryDirectory() as proj_dir:
        # Fix needed for Windows:
        # The default temporary directory has a ~ in it, and LaTeX really hates
        # that.
        proj_dir = os.path.realpath(proj_dir)

        # Put default template into template directory
        file_name = eq.template.put(proj_dir)

        # Loop while the content changes on write
        with open(file_name, 'r') as old_file:
            old_content = old_file.read()

        while True:
            # Open template with editor
            sp.run([editor, file_name])

            # Compare with new content
            with open(file_name, 'r') as new_file:
                new_content = new_file.read()

            if new_content == old_content:
                break # out of edition loop
            # else:
            old_content = new_content

            # Do the rendering, and copy to the clipboard
            # Use latexmk to ensure full compilation
            result = sp.run(
                [latexmk, '-pdf', f'-output-directory="{proj_dir}"', file_name])
            if result.returncode != 0:
                # Something went wrong with the compilation (user error?)
                # Loop back to edition.
                error('Something went wrong with compiling the project.')
                continue # to another edition pass.

            # Finished producing a PDF.
            # Now convert it to a white background PNG with imagemagick.
            pdf_name = os.path.splitext(file_name)[0] + '.pdf'
            png_name = os.path.splitext(file_name)[0] + '.png'
            density = arguments.get('--density', 300)
            result = sp.run([*magick, '-flatten', '-colorspace', 'RGB',
                '-background', 'white', '-density', density, pdf_name,
                png_name])

            if result.returncode != 0:
                error('Something went wrong with converting the PDF to a PNG.')
                log('Press any key to go back to edition.')
                input()
                continue # to another edition pass.
            # else: success

            # Copy the PNG into the clipboard
            result = copy(png_name)
            if result.returncode != 0:
                error('Something went wrong with copying the PNG to your '
                      'clipboard.')
                log('Press any key to go back to edition.')
                input()
                continue # to another edition pass.

            # loop to another edition pass


import gzip
import pickle
from pathlib import Path
import numpy as np
import torch
from torch import nn, optim
from torch.utils.data import DataLoader, TensorDataset
import torch.nn.functional as F

loss_func = F.cross_entropy

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

def loss_batch(model, loss_func, xb, yb, opt=None):
    loss = loss_func(model(xb), yb)

    if opt is not None:
        loss.backward()
        opt.step()
        opt.zero_grad()

    return loss.item(), len(xb)

class Mnist_CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1)
        self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1)
        self.conv3 = nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1)

    def forward(self, xb):
        xb = xb.view(-1, 1, 28, 28)
        xb = F.relu(self.conv1(xb))
        xb = F.relu(self.conv2(xb))
        xb = F.relu(self.conv3(xb))
        xb = F.avg_pool2d(xb, 4)
        return xb.view(-1, xb.size(1))

class Lambda(nn.Module):
    def __init__(self, func):
        super().__init__()
        self.func = func

    def forward(self, x):
        return self.func(x)

def preprocess(x, y):
    return x.view(-1, 1, 28, 28).to(device), y.to(device)

class WrappedDataLoader:
    def __init__(self, dl, func):
        self.dl = dl
        self.func = func

    def __len__(self):
        return len(self.dl)

    def __iter__(self):
        batches = iter(self.dl)
        for b in batches:
            yield (self.func(*b))

def fit(epochs, model, loss_func, opt, train_dl, valid_dl):
    for epoch in range(epochs):
        model.train()
        for xb, yb in train_dl:
            loss_batch(model, loss_func, xb, yb, opt)

        model.eval()
        with torch.no_grad():
            losses, nums = zip(
                *[loss_batch(model, loss_func, xb, yb) for xb, yb in valid_dl]
            )
        val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)

        print(epoch, val_loss)

def get_data(bs=100):
    DATA_PATH = Path("data")
    PATH = DATA_PATH / "mnist"
    FILENAME = "mnist.pkl.gz"
    with gzip.open((PATH / FILENAME).as_posix(), "rb") as f:
        ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f, encoding="latin-1")
    x_train, y_train, x_valid, y_valid = map(torch.tensor, (x_train, y_train, x_valid, y_valid))
    train_ds = TensorDataset(x_train, y_train)
    train_dl = DataLoader(train_ds, batch_size=100, shuffle=True)
    train_dl = WrappedDataLoader(train_dl, preprocess)

    valid_ds = TensorDataset(x_valid, y_valid)
    valid_dl = DataLoader(valid_ds, batch_size=bs * 2)
    valid_dl = WrappedDataLoader(valid_dl, preprocess)

    return (train_dl, valid_dl)

def get_model(lr=0.1):
    model = Mnist_CNN().to(device)
    opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
    return model, opt

def seq():
    model = nn.Sequential(
        nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        nn.AdaptiveAvgPool2d(1),
        Lambda(lambda x: x.view(x.size(0), -1)),
    )
    return model

train_dl, valid_dl = get_data(bs=100)
model, opt = get_model()
epochs = 2
fit(epochs, model, loss_func, opt, train_dl, valid_dl)

""


from setuptools import setup, find_packages

version = '0.1'

setup(name='fluegelform',
      version=version,
      description="Dexterity-based form generator for Plone.",
      long_description=open("README.rst").read() + "\n" +
                       open("CHANGES.rst").read(),
      # Get more strings from
      # http://pypi.python.org/pypi?:action=list_classifiers
      classifiers=[
        "Programming Language :: Python",
        "Framework :: Plone",
        ],
      keywords='plone form ploneformgen dexterity',
      author='David Glick',
      author_email='dglick@gmail.com',
      url='http://github.com/davisagli/fluegelform',
      license='GPL',
      packages=find_packages('src'),
      package_dir = {'': 'src'},
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'lxml',
          'repoze.xmliter',
          'setuptools',
          'Products.CMFPlone',
          'plone.app.blocks',
          'plone.app.deco',
          'plone.app.dexterity',
          'zope.interface',
          # -*- Extra requirements: -*-
      ],
      entry_points="""
      [z3c.autoinclude.plugin]
      target = plone
      """,
      )


"""A setuptools based setup module.

See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='oc_ica',
    description='OC_ICA models and analysis.',
    long_description=long_description,
    author='Jesse Livezey and Alex Bujan',
    author_email='jesse.livezey@gmail.com')


import requests
import pandas as pd
import csv
import json
import datetime
from datetime import timezone
from time import gmtime, strftime

Headers = {
     'X-CSRFToken': 'token',
     'Authorization': 'Bearer Authorization'
}

url = 'https://sagebrush.pamdas.org/api/v1.0/observations/?subject_id=ee76e226-c05e-400b-9ae9-db8b11b5359b'

response= requests.get(url, headers=Headers, timeout=20)

data = response.json().get("results", []) or []

def extract_property_value(properties, prop_name):
    if not isinstance(properties, list):
        return "0"
    return next((prop["value"] for prop in properties if isinstance(prop, dict) and prop.get("name")== prop_name), '0')

df = pd.DataFrame({
    'humidity': [extract_property_value(obs['device_status_properties'], 'humidity') for obs in data],
    'recorded_at': [obs.get['recorded_at','0'] for obs in data],
    'battery': [extract_property_value(obs['device_status_properties'], 'battery')for obs in data],
    'external_temperature': [extract_property_value(obs['device_status_properties'],'external_temperature')for obs in data],
    'internal_temperature': [extract_property_value(obs['device_status_properties'],'internal_temperature')for obs in data]
})

# Print the dataframe
print(df)

# def get_date(df):
#     # Getting key
#     recorded_at_key = list(df['data']['results'][0].keys())[2]

#     print(recorded_at_key.capitalize())

#     # Getting all the recorded_at value
#     recorded_at = [df['data'][3][i][recorded_at_key] for i in range(0, len(df['data'][3]))]

#     for index, time in enumerate(recorded_at):

#         # Convert recorded_at to datetime format
#         new_timestamp = datetime.datetime.strptime(time, "%Y-%m-%dT%H:%M:%S%z")

#         #converts the new_timestamp object to GMT.
#         new_timestamp_gmt = new_timestamp.astimezone(timezone.utc)

#         # Format new_timestamp_gmt in GMT
#         new_timestamp_gmt_formatted = new_timestamp_gmt.strftime("%A, %Y-%m-%d %H:%M:%S")

#         dates_list = ('Timestamp: ', new_timestamp_gmt_formatted)

#         print(dates_list)

#     return dates_list

# print(get_date(df))


#collections module: counter, namedtuple, orderedDirect, defaultdict, deque

from collections import Counter

a= 'aaabbbccc'
counter_char= Counter(a)  #devuelve un diccionario con cada letra y su conteo
a.counter.keys() #devuelve una letra de cada
a.counter.values() #devuelve conteo de cada letra
a.most_common() #devuelve moda. si dentro del arg se pone num int, (1)-> más repetido, (2) -> segundo más repetido, etc
lista=list(a.elements()) #devuelve cada elemento dentro de una lista
#---
#namedtuple

from collections import namedtuple

Point= namedtuple ('Point', 'x,y')
pt= Point(1,-4)
print(pt.x, pt.y)

#--
# order Dict
from collections import OrderedDict

ordered_dict= OrderedDict()   #crea un dict ordenado
ordered_dict['a'] = 1
ordered_dict['b'] = 2
ordered_dict['c'] = 3
ordered_dict['d'] = 4
ordered_dict['e'] = 5

#--
# default dict

from collections import defaultdict

d= defaultdict(int)  #devuelve 0 si no existe
d['a'] = 1
d['b'] = 2

#---
d= defaultdict(float) #devuelve 0.0 si no existe
d['a'] = 1
d['b'] = 2

#----
#deque

from collections import deque
d=deque()  #devuelve lista
d.append(1)
d.append(2)
d.appendleft(0) #pone elementos en el punto0

d.pop() #remove last element
d.popleft() #remueve primer elemento
d.extend([4,5,6]) #igual que extend lista
d.extendleft({-1,0})

d.rotate()  #invierte


# -*- coding: utf-8 -*-
import sys

import click

import click_spinner

from .. import api
from .. import cli
from .. import options
from ..helpers.datetime import parse_psql_date_str, reltime

@cli.cli.command()
@click.option('--limit', '-n', nargs=1, default=20,
              help='List N latest releases')
@options.app
def releases(app, limit):
    """
    List application releases

    More:

    $ asyncy releases:rollback
    """
    cli.user()
    cli.assert_project()

    # click.echo(click.style('Releases', fg='magenta'))
    # click.echo(click.style('========', fg='magenta'))

    with click_spinner.spinner():
        res = api.Releases.list(app, limit=limit)

    res = sorted(res, key=lambda elem: elem['id'])

    if res:
        from texttable import Texttable
        table = Texttable(max_width=800)
        table.set_deco(Texttable.HEADER)
        table.set_cols_align(['l', 'l', 'l', 'l'])
        all_releases = [['VERSION', 'STATUS', 'CREATED', 'MESSAGE']]
        for release in res:
            date = parse_psql_date_str(release['timestamp'])
            all_releases.append([
                f'v{release["id"]}',
                release['state'].capitalize(),
                reltime(date),
                release['message']
            ])
        table.add_rows(rows=all_releases)
        click.echo(table.draw())
    else:
        click.echo(f'No releases yet for app {app}.')

@cli.cli.command(aliases=['releases:rollback'])
@click.argument('version', nargs=1, required=False)
@options.app
def releases_rollback(version, app):
    """
    Rollback release to a previous release.
    """
    cli.user()
    cli.assert_project()

    if not version:
        click.echo(f'Getting latest release for app {cli.get_app_name()}... ',
                   nl=False)
        with click_spinner.spinner():
            res = api.Releases.get(app=app)
            version = int(res[0]['id']) - 1
        click.echo(click.style('√', fg='green'))

    if int(version) == 0:
        click.echo('Unable to rollback a release before v1.')
        sys.exit(1)

    click.echo(f'Rolling back to v{version}... ', nl=False)
    with click_spinner.spinner():
        res = api.Releases.rollback(version=version, app=app)
    click.echo(click.style('√', fg='green'))
    click.echo(f'Deployed new release... ' +
               click.style(f'v{res["id"]}', bold=True, fg='magenta'))


'''Train CIFAR10 with PyTorch.'''
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn

import torchvision
import torchvision.transforms as transforms
import torch.utils.data as data

from sklearn.metrics import matthews_corrcoef, f1_score, accuracy_score
from sklearn.metrics import precision_recall_fscore_support as score
from sklearn.metrics import roc_curve, roc_auc_score

import numba

import os
import argparse
from tqdm import tqdm, trange
from data import CelebA

from utils import get_scores, get_embeddings, get_scores_and_embeddings, get_neighbors,test_auc,get_dataset_img_info

import matplotlib
matplotlib.use('agg')
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.manifold import MDS, TSNE
from sklearn.decomposition import PCA
import umap
import time
from scipy import stats

from models.CustomModel import ResNet18, vgg11, AutoEncoder, SugiyamaNet, ComboModel, VAE, SugiyamaNet
from models.inception import inception_v3
import train

def write_data(args, data, clear=False):
    filename = f"{args.net}_{args.activation_name}_{args.lr}.csv"
    if clear: open(filename, 'w').close()
    f=open(filename, "a+");
    f.write(data);
    f.close()

attrs_default = [
    '5_o_Clock_Shadow', 'Arched_Eyebrows', 'Attractive', 'Bags_Under_Eyes',
    'Bald', 'Bangs', 'Big_Lips', 'Big_Nose', 'Black_Hair', 'Blond_Hair',
    'Blurry', 'Brown_Hair', 'Bushy_Eyebrows', 'Chubby', 'Double_Chin',
    'Eyeglasses', 'Goatee', 'Gray_Hair', 'Heavy_Makeup', 'High_Cheekbones',
    'Male', 'Mouth_Slightly_Open', 'Mustache', 'Narrow_Eyes', 'No_Beard',
    'Oval_Face', 'Pale_Skin', 'Pointy_Nose', 'Receding_Hairline',
    'Rosy_Cheeks', 'Sideburns', 'Smiling', 'Straight_Hair', 'Wavy_Hair',
    'Wearing_Earrings', 'Wearing_Hat', 'Wearing_Lipstick', 'Wearing_Necklace',
    'Wearing_Necktie', 'Young'
]

model_dict = {
        "classifier": (train.classifier,ResNet18),
        "ae":         (train.ae, AutoEncoder),
        "vae":        (train.vae, VAE),
        'gan':        (train.gan, None),
        "dre":        (train.dre_scratch,vgg11),
        "imagenet":   (train.imagenet, inception_v3)
    }

device = 'cuda' if torch.cuda.is_available() else 'cpu'

def get_args():
    parser = argparse.ArgumentParser(description='PyTorch celeba weird Training')
    parser.add_argument('--attrs', dest='attrs', default=attrs_default, nargs='+', help='attributes to learn')
    parser.add_argument('--lr', default=1e-3, type=float, help='learning rate')
    parser.add_argument('--img_size', default=64, type=int, help='img size')
    parser.add_argument('--epochs', default=20, type=int, help=' ')
    parser.add_argument('--batch_size', default=128, type=int, help='minibatch size')
    parser.add_argument('--data', default='CelebA', type=str,  choices=['CelebA', 'CelebA-HQ'])
    parser.add_argument('--net', default="classifier", type=str, choices=model_dict.keys())
    parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
    parser.add_argument('--data_path', dest='data_path', type=str, default='./data/celebA')
    parser.add_argument('--attr_path', dest='attr_path', type=str, default='./data/celebA/list_attr_celeba.txt')
    parser.add_argument('--remove_att', type=str, default="")
    parser.add_argument('--remove_att_val', type=int, default=-1)
    parser.add_argument('--trainset_total', type=int, default=50000)
    parser.add_argument('--heldout_total', type=int, default=10000)
    parser.add_argument('--outlier_total', type=int, default=1000)
    parser.add_argument("--save_dir", type=str,  default="pics")
    parser.add_argument("--do_train", action='store_true', help="  ")
    parser.add_argument("--do_dre", action='store_true', help="")
    parser.add_argument("--cluster_eval_type", type=str, default="", help="??")
    return parser.parse_args()

@torch.no_grad()
def calc_inception_distr(model, dataset):
    model.eval()
    device = next(model.parameters()).device
    all_logits = torch.empty(0).to(device)

    print("pre-parsing the training data for average distribution")
    for images, labels, _ in tqdm(dataset):
        all_logits = torch.cat([all_logits, model(images.to(device))], dim = 0)

    model.average_distr = torch.log_softmax(all_logits, dim=1).mean(0)

def main():
    args = get_args()
    datasets = all_celeba(args)
    trainset_total = args.trainset_total
    heldout_total  = args.heldout_total
    outlier_total  = args.outlier_total
    inlier_total   = heldout_total - outlier_total

    # Model
    print('==> Building model..')

    train_func, modelfunc  = model_dict[args.net]

    model_path = args.net + ".pth"

    if args.net == 'imagenet':
        model = modelfunc(model_path).to(device)
        for name, param in model.named_parameters():
            param.requires_grad = False

        calc_inception_distr(model,  datasets.train_dataloader)
        bs = args.batch_size
    else:
        model = modelfunc().to(device)
        bs = 2048

        if args.do_train:
            optimizer  = optim.Adam(model.parameters(), lr=args.lr) if args.net !='dre' else optim.SGD(model.parameters(), lr=.01)
            for e in range(args.epochs):
                train_func(model, optimizer, datasets.train_dataloader, datasets.eval_dataloader)
            torch.save(model.state_dict(), model_path)
        else:
            model.load_state_dict(torch.load(model_path))
    model.eval()

    original_train_zs   = get_embeddings(model, datasets.train_dataloader)
    original_inlier_zs  = get_embeddings(model, datasets.inlier_dataloader)
    original_outlier_zs = get_embeddings(model, datasets.outlier_dataloader)

    if args.do_dre:
        model = get_2phase_dre(model,datasets, bs)
        model.eval()

    print("getting embeddings")

    train_scores,   train_zs   = get_scores_and_embeddings(model, datasets.train_dataloader)
    inlier_scores,  inlier_zs  = get_scores_and_embeddings(model, datasets.inlier_dataloader)
    outlier_scores, outlier_zs = get_scores_and_embeddings(model, datasets.outlier_dataloader)

    eval_scores    = np.concatenate([inlier_scores, outlier_scores])
    eval_zs        = np.concatenate([inlier_zs, outlier_zs])
    auc            = test_auc(inlier_scores, outlier_scores)

    top1k_scores = np.argpartition(eval_scores, outlier_total)[:outlier_total]
    inlier_total   =  heldout_total  - outlier_total
    outlier_selection_percent = (top1k_scores > inlier_total).sum() / outlier_total
    print(f"percentage of outlier selected: {outlier_selection_percent*100}")

    train_neighbors, eval_neighbors, train_subset = get_neighbors(model, datasets)

    if args.cluster_eval_type != "":
        print("kmeans or inception score or something")

    trans = umap.UMAP(n_neighbors=15,verbose=True).fit(eval_zs)
    embs_train      = trans.transform(train_zs[train_subset])
    embs_inlier     = trans.transform(inlier_zs)
    embs_outlier    = trans.transform(outlier_zs)

    total_points = embs_train.shape[0] + embs_inlier.shape[0] + embs_outlier.shape[0]

    train_scores,   tlabels, t_paths, _ = get_dataset_img_info(model, datasets.train_dataloader)  #this is probably what I want
    inlier_scores,  ilabels, i_paths, _ = get_dataset_img_info(model, datasets.inlier_dataloader)  #this is probably what I want
    outlier_scores, olabels, o_paths, _ = get_dataset_img_info(model, datasets.outlier_dataloader)  #this is probably what I want

    import pdb; pdb.set_trace()

    indices     = np.array(["indices"]+list(range(total_points)))
    data_type   = np.array(["datatype"]+["train"] * len(train_subset) + ["inlier"]*embs_inlier.shape[0] + ["outlier"]*embs_outlier.shape[0])
    labels      = np.concatenate([["data_label"], tlabels[train_subset], ilabels, olabels])
    paths       = np.concatenate([["data_path"],  t_paths[train_subset], i_paths, o_paths])
    scores      = np.concatenate([["shift_score"],  train_scores[train_subset], inlier_scores, outlier_scores])
    og_embs     = np.concatenate([[["emb_x","emb_y"]], embs_train, embs_inlier, embs_outlier])
    close_train = np.concatenate([[[str(i)+"_closest_train" for i in range(1, 21)]], train_neighbors])
    close_eval  = np.concatenate([[[str(i)+"_closest_eval"  for i in range(1, 21)]], eval_neighbors])

    savedata =  np.vstack([indices,data_type,labels,paths,scores,og_embs.T,close_train.T,close_eval.T]).T
    filename = f'dre_{args.net}.csv' if args.do_dre else f'{args.net}.csv'
    np.savetxt(filename,savedata, delimiter=',', fmt='%s')

    print("here")

    exit()
    do_plots(15, .1)

    '''for i, top10 in enumerate(np.argpartition(eval_scores, 10)[:10]):
        prep_img, _, path = eval_CelebA_data.__getitem__(top10)
        get_scorecam_example(net, prep_img.to(device), path, os.path.join("imgs",f"top{i}_outlier"), dr_model=dr_model, target_layer=2, invert_sigmoid=True)

    for i, top10 in enumerate(np.argpartition(eval_scores, -10)[-10:]):
        prep_img, _, path = eval_CelebA_data.__getitem__(top10)
        get_scorecam_example(net, prep_img.to(device), path, os.path.join("imgs",f"top{i}_inlier"), dr_model=dr_model, target_layer=2, invert_sigmoid=False)
    '''

def do_plots(neighbors,dist):
    #x_emb_dtest = TSNE(n_components=2, perplexity=neighbors, verbose=True).fit_transform(dr_eval_zs.cpu().numpy())
    x_emb_dtest = umap.UMAP(n_neighbors=neighbors,verbose=True).fit_transform(dr_eval_zs.cpu().numpy())
    #x_emb_dtest_small = x_emb_dtest[(x_emb_dtest[:,0] >= 12) & (x_emb_dtest[:,1] <= 0)]
    #make_plot2d(x_emb_dtest_small, ["test"] * x_emb_dtest_small.shape[0],        "true labels",                             "test.png")

    #x_emb_test = TSNE(n_components=2, perplexity=neighbors, verbose=True).fit_transform(torch.cat([inlier_zs, outlier_zs]).cpu().numpy())
    x_emb_test = umap.UMAP(n_neighbors=neighbors, verbose=True).fit_transform(torch.cat([inlier_zs, outlier_zs]).cpu().numpy())

    #do pretty graphs

    bins = 8
    graph_scores = np.array((stats.mstats.rankdata(eval_scores) / len(eval_scores)) * bins).astype(int)
    labels_true = ['inlier'] * len(inlier_zs) + ['outlier'] * len(outlier_zs)
    labels_selected = ["not selected"] * len(eval_scores)
    labled_intersected = ['excluded inlier'] * len(inlier_zs) + ['excluded outlier'] * len(outlier_zs)
    for s in top1k_scores:
        labels_selected[s] = "selected"
        if s > len(inlier_zs):
            labled_intersected[s] = 'intersection'

    make_plot2d(x_emb_dtest, labels_true,        "true labels",                             "true_labels_density.png")
    make_plot2d(x_emb_dtest[::-1], labels_true[::-1],        "true labels",                             "true_labels_density_flipped.png")
    make_plot2d(x_emb_dtest, labels_selected,    "labels selected by density_score",        "labled_selected_density.png")
    make_plot2d(x_emb_dtest, labled_intersected, "compare true labels to density selected", "labels_intersected_density.png")
    make_plot2d_scores(x_emb_dtest, labels_true, graph_scores, "density scores plot",       "score_density.png")

    make_plot2d(x_emb_test, labels_true,        "true labels",                             "true_labels_original.png")
    make_plot2d(x_emb_test[::-1], labels_true[::-1],        "true labels",                             "true_labels_original_flipped.png")
    make_plot2d(x_emb_test, labels_selected,    "labels selected by density_score",        "labled_selected_original.png")
    make_plot2d(x_emb_test, labled_intersected, "compare true labels to density selected", "labels_intersected_original.png")
    make_plot2d_scores(x_emb_test, labels_true, graph_scores, "density scores plot",       "score_plot_original.png")

def get_2phase_dre(net,datasets, batch_size = 2048):
    train_zs   = get_embeddings(net, datasets.train_dataloader)
    inlier_zs  = get_embeddings(net, datasets.inlier_dataloader)
    outlier_zs = get_embeddings(net, datasets.outlier_dataloader)

    epochs = 100
    size = net.get_latent_size()
    #compute dr data
    dr_train_loader        = torch.utils.data.DataLoader(train_zs, batch_size=batch_size, shuffle=True,drop_last=False)
    dr_test_loader         = torch.utils.data.DataLoader(torch.cat([inlier_zs, outlier_zs]), batch_size=batch_size, shuffle=True)
    dr_inlier_loader       = torch.utils.data.DataLoader(inlier_zs, batch_size=batch_size, shuffle=False, drop_last=False)
    dr_outlier_loader      = torch.utils.data.DataLoader(outlier_zs, batch_size=batch_size, shuffle=False, drop_last=False)

    torch.cuda.empty_cache()
    dr_model = train.dre(SugiyamaNet, size, dr_train_loader, dr_test_loader, dr_inlier_loader, dr_outlier_loader, epochs, device)

    return ComboModel(net, dr_model)

class all_celeba():
    def __init__(self, args):
        trainset_total = args.trainset_total
        heldout_total  = args.heldout_total
        outlier_total  = args.outlier_total
        inlier_total   = heldout_total - outlier_total
        batch_size = args.batch_size

        if args.attrs[0] == args.remove_att:
            exit("cant remove att to be classified")

        kwargs = {'num_workers': 6, 'pin_memory': True} if torch.cuda.is_available() else {}

        print(args.attrs, args.remove_att, args.remove_att_val, "loading dataset")

        train_dataset = CelebA(args.data_path, args.attr_path, args.img_size, 'train', args.attrs)

        outlier_train,       train_dataset = train_dataset.split_attribute(args.remove_att, args.remove_att_val)
        heldout_inlier_data, train_dataset = train_dataset.split(inlier_total)
        train_dataset, _ = train_dataset.split(trainset_total)

        heldout_outlier_data, _            = outlier_train.split(outlier_total)

        assert len(train_dataset)        == trainset_total
        assert len(heldout_inlier_data)  == inlier_total
        assert len(heldout_outlier_data) == outlier_total
        # Data
        self.train_dataloader = data.DataLoader(
            train_dataset, batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )

        self.inlier_dataloader = data.DataLoader(
            heldout_inlier_data, batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )
        self.outlier_dataloader = data.DataLoader(
            heldout_outlier_data, batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )

        self.eval_CelebA_data = CelebA.combine(heldout_inlier_data, heldout_outlier_data)

        self.eval_dataloader = data.DataLoader(
            self.eval_CelebA_data, batch_size=args.batch_size,
            shuffle=True, drop_last=False, **kwargs
        )

        self.perfect_user_train_dataloader = data.DataLoader(
            CelebA.combine(train_dataset, heldout_outlier_data), batch_size=args.batch_size,
            shuffle=False, drop_last=False, **kwargs
        )

if __name__ == "__main__":
    main()


import nltk
import urllib.request
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
import matplotlib.pylab as plt

# Webscrape and frequency analyze the lastest Health News
# Install nltk: $ pip install --user -U nltk
# To load modules open Python shell and enter >>> import nltk >>> nltk.download()

response = urllib.request.urlopen('https://www.healthline.com/health-news')
html = response.read()
#print(html)
soup = BeautifulSoup(html, 'html5lib')
text = soup.get_text(strip = True)
print(text)
# tokenize words
tokens  =  [t for t in text.split()]
print(tokens)

# remove stop words (am, at, the, for) and count frequency
sr = stopwords.words('english')
chars = ["{", "=", "-", "}", "+", "_", "--", ">", "<", "<=", ">=", "—", "var"]
conjunctions = ["but", "But", "and", "or", "nor", "because", "although", "since", "unless", "while", "where"]
conjugations =  ["Is", "is", "Am", "am", "Are", "are", "Was", "was", "Were", "were", "Being", "being", "Been", "been"]
clean_tokens = tokens[:]
for token in tokens:
	if token in sr:
		clean_tokens.remove(token)
# calculate frequency
freq = nltk.FreqDist(clean_tokens)
wordFreqDic = {}
freqDic5 = {}
freqDic10 = {}
freqDic25 = {}
freqDic50 = {}
freqDic100 = {}

# function to remove special characters and conjunnctions
def removeWord(wordList, wordDic):
	for word in wordList:
		if word in wordDic:
			del wordDic[word]

# bucket dictionaries by increasing word frequency
for key,val in freq.items():
	if val >= 5:
		#print(key)
		freqDic5.update({key: val})
		removeWord(chars, freqDic5)
		removeWord(conjunctions, freqDic5)
		removeWord(conjugations, freqDic5)
	elif val >= 10:
		#print(key)
		freqDic10.update({key: val})
		for char in chars:
			if char in chars:
				del freqDic10[char]
	elif val >= 25:
		#print(key)
		freqDic25.update({key: val})
		for char in chars:
			if char in chars:
				del freqDic25[char]
	elif val >= 50:
		#print(key)
		freqDic50.update({key: val})
		for char in chars:
			if char in chars:
				del freqDic50[char]
	elif val >= 100:
		#print(key)
		freqDic100.update({key: val})
		for char in chars:
			if char in chars:
				del freq100[char]
	else:
		pass
	wordFreqDic.update({key: val})
	#print(str(key) + ":" + str(val))

totalDicLen = len(wordFreqDic)
print("Word Occurrences: " + str(totalDicLen))
# bucket out words by increasing  number of occurrences
freqDic5Len = len(freqDic5)
if freqDic5Len > 0:
	over5 = freqDic5Len/totalDicLen
	print("###############")
	print(">= 5 Word Occurrences:")
	print(str(freqDic5Len)+ " words occuring 5 or more times, or " + str(round((100*over5),2)) + "% of the total")
	print(" ")
	print(freqDic5)
	print("---------------------")
freqDic10Len = len(freqDic10)
if freqDic10Len > 0:
	over10 = freqDic10Len/totalDicLen
	print("###############")
	print(">= 10 Word Occurrences:")
	print(str(freqDic10Len)+ " words occuring 10 or more times, or " + str(round((100*over10),2)) + "% of the total")
	print(" ")
	print(freqDic10)
	print("---------------------")
freqDic25Len = len(freqDic25)
if freqDic25Len > 0:
	over25 = freqDic25Len/totalDicLen
	print("###############")
	print(">= 25 Word Occurrences:")
	print(str(freqDic25Len)+ " words occuring 5 or more times, or " + str(round((100*over25),2)) + "% of the total")
	print(" ")
	print(freqDic5)
	print("---------------------")
freqDic50Len = len(freqDic50)
if freqDic50Len > 0:
	over50 = freqDic50Len/totalDicLen
	print("###############")
	print(">= 50 Word Occurrences:")
	print(str(freqDic50Len)+ " words occuring 5 or more times, or " + str(round((100*over50),2)) + "% of the total")
	print(" ")
	print(freqDic50)
	print("---------------------")
freqDic100Len = len(freqDic100)
if freqDic100Len > 0:
	over100 = freqDic100Len/totalDicLen
	print("###############")
	print(">= 100 Word Occurrences:")
	print(str(freqDic100Len)+ " words occuring 5 or more times, or " + str(round((100*over100),2)) + "% of the total")
	print(" ")
	print(freqDic100)
	print("---------------------")
if freqDic5Len == 0 and freqDic10Len == 0 and freqDic25Len == 0 and freqDic50Len == 0 and freqDic100Len == 0:
	print("###############")
	print(" >= 1 Word Occurences: ")
	print(wordFreqDic)
	print("---------------------")

######PLOTS###########
# plot function
def plotWords(wordDic):
	plt.plot(*zip(*sorted(wordDic.items())))
	plt.show()

if freqDic5Len != 0:
	plotWords(freqDic5)
if freqDic10Len != 0:
	plotWords(freqDic10)
if freqDic25Len != 0:
	plotWords(freqDic25)
if freqDic50Len != 0:
	plotWords(freqDic50)
if freqDic100Len != 0:
	plotWords(freqDic100)


import os
import sys
import argparse
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
import tensorflow as tf

from common_utils import input_pipeline, console
from models import network

def validate(filenames, num_classes, ckpt_dir="./"):
    """Performs validation run on a tfrecord dataset files. =
    Also calculates accuracy using multiple patches from an image.

    Args:
        filenames: Tfrecord filenames.
        num_classes: Number of classes in the softmax classification layer.
        ckpt_dir: Location where the checkpoints are stored.
    """
    use_inception = False
    use_large = True

    pt = console.ProgressTracker()
    sess = tf.Session()

    iterator = input_pipeline.make_image_patches(filenames, use_large=use_large)
    images, labels = iterator.get_next()

    """
        Network model
    """
    if use_inception:
        logits, _ = network.build_net(images, num_classes, is_training=False)
    else:
        logits, _ = network.build_nasnet(images, num_classes, is_training=False, use_large=use_large)

    """
        Results
    """
    with tf.name_scope('results'):
        labels = tf.expand_dims(labels, 0)
        correct_prediction = tf.equal(tf.cast(labels, tf.int64), tf.argmax(logits[:1], 1))

        accuracy = tf.reduce_sum(tf.cast(correct_prediction, tf.float32), name='accuracy')
        in_top_5 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(logits[:1], labels, k=5), tf.float32),
                                 name='top_5_predictions')
        in_top_10 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(logits[:1], labels, k=10), tf.float32),
                                  name='top_10_predictions')

        logits = tf.expand_dims(tf.reduce_mean(tf.nn.softmax(logits), axis=0), 0)
        correct_prediction = tf.equal(tf.cast(labels, tf.int64), tf.argmax(logits, 1))

        accuracy_averaged = tf.reduce_sum(tf.cast(correct_prediction, tf.float32), name='accuracy_averaged')
        in_top_5_averaged = tf.reduce_sum(tf.cast(tf.nn.in_top_k(logits, labels, k=5), tf.float32),
                                          name='top_5_predictions_averaged')
        in_top_10_averaged = tf.reduce_sum(tf.cast(tf.nn.in_top_k(logits, labels, k=10), tf.float32),
                                           name='top_10_predictions_averaged')

    sess.run(tf.global_variables_initializer())

    """
        Savers
    """
    pt.info(">> Restoring trained model.")
    saver = tf.train.Saver()
    if os.path.isdir(ckpt_dir):
        saver.restore(sess, tf.train.latest_checkpoint(ckpt_dir))
    else:
        saver.restore(sess, ckpt_dir)

    """
        Validation
    """
    pt.info(">> Validating...")

    top_acc = {
        "top_1": 0,
        "top_5": 0,
        "top_10": 0,
        "top_1_avg": 0,
        "top_5_avg": 0,
        "top_10_avg": 0
    }
    examples = 0
    while True:
        try:
            top1, top5, top10, top1_avg, top5_avg, top10_avg = sess.run([
                accuracy, in_top_5, in_top_10, accuracy_averaged, in_top_5_averaged, in_top_10_averaged
            ])
            top_acc["top_1"] += top1
            top_acc["top_5"] += top5
            top_acc["top_10"] += top10
            top_acc["top_1_avg"] += top1_avg
            top_acc["top_5_avg"] += top5_avg
            top_acc["top_10_avg"] += top10_avg
            examples += 1

            results = "Results: " + ", ".join([
                "{:8} {:.4f}%".format(name, val / examples * 100) for name, val in top_acc.items()
            ])
            pt.progress_info("Images evaluated:{:6d} | {}", [examples, results])
        except tf.errors.OutOfRangeError:
            pt.info(">> Validation completed.")
            break

    results = ["{:8} {:.4f}%".format(name, val / examples * 100) for name, val in top_acc.items()]
    pt.info("Results: " + ", ".join(results))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--val', type=str, required=True)
    parser.add_argument('--ckpt_dir', type=str, required=True,
                        help='Directory of the checkpoints.')
    args = parser.parse_args()

    if len(sys.argv) == 1:
        parser.print_help()
        exit(1)

    val = [os.path.join(args.val, name) for name in os.listdir(args.val) if name[-9:] == ".tfrecord"]

    validate(val, num_classes=1243, ckpt_dir=args.ckpt_dir)


import requests
from bs4 import BeautifulSoup

url = "https://www.clcoding.com/"
r = requests.get(url)

soup= BeautifulSoup(r.content, 'html.parser')
links = soup.find_all('a')
for link in links:
    print(link.get('href'))


# Create tuples
import csv
with open("wikiHow_categories_hierarchy.ttl") as f:
    reader = csv.reader(f)
    data = list(list(rec) for rec in csv.reader(f, delimiter='\t'))

# print(data[0])
import re
articles = ['a', 'an', 'of', 'the', 'is', 'and', 'for']

def title_except(s, exceptions):
    word_list = re.split('-', s)       # re.split behaves as expected
    final = [word_list[0].capitalize()]
    for word in word_list[1:]:
        final.append(word if word in exceptions else word.capitalize())
    return "-".join(final)

def cleanup(val):
    new_val = ""
    if val.startswith('ns1'):
        new_val = val[4:]
    elif val.startswith(''):
        new_val = val.replace("<http://www.wikihow.com/category:", "")
        new_val = new_val[:-1]

    return title_except(new_val, articles)

# print(cleanup('finance-and-business'))

new_data = [(cleanup(val[0]), cleanup(val[2][:-2])) for val in data]

print(new_data[1])

# a = [(1, 1), (4, 3), (5, 3), (6, 3), (7, 7), (8, 7), (9, 7), (2, 1), (3, 1)]

a = new_data
# print(a)
# pass 1: create nodes dictionary
nodes = {}
for i in a:
    id, parent_id = i
    nodes[id] = {'name': id}

# pass 2: create trees and parent-child relations
forest = []
for i in a:
    id, parent_id = i
    node = nodes[id]

    # either make the node a new tree or link it to its parent
    if id == parent_id:
        # start a new tree in the forest
        forest.append(node)
    else:
        # add new_node as child to parent
        parent = nodes[parent_id]
        if not 'children' in parent:
            # ensure parent has a 'children' field
            parent['children'] = []
        children = parent['children']
        children.append(node)

import json

with open('know-how.json', 'w') as f:
    json.dump(forest[0], f, indent=4)


import numpy as np
import torch
from torch.autograd import Variable

dtype = torch.FloatTensor

def pull_arm(bandit):
      result = np.random.randn(1)
      if result > bandit:
          return 1
      else:
          return -1

bandits = [0.1, 1, -0.4, -5]
num_bandits = len(bandits)

w = Variable(torch.ones(num_bandits).type(dtype), requires_grad=True)
rewards = np.zeros(num_bandits)

lr = 0.001

for t in range(1000):
  if np.random.rand(1) < 0.1:
    action = np.random.randint(0, num_bandits)
  else:
    action = np.argmax(w.data)
  reward = pull_arm(bandits[action])
  loss = -torch.log(w[action]) * reward
  rewards[action] += reward
  loss.backward()
  w.data -=  lr* w.grad.data
  w.grad.data.zero_()

print(rewards)


import configparser
import logging
import os

from telegram.common import options

logger = logging.getLogger(__name__)
config_stack = None

DEFAULT_CONF = 'ubuntu-system-tests.conf'
KEY_DEFAULT = 'default'

CONFIG_OPTIONS = [
    options.Option(
        'device_serial', help_string='Device serial'),
    options.Option(
        'device_password', help_string='Device password', mandatory=True),
    options.Option(
        'output_dir', mandatory=True,
        help_string='Directory to store the tests results and artifacts.\n'
                    'WARNING: This directory will have all existing content '
                    'deleted!'),
    options.Option(
        'sim_0_pin', mandatory=True,
        help_string='SIM pin for the card in the first slot'),
    options.Option(
        'sim_1_pin',
        help_string='SIM pin for the card in the second slot'),
    options.Option(
        'wifi_ssid', mandatory=True,
        help_string='Wi-Fi SSID used to connect during setup wizard'),
    options.Option(
        'wifi_password', mandatory=True,
        help_string='Wi-Fi password used to connect during setup wizard'),
    options.Option(
        'device_security', default='Passcode',
        help_string='The security method to select during setup wizard'),
    options.Option(
        'bluetooth_device_name',
        help_string='Name of a nearby bluetooth pairing device'),
    options.Option(
        'device_phone_number',
        help_string="The phone number of the SIM card inserted in the first "
                    "SIM slot. This must be formatted exactly as you would "
                    "dial it."),
    options.Option(
        'device_phone_number2',
        help_string="The phone number of the SIM card inserted in the second "
                    "SIM slot. If the device has only one SIM slot, you can "
                    "leave this blank. This must be formatted exactly as you "
                    "would dial it."),
    options.Option(
        'telephony_service_number1',
        help_string="The phone number to call and message from "
                    "in the telephony service. In Twilio go to "
                    "https://www.twilio.com/user/account/"
                    "phone-numbers/incoming and click on the number "
                    "you want to use, then copy the 'Phone Number' "
                    "field exactly."),
    options.Option(
        'telephony_service_number2',
        help_string="The second phone number to call and message from "
                    "in the telephony service. In Twilio go to "
                    "https://www.twilio.com/user/account/"
                    "phone-numbers/incoming and click on the number "
                    "you want to use, then copy the 'Phone Number' "
                    "field exactly."),
    options.Option(
        # this is a private key, that will not be exported to
        # autopilot directly
        '_twilio_account_sid', mandatory=True,
        help_string='Account SID for Twilio. Please log into '
                    'https://www.twilio.com/login '
                    'and select \'Dashboard -> Show API Credentials\'.'
                    'Then copy and paste the Account SID value here.'),
    options.Option(
        # this is a private key, that will not be exported to
        # autopilot directly
        '_twilio_auth_token', mandatory=True,
        help_string='Auth token for Twilio account. Please log into '
                    'https://www.twilio.com/login '
                    'and select \'Dashboard -> Show API Credentials\'.'
                    'Then copy and paste the Auth token value here.'),
    options.Option(
        # this is a private key, that will not be exported to
        # autopilot directly
        'max_unity8_retry_delay', default=30000,
        help_string='The boundary for how long we should be retrying until '
                    'unity8 is started'),
    options.Option(
        'app_startup_cold_runs', default=0,
        help_string='How many iterations to run for the cold start app '
                    'performance tests?'),
    options.Option(
        'app_startup_hot_runs', default=0,
        help_string='How many iterations to run for the hot start app '
                    'performance tests?'),
    options.Option(
        'email_outlook', default='platform_qa_test@outlook.com',
        help_string='@outlook.com email address to use for email tests'),
    options.Option(
        'password_outlook', mandatory=True,
        help_string='@outlook.com password to use for email tests'),
    options.Option(
        'email_yahoo', default='platform_qa_test_account@yahoo.com',
        help_string='@yahoo.com email address to use for email tests'),
    options.Option(
        'password_yahoo', mandatory=True,
        help_string='@yahoo.com password to use for email tests'),
    options.Option(
        'username_imap', default='platform-qa-test-account',
        help_string='IMAP username to use for email tests'),
    options.Option(
        'password_imap', mandatory=True,
        help_string='IMAP password to use for email tests'),
    options.Option(
        'pictures_scalability_runs',
        help_string='The different loads used for scalability pictures tests '
                    'separated by comma'),
    options.Option(
        '_practitest_project_id',
        help_string='Practitest project ID to use for retrieving test lists.'),
    options.Option(
        '_practitest_api_token',
        help_string='Practitest api token used for authentication.'),
    options.Option(
        'country_sim', default='United States',
        help_string='The name of the country of the SIM card to use for '
                    'telegram login'),
]

def get_user_config_dir():
    """Return the path to the user configuration directory."""
    conf_dir = os.environ.get(
        'XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
    return conf_dir

def get_device_config_file_path():
    """Return the path of the config file copied to the device."""
    return '/tmp/ubuntu-system-tests.conf'

def get_device_config_section():
    """Return the name of config section to use."""
    return os.environ.get('CONFIG_SECTION')

def get_config_stack_from_file(file_path, config_section=None):
    """Return config stack from specified file path."""
    config = UbuntuSystemTestsConfig(file_path, section=config_section)
    config.get_config_from_file()
    return config

def get_device_config_stack():
    """Return the config stack variable"""
    global config_stack
    if not config_stack:
        config_stack = get_config_stack_from_file(
            get_device_config_file_path(), get_device_config_section())
    return config_stack

def get_test_config_values(config_stack):
    """ Return a string containing the comma separated key=value options.
    The list includes the config values and the temporal ones as well
    """
    return config_stack.get_autopilot_config_string()

class UbuntuSystemTestsConfig():

    """Class to get and save test configuration data."""

    def __init__(self, file_name=None, section=None, options=None):
        """
        Construct the config and parser object.
        :param file_name: Name of config file to load. Default file is used if
                          value is not specified.
        :param section: Name of section to use in config file. Default section
                        is used if not specified.
        :param options: List of options that should be specified in config.
                        Default list of options is used if not specified.
        """
        user_config_dir = get_user_config_dir()
        if not os.path.exists(user_config_dir):
            os.mkdir(user_config_dir)
        self.file_path = os.path.join(
            user_config_dir, file_name or DEFAULT_CONF)
        self.section = section or KEY_DEFAULT
        self.parser = configparser.ConfigParser(
            allow_no_value=True, empty_lines_in_values=False,
            default_section=KEY_DEFAULT)
        self.config = {}
        self.options = options or CONFIG_OPTIONS

    def set(self, option_name, value):
        """Set a config value.
        :param option_name: Name of config option.
        :param value: Value to set.
        """
        self.config[option_name] = value
        self.save()

    def get(self, option_name, default=''):
        """Get a config value.
        :param option_name: Name of config option to get.
        :param default: Value to return as default if item doesn't exist.
        :return: Config option value if it exists. If the config option itself
        doesn't exist, return None. Otherwise if the option exists but no value
        is defined return the default value."""
        return self.config.get(option_name, default)

    def save(self):
        """Save the config to file."""
        mode = 'r+' if os.path.exists(self.file_path) else 'w'
        with open(self.file_path, mode) as f:
            self.parser.write(f)

    def get_config_from_string(self, string):
        """Load config from a string. This is used for selftests only.
        :param string: Config string to load from.
        """
        self.parser.read_string(string)
        self._save_config_for_section()

    def get_config_from_file(self):
        """Load config from file."""
        self.parser.read(self.file_path)
        self._save_config_for_section()

    def _save_config_for_section(self):
        """Save the current config for the specified section only."""
        self.config = self.parser[self.section]

    def get_missing_options(self, silent=False):
        """Check through each required option and get a value for each one
        that is not present.
        :param silent: Whether to query stdin for an input value or take a
                       default value where it exists.
        """
        save_required = False
        for option in self.options:
            if option.name in self.config:
                # name exists, check if its mandatory and if a value exists
                if option.mandatory and not self.config.get(option.name):
                    # mandatory and no value exists, so get one
                    self._get_option_value(option, silent)
                    save_required = True
            else:
                # option name not in config, get a value
                self._get_option_value(option, silent)
                save_required = True
        self._save_if_required(save_required)

    def _save_if_required(self, save_required):
        """Save config if it is required.
        :param save_required: Whether to save or not.
        """
        if save_required:
            self.save()

    def _get_option_value(self, option, silent):
        """Request a value for specified option.
        :param option: Required option.
        :param silent: Whether to query stdin or take default value.
        """
        if silent:
            default = option.default
            if default:
                self.config[option.name] = option.default
                logger.warning(
                    'In silent mode no value specified for option: "{o}". '
                    'Using default value "{v}".'.format(
                        o=option.name, v=default))
            else:
                logger.warning(
                    'In silent mode no default value specified for option: '
                    '"{o}". Continuing with value unset.'.format(
                        o=option.name))
        else:
            self._get_option_value_from_stdin(option)

    def _get_option_value_from_stdin(self, option):
        """Request option value from stdin.
        :param option: Required option.
        """
        help_msg = self._get_option_help_text(option)
        print(help_msg, end='')
        value = self._get_value_from_user_or_default(option)
        self.config[option.name] = value

    def _get_option_help_text(self, option):
        """Return help message for specified option.
        :param option: Required option.
        """
        help_msg = option.help_string
        default = option.default
        if default:
            help_msg += ' (leave blank to use default value: {})'.format(
                default)
        elif not option.mandatory:
            help_msg += ' (leave blank if not required)'
        help_msg += ': '
        return help_msg

    def _get_value_from_user_or_default(self, option):
        """Return value from stdin or default value if no stdin value."""
        return input() or option.default

    def get_autopilot_config_string(self):
        """Return the configuration in a string usable by Autopilot.

        That is, a list of key=value pairs serparated by comma.

        Any keys starting with _ will be ignored.

        """
        config = ''
        for key, value in self.config.items():
            if not key.startswith('_'):
                config += '{k}={v},'.format(k=key, v=value)

        # remove trailing ',' if it exists
        if config.endswith(','):
            config = config[:-1]
        return config


import pandas as pd
from scipy.sparse import coo_matrix
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from movie_stream.models import Rating, Movie, LightFMModel
import pickle
from movie_stream.users.models import User

def get_recommendations(userid):
    """Get movie and follow recommendations for a user.

    Args:
        userid (int): id of a user in database
    """
    ratings_df = pd.DataFrame.from_records(
        Rating.objects.all().values('movie_id', 'rating', 'user_id'))
    movies_df = pd.DataFrame.from_records(
        Movie.objects.all().values('id', 'title'))
    ratings_df = ratings_df.pivot(
        index='user_id', columns='movie_id', values='rating')
    train_sparse = coo_matrix(ratings_df)

    model = pickle.loads(
        LightFMModel.objects.all().last().pickle_model)  # get last model
    # should really just store n_users and n_tems in database
    # so we don't have to load the whold database everytime
    n_users, n_items = train_sparse.shape

    user_row = ratings_df.index.tolist().index(userid)
    # This will fail unless model has run at least once for user
    scores = model.predict(user_row, np.arange(n_items))
    # rank them in order of most liked to least
    top_items = movies_df['id'].values[np.argsort(-scores)]

    # Now we can use the feature vectors of each user to find who
    # is the most similar to them. (Likes the same type of movies)
    similarities = cosine_similarity(model.user_embeddings)
    ind = np.argpartition(similarities[user_row], -6)[-6:]

    to_follow_ids = ratings_df.index[ind].tolist()
    # don't want to recomend to follow user self
    to_follow_ids.remove(userid)
    return top_items[:10], to_follow_ids


#User function Template for python3
from collections import defaultdict ,deque
class Solution:
    def findOrder(self,alien_dict, N, K):
    # code here
        graph = defaultdict(list)
        degree = defaultdict(int)

        for i in range(len(alien_dict) -1 ):

            edges = self.comper(alien_dict[i] ,alien_dict[i+1])

            if edges:

                graph[edges[0]].append(edges[1])
                degree[edges[0]] = 0 if  edges[0] not in degree else degree[edges[0]]
                degree[edges[1]] += 1

        que = deque()

        for i in range(k):
            if not degree[chr(97+i)]:
                que.append(chr(97+i))
        order = []

        while que:
            curNode = que.popleft()
            order.append(curNode)
            for nbr in graph[curNode]:
                degree[nbr] -=1
                if not degree[nbr]:
                    que.append(nbr)

        return "".join(order)

    def comper(self , str1 , str2 ):
        i = j = 0
        while i < len(str1) and j < len(str2):
            if str1[i] != str2[j]:
                return (str1[i] ,str2[j])
            i += 1
            j += 1


__all__ = ['scaffold']

from gevent import monkey

monkey.patch_all()
import csv
import gevent
from typing import List

from src.BusinessCentralLayer.setting import *

command_set = {

    # ---------------------------------------------
    # 部署接口
    # ---------------------------------------------
    'deploy': "部署项目（定时任务/Flask 开启与否取决于yaml配置文件）",
    # ---------------------------------------------
    # 调试接口
    # ---------------------------------------------
    "decouple": "立即唤醒一次subs_ddt链接解耦任务",
    "overdue": "立即执行一次过时链接清洗任务",
    "run": "立即执行一次采集任务（强制使用协程加速）",
    "force_run": "强制执行采集任务",
    "remain": "读取剩余订阅数量",
    "ping": "测试数据库连接",
    "panel": "打开桌面前端面板",
    "packer": "打包生成桌面客户端（windows可用）",
    "launcher": "返回采集器的单步启动接口",
    # ---------------------------------------------
    # 调用示例
    # ---------------------------------------------
    "example": "python main.py ping"
}

class _ConfigQuarantine(object):
    def __init__(self):

        self.root = [
            SERVER_DIR_CLIENT_DEPORT, SERVER_PATH_DEPOT_VCS,
            SERVER_DIR_DATABASE_CACHE, SERVER_DIR_CACHE_BGPIC
        ]
        self.flag = False

    def set_up_file_tree(self, root):
        """
        --/qinse/V2RaycSpider{verNum}
            --BCL
            --BLL
            --BVL
            --Database
                --client_depot
                    --vcs.csv
                --logs
                    --*error.log
                    --*runtime.log
                --temp_cache
                    --*AnyTempCacheFile...
                --*CrawlFetchHistory.txt
                --fake_useragent_0.1.11.json
            --*tests
        """

        # 检查默认下载地址是否残缺 深度优先初始化系统文件
        for child_ in root:
            if not os.path.exists(child_):
                self.flag = True
                try:
                    # 初始化文件夹
                    if os.path.isdir(child_) or not os.path.splitext(child_)[-1]:
                        os.mkdir(child_)
                        logger.success(f"系统文件链接成功->{child_}")
                    # 初始化文件
                    else:
                        if child_ == SERVER_PATH_DEPOT_VCS:
                            try:
                                with open(child_, 'w', encoding='utf-8', newline='') as fpx:
                                    csv.writer(fpx).writerow(['version', 'title'])
                                logger.success(f"系统文件链接成功->{child_}")
                            except Exception as ep:
                                logger.exception(f"Exception{child_}{ep}")
                except Exception as ep:
                    logger.exception(ep)

    @staticmethod
    def check_config():
        if not all(SMTP_ACCOUNT.values()):
            logger.warning('您未正确配置<通信邮箱>信息(SMTP_ACCOUNT)')
        if not SERVER_CHAN_SCKEY:
            logger.warning("您未正确配置<Server酱>的SCKEY")
        if not all([REDIS_SLAVER_DDT.get("host"), REDIS_SLAVER_DDT.get("password")]):
            logger.warning('您未正确配置<Redis-Slave> 本项目的部分功能将毫无意义')
        if not all([REDIS_MASTER.get("host"), REDIS_MASTER.get("password")]):
            logger.error("您未正确配置<Redis-Master> 此配置为“云彩姬”的核心组件，请配置后重启项目！")
            exit()

    def run(self):
        try:
            if [cq for cq in reversed(self.root) if not os.path.exists(cq)]:
                logger.warning('系统文件残缺！')
                logger.debug("启动<工程重构>模块...")
                self.set_up_file_tree(self.root)
            self.check_config()

        finally:
            if self.flag:
                logger.success(">>> 运行环境链接完成，请重启项目")
                logger.warning(">>> 提醒您正确配置Chrome及对应版本的ChromeDriver")
                exec("if self.flag:\n\texit()")

_ConfigQuarantine().run()

class _ScaffoldGuider(object):
    # __slots__ = list(command_set.keys())

    def __init__(self):
        # 脚手架公开接口
        self.scaffold_ruler = [i for i in self.__dir__() if i.startswith('_scaffold_')]

    @logger.catch()
    def startup(self, driver_command_set: List[str]):
        """
        仅支持单进程使用
        @param driver_command_set: 在空指令时列表仅有1个元素，表示启动路径
        @return:
        """
        # logger.info(f">>> {' '.join(driver_command_set)}")

        # -------------------------------
        # TODO 优先级0：预处理指令集
        # -------------------------------
        # CommandId or List[CommandId]
        driver_command: List[str] = []

        # 未输入任何指令 列出脚手架简介
        if len(driver_command_set) == 1:
            print("\n".join([f">>> {menu[0].ljust(20, '-')}|| {menu[-1]}" for menu in command_set.items()]))
            return True
        # 输入立即指令 转译指令
        elif len(driver_command_set) == 2:
            driver_command = [driver_command_set[-1].lower(), ]
        # 输入指令集 转译指令集
        elif len(driver_command_set) > 2:
            driver_command = list(set([command.lower() for command in driver_command_set[1:]]))

        # 捕获意料之外的情况
        if not isinstance(driver_command, list):
            return True
        # -------------------------------
        # TODO 优先级1：解析运行参数
        # -------------------------------

        # TODO --help 帮助菜单（继续完善相关功能）
        # 使用该参数时系统不解析运行指令
        if '--help' in driver_command:
            logger.info(f">>>GuiderHelp || 帮助菜单")
            driver_command.remove("--help")
            for command_ in driver_command:
                introduction = command_set.get(command_)
                if introduction:
                    print(f"> {command_.ljust(20, '-')}|| {introduction}")
                else:
                    print(f"> {command_}指令不存在")
            return True

        # -------------------------------
        # TODO 优先级2：运行单线程指令
        # -------------------------------

        # 协程任务队列
        task_list = []

        # 测试数据库连接
        while driver_command.__len__() > 0:
            _pending_command = driver_command.pop()
            try:
                task_list.append(gevent.spawn(eval(f"self._scaffold_{_pending_command}")))
            except Exception as e:
                logger.warning(f'脚手架暂未授权指令<{_pending_command}> {e}')
        else:
            # 并发执行以上指令
            gevent.joinall(task_list)

        # -------------------------------
        # TODO 优先级3：自定义参数部署（阻塞线程）
        # -------------------------------

        if 'deploy' in driver_command:
            self._scaffold_deploy()

    @staticmethod
    def _scaffold_deploy():
        logger.info(f"<ScaffoldGuider> Deploy || MainProcess")
        from src.BusinessCentralLayer.middleware.interface_io import SystemInterface as app
        app.run(deploy_=True)

    @staticmethod
    def _scaffold_decouple():
        logger.info(f"<ScaffoldGuider> Decouple || General startup")

        from src.BusinessLogicLayer.plugins.ddt_subs import SubscribesCleaner
        SubscribesCleaner(debug=True).interface()

    @staticmethod
    def _scaffold_overdue():
        logger.info(f"<ScaffoldGuider> Overdue || Redis DDT")
        from src.BusinessCentralLayer.middleware.interface_io import SystemInterface as app
        app.ddt()

    @staticmethod
    def _scaffold_run():
        if not os.path.exists(CHROMEDRIVER_PATH):
            logger.error(f"<ScaffoldGuider> ForceRun || ChromedriverNotFound ||"
                         f" 未查找到chromedriver驱动，请根据技术文档正确配置\n"
                         f">>> https://github.com/QIN2DIM/V2RayCloudSpider")
            return False

        logger.info(f"<ScaffoldGuider> Run || MainCollector")
        from src.BusinessCentralLayer.middleware.interface_io import SystemInterface as app
        app.run(deploy_=False)

    @staticmethod
    def _scaffold_force_run():

        if not os.path.exists(CHROMEDRIVER_PATH):
            logger.error(f"<ScaffoldGuider> ForceRun || ChromedriverNotFound ||"
                         f" 未查找到chromedriver驱动，请根据技术文档正确配置\n"
                         f">>> https://github.com/QIN2DIM/V2RayCloudSpider")
            return False

        logger.info(f"<ScaffoldGuider> ForceRun || MainCollector")
        from src.BusinessCentralLayer.setting import CRAWLER_SEQUENCE
        from src.BusinessLogicLayer.apis.vulcan_ash import ForceRunRelease
        ForceRunRelease(task_docker=CRAWLER_SEQUENCE).interface()

    @staticmethod
    def _scaffold_remain():
        from src.BusinessCentralLayer.middleware.redis_io import RedisClient
        logger.info(f"<ScaffoldGuider> Remain || {RedisClient().subs_info()}")

    @staticmethod
    def _scaffold_ping():
        from src.BusinessCentralLayer.middleware.redis_io import RedisClient
        logger.info(f"<ScaffoldGuider> Ping || {RedisClient().test()}")

    @staticmethod
    def _scaffold_panel():
        from src.BusinessCentralLayer.middleware.interface_io import SystemInterface
        SystemInterface.system_panel()

    @staticmethod
    def _scaffold_packer():
        pass
        # packer()

    @staticmethod
    def _scaffold_launcher(name: str = None) -> dict:
        from src.BusinessLogicLayer.cluster.slavers import actions
        from src.BusinessLogicLayer.apis.ghost_filler import gevent_ghost_filler
        response = {"entropy": actions.__entropy__, "actions": actions}
        if name is None:
            return response
        else:
            gevent_ghost_filler(docker=eval("actions.name"), silence=True)

scaffold = _ScaffoldGuider()

def packer(ico_path: str = None, output_dir=None, py_panel=None):
    """
    打包panel
    :param ico_path:
    :param output_dir:
    :param py_panel:
    :return:
    """
    # ----------------------------
    # 参数整理
    # ----------------------------
    if ico_path is None:
        ico_path = join(PANEL_DIR_ROOT, 'logo.ico')
    if output_dir is None:
        output_dir = join(PANEL_DIR_ROOT, 'bin')
        if not os.path.exists(output_dir):
            os.mkdir(output_dir)
    if py_panel is None:
        py_panel = join(PANEL_DIR_ROOT, 'panel.py')
    # ----------------------------
    # 根据不同的操作系统选择不同的打包方案
    # > Windows:Pyinstaller
    # https://pyinstaller.readthedocs.io/en/stable/usage.html#options
    # ----------------------------

    if platform.startswith('win'):
        pass
    elif platform.startswith('linux'):
        pass
    elif platform.startswith('darwin'):
        pass

if __name__ == '__main__':
    packer()


from mode_switcher.modes import Mode
import json
import os.path

def main():
    mode = Mode()
    mode_path = './input/scenario_2/mode.json'
    mode_type = None

    if os.path.isfile(mode_path):
        with open(mode_path) as f:
            mode_type = json.load(f)

    if mode_type:
        mode.run(mode_type['mode'])
    else:
        mode.run('normal')

if __name__ == '__main__':
    main()


# Optical Flow
import cv2
import numpy as np
from matplotlib import pyplot as plt

# Load the images "OF1.jpg" and "OF2.jpg" and change them to grayscale.
img1 = cv2.imread('OF1.jpg')
img2 = cv2.imread('OF2.jpg')

b,g,r = cv2.split(img1) # Changing the order from bgr to rgb so that matplotlib can show it
img1 = cv2.merge([r,g,b])
b,g,r = cv2.split(img2) # Changing the order from bgr to rgb so that matplotlib can show it
img2 = cv2.merge([r,g,b])

gray1 = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY)
gray2 = cv2.cvtColor(img2, cv2.COLOR_RGB2GRAY)

# plt.figure(figsize = (18,18))
# plt.subplot(1,2,1)
# plt.imshow(gray1, cmap = 'gray')
# plt.subplot(1,2,2)
# plt.imshow(gray2, cmap = 'gray')

# They look pretty much the same. We can now use optical flow to find out how the objects
# in the picutures have moved. We do this by first using [cv2.goodFeaturesToTrack]
# (https://docs.opencv.org/3.4/dd/d1a/group__imgproc__feature.html#ga1d6bb77486c8f92d79c8793ad995d541)
# to find features in the first image. You can play around with the parameters to see the difference.
feat1 = cv2.goodFeaturesToTrack(gray1, maxCorners=100, qualityLevel=0.4, minDistance=7)
# The function finds the most prominent corners in the image or in the specified image region

# Next we use the function [cv2.calcOpticalFlowPyrLK]
# (https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323)
# to track the features in the next image.
feat2, status, error = cv2.calcOpticalFlowPyrLK(gray1, gray2, feat1, None)

# We now have the location of the features from the first image in the second image. To find the movement we can draw a line between the keypoints.
# for i in range(len(feat1)):
#     cv2.line(img2, (feat1[i][0][0], feat1[i][0][1]), (feat2[i][0][0], feat2[i][0][1]), (0, 255, 0), 2)
#     cv2.circle(img2, (feat1[i][0][0], feat1[i][0][1]), 5, (0, 255, 0), -1)

# plt.figure(figsize=(15,15))
# plt.imshow(img2)

# ## 4a) Change the code such that only the keypoints that have moved will be showed in the image.

# # feat1 = cv2.goodFeaturesToTrack(gray1, maxCorners=100, qualityLevel=0.4, minDistance=7)
# # feat2 = cv2.goodFeaturesToTrack(gray2, maxCorners=100, qualityLevel=0.4, minDistance=7)
# # for i in range(len(feat1)):
# #     if feat1[i][0][0] != feat2[i][0][0] and feat1[i][0][1] != feat2[i][0][1]:
# #         cv2.line(img2, (feat1[i][0][0], feat1[i][0][1]), (feat2[i][0][0], feat2[i][0][1]), (0, 255, 0), 2)
# #         cv2.circle(img2, (feat1[i][0][0], feat1[i][0][1]), 5, (0, 255, 0), -1)
# # plt.figure(figsize=(15,15))
# # plt.imshow(img2)

# plt.show()

### Dense Optical Flow
# The sparse optical flow finds the flow of the detected keypoints.
#  We will now try to use dense optical flow which, finds the flow of all the points in the picture.
# For this example we use the same two images, so we don't have to load them again.
# To find the optical flow we use the function [cv2.calcOpticalFlowFarneback]
# (https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af).
# Check out the description to see what all the different parameters does, and try to change them to see the difference.
flow = cv2.calcOpticalFlowFarneback(gray1, gray2, None, 0.5, 3, 15, 3, 5, 1.5, 0)
# The function returns an array containing the flow vector for every pixel.
# This can be changed to a magnitude and an angle using the function cv2.cartToPolar.

# Find a way to represent the flow in the image. For example by drawing vectors the relevant places
# or by making a new image with colors representing the flow of every pixel.
# An example of retrieving the magnitude and angle is shown below, which you can use if you like.
mag, ang = cv2.cartToPolar(flow[:,:,0], flow[:,:,1]) # Retrieving the magnitude and angle of every pixel

### Fill in some code to represent the flow here ###
old_shape = img1.shape
prev_img_gray = gray1
current_img = img2
current_img_gray= gray2
assert current_img.shape == old_shape # assert: return TRUE if the conditions is true
hsv = np.zeros_like(img1)
hsv[...,1] = 255 # all the element of the first column set to 255
flow = None
flow = cv2.calcOpticalFlowFarneback(prev=prev_img_gray,
                                      next=current_img_gray, flow=flow,
                                      pyr_scale=0.8, levels=15, winsize=5,
                                      iterations=10, poly_n=5, poly_sigma=0,
                                      flags=10)
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv[..., 0] = ang*180/np.pi/2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
cv2.imshow(img2,rgb)
plt.show()


from apscheduler.schedulers.background import BackgroundScheduler
from lib.providers import wrapper
from lib.mongo import timeseries
from lib import tools
from datetime import datetime
import time
import json

providers = tools.read_json('static/providers.json')
centers = tools.read_json('static/centers.json')
predictions = tools.read_json('static/predictions.json')

def update(provider, city, predict):
    # Get the information for the city
    try:
        stations = wrapper.collect(provider, city)
    except:
        return
    # Update the database
    if predict == 'Yes':
        timeseries.update_city(stations, city)
    # Save the data for the map
    geojson = tools.json_to_geojson(stations)
    with open('static/geojson/{0}.geojson'.format(city), 'w') as outfile:
        json.dump(geojson, outfile)
    # Tell the server the city's data was updated
    with open('static/updates/{0}.txt'.format(city), 'w') as outfile:
        outfile.write(datetime.now().isoformat())

if __name__ == '__main__':
    scheduler = BackgroundScheduler()
    for provider, cities in providers.items():
        for city in cities:
            time.sleep(3)
            update(provider, city, predictions[city])
            scheduler.add_job(update, 'interval', seconds=60,
                              args=[provider, city, predictions[city]],
                              misfire_grace_time=50, coalesce=True)
    scheduler.start()
    while True:
        time.sleep(10e-1000000)


#!/usr/bin/env python
import logging
import unittest
import docker
import time
import os

import kuberay_utils.utils as utils

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

# Image version
ray_version = '1.9.0'

# Docker images
ray_image = 'rayproject/ray:1.9.0'
kuberay_operator_image = 'kuberay/operator:nightly'
kuberay_apiserver_image = 'kuberay/apiserver:nightly'

class BasicRayTestCase(unittest.TestCase):
    cluster_template_file = 'tests/config/ray-cluster.mini.yaml.template'

    @classmethod
    def setUpClass(cls):
        # Ray cluster is running inside a local Kind environment.
        # We use port mapping to connect to the Kind environment
        # from another local ray container. The local ray container
        # outside Kind environment has the same ray version as the
        # ray cluster running inside Kind environment.
        utils.delete_cluster()
        utils.create_cluster()
        images = [ray_image, kuberay_operator_image, kuberay_apiserver_image]
        utils.download_images(images)
        utils.apply_kuberay_resources(images, kuberay_operator_image, kuberay_apiserver_image)
        utils.create_kuberay_cluster(BasicRayTestCase.cluster_template_file,
                                     ray_version, ray_image)

    def test_simple_code(self):
        # connect from a ray container client to ray cluster
        # inside a local Kind environment and run a simple test
        client = docker.from_env()
        container = client.containers.run(ray_image,
                                          remove=True,
                                          detach=True,
                                          tty=True,
                                          network_mode='host')
        rtn_code, output = container.exec_run(['python',
                                               '-c', '''
import ray
ray.init(address='ray://127.0.0.1:10001')

def retry_with_timeout(func, count=90):
    tmp = 0
    err = None
    while tmp < count:
        try:
            return func()
        except Exception as e:
            err = e
            tmp += 1
    assert err is not None
    raise err

@ray.remote
def f(x):
    return x * x

def get_result():
    futures = [f.remote(i) for i in range(4)]
    print(ray.get(futures))
    return 0
rtn = retry_with_timeout(get_result)
assert rtn == 0
'''],
                                              demux=True)
        stdout_str, stderr_str = output

        container.stop()

        if stdout_str != b'[0, 1, 4, 9]\n':
            logger.error('test_simple_code returns {}'.format(output))
            raise Exception(('test_simple_code returns invalid result. ' +
                             'Expected: {} Actual: {} Stderr: {}').format(
                b'[0, 1, 4, 9]', stdout_str, stderr_str))
        if rtn_code != 0:
            msg = 'invalid return code {}'.format(rtn_code)
            logger.error(msg)
            raise Exception(msg)

        client.close()

    def test_cluster_info(self):
        # connect from a ray container client to ray cluster
        # inside a local Kind environment and run a test that
        # gets the amount of nodes in the ray cluster.
        client = docker.from_env()
        container = client.containers.run(ray_image,
                                          remove=True,
                                          detach=True,
                                          tty=True,
                                          network_mode='host')
        rtn_code, output = container.exec_run(['python',
                                               '-c', '''
import ray
ray.init(address='ray://127.0.0.1:10001')

print(len(ray.nodes()))
'''],
                                              demux=True)
        stdout_str, _ = output

        container.stop()

        if stdout_str != b'2\n':
            logger.error('test_cluster_info returns {}'.format(output))
            raise Exception(('test_cluster_info returns invalid result. ' +
                             'Expected: {} Actual: {}').format(b'2',
                                                               stdout_str))
        if rtn_code != 0:
            msg = 'invalid return code {}'.format(rtn_code)
            logger.error(msg)
            raise Exception(msg)

        client.close()

class RayFTTestCase(unittest.TestCase):
    cluster_template_file = 'tests/config/ray-cluster.ray-ft.yaml.template'

    @classmethod
    def setUpClass(cls):
        if not utils.ray_ft_supported(ray_version):
            raise unittest.SkipTest("ray ft is not supported")
        utils.delete_cluster()
        utils.create_cluster()
        images = [ray_image, kuberay_operator_image, kuberay_apiserver_image]
        utils.download_images(images)
        utils.apply_kuberay_resources(images, kuberay_operator_image, kuberay_apiserver_image)
        utils.create_kuberay_cluster(RayFTTestCase.cluster_template_file,
                                     ray_version, ray_image)

    def test_kill_head(self):
        # This test will delete head node and wait for a new replacement to
        # come up.
        utils.shell_assert_success(
            'kubectl delete pod $(kubectl get pods -A | grep -e "-head" | awk "{print \$2}")')

        # wait for new head node to start
        time.sleep(80)
        utils.shell_assert_success('kubectl get pods -A')

        # make sure the new head is ready
        # shell_assert_success('kubectl wait --for=condition=Ready pod/$(kubectl get pods -A | grep -e "-head" | awk "{print \$2}") --timeout=900s')
        # make sure both head and worker pods are ready
        rtn = utils.shell_run(
            'kubectl wait --for=condition=ready pod -l rayCluster=raycluster-compatibility-test --all --timeout=900s')
        if rtn != 0:
            utils.shell_run('kubectl get pods -A')
            utils.shell_run(
                'kubectl describe pod $(kubectl get pods | grep -e "-head" | awk "{print \$1}")')
            utils.shell_run(
                'kubectl logs $(kubectl get pods | grep -e "-head" | awk "{print \$1}")')
            utils.shell_run(
                'kubectl logs -n $(kubectl get pods -A | grep -e "-operator" | awk \'{print $1 "  " $2}\')')
        assert rtn == 0

    def test_ray_serve(self):
        client = docker.from_env()
        container = client.containers.run(ray_image, remove=True, detach=True, stdin_open=True, tty=True,
                                          network_mode='host', command=["/bin/sh", "-c", "python"])
        s = container.attach_socket(
            params={'stdin': 1, 'stream': 1, 'stdout': 1, 'stderr': 1})
        s._sock.setblocking(0)
        s._sock.sendall(b'''
import ray
import time
import ray.serve as serve
import os
import requests
from ray._private.test_utils import wait_for_condition

def retry_with_timeout(func, count=90):
    tmp = 0
    err = None
    while tmp < count:
        try:
            return func()
        except Exception as e:
            err = e
            tmp += 1
    assert err is not None
    raise err

ray.init(address='ray://127.0.0.1:10001')

@serve.deployment
def d(*args):
    return f"{os.getpid()}"

d.deploy()
pid1 = ray.get(d.get_handle().remote())

print('ready')
        ''')

        count = 0
        while count < 90:
            try:
                buf = s._sock.recv(4096)
                logger.info(buf.decode())
                if buf.decode().find('ready') != -1:
                    break
            except Exception as e:
                pass
            time.sleep(1)
            count += 1
        if count >= 90:
            raise Exception('failed to run script')

        # kill the gcs on head node. If fate sharing is enabled
        # the whole head node pod will terminate.
        utils.shell_assert_success(
            'kubectl exec -it $(kubectl get pods -A| grep -e "-head" | awk "{print \\$2}") -- /bin/bash -c "ps aux | grep gcs_server | grep -v grep | awk \'{print \$2}\' | xargs kill"')
        # wait for new head node getting created
        time.sleep(10)
        # make sure the new head is ready
        utils.shell_assert_success(
            'kubectl wait --for=condition=Ready pod/$(kubectl get pods -A | grep -e "-head" | awk "{print \$2}") --timeout=900s')

        s._sock.sendall(b'''
def get_new_value():
    return ray.get(d.get_handle().remote())
pid2 = retry_with_timeout(get_new_value)

if pid1 == pid2:
    print('successful: {} {}'.format(pid1, pid2))
    sys.exit(0)
else:
    print('failed: {} {}'.format(pid1, pid2))
    raise Exception('failed')
        ''')

        count = 0
        while count < 90:
            try:
                buf = s._sock.recv(4096)
                logger.info(buf.decode())
                if buf.decode().find('successful') != -1:
                    break
                if buf.decode().find('failed') != -1:
                    raise Exception('test failed {}'.format(buf.decode()))
            except Exception as e:
                pass
            time.sleep(1)
            count += 1
        if count >= 90:
            raise Exception('failed to run script')

        container.stop()
        client.close()

    def test_detached_actor(self):
        # This test will run a ray client and start a detached actor at first.
        # Then we will kill the head node and kuberay will start a new head node
        # replacement. Finally, we will try to connect to the detached actor again.
        client = docker.from_env()
        container = client.containers.run(ray_image, remove=True, detach=True, stdin_open=True, tty=True,
                                          network_mode='host', command=["/bin/sh", "-c", "python"])
        s = container.attach_socket(
            params={'stdin': 1, 'stream': 1, 'stdout': 1, 'stderr': 1})
        s._sock.setblocking(0)
        s._sock.sendall(b'''
import ray
import time

def retry_with_timeout(func, count=90):
    tmp = 0
    err = None
    while tmp < count:
        try:
            return func()
        except Exception as e:
            err = e
            tmp += 1
    assert err is not None
    raise err

ray.init(address='ray://127.0.0.1:10001')

@ray.remote
class A:
    def ready(self):
        import os
        return os.getpid()

a = A.options(name="a", lifetime="detached", max_restarts=-1).remote()
res1 = ray.get(a.ready.remote())
print('ready')

        ''')

        count = 0
        while count < 90:
            try:
                buf = s._sock.recv(4096)
                logger.info(buf.decode())
                if buf.decode().find('ready') != -1:
                    break
            except Exception as e:
                pass
            time.sleep(1)
            count += 1
        if count >= 90:
            raise Exception('failed to run script')

        # kill the gcs on head node. If fate sharing is enabled
        # the whole head node pod will terminate.
        utils.shell_assert_success(
            'kubectl exec -it $(kubectl get pods -A| grep -e "-head" | awk "{print \\$2}") -- /bin/bash -c "ps aux | grep gcs_server | grep -v grep | awk \'{print \$2}\' | xargs kill"')
        # wait for new head node getting created
        time.sleep(10)
        # make sure the new head is ready
        utils.shell_assert_success(
            'kubectl wait --for=condition=Ready pod/$(kubectl get pods -A | grep -e "-head" | awk "{print \$2}") --timeout=900s')

        s._sock.sendall(b'''
def get_detached_actor():
    return ray.get_actor("a")
a = retry_with_timeout(get_detached_actor)

def get_new_value():
    return ray.get(a.ready.remote())
res2 = retry_with_timeout(get_new_value)

if res1 != res2:
    print('successful: {} {}'.format(res1, res2))
    sys.exit(0)
else:
    print('failed: {} {}'.format(res1, res2))
    raise Exception('failed')
        ''')

        count = 0
        while count < 90:
            try:
                buf = s._sock.recv(4096)
                logger.info(buf.decode())
                if buf.decode().find('successful') != -1:
                    break
                if buf.decode().find('failed') != -1:
                    raise Exception('test failed {}'.format(buf.decode()))
            except Exception as e:
                pass
            time.sleep(1)
            count += 1
        if count >= 90:
            raise Exception('failed to run script')

        container.stop()
        client.close()

class RayServiceTestCase(unittest.TestCase):
    service_template_file = 'tests/config/ray-service.yaml.template'
    service_serve_update_template_file = 'tests/config/ray-service-serve-update.yaml.template'
    service_cluster_update_template_file = 'tests/config/ray-service-cluster-update.yaml.template'

    @classmethod
    def setUpClass(cls):
        if not utils.ray_service_supported(ray_version):
            raise unittest.SkipTest("ray service is not supported")
        # Ray Service is running inside a local Kind environment.
        # We use the Ray nightly version now.
        # We wait for the serve service ready.
        # The test will check the successful response from serve service.
        utils.delete_cluster()
        utils.create_cluster()
        images = [ray_image, kuberay_operator_image, kuberay_apiserver_image]
        utils.download_images(images)
        utils.apply_kuberay_resources(images, kuberay_operator_image, kuberay_apiserver_image)
        utils.create_kuberay_service(
            RayServiceTestCase.service_template_file, ray_version, ray_image)

    def test_ray_serve_work(self):
        time.sleep(5)
        curl_cmd = 'curl  -X POST -H \'Content-Type: application/json\' localhost:8000 -d \'["MANGO", 2]\''
        utils.wait_for_condition(
            lambda: utils.shell_run(curl_cmd) == 0,
            timeout=15,
        )
        utils.create_kuberay_service(
            RayServiceTestCase.service_serve_update_template_file,
            ray_version, ray_image)
        curl_cmd = 'curl  -X POST -H \'Content-Type: application/json\' localhost:8000 -d \'["MANGO", 2]\''
        time.sleep(5)
        utils.wait_for_condition(
            lambda: utils.shell_run(curl_cmd) == 0,
            timeout=60,
        )
        utils.create_kuberay_service(
            RayServiceTestCase.service_cluster_update_template_file,
            ray_version, ray_image)
        time.sleep(5)
        curl_cmd = 'curl  -X POST -H \'Content-Type: application/json\' localhost:8000 -d \'["MANGO", 2]\''
        utils.wait_for_condition(
            lambda: utils.shell_run(curl_cmd) == 0,
            timeout=180,
        )

def parse_environment():
    global ray_version, ray_image, kuberay_operator_image, kuberay_apiserver_image
    for k, v in os.environ.items():
        if k == 'RAY_IMAGE':
            ray_image = v
            ray_version = ray_image.split(':')[-1]
        elif k == 'OPERATOR_IMAGE':
            kuberay_operator_image = v
        elif k == 'APISERVER_IMAGE':
            kuberay_apiserver_image = v

if __name__ == '__main__':
    parse_environment()
    logger.info('Setting Ray image to: {}'.format(ray_image))
    logger.info('Setting Ray version to: {}'.format(ray_version))
    logger.info('Setting KubeRay operator image to: {}'.format(kuberay_operator_image))
    logger.info('Setting KubeRay apiserver image to: {}'.format(kuberay_apiserver_image))
    unittest.main(verbosity=2)


# django imports
from django.conf import settings

# sendgrid imports
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import (
    Mail,
    Attachment,
    FileContent,
    FileName,
    FileType,
    Disposition,
    Content,
)

# additional imports
from typing import Tuple
import os

class SendGrid:
    SENDGRID_USE = settings.SENDGRID_USE
    SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY', '')
    SENDGRID_TEMPLATE_ID = os.environ.get('SENDGRID_TEMPLATE_ID', '')
    SENDGRID_SENDER_EMAIL = os.environ.get('SENDGRID_SENDER_EMAIL', '')

    def __init__(self) -> None:

        '''
        This is the constructor for SendGrid class.
        In future a logger will also be initialized here.
        '''

        if self.SENDGRID_USE:
            self.sendgrid_client = SendGridAPIClient(self.SENDGRID_API_KEY)

    def mail_object(self, to_email: str, verification_link: str) -> Mail:

        '''
        This method will create a Mail object which will be sent by the email,
        Mail object consists of from_email, to_emails, subject, template_id and dynamic_template_data
        '''

        mail = Mail(
            from_email=self.SENDGRID_SENDER_EMAIL,
            to_emails=to_email,
            subject='SSM Forgot Password hero',
        )
        mail.template_id = self.SENDGRID_TEMPLATE_ID
        mail.dynamic_template_data = {'verification_link': verification_link}

        return mail

    def send_email(self, to_email: str, verification_link: str) -> Tuple[bool, str]:

        '''
        This method will send the mail and returns a tuple of bool and str,
        corresponding to success and success message or error and error message(in case of failure)
        '''

        if not self.SENDGRID_USE:
            return True, 'Mail was not sent because sendgrid use is set to false'
        try:
            mail = self.mail_object(to_email, verification_link)
            sendgrid_response = self.sendgrid_client.send(mail)
            return True, str(sendgrid_response.headers)
        except Exception as e:
            return False, str(e)


import numpy as np
import matplotlib.pyplot as plt
import cv2

image = plt.imread('animation.jpg')
grayscale = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)

g_width = grayscale.shape[0]
g_height = grayscale.shape[1]

kernel1 = np.array([[-1, -1, -1],[-1, 8, -1],[-1, -1, -1]])

pad_quantity = int((len(kernel1) - 1) / 2)

padded_g = pad_arr = np.pad(grayscale, pad_quantity)

processed_img1 = cv2.filter2D(padded_g, -1, kernel1, cv2.BORDER_CONSTANT)
processed_img1 = processed_img1[pad_quantity:g_width+pad_quantity, pad_quantity:g_height+pad_quantity]

print(padded_g[0:5,padded_g.shape[1]-5:])
print(processed_img1)

m, n = kernel1.shape

y, x = padded_g.shape

processed_img2 = np.zeros((g_width, g_height))

for i in range(g_width):
    for j in range(g_height):
        processed_img2[i][j] = int(np.sum(padded_g[i:i+m, j:j+n]*kernel1))

        if(processed_img2[i][j] > 255):
            processed_img2[i][j] = 255
        if(processed_img2[i][j] < 0):
            processed_img2[i][j] = 0

img_set = [image,grayscale,processed_img1, processed_img2]
title_set = ['RGB','Grayscale','Neighborhood', 'BuiltIn']
cmap_set = ['','gray','gray','gray']

n = len(img_set)
plt.figure(figsize=(15,15))

for i in range (n):
    img = img_set[i]
    plt.subplot(2,3,i+1)
    plt.title(title_set[i])
    ch = len(img_set[i].shape)
    if ch == 3:
        plt.imshow(img_set[i])
    else:
        plt.imshow(img_set[i],cmap=cmap_set[i])

    plt.savefig('Neighborhood_BuiltIn_Comparision.jpg')
plt.show()


# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
from mtcnn.model import PNet,RNet,ONet,load_weight
from mtcnn.run import run_pnet,run_rnet,run_onet
from mtcnn.utils import show_bboxes
from mtcnn.bboxUtils import nms,calibrate_box,convert_to_square,get_image_crop,correct_bboxes

import torch
from torch.autograd import Variable

def build_pyramid(image,min_face_size=20.0):
    width,height = image.size
    min_length = min(width,height)

    min_detection_size = 12
    factor = 0.707

    scales = []

    m = min_detection_size / min_face_size
    min_length *= m

    factor_count = 0
    while min_length > min_detection_size:
        scales.append(m * factor **factor_count)
        min_length *= factor
        factor_count += 1

    return scales

pnet_weight = "weights/pnet.npy"
p_net = PNet()
load_weight(p_net,pnet_weight)

file = "1.jpg"
image = Image.open(file)

scales = build_pyramid(image)

bounding_boxes = []
for s in scales:
    boxes = run_pnet(image,p_net,s,0.6)
    bounding_boxes.append(boxes)

# 经过P-Net 后，所有可能包含人脸的box
bounding_boxes = [i for i in bounding_boxes if i is not None]
bounding_boxes = np.vstack(bounding_boxes)

print("经过P-Net 后，所有可能包含人脸的box个数:{}".format(bounding_boxes.shape[0]))

img1 = show_bboxes(image,bounding_boxes[:,0:4])

keep = nms(bounding_boxes,0.7)
bounding_boxes = bounding_boxes[keep]
print("NMS删除重叠较多的box后，box个数:{}".format(bounding_boxes.shape[0]))

img2 = show_bboxes(image,bounding_boxes[:,0:5])

bounding_boxes = calibrate_box(bounding_boxes,bounding_boxes[:,5:])

img3 = show_bboxes(image,bounding_boxes[:,0:4])

plt.figure()
plt.subplot(221)
plt.imshow(img1)

plt.subplot(222)
plt.imshow(img2)

plt.subplot(223)
plt.imshow(img3)

#####
###　STAGE 2
####
rnet = RNet()
load_weight(rnet,"weights/rnet.npy")
bounding_boxes = run_rnet(image,bounding_boxes,rnet,0.7)

onet = ONet()
load_weight(onet,"weights/onet.npy")
bounding_boxes,landmarks = run_onet(image,bounding_boxes,onet,0.8)

img4 = show_bboxes(image,bounding_boxes,landmarks)

plt.subplot(224)
plt.imshow(img4)
plt.show()


import jsonschema
import requests

from tests.conftest import base_url, register_new_valid_user, register_new_invalid_user
from utils import load_schema

def test_successful_registration():
    schema = load_schema('register_user.json')

    response = requests.post(
        url=f'{base_url}/register',
        json=register_new_valid_user,
    )

    jsonschema.validate(response.json(), schema)
    assert (
        response.status_code == 200
    ), f"Status code {response.status_code} is incorrect"

def test_unsuccessful_registration():
    response = requests.post(url=f'{base_url}/register', json=register_new_invalid_user)

    assert (
        response.status_code == 400
    ), f"Status code {response.status_code} is incorrect"
    assert (
        response.json()['error'] == 'Missing password'
    ), f"Error message {response.json()['error']} is incorrect"