import numpy as np # Linear algebra.
import pandas as pd # Data processing.
from sklearn.model_selection import train_test_split
from matplotlib import pyplot as plt

#%%
data_df = pd.read_csv('/Users/carrotkr/Dropbox/UCI_Credit_Card.csv')
print(data_df.head())
print(data_df.info())
print(data_df.describe())
print(data_df.isnull().sum())
print(data_df.columns)

data = data_df.copy()
data = data.drop(['ID'], axis=1)
print(data.info())

#%% Targer variable.
print(data_df['default.payment.next.month'].unique())
print(data_df['default.payment.next.month'].value_counts())

Y = data['default.payment.next.month'].copy()
del data['default.payment.next.month']
Y = np.array(Y)

num_features = len(data.columns)

#%% Standardization.
data_std = data.astype('int')
print(data_std.info())

from sklearn.preprocessing import StandardScaler

data_std = StandardScaler().fit_transform(data_std)

#%%
X = data_std.copy()
X = np.insert(X, 0, 1, axis=1)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=33)

#%%
# Reference.
#   Implementing a Lasso solver from scratch in Python by Daniel O'Connor.
def clip(beta, alpha):
    clipped = np.minimum(beta, alpha)
    clipped = np.maximum(clipped, -alpha)

    return clipped

def prox_l1_norm(beta_hat, alpha, penalizeAll=True):
    output = beta_hat - clip(beta_hat, alpha)

    if not penalizeAll:
        output[0] = beta_hat[0]

    return output

def lasso_prox_grad(X, Y, Lambda):
    max_iter = 300
    alpha = 0.005

    beta = np.zeros(num_features+1)
    cost_function_vals = np.zeros(max_iter)

    for i in range(max_iter):
        gradient = X.T @ (X @ beta - Y)
        beta = prox_l1_norm(beta - alpha*gradient, alpha*Lambda)

        cost_function_vals[i] = 0.5*np.linalg.norm(X @ beta - Y)**2 + Lambda*np.sum(np.abs(beta))

        print('Iteration: ', i, 'Objective function value: ', cost_function_vals[i])

    return beta, cost_function_vals

#%%
num_non_zero_beta = 3

prm = np.random.permutation(num_features+1)
beta_true = np.zeros(num_features+1)
beta_true[prm[0:num_non_zero_beta]] = 5*np.random.randn(num_non_zero_beta)

#%% Logistic regression.
def sigmoid_function(x):
    return 1 / (1+np.exp(-x))

#%%
theta = np.zeros(X_train.shape[1])
num_epochs = 10

Lambda = 10

for epoch in range(num_epochs):
    hypothesis = sigmoid_function(X_train @ theta)

    beta, cost_function_vals = lasso_prox_grad(X_train, Y_train, Lambda)

plt.figure()
plt.stem(beta, markerfmt='C1o')
plt.stem(beta_true)

#%%
from sklearn.linear_model import Lasso

lambda_values = (0.1, 0.05, 0.01, 0.001, 0.0001)

for index, Lambda in enumerate(lambda_values):
    lasso = Lasso(alpha=Lambda).fit(X_train, Y_train)

    print('\nLambda: ', Lambda)
    print('Training set accuracy: ', lasso.score(X_train, Y_train))
    print('Test set accuracy: ', lasso.score(X_test, Y_test))
    print('Lasso - Number of features: ', np.sum(lasso.coef_ != 0))
    print(lasso.coef_)
    for index_feature in range(num_features):
        if (lasso.coef_[index_feature+1] != 0):
            print('\'Index\'', index_feature, 'Feature: ', data.columns[index_feature])

Logistic Regression with L1-norm Regularization from Scratch.py


import os
import sys
from collections import deque
from pathlib import Path
import typing
from dataclasses import fields

sys.path.append(".")  # add root of project to path

# torch & hf
import torch
from torch import nn
from torch.utils.data import DataLoader
from accelerate import Accelerator
from transformers import get_linear_schedule_with_warmup, HfArgumentParser
from datasets import load_dataset

# logging & etc
from torchinfo import summary
from torchview import draw_graph
import wandb
import numpy as np
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
from tqdm.auto import tqdm
import yaml
from rich.console import Console

# plotting
import matplotlib.pyplot as plt
from util.plotting import plot_first_batch

console = Console()

# local imports
from configs.args import TrainingArgs, ModelArgs, CollatorArgs
from configs.validation import validate_args
from util.remote import wandb_update_config, wandb_init, push_to_hub
from model.simple_mlp import SimpleMLP
from collators import get_collator

MODEL_CLASS = SimpleMLP

def train_epoch(epoch):
    global global_step
    losses = deque(maxlen=training_args.log_every_n_steps)
    step = 0
    console_rule(f"Epoch {epoch}")
    last_loss = None
    for batch in train_dl:
        with accelerator.accumulate(model):
            y = model(batch["image"])
            loss = loss_func(y, batch["target"])
            accelerator.backward(loss)
            accelerator.clip_grad_norm_(
                model.parameters(), training_args.gradient_clip_val
            )
            optimizer.step()
            scheduler.step()
            optimizer.zero_grad()
        losses.append(loss.detach())
        if (
            step > 0
            and step % training_args.log_every_n_steps == 0
            and accelerator.is_main_process
        ):
            last_loss = torch.mean(torch.tensor(losses)).item()
            wandb_log("train", {"loss": last_loss}, print_log=False)
        if (
            training_args.do_save
            and global_step > 0
            and global_step % training_args.save_every_n_steps == 0
        ):
            save_checkpoint()
        if training_args.n_steps is not None and global_step >= training_args.n_steps:
            return
        if (
            training_args.eval_every_n_steps is not None
            and global_step > 0
            and global_step % training_args.eval_every_n_steps == 0
            and accelerator.is_main_process
        ):
            if training_args.do_full_eval:
                evaluate()
            else:
                evaluate_loss_only()
            console_rule(f"Epoch {epoch}")
        step += 1
        global_step += 1
        if accelerator.is_main_process:
            pbar.update(1)
            if last_loss is not None:
                pbar.set_postfix({"loss": f"{last_loss:.3f}"})

def evaluate(device="cpu"):
    eval_model = create_latest_model_for_eval(device)
    if accelerator.is_main_process:
        y_true = []
        y_pred = []
        console_rule("Evaluation")
        for batch in val_dl:
            batch = move_batch_to_device(batch, "cpu")
            y = eval_model(batch["image"])
            y_true.append(batch["target"].cpu().numpy())
            y_pred.append(y.argmax(-1).cpu().numpy())
        y_true = np.concatenate(y_true)
        y_pred = np.concatenate(y_pred)
        acc = accuracy_score(y_true, y_pred)
        f1 = f1_score(y_true, y_pred, average="macro")
        precision = precision_score(y_true, y_pred, average="macro", zero_division=0)
        recall = recall_score(y_true, y_pred, average="macro")
        wandb_log(
            "val", {"acc": acc, "f1": f1, "precision": precision, "recall": recall}
        )
        evaluate_loss_only()

def evaluate_loss_only():
    model.eval()
    losses = []
    console_rule("Evaluation")
    for batch in val_dl:
        y = model(batch["image"])
        loss = loss_func(y, batch["target"])
        losses.append(loss.detach())
    wandb_log("val", {"loss": torch.mean(torch.tensor(losses)).item()})

def main():
    global accelerator, training_args, model_args, collator_args, train_dl, val_dl, optimizer, scheduler, model, global_step, pbar, loss_func

    global_step = 0

    accelerator = Accelerator()

    loss_func = nn.CrossEntropyLoss()

    # parse args
    (
        training_args,
        model_args,
        collator_args,
    ) = parse_args([TrainingArgs, ModelArgs, CollatorArgs])

    # check if run name is specified
    if training_args.run_name is None:
        raise ValueError("run_name must be specified")
    if (
        training_args.do_save
        and (Path(training_args.checkpoint_path) / training_args.run_name).exists()
    ):
        raise ValueError(f"run_name {training_args.run_name} already exists")

    # wandb
    if accelerator.is_main_process:
        wandb_name, wandb_project, wandb_dir, wandb_mode = (
            training_args.run_name,
            training_args.wandb_project,
            training_args.wandb_dir,
            training_args.wandb_mode,
        )
        wandb_init(wandb_name, wandb_project, wandb_dir, wandb_mode)
        wandb.run.log_code()

    # log args
    console_rule("Arguments")
    console_print(training_args)
    console_print(model_args)
    console_print(collator_args)
    if accelerator.is_main_process:
        wandb_update_config(
            {
                "training": training_args,
                "model": model_args,
            }
        )
    validate_args(training_args, model_args, collator_args)

    # Distribution Information
    console_rule("Distribution Information")
    console_print(f"[green]accelerator[/green]: {accelerator}")
    console_print(f"[green]n_procs[/green]: {accelerator.num_processes}")
    console_print(f"[green]process_index[/green]: {accelerator.process_index}")

    # model
    model = MODEL_CLASS(model_args)
    console_rule("Model")
    print_and_draw_model()

    # dataset
    console_rule("Dataset")

    console_print(f"[green]dataset[/green]: {training_args.dataset}")
    console_print(f"[green]train_split[/green]: {training_args.train_split}")
    console_print(f"[green]val_split[/green]: {training_args.val_split}")

    train_ds = load_dataset(training_args.dataset, split=training_args.train_split)
    val_ds = load_dataset(training_args.dataset, split=training_args.val_split)

    console_print(f"[green]train[/green]: {len(train_ds)}")
    console_print(f"[green]val[/green]: {len(val_ds)}")

    # collator
    collator = get_collator(collator_args)

    # plot first batch
    if accelerator.is_main_process:
        first_batch = collator([train_ds[i] for i in range(training_args.batch_size)])
        plot_first_batch(first_batch, training_args)
        plt.savefig("figures/first_batch.png")

    # dataloader
    train_dl = DataLoader(
        train_ds,
        batch_size=training_args.batch_size,
        shuffle=True,
        collate_fn=collator,
        drop_last=True,
    )

    val_dl = DataLoader(
        val_ds,
        batch_size=training_args.batch_size,
        shuffle=False,
        collate_fn=collator,
    )

    # optimizer
    optimizer = torch.optim.AdamW(model.parameters(), lr=training_args.lr)

    # scheduler
    if training_args.lr_schedule == "linear_with_warmup":
        scheduler = get_linear_schedule_with_warmup(
            optimizer,
            num_warmup_steps=training_args.lr_warmup_steps,
            num_training_steps=training_args.n_steps,
        )
    else:
        raise NotImplementedError(f"{training_args.lr_schedule} not implemented")

    # accelerator
    model, optimizer, train_dl, val_dl, scheduler = accelerator.prepare(
        model, optimizer, train_dl, val_dl, scheduler
    )

    # evaluation
    if training_args.eval_only:
        console_rule("Evaluation")
        seed_everything(training_args.seed)
        evaluate()
        return

    # training
    console_rule("Training")
    seed_everything(training_args.seed)
    pbar_total = training_args.n_steps
    training_args.n_epochs = training_args.n_steps // len(train_dl) + 1
    console_print(f"[green]n_epochs[/green]: {training_args.n_epochs}")
    console_print(
        f"[green]effective_batch_size[/green]: {training_args.batch_size*accelerator.num_processes}"
    )
    pbar = tqdm(total=pbar_total, desc="step")
    for i in range(training_args.n_epochs):
        train_epoch(i)
    console_rule("Evaluation")
    seed_everything(training_args.seed)
    evaluate()

    # save final model
    console_rule("Saving")
    if training_args.do_save:
        save_checkpoint()

    # wandb sync reminder
    if accelerator.is_main_process and training_args.wandb_mode == "offline":
        console_rule("Weights & Biases")
        console_print(
            f"use \n[magenta]wandb sync {Path(wandb.run.dir).parent}[/magenta]\nto sync offline run"
        )

# helper functions (change them in parent repository if needed)

def parse_args(argument_classes):
    parser = HfArgumentParser(argument_classes)
    type_dicts = []
    for arg_class in argument_classes:
        resolved_hints = typing.get_type_hints(arg_class)
        field_names = [field.name for field in fields(arg_class)]
        resolved_field_types = {name: resolved_hints[name] for name in field_names}
        type_dicts.append(resolved_field_types)
    if len(sys.argv) > 1 and sys.argv[1].endswith(".yml"):
        with open(sys.argv[1], "r") as f:
            args_dict = yaml.load(f, Loader=yaml.Loader)
        # additonally parse args from command line
        parsed_args = parser.parse_args_into_dataclasses(sys.argv[2:])
        args_in_argv = sys.argv[2:]
        updated_args = []
        for arg in args_in_argv:
            if "--" in arg:
                arg = arg.replace("--", "")
                updated_args.append(arg)
            if "=" in arg:
                arg1, arg2 = arg.split("=")
                updated_args.append(arg1)
                updated_args.append(arg2)
        # remove every second element (values)
        updated_args = updated_args[::2]
        args_in_argv = updated_args
        # update args from yaml (only if not specified in command line)
        for k, v in args_dict.items():
            for j, arg_class in enumerate(parsed_args):
                if hasattr(arg_class, k) and k not in args_in_argv:
                    setattr(arg_class, k, type_dicts[j][k](v))
    else:
        parsed_args = parser.parse_args_into_dataclasses()
    return parsed_args

def print_and_draw_model():
    bsz = training_args.batch_size
    dummy_input = model.dummy_input
    # repeat dummy input to match batch size (regardless of how many dimensions)
    if isinstance(dummy_input, torch.Tensor):
        dummy_input = dummy_input.repeat((bsz,) + (1,) * (len(dummy_input.shape) - 1))
        console_print(f"[green]input shape[/green]: {dummy_input.shape}")
    elif isinstance(dummy_input, list):
        dummy_input = [
            x.repeat((bsz,) + (1,) * (len(x.shape) - 1)) for x in dummy_input
        ]
        console_print(f"[green]input shapes[/green]: {[x.shape for x in dummy_input]}")
    model_summary = summary(
        model,
        input_data=dummy_input,
        verbose=0,
        col_names=[
            "input_size",
            "output_size",
            "num_params",
        ],
    )
    console_print(model_summary)
    Path("figures").mkdir(exist_ok=True)
    if accelerator.is_main_process:
        _ = draw_graph(
            model,
            input_data=dummy_input,
            save_graph=True,
            directory="figures/",
            filename="model",
            expand_nested=True,
        )
        # remove "figures/model" file
        os.remove("figures/model")

def console_print(*args, **kwargs):
    if accelerator.is_main_process:
        console.print(*args, **kwargs)

def console_rule(*args, **kwargs):
    if accelerator.is_main_process:
        console.rule(*args, **kwargs)

def wandb_log(prefix, log_dict, round_n=3, print_log=True):
    if accelerator.is_main_process:
        log_dict = {f"{prefix}/{k}": v for k, v in log_dict.items()}
        wandb.log(log_dict, step=global_step)
        if print_log:
            log_dict = {k: round(v, round_n) for k, v in log_dict.items()}
            console.log(log_dict)

def seed_everything(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)

def save_checkpoint(name_override=None):
    accelerator.wait_for_everyone()
    checkpoint_name = training_args.run_name
    if name_override is not None:
        name = name_override
    else:
        name = f"step_{global_step}"
    checkpoint_path = Path(training_args.checkpoint_path) / checkpoint_name / name
    if name_override is None:
        # remove old checkpoints
        if checkpoint_path.exists():
            for f in checkpoint_path.iterdir():
                os.remove(f)
    # model
    model.save_model(checkpoint_path, accelerator)
    if accelerator.is_main_process:
        # training args
        with open(checkpoint_path / "training_args.yml", "w") as f:
            f.write(yaml.dump(training_args.__dict__, Dumper=yaml.Dumper))
        # collator args
        with open(checkpoint_path / "collator_args.yml", "w") as f:
            f.write(yaml.dump(collator_args.__dict__, Dumper=yaml.Dumper))
    accelerator.wait_for_everyone()
    return checkpoint_path

def create_latest_model_for_eval(device="cpu"):
    checkpoint_path = save_checkpoint("latest")
    if accelerator.is_main_process:
        eval_model = MODEL_CLASS.from_pretrained(checkpoint_path)
        eval_model.eval()
        eval_model = eval_model.to(device)
    return eval_model

def move_batch_to_device(batch, device):
    if isinstance(batch, dict):
        for k, v in batch.items():
            if isinstance(v, torch.Tensor):
                batch[k] = move_batch_to_device(batch[k], device)
            elif isinstance(v, list):
                # recursively move list of tensors to device
                batch[k] = move_batch_to_device(batch[k], device)
    elif isinstance(batch, list):
        batch = [move_batch_to_device(x, device) for x in batch]
    elif isinstance(batch, torch.Tensor):
        batch = batch.to(device)
    return batch

# main

if __name__ == "__main__":
    main()


import wx
import datetime
import six

if 'phoenix' in wx.PlatformInfo:
    from wx import Control
else :
    from wx import PyControl as Control

class Footer(Control):
    def __init__(self,
            parent,
            id=-1,
            pos=wx.DefaultPosition,
            size=wx.DefaultSize,
            style=wx.NO_BORDER,
            name="footer",
        ):
        self.hauteur = 24
        self.afficherColonneDroite = True

        self.listview = None
        self.dictColonnes = {}
        self.dictTotaux = {}
        self.listeImpression = []
        Control.__init__(self, parent, id=id, pos=pos, size=size, style=style, name=name)
        self.SetInitialSize(size)

        self.Bind(wx.EVT_PAINT, self.OnPaint)
        self.Bind(wx.EVT_ERASE_BACKGROUND, self.OnErase)
        self.Bind(wx.EVT_SIZE, self.MAJ_affichage)

    def MAJ_affichage(self, event=None):
        self.Refresh()

    def MAJ_totaux(self):
        self.dictTotaux = {}
        for track in self.listview.innerList :
            for nomColonne, dictColonne in self.dictColonnes.items() :
                if dictColonne["mode"] == "total" :
                    if hasattr(track, nomColonne) :
                        total = getattr(track, nomColonne)
                        if (nomColonne in self.dictTotaux) == False :
                            # Format classique (numérique)
                            self.dictTotaux[nomColonne] = 0
                            # Autre format
                            if "format" in dictColonne :
                                if dictColonne["format"] in ("temps", "duree") :
                                    self.dictTotaux[nomColonne] = datetime.timedelta(0)
                        if total != None :
                            self.dictTotaux[nomColonne] += total

    def MAJ(self):
        self.MAJ_totaux()
        self.MAJ_affichage()

    def DrawColonne(self, dc, x, largeur, label="", alignement=None, couleur=None, font=None):
        """ Dessine une colonne """
        render = wx.RendererNative.Get()
        options = wx.HeaderButtonParams()
        options.m_labelText = label
        if alignement : options.m_labelAlignment = alignement
        if couleur : options.m_labelColour = couleur
        if font : options.m_labelFont = font
        render.DrawHeaderButton(self, dc, (x, 1, largeur, self.hauteur), params=options)

    def Paint(self, dc):
        """Draws the ticker text at the current offset using the provided DC"""
        defaultFont = self.GetFont()
        dc.SetFont(defaultFont)

        x = 0 - self.listview.GetScrollPos(wx.HORIZONTAL)
        self.listeImpression = []
        dernierTexte = ""
        for (indexColonne, col) in enumerate(self.listview.columns):
            texte = ""
            font = wx.Font(8, wx.SWISS, wx.NORMAL, wx.NORMAL)
            couleur = wx.Colour(140, 140, 140)
            largeur = col.width
            #largeur = self.listview.GetColumnWidth(indexColonne)
            converter = col.stringConverter
            nom = col.valueGetter
            if col.align == "left" : alignement = wx.ALIGN_LEFT
            if col.align == "centre" : alignement = wx.ALIGN_CENTER
            if col.align == "right" : alignement = wx.ALIGN_RIGHT

            # Recherche infos personnalisées à afficher dans la colonne
            mode = None
            if nom in self.dictColonnes :
                infoColonne = self.dictColonnes[nom]
                mode = infoColonne["mode"]

                # Valeur : TOTAL
                if mode == "total" :
                    if nom in self.dictTotaux :
                        texte = self.dictTotaux[nom]
                    else :
                        # Total format classique (numérique)
                        texte = 0
                        # Autres formats de total
                        if "format" in infoColonne :
                            if infoColonne["format"] in ("temps", "duree") :
                                texte = datetime.timedelta(0)
                    if converter != None :
                        texte = converter(texte)
                    if six.PY2:
                        liste_types = (int, float, long)
                    else :
                        liste_types = (int, float)
                    if type(texte) in liste_types :
                        texte = str(texte)

                # Valeur : NOMBRE
                if mode == "nombre" :
                    nombre = len(self.listview.innerList)
                    if nombre > 1 :
                        texte = u"%d %s" % (nombre, infoColonne["pluriel"])
                    else :
                        texte = u"%d %s" % (nombre, infoColonne["singulier"])

                # Valeur : TEXTE
                if mode == "texte" :
                    texte = infoColonne["texte"]

                # Paramètres personnalisés
                if "alignement" in infoColonne : alignement = infoColonne["alignement"]
                if "font" in infoColonne : font = infoColonne["font"]
                if "couleur" in infoColonne : couleur = infoColonne["couleur"]

            # Pour éviter les bords si les cases sont vides
            ajustement = 0
            if mode != "total" and dernierTexte == "" :
                ajustement = 5

            self.DrawColonne(dc, x-ajustement, largeur+ajustement, texte, alignement, couleur, font)
            x += largeur

            # Mémorisation pour impression
            self.listeImpression.append({"texte" : texte, "alignement" : alignement})

            if mode == "total" :
                dernierTexte = texte
            else :
                dernierTexte = ""

        # Dernière colonne de remplissage
        if self.afficherColonneDroite :
            self.DrawColonne(dc, x, self.GetSize()[0]-x)

    def GetDonneesImpression(self, typeInfo="texte"):
        """ Renvoie infos pour impression """
        """ typeInfo = "texte" ou "alignement" """
        listeDonnees = []
        for info in self.listeImpression :
            listeDonnees.append(info[typeInfo])
        return listeDonnees[1:]

    def OnPaint(self, evt):
        dc = wx.BufferedPaintDC(self)
        dc.Clear()
        self.Paint(dc)

    def OnErase(self, evt):
        """Noop because of double buffering"""
        pass

    def AcceptsFocus(self):
        """Non-interactive, so don't accept focus"""
        return False

    def DoGetBestSize(self):
        """Width we don't care about, height is either -1, or the character
        height of our text with a little extra padding
        """
        return (100, self.hauteur-1)

    def ShouldInheritColours(self):
        """Don't get colours from our parent..."""
        return False

if __name__ == '__main__':
    """ DEMO FRAME"""
    app = wx.App()
    f = wx.Frame(None)
    p = wx.Panel(f)
    t = Footer(p)
    s = wx.BoxSizer(wx.VERTICAL)
    s.Add(t, flag=wx.GROW, proportion=1)
    p.SetSizer(s)
    f.Show()
    app.MainLoop()


import os
import pickle
from pathlib import Path

import numpy as np
import pandas as pd
import utils

only_process_test_sites = True

data_folder = utils.get_data_folder()
summary_path = data_folder / 'file_summary.csv'
train_waypoints_path = data_folder / 'train_waypoints.csv'
submission_folder = data_folder / 'submissions'
submission_path = submission_folder / 'probe_max_submission.csv'
df = pd.read_csv(summary_path)

# Loop over all file paths and keep track of the max train floor, X and Y for
# all waypoints
waypoint_counts = {}
waypoint_floor = []
waypoint_x = []
waypoint_y = []
all_waypoints = []
for i in range([100, df.shape[0]][1]):
  print(i)
  if (df['mode'][i] == 'train') and (not only_process_test_sites or
                                     df.test_site[i]):
    pickle_path = data_folder / (
        str(Path(df.ext_path[i]).with_suffix('')) + '_reshaped.pickle')
    with open(pickle_path, 'rb') as f:
      trajectory = pickle.load(f)

    site = df.site_id[i]
    floor = df.level[i]
    for w_id in range(trajectory['waypoint'].shape[0]):
      x = trajectory['waypoint']['x_waypoint'][w_id]
      y = trajectory['waypoint']['y_waypoint'][w_id]
      k = (site, floor, x, y)
      if k in waypoint_counts:
        waypoint_counts[k] += 1
      else:
        waypoint_counts[k] = 1

      waypoint_floor.append(floor)
      waypoint_x.append(x)
      waypoint_y.append(y)

      fn = Path(df['ext_path'][i]).stem
      all_waypoints.append(tuple([fn] + list(k)))

max_floor = np.array(waypoint_floor).max()
max_x = np.array(waypoint_x).max()
max_y = np.array(waypoint_y).max()
print(f"Max floor: {max_floor}")
print(f"Max x: {max_x}")
print(f"Max y: {max_y}")
train_waypoints = pd.DataFrame(np.array(all_waypoints))
train_waypoints.columns = ["fn", "site_id", "level", "x", "y"]
train_waypoints.to_csv(train_waypoints_path, index=False)

sample_submission = pd.read_csv(data_folder / 'sample_submission.csv')

# Generate the probing default dummy submission
sample_submission['floor'] = max_floor + 1
sample_submission['x'] = max_x + 1
sample_submission['y'] = max_y + 1
Path(submission_folder).mkdir(parents=True, exist_ok=True)
sample_submission.to_csv(submission_path, index=False)


#!usr/bin/env python3

"""
This program uses API to get random quotes from the api endpoint "api.kanye.rest".
"""

from tkinter import *
import requests

def get_quote():
    response = requests.get(url="https://api.kanye.rest")
    response.raise_for_status()
    # Use json() method to get the data and filter it using the key "quote"
    quote = response.json()["quote"]
    # Item we want to configure here is quote_text and the value that we want to change is text
    canvas.itemconfig(quote_text, text=quote)

window = Tk()
window.title("Kanye Says...")
window.config(padx=50, pady=50)

canvas = Canvas(width=300, height=414)
background_img = PhotoImage(file="background.png")
canvas.create_image(150, 207, image=background_img)
quote_text = canvas.create_text(150, 207, text="Kanye Quote Goes HERE", width=250, font=("Arial", 30, "bold"), fill="white")
canvas.grid(row=0, column=0)

kanye_img = PhotoImage(file="kanye.png")
kanye_button = Button(image=kanye_img, highlightthickness=0, command=get_quote)
kanye_button.grid(row=1, column=0)

window.mainloop()


from django.db.models.signals import post_save

from vk_api_wrapper.models import VkDialog
from .models import ChatFolder

def create_chat_folder_on_vk_dialog_created(sender, instance, created, *args, **kwargs):
    if created:
        ChatFolder.objects.create(
            title=instance.title,
            author_id=instance.user_id,
            type='chat',
            typeForElasticSearchPleaseDontTouchMe='chat',
            vk_dialog=instance,
            icon=instance.photo
        )

post_save.connect(create_chat_folder_on_vk_dialog_created, VkDialog)


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from routes.police import police_router
from dotenv import load_dotenv

load_dotenv('.env')

app = FastAPI(
        docs_url='/api/docs',
        redoc_url='/api/redoc',
        openapi_url='/api/openapi.json'
        )

app.add_middleware(
    CORSMiddleware,
    allow_origins=['*'],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

app.include_router(police_router, prefix='/api')


#!/usr/bin/env python2.7
import copy
import ctypes
from collections import OrderedDict
from pprint import pprint
import veriloglang
import re
import string
class ctypes2sv(object):
  """
  ToDo: handle nonprintable char
  """
  pythontype2bitlen=dict(
c_byte          = ctypes.sizeof(ctypes.c_byte)*8,               # 8
c_char          = ctypes.sizeof(ctypes.c_char)*8,               # 8
c_double        = ctypes.sizeof(ctypes.c_double)*8,             # 64
c_float         = ctypes.sizeof(ctypes.c_float)*8,              # 32
c_int           = ctypes.sizeof(ctypes.c_int)*8,                # 32
c_long          = ctypes.sizeof(ctypes.c_long)*8,               # 64
c_longdouble    = ctypes.sizeof(ctypes.c_longdouble)*8,         # 128
c_short         = ctypes.sizeof(ctypes.c_short)*8,              # 16
c_ubyte         = ctypes.sizeof(ctypes.c_ubyte)*8,              # 8
c_uint          = ctypes.sizeof(ctypes.c_uint)*8,               # 32
c_ulong         = ctypes.sizeof(ctypes.c_ulong)*8,              # 64
c_ulonglong     = ctypes.sizeof(ctypes.c_ulonglong)*8,          # 128
c_ushort        = ctypes.sizeof(ctypes.c_ushort)*8,             # 16
)
  pythontype2sv=dict(
c_byte          = 'byte',
c_char          = 'byte',
c_double        = 'real',
c_float         = 'shortreal',
c_int           = 'int',
c_long          = 'longint',
c_longdouble    = 'c_longdouble',                  # user defined type
c_short         = 'shortint',
c_ubyte         = 'byte unsigned',
c_uint          = 'int unsigned',
c_ulong         = 'longint unsigned',
c_ulonglong     = 'c_ulonglong',                   # user defined type
c_ushort        = 'shortint unsigned',
)

  def __init__(self):
    """
    """
    self.classdb=OrderedDict()
    self.un_classdb=OrderedDict()
    self.imports={}
    self.endians=[]
    self.structlines=[]
    self.unionlines=[]
    self.classlines=[]
    self.const2value=OrderedDict()
    self.enumdb=OrderedDict()
    self.struct2bitlen={}
  def read_ctypes(self,fname):
    """
    """
    dict_p=re.compile(r'\s*=\s*{')
    with open(fname,'r') as fh:
      lines=fh.readlines()
      line=''
      while lines:
        line=lines.pop(0)
        line=line.rstrip()
        if line=='' or line.lstrip().startswith('#'):continue
        if line.startswith('from '):
          module,item=map(str.strip,line[5:].split(' import '))
          self.imports.setdefault(module,[]).append(item)

        elif line.startswith('ctypes_format=('):
          while True:
            line=lines.pop(0)
            line=line.strip()
            if line=='' or line.lstrip().startswith('#'):continue
            if line==')': break
        elif line.startswith('class un_'):
          lines=self.extract_union(line,lines)
        elif line.startswith('class '):
          lines=self.extract_struct(line,lines)
        elif line.startswith('assert '):
          pass
        elif dict_p.search(line):
          while True:
            line=lines.pop(0)
            line=line.strip()
            if line=='' or line.lstrip().startswith('#'):continue
            if line=='}': break
        elif line.startswith('def '):
          while True:
            if len(lines)==0: break
            line=lines.pop(0)
            if line[0] != ' ':
              lines=[line]+lines
              break
        elif 'dict(' in line:
          lines=self.extract_enum(line,lines)
        elif line[0]!=' ' and '=' in line:
          k,v=map(str.strip,line.split('#')[0].split('='))
          if v[-1]=="'":v=v[:-1]+'"'
          if v[0]=="'":v='"'+v[1:]
          if v=='long(-1)':
            v="longint'(-1)"
          self.const2value[k]=v

  def extract_struct(self,line,lines):
    """
    """
    idx=line.index('(')
    classname=line[6:idx]
    if 'BigEndian' in line:
      endian='big'
    else:
      endian='little'
    structdb=(endian,)
    while True:
      line=lines.pop(0)
      line=line.rstrip()
      if line=='' or line.lstrip().startswith('#'):continue
      if line.startswith('  _pack'):
        continue
      if '(' in line:
        fields=line[line.index('(')+1:line.index(')')]
        fs=map(lambda x: x.strip(),fields.split(','))
        if len(fs)==2:
          if '*' in fs[1]:
            ftype,fsize=map(lambda x:x.strip(),fs[1][7:].split('*'))
            if fsize in self.const2value:
              fsize=self.const2value[fsize]
            structdb+=((self.replace_vlog_keywords(fs[0].strip("'")),'bit [%d:0]'%(eval(fsize)*self.pythontype2bitlen[ftype]-1)),)
          else:
            structdb+=((self.replace_vlog_keywords(fs[0].strip("'")),fs[1][7:] if fs[1].startswith('ctypes.') else fs[1]),)
        elif len(fs)==3:
          bitlen=self.pythontype2bitlen[fs[1][7:]]
          bitcount=int(fs[2])
          bitfields=[(self.replace_vlog_keywords(fs[0].strip("'")),fs[1][7:],int(fs[2]))]
          while bitlen > bitcount:
            line=lines.pop(0)
            line=line.rstrip()
            if line=='' or line.lstrip().startswith('#'):continue
            if '(' in line:
              fields=line[line.index('(')+1:line.index(')')]
              fs=map(lambda x: x.strip(),fields.split(','))
              if len(fs)==2:
                structdb+=((self.replace_vlog_keywords(fs[0].strip("'")),fs[1][7:] if fs[1].startswith('ctypes.') else fs[1]),)
                break
              bitlen=self.pythontype2bitlen[fs[1][7:]]
              bitcount+=int(fs[2])
              bitfields.append((self.replace_vlog_keywords(fs[0].strip("'")),fs[1][7:],int(fs[2])))
            if line.endswith(']'): break
          structdb+=(bitfields,)
      if line.endswith(']'): break
    self.classdb[classname]=structdb
    return lines

  def extract_enum(self,line,lines):
    """
    """
    name=line.split('#')[0].split('=')[0].strip()
    self.enumdb[name]=()
    while True:
      line=lines.pop(0)
      line=line.rstrip()
      if line=='' or line.lstrip().startswith('#'):continue
      if line.startswith(')'): break
      k,v=map(str.strip,line.split('#')[0].split('='))
      if v[-1]==',':
        v=v[:-1].strip()
      if v[-1]=="'":v=v[:-1]+'"'
      if v[0]=="'":v='"'+v[1:]
      self.enumdb[name]+=((k,v),)
    return lines

  def extract_union(self,line,lines):
    """
    """
    idx=line.index('(')
    classname=line[9:idx]
    structdb=()
    while True:
      line=lines.pop(0)
      line=line.rstrip()
      if line=='' or line.lstrip().startswith('#'):continue
      if '(' in line:
        fields=line[line.index('(')+1:line.index(')')]
        fs=map(lambda x: x.strip(),fields.split(','))
        if len(fs)==2:
          if '*' in fs[1]:
            ftype,fsize=map(lambda x:x.strip(),fs[1][7:].split('*'))
            if fsize in self.const2value:
              fsize=self.const2value[fsize]
            structdb+=((self.replace_vlog_keywords(fs[0].strip("'")),eval(fsize)),)
          else:
            structdb+=((self.replace_vlog_keywords(fs[0].strip("'")),fs[1][7:] if fs[1].startswith('ctypes.') else fs[1]),)
        elif len(fs)==3:
          raise Extract_Error
      if ']' in line:
        break

      if len(lines)==0: break
    self.un_classdb[classname]=structdb
    return lines

  def structgen(self):
    """
    """
    for sname,structdb in self.classdb.items():
      endian=structdb[0]
      self.endians.append(sname)
      if endian=='little':
        self.classlines.append(self.endian_swap_class(sname,structdb[1:]))
      self.structlines.append(self.create_struct(sname,structdb[1:]))
      self.structlines.append(self.create_type_array(sname,structdb[1:]))
  def uniongen(self):
    """
    """
    for sname,structdb in self.un_classdb.items():
      self.unionlines.append(self.create_union(sname,structdb))
  def endian_swap_class(self,name,db):
    """
    """

  def create_union(self,name,db):
    """
    """
    return """typedef union {
%s
} un_%s;"""%('\n'.join(map(lambda x: '  %-20s %s;'%(('byte unsigned','bytes[%d]'%x[1]) if x[0]=='bytes' else (x[1],x[0])),db)),name)
  def create_struct(self,name,db):
    """
    """
    def bitfields(bits):
      return '\n'.join(map(lambda x: '  %-20s %s;'%('bit [%d:0]'%(x[2]-1) if x[2] > 1 else 'bit',x[0]),bits))
    return """typedef struct packed {
%s
} %s;"""%('\n'.join(map(lambda x: bitfields(x) if isinstance(x,list) else '  %-20s %s;'%(self.pythontype2sv[x[1]] if x[1] in self.pythontype2sv else x[1],x[0]),db)),name)

  def create_type_array(self,name,db):
    """
    """
    def nestlen(db):
      return sum(map(lambda x: int(len(x)) if isinstance(x,list) else 1,db))
    def bitfields(bits):
      return ','.join(map(lambda x: '%d'%x[2],bits))
    def bitfield(bit):
      if bit.startswith('bit ['):
        return str(int(bit[5:bit.index(':')])+1)
      else:
        return str(self.struct2bitlen[bit])

    self.struct2bitlen[name]=sum(map(lambda x: int(sum(map(int,bitfields(x).split(','))) if isinstance(x,list) else '%d'%self.pythontype2bitlen[x[1]] if x[1] in self.pythontype2sv else bitfield(x[1])),db))
    return """byte unsigned %s_type[%d]='{%s};"""%(name,nestlen(db), ','.join(map(lambda x: bitfields(x) if isinstance(x,list) else '%d'%self.pythontype2bitlen[x[1]] if x[1] in self.pythontype2sv else bitfield(x[1]),db)))

  def enumgen(self):
    lines=[]
    for name in self.enumdb:
      lines.append('''int %s[string] = '{
%s
  };'''%(name,',\n'.join(map(lambda x: '    {0:<45s}: {1}'.format('"%s"'%x[0],self.hexnum(x[1])),self.enumdb[name]))))
    return lines
  def enumgen_2(self):
    lines=[]
    for name in self.enumdb:
      lines.append('''typedef enum {
%s
  } %s;'''%(',\n'.join(map(lambda x: '    {0:<45s}= {1}'.format(x[0],self.hexnum(x[1])),self.enumdb[name])),name))
    return lines
  def enumgen_1(self):
    lines=[]
    for name in self.enumdb:
      lines.append('''virtual class %s_wrap;
  typedef enum {
%s
  } %s;
endclass'''%(name,',\n'.join(map(lambda x: '    {0:<45s}= {1}'.format(x[0],self.castint(self.hexnum(x[1]))),self.enumdb[name])),name))
    return lines
  def write_hdr(self,fname):
    """
    """
    lines=[]
    lines.append('package %s;'%fname[:-2])
    # import
    if self.imports:
      lines.extend(reduce(lambda x,y: x+y,map(lambda x: map(lambda y: 'import %s::%s;'%(x,y), self.imports[x]),self.imports)))
    # constant
    lines.extend(map(lambda x: 'localparam %-45s = %s;'%(x,self.hexnum(self.const2value[x])),self.const2value))
    # enum
    lines.extend(self.enumgen())

    # struct
    lines.extend(self.structlines)
    # union
    lines.extend(self.unionlines)
    lines.append('endpackage')
    with open(fname,'w') as fh:
      fh.write('\n'.join(lines))

  def replace_vlog_keywords(self,name):
    """
    """
    for group in veriloglang.veriloglang.keywords:
      if name in group:
         name+='qq'
    return name

  def hexnum(self,num):
    """
    """
    res_num=''
    while True:
      if '0x' not in num: break
      idx=num.index('0x')
      res_num+=num[:idx]
      num=num[idx+2:]
      idx=len(num)
      for i,c in enumerate(num):
        if c not in string.hexdigits:
          idx=i
          break
      res_num+="%d'h"%(idx*4)
      res_num+=num[:idx]
      num=num[idx:]
    return res_num+num
  def castint(self,num):
    """
    """
    if not num.startswith('"'):
      num="int'(%s)"%num
    return num

class Extract_Error:
  pass

if __name__=='__main__':
  import sys
  argc=len(sys.argv)
  if argc > 1: ctypes_hdr=sys.argv[1]
  if argc > 2: sv_hdr=sys.argv[2]
  obj=ctypes2sv()
  obj.read_ctypes(ctypes_hdr)
  #pprint(obj.classdb)
  #pprint(obj.un_classdb)
  obj.structgen()
  obj.uniongen()
  obj.write_hdr(sv_hdr)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 14 13:53:48 2019

@author: vladgriguta
"""

import numpy as np
import matplotlib.pyplot as plt
import itertools

def plot_confusion_matrix(cm, classes, normalize=False,directory='RFres/', title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Showing normalized confusion matrix")
    else:
        print('Showing confusion matrix, without normalization')
    #print(cm)
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=90)
    plt.yticks(tick_marks, classes)
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.savefig(directory+title+'.png')
    plt.gcf().clear()

def plot_feature_importance(name_of_features,pipeline,title,directory='RFres/'):
    #make plot of feature importances
    clf=pipeline.steps[0][1] #get classifier used. zero because only 1 step.
    importances = pipeline.steps[0][1].feature_importances_
    std = np.std([tree.feature_importances_ for tree in clf.estimators_],
             axis=0)
    indices = np.argsort(importances)[::-1]
    feature_names_importanceorder=[]
    for f in range(len(indices)):
        #print("%d. feature %d (%f) {0}" % (f + 1, indices[f], importances[indices[f]]), feature_names[indices[f]])
        feature_names_importanceorder.append(str(name_of_features[indices[f]]))
    plt.figure()
    plt.title(title)
    plt.bar(range(len(indices)), importances[indices],
           color="r", yerr=std[indices], align="center")
    plt.xticks(range(len(indices)), indices)
    plt.xlim([-1, len(indices)])
    plt.xticks(range(len(indices)), feature_names_importanceorder, rotation='vertical')
    plt.tight_layout()
    plt.savefig(directory+title+'.png',format='png')
    plt.gcf().clear()


#!/usr/bin/env python

import numpy as np
import rospy
import rospkg

import cv2
from cv_bridge import CvBridge, CvBridgeError
from sensor_msgs.msg import Image
from asc.msg import *

if __name__ == "__main__":
    rospy.init_node('DummyPerception')
    rate = rospy.Rate(100)
    pcb = cv2.imread('pcb.png')
    pcb_processed = cv2.imread('pcb_blob.png')

    image_pub = rospy.Publisher("camera",Image,queue_size=1)
    image_pub2 = rospy.Publisher("pcb_processed",Image,queue_size=1)
    asc_pub = rospy.Publisher("/PerceptionNewNodeListener",NodeInfo,queue_size=1)
    bridge = CvBridge()
    imageMsg=bridge.cv2_to_imgmsg(pcb, "bgr8")
    imageMsg2=bridge.cv2_to_imgmsg(pcb_processed, "bgr8")

    cameraNodeInfo=NodeInfo()
    cameraNodeInfo.nodeId='DummyCamera'
    cameraNodeInfo.purposeId='Camera'
    cameraNodeInfo.nodeTopic='camera'
    cameraNodeInfo.description='This is simple dummy camera simulation that reads an image and send it as ros message'

    blobNodeInfo=NodeInfo()
    blobNodeInfo.nodeId='Dummy_Pcb_blob'
    blobNodeInfo.purposeId='Pcb_Blob'
    blobNodeInfo.nodeTopic='pcb_processed'
    blobNodeInfo.description='This is simple dummy model that supposedly find pcb blob, publish it. Currently it reads an image and send it as ros message'

    for i in range(20): ## Currently, this topic is not rosservice, so this kind of hack is necessary.
        asc_pub.publish(blobNodeInfo)
        rate.sleep()
        asc_pub.publish(cameraNodeInfo)
        rate.sleep()

    rate.sleep()
    while not rospy.is_shutdown():
        image_pub.publish(imageMsg)
        image_pub2.publish(imageMsg2)

        rate.sleep()


from utils import MLUtilities as util
import scipy as sp
from sklearn.model_selection import train_test_split
from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier
from sklearn.svm import LinearSVC
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt
import seaborn as sns

# read data
params = util.get_config_file()
df = util.read_csv_file(params['mandatory']['train_csv_file_path'])

# lets see the some sample data
print(df.head())

# analyze data
print("Missing values per column:")
is_null = df.apply(util.num_missing, axis=0)
print(sp.transpose(is_null))

# preparing features and labels
y = sp.array(df[[-1]])
X = sp.array(df.drop(['Class', 'Time'], 1))

# appearently there is no null value in the data frame, and all the values looks good
# divide training and test set and check no. of positives and negatives example percentage in each case
# crossvalidation and training data should contain balanced number of positive and negative examples

# divide training, cross validation data
# create training and test sets
X_train, X_cross_val, y_train, y_cross_val = train_test_split(X, y, test_size=0.3, random_state=0)

# need to find the number of 0's and 1's in
temp = 0
for y1 in y_train:
    if y1 == 1:
        temp += 1

print("number of ones in y_train : ")
print(temp)

# need to find the number of 0's and 1's in
temp = 0
for y1 in y_cross_val:
    if y1 == 1:
        temp += 1

print("number of ones in y_cross_val : ")
print(temp)

# perform classification and predict the test set
clf = OneVsRestClassifier(LinearSVC(random_state=0)).fit(X_train, y_train)

# predict the values on cross validation data
y_predict = clf.predict(X_cross_val)

# find error of prediction on cross validation data
print("Accuracy :")
accuracy_score(y_cross_val, y_predict)

# lets check the false positive and false negative predictions
# absolute values will be much more helpful
false_positive = 0
false_negative = 0
for i in range(1, len(y_cross_val)):
    if y_cross_val[i] - y_predict[i] == -1:
        false_negative += 1
    if y_cross_val[i] - y_predict[i] == 1:
        false_positive += 1
print("false positive : ")
print(false_positive)

print("false negative : ")
print(false_negative)


"""
User interface program that interacts with program interface by importing the packages.
"""
from utils import database

database.create_table()

def add_books_prompt():
    title = input("Enter Title: ")
    author = input("Enter book author: ")
    database.insert_data(title, author)

def list_books_prompt():
    rows = database.list_data()
    try:
        for row in rows:
            read = 'Yes' if row[2] else 'No'
            print(f"{row[0]} by {row[1]} - Read: {read}")
    except Exception as e:
        print("Error", e)

def mark_book_as_read_prompt():
    title = input("Enter title of the book you read: ")
    database.mark_book_read(title)

def delete_books_prompt():
    title = input("Enter the title of the book you wish to delete: ")
    database.delete_book(title)

USER_CHOICES = """
        Enter:
        'a' to add a new book
        'l' to list all books
        'r' to mark the book as read
        'd' to delete a book
        'q' to quit
        """

func = {'a': add_books_prompt,
        'l': list_books_prompt,
        'r': mark_book_as_read_prompt,
        'd': delete_books_prompt}

def menu():
    choice = input(f"{USER_CHOICES} ")
    while choice != 'q':
        if choice in func.keys():
            select = func[choice]
            select()
        else:
            print("Invalid argument, please try again")
        choice = input(f"{USER_CHOICES} ")

menu()


from .parser import Class, Method, Attr, Object, Int, Str, Block, Assign, \
        Dispatch, StaticDispatch, Plus, Sub, Mult, Div, Lt, Le, Eq, \
        If, While, Let, Case, New, Isvoid, Neg, Not, Bool

from collections import defaultdict, MutableMapping, Set
import warnings
from copy import deepcopy

class SemantError(Exception):
    pass

class SemantWarning(Warning):
    pass

#classes_dict = {}
#inheritance_graph = defaultdict(set)  # format {'classname': {'childclass1', 'childclass2'}}

def install_base_classes(ast):
    """purpose of this is to add base classes always available in the language"""
    objc = Class("Object", None, [
        Method('abort', [], 'Object', None),  # aborts the program
        Method('type_name', [], 'String', None),  # returns string repr of classname
        Method('copy', [], 'SELF_TYPE', None),  # object copy
    ])
    ioc = Class("IO", "Object", [
        Method('out_string', [('arg', 'String')], 'SELF_TYPE', None),  # outputs a string to stdio
        Method('out_int', [('arg', 'Int')], 'SELF_TYPE', None),  # outputs a int to stdio
        Method('in_string', [], 'String', None),  # inputs a string from stdio
        Method('in_int', [], 'Int', None),  # inputs a string from stdio
    ])
    intc = Class("Int", "Object", [
        Attr('_val', '_prim_slot', None)  # unboxed value, untyped
    ])
    boolc = Class("Bool", "Object", [
        Attr('_val', '_prim_slot', None)  # unboxed value, untyped
    ])
    stringc = Class("String", "Object", [
        Attr('_val', 'Int', None),  # string length
        Attr('_str_field', '_prim_slot', None),  # untyped string
        Method('length', [], 'Int', None),  # returns the string length
        Method('concat', [('arg', 'String')], 'String', None),  # str concatenation
        Method('substr', [('arg1', 'Int'), ('arg2', 'Int')], 'String', None),  # str subselection
    ])
    ast += [objc, ioc, intc, boolc, stringc]

def build_inheritance_graph(ast):
    global classes_dict, inheritance_graph
    classes_dict = {}
    inheritance_graph = defaultdict(set)  # format {'classname': {'childclass1', 'childclass2'}}
    for cl in ast:
        if cl.name in classes_dict:
            raise SemantError("class %s already defined" % cl.name)
        classes_dict[cl.name] = cl
        if cl.name == "Object":
            continue  # Object has no parent
        inheritance_graph[cl.parent].add(cl.name)

def check_for_undefined_classes():
    initial_parents = list(inheritance_graph.keys())
    for parentc in initial_parents:
        if parentc not in classes_dict and parentc != "Object":
            warnings.warn("classes %s inherit from an undefined parent %s" % (inheritance_graph[parentc], parentc), SemantWarning)
            inheritance_graph['Object'] |= inheritance_graph[parentc]  # intermediate class does not exist so make these classes inherit from Object
            del inheritance_graph[parentc]

def impede_inheritance_from_base_classes():
    for parent in ['String', 'Int', 'Bool']:
        for cl_name in inheritance_graph[parent]:
            raise SemantError("Class %s cannot inherit from base class %s" % (cl_name, parent))

def visit_inheritance_tree(start_class, visited):
    visited[start_class] = True

    if start_class not in inheritance_graph.keys():
        return True

    for childc in inheritance_graph[start_class]:
        #print("%s to %s" % (start_class, childc))
        visit_inheritance_tree(childc, visited)

    return True

def check_for_inheritance_cycles():
    visited = {}
    for parent_name in inheritance_graph.keys():
        visited[parent_name] = False
        for cl_name in inheritance_graph[parent_name]:
            visited[cl_name] = False
    visit_inheritance_tree("Object", visited)
    for k,v in visited.items():
        if not v:
            raise SemantError("%s involved in an inheritance cycle." % k)

class VariablesScopeDict(MutableMapping):
    """dictionary of varname->type that represent variable scope in the ast"""

    def __init__(self):
        self.store = [dict()]

    def __getitem__(self, key):
        for scope in self.store[::-1]:
            if key in scope:
                return scope[key]
        raise KeyError(key)

    def __setitem__(self, key, value):
        self.store[-1][key] = value

    def __delitem__(self, key):
        del self.store[-1][key]

    def __iter__(self):
        raise NotImplementedError

    def __len__(self):
        raise NotImplementedError

    def new_scope(self):
        self.store.append(dict())

    def destroy_scope(self):
        del self.store[-1]

class VariablesScopeSet(Set):
    """dictionary of varnames that represent variable scope in the ast"""

    def __init__(self):
        self.store = [set()]

    def __iter__(self):
        raise NotImplementedError

    def __contains__(self, value):
        for scope in self.store[::-1]:
            if value in scope:
                return True
        return False

    def __len__(self):
        raise NotImplementedError

    def add(self, value):
        """just call add on the last set"""
        self.store[-1].add(value)

    def new_scope(self):
        self.store.append(set())

    def destroy_scope(self):
        del self.store[-1]

def check_scopes_and_infer_return_types(cl):
    # this function does scope checking and type inference together because
    # the latter is dependent on the first
    variable_scopes = VariablesScopeDict()
    # in COOL methods are not in the same variable scope, which means we can
    # have methods with same name as variables
    attr_seen = set()
    method_seen = set()
    for feature in cl.feature_list:
        # first add all attributes to the scope
        if isinstance(feature, Attr):
            if feature.name in attr_seen:
                raise SemantError("attribute %s is already defined" % feature.name)
            attr_seen.add(feature.name)

            if feature.type == "SELF_TYPE":
                realtype = cl.name
            else:
                realtype = feature.type

            variable_scopes[feature.name] = realtype
    for feature in cl.feature_list:
        if isinstance(feature, Attr):
            traverse_expression(feature.body, variable_scopes, cl)
        elif isinstance(feature, Method):
            if feature.name in method_seen:
                raise SemantError("method %s is already defined" % feature.name)
            method_seen.add(feature.name)
            variable_scopes.new_scope()

            formals_seen = set()
            for formal in feature.formal_list:
                if formal in formals_seen:
                    raise SemantError("formal %s in method %s is already defined" % (formal[0], feature.name))
                formals_seen.add(formal)
                variable_scopes[formal[0]] = formal[1]

            traverse_expression(feature.body, variable_scopes, cl)
            variable_scopes.destroy_scope()

def lowest_common_ancestor(*classes):
    """return the lowest common parent of cl1 and cl2"""
    def ascend_tree(cl):
        yield cl.name
        if cl.parent:
            yield from ascend_tree(classes_dict[cl.parent])

    inheritance_paths = []
    for cl in classes:
        path = list(ascend_tree(cl))
        path.reverse()
        inheritance_paths.append(path)

    smallest_inheritance_path_length = min(len(x) for x in inheritance_paths)

    for step in range(smallest_inheritance_path_length):
        inheritance_points = [x[step] for x in inheritance_paths]
        if len(set(inheritance_points)) > 1:
            return inheritance_paths[0][step-1]  # any would do

    # if we are here, means objects have the same type
    return classes[0].name

def traverse_expression(expression, variable_scopes, cl):
    if isinstance(expression, Isvoid):
        traverse_expression(expression.body, variable_scopes, cl)
        expression.return_type = "Bool"
    elif any(isinstance(expression, X) for X in [Eq, Lt, Le]):
        traverse_expression(expression.first, variable_scopes, cl)
        traverse_expression(expression.second, variable_scopes, cl)
        expression.return_type = "Bool"
    elif isinstance(expression, Neg):
        traverse_expression(expression.body, variable_scopes, cl)
        expression.return_type = "Int"
    elif isinstance(expression, Not):
        traverse_expression(expression.body, variable_scopes, cl)
        expression.return_type = "Bool"
    elif any(isinstance(expression, X) for X in [Plus, Sub, Mult, Div]):
        traverse_expression(expression.first, variable_scopes, cl)
        traverse_expression(expression.second, variable_scopes, cl)
        expression.return_type = "Int"
    elif isinstance(expression, While):
        traverse_expression(expression.predicate, variable_scopes, cl)
        traverse_expression(expression.body, variable_scopes, cl)
    elif isinstance(expression, Let):
        # LET creates a new scope
        variable_scopes.new_scope()
        variable_scopes[expression.object] = expression.type
        traverse_expression(expression.init, variable_scopes, cl)
        traverse_expression(expression.body, variable_scopes, cl)
        variable_scopes.destroy_scope()
        expression.return_type = expression.body.return_type
    elif isinstance(expression, Block):
        last_type = None
        for expr in expression.body:
            traverse_expression(expr, variable_scopes, cl)
            last_type = getattr(expr, 'return_type', None)
        expression.return_type = last_type
    elif isinstance(expression, Assign):
        traverse_expression(expression.body, variable_scopes, cl)
        traverse_expression(expression.name, variable_scopes, cl)
        expression.return_type = expression.name.return_type  # type comes from var declaration
    elif isinstance(expression, Dispatch) or isinstance(expression, StaticDispatch):
        traverse_expression(expression.body, variable_scopes, cl)
        for expr in expression.expr_list:
            traverse_expression(expr, variable_scopes, cl)

        # REDUNDANT code, copied from type_check because we need to infer
        # the dispatch return type from the called method type

        # in case it's self, use current class name
        if expression.body == "self":
            bodycln = cl.name
        else:
            bodycln = expression.body.return_type

        called_method = None
        if bodycln in classes_dict:
            bodycl = classes_dict[bodycln]
            for feature in bodycl.feature_list:
                if isinstance(feature, Method) and feature.name == expression.method:
                    called_method = feature
        if not called_method:
            raise SemantError("Tried to call the undefined method %s in class %s" % (expression.method, bodycl.name))

        if called_method.return_type == "SELF_TYPE":
            method_type = bodycl.name
        else:
            method_type = called_method.return_type

        expression.return_type = method_type
    elif isinstance(expression, If):
        traverse_expression(expression.predicate, variable_scopes, cl)
        traverse_expression(expression.then_body, variable_scopes, cl)
        traverse_expression(expression.else_body, variable_scopes, cl)
        then_type = classes_dict[expression.then_body.return_type]
        else_type = classes_dict[expression.else_body.return_type]
        ret_type = lowest_common_ancestor(then_type, else_type)
        expression.return_type = ret_type
    elif isinstance(expression, Case):
        traverse_expression(expression.expr, variable_scopes, cl)
        branch_types = []
        for case in expression.case_list:
            variable_scopes.new_scope()  # every branch of case has its own scope
            variable_scopes[case[0]] = case[1]
            traverse_expression(case[2], variable_scopes, cl)
            branch_types.append(classes_dict[case[2].return_type])
        expression.return_type = lowest_common_ancestor(*branch_types)
    elif isinstance(expression, Object):
        if expression.name == "self":
            expression.return_type = cl.name
            return
        if expression.name not in variable_scopes:
            raise SemantError("variable %s not in scope" % expression.name)
        expression.return_type = variable_scopes[expression.name]
    elif isinstance(expression, New):
        if expression.type == "SELF_TYPE":
            expression.return_type = cl.name
            return
        expression.return_type = expression.type
    elif isinstance(expression, Int):
        expression.return_type = "Int"
    elif isinstance(expression, Bool):
        expression.return_type = "Bool"
    elif isinstance(expression, Str):
        expression.return_type = "String"

def expand_inherited_classes(start_class="Object"):
    """apply inheritance rules through the class graph"""
    cl = classes_dict[start_class]
    if cl.parent:
        parentcl = classes_dict[cl.parent]

        # Not performant, but cleaner
        attr_set_in_child = [i for i in cl.feature_list if isinstance(i, Attr)]
        attr_set_in_parent = [i for i in parentcl.feature_list if isinstance(i, Attr)]

        for attr in attr_set_in_child:
            for pattr in attr_set_in_parent:
                if attr.name == pattr.name:
                    raise SemantError("Attribute cannot be redefined in child class %s" % cl.name)

        method_set_in_child = [i for i in cl.feature_list if isinstance(i, Method)]
        method_set_in_parent = [i for i in parentcl.feature_list if isinstance(i, Method)]

        def extract_signatures(method_set):
            method_signatures = {}
            for method in method_set:
                method_signatures[method.name] = {}
                for formal in method.formal_list:
                    method_signatures[method.name][formal[0]] = formal[1]
                method_signatures[method.name]['return'] = method.return_type
            return method_signatures

        method_signatures_for_child = extract_signatures(method_set_in_child)
        method_signatures_for_parent = extract_signatures(method_set_in_parent)

        methods_in_child = set()
        for method in method_set_in_child:
            methods_in_child.add(method.name)
            if method.name in method_signatures_for_parent:
                parent_signature = method_signatures_for_parent[method.name]
                child_signature = method_signatures_for_child[method.name]
                if parent_signature != child_signature:
                    raise SemantError("Redefined method %s cannot change arguments or return type of the parent method" % method.name)

        # finished checks, now apply inheritance by simply copying definitions
        # DEEPCOPY of sub-asts to avoid interference with type inference and
        # type checking (all type inference is stored inside return_type
        # fields)
        # insert instead of append so they are evaluated earlier
        for method in method_set_in_parent:
            if method.name not in methods_in_child:
                new_method = deepcopy(method)
                new_method.inherited_from = cl.parent  # used in codegen, to reuse function bodies
                cl.feature_list.insert(0, new_method)
        for attr in attr_set_in_parent:
            cl.feature_list.insert(0, deepcopy(attr))

    # descend down the inheritance tree, applying the same function
    all_children = inheritance_graph[start_class]
    for child in all_children:
        expand_inherited_classes(child)

def is_conformant(childclname, parentclname):
    """check whether childcl is a descendent of parentcl"""
    if childclname == parentclname:
        return True
    for clname in inheritance_graph[parentclname]:
        if is_conformant(childclname, clname):
            return True
    return False

def type_check(cl):
    """make sure the inferred types match the declared types"""
    for feature in cl.feature_list:
        if isinstance(feature, Attr):
            if feature.type == "SELF_TYPE":
                realtype = cl.name
            else:
                realtype = feature.type

            if feature.body:
                type_check_expression(feature.body, cl)
                childcln = feature.body.return_type
                parentcln = realtype
                if not is_conformant(childcln, parentcln):
                    raise SemantError("Inferred type %s for attribute %s does not conform to declared type %s" % (childcln, feature.name, parentcln))
        elif isinstance(feature, Method):
            for formal in feature.formal_list:
                if formal[1] == "SELF_TYPE":
                    raise SemantError("formal %s cannot have type SELF_TYPE" % formal[0])
                elif formal[1] not in classes_dict:
                    raise SemantError("formal %s has a undefined type" % formal[0])

            if feature.return_type == "SELF_TYPE":
                realrettype = cl.name
            else:
                realrettype = feature.return_type

            if feature.body is None:
                # for internal classes, some methods body are not defined
                # assume they return SELF_TYPE
                if feature.return_type == "SELF_TYPE":
                    returnedcln = cl.name
                else:
                    returnedcln = feature.return_type
            else:
                type_check_expression(feature.body, cl)
                returnedcln = feature.body.return_type

            declaredcln = realrettype
            if returnedcln is None:
                warnings.warn("untyped content for method %s with declared type %s" % (feature.name, declaredcln), SemantWarning)
            else:
                if not is_conformant(returnedcln, declaredcln):
                    raise SemantError("Inferred type %s for method %s does not conform to declared type %s" % (returnedcln, feature.name, declaredcln))

def type_check_expression(expression, cl):
    """make sure types validate at any point in the ast"""
    if isinstance(expression, Case):
        type_check_expression(expression.expr, cl)
        for case in expression.case_list:
            type_check_expression(case[2], cl)
    elif isinstance(expression, Assign):
        type_check_expression(expression.body, cl)
        if not is_conformant(expression.body.return_type, expression.name.return_type):
            raise SemantError("The inferred type %s for %s is not conformant to declared type %s" % (expression.body.return_type, expression.name.name, expression.name.return_type))
    elif isinstance(expression, If):
        type_check_expression(expression.predicate, cl)
        type_check_expression(expression.then_body, cl)
        type_check_expression(expression.else_body, cl)
        if expression.predicate.return_type != "Bool":
            raise SemantError("If statements must have boolean conditions")
    elif isinstance(expression, Let):
        type_check_expression(expression.init, cl)
        if expression.init:  # some let expression auto-initialize with default values
            if not is_conformant(expression.init.return_type, expression.type):
                raise SemantError("The inferred type %s for let init is not conformant to declared type %s" % (expression.init.return_type, expression.type))
    elif isinstance(expression, Block):
        for line in expression.body:
            type_check_expression(line, cl)
    elif isinstance(expression, Dispatch) or isinstance(expression, StaticDispatch):
        type_check_expression(expression.body, cl)
        # dispatch to current instance (self)
        if expression.body == "self":
            bodycln = cl.name
        else:
            bodycln = expression.body.return_type
        if isinstance(expression, StaticDispatch):
            # additional check on static dispatch
            if not is_conformant(bodycln, expression.type):
                raise SemantError("Static dispatch expression (before @Type) does not conform to declared type {}".format(expression.type))

        called_method = None
        if bodycln in classes_dict:
            bodycl = classes_dict[bodycln]
            for feature in bodycl.feature_list:
                if isinstance(feature, Method) and feature.name == expression.method:
                    called_method = feature
        if not called_method:
            raise SemantError("Tried to call the undefined method %s in class %s" % (expression.method, bodycl.name))
        if len(expression.expr_list) != len(called_method.formal_list):
            raise SemantError("Tried to call method {} in class {} with wrong number of arguments".format(called_method.name, bodycl.name))
        else:
            # check conformance of arguments
            for expr, formal in zip(expression.expr_list, called_method.formal_list):
                if not is_conformant(expr.return_type, formal[1]):
                    raise SemantError("Argument {} passed to method {} in class {} is not conformant to its {} declaration".format(expr.return_type, called_method.name, bodycl.name, formal[1]))
    elif isinstance(expression, While):
        type_check_expression(expression.predicate, cl)
        type_check_expression(expression.body, cl)
        if expression.predicate.return_type != "Bool":
            raise SemantError("While statement must have boolean conditions")
    elif isinstance(expression, Isvoid):
        type_check_expression(expression.body, cl)
    elif isinstance(expression, Not):
        type_check_expression(expression.body, cl)
        if expression.body.return_type != "Bool":
            raise SemantError("Not statement require boolean values")
    elif isinstance(expression, Lt) or isinstance(expression, Le):
        type_check_expression(expression.first, cl)
        type_check_expression(expression.second, cl)
        if expression.first.return_type != "Int" or expression.second.return_type != "Int":
            raise SemantError("Non-integer arguments cannot be check with < == or <=")
    elif isinstance(expression, Neg):
        type_check_expression(expression.body, cl)
        if expression.body.return_type != "Int":
            raise SemantError("Negative statement require integer values")
    elif any(isinstance(expression, X) for X in [Plus, Sub, Mult, Div]):
        type_check_expression(expression.first, cl)
        type_check_expression(expression.second, cl)
        if expression.first.return_type != "Int" or expression.second.return_type != "Int":
            raise SemantError("Arithmetic operations require integers")
    elif isinstance(expression, Eq):
        type_check_expression(expression.first, cl)
        type_check_expression(expression.second, cl)
        type1 = expression.first.return_type
        type2 = expression.second.return_type
        if (type1 == "Int" and type2 == "Int") or \
           (type1 == "Bool" and type2 == "Bool") or \
           (type1 == "String" and type2 == "String"):
            pass  # comparing basic types together is ok
        else:
            raise SemantError("Comparison is only possible among same base types")

def semant(ast):
    install_base_classes(ast)
    build_inheritance_graph(ast)
    check_for_undefined_classes()
    impede_inheritance_from_base_classes()
    check_for_inheritance_cycles()
    expand_inherited_classes()
    for cl in classes_dict.values():
        check_scopes_and_infer_return_types(cl)
    for cl in classes_dict.values():
        type_check(cl)
    return classes_dict


from typing import List
from textwrap import dedent
from common import DatabaseConnection
from common.db_tools import dict2asyncpg_param
from common.exceptions import I4cClientError
from models.log import DataPointLog

async def check_data_id(conn, device, timestamp, sequence, data_id):
    sql = dedent("""\
        select data_id from log
        where
          device = $1
          and "timestamp" = $2
          and sequence = $3
        """)
    r = await conn.fetchrow(sql, device, timestamp, sequence)
    if r and r[0] != data_id:
        raise I4cClientError("data_id update is not allowed")

async def put_log_write(credentials, datapoints: List[DataPointLog], *, override=False, pconn=None):
    on_conflict = "DO NOTHING" if not override else """\
                   DO UPDATE SET
                     instance = EXCLUDED.instance,
                     value_num = EXCLUDED.value_num,
                     value_text = EXCLUDED.value_text,
                     value_extra = EXCLUDED.value_extra,
                     value_aux = EXCLUDED.value_aux
                   """
    sql = dedent(f"""\
                 insert into log
                 (
                    device,
                    instance,
                    "timestamp",
                    sequence,
                    data_id,
                    value_num,
                    value_text,
                    value_extra,
                    value_aux
                   )
                   values ($1, $2, $3, $4, $5, $6, $7, $8, $9)
                   ON CONFLICT (device, "timestamp", sequence)
                   {on_conflict}""")

    async with DatabaseConnection(pconn) as conn:
        async with conn.transaction():
            for d in datapoints:
                if override:
                    await check_data_id(conn, d.device, d.timestamp,
                                   d.sequence, d.data_id)
                await conn.execute(sql,
                               d.device, d.instance, d.timestamp,
                               d.sequence, d.data_id, d.value_num,
                               d.value_text, d.value_extra, dict2asyncpg_param(d.value_add))


#!/usr/bin/env python3

import sys
import rospy
import cv2
import numpy as np

from geometry import CameraObj
from ssdlocalization import Observation, Database, vector_to_msg
from perceptbot_camera.msg import ObjDetection, ImageWithTransform
from image_geometry import PinholeCameraModel as Camera
from model_loader import Stopwatch
from visualization_msgs.msg import MarkerArray, Marker
from geometry_msgs.msg import Point, Pose
from time import sleep

from ssdlite_mobilenet import SSDlite
from yolo_v3 import YoloV3
from tiny_yolo_v3 import TinyYoloV3

# dictionary of model classes available to object detector for lookup with launch param
# pass in string name when loading module to use appropriate SSD model
models = {"yolo": YoloV3, "tiny_yolo": TinyYoloV3, "ssdlite": SSDlite}

class ObjectDetector:
    def __init__(self, model):
        self.database = Database()
        self.stopwatch = Stopwatch()
        self.stopwatch.start()
        self.model = model
        self.stopwatch.lap()
        self.pub = None
        self.broadcast_markers = False
        self.vis_pub = None
        rospy.init_node('obj_detection', anonymous=True)

    def call_model(self, iwt):
        if self.model.is_busy:
            return
        img_msg = iwt.compressedImage
        is_stable = iwt.is_stable
        camera_transform = np.reshape(np.array(iwt.flat_transform), (4, 4))
        str_msg = img_msg.data
        buf = np.ndarray(shape=(1, len(str_msg)), dtype=np.uint8, buffer=img_msg.data)
        image = cv2.imdecode(buf, cv2.IMREAD_ANYCOLOR)

        obj_detections = self.model.process_image(image, img_msg.header)
        if obj_detections:
            camera_obj = CameraObj(camera_transform, img_msg.header.stamp, image)
            camera_obj.set_camera_fov(110, 88)  #THIS HARD CODE NEEDS TO BE DONE AS VARIABLE
            self.publish_messages(obj_detections)
            obs = [Observation.from_ros_msg(camera_obj, obj) for obj in obj_detections]
            self.show_observations(obs, is_stable)
            if is_stable:
                self.database.match_observations(obs)

    def publish_messages(self, obj_detections):
        if self.pub:
            for obj in obj_detections:
                self.pub.publish(obj)

    # turn on broadcasts if being used in ROS context
    def activate_ROS_communication(self, broadcast_markers=True, refresh_duration=5):
        self.pub = rospy.Publisher('/obj_detection', ObjDetection, queue_size=50)
        self.listen()
        self.broadcast_markers = broadcast_markers
        if broadcast_markers:
            self.vis_pub = rospy.Publisher('/visualization_marker_array', MarkerArray, queue_size=1)
            self.show_matches(refresh_duration)
        else:
            rospy.spin()

    def listen(self):
        # buffer may need to be very large to overcome OS queue and ensure only latest
        # message is used...
        rospy.Subscriber('/iwt_camera_stream', ImageWithTransform,
                         callback=self.call_model, queue_size=1, buff_size=2000000)

    #publish RVIZ markers for matches
    def show_matches(self, refresh_duration=5):
        lifetime = rospy.Duration(refresh_duration + 10)
        while not rospy.is_shutdown():
            time = rospy.Time.now()
            #  HACK FOR NOW TO NOT SHOW PEOPLE
            markers = [self.make_match_marker(key, match, time, lifetime)
                       for key, keyList in self.database.matches.items()
                       for match in keyList
                       if key != "person"]
            if markers:
                self.vis_pub.publish(MarkerArray(markers))
                self.vis_pub.publish(MarkerArray(self.annotate_matches(markers)))
            sleep(refresh_duration)

    #publish RVIZ markers for observations
    def show_observations(self, observations, is_stable):
        if self.broadcast_markers and observations:
            time = rospy.Time.now()
            lifetime = rospy.Duration(30 if is_stable else 2) #unstable observations only shown briefly
            markers = [self.make_observation_marker(obs, time, lifetime, is_stable) for obs in observations]
            self.vis_pub.publish(MarkerArray(markers))
            self.vis_pub.publish(MarkerArray(self.annotate_observations(markers)))

    # add text markers for each match
    @staticmethod
    def annotate_matches(marker_list):
        for marker in marker_list:
            marker.ns = "text-" + marker.ns
            marker.type = 9
            pose = Pose()
            pose.position.x = marker.pose.position.x
            pose.position.y = marker.pose.position.y
            pose.position.z = marker.pose.position.z + marker.scale.z + 0.1
            marker.pose = pose
            marker.scale.z = 0.15
        return marker_list

    # add text markers for observations
    @staticmethod
    def annotate_observations(marker_list):
        for marker in marker_list:
            marker.ns = "text-" + marker.ns
            marker.type = 9
            marker.pose = Pose()
            marker.pose.position.x = marker.points[1].x
            marker.pose.position.y = marker.points[1].y
            marker.pose.position.z = marker.points[1].z
            marker.scale.z = 0.1
        return marker_list

    # construct a RVIZ marker from a match
    @staticmethod
    def make_match_marker(key, match, time, lifetime):
        marker = Marker(ns="Match:" + key, id=match.id, type=3,
                        text=key, pose=match.pose, lifetime=lifetime)
        marker.header.frame_id = "map"
        marker.header.stamp = time
        marker.scale.z = match.estimated_height * 0.5
        marker.scale.x = 0.1
        marker.scale.y = 0.1
        marker.color.a = 1.0
        marker.color.r = 0.25
        marker.color.g = 0.75
        marker.color.b = 0.25
        marker.colors = [marker.color]
        return marker

    #construct a RVIZ marker from an observation
    @staticmethod
    def make_observation_marker(observation, time, lifetime, is_stable):
        marker = Marker(ns="Observe:" + observation.class_label, id=observation.id, type=0,
                        text=observation.class_label, lifetime=lifetime)
        marker.id = observation.id + (0 if is_stable else 1000) # reuse namespace at different range
        marker.points = [vector_to_msg(observation.position(), Point()),
                         vector_to_msg(observation.position() + observation.bearing, Point())]
        marker.header.frame_id = "map"
        marker.header.stamp = time
        marker.scale.x = 0.01
        marker.scale.y = 0.03
        marker.color.a = 1.0
        if is_stable:
            marker.color.r = 0.25
            marker.color.g = 0.25
            marker.color.b = 0.75
        else:
            marker.color.r = 0.75
            marker.color.g = 0.25
            marker.color.b = 0.25
        marker.colors = [marker.color]
        return marker

# model name can be based in as the first parameter
if __name__ == '__main__':
    if len(sys.argv) < 2:
        model = YoloV3() # default model
    else:
        model_class = models.get(sys.argv[1], YoloV3)
        model = model_class()
    o = ObjectDetector(model)
    o.activate_ROS_communication()


import pandas as pd
from numpy import random
import numpy

from utils import (SimpleMovingAvrage, ExponentialMovingAvrage, CumulativeMovingAvrage, ADX, plus_di,minus_di, adx)
from utils import (stop_buy, stop_sell, risk_reward_ratio, target)

prices = pd.Series(random.randint(low=1, high=10, size=10))

# simple_moving_avrage = ExponentialMovingAvrage(prices, 5)
# t = simple_moving_avrage.update_moving_avrage(22)

close_price = prices[1]
target = target(close_price, 1)
stop_buy = stop_buy(prices, 0.1)
stop_sell = stop_sell(prices, 0.1)
rr_buy = risk_reward_ratio(_target=target, _stop=stop_buy)
rr_sell = risk_reward_ratio(_target=target, _stop=stop_sell)

moving_average_old = ExponentialMovingAvrage(prices, 5)
ema = moving_average_old.update_moving_avrage(close_price)

# ================================================= ADX PART ========================================================

data_prices = pd.DataFrame(numpy.random.randint(low=19503,
                                                high=26152,
                                                size=(100, 5)),
                                                columns=['open', 'high', 'low', 'close', 'values'])

obj_prices = ADX(prices=prices, lookback=14)
plus = plus_di(prices, obj_prices.get_adx_plus())
minus = minus_di(prices, obj_prices.get_adx_minus())
adx = adx(prices, obj_prices.get_adx_smooth())

plus = plus.dropna()
minus = minus.dropna()
adx = adx.dropna()

plus.tail()
minus.tail()
adx.tail()

test_adx = obj_prices = ADX(prices=prices, lookback=14)

obj_prices.update_price(open=22310, high=19963, low=20135, close=25000, values=23555)

plus_new = obj_prices.update_adx_plus()
minus_new = obj_prices.update_adx_minus()
adx_new = obj_prices.update_adx_smooth()

plus_new = plus_new.dropna()
plus_new.tail()
minus_new = minus_new.dropna()
minus_new.tail()
adx_new = adx_new.dropna()
adx_new.tail()

# =======================================================================================================================
Trading = False

if (close_price > ema) and (data_prices['plus_di'][14] > data_prices['minus_di'][14]):
    if rr_buy > 1.5:
        if not Trading:
            print('buy')
            Trading = True
        else:
            print('You are trading')
    else:
        print('risk reward ratio is less than 1.5 ')
elif (close_price < ema) and (data_prices['plus_di'][14] < data_prices['minus_di'][14]):
    if rr_sell > 1.5:
        if not Trading:
            Trading = True
            print('sell')
        else:
            print('You are trading')
    else:
        print('risk reward ratio is less than 1.5 ')
else:
    print('There are no conditions for Trading currency')


import io
import logging
from lottie.exporters import exporters
from lottie.importers import importers

def parse_tgs_to_json(input_file: bytes) -> str:
    infile = io.BytesIO(input_file)

    for p in importers:
        if 'tgs' in p.extensions:
            importer = p
            break

    outfile = io.StringIO()
    exporter = exporters.get_from_extension('json')
    an = importer.process(infile)
    exporter.process(an, outfile)
    return outfile.getvalue()

def parse_json_to_tgs(input_file: str) -> bytes:
    infile = io.BytesIO(input_file.encode())

    for p in importers:
        if 'json' in p.extensions:
            importer = p
            break

    outfile = io.BytesIO()
    exporter = exporters.get_from_extension('tgs')
    an = importer.process(infile)
    exporter.process(an, outfile)
    return outfile.getvalue()

async def distort(target: bytes, config: list):
    distorted = parse_tgs_to_json(target)

    for cf in config:
        distorted = distorted.replace(cf[0], cf[1])

    outFile = io.BytesIO()
    outFile.name = "distorted.tgs"
    try:
        outFile.write(parse_json_to_tgs(distorted))
    except:
        return None
    outFile.seek(0)
    return outFile

async def Distorting(input_file: bytes, configs: list):
    _num = 0
    for conf in configs:
        try:
            distorted = await distort(
                target=input_file,
                config=conf
            )
        except Exception as err:
            logging.error(f'Error in Distorter:\n{err}')
            continue

        _num += 1
        if distorted:
            yield _num, distorted
        else:
            yield None, None


import sys
import grp
import pwd
import subprocess
import os

def create_group(group):
	"""Create a Unix group"""
	try:
		info = grp.getgrnam(group)
		gid = info.gr_gid
	except KeyError:
		subprocess.call(["groupadd", "-r", group])
		info = grp.getgrnam(group)
		gid = info.gr_gid
	return gid

def create_user(user, home, gid):
	"""Create a Unix user"""
	try:
		info = pwd.getpwnam(user)
		uid = info.pw_uid
	except KeyError:
		subprocess.call(["useradd", "-r", "-g", str(gid),
						"-s", "/bin/false", "-d", home, user])
		info = pwd.getpwnam(user)
		uid = info.pw_uid
	return uid

def make_dir(path, uid, gid):
	"""Make a directory"""
	if not os.path.exists(path):
		os.mkdir(path)
	if uid is not None:
		os.chown(path, uid, gid)
		for _file in os.listdir(path):
			os.chown(os.path.join(path, _file), uid, gid)


import requests
from parsel import Selector
from time import sleep
from tech_news.Tecmundo_scraper import Scrap_tecmundo
from tech_news.database import create_news

# https://softbranchdevelopers.com/python-requests-library-exception-handling-advanced-request-get-parameters/#all-exceptions
def fetch(url):
    resp = ""
    try:
        sleep(1)
        resp = requests.get(url, timeout=3)
    except requests.exceptions.RequestException:
        return None
    finally:
        if resp != "" and resp.status_code == 200:
            return resp.text
        return None

def scrape_noticia(html_content):
    selector = Selector(text=html_content)
    scraped_news = Scrap_tecmundo(selector)
    return scraped_news.mount()

# header Notícias have more hidden news with class .tec--card__title__link, so
#  use id #js-main to restrict to only main news
def scrape_novidades(html_content):
    selector = Selector(text=html_content)
    fresh_news = selector.css(
        "#js-main .tec--card__title__link::attr(href)"
    ).getall()
    return fresh_news

def scrape_next_page_link(html_content):
    selector = Selector(text=html_content)
    next_page = selector.css("#js-main .tec--btn::attr(href)").get()
    return next_page

def add_news_to_database(requested_news):
    news_to_insert = []
    for url in requested_news:
        html_content = fetch(url)
        new = scrape_noticia(html_content)
        news_to_insert.append(new)
    create_news(news_to_insert)
    return news_to_insert

def get_tech_news(amount):
    pagination = ""
    base_url = "https://www.tecmundo.com.br/novidades"
    html_content = fetch(base_url + pagination)
    latest_news = scrape_novidades(html_content)

    while len(latest_news) < amount:
        pagination = (scrape_next_page_link(html_content)).split("/novidades")[
            1
        ]
        html_content = fetch(base_url + pagination)
        latest_news.extend(scrape_novidades(html_content))
    requested_news = latest_news[:amount]
    return add_news_to_database(requested_news)


from django.shortcuts import render

# Create your views here.
import Preprocessing as preprocess

from django.http import JsonResponse
import pickle
from django.views.decorators.csrf import csrf_exempt
import json
@csrf_exempt
def Classifier(request):
    try:
        print(request.body.decode('utf-8'))
        data = request.body.decode('utf-8')
        dict=json.loads(data)
        Rawtweet= dict['Raw_tweet']
        CleanTweet=preprocess.preprocess_tweet(Rawtweet)
        vectorizer = pickle.load(open('./Vectorizer','rb'))
        model = pickle.load(open('./LRclassifier','rb'))
        pred = model.predict(vectorizer.transform([Rawtweet]))[0]
        if pred==0:
            return JsonResponse({"Msg":'NoHate'} ,status=201)
        else:
            return JsonResponse({"Msg":'Hateful Tweet'},status=201 )
    except Exception as e:
        return JsonResponse({"Msg":'Unexpected Error'+str(e)},status=500)