import argparse
import os
import sys

import numpy as np
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from torch.autograd import Variable

from generate_input import get_data
from model import ConstrainedModel
import utils

from torch.utils.tensorboard import SummaryWriter
import logging

logging.basicConfig(
    format="%(asctime)s %(levelname)-8s %(message)s",
    level=logging.INFO,
    datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger(__name__)

from time import time
from collections import defaultdict

def train_nn(
    model,
    train_loader,
    hps,
    model_dir,
    alpha,
    writer,
    beta=None,
):
    # Check Device configuration
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    logger.info(f"Using device {device}")

    learning_rate = hps.learning_rate
    n_epochs = hps.n_epochs

    criterion = nn.MSELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

    # Define loss stopping criteria
    ep_stop = 20
    d_loss = []
    conv_loss = 1e-3

    model.to(device)
    logger.info("Training starts!")

    model.train()
    init_wd = []
    for layer in model.layers_weight:
        init_wd += [torch.flatten(layer.data).cpu().numpy()]

    for epoch in range(n_epochs):
        for idx, data in enumerate(train_loader):
            inputs, targets = data
            inputs, targets = inputs.float(), targets.float()

            inputs, targets = (
                Variable(inputs).to(device),
                Variable(targets).to(device),
            )

            # Forward pass
            hidden_states, outputs, _ = model(inputs)

            loss_model = criterion(outputs, targets)

            # Load the constraints to the network
            dist_reg = model.compute_weight_regularizer(norm=hps.norm)
            l_reg = model.compute_lregularizer(norm=hps.norm)

            loss = loss_model + alpha * (
                beta * dist_reg + (1 - beta) * l_reg
            )

            # Backpropagation and optimization
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            writer.add_scalar(
                "training loss",
                loss.item(),
                idx + epoch * len(train_loader),
            )

        if epoch == 0:
            utils.save_checkpoints(
                    {
                        "state_dict": model.state_dict(),
                        "optim_dict": optimizer.state_dict(),
                        "loss": loss,
                        "inputs": inputs,
                        "outputs": outputs,
                        "targets": targets,
                        "hidden_states": hidden_states,
                        "init_wd": init_wd,
                    },
                    checkpoint=os.path.join(
                        model_dir, "checkpoints", "init.pth"
                    ),
                )
        if len(d_loss) >= ep_stop:
            d_loss.pop(0)
        d_loss.append(loss.item())

        if (epoch + 1) % ep_stop == 0:
            acc_avg, acc_max, acc_min, acc_std = utils.evaluate(
                outputs, targets
            )
            accuracies = [acc_avg, acc_max, acc_min, acc_std]
            if (
                np.abs(
                    np.mean(d_loss[: ep_stop // 2])
                    - np.mean(d_loss[ep_stop // 2 :])
                )
                > conv_loss
            ):
                continue
            else:
                logger.info(
                    f"Current epoch is {epoch} and accuracy is {acc_avg}."
                )
                utils.save_checkpoints(
                    {
                        "epoch": epoch,
                        "state_dict": model.state_dict(),
                        "optim_dict": optimizer.state_dict(),
                        "loss": loss,
                        "inputs": inputs,
                        "outputs": outputs,
                        "targets": targets,
                        "hidden_states": hidden_states,
                        "init_wd": init_wd,
                        "accuracy": accuracies,
                    },
                    checkpoint=os.path.join(
                        model_dir, "checkpoints", "last.pth"
                    ),
                )
                print("Checkpoint saved")
                break

def reg_train(hps, model_dir, alpha):
    if isinstance(alpha, (int, float)):
        alpha = [alpha]
    alpha = (
        torch.arange(0.0, 0.009, 0.001)
        if alpha is None
        else np.array(alpha)
    )
    for a in alpha:
        beta = torch.FloatTensor([0.0, 1.0])
        for b in beta:
            logger.info(f"alpha = {a}, beta = {b}")
            # Initialize new dataset and model for each alpha value
            train_loader, test_loader = get_data(hps)
            model = ConstrainedModel(
                hps.n_bits,
                hps.hidden_size,
                hps.n_bits,
                hps.n_spatial_dims,
                hps.norm,
            )

            # Creates directories for each alpha and beta value
            alpha_dir = os.path.join(
                model_dir,
                f"alpha_{round(a.item(),5)}_beta_{round(b.item(),3)}",
            )
            writer = SummaryWriter(log_dir=os.path.join(alpha_dir, "logs"))
            if not os.path.exists(alpha_dir):
                os.makedirs(alpha_dir)
                logger.info(f"Created directory {alpha_dir}")

            # Initialize training
            train_nn(
                model,
                train_loader,
                hps,
                alpha_dir,
                a,
                writer,
                b
            )
    logger.info("Experiments are completed!")

parser = argparse.ArgumentParser(
    description="Process net with distance constraints and L1/L2 regularization."
)
parser.add_argument("--seed", type=int, default=None)
parser.add_argument("--alpha", type=float, default=None)

args = parser.parse_args()

logger.info(f"Passed arguments {args}")

# Initialize the training
manualSeed = args.seed
random.seed(manualSeed)
torch.manual_seed(manualSeed)

torch.cuda.manual_seed(manualSeed)
torch.cuda.manual_seed_all(manualSeed)
torch.backends.cudnn.enabled = False
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True

json_path = os.path.join(os.getcwd(), "hps.json")
assert os.path.isfile(
    json_path
), "No json configuration file found at {}".format(json_path)
hps = utils.Params(json_path)

model_dir = os.path.join(
    os.getcwd(), f"trial{hps.trial}", f"seed{manualSeed}trained_model"
)
if not os.path.exists(model_dir):
    os.makedirs(model_dir)
    logger.info(f"Created directory {model_dir}")

reg_train(
    hps,
    model_dir,
    alpha=args.alpha,
)


import torch

def gram_schmidt(input_mat, reverse=False, modified=False):
    """ Carries out the Gram-Schmidt orthogonalization of a matrix.

    Arguments:
        input_mat (torch.Tensor): A quadratic matrix that will be turned into an
            orthogonal matrix.
        reverse (bool): Starts gram Schmidt method beginning from the last
            column if set to True.
        modified (bool): Uses modified Gram-Schmidt as described.
    """
    mat_size = input_mat.shape[0]
    Q = torch.zeros(mat_size, mat_size,
                    device=input_mat.device, dtype=input_mat.dtype)

    if modified:
        if reverse:
            outer_iterator = range(mat_size - 1, -1, -1)
            def inner_iterator(k): return range(k, -1, -1)
        else:
            outer_iterator = range(mat_size)
            def inner_iterator(k): return range(k+1, mat_size)

        # This implementation mostly follows the description from
        # https://www.math.uci.edu/~ttrogdon/105A/html/Lecture23.html
        # The more complex form is due to pytorch not allowing for inplace
        # operations of variables needed for gradient computation.
        v = input_mat
        for j in outer_iterator:
            Q[:, j] = v[:, j] / torch.norm(v[:, j])

            v_old = v
            v = torch.zeros(mat_size, mat_size,
                            device=input_mat.device, dtype=input_mat.dtype)

            for i in inner_iterator(j):
                v[:, i] = v_old[:, i] \
                          - (torch.dot(Q[:, j].clone(), v_old[:, i])
                             * Q[:, j].clone())

    elif not modified:
        if reverse:
            outer_iterator = range(mat_size - 1, -1, -1)
            def inner_iterator(k): return range(mat_size - 1, k, -1)
        else:
            outer_iterator = range(mat_size)
            def inner_iterator(k): return range(k)

        for j in outer_iterator:
            v = input_mat[:, j]
            for i in inner_iterator(j):
                p = torch.dot(Q[:, i].clone(), v) * Q[:, i].clone()
                v = v - p

            Q[:, j] = v / torch.norm(v)

    return Q

def gram_schmidt_batched(input_mat, reverse=False, modified=False):
    """ Carries out the Gram-Schmidt orthogonalization of a matrix on an
        entire batch.

    Arguments:
        input_mat (torch.Tensor): A tensor containing quadratic matrices each of
            which will be orthogonalized of shape (batch_size, m, m).
        reverse (bool): Starts gram Schmidt method beginning from the last
            column if set to True.
        modified (bool): Uses modified Gram-Schmidt as described.

    Returns:
        Q (torch.Tensor): A batch of orthogonal matrices of same shape as
            input_mat.
    """
    batch_size = input_mat.shape[0]
    mat_size = input_mat.shape[1]
    Q = torch.zeros(batch_size, mat_size, mat_size,
                    device=input_mat.device, dtype=input_mat.dtype)

    if modified:
    #TODO implement batched version
        for i in range(input_mat.shape[0]):
            q = gram_schmidt(input_mat[i], reverse, modified)
            Q[i] = q
    elif not modified:
        if reverse:
            raise NotImplementedError
        else:
            outer_iterator = range(mat_size)
            def inner_iterator(k): return range(k)

        for j in outer_iterator:
            v = input_mat[:, :, j].view(batch_size, mat_size, 1)
            for i in inner_iterator(j):
                q_squeezed = Q[:, :, i].view(batch_size, 1, mat_size).clone()
                dot_products = torch.bmm(q_squeezed, v)
                p = dot_products.repeat((1, mat_size, 1)) \
                    * Q[:, :, i].unsqueeze(2).clone()
                v = v - p

            Q[:, :, j] = v.squeeze() / torch.norm(v, dim=1).repeat(1, mat_size)

    return Q


from django.urls import path
from .views import (
    ClientTableView, ClientView, ContractingView, EstablishmentView, GetClientsView, GetCompaniesToCreateClient, GetCompaniesToCreateEstablishment, GetEstablishmentsToCreateClient, GetPriceTablesToCreateClient, SpecificClient, SpecificClientTable, SpecificContracting, SpecificEstablishment, CompanyView, SpecificCompany
)

urlpatterns = [
    path('contracting', ContractingView.as_view()),
    path('contracting/<contracting_code>', SpecificContracting.as_view()),
    #Company
    path('company', CompanyView.as_view()),
    path('company/<company_compound_id>', SpecificCompany.as_view()),
    # Establishment
    path('establishment', EstablishmentView.as_view()),
    path('establishment/<establishment_compound_id>', SpecificEstablishment.as_view()),
    # Client table
    path('client_table', ClientTableView.as_view()),
    path('client_table/<client_table_compound_id>', SpecificClientTable.as_view()),
    # Client
    #  path('client_tables_to_create_client', GetClientTablesToCreateClient.as_view()),
    path('companies_to_create_client', GetCompaniesToCreateClient.as_view()),
    path('establishments_to_create_client/<client_table_compound_id>', GetEstablishmentsToCreateClient.as_view()),
    path('price_tables_to_create_client/<company_compound_id>', GetPriceTablesToCreateClient.as_view()),
    path('get_clients/<client_table_compound_id>', GetClientsView.as_view()),
    path('client', ClientView.as_view()),
    path('client/<client_compound_id>', SpecificClient.as_view()),
    path('companies_to_create_establishment', GetCompaniesToCreateEstablishment.as_view()),

]


# IMPORTING LIBRARIES
import pickle
import pandas as pd
import webbrowser

import dash
import dash_html_components as html
import dash_core_components as dcc
import dash_bootstrap_components as dbc
from dash.dependencies import Input, Output , State

import plotly.express as px

from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer

import numpy as np
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import base64

# READING AND PREPARING DATA
df = pd.read_csv('scrappedreviews.csv')
df.head()
df.shape

reviews = list(df.Reviews)

# LOADING PICKLE MODELS
file = open("pickle_model1.pkl", 'rb')
pickle_model = pickle.load(file, encoding = ('ISO-8859-1'))

file1 = open("features1.pkl", 'rb')
vocab = pickle.load(file1, encoding = ('ISO-8859-1'))

# Function checking whether a given review is Positive or Negative
def check_review(reviewText):
    transformer = TfidfTransformer()
    loaded_vec = CountVectorizer(decode_error="replace", vocabulary = vocab)
    vectorised_review = transformer.fit_transform(loaded_vec.fit_transform([reviewText]))

    return pickle_model.predict(vectorised_review)

# Function for directly opening the browser
def open_browser():
    webbrowser.open_new("http://127.0.0.1:8050/")

# Preparing data for Pie-Chart and Wordcloud
output_list = list(map(check_review, reviews))

# Pie-Chart
ratings = output_list
positive = []
negative = []

for rating in ratings:
    if (rating == 0):
        negative.append(rating)
    else:
        positive.append(rating)

perc_p = int(((len(positive)/len(ratings))*100))
perc_n = int(((len(negative)/len(ratings))*100))

data = {'Reviews': ['Positive', 'Negative'], 'Percentage': [perc_p, perc_n]}
df1 = pd.DataFrame(data)

sreviews = str(reviews)

df2 = pd.read_csv('scrappedreviews.csv')
df2 = df2['Reviews']

# Wordcloud
wc = WordCloud(max_font_size = 80, max_words = 100, width = 800, height = 400, background_color="black").generate(sreviews)
wc.to_file('img.png')

image_filename = 'img.png'
encoded_image = base64.b64encode(open(image_filename, 'rb').read())

# MAIN LAYOUT OF APPLICATION
app = dash.Dash(__name__, external_stylesheets = [dbc.themes.CYBORG])

app.layout = html.Div(children = [
    html.Div(children = [
    html.H1(id = 'Main_title', children = 'SENTIMENT ANALYSIS WITH INSIGHTS', style={
    'textAlign': 'center',
    'color': 'white',
    'font-size': 50,
    'margin-top': '20px'}),
    html.H1(children = 'DASHBOARD', style = {'backgroundColor': 'yellow', 'textAlign': 'center', 'color': 'black'})
    ]),

    html.H1(children = "Select any review from the dropdown list and check whether it's Positive or Negative", style = {'font-size': 20, 'margin-top': '80px', 'font-style': 'bold', 'color': 'white'}),
    html.Div([dcc.Dropdown(id = 'dropdown',
                           options = [{'label': v[:260], 'value': v} for v in df.Reviews],
                           style = {'width':'98%', 'height':50, 'margin': '10px'},
                           searchable = False),
                           html.H1(children = None, id='result1', style = {'margin': '10px', 'margin-bottom': '40px'})]),

    html.Div(children = [html.Div(className = 'row',
    children = [html.Div(className = 'six columns div-user-controls bg-grey',
             children = [html.H1(children = '''PIE-CHART''', style = {'color': 'yellow'}),
                         html.H1(children = "Showing Positive and Negative Reviews as percentages", style = {'font-size': 15, 'font-style': 'italic', 'color': 'white'}),
                         html.Div([dcc.Graph(id = 'pie-chart',
                                             figure = px.pie(df1, values='Percentage', names = 'Reviews', title='POSITIVE AND NEGATIVE REVIEWS'),
                                             style={'height': 400, 'width':600, 'margin-top': '25px', 'margin-bottom': '80px', 'margin': '10px'})])]),
                         html.Div(className = 'six columns div-user-controls bg-grey',
                                  children = [html.H1(children = 'WORDCLOUD', style = {'color': 'yellow'}),
                                              html.H1(children = "Showing the most frequently used words in the list of Reviews", style = {'font-size': 15, 'font-style': 'italic', 'color': 'white'}),
                                              html.Img(id = 'wordcloud',
                                                       src = app.get_asset_url('img.png'),
                                                       style={'height': 400, 'width':600, 'margin-top': '25px', 'margin-bottom': '80px', 'margin': '10px'})])])]),

    html.H1(children = "Enter any review of your choice and check whether it's Positive or Negative", style = {'font-size' : 20, 'margin-top': '80px', 'font-style': 'bold', 'color': 'white'}),
    html.H1([dcc.Textarea(id = 'textarea_review',
                           placeholder = 'Enter the review here.....',
                           style = {'width':'98%', 'height':100, 'margin': '10px'}),

              dbc.Button(children = 'Find Review',
                         id = 'button_review',
                         color = 'success',
                         style= {'width':'98%', 'margin': '10px', 'color': 'black'}),
                         html.H1(children = None, id='result')])], style = {'backgroundColor': 'lightblack'})

# FUNCTION CALLBACKS
# Callback 1
@app.callback(
    Output('result', 'children'),
    [Input('button_review', 'n_clicks')],
    [State('textarea_review', 'value')])
def update_output(n_clicks, textarea_value):

    if (n_clicks > 0):

        response = check_review(textarea_value)
        if (response[0] == 0):
            result = 'NEGATIVE'
        elif (response[0] == 1 ):
            result = 'POSITIVE'
        else:
            result = 'UNKNOWN'

        return result

    else:
        return ""

# Callback 2
@app.callback(
    dash.dependencies.Output('result1', 'children'),
    [dash.dependencies.Input('dropdown', 'value')])
def update_output1(value):

    response1 = check_review(value)
    if (response1[0] == 0):
        result1 = 'NEGATIVE'
    elif (response1[0] == 1 ):
        result1 = 'POSITIVE'
    else:
        result1 = 'UNKNOWN'

    return result1

# MAIN FUNCTION
if __name__ == '__main__':
    open_browser()
    app.run_server()


import numpy as np
import matplotlib.pyplot as plt

# Load and display an image from CSV file
digits_file = 'digits.csv'
digits = np.loadtxt(digits_file, delimiter=',')
print("Datatype of digits:", type(digits))

# Select and reshape a row from digits array
im = digits[21, 1:]
im_sq = np.reshape(im, (28, 28))

# Display the reshaped image using matplotlib
plt.imshow(im_sq, cmap='Greys', interpolation='nearest')
plt.show()

# Load data from a file with a header and specific columns
header_data_file = 'digits_header.txt'
data = np.loadtxt(header_data_file, delimiter='\t', skiprows=1, usecols=(0, 2))
print("Loaded data:", data)

# Load and process string data from a file
seaslug_file = 'seaslug.txt'
data_str = np.loadtxt(seaslug_file, delimiter='\t', dtype=str)
print("First element of data_str:", data_str[0])

# Load and process numeric data from the same file
data_float = np.loadtxt(seaslug_file, delimiter='\t', dtype='float', skiprows=1)
print("10th element of data_float:", data_float[9])

# Create a scatterplot using loaded data
plt.scatter(data_float[:, 0], data_float[:, 1])
plt.xlabel('time (min.)')
plt.ylabel('percentage of larvae')
plt.show()

# Load and display structured data from a CSV file
titanic_file = 'titanic.csv'
d = np.recfromcsv(titanic_file)
print("First three entries of d:", d[:3])

14 - Introduction to Importing Data in Python/01 - Introduction and flat files/02 - Using Numpy to Import flat files.py


import datetime
from dataclasses import dataclass
from enum import Enum
from typing import Dict
from uuid import uuid4

import asyncpg

class EventType(Enum):
    LINK_RESOLVED = "link_resolved"

@dataclass(frozen=True)
class Event:
    id: str
    type: str
    payload: Dict
    created_at: datetime.datetime

    @staticmethod

    async def publish(connection: asyncpg.Connection, type: EventType, payload: Dict) -> 'Event':
        event_id = uuid4()
        row = await connection.fetchrow("""
           INSERT INTO event_log
           (id, type, payload, created_at)
           VALUES
           ($1, $2, $3, NOW())
           RETURNING id, type, payload, created_at
        """, event_id, type.value, payload)

        return Event(**row)


# coding: utf-8

# In[96]:

import requests
from bs4 import BeautifulSoup

# requests.get (URL) + requests.content - keep HTML of page.
r = requests.get("https://www.pythonhow.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/")
c = r.content

# beautify HTML
soup = BeautifulSoup(c, "html.parser")

# logic - grab HTML, go through HTML's and find relevant tags for data we want to get.
# we should find the elements which identifies
all = soup.find_all("div",{"class":"propertyRow"})
# each element of the "all" have same methods of soup.find_all ...
all[0].find("h4",{"class":"propPrice"}).text.replace("\n","")
# get page number dynamically
num_of_pages = soup.find_all("a", {"class":"Page"})[-1].text
print(num_of_pages)

# In[98]:

#now we'd like to extract the data by iterating
l = []
#base_url="https://www.pythonhow.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/t=0&s="
for page in range(0,int(num_of_pages)*10,10):
    r = requests.get(base_url + str(page) + ".html")
    c = r.content
    soup = BeautifulSoup(c, "html.parser")
    all = soup.find_all("div",{"class":"propertyRow"})
    for item in all:
        d = {}
        d["Address"] = item.find_all("span", {"class":"propAddressCollapse"})[0].text
        try:
            d["Locality"] = item.find_all("span", {"class":"propAddressCollapse"})[1].text
        except:
            d["Locality"] = None
        d["Price"] = item.find("h4",{"class":"propPrice"}).text.replace("\n","")
        try:
            d["Beds"] = item.find("span", {"class":"infoBed"}).find("b").text
        except:
            d["Beds"] = None

        try:
            d["Area"] = item.find("span", {"class":"infoSqFt"}).find("b").text
        except:
            d["Area"] = None

        try:
            d["Full Baths"] = item.find("span", {"class":"infoValueFullBath"}).find("b").text
        except:
            d["Full Baths"] = None

        try:
            d["Half Baths"] = item.find("span", {"class":"infoValueHalfBath"}).find("b").text
        except:
            d["Full baths"] = None

        for column_group in item.find_all("div", {"class":"columnGroup"}):
            for feature_group, feature_name in zip(column_group.find_all("span", {"class":"featureGroup"}),
                                                  column_group.find_all("span", {"class":"featureName"})):
                if "Lot Size" in feature_group.text:
                    d["Lot Size"] = feature_name.text
        l.append(d)

# In[99]:

import pandas
df = pandas.DataFrame(l)

# In[85]:

df

# In[100]:

df.to_csv("WebScrapperOutput.csv")


from django.conf.urls import url
from django.urls import path
from first_app import views

app_name= "First_app"

urlpatterns= [
    path('',views.index,name='index'),
    path('add_albam/',views.albam_form,name='albam_form'),
    path('add_musician/',views.musician_form,name='musician_form'),
    path('albam_list/<int:artist_id>/',views.albam_list,name='albam_list'),
    path('edit_artist/<int:artist_id>/',views.edit_artist,name='edit_artist'),
    path('edit_albam/<int:albam_id>/',views.edit_albam,name='edit_albam'),
    path('delete_albam/<int:albam_id>/',views.delete_albam,name='delete_albam'),
    path('delete_artist/<int:artist_id>/',views.delete_musician,name='delete_artist'),

]


import numpy as np
from scipy.sparse import csc_matrix, csgraph, find
import scipy.sparse as sp
from .. import inputs

def get_local_problems_structure(DUAL_1, GID_1, adjacencies, ts):
    ds_edges, local_ID_edges, entity_ID_edges = get_dual_structure(DUAL_1, adjacencies, 2)
    edge_local_problems, edge2face_connections = get_prolongation_operator_local_problems(adjacencies, ds_edges, DUAL_1, local_ID_edges, entity_ID_edges, [], ts)

    ds_faces, local_ID_faces, entity_ID_faces = get_dual_structure(DUAL_1, adjacencies, 1)
    face_local_problems, face2internal_connections = get_prolongation_operator_local_problems(adjacencies, ds_faces, DUAL_1, local_ID_faces, entity_ID_edges, edge2face_connections, ts)

    ds_internal, local_ID_internal, entity_ID_internal = get_dual_structure(DUAL_1, adjacencies, 0)
    internal_local_problems, _ = get_prolongation_operator_local_problems(adjacencies, ds_internal, DUAL_1, local_ID_internal, entity_ID_faces, face2internal_connections, ts)
    # entity_ID=np.vstack([entity_ID_edges, entity_ID_faces, entity_ID_internal]).max(axis=0)
    local_ID=np.vstack([local_ID_edges, local_ID_faces, local_ID_internal]).max(axis=0)
    return [edge_local_problems, face_local_problems, internal_local_problems], local_ID

# @profile
def get_prolongation_operator_local_problems(adjacencies, entities, DUAL_1, local_ID, entity_ID_up, external_connections_in, ts):
    local_problems=[]
    print(len(entities[0]))
    external_connections_out = []
    map_v=np.zeros_like(local_ID)
    map_l=np.zeros_like(local_ID)
    for i in range(len(entities[0])):
        local_faces = entities[0][i] # Local faces internal internal
        adjs = adjacencies[local_faces]
        lines=[adjs[:,0], adjs[:,1], adjs[:,0], adjs[:,1]]
        cols=[adjs[:,1], adjs[:,0], adjs[:,0], adjs[:,1]]
        data=[ts[local_faces],ts[local_faces],-ts[local_faces],-ts[local_faces]]
        internal_off_diag_entries = np.concatenate([local_faces, local_faces])
        internal_diag_entries = [local_faces, local_faces]
        external_matrices=[]

        for j in range(1,len(entities)):
            if len(entities[j][0])>0:
                external_faces = entities[j][i]
                adjs_int_ext = adjacencies[external_faces]
                int_pos = DUAL_1[adjs_int_ext]==DUAL_1[adjs_int_ext].min()
                internal_gids=adjs_int_ext[int_pos]
                #_____External influences____
                external_gids = adjs_int_ext[~int_pos]
                entity_up_ids = np.unique(entity_ID_up[external_gids]).astype(int)
                # import pdb; pdb.set_trace()
                # print(entity_up_ids)
                l=local_ID[internal_gids]
                map_l[np.unique(external_gids)]=np.arange(len(np.unique(external_gids)))
                c=map_l[external_gids]
                d=external_faces.copy()
                external_matrix=csc_matrix((d, (l, c)), shape = (l.max()+1, c.max()+1), dtype=np.float32)
                if len(external_connections_in)>0:
                    aa=np.hstack([external_connections_in[e] for e in entity_up_ids])
                    map_l[np.unique(aa[0,:])]=np.arange(len(np.unique(aa[0,:])))#[np.argsort(entity_ID_up[external_gids])]
                    ls=map_l[aa[0,:]]
                    map_v[np.unique(aa[1,:])]=np.arange(len(np.unique(aa[1,:])))#[np.argsort(internal_gids)]

                    cs=map_v[aa[1,:]]
                    # import pdb; pdb.set_trace()
                    # asort=np.lexsort((aa[1,:],aa[0,:]))
                    # ls=ls[asort]
                    # cs=cs[asort]
                    '''
                    asort=np.lexsort((edges, labels))
                    slabels=labels[asort]
                    sedges=edges[asort]
                    '''
                    # map_l[np.unique(aa[0,:])]=range(len(np.unique(aa[0,:])))
                    g_lines=np.tile(np.unique(adjs),len(np.unique(aa[1,:])))
                    g_cols=np.repeat(np.unique(aa[1,:]),len(np.unique(adjs)))
                    matrix_connection=csc_matrix((np.arange(len(ls))+1,(ls, cs)), shape=(ls.max()+1,cs.max()+1))

                else:
                    matrix_connection=[]
                    g_lines=np.tile(np.unique(adjs),len(external_gids))
                    g_cols=np.repeat(np.sort(external_gids),len(np.unique(adjs)))

                external_connections_out.append([g_lines, g_cols])
                external_matrices.append([external_matrix, external_faces, external_gids, entity_up_ids, matrix_connection, np.unique(g_cols)])
                #_____Internal influences____
                lines.append(internal_gids)
                cols.append(internal_gids)
                data.append(-ts[external_faces])
                internal_diag_entries.append(external_faces)
                #________________________
        lines=local_ID[np.concatenate(lines)]
        cols=local_ID[np.concatenate(cols)]
        data=np.concatenate(data)
        internal_diag_entries=np.concatenate(internal_diag_entries)
        # import pdb; pdb.set_trace()
        if len(lines)>0:
            acumulator = lines*(cols.max()+1)+cols
            internal_matrix=csc_matrix((np.ones_like(data), (lines, cols)), shape = (lines.max()+1, cols.max()+1), dtype=np.float32)
            # import pdb; pdb.set_trace()
            internal_gids = np.unique(adjs)
            i_matrix_structure = [internal_matrix, internal_off_diag_entries, internal_diag_entries, acumulator, internal_gids]
        else:
            i_matrix_structure=[]
        local_problems.append([i_matrix_structure, external_matrices])
    return local_problems, external_connections_out

def get_dual_and_primal_1(centroids):
    maxs=centroids.max(axis=0)
    mins=centroids.min(axis=0)
    cr1=np.array(inputs.multiscale_and_multilevel_inputs['multiscale']['coarsening_ratio_1'])
    n_blocks=np.array(inputs.finescale_inputs['mesh_generation_parameters']['n_blocks'])
    block_size=np.array(inputs.finescale_inputs['mesh_generation_parameters']['block_size'])
    n_duals=n_blocks//cr1+1
    second_line = (n_blocks-(n_duals-1)*cr1)//2
    xd=[]
    for i in range(3):
        xd.append(np.arange(second_line[i]+cr1[i],(n_duals[i]-1)*cr1[i],cr1[i]))
        if len(xd[i])>0:
            xd[i]=np.unique(np.concatenate([[mins[i], maxs[i]],(xd[i]+0.5)*block_size[i]]))
        else:
            xd[i]=np.unique(np.array([mins[i], maxs[i]]))
    d=np.zeros(len(centroids))

    for i in range(3):
        for x in xd[i]:
            d+=centroids[:,i]==x
    DUAL_1=d

    volumes=np.arange(len(centroids))
    GID_1=-np.ones_like(volumes)
    xp=np.array(mins[0]-1)
    xp=np.append(xp,(xd[0][:-1]+xd[0][1:])/2)
    xp=np.append(xp,maxs[0]+1)

    yp=np.array(mins[1]-1)
    yp=np.append(yp,(xd[1][:-1]+xd[1][1:])/2)
    yp=np.append(yp,maxs[1]+1)

    zp=np.array(mins[2]-1)
    zp=np.append(zp,(xd[2][:-1]+xd[2][1:])/2)
    zp=np.append(zp,maxs[2]+1)
    count=0
    for i in range(len(xp)-1):
        vx=volumes[(centroids[:,0]>=xp[i]) & (centroids[:,0]<xp[i+1])]
        for j in range(len(yp)-1):
            vy=vx[(centroids[:,1][vx]>=yp[j]) & (centroids[:,1][vx]<yp[j+1])]
            for k in range(len(zp)-1):
                vz=vy[(centroids[:,2][vy]>=zp[k]) & (centroids[:,2][vy]<zp[k+1])]
                GID_1[vz]=count
                count+=1
    return GID_1, DUAL_1

def get_dual_structure(DUAL_1, adjs, entity):
    dual_adjs=DUAL_1[adjs]
    all_faces=np.arange(len(adjs))
    ee=(dual_adjs==entity).sum(axis=1)==2

    ev=adjs[ee]
    edges=np.unique(ev)
    mapd=np.arange(adjs.max()+1)
    mapd[edges]=np.arange(len(edges))
    adjs0=adjs[:,0][ee]
    adjs1=adjs[:,1][ee]
    lines=np.concatenate([mapd[adjs0],mapd[adjs1]])
    cols=np.concatenate([mapd[adjs1],mapd[adjs0]])
    data=np.ones(len(lines))
    graph=csc_matrix((data,(lines,cols)),shape=(len(edges),len(edges)))
    n_l,labels=csgraph.connected_components(graph,connection='strong')

    # asort=np.argsort(labels)
    asort=np.lexsort((edges, labels))
    slabels=labels[asort]
    sedges=edges[asort]
    pos=np.array([0])
    pos=np.append(pos,np.arange(len(labels)-1)[(-slabels[:-1]+slabels[1:])==1]+1)
    pos=np.append(pos,len(labels))
    vs=np.concatenate([range(pos[i]-pos[i-1]) for i in range(1,len(pos))])

    entity_ID=-np.ones(len(DUAL_1))
    entity_ID[edges]=labels
    local_ID=entity_ID.copy()
    local_ID[sedges]=vs
    entities=[]
    for i in range(entity, 4):
        faces=all_faces[(dual_adjs.min(axis=1)==entity) & (dual_adjs.max(axis=1)==i)]
        ents=entity_ID[adjs[faces]].max(axis=1).astype(int)
        asort=np.argsort(ents)
        sents=ents[asort]
        sfaces=faces[asort]
        pos=np.arange(len(faces)-1)[(-sents[:-1]+sents[1:])==1]
        entities.append(np.split(sfaces,pos+1))
    return entities, local_ID.astype(int), entity_ID


import gc
from time import sleep
from datetime import datetime
import psutil

LVE = 0.5
BLOCKSIZE = 1024 * 1024

def plog(s):
    print("%s %s" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),s))

def memorySpace():
    mem = psutil.virtual_memory()
    plog(int((mem.available - (mem.total*LVE)) / BLOCKSIZE))
    return int((mem.available - (mem.total*LVE)) / BLOCKSIZE)

def memoryUp(lst):
    while memorySpace() > 0:
        lst.append(1 << 7864320)

def memoryDown(lst):
    while len(lst) > 0:
        if memorySpace() < 0:
            del(lst[0])
            gc.collect()
        else:
            break

if __name__ == "__main__":
    lst=[]
    while True:
        for idx in range(0,20):
            memoryDown(lst)
            if idx == 0:
                memoryUp(lst)

            sleep(3)


from ersa.parser import *
import pytest

def test_SharedSegment():
    with pytest.raises(AssertionError):
        SharedSegment(0)
    for n in [14, 16]:
        with pytest.raises(AssertionError):
            SharedSegment(n * [1])

def test_read_matchfile():
    path = "ersa/tests/test_data/test_LL.match"
    s_list = read_matchfile(path)
    len = 0
    for s in s_list:
        len += 1
    assert len == 14

    path = "ersa/tests/test_data/test_LL_haploscores.match"
    s_list = read_matchfile(path, haploscores=True)
    len = 0
    for s in s_list:
        len += 1
    assert len == 14

def test_get_pair_dict():
    path = "ersa/tests/test_data/test_LL.match"
    pair_dict = get_pair_dict(path, 2.5)
    assert len(pair_dict['TestA:TestB']) == 7

    pair_dict = get_pair_dict(path, 2.5, "TestA")
    with pytest.raises(KeyError):
        assert len(pair_dict['TestB:TestC']) == 9

def test_merge_segments():
    path = "ersa/tests/test_data/test_merge.match"
    pair_dict = get_pair_dict(path, 0)

    # no merging when merge_len <= 0
    for i in range(-1, 1):
        for pair, segs in pair_dict.items():
            n = len(segs)
            new_segs = merge_segments(segs, i)
            assert len(new_segs) == n

    for pair, segs in pair_dict.items():
        new_segs = merge_segments(segs, 2)
        if pair == 'TestA:TestB':
            assert len(new_segs) == 4
        elif pair == 'TestB:TestC':
            assert len(new_segs) == 3

    for pair, segs in pair_dict.items():
        new_segs = merge_segments(segs, 99)
        if pair == 'TestA:TestB':
            assert len(new_segs) == 2
        elif pair == 'TestB:TestC':
            assert len(new_segs) == 2

    for pair, segs in pair_dict.items():
        new_segs = merge_segments(segs, 100)
        if pair == 'TestA:TestB':
            assert len(new_segs) == 2
        elif pair == 'TestB:TestC':
            assert len(new_segs) == 1

    for pair, segs in pair_dict.items():
        new_segs = merge_segments(segs, 500)
        if pair == 'TestA:TestB':
            assert len(new_segs) == 2
        elif pair == 'TestB:TestC':
            assert len(new_segs) == 1


#!/usr/bin/env python3

# standards
import re
from typing import Any, Dict, Tuple, no_type_check

# 3rd parties
import lxml.etree as ET

# klon
from .utils import is_element

def build_etree(tag: str, *args) -> ET._Element:
    if not isinstance(tag, str):
        raise ValueError(f'Tag must be a str, got {tag!r}')
    attrib, args = _compile_attrib(*args)
    tag, attrib = _parse_css_style_tags(tag, attrib)
    element = ET.Element(tag, attrib)
    _append_children(element, args)
    return element

def _compile_attrib(*args) -> Tuple[Dict, Tuple[Any, ...]]:
    attrib = {}
    if len(args) > 0 and isinstance(args[0], dict):
        attrib.update(args[0])
        args = args[1:]
    return attrib, args

def _parse_css_style_tags(tag: str, attrib: Dict) -> Tuple[str, Dict]:
    match = re.search(r'^(.+?)(\#|\.)(.+)', tag)
    if match:
        tag, key, value = match.groups()
        key = {'#': 'id', '.': 'class'}[key]
        attrib[key] = value
    return tag, attrib

@no_type_check  # until lxml-stubs improves
def _append_children(element: ET._Element, args: Tuple[Any, ...]) -> None:
    text_anchor = None
    for child in args:
        if child in (None, (), []):
            pass
        elif isinstance(child, str):
            if text_anchor is None:
                element.text = (element.text or '') + child
            else:
                text_anchor.tail = (text_anchor.tail or '') + child
        else:
            if not is_element(child):
                child = build_etree(*child)
            element.append(child)
            text_anchor = child


import time
import boto3
import json

def watch_sqs(sqs_url:str, waittime:int=0, region:str='us-east-1', purge_queue:bool=False) -> list:
    '''
    Monitor an SQS for new messages. This is a blocking function that will
    wait until an SQS message appears.

    Parameters
    ----------
    sqs_url : str
        The url of the SQS to monitor.
    waittime : int, optional
        Time in seconds to wait for querying the SQS for new messages.
        The default is 0.
    region : str, optional
        AWS region the SQS is in.
        The default is 'us-east-1'.
    purge_queue : bool, optional
        If true, each time a message is found, the entire SQS will
        be purged of all messages before proceeding.  This is useful if you
        have many workers that could submit messages to the SQS, but you only
        want 1 downstream job to trigger.
        The default is False.

    Returns
    -------
    list
        SQS messages.

    '''

    sqs_client = boto3.client('sqs', region_name=region)

    while True:
        print("watching sqs")

        #check if messages in queue
        response = sqs_client.receive_message(
                                    QueueUrl=sqs_url,
                                    #20sec is the max allowed for sqs long polling,
                                    #using time.sleep for longer polling
                                    WaitTimeSeconds=min(waittime,20)
                                    )
        if waittime > 20:
            time.sleep(waittime-20)

        #if we found messages, delete from queue and return
        #back to parent
        lst = []
        if 'Messages' in list(response.keys()):
            print("Message found in sqs. Extracting.")
            if purge_queue:
                sqs_client.purge_queue( QueueUrl=sqs_url)
            else:
                for message in response['Messages']:
                    #leaving this code behind in case I want the SQS message contents
                    #recieptHandle = response['Messages'][0]['ReceiptHandle']
                    #body = json.loads(response['Messages'][0]['Body'])
                    #filename = body['Records'][0]['s3']['object']['key']

                    recieptHandle = message['ReceiptHandle']
                    lst.append(recieptHandle)

                    sqs_client.delete_message(
                                                QueueUrl=sqs_url,
                                                ReceiptHandle  = recieptHandle)
            break

    print("SQS: wait completed, moving on. ")
    return lst

#-------------------------------------------------------------------------------------

def create_sqs_s3(queue_name:str, account_number:str, s3_bucket_name:str) -> dict:
    '''

    Deprecated: use CDK instead

    Create an SQS that accepts notifications from S3 bucket triggers. Note this
    function uses the SDK and is expected to be replaced with CDK in the
    future.

    Parameters
    ----------
    queue_name : str
        Name of the SQS
    account_number : str
        Account where the SQS will be created.
    s3_bucket_name : str
        Name of the s3 bucket to accept messages from.

    Returns
    -------
    dict
        SQS queue attributes.

    '''

    #create the queue then set the access policy to enable s3 notifications
    client = boto3.client('sqs')
    response = client.create_queue(  QueueName=queue_name )

    queue_attributes = client.get_queue_attributes(QueueUrl=response['QueueUrl'],
                                            AttributeNames=['All'])

    s3_policy_update={}
    s3_policy_update['Policy'] = \
        {
        "Version": "2012-10-17",
        "Id": "auto-setup",
        "Statement": [
            {
                "Sid": "s3-events",
                "Effect": "Allow",
                "Principal": {
                    "Service": "s3.amazonaws.com"
                },
                "Action": [
                    "SQS:SendMessage"
                ],
                "Resource": queue_attributes['Attributes']['QueueArn'],
                "Condition": {
                    "ArnLike": {
                        "aws:SourceArn": "arn:aws:s3:*:*:"+ s3_bucket_name
                    },
                    "StringEquals": {
                        "aws:SourceAccount": account_number
                    }
                }
            }
        ]
    }

    s3_policy_update['Policy'] = json.dumps(s3_policy_update['Policy'])

    response = client.set_queue_attributes(
                                    QueueUrl=response['QueueUrl'],
                                    Attributes= s3_policy_update
                                )

    return queue_attributes

#-------------------------------------------------------------------------------------

def delete_sqs(queue_name:str) -> None:
    '''

    Deprecated: use CDK instead

    Delete an SQS. Note this
    function uses the SDK and is expected to be replaced with CDK in the
    future.

    Parameters
    ----------
    queue_name : str
        Name of the SQS to delete.

    Returns
    -------
    None

    '''

    client = boto3.client('sqs')
    response = client.list_queues(QueueNamePrefix = queue_name)

    if "QueueUrls" not in response.keys():
        print("QueueDoesNotExist.")
        return None

    for QueueUrl in response['QueueUrls']:
        try:
            client.delete_queue(
                                QueueUrl=QueueUrl
                            )
        except Exception as e:
           if "QueueDoesNotExist" in str(e):
               print("QueueDoesNotExist.")
           else:
               raise e


#!/usr/local/bin/python3
import shutil
import os
import sys
import re
import subprocess
import multiprocessing
from .pip_packages import get_requirements
from .hardening_manifest_functions import *

def gem_environment_check():
    """Check to make sure environment requirements are met"""
    print('\nChecking environment...\n')
    # Checking for gem
    if not shutil.which('gem'):
        sys.exit('ERROR: It seems gem is not installed '
                 'or is not in path, please fix to continue.\n')

    # Make sure gem-packages.txt has content
    file_size = os.stat("requirements_files/gem-packages.txt")
    if file_size.st_size == 0:
        sys.exit('ERROR: gem-packages.txt does not have any content.'
                 ' Please add required packages there.\n')

def get_gem_deps(package_string: str):
    """ Use the user provided requirement to get all dependencies """
    # Possible values for package_string:
    # package_name OR package_name <boolOperator><version>
    #   possible bool operators <, >, <=, >=, and =
    pkg_dep_list = []
    failed_list = []
    try:
        # If there is a bool operator in the string we need to capture that
        # and use it in the command for versioning
        char_list = [">", "<", "="]
        has_version_spec = [characters in char_list for characters in package_string]
        if True in has_version_spec:
            pkg_name, pkg_version_spec = package_string.split(" ")
            dependency_command_specific_version = "gem install {} -rq -v '{}' --explain 2> /dev/null".format(pkg_name, pkg_version_spec)
            output = subprocess.check_output(dependency_command_specific_version, shell=True)
            for word in output.decode('utf-8').split('\n')[1:]:
                pkg = word.strip()
                pkg_dep_list.append(pkg)
            return(pkg_dep_list, failed_list)
        else:
            # If just package_name then use this command
            dependency_command_latest_version =  "gem install {} -rq --explain 2> /dev/null".format(package_string)
            output = subprocess.check_output(dependency_command_latest_version, shell=True)
            for word in output.decode('utf-8').split('\n')[1:]:
                pkg = word.strip()
                pkg_dep_list.append(pkg)
            return(pkg_dep_list, failed_list)
    except:
        # If both failed add this package to the failed list
        failed_list.append(package_string)
        return(pkg_dep_list, failed_list)

def create_dep_list(requirements: list):
    """ Create master and failed list of packages """
    print("Retrieving metadata about gems...\n")
    master_list = []
    master_failed_list = []
    with multiprocessing.Pool(processes=15) as pool:
        vals = pool.imap(get_gem_deps, requirements)
        # Dedup and add packages to master or failed list
        for pkgs_list in vals:
            packages_list, failed_list = pkgs_list
            for pkg in packages_list:
                if pkg not in master_list:
                    master_list.append(pkg)
            for pkg in failed_list:
                if pkg not in master_failed_list:
                    master_failed_list.append(pkg)
    # Remove any null entries that may have been
    # included when parsing and decoding in previous steps
    try:
        master_list.remove('')
    except:
        pass
    return(master_list, master_failed_list)

def get_sha(url: str):
    """ The package's hash lives on their site so we need to curl for it """
    # Example response (value of output var below)
    # b'
SHA 256 checksum:
\n
\n      6636e0262fec9b2b1b40775c5702944a36ad5649408e5a2e25f844760d4950f2\n'
    get_sha_command = "curl -s {} 2> /dev/null | grep -i 'gem__sha' -A1 -B1".format(url)
    output = subprocess.check_output(get_sha_command, shell=True)
    decoded_list = output.decode('utf-8').split('\n')
    sha_type_string = decoded_list[0]
    sha_value_string = decoded_list[2]

    # Parse HTML response for the sha type
    sha_type_string_2 = sha_type_string.split(">")[1]
    sha_type_uppercase = sha_type_string_2.split(" checksum")[0]
    sha_type = sha_type_uppercase.lower()

    # Strip spaces from string
    pkg_sha = sha_value_string.strip()
    return(pkg_sha, sha_type)

def split_name(package: str):
    """ Use regex to properly split the string into name and version spec """
    version_tuple = re.search('(-\d{1,10}\.\d{1,10}\.\d{1,10}-?.{0,50})', package)
    version_string = version_tuple.groups()[0]
    version = version_string.split("-", 1)[1]

    name = re.split('(-\d{1,10}\.\d{1,10}\.\d{1,10}-?.{0,50})', package)
    pkg_name = name[0]
    return(pkg_name, version)

def build_a_pkg_dict(package: str):
    """ For each package get build its dictionary entry """
    # Example string: concurrent-ruby-1.1.8
    try:
        pkg_name, pkg_version = split_name(package)

        # Can build the file location and hash location URLs with just this info above
        filename = "{}.gem".format(package)
        pkg_url = "https://rubygems.org/downloads/{}.gem".format(package)
        sha_url = "https://rubygems.org/gems/{}/versions/{}".format(pkg_name, pkg_version)

        pkg_sha, sha_type_string = get_sha(sha_url)
        sha_type = sha_type_string.replace(" ", "")

        # Create pkg dictionary
        pkg_entry = {
            'pkg_filename': filename,
            'sha_url': sha_url,
            'pkg_url': pkg_url,
            'pkg_sha': pkg_sha,
            'sha_type': sha_type
            }
        success = True
        return(success, pkg_entry)
    except:
        success = False
        return(success, package)

def build_master_pkg_dicts(packages: list):
    """ Build dictionaries with neccessary values and build URLs """
    master_list_pkg_dicts = []
    failed_list = []
    with multiprocessing.Pool(processes=15) as pool:
        vals = pool.imap(build_a_pkg_dict, packages)
        # add packages to master or failed list
        for pkg_tuple in vals:
            success = pkg_tuple[0]
            if success:
                pkg_dict = pkg_tuple[1]
                master_list_pkg_dicts.append(pkg_dict)
            else:
                pkg_name = pkg_tuple[1]
                failed_list.append(pkg_name)
    return(master_list_pkg_dicts, failed_list)

def gem_create_hardening_manifest(force_option: bool, multi_pkg_types: bool):
    """ Main Function """
    # Based on whether were running multiple packages do the gem env check
    if not multi_pkg_types:
        gem_environment_check()

    # Based on if the manifest exists and the options in cli set some required vars
    manifest_exist = os.path.isfile("hardening_manifest/hardening_manifest.yaml")
    if force_option:
        append_to_manifest = False
        if manifest_exist:
            append_to_manifest = True
    else:
        if not multi_pkg_types:
            append_to_manifest = check_for_existing_manifest()
        else:
            append_to_manifest = True

    # Get user's packages
    print("Getting gem packages and their dependencies...\n")
    path = '{}/requirements_files/gem-packages.txt'.format(os.getcwd())
    user_required_pkgs_list = get_requirements(path)

    # Get dependencies
    master_list, failed_list = create_dep_list(user_required_pkgs_list)

    # Gathering metadata required for each package
    master_list_of_metadata, master_failed_list = build_master_pkg_dicts(master_list)
    for entry in failed_list:
        if entry not in master_failed_list:
            master_failed_list.append(entry)

    if not multi_pkg_types:
        print("Creating hardening manifest...\n")

    # Decide whether we need to append to manifest as well as dedup new and existing list if we do
    if append_to_manifest:
        try:
            master_pkgs_metadata = sha_compare_existing_manifest_to_requirements(master_list_of_metadata)
        except:
            pass
    else:
        master_pkgs_metadata = master_list_of_metadata

    # Build the manifest
    manifest_resources_section = generate_resources_section(master_pkgs_metadata, append_to_manifest)
    generate_hardening_manifest(manifest_resources_section)

    # Based on whether this is multiple package managers and if anything failed print to screen.
    if not multi_pkg_types:
        if master_failed_list:
            print('ERROR: Packages not found:')
            for package in master_failed_list:
                print(' - {}'.format(package))
        else:
            print('All packages found!')
        print("\nHardening manifest located at {}/hardening_manifest/hardening_manifest.yaml\n".format(os.getcwd()))
    if multi_pkg_types:
        return (master_failed_list)


from typing import Any, Dict, Type, Union

from .. import base
from .model import Model
from .nested import Nested

class Inline(Model, Nested):
    def __init__(
        self,
        props: Dict[
            str,
            Union[Model, base.SchemaAttribute, Type[Model], Type[base.SchemaAttribute]],
        ],
        **kwargs: Any,
    ):
        for key, value in props.items():
            if isinstance(value, type):
                props[key] = value()
        self.__props__ = props
        Nested.__init__(self, model=self, **kwargs)


#!/usr/bin/env python

import numpy as np
from scipy.spatial import ConvexHull

def PolyArea2D(pts):
    lines = np.hstack([pts,np.roll(pts,-1,axis=0)])
    area = 0.5*abs(sum(x1*y2-x2*y1 for x1,y1,x2,y2 in lines))
    return area

isQuitTrue = False
pointList = []

print("Program to Calculate Convex Hull Area")

while(isQuitTrue==False):
    myInput = input("Press 'c' To Enter Coordinates or Press 'q' To Exit: ")

    if(myInput=="q" or myInput=="Q"):
        isQuitTrue = True
        break

    elif(myInput=="c" or myInput=="C"):
        xCoordinate = int(input("Enter x coordinate: "))
        yCoordinate = int(input("Enter y coordinate: "))
        pointList.append([xCoordinate,yCoordinate])

        continueInput = input("To continue adding points precc 'c', to calculate area press 'a': ")
        if(continueInput=="c" or continueInput =="C"):
            continue
        elif(myInput=="a" or myInput=="A"):
            isQuitTrue = True
    else:
        print("Invalid input try again\n")

print("Your list of points: ",pointList)
print("Area: ", PolyArea2D(pointList))

points = np.array([(1, 2), (3, 4), (3, 6), (2, 4.5), (2.5, 5)])
hull = ConvexHull(pointList)

np.random.seed(1)
random_points = np.random.uniform(0, 6, (100, 2))
#print("randpoints: ",random_points)

# get array of boolean values indicating in hull if True
in_hull = np.all(np.add(np.dot(random_points, hull.equations[:,:-1].T),
                        hull.equations[:,-1]) <= 0.5, axis=1)

random_points_in_hull = random_points[in_hull]

#print(random_points_in_hull)

howManyNumInHull = 0

for i in range(len(random_points_in_hull)):
    howManyNumInHull+=1
print("What percent is in this area: ",howManyNumInHull/100)


# -*- coding: utf-8 -*-
"""
Created on Mon Nov 12 22:33:31 2018

@author: bpou
"""
#from matplotlib.pyplot import show
import networkx
BJ = 'Beijing'
SZ = 'Shenzhen'
GZ = 'Guangzhou'
WH = 'Wuhan'
HLG = 'Heilongjiang'
NY = 'New York City'
CM = 'Chiangmai'
SG = 'Singapore'

air_route = {
    BJ : {SZ, GZ, WH, HLG, NY},
    GZ : {WH, BJ, CM, SG},
    SZ : {BJ, SG},
    WH : {BJ, GZ},
    HLG : {BJ},
    CM : {GZ},
    NY : {BJ}
}

air_route = networkx.Graph(air_route)

#networkx.draw(air_route, with_labels=True)

def search_destination(graph, start, destination):
    pathes = [[start]]
    visited = set()
    chosen_pathes = []
    while pathes:
        path = pathes.pop(0)
        #print (path)
        frontier = path[-1]
        if frontier in visited:
            continue
        # get new lines

        for city in graph[frontier]:
            new_path = path + [city]
            pathes.append(new_path)
            if city == destination:
               # print (new_path)
                return new_path

        visited.add(frontier)
    return chosen_pathes

def draw_route(cities): return ' ✈ -> '.join(cities)

print(draw_route(search_destination(air_route, SZ, CM)))


import os
import pefile

__LIST_OF_DLLS = {
    'ADVAP132.DLL', # Advanced Win32 application programming interfaces
    'AWFAXP32.DLL', # Mail API fax transport
    'AWFXAB32.DLL', # Address book
    'AWPWD32.DLL', # Security support
    'AWRESX32.DLL', # Resource Executor
    'AWUTIL32.DLL', # At Work Security Support
    'BHNETB.DLL', # Network monitor SMS client
    'BHSUPP.DLL', # Network monitor SMS client
    'CCAPI.DLL', # Microsoft Network component
    'CCEI.DLL', # Microsoft Network component
    'CCPSH.DLL', # Microsoft Network component
    'CCTN20.DLL', # Microsoft Network component
    'CMC.DLL', # Common messaging calls for Mail API 1.0
    'COMCTL32.DLL', # User Experience Controls Library
    'COMDLG32.DLL', # Common Dialogue Library
    'CRTDLL.DLL', # Microsoft C Runtime Library
    'DCIMAN.DLL', # Display Control Interface Manager
    'DCIMAN32.DLL', # Display Control Interface Manager
    'DSKMAINT.DLL', # Disk Utilities engine
    'GDI32.DLL', # GDI Client DLL
    'GROUP.DLL', # policy support
    'HYPERTERM.DLL', # Terminal DLL
    'KERNL32.DLL', # Windows NT BASE API Client DLL
    'LZ32.DLL', # LZ Expand/Compress API DLL
    'MAPI.DLL', # Mail / Exchange component
    'MAPI32.DLL', # Extended MAPI 1.0 for Windows NT
    'MFC30.DLL', # Shared MFC DLL
    'MPR.DLL', # Multiple Provider Router DLL
    'MSPST32.DLL', # Microsoft Personal Folder/Address Book Service Provider
    'MSFS32.DLL', # MAPI 1.0 Service Providers for Microsoft Mail
    'MSNDUI.DLL', # Microsoft Network component
    'MSNET32.DLL', # Microsoft 32-bit Network API Library
    'MSSHRUI.DLL', # Shell extensions for sharing
    'MSVIEWUT.DLL', # Service data-link libraries for display engines
    'NAL.DLL', # Network monitor SMS client
    'NDIS30.DLL', # Network monitor SMS client
    'NETAPI.DLL', # Network API
    'NETAPI32.DLL', # Net Win32 API DLL
    'NETBIOS.DLL', # NetBIOS API Library
    'NETDI.DLL', # Net Device installer
    'NETSETUP.DLL', # Network server-based setup
    'NWAB32.DLL', # Address book provider
    'NWNET32.DLL', # NetWare client
    'NWNP32.DLL', # NetWare component
    'OLEDLG.DLL', # Microsoft Windows OLE 2.0 User Interface Support
    'POWERCFG.DLL', # Advanced Power Management Control Panel
    'RASPI.DLL', # Automated Software Profile, Analysis, Removal and Signature Information
    'RASAPI16.DLL', # Remote Access Services 16-bit API Library
    'RASAPI32.DLL', # Remote Access 16-bit API Library
    'RPCRT4.DLL', # Remote Procedure Call Runtime
    'RPCLTC1.DLL', # Remote Procedure Call libraries
    'RPCTLC3.DLL', # Remote Procedure Call libraries
    'RPCTLC5.DLL', # Remote Procedure Call libraries
    'RPCTLC6.DLL', # Remote Procedure Call libraries
    'RPCTLS3.DLL', # Remote Procedure Call libraries
    'RPCTLS5.DLL', # Remote Procedure Call libraries
    'RPCTLS6.DLL', # Remote Procedure Call libraries
    'RPCNS4.DLL', # Remote Procedure Call Name Service Client
    'RSRC32.DLL', # Resource Meter
    'SAPNSP.DLL', # Winsock data-link library
    'SECUR32.DLL', # Security Support Provider Interface
    'SHELL32.DLL', # Windows Shell Common DLL
    'SLENH.DLL', # Advanced Power Management options
    'SHLWAPI.DLL', # Library for UNC and URL Paths, Registry Entries and Color Settings
    'UMDM32.DLL', # Universal Modem Driver component
    'USER32.DLL', # USER API Client DLL
    'VERSION.DLL', # Version Checking and File Installation Libraries
    'WININET.DLL', # Internet Extensions for Win32
    'WINMM.DLL', # MCI API DLL
    'WINREG.DLL', # Remote Registry support
    'WINSOCK.DLL', # Socket API for Windows
    'WS2.DLL', # 32.DLL Windows Socket 2.0 32-Bit DLL
    'WSOCK32.DLL', # Windows Socket 32-Bit DLL
}

__SET_OF_DLLS = set(__LIST_OF_DLLS)

# https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-_image_file_header
__FILE_HEADER = [
    'Machine', # The architecture type of the computer.
    'NumberOfSections', # The number of sections.
    'TimeDateStamp', # The low 32 bits of the time stamp of the image.
    'PointerToSymbolTable', # The offset of the symbol table, in bytes, or zero if no COFF symbol table exists.
    'NumberOfSymbols', # The number of symbols in the symbol table.
    'SizeOfOptionalHeader', # The size of the optional header, in bytes.
    'Characteristics' # The characteristics of the image.
]

# https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-image_optional_header32
__OPTIONAL_HEADER = [
    'Magic', # The state of the image file.
    'MajorLinkerVersion', # The major version number of the linker.
    'MinorLinkerVersion', # The minor version number of the linker.
    'SizeOfCode', # The size of the code section, in bytes, or the sum of all such sections if there are multiple code sections.
    'SizeOfInitializedData', # The size of the initialized data section, in bytes, or the sum of all such sections if there are multiple initialized data sections.
    'SizeOfUninitializedData', # The size of the uninitialized data section, in bytes, or the sum of all such sections if there are multiple uninitialized data sections.
    'AddressOfEntryPoint', # A pointer to the entry point function, relative to the image base address.
    'BaseOfCode', # A pointer to the beginning of the code section, relative to the image base.
    'BaseOfData', # A pointer to the beginning of the data section, relative to the image base.
    'ImageBase', # The preferred address of the first byte of the image when it is loaded in memory.
    'SectionAlignment', # The alignment of sections loaded in memory, in bytes.
    'FileAlignment', # The alignment of the raw data of sections in the image file, in bytes.
    'MajorOperatingSystemVersion', # The major version number of the required operating system.
    'MinorOperatingSystemVersion', # The minor version number of the required operating system.
    'MajorImageVersion', # The major version number of the image.
    'MinorImageVersion', # The minor version number of the image.
    'MajorSubsystemVersion', # The major version number of the subsystem.
    'MinorSubsystemVersion', # The minor version number of the subsystem.
    'Reserved1',  # (Win32VersionValue) This member is reserved and must be 0.
    'SizeOfImage', # The size of the image, in bytes, including all headers.
    'SizeOfHeaders', # The combined size of the following items, rounded to a multiple of the value specified in the FileAlignment member.
    'CheckSum', # The image file checksum.
    'Subsystem', # The subsystem required to run this image.
    'DllCharacteristics', # The DLL characteristics of the image.
    'SizeOfStackReserve', # The number of bytes to reserve for the stack.
    'SizeOfStackCommit', # The number of bytes to commit for the stack.
    'SizeOfHeapReserve', # The number of bytes to commit for the local heap.
    'SizeOfHeapCommit', # This member is obsolete.
    'LoaderFlags', # The number of directory entries in the remainder of the optional header.
    'NumberOfRvaAndSizes' # A pointer to the first IMAGE_DATA_DIRECTORY structure in the data directory.
]

__DIRECTORY_ENTRY_TYPES = {
    'IMAGE_DIRECTORY_ENTRY_EXPORT' : 0,
    'IMAGE_DIRECTORY_ENTRY_IMPORT' : 1,
    'IMAGE_DIRECTORY_ENTRY_RESOURCE' : 2,
    'IMAGE_DIRECTORY_ENTRY_EXCEPTION' : 3,
    'IMAGE_DIRECTORY_ENTRY_SECURITY' : 4,
    'IMAGE_DIRECTORY_ENTRY_BASERELOC' : 5,
    'IMAGE_DIRECTORY_ENTRY_DEBUG' : 6,
    'IMAGE_DIRECTORY_ENTRY_COPYRIGHT' : 7,
    'IMAGE_DIRECTORY_ENTRY_GLOBALPTR' : 8,
    'IMAGE_DIRECTORY_ENTRY_TLS' : 9,
    'IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG' : 10,
    'IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT' : 11,
    'IMAGE_DIRECTORY_ENTRY_IAT' : 12,
    'IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT' : 13,
    'IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR' : 14,
    'IMAGE_DIRECTORY_ENTRY_RESERVED' : 15
}
# https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#section-table-section-headers
__SECTION_HEADER = [
    'VirtualSize',
    'VirtualAddress',
    'SizeOfRawData',
    'PointerToRawData',
    'PointerToRelocations',
    'PointerToLinenumbers',
    'NumberOfRelocations',
    'NumberOfLinenumbers',
    'Characteristics'
]

__RESOURCE_DIRECTORY_TABLE = [
    'Characteristics',
    'MajorVersion',
    'MinorVersion',
    'NumberOfIdEntries',
    'NumberOfNamedEntries',
    'TimeDateStamp'
]

def extract_feature(file_path):
    with open(file_path, 'rb') as f:
        file_data = f.read()
    try:
        pe = pefile.PE(data=file_data)
        ret = dict()
        # Feature From DLLs referred
        for dll in __LIST_OF_DLLS:
            ret[dll] = 0
        if hasattr(pe, 'DIRECTORY_ENTRY_IMPORT'):
            for lib in pe.DIRECTORY_ENTRY_IMPORT:
                if lib.dll != None:
                    if lib.dll.decode().upper() in __SET_OF_DLLS :
                        ret[lib.dll.decode().upper()] = 1
        # Feature From COFF file header
        for member in __FILE_HEADER:
            ret[member] = getattr(pe.FILE_HEADER, member, -1)
        # Feature From Optional header: standard fields and Windows specific fields
        for member in __OPTIONAL_HEADER:
            ret[member] = getattr(pe.OPTIONAL_HEADER, member, -1)
        # Feature From Optional header: data directories
        for key in __DIRECTORY_ENTRY_TYPES.keys():
            ret[f'{key}:VirtualAddress'] = -1
            ret[f'{key}:Size'] = -1
        for structure in pe.OPTIONAL_HEADER.DATA_DIRECTORY:
            ret[f'{structure.name}:VirtualAddress'] = structure.VirtualAddress
            ret[f'{structure.name}:Size'] = structure.Size
        # Feature From Section headers
        for section_name in ['text', 'data', 'resource']:
            for member_name in __SECTION_HEADER :
                ret[f'{section_name}:{member_name}'] = -1
        for section in pe.sections:
            scn = section.Name
            if section.Name == b'.text\x00\x00\x00':
                for member in __SECTION_HEADER:
                    ret[f'text:{member}'] = getattr(section, member, -1)
            elif section.Name == b'.data\x00\x00\x00':
                for member in __SECTION_HEADER:
                    ret[f'data:{member}'] = getattr(section, member, -1)
            elif section.Name == b'.rsrc\x00\x00\x00':
                for member in __SECTION_HEADER:
                    ret[f'resource:{member}'] = getattr(section, member, -1)

        # Feature From Resource directory table & resources
        for resource_type in ['Cursors', 'Bitmaps', 'Icons', 'Menus', 'Dialogs', 'Strings', 'Fonts', 'Group Cursors', 'Group Icons']:
            ret[resource_type] = 0
        for resource_struct in __RESOURCE_DIRECTORY_TABLE:
            ret[f'Resource:{resource_struct}'] = -1
        if hasattr(pe, 'DIRECTORY_ENTRY_RESOURCE'):
            for resource_struct in __RESOURCE_DIRECTORY_TABLE:
                ret[f'Resource:{resource_struct}'] = getattr(pe.DIRECTORY_ENTRY_RESOURCE.struct, resource_struct, -1)
            for resource_type in pe.DIRECTORY_ENTRY_RESOURCE.entries:
                # Cursors
                if resource_type.struct.Id == 1:
                    ret['Cursors'] += len(resource_type.directory.entries)
                # Bitmaps
                elif resource_type.struct.Id == 2:
                    ret['Bitmaps'] += len(resource_type.directory.entries)
                # Icons
                elif resource_type.struct.Id == 3:
                    ret['Icons'] += len(resource_type.directory.entries)
                # Menus
                elif resource_type.struct.Id == 4:
                    ret['Menus'] += len(resource_type.directory.entries)
                # Dialogs
                elif resource_type.struct.Id == 5 :
                    ret['Dialogs'] += len(resource_type.directory.entries)
                # Strings
                elif resource_type.struct.Id == 6:
                    ret['Strings'] += len(resource_type.directory.entries)
                # Fonts
                elif resource_type.struct.Id == 8:
                    ret['Fonts'] += len(resource_type.directory.entries)
                # Group Cursors
                elif resource_type.struct.Id == 12:
                    ret['Group Cursors'] += len(resource_type.directory.entries)
                # Group Icons
                elif resource_type.struct.Id == 14:
                    ret['Group Icons'] += len(resource_type.directory.entries)
        return ret
    except pefile.PEFormatError as pefe:
        print("PEFormatError", pefe, os.path.basename(file_path))
        return None
    except Exception as e:
        print("Exception", e, os.path.basename(file_path))
        return None


""" https://leetcode.com/problems/sum-of-all-odd-length-subarrays/"""

import time
from typing import List

class Solution:
    def sumOddLengthSubarrays(self, arr: List[int]) -> int:
        sum_odd = 0
        n = len(arr)
        for i in range(1, n + 1, 2):
            for j in range(n - i + 1):
                sum_odd += sum(arr[j: j + i])
        return sum_odd

    def sumOddLengthSubarrays2(self, arr: List[int]) -> int:
        res = 0
        n = len(arr)
        for i in range(n):
            res += ((i + 1) * (n - i) + 1) // 2 * arr[i]
        return res

if __name__ == "__main__":
    start_time = time.time()
    print(Solution().sumOddLengthSubarrays([1, 4, 2, 5, 3]))
    print(Solution().sumOddLengthSubarrays([4]))

    print("--- %s seconds ---" % (time.time() - start_time))


import pytest
from time import sleep
from src.collect_functions import collect

# Testing the greetings

@pytest.fixture(name='greeting',scope='module') # Desse modo podemos usar a função greeting mais de uma vez guardando o resultado dela em cache
def greeting():
    greeting = 'hello'
    sleep(3)
    return greeting

class TestCollectInfo(object):
    def test_enter_params(self, greeting):
        actual = collect.collect_info('jon',60,1.70,greeting) # usando função greeting
        expected = 'hello Name: jon, Weight: 60.0, Height: 1.7, IMC: 21'
        assert actual == expected, f'actual:{actual}, expected:{expected}'

    def test_outer(self, greeting):
        actual = collect.collect_info('jon',60,2.0,greeting) # usando dnv
        expected = 'hello Name: jon, Weight: 60.0, Height: 2.0, IMC: 15'
        assert actual == expected, f'actual:{actual}, expected:{expected}'


#/usr/bin/env python

from itertools import product
import sys
import yaml

clangVersions = (
    '3.8',
    '3.7',
    '3.6',
)

perlVersions = (
    '5.24.0',
    '5.22.0',
    '5.20.0',
)

qtVersions = (
    '5.7.0',
    '5.6.0',
    '5.5.0',
    '5.4.0',
    '5.3.0',
    '5.2.0',
)

config = {
    'matrix': {
        'include': []
    }
}

for (clangVersion, qtVersion, perlVersion) in product(clangVersions, qtVersions, perlVersions):
    d = {
        'env': 'CLANG_VERSION={0} QT_VERSION={1} PERL_VERSION={2}'.format(
            clangVersion, qtVersion, perlVersion
        ),
        'addons': {
            'apt': {
                'sources': [
                    'ubuntu-toolchain-r-test',
                    'llvm-toolchain-precise-{0}'.format(clangVersion),
                    'george-edison55-precise-backports',
                ],
                'packages': [
                    'clang-{0}'.format(clangVersion),
                    'libclang-{0}-dev'.format(clangVersion),
                    'llvm-{0}-dev'.format(clangVersion),
                    'cmake',
                    'cmake-data',
                    'libgtest-dev',
                    'libedit-dev',
                ]
            }
        }
    }
    config['matrix']['include'].append(d)

yaml.dump(config, sys.stdout, default_flow_style=False)
print(len(config['matrix']['include']))


from typing import Tuple, List
import numpy as np
from sklearn.ensemble import RandomForestClassifier
import optuna
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold, cross_val_score

def hyper_opt_random_forest(
    Model: RandomForestClassifier,
    X: np.ndarray,
    y: np.ndarray,
    timeout: int,
    seed: int,
    **kwargs,
) -> dict:
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)

    def Objective(trial: optuna.Trial):
        param = {
            "n_estimators": trial.suggest_int("n_estimators", 50, 1000),
            "max_depth": trial.suggest_int("max_depth", 4, 50),
            "min_samples_split": trial.suggest_int("min_samples_split", 1, 150),
            "min_samples_leaf": trial.suggest_int("min_samples_leaf", 1, 60),
            "random_state": trial.suggest_categorical("random_state", [seed]),
        }

        model = Model(**param)
        score = cross_val_score(model, X, y, scoring="roc_auc", cv=skf).mean()

        return score

    study = optuna.create_study(
        direction="maximize",
        study_name="RandomForest optimization",
    )
    study.optimize(Objective, gc_after_trial=True, timeout=timeout)

    model = Model()
    base_score = cross_val_score(model, X, y, scoring="roc_auc", cv=skf).mean()

    hyperopt_score = study.best_value
    best_params = study.best_params

    if base_score > hyperopt_score:
        best_params = {}

    best_params["random_state"] = seed

    model = Model(**best_params)
    model.fit(X, y)

    return best_params


#!/usr/bin/env fslpython

import sys
import subprocess
import vtk
import vtk.util.numpy_support as vtknp
import nibabel.nifti1 as nii
import scipy.ndimage.morphology

print('NB: make_mesh.py assumes radiological orientation!')

infile = sys.argv[1]
outroot = sys.argv[2]
threshold = float(sys.argv[3])
distweight = float(sys.argv[4])
iterations = int(sys.argv[5])

img = nii.Nifti1Image.load(infile)
zooms = img.get_header().get_zooms()
vol = img.get_data()

if distweight == 0.0:
    print('Not using distance weighting')
else:
    print('Using distance weighting')

distmap = scipy.ndimage.morphology.distance_transform_edt(vol < 95)
vol *= distweight * distmap ** 2 / 100 + 1

imd = vtk.vtkImageData()
imd.SetDimensions(vol.shape[0], vol.shape[1], vol.shape[2])
imd.SetSpacing(zooms[0], zooms[1], zooms[2])
imd.GetPointData().SetScalars(vtknp.numpy_to_vtk(vol.flatten('F'), deep = 1))

mcf = vtk.vtkImageMarchingCubes()
mcf.SetInputData(imd)
mcf.SetValue(0, threshold)
mcf.Update()

writer = vtk.vtkPolyDataWriter()
writer.SetInputData(mcf.GetOutput())
writer.SetFileName(outroot + '_orig_mesh.vtk')
writer.SetFileTypeToASCII()
writer.Write()

ndt = scipy.ndimage.morphology.distance_transform_edt(vol > threshold)
ndtimg = nii.Nifti1Image(ndt, img.get_affine())
ndtimg.to_filename(outroot + '_distmap')

subprocess.call("run_mesh_utils --doDeformSurface -i {0} -m {1} -o {2} --w_im=-0.2 --w_tan=1.0 --w_tri=1.0 --w_norm=0.0 -d {3} -t 0.2".format(
        outroot + '_distmap', outroot + '_orig_mesh.vtk', outroot + '_deformed_mesh.vtk', iterations), shell = True)


import os
import os.path
import pickle
from copy import copy

os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

import numpy as np
import matplotlib.pyplot as plt
import rosbag
from glob import glob

import pinocchio as pino
from scipy.signal import butter, filtfilt
from scipy.fft import fft
from scipy.interpolate import interp1d
from scipy.spatial.transform import Rotation as Rot

from manifold_planning.utils.manipulator import Iiwa
from manifold_planning.utils.feasibility import check_if_plan_valid, compute_cartesian_losses

from manifold_planning.utils.constants import Table1, Table2, Cup, Limits

root_dir = os.path.dirname(__file__)
package_dir = os.path.dirname(root_dir)

robot_file = os.path.join(root_dir, "manifold_planning", "iiwa_cup.urdf")
pino_model = pino.buildModelFromUrdf(robot_file)
pino_data = pino_model.createData()
joint_id = pino_model.getFrameId("F_link_cup")

package_path = os.path.join(os.path.dirname(__file__), "..")
file_name = os.path.join(package_path, "data/kinodynamic7fixed_12kg_validated/test/data.tsv")
data = np.loadtxt(file_name, delimiter="\t").astype(np.float32)#[:3]

def compute_object_coordinates(xyz, R):
    h = Cup.height
    w = Cup.width
    xyz_cuboid = np.array([
        # corners
        [w, w, h], [w, w, -h], [w, -w, h], [w, -w, -h],
        [-w, w, h], [-w, w, -h], [-w, -w, h], [-w, -w, -h],
        ## middle points on the edges
        # [w, w, 0], [w, -w, 0], [-w, w, 0], [-w, -w, 0],
        # [w, 0, h], [w, 0, -h], [-w, 0, h], [-w, 0, -h],
        # [0, w, h], [0, w, -h], [0, -w, h], [0, -w, -h],
        ## middle points on the faces
        # [w, 0, 0], [-w, 0, 0],
        # [0, w, 0], [0, -w, 0],
        # [0, 0, h], [0, 0, -h],
    ])[np.newaxis, :, :, np.newaxis]
    xyz_object = xyz[:, np.newaxis] + (R[:, np.newaxis] @ xyz_cuboid)[..., 0]
    return xyz_object

def cal_d(fs, dts):
    dfs = np.zeros_like(fs)
    dfs[1: -1] = (fs[2:, :] - fs[:-2, :]) / (dts[2:] - dts[:-2])
    dfs[0] = dfs[1]
    dfs[-1] = dfs[-2]
    return dfs

def rnea(q, dq, ddq):
    taus = []
    for i in range(len(q)):
        taus.append(pino.rnea(pino_model, pino_data, q[i], dq[i], ddq[i]))
    return np.array(taus)

def get_vel_acc(t, q_m):
    # Position
    bv, av = butter(6, 40, fs=1000)
    b, a = butter(6, 40, fs=1000)
    bt, at = butter(6, 30, fs=1000)
    # bv, av = butter(6, 4, fs=1000)
    # b, a = butter(6, 4, fs=1000)
    q_m_filter = q_m

    # Velocity
    dq_m = cal_d(q_m, t)
    dq_m_filter = filtfilt(bv, av, dq_m.copy(), axis=0)

    # Acceleration
    ddq_m = cal_d(dq_m_filter, t)
    ddq_m_filter = filtfilt(b, a, ddq_m.copy(), axis=0)

    torque = rnea(q_m.astype(np.float32), dq_m.astype(np.float32), ddq_m.astype(np.float32))
    # Torque
    tau_m = torque
    tau_m_filter = filtfilt(bt, at, tau_m.copy(), axis=0)

    return q_m_filter, dq_m_filter, ddq_m_filter, tau_m_filter, q_m, dq_m, ddq_m, tau_m

def compute_vel_acc_tau(t, q, qd, qd_dot, qd_ddot):
    desired_torque = rnea(qd, qd_dot, qd_ddot)
    q, dq, ddq, torque, q_, dq_, ddq_, torque_ = get_vel_acc(t[:, np.newaxis], q)
    return q, dq, ddq, torque, qd, qd_dot, qd_ddot, desired_torque

def forwardKinematics(q):
    xyz = []
    R = []
    for i in range(len(q)):
        pino.forwardKinematics(pino_model, pino_data, q[i])
        pino.updateFramePlacements(pino_model, pino_data)
        xyz_pino = pino_data.oMf[joint_id].translation
        R_pino = pino_data.oMf[joint_id].rotation
        xyz.append(copy(xyz_pino))
        R.append(copy(R_pino))
    xyz = np.stack(xyz)
    R = np.stack(R)
    return xyz, R

def compute_vertical_constraints(q):
    xyz, R = forwardKinematics(q)
    vertical_loss = 1. - R[:, 2, 2]
    angles = []
    for i in range(len(R)):
        angles.append(Rot.from_matrix(R[i]).as_euler("xyz"))
    return vertical_loss, np.array(angles)

def if_valid_box_constraints(q, zh1, zh2):
    allowed_violation = 0.02
    xyz, R = forwardKinematics(q)

    def dist2box(xyz, xl, yl, zl, xh, yh, zh):
        l = np.stack([xl, yl, zl], axis=-1)
        h = np.stack([xh, yh, zh], axis=-1)
        xyz_dist = np.max(np.stack([l - xyz, np.zeros_like(xyz), xyz - h], axis=-1), axis=-1)
        dist = np.sqrt(np.sum(np.square(xyz_dist), axis=-1) + 1e-8)
        return dist

    def inside_box(xyz, xl, yl, zl, xh, yh, zh):
        pxl = xyz[..., 0] > xl + allowed_violation
        pxh = xyz[..., 0] < xh - allowed_violation
        pyl = xyz[..., 1] > yl + allowed_violation
        pyh = xyz[..., 1] < yh - allowed_violation
        pzl = xyz[..., 2] > zl + allowed_violation
        pzh = xyz[..., 2] < zh - allowed_violation
        return np.all(np.stack([pxl, pxh, pyl, pyh, pzl, pzh], axis=-1), axis=-1)

    robot_dist2box_1 = dist2box(xyz, Table1.xl, Table1.yl, -1e10, Table1.xh, Table1.yh, zh1 - Cup.height)
    robot_dist2box_2 = dist2box(xyz, Table2.xl, Table2.yl, -1e10, Table2.xh, Table2.yh, zh2 - Cup.height)
    inside_box_1 = inside_box(xyz, Table1.xl, Table1.yl, -1e10, Table1.xh, Table1.yh, zh1 - Cup.height)
    inside_box_2 = inside_box(xyz, Table2.xl, Table2.yl, -1e10, Table2.xh, Table2.yh, zh2 - Cup.height)

    xyz_object = compute_object_coordinates(xyz, R)

    object_inside_box_1 = inside_box(xyz_object, Table1.xl, Table1.yl, -1e10, Table1.xh, Table1.yh,
                                     zh1 - Cup.height)
    object_inside_box_2 = inside_box(xyz_object, Table2.xl, Table2.yl, -1e10, Table2.xh, Table2.yh,
                                     zh2 - Cup.height)

    return not np.any(inside_box_1) and not np.any(inside_box_2) and \
           np.all(robot_dist2box_1 > 0.15 - allowed_violation) and \
           np.all(robot_dist2box_2 > 0.15 - allowed_violation) and \
           not np.any(object_inside_box_1) and not np.any(object_inside_box_2)

def read_bag(bag):
    time = []
    joint_state_desired = []
    joint_state_actual = []
    joint_state_error = []
    planning_time = None
    success = None
    planned_motion_time = None

    for topic, msg, t_bag in bag.read_messages():
        if topic in ["/bspline_ff_kino_joint_trajectory_controller/state"]:
            n_joints = len(msg.joint_names)
            time.append(msg.header.stamp.to_sec())
            joint_state_desired.append(np.concatenate(
                [msg.desired.positions, msg.desired.velocities, msg.desired.accelerations, msg.desired.effort]))
            joint_state_actual.append(np.concatenate(
                [msg.actual.positions, msg.actual.velocities, msg.actual.accelerations, msg.actual.effort]))
            joint_state_error.append(np.concatenate(
                [msg.error.positions, msg.error.velocities, msg.error.accelerations, msg.error.effort]))
        elif topic == "/neural_planner/status":
            planning_time = msg.planning_time
            planned_motion_time = msg.planned_motion_time
            success = msg.success
    desired = np.array(joint_state_desired).astype(np.float32)
    actual = np.array(joint_state_actual).astype(np.float32)
    error = np.array(joint_state_error).astype(np.float32)
    t = np.array(time)
    t -= t[0]
    t = t.astype(np.float32)
    return dict(t=t, actual=actual, desired=desired, error=error, planning_time=planning_time, success=success,
                planned_motion_time=planned_motion_time)

def compute_metrics(bag_path):
    # bag_path = os.path.join(package_dir, "bags", bag_path)
    # bag_path = os.path.join(package_dir, "bags/ours_nn/K9.bag")
    bag_file = rosbag.Bag(bag_path)
    i = int(bag_path.split("/")[-1][:-4])
    data_i = data[i]
    zh1 = data_i[16]
    zh2 = data_i[19]

    result = {}

    bag_dict = read_bag(bag_file)
    q, dq, ddq, torque, qd, qd_dot, qd_ddot, torqued = compute_vel_acc_tau(bag_dict["t"],
                                                                           bag_dict["actual"][:, :7],
                                                                           bag_dict["desired"][:, :7],
                                                                           bag_dict["desired"][:, 7:14],
                                                                           bag_dict["desired"][:, 14:21])
    #plt.plot(bag_dict["t"], qd_dot[..., :2], 'r')
    # plt.plot(bag_dict["t"], dq[..., :2], 'm')
    moving = np.linalg.norm(qd_dot, axis=-1) > 5e-2
    if not np.any(moving):
        result["valid"] = 0
        result["finished"] = 0
        result["planning_time"] = bag_dict["planning_time"]
        return result
    q = q[moving]
    dq = dq[moving]
    ddq = ddq[moving]
    torque = torque[moving]
    qd = qd[moving]
    qd_dot = qd_dot[moving]
    qd_ddot = qd_ddot[moving]
    torqued = torqued[moving]
    t = bag_dict["t"][moving]

    #plt.plot(t, qd_dot[..., :2], 'b')
    #plt.show()

    vertical_loss, angles = compute_vertical_constraints(q)

    box_constraints = if_valid_box_constraints(q, zh1, zh2)
    vertical_constraints = np.all(vertical_loss < 0.05)
    alpha_beta = np.sum(np.abs(angles[:, :2]), axis=-1)

    torque_limits = 1.0 * np.array([320, 320, 176, 176, 110, 40, 40], dtype=np.float32)[np.newaxis]
    q_dot_limits = 1.0 * np.array([1.4835, 1.4835, 1.7453, 1.3090, 2.2689, 2.3562, 2.3562], dtype=np.float32)[np.newaxis]
    q_ddot_limits = 10. * q_dot_limits
    torque_constraints = np.all(np.abs(torque) < torque_limits)
    dq_constraints = np.all(np.abs(dq) < q_dot_limits)
    ddq_constraints = np.all(np.abs(ddq) < q_ddot_limits)
    #for i in range(7):
    #    plt.subplot(331 + i)
    #    plt.plot(np.abs(dq[:, i]))
    #    plt.plot([0, dq.shape[0]], [q_dot_limits[0, i], q_dot_limits[0, i]])
    #plt.show()
    #for i in range(7):
    #    plt.subplot(331 + i)
    #    plt.plot(np.abs(ddq[:, i]))
    #    plt.plot([0, ddq.shape[0]], [q_ddot_limits[0, i], q_ddot_limits[0, i]])
    #plt.show()
    #for i in range(7):
    #    plt.subplot(331 + i)
    #    plt.plot(np.abs(torque[:, i]))
    #    plt.plot([0, torque.shape[0]], [torque_limits[0, i], torque_limits[0, i]])
    #plt.show()

    #dist_2_goal = np.linalg.norm(data_i[7:14] - q[-1])
    #finished = dist_2_goal < 0.2
    xyz_final, _ = forwardKinematics([q[-1]])
    xyz_desired, _ = forwardKinematics([data_i[7:14]])
    dist_2_goal = np.linalg.norm(xyz_final[0] - xyz_desired[0])
    finished = dist_2_goal < 0.2
    #valid = box_constraints and vertical_constraints and torque_constraints and dq_constraints and ddq_constraints and finished
    valid = box_constraints and vertical_constraints and torque_constraints and dq_constraints and finished
    #valid = box_constraints and vertical_constraints and finished

    ee, _ = forwardKinematics(q)
    ee_d, _ = forwardKinematics(qd)

    # plt.plot(ee[..., 0], ee[..., 1], 'r')
    # plt.plot(ee_d[..., 0], ee_d[..., 1], 'g')
    # plt.plot(bag_dict['puck_pose'][:, 0], bag_dict['puck_pose'][:, 1], 'b')
    # plt.show()

    #for i in range(6):
    #   plt.subplot(321 + i)
    #   plt.plot(t, q[..., i], 'r')
    #   plt.plot(t, qd[..., i], 'g')
    #plt.show()

    # for i in range(6):
    #    plt.subplot(321 + i)
    #    plt.plot(t, np.abs(q[..., i] - qd[..., i]))
    #    plt.plot(t, np.linalg.norm(ee_d[..., :2] - ee[..., :2], axis=-1))
    # plt.show()

    movement_time = t[-1] - t[0]

    dt = np.diff(t)
    integral = lambda x: np.sum(np.abs(x)[1:] * dt)
    # reduce_integral = lambda x: integral(np.sum(np.abs(x), axis=-1))
    reduce_integral = lambda x: integral(np.linalg.norm(x, axis=-1))

    result["valid"] = int(valid)
    result["finished"] = int(finished)
    result["vertical_loss"] = integral(vertical_loss)
    result["planning_time"] = bag_dict["planning_time"]
    result["motion_time"] = movement_time

    result["alpha_beta"] = integral(alpha_beta)
    result["joint_trajectory_error"] = reduce_integral(qd - q)
    result["cartesian_trajectory_error"] = reduce_integral(ee_d - ee)

    print(result)
    return result

#planners = ["ours", "nlopt", "sst", "cbirrt", "mpcmpnet"]
#planners = ["nlopt", "sst", "cbirrt", "mpcmpnet"]
#planners = ["ours"]
#planners = ["ours_long"]
#planners = ["cbirrt"]
#planners = ["nlopt"]
#planners = ["mpcmpnet"]
#planners = ["sst"]
#planners = ["ours_n10"]
#planners = ["ours_n20"]
#planners = ["ours_l256", "ours_l512", "ours_l1024"]
#planners = ["ours_l64"]
#planners = ["ours_l128"]
#planners = ["ours_l128_long", "ours_l256_long", "ours_l3072_long"]
#planners = ["ours_l1024_long"]
#planners = ["ours_l512_long"]
#planners = ["ours_l64_long"]
#planners = ["ours_l2048s"]
#planners = ["ours_10kg"]
#planners = ["ours_12kg"]
planners = ["ours_13kg"]
#planners = ["ours_16kg"]
#planners = ["ours_18kg"]
#planners = ["ours_l2048_long"]
package_dir = "/home/piotr/b8/ah_ws/data"
for planner in planners:
    print(planner)
    dir_path = os.path.join(package_dir, "kino_exp", planner)
    sp = dir_path.replace("data", "results")
    os.makedirs(sp, exist_ok=True)
    for i, p in enumerate(glob(os.path.join(dir_path, "*.bag"))):
    #for i, p in enumerate([os.path.join(dir_path, f"{str(x).zfill(3)}.bag") for x in [16, 23, 24, 42, 44, 67, 72]]):
    #for i, p in enumerate([os.path.join(dir_path, f"{str(x).zfill(3)}.bag") for x in [1, 12, 16, 24, 39, 42, 44, 65, 74, 91]]):
    #for i, p in enumerate([os.path.join(dir_path, f"{str(x).zfill(3)}.bag") for x in [0, 58]]):
    #for i, p in enumerate([os.path.join(dir_path, f"{str(x).zfill(3)}.bag") for x in [24, 44, 53, 80, 93, 94]]):
    #for i, p in enumerate(glob(os.path.join(dir_path, "00*.bag"))):
    #for i, p in enumerate(glob(os.path.join(dir_path, "*42.bag"))):
    #for i, p in enumerate([os.path.join(dir_path, f"{str(x).zfill(3)}.bag") for x in [0, 6, 16, 21, 23, 24, 42, 44, 65, 66, 67, 72, 92]]):
        print(i)
        d = compute_metrics(p)
        save_path = p[:-3] + "res"
        save_path = save_path.replace("data", "results")
        with open(save_path, 'wb') as fh:
            pickle.dump(d, fh)


import os, time
from colorama import Fore

try:
    from selenium import webdriver
except ImportError:
    print("• Installing selenium...")
    os.system("pip install --upgrade selenium >nul")
    print("• Selenium installed.")
    from selenium import webdriver

cwd = os.getcwd() # Gets current working directory path
browser = ""

while True:
    browser = input(f"{Fore.LIGHTYELLOW_EX}Enter browser (chrome / edge): {Fore.RESET}").lower()
    if browser == "edge" or browser == "chrome":
        print(f"Browser set to {Fore.LIGHTGREEN_EX}%s{Fore.RESET}" % browser.capitalize())
        break

if browser == 'chrome':
    driver_path = cwd + "\\chromedriver.exe"
else:
    driver_path = cwd + "\\edgedriver.exe"

if os.path.exists(driver_path): # Checks if a chromedriver is installed
    print(f"{Fore.LIGHTMAGENTA_EX}Created by Neek8044{Fore.RESET}\n")
    print("Make sure Chrome / Edge is installed with the latest update\n")
    video_url = input("Video URL: ")

    Loop = False
    while Loop:
        try:
            duration = int(input("Video Duration (seconds): ")) - 3
            Loop = True
        except:
            continue

    if browser.lower() == 'chrome':
        # Configuring drivers to work with C:\chromedriver.exe
        driver_1 = webdriver.Chrome(executable_path = cwd + "\\chromedriver.exe")
        driver_2 = webdriver.Chrome(executable_path = cwd + "\\chromedriver.exe")
        driver_3 = webdriver.Chrome(executable_path = cwd + "\\chromedriver.exe")
        driver_4 = webdriver.Chrome(executable_path = cwd + "\\chromedriver.exe")
    else:
        # Configuring drivers to work with C:\chromedriver.exe
        driver_1 = webdriver.Edge(executable_path = cwd + "\\edgedriver.exe")
        driver_2 = webdriver.Edge(executable_path = cwd + "\\edgedriver.exe")
        driver_3 = webdriver.Edge(executable_path = cwd + "\\edgedriver.exe")
        driver_4 = webdriver.Edge(executable_path = cwd + "\\edgedriver.exe")

    # Starts drivers
    driver_1.get(video_url)
    driver_2.get(video_url)
    driver_3.get(video_url)
    driver_4.get(video_url)

    while True:
        time.sleep(duration)
        driver_1.refresh()
        driver_2.refresh()
        driver_3.refresh()
        driver_4.refresh()

else: # If no driver can be found, show error.
    print(f"{Fore.LIGHTRED_EX}ERROR: No driver could be found in the current working directory.\nLaunch installer and restart the program.")

input(f"\n\n{Fore.LIGHTBLUE_EX}Press enter key to exit...{Fore.RESET}") # Keep the screen alive by waiting for any input


from BO_interfaces.BO import *
import os
import argparse

# Command to run VASP executable.
VASP_RUN_COMMAND = 'mpirun -np 56 vasp_ncl'
# Define the name for output file.
OUTFILENAME = 'vasp.out'
# Define the path direct to the VASP pesudopotential.
VASP_PP_PATH = '/home/smoayedp/pseudopotential'

def parse_argument():
    """
    bounds: The parameter that determines the range of shift in
            Cartetian coordinates in (x,y,z) format

    kappa: The parameter to control exploration and exploitation.
           exploitation 0 <-- kappa --> 10 exploration

    threshold: Convergence threshold of Bayesian optimization process.
    """
    parser = argparse.ArgumentParser(description='params')
    parser.add_argument('--bounds', dest='bounds',
                        type=tuple, default=[2, 2, 2])
    parser.add_argument('--kappa', dest='kappa', type=float, default=4)
    parser.add_argument('--threshold', dest='threshold',
                        type=float, default=0.05)
    parser.add_argument('--import_kpath', dest='import_kpath',
                        type=bool, default=False)

    return parser.parse_args()

def main():
    args = parse_argument()
    k = args.kappa
    br = args.br
    import_kpath = args.import_kpath
    os.environ['VASP_PP_PATH'] = VASP_PP_PATH
    if os.path.exists('POSCAR_shifted'):
        os.remove('POSCAR_shifted')

    calculate(command=VASP_RUN_COMMAND, outfilename=OUTFILENAME,
              method='dftu', import_kpath=import_kpath)
    outcar = open("dftu/scf/OUTCAR")
    for l in outcar.readlines():
        if "energy without entropy" in l:
            energy = float(l.split()[7])
    with open("BO_result.txt", "w") as res_file:
        print(0, 0, 0, energy, file=res_file)

    threshold = args.threshold
    for i in range(100):
        if os.path.exists('POSCAR_shifted'):
            os.rename('POSCAR_shifted', 'POSCAR_shifted_' + str(i))

        bayesianOpt = bayesOpt_DFTU(path='./')
        obj_next = bayesianOpt.bo()
        if i == 0:
            obj_ref = obj_next
        else:
            if (np.abs(obj_next[0] - obj_ref[0]) < threshold) and (np.abs(obj_next[1] - obj_ref[1]) < threshold) and \
                    (np.abs(obj_next[2] - obj_ref[2]) < threshold):
                break
            obj_ref = obj_next

        calculate(command=VASP_RUN_COMMAND, outfilename=OUTFILENAME,
                  method='dftu', import_kpath=import_kpath)
        outcar = open("dftu/scf/OUTCAR")
        for l in outcar.readlines():
            if "energy without entropy" in l:
                energy = float(l.split()[7])
        with open("BO.txt", "a") as res_file:
            print(energy, file=res_file)

if __name__ == "__main__":
    main()


from django.urls import path, include
from apps.viajeros.views import index,viajeroCreate,viajeroEdit,viajeroDelete
app_name = "viajeros";

urlpatterns = [
    path('', index, name='index'),
    path('nuevo/', viajeroCreate, name='viajeroCreate'),
    path('actualizar/<int:id_viajero>/', viajeroEdit, name='viajeroEdit'),
    path('eliminar/<int:id_viajero>/', viajeroDelete, name='viajeroDelete'),
]


import json
#coding=utf-8
import time
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
import logging
import os
# Import WebClient from Python SDK (github.com/slackapi/python-slack-sdk)
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError

class WebDriver(object):

    def __init__(self):
        self.options = Options()

        self.options.binary_location = '/opt/headless-chromium'
        self.options.add_argument('--headless')
        self.options.add_argument('--no-sandbox')
        self.options.add_argument('--start-maximized')
        self.options.add_argument('--start-fullscreen')
        self.options.add_argument('--single-process')
        self.options.add_argument('--disable-dev-shm-usage')

    def get(self):
        driver = Chrome('/opt/chromedriver', options=self.options)
        return driver

def lambda_handler(event, context):
	instance_ = WebDriver()
	driver = instance_.get()
	options = Options()
	a = os.listdir('/tmp')
	for x in a:
		print(x)
	URL = os.environ.get("URL")

	driver.get(URL)

	S = lambda X: driver.execute_script('return document.body.parentNode.scroll'+X)
	driver.set_window_size(S('Width'),S('Height'))
	#driver.set_window_size(1920, 1080)
	# May need manual adjustment
	driver.find_element_by_tag_name('body').screenshot('/tmp/web_screenshot.png')

	driver.quit()
	# WebClient instantiates a client that can call API methods
	# When using Bolt, you can use either `app.client` or the `client` passed to listeners.
	client = WebClient(token=os.environ.get("SLACK_BOT_TOKEN"))
	logger = logging.getLogger(__name__)
	# The name of the file you're going to upload
	file_name = "/tmp/web_screenshot.png"
	# ID of channel that you want to upload file to
	channel_id= os.environ.get("channel_id")

	#client.chat_postMessage(channel=channel_id, text="hello")
	try:
		# Call the files.upload method using the WebClient
		# Uploading files requires the `files:write` scope
		result = client.files_upload(
			channels=channel_id,
			initial_comment="Ops-Analytics Job Status: <"+ URL +"|Ops360 link>",
			file=file_name,
		)
		# Log the result
		logger.info(result)

	except SlackApiError as e:
		logger.error("Error uploading file: {}".format(e))

		return {
			'statusCode': 200,
			'body': json.dumps('Hello from Lambda!')
		}


from datetime import datetime
from unittest import TestCase
from django.http import HttpRequest
from .models import Article, add_article

class TestArticleModel(TestCase):
    def setUp(self):
        self.data = (
            ('hello', ['ni hao me']),
            ('hello2', ['ni hao me']),
        )
        self.ids = [
            'nyduwGYBt6UyLsIJHvAf',
            'iidtwGYBt6UyLsIJq_CJ',
            'hydxwGYBt6UyLsIJz_GU',
        ]

    def init_test_data(self):
        pass

    def test_add(self):
        id = 0
        for title, tags in self.data:
            article = Article(meta={'id': self.ids[id]}, title=title, tags=tags)
            article.body = ''' looong text xxxxxx xxxx '''
            article.published_from = datetime.now()
            article.save()
            id += 1

        for i in range(2):
            article = Article.get(id=self.ids[i])
            assert article.title == self.data[i][0]
            assert article.tags == self.data[i][1]

    def test_mget(self):
        ret = Article.mget(self.ids)
        assert len(ret) == len(self.ids)

    def test_search_body(self):
        assert 1 == 1

    def tearDown(self):
        self.data = None


#!/usr/bin/python
# -*- coding: utf-8 -*-
import logging
import tornado

class BaseHandler(tornado.web.RequestHandler):
    """ The base handlers for all handlers.
    """
    def _handle_request_exception(self, exception):
        """ Error handler of the application.

        Args:
            self: the BaseHandler itself
            exception: the exception pass to this handler
        """
        exception_type_name = type(exception).__name__
        if exception_type_name == 'HTTPError':
            self.set_status(exception.status_code)
            logging.warn('HTTPError[Code=%d] occurred: %s.' % \
                (exception.status_code, exception.log_message))
        else:
            self.set_status(500)
            logging.error('Exception occurred: %s' % exception)

        self.render('error.html', status_code=self.get_status())

    def get_user_locale(self):
        """ Get language settings of users.

        Args:
            self: the BaseHandler itself
        """
        user_locale = self.get_cookie('user_locale')

        if not user_locale:
            # Fallback to browser based locale detection
            user_locale = self.get_browser_locale().code
            self.set_cookie('user_locale', user_locale)

        return tornado.locale.get(user_locale)