from library.argparse import argparse
import importlib

def argparse_cli(request_uri=["--uri", "-u"]) -> list:
    parser = argparse.ArgumentParser()

    try:
        # 获取uri
        parser.add_argument("--uri", "-u", required=True, help="request uri", type=str)
        args = parser.parse_known_args()[0]
        uri = args.uri
        uri_list = uri.strip("/").split("/")

        if len(uri_list) != 3:
            raise Exception("request uri error")

        module, controller, method = uri_list
    except Exception as e:
        print("analysis request uri error: uri={}".format(uri))
        exit()

    try:
        # 验证参数
        params_path = "config.params.{}.{}".format(module, controller)
        params_file = importlib.import_module(params_path)
        params_info = getattr(params_file, "param_check")[method]
        for field, param in params_info.items():
            field = "--" + field
            parser.add_argument(field, **param)

        args = parser.parse_args()
    except Exception as e:
        print("check params error:")
        exit()

    args = vars(args)
    del args["uri"]

    return uri, args


'''
 SDD TRAJECTORY PREDICTION TRAINING
 MADE BY DOOSEOP CHOI (d1024.choi@etri.re.kr)
 DATE : 2019-12-06
'''

import argparse
import os
import pickle
import time

from sdd_model import Model
from sdd_utils import DataLoader
from sdd_functions import *

def print_training_info(args):

    print('-----------------------------------------------------------')
    print('------------ Encoder-Decoder LSTM INFORMATION -------------')
    print('.dataset_num %d, exp id %d' %(args.dataset_num, args.exp_id))
    print('.network structure: LSTM')
    print('   rnn size (%d)' % (args.rnn_size))
    print('   num layers (%d)' % (args.num_layers))
    print('.training setting')
    print('   num epochs (%d)' % args.num_epochs)
    print('   batch size (%d)' % args.batch_size)
    print('   obs_length (%d)' % args.obs_length)
    print('   pred_length (%d)' % args.pred_length)
    print('   map size (%d)' % args.map_size)
    print('   learning rate (%.5f)' % args.learning_rate)
    print('   reg. lambda (%.5f)' % args.lambda_param)
    print('   gamma param (%.5f)' % args.gamma_param)
    print('   grad_clip (%.2f)' % args.grad_clip)
    print('   keep prob (%.2f)' % args.keep_prob)
    print('   load pretrained (%d)' % args.load_pretrained)
    print('   start epoch (%d)' % args.start_epoch)
    print('   data load step (%d)' % args.data_load_step)
    print('------------------------------------------------------------')

def main():
    parser = argparse.ArgumentParser()

    # # network structure : LSTM
    parser.add_argument('--rnn_size', type=int, default=256, help='size of RNN hidden state')
    parser.add_argument('--num_layers', type=int, default=1, help='number of layers in the RNN')
    parser.add_argument('--model', type=str, default='lstm', help='rnn, gru, or lstm')
    parser.add_argument('--input_dim', type=int, default=2, help='dimension of input vector')

    # # training setting
    parser.add_argument('--dataset_path', type=str, default='./dataset', help='dataset path')
    parser.add_argument('--exp_id', type=int, default=0, help='dataset id')
    '''
    dataset number | random seed
    -----------------------------
           0       |      2
    -----------------------------
           1       |      2
    -----------------------------
           2       |      2
    -----------------------------
           3       |      2
    -----------------------------
           4       |      2
    '''
    parser.add_argument('--dataset_num', type=int, default=0, help='dataset number')
    parser.add_argument('--gpu_num', type=int, default=2, help='gpu device id utilized for training and validation')
    parser.add_argument('--load_pretrained', type=int, default=0, help='want to load pre-trained network?')
    parser.add_argument('--batch_size', type=int, default=4, help='minibatch size')
    parser.add_argument('--obs_length', type=int, default=8, help='observation sequence length')
    parser.add_argument('--pred_length', type=int, default=12, help='prediction sequence length')
    parser.add_argument('--num_epochs', type=int, default=300, help='number of epochs')
    parser.add_argument('--model_dir', type=str, default='save', help='directory to save model to')
    parser.add_argument('--grad_clip', type=float, default=1.5, help='clip gradients at this value')
    parser.add_argument('--learning_rate', type=float, default=0.0001, help='learning rate')
    parser.add_argument('--lambda_param', type=float, default=0.0001, help='l2-regularization weight')
    parser.add_argument('--gamma_param', type=float, default=0.001, help='IRL regularization weight')
    parser.add_argument('--keep_prob', type=float, default=0.8, help='dropout keep probability')
    parser.add_argument('--patient_thr', type=float, default=100, help='threshold for early stopping')
    parser.add_argument('--data_load_step', type=int, default=3, help='data_load_step for training')
    parser.add_argument('--start_epoch', type=int, default=0, help='start epoch value')
    parser.add_argument('--min_avg_loss', type=float, default=100000.0, help='min avg loss')

    # # map info
    parser.add_argument('--map_size', type=int, default=96, help='width of map image')

    args = parser.parse_args()
    train(args)

def train(args):

    # print training information
    print_training_info(args)

    # assign gpu device
    if (args.gpu_num == 0):
        os.environ["CUDA_VISIBLE_DEVICES"] = "0"
    elif (args.gpu_num == 1):
        os.environ["CUDA_VISIBLE_DEVICES"] = "1"
    elif (args.gpu_num == 2):
        os.environ["CUDA_VISIBLE_DEVICES"] = "2"
    elif (args.gpu_num == 3):
        os.environ["CUDA_VISIBLE_DEVICES"] = "3"

    # training data preparation (utils.py)
    data_loader = DataLoader(args)

    # check if there is pre-trained network
    args.model_dir = args.model_dir + '_' + str(args.dataset_num) + '_' + str(args.exp_id)
    if args.model_dir != '' and not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)
        args.load_pretrained = 0

    # load saved training parameters or save current parameters
    start_epoch = args.start_epoch
    if (args.load_pretrained == 1):
        with open(os.path.join(args.model_dir, 'config.pkl'), 'rb') as f:
            args = pickle.load(f)
            args.load_pretrained = 1
    else:
        with open(os.path.join(args.model_dir, 'config.pkl'), 'wb') as f:
            pickle.dump(args, f)

    # network model definition (model.py)
    model = Model(args)

    with tf.Session() as sess:

        # initialize network
        tf.global_variables_initializer().run()
        saver = tf.train.Saver(tf.global_variables())

        # load the latest trained parameters
        if (args.load_pretrained == 1):
            ckpt = tf.train.get_checkpoint_state(args.model_dir)
            saver.restore(sess, ckpt.model_checkpoint_path)
            print(">> loaded model: ", ckpt.model_checkpoint_path)

        best_epoch = 0
        patient = 0
        min_avg_loss = 10000.0
        if (args.load_pretrained == 1):
            min_avg_loss = args.min_avg_loss
        print('>> training and validation start from epoch %d  with min_avg_loss %.4f' % (start_epoch, min_avg_loss))

        # start training and validation !!!!
        for e in range(start_epoch, args.num_epochs):

            state = model.init_state_enc.eval()

            rand_list = [x for x in range(data_loader.num_train_scenes)]
            random.shuffle(rand_list)

            # # Train one epoch -------------------------------------------------------
            printProgressBar(0, data_loader.num_batches - 1, prefix='Progress:', suffix='Complete', length=50, epoch=e)
            start = time.time()

            for b in range(data_loader.num_batches):

                # load mini-batch
                xo, xp, xoo, xpo, did = data_loader.next_batch(rand_list[b * args.batch_size:(b + 1) * args.batch_size])
                mo = make_map_batch(xo, did, data_loader.map, args.map_size)

                # step 1) simulate current policy
                feed = {model.gt_traj_enc: xoo,
                        model.gt_map_enc: mo,
                        model.init_state_enc: state,
                        model.output_keep_prob: 1.0}

                xoo_policy = reshape_est_traj(sess.run([model.predictions_enc], feed), args.batch_size, args.obs_length)
                mo_policy = make_map_batch_for_policy(xo, xoo, xoo_policy, did, data_loader.map, args.map_size)

                # step2) train rnn
                feed = {model.gt_traj_enc: xoo,
                        model.gt_traj_dec: xpo,
                        model.gt_map_enc: mo,
                        model.policy_map_enc: mo_policy,
                        model.init_state_enc: state,
                        model.output_keep_prob: args.keep_prob}
                train_loss, _ = sess.run([model.cost_pos_dec, model.train_op_pose], feed)

                # step3) train reward
                train_loss, _ = sess.run([model.cost_reward, model.train_op_reward], feed)

                printProgressBar(b, data_loader.num_batches - 1, prefix='Progress:', suffix='Complete', length=50, epoch=e)
            end = time.time()

            # # Validation ------------------------------------------------------------
            valid_loss_list = []
            state = model.init_state_enc.eval()
            traj_list = [x for x in range(data_loader.num_val_scenes)]
            for b in range(int(data_loader.num_val_scenes/args.batch_size)):

                xo, xp, xoo, xpo, did = data_loader.next_batch_valid(traj_list[b * args.batch_size:(b + 1) * args.batch_size])
                mo = make_map_batch(xo, did, data_loader.map, args.map_size)

                # # train rnn
                feed = {model.gt_traj_enc: xoo,
                        model.gt_traj_dec: xpo,
                        model.gt_map_enc: mo,
                        model.init_state_enc: state,
                        model.output_keep_prob: 1.0}

                # run prediction
                valid_loss = sess.run(model.cost_valid, feed)
                valid_loss_list.append(valid_loss)

            # show current performance
            print('>> cur cost: %.4f, cur best: %.4f (time %.1fs, p-lvl %02d, %.1f hrs left)'
                  % (np.mean(valid_loss_list), min_avg_loss, (end - start), patient, ((end - start) * (args.num_epochs - e - 1) / 3600.0)))

            # save every breakthrough
            if (min_avg_loss > np.mean(valid_loss_list)):
                best_epoch = e
                patient = 0
                min_avg_loss = np.mean(valid_loss_list)
                checkpoint_path = os.path.join(args.model_dir, 'model.ckpt')
                saver.save(sess, checkpoint_path, global_step=e)
                print(">> model saved to {}".format(checkpoint_path))

                with open(os.path.join(args.model_dir, 'config.pkl'), 'wb') as f:
                    args.start_epoch = e
                    args.min_avg_loss = min_avg_loss
                    pickle.dump(args, f)
                print(">> args.start_epoch & args.min_avg_loss are updated and saved to {}".format('config.pkl'))
            else:
                patient += 1

            # # Save log
            if e % 10 == 0:
                file_name_txt = 'current_progress_' + str(e) + '.txt'
                file = open(os.path.join(args.model_dir, file_name_txt), "w")
                file.write('current best val. loss: ' + str(min_avg_loss) + '\n')
                file.write('p-evel: ' + str(patient) + '\n')
                file.write('remaining time: ' + str(((end - start) * (args.num_epochs - e - 1) / 3600.0)) + '\n')
                file.close()

            # early stop
            if (patient > args.patient_thr):
                print_training_info(args, min_avg_loss)
                print('>> early stop triggered ...')
                print('>> Best performance occured at %d epoch, corresponding avg. loss %.4f' % (
                best_epoch, min_avg_loss))
                print('>> goodbye ...')
                break

if __name__ == '__main__':
    main()


import pandas as pd
import joblib
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split

# Função para determinar a estação do ano
def get_season(date):
    month = date.month
    if month in [9, 10, 11]:
        return 'Primavera'
    elif month in [12, 1, 2]:
        return 'Verão'
    elif month in [3, 4, 5]:
        return 'Outono'
    else:
        return 'Inverno'

# Importação e Preparação dos Dados
data_path = "path_to_your_data/Itajai_2013_2021_a.xlsx"
df = pd.read_excel(data_path)
df['Estacao'] = df['Data'].apply(get_season)
df_seasonal = df.groupby(['Ano', 'Estacao']).mean().reset_index()
df_seasonal.drop(columns=['Dia', 'Mes'], inplace=True)

# Divisão dos dados em treino e teste
X = df_seasonal.drop(columns=['Aedes', 'Data'])
y = df_seasonal['Aedes']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Treinamento do Modelo
model = RandomForestRegressor(n_estimators=100, random_state=42)
model.fit(X_train, y_train)

# Salvar o modelo treinado
model_filename = "random_forest_model.pkl"
joblib.dump(model, model_filename)

# Carregar o modelo e fazer previsões
loaded_model = joblib.load(model_filename)
predictions = loaded_model.predict(X_test)

# Aqui, você pode continuar a processar as previsões conforme necessário


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Nov 10 23:28:29 2017
On my honor, I have not given, nor recieved, nor witnessed any unauthorized
assistance on this work.
I worked alone, and consulted the following resources:
    https://docs.opencv.org/2.4/modules/highgui/doc/user_interface.html
    http://uberhip.com/python/image-processing/opencv/2014/10/26/warping-brien/
    https://www.learnopencv.com/homography-examples-using-opencv-python-c/
    https://gist.github.com/nikgens/1a129d620978a4abc6a9a30f5f66e0d3

@author: Hannah Holman (hholman@rollins.edu)
"""
import cv2
import numpy as np

refPt = []
destPt = []

def get_src_coordinates(event, x, y, flags, param):
    """ Method to work with mouse callback in order to collect a set of
        user specified click points on source image.
        Appends points to global array variable, refPt

        Args:
            event: mouse event
            x, y: mouse click coordinates
            flags: None
            param (np.ndarray): input image
    """
    #instanitate global variables in local context
    global refPt
    img = param.copy()

    #When user clicks:
    if event == cv2.EVENT_LBUTTONDOWN:
        #Append click coordinates to global array variable
        refPt.append((x, y))
        print(len(refPt), "source points selected")

        #Draw circle to show click point
        img = cv2.circle(img, refPt[-1], 2, (255, 255, 0), 2)
        cv2.imshow('image', img)
        temp = refPt

        #Once four points have been selected, organize array
        if len(refPt) == 4:
            temp = np.array(temp)

            #top left and bottom right
            pt_sum = temp.sum(axis = 1)
            refPt[0] = temp[np.argmin(pt_sum)]
            refPt[2] = temp[np.argmax(pt_sum)]

            #top right and bottom left
            diff = np.diff(temp, axis = 1)
            refPt[1] = temp[np.argmin(diff)]
            refPt[3] = temp[np.argmax(diff)]
            return

def get_dest_coordinates(event, x, y, flags, param):
    """ Method to work with mouse callback in order to collect a set of
        user specified click points on destination image.
        Appends points to global array variable, destPt

        Args:
            event: mouse event
            x, y: mouse click coordinates
            flags: None
            param (np.ndarray): input image
    """
    #Instantiate global variables in local context
    global destPt
    img = param.copy()

    #When user clicks
    if event == cv2.EVENT_LBUTTONDOWN:
        #Append click coordinates to global array variable
        destPt.append((x, y))
        print(len(destPt), "destination points selected")

        #Draw circle to show click point
        img = cv2.circle(img, destPt[-1], 2, (255, 255, 0), 2)
        cv2.imshow('image 2', img)
        temp = destPt

        #When four points have been selected, organize array and return
        if len(destPt) == 4:
            temp = np.array(temp)

            #top left and bottom right
            pt_sum = temp.sum(axis = 1)
            destPt[0] = temp[np.argmin(pt_sum)]
            destPt[2] = temp[np.argmax(pt_sum)]

            #top right and bottom left
            diff = np.diff(temp, axis = 1)
            destPt[1] = temp[np.argmin(diff)]
            destPt[3] = temp[np.argmax(diff)]
            return

def transform_upright(image, pts):
    """Transforms the perspective of an image to a "face-on" view

       Args:
           image (numpy.ndarray): an input image to be transformed

           pts (array): an array of points on an input image to be used as source points
               for a perspective transform

       Returns:
           cpy (numpy.ndarray): a transformed image
    """
    #Instantiate global variables in local context
    global refPt

    #Retrieve the min and max values in the reference points array
    minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(refPt)

    cpy = image.copy()
    pts = np.float32(pts)

    #Use max value to create a large enough desination image with square shape
    dest_pts = np.float32([[0, 0], [maxVal, 0], [maxVal, maxVal], [0, maxVal]])

    M = cv2.getPerspectiveTransform(pts, dest_pts)

    maxVal = np.int32(maxVal)
    cpy = cv2.warpPerspective(image, M, (maxVal, maxVal))

    return cpy

def transform_homography(img_src, img_dst, pts1, pts2):
    """Warps a source image to a destination image based on a set of source
       and destination points

       Args:
           img_src (numpy.ndarray): source image to be warped
           img_dest (numpy.ndarray): destination image to be warped to
           pts1 (array): array of coordinates to be used to compute homography
           pts2 (array): array of coordinates to be used to compute homography

       Returns:
           image_dst (numpy.ndarray): the final image with warped image on top
    """
    image_src = img_src.copy()
    image_dst = img_dst.copy()
    mask = image_dst.copy()

    h, status = cv2.findHomography(pts1, pts2)

    #create masks of both images around the portion of the image to be merged
    mask = cv2.warpPerspective(image_src, h, (image_dst.shape[1], image_dst.shape[0]))
    cv2.fillConvexPoly(image_dst, np.int32(pts2), 0, cv2.LINE_AA)

    image_dst = image_dst + mask

    #return image_dst
    return image_dst

def foreground_warp(img):
    """Takes an input image and transforms the perspective to a face-on view,
       and fills an array of points with the four corners fo the transformed
       image

       Args:
           img (numpy.ndarray): image to be transformed

       Returns:
           img_transform (numpy.ndarray): transformed image
           src_pts_transform (array of points): array containign the four
               corners of the output image
    """
    global refPt
    src_pts = []

    img_src = img.copy()
    src_pts = refPt
    src_pts = np.float32(src_pts)
    refPt = np.array(refPt)

    img_transform = transform_upright(img_src, src_pts)

    #build an array with four corner points of transformed image to be used
    #when computing homography with final dest image later
    h, w = img_transform.shape[:2]
    src_pts_transform = np.empty((0,2),dtype=np.float32)
    src_pts_transform = np.append(src_pts_transform, [(0,0)], axis=0)
    src_pts_transform = np.append(src_pts_transform, [(w - 1, 0)], axis=0)
    src_pts_transform = np.append(src_pts_transform, [(w - 1, h - 1)], axis=0)
    src_pts_transform = np.append(src_pts_transform, [(0, h - 1)], axis=0)

    return img_transform, src_pts_transform

def warp_to_background(img):
    """Creates an array of destination points and copies an input image in
       order to preserve original for later manipulation

       Args
           img (numpy.ndarray): image to be preserved

       Returns:
           img_dest (numpy.ndarray): copied image to be used as destination for
               perspective transformation

           dest_pts (np.float32): list of destination points for computing
               homography
    """
    dest_pts = []

    if len(dest_pts) == 0:
        img_dest = img.copy()
        dest_pts = destPt
        dest_pts = np.float32(dest_pts)
        cv2.destroyAllWindows()
        return img_dest, dest_pts
    else:
        print("invalid")
        cv2.destroyAllWindows()
        return None, None

################################################################

#process first image
while (1):

    src = cv2.imread('comic1.jpg')
    #src = cv2.resize(src, (1500,900))
    dst = cv2.imread('billboard.jpg')

    cv2.namedWindow('image')
    cv2.imshow('image', src)

    #set the "on click" to get the source coordinates, param is the source image
    cv2.setMouseCallback('image', get_src_coordinates, param = src)

    #Wait for user to select points and press a key. If keypress = 'b', go back
    #and erase selected points
    if cv2.waitKey(0) == 98:
        refPt = []
        print("Source point selection restarted")
    elif cv2.waitKey(0) and len(refPt) == 4:
        img_transform, src_pts_transform = foreground_warp(src)
        cv2.destroyAllWindows()
        break
    elif len(refPt) > 4:
        print("Error: Too many points. Points set to 0, please start again")
        refPt = []
    else:
        print("Source points selected: ", len(refPt),
              "- Please select", 4 - len(refPt), "more points")

#process second image
while (1):

    cv2.namedWindow('image 2')
    cv2.imshow('image 2', dst)

    #set the "on click" to get the destination coordinates, param is the dest image
    cv2.setMouseCallback('image 2', get_dest_coordinates, param = dst)

    #Wait for user to select points and press a key. If keypress = 'b', go back
    #and erase selected points
    if cv2.waitKey(0) == 98:
        destPt = []
        print("Destination point selection restarted")
    elif cv2.waitKey(0) and len(destPt) == 4:
        img_dest, dest_pts = warp_to_background(dst)
        break
    elif len(destPt) > 4:
        print("Error: Too many points. Points set to 0, please start again")
        destPt = []
    else:
        print("Destination points selected:", len(destPt),
              "- Please select", 4 - len(destPt), "more points")

#find the homography and transform images
output = transform_homography(img_transform, img_dest, src_pts_transform, dest_pts)
cv2.imshow("Output", output)
cv2.waitKey(0)
cv2.destroyAllWindows()


import time
from typing import Dict, List

import pytest
import requests
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker, clear_mappers

import config
from domain.models.batch import Batch
from domain.models.order_line import OrderLine
from entrypoints.flask_app import app
from adapters.orm import mapper_registry, start_mappers

'''
Updates:
    -   commented out start_mappers() in session method
        because we our orm and flask app is tightly coupled and we are already starting a mapper in flask_app module.
        calling the method twice throws ArgumentError: already has a primary mapper defined
'''

@pytest.fixture(scope="session")
def session():
    engine = create_engine(config.get_db_url())
    mapper_registry.metadata.create_all(engine)
    # start_mappers()
    session = sessionmaker(bind=engine)()
    yield session
    clear_mappers()

@pytest.fixture()
def add_stock(session):
    batches_added = set()
    skus_added = set()

    def _add_stock(batches):
        for ref, sku, qty, eta in batches:
            session.execute(text('insert into batches (reference, sku, quantity, eta) values'
                                 f'("{ref}","{sku}",{qty},"{eta}")'))

            batches_added.add(list(session.execute(text('select id from batches where sku=:sku and reference=:ref'),
                                                   dict(sku=sku, ref=ref)))[0][0])
            skus_added.add(sku)

        session.commit()

    yield _add_stock

    for batch_id in batches_added:
        session.execute(
            text("DELETE FROM allocations WHERE batch_id=:batch_id"),
            dict(batch_id=batch_id),
        )
        session.execute(
            text("DELETE FROM batches WHERE id=:batch_id"), dict(batch_id=batch_id),
        )

    for sku in skus_added:
        session.execute(
            text("DELETE FROM order_lines WHERE sku=:sku"), dict(sku=sku),
        )

        session.commit()

@pytest.fixture()
def add_batch_and_allocations(session):
    order_lines_added = set()
    batch_added = set()

    def _add_batch_and_allocate_lines(batch: Dict, lines: List, unallocated_order_id =''):
        session.execute(text('insert into batches (reference, sku, quantity, eta) values'
                             f'("{batch["reference"]}","{batch["sku"]}",{batch["quantity"]},"{batch["eta"]}")'))

        batch_id = list(session.execute(text('SELECT id from batches'
                                             f' where reference =:batch_ref'), dict(batch_ref=batch['reference'])))[0][
            0]
        batch_added.add(batch_id)
        for order_id, sku, qty in lines:
            session.execute(text('insert into order_lines (order_id, sku, quantity) values'
                                 f'("{order_id}","{sku}",{qty})'))
            order_lines_added.add(order_id)

        order_line_ids = []
        for order_id in order_lines_added:
            if order_id != unallocated_order_id:
                order_line_ids.append(list(session.execute(text('SELECT id FROM order_lines WHERE order_id = :order_id'),
                                                       {'order_id': order_id}))[0][0])

        for line_id in order_line_ids:

            session.execute(text('insert into allocations (batch_id, order_line_id) values'
                                 f'({batch_id},{line_id})'))
        session.commit()

    yield _add_batch_and_allocate_lines

    for batch_id in batch_added:
        session.execute(
            text("DELETE FROM allocations WHERE batch_id=:batch_id"),
            dict(batch_id= batch_id),
        )
        session.execute(
            text("DELETE FROM batches WHERE id=:batch_id"), dict(batch_id=batch_id),
        )

    for order_id in order_lines_added:
        session.execute(
            text("DELETE FROM order_lines WHERE order_id=:order_id"), dict(order_id=order_id),
        )

    session.commit()

def wait_for_webapp_to_come_up():
    deadline = time.time() + 10
    url = config.get_api_url()
    while time.time() < deadline:
        try:
            return requests.get(url)
        except ConnectionError:
            time.sleep(0.5)
    pytest.fail("API never came up")

@pytest.fixture()
def restart_api():
    app.run(host='0.0.0.0', port=80, debug=True)
    time.sleep(0.5)
    return
#     # wait_for_webapp_to_come_up()


import cv2
import numpy as np
# masks are the binary images that indicate the pixel in which an operation is to be
#performed
img=cv2.imread("blackwhite.jpg")
print(img.shape)
img2=np.zeros((339,509,3),np.uint8)
cv2.rectangle(img2,(200,0),(300,100),(255,255,255),-1)
bitand=cv2.bitwise_and(img,img2)
bitor=cv2.bitwise_or(img,img2)
bitnot=cv2.bitwise_not(img)
bitxor=cv2.bitwise_xor(img,img2)
cv2.imshow("image",img)
cv2.imshow("image2",img2)
cv2.imshow("image3",bitand)
cv2.imshow("image4",bitor)
cv2.imshow("image5",bitnot)
cv2.imshow("image6",bitxor)

cv2.waitKey(0)
cv2.destroyAllWindows()


""" Special configs for the Logger """
import logging
import time
from functools import wraps
import os

supress_error = False

def basiclogger(loggerobj, loggername, printline = False):
    """ This is a very basic logger, which logs every Level and writes the time
    and a Message.

    arguments:

        -- loggerobj (str): Name for the loggerobject to later call on
        -- loggername (str): Name of the loggingfile
        -- printline (bool): Condition if the logger should log a restarting line into the log to see all restarts/crashes
    """
    logger = logging.getLogger(loggerobj)
    logger.setLevel(logging.DEBUG)
    dirpath = os.path.dirname(__file__)
    subfoldername = "logs"
    savepath = os.path.join(dirpath, '..', subfoldername, "{}.log".format(loggername))
    # print(savepath)
    fh = logging.FileHandler(savepath)
    fh.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        '%(asctime)s - %(message)s', "%Y-%m-%d %H:%M")
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    # Logging the Start of the Programm. If it runs a whole evening, each restart means a previous crash.
    if printline:
        template = "{:-^80}"
        logger.debug(template.format("Restarting the Programm",))

def logerror(func):
    """ Logs every time an error occours """
    @wraps(func)
    def wrapper(*args, **kwargs):
        if supress_error:
            logger = logging.getLogger('debuglog')
            try:
                func(*args, **kwargs)
                # return func(*args, **kwargs)
            except Exception:
                logger.exception
                logger.exception("The function {} could not be fully excecuted!".format(func.__name__))
                print("The function {} could not be fully excecuted!".format(func.__name__))
        else:
            func(*args, **kwargs)
    return wrapper


import cv2, sys, time
import matplotlib.pyplot as plt
import subprocess

def Foto(Nombre):

#video_capture = cv2.VideoCapture(0)
 video_capture=cv2.VideoCapture("/dev/video0") #Escogiendo la camara
 # Check success
 if not video_capture.isOpened():
    raise Exception("Could not open video device")
 # Read picture. ret === True on success
 ret, frame = video_capture.read()
 # Close device
 video_capture.release()
 #print(ret)
 #img = cv2.imread('1.jpg',1)  #Leyendo imagenes
 #print(frame.shape)
 frameRGB = frame[:,:,::-1] # BGR => RGB
 #cv2.imshow(frameRGB, img)
 #cv2.imwrite('/home/debian/MicroSD/Codigos/Fotos/CameraPhoto.jpg',frameRGB)
 a='/home/debian/'
 cv2.imwrite(a+Nombre ,frameRGB)
 cv2.imshow('image',frameRGB)
 cv2.waitKey(0)
 cv2.destroyAllWindows()

print("Tomando Imagen")
Name="Camara.jpg"
Foto(Name)
print("FOto tomada")
bashCommand = "mv /home/debian/"+Name+" /home/debian/MicroSD/Codigos/Fotos"
process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
output, error = process.communicate()


import os
from django.contrib.gis.utils import LayerMapping
from .models import Counties, Constituencies

counties_mapping = {
    'objectid' : 'OBJECTID',
    'id_field' : 'ID_',
    'county_nam' : 'COUNTY_NAM',
    'const_code' : 'CONST_CODE',
    'constituen' : 'CONSTITUEN',
    'county_cod' : 'COUNTY_COD',
    'shape_leng' : 'Shape_Leng',
    'shape_area' : 'Shape_Area',
    'geom' : 'MULTIPOLYGON',
}

county_shp = os.path .abspath(os.path.join(os.path.dirname(__file__),'data/counties/counties.shp'))

def load_counties_shapefile(verbose=True):
	lm = LayerMapping(Counties, county_shp, counties_mapping, transform= False, encoding='iso-8859-1')
	lm.save(strict=True,verbose=verbose)

constituencies_mapping = {
    'objectid' : 'OBJECTID',
    'county_nam' : 'COUNTY_NAM',
    'const_code' : 'CONST_CODE',
    'constituen' : 'CONSTITUEN',
    'county_ass' : 'COUNTY_ASS',
    'county_a_1' : 'COUNTY_A_1',
    'regist_cen' : 'REGIST_CEN',
    'registrati' : 'REGISTRATI',
    'county_cod' : 'COUNTY_COD',
    'shape_leng' : 'Shape_Leng',
    'shape_area' : 'Shape_Area',
    'geom' : 'MULTIPOLYGON',
}

constituency_shp = os.path .abspath(os.path.join(os.path.dirname(__file__),'data/constituencies/constituencies.shp'))

def load_constituencies_shapefile(verbose=True):
	lm = LayerMapping(Constituencies, constituency_shp, constituencies_mapping, transform= False, encoding='iso-8859-1')
	lm.save(strict=True,verbose=verbose)


# 2018 Nicola Dinsdale
# Dependencies: pdffigures2-assembly-0.0.12-SNAPSHOT.jar --> extracts the information from the pdf to a json file.
import re
import numpy as np
import csv
from pprint import pprint
import matplotlib.pyplot as plt
import string
import word2vec
import gensim
from sklearn.pipeline import Pipeline
from sklearn.ensemble import ExtraTreesClassifier, ExtraTreesRegressor
from sklearn.svm import SVC, SVR
from tabulate import tabulate
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from gensim.models.word2vec import Word2Vec
from collections import Counter, defaultdict
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.metrics import accuracy_score
from sklearn.cross_validation import cross_val_score
from sklearn.cross_validation import StratifiedShuffleSplit
import zipfile
import json
import argparse
import os
import subprocess
import sys
import PyPDF2
import pdfminer

#Project files
from Embeddings import TfidfEmbeddingVectorizer
from re_functions import *
from regressor_functions import *

def get_info(path):
    with open(path, 'rb') as f:
        pdf =PyPDF2.PdfFileReader(f)
        info = pdf.getDocumentInfo()
        number_of_pages = pdf.getNumPages()
    author = info.author
    creator = info.creator
    subject = info.subject
    title = info.title

    return info, number_of_pages

################################################################################
# Get the file name from the input argument
# Input parser
# THIS SHOULD BE IN THE FORM OF A PDF
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-f', '--file', required=True, help="path to f file, f should be a PDF")
args = parser.parse_args()
file_pth = args.file
print (file_pth)
assert os.path.isfile(file_pth)

#use pyPDF2 to get some information quickly
info, number_of_pages = get_info(file_pth)

#Run the jar to create the json files and extract the figures
print ('EXTRACTING THE BODY OF THE DOCUMENT')
x = subprocess.call(['java','-jar', 'pdffigures2-assembly-0.0.12-SNAPSHOT.jar', file_pth,'-g', 'g_'])
# We cannot run this extraction on the older pdfs so force an abort if we have not succeeded
if x == 1:
    print ('ERROR: FAILED TO EXTRACT INFORMATION FROM PDF')
    sys.exit()

print ('EXTRACTING THE IMAGES FROM THE DOCUMENT')
y = subprocess.call(['java', '-jar', 'pdffigures2-assembly-0.0.12-SNAPSHOT.jar', file_pth, '-m', 'm_'])
if y == 1:
    print('Failed to extract images')
no = int([float(s) for s in re.findall(r'-?\d+\.?\d*', file_pth)][0])     #Use re to extract the number
filename = 'g_id_article_' + str(no) + '.json'
#Read in the pdf structure in the form of a json --> dictionary of dictionaries
print ('Reading in file:')
with open(filename) as json_data:
    e = json.load(json_data)
    json_data.close()

#Search the sections to try and find the sentence, its only going to be on page one or two
#Some of the articles store the abstract as a seperate item so look for this and search it as well7
textstore = []
print ('SEARCHING FOR ABSTRACT')
try:
    Abstract = e['abstractText']
    print ('\t Abstract object found')
    print ('\t Gathering Text from abstract')
    textstore.append(Abstract['text'])
except:
    print ('\t No abstract object ')

print ('GATHERING MAIN TEXT')

sectionDict = e['sections']
for i in range(len(sectionDict)):
    dictionary = sectionDict[i]
    subdictionary = dictionary['paragraphs']
    if len(subdictionary) > 1:
        for j in range(len(subdictionary)):
            sub = subdictionary[j]
            page = sub['page']
            if page == 0 or 1:          #The identifier sentence will only be on the first or second page of the article if it exists
                text = sub['text']
                textstore.append(text)
    if len(subdictionary) == 1:
        subdictionary = dictionary['paragraphs'][0]
        page = subdictionary['page']
        if page == 0 or 1:          #The identifier sentence will only be on the first or second page of the article if it exists
            text = subdictionary['text']
            textstore.append(text)
    if len(subdictionary) == 0:
        subdict = sectionDict[i]
        paradict = subdict['paragraphs']
        for j in range(0, len(paradict)):
            finaldict = paradict[j]
            page = finaldict['page']
            if page == 0 or 1:
                text = finaldict['text']
                textstore.append(text)

sentences = []
for para in textstore:
    s = split_into_sentences(para)
    for sent in s:
        sent = sent.lower()                                         #Make everything lower case
        sent = re.sub('['+string.punctuation+']', '', sent)
        words = sent.split()
        sentences.append(words)
sentences = np.array(sentences)

fittedModel = idenitifier_regressor()
pred_id = fittedModel.predict(sentences)

#Do the same for the captions
caps = []
pageno = []
tableFlag = 0
figDict = e['figures']
for k in range(len(figDict)):
    fig = figDict[k]
    if fig['figType'] == 'Figure':
        caps.append(fig['caption'])
        pageno.append(fig['page'])
    if fig['figType'] == 'Table':
        tableFlag += 1
capStore = []
for cap in caps:
    cap = cap.lower()       #Make everything lower case
    cap = re.sub('['+string.punctuation+']', '', cap)
    words = cap.split()
    capStore.append(words)

capmodel = caption_regressor(verbose = 1)
pred_cap = capmodel.predict(capStore)
args = np.argsort(pred_cap)
args = args[::-1]

# Run the analysis given the model
flagid = 0
sid = []
pid = []

for i in range(sentences.shape[0]):
    if pred_id[i] > 0.9:
        sid.append(sentences[i])
        pid.append(pred_id[i])
        flagid = 1
if flagid == 0:
    for i in range(sentences.shape[0]):
        if pred_id[i] > 0.8:
            sid.append(sentences[i])
            pid.append(pred_id[i])
            flagid = 1
if flagid == 0:
    #print ('We dont think there is an indicator sentence')
    for i in range(sentences.shape[0]):
        if pred_id[i] > 0.7:
            sid.append(sentences[i])
            pid.append(pred_id[i])
            flagid = 1
if flagid == 0:
    print ("Couldn't find an idenitifer sentence")

I = np.argmax(pid)

print ('#########################################################')
print ('# ~~~~~~~~~~~~~~~~~~~~~~ Summary ~~~~~~~~~~~~~~~~~~~~~~ #')

print ('TITLE:')
print (info.title)
print ('\n')

if flagid != 0:
    print ('IDENTIFIER SENTENCE:')
    print ('Identifier Found:')
    #Stitch the sentence back together
    if pid[I] < 0.8:
        print ('We dont think there is an identifier sentence. This was the highest scorer:')
    identifier = sid[I]
    s = " "         #Use space as the thing to knit the sentences back together
    identifier = s.join(identifier)
    print ('\n',identifier)
    print ('This identifier scored: ', pid[I])

    familyFlag, i = family_check(identifier)
    if familyFlag == 1:
        print ('We think that there is a family connection because we found:')
        print (i)

    unusualFlag, i = unusual_check(identifier)
    if unusualFlag == 1:
        print ('We think that this condition may be atypical because we found:')
        print (i)

print ('\n \n FIGURE CAPTIONS:')
print ('\t WARNING: pdffigures2 doesnt always extract all the figures succesfully \n')
for a in args:
    print ('- We are %2f confident this figure has a face in it:' % pred_cap[a])
    c = capStore[a]
    s = " "         #Use space as the thing to knit the sentences back together
    c = s.join(c)
    print (c)
    print ('Found on page', pageno[a])
    print ('Searching for identifiers:')
    identifiers = indentifier_analyis(c)
    print (identifiers)
    familyFlag, i = family_check(c)
    if familyFlag == 1:
        print ('- We think that there is a family connection because we found:')
        print (i)
    unusualFlag, i = unusual_check(c)
    if unusualFlag == 1:
        print ('- We think that this condition may be atypical because we found:')
        print (i)
    surgery_flag, i = surgery(c)
    if surgery_flag == 1:
        print ('- WARNING: We think that the person may have had reconstructive surgery because we found:')
        print (i)
    flag, i = unaffectedCheck(c)
    if flag == 1:
        print ('- WARNING: We think that someone in the photo is not affected because we found:')
        print (i)
    print ('\n')

if tableFlag != 0:
    print ('We think there is information held in a table in this article, we found ', tableFlag, ' tables')


# BeautifulSoup basics
# Selecting using CSS style selectors using .select()
# .select() returns a list of matching elements; cleaner alternative to .find() and .find_all()

# Select by id = #foobar
# Select by class = .foobar
# Select by children = foo > bar
# Select by descendants = foo bar

from bs4 import BeautifulSoup

# Mockup HTML file inside a multi-lined string
html = """
<!DOCTYPE html>

  <meta charset="UTF-8">
  <title>First HTML Page</title>

hi

more text.

This list item is special.

This list item is also special.

This list item is not special.

bye

"""

# Setup BeautifulSoup to read the mockup HTML content and parse the content using html.parser
soup = BeautifulSoup(html, "html.parser")

# Select by id
print("Select matching elements by CSS selector id")
matching_elements = soup.select("#first")       # Returns matching element(s) enclosed in a list
print(matching_elements)
print("\n")

# Select the 1st matching element by ID
print("Select matching 1st element by CSS selector id:")
matching_elements = soup.select("#first")[0]    # Returns a specific matching element; no long enclosed in a list
print(matching_elements)
print("\n")

# Select by class
print("Select matching elements by CSS selector class:")
matching_elements = soup.select(".special")
print(matching_elements)
print("\n")

# Select by element
print("Select matching elements by element name:")
matching_elements = soup.select("div")
print(matching_elements)
print("\n")

# Select by attribute
print("Select matching elements by attribute:")
matching_elements = soup.select("[data-example]")
print(matching_elements)
print("\n")


# -*- coding: utf-8 -*-
"""
Created on Sat Jul 14 12:43:03 2018

@author: vs362
"""

import h5py
import numpy as np
from devices_functions import *

SLASH = '/' #'/' for Mac addresses or '\\' for PC

class Amplifier(object):

    def __init__(self, path, name):
        self.name = name
        self.device_folder = path + self.name + SLASH
        self.data_file = self.device_folder + name + u'.hdf5'
        self.load_attributes()

        self.name = name
        self.device_folder = path + self.name + SLASH
        self.data_file = self.device_folder + name + u'.hdf5'
        self.save_attributes()

    #   saves all device data into 'results' group of the hdf5 file, overwrites existing data!
    def save_attributes(self):
        hdf5_file = h5py.File(self.data_file,'r+')
        try:
            if 'results' not in hdf5_file.keys():
                hdf5_file.create_group('results')
            attributes = [a for a in dir(self) if not a.startswith('__') and not callable(getattr(self,a)) ]
            for field in attributes:
                hdf5_file['results'].attrs[field] = getattr(self,field)
        finally:
            hdf5_file.close()

#   loads existing 'results' group from hdf5 file to python memory
    def load_attributes(self):
        hdf5_file = h5py.File(self.data_file,'r')
        try:
            if 'results' in hdf5_file.keys():
                for attr in hdf5_file['results'].attrs:
                    setattr(self, attr, hdf5_file['results'].attrs.get(attr) )
        finally:
            hdf5_file.close()

    def add_flux_sweep(self, date, time, meas_name='"CH1_S11_1"'):
        self.flux_sweep_date = date
        self.flux_sweep_time = time
        self.flux_sweep_meas_name = meas_name
        self.save_attributes()

#   Adds different components to attenuation of drive/signal/pump
    def add_component_to_line_attenuation(self, component_name, date, time, meas_name, pump=False, signal=False, drive=False):
        hdf5_file = h5py.File(self.data_file,'r+')
        try:
            if pump and 'pump_attenuation' not in hdf5_file.keys():
                hdf5_file.create_group('pump_attenuation')
            elif signal and 'signal_attenuation' not in hdf5_file.keys():
                hdf5_file.create_group('signal_attenuation')
            elif drive and 'drive_attenuation' not in hdf5_file.keys():
                hdf5_file.create_group('drive_attenuation')

            if pump:
                grp = hdf5_file['pump_attenuation'].create_group(component_name)
            elif signal:
                grp = hdf5_file['signal_attenuation'].create_group(component_name)
            elif drive:
                grp = hdf5_file['drive_attenuation'].create_group(component_name)
            grp.attrs['date'] = date
            grp.attrs['time'] = time
            grp.attrs['meas_name'] = meas_name
        finally:
            hdf5_file.close()

#   Fits stark shift to get g4. Before using make sure you saved drive attenuation
    def fit_stark_shift(self, meas_name, flux_calibrated=True):

        hdf5 = h5py.File(self.data_file)
        self.stark_meas_name = meas_name
        currents = np.loadtxt(self.device_folder+'stark'+SLASH+'curr.txt')*1e-3
        dates = np.loadtxt(self.device_folder+'stark'+SLASH+'date.txt')
        times = np.loadtxt(self.device_folder+'stark'+SLASH+'time.txt')
        drive_freqs = np.loadtxt(self.device_folder+'stark'+SLASH+'drive.txt')*1e9
        if flux_calibrated: fluxes = Flux(currents,self.a,self.b)
        Kerr_Data = np.empty(0)
        K_prime_Data = np.empty(0)
        try:
            for j, current in enumerate(currents):
                date = float_to_DateTime( dates[j] )
                time = float_to_DateTime( times[j] )
                f_d = drive_freqs[j]

                #linear scattering data
                f_0 = np.asarray(hdf5[date][time]['fits'][meas_name].get('f0'))
                k_c = np.asarray(hdf5[date][time]['fits'][meas_name].get('kc'))
                k_i = np.asarray(hdf5[date][time]['fits'][meas_name].get('ki'))
                k_t = k_c + k_i

                line_attenuation = attenuation(f_d, self.data_file, 'drive_attenuation')

                if 'powers' in hdf5[date][time]['LIN'].keys():
                    log_powers = np.asarray(hdf5[date][time]['LIN'].get('powers'))
                elif 'powers_swept' in hdf5[date][time]['LIN'].keys():
                    log_powers = np.asarray(hdf5[date][time]['LIN'].get('powers_swept'))
                log_powers = log_powers + line_attenuation
                powers = dBm_to_Watts(log_powers)

                # fit the slope, this formula takes into account some non RWA corrections
                deg = 6
                Kerr = np.polyfit(24/(2*np.pi*h)*powers*k_c[0]/f_0[0]*((2*f_d/(f_0[0]+f_d))**2)/( (f_d-f_0[0])**2 + (k_t[0]/2)**2*((2*f_d/(f_0[0]+f_d))**2) ),f_0,deg)[deg-1]    # in Hz
#                K_prime = np.polyfit(1/(2*np.pi*h)*powers*k_c[0]/f_0[0]*((2*f_d/(f_0[0]+f_d))**2)/( (f_d-f_0[0])**2 + (k_t[0]/2)**2*((2*f_d/(f_0[0]+f_d))**2) ),f_0,2)[0]
                Kerr_Data = np.append(Kerr_Data, Kerr)
#                K_prime_Data = np.append(K_prime_Data,K_prime)
            if 'fit_stark_shift' in hdf5.keys():
                del hdf5['fit_stark_shift']
            print(len(currents))
            grp = hdf5.create_group('fit_stark_shift')
            grp.create_dataset('current', data = currents)
            grp.create_dataset('date', data = dates)
            grp.create_dataset('time', data = times)
            grp.create_dataset('drive_freq', data = drive_freqs)
            if flux_calibrated: grp.create_dataset('flux', data = fluxes)
            grp.create_dataset('g4', data = Kerr_Data)
#            grp.create_dataset('K_prime', data = K_prime_Data)
        finally:
            hdf5.close()

    #   Loads the data about measured gain points into hdf5 file, then you can use it to plot all kinds of things
    def load_nonlin_characterization(self, meas_name='"CH1_S11_1"',sweep_name='nonlin_characterization'):

        self.pumping_meas_name = meas_name
        nominal_gain_array = np.loadtxt(self.device_folder + sweep_name + SLASH + 'gains.txt')
        nominal_gain_array = [nominal_gain_array]

        hdf5 = h5py.File(self.data_file)
        try:
            if sweep_name in hdf5.keys():
                del hdf5[sweep_name]
            grp = hdf5.create_group(sweep_name)
            grp.create_dataset('gains', data = nominal_gain_array)
        finally:
            hdf5.close()

        for log_g in nominal_gain_array:
            temp_folder = sweep_name + SLASH + 'Gain_'+str(int(log_g))+SLASH
            data_dict = {}
            data_dict['current'] = np.loadtxt(self.device_folder+temp_folder+'curr.txt')*1e-3
            data_dict['date_linear'] = np.loadtxt(self.device_folder+temp_folder+'date_linear.txt')
            data_dict['time_linear'] = np.loadtxt(self.device_folder+temp_folder+'time_linear.txt')
            data_dict['IMD_time'] = np.loadtxt(self.device_folder+temp_folder+'IMD_time.txt')
            data_dict['IMD_date'] = np.loadtxt(self.device_folder+temp_folder+'IMD_date.txt')
            if log_g:
                data_dict['date'] = np.loadtxt(self.device_folder+temp_folder+'date.txt')
                data_dict['time'] = np.loadtxt(self.device_folder+temp_folder+'time.txt')
                data_dict['Pump_power'] = np.loadtxt(self.device_folder+temp_folder+'pump_power.txt')
                data_dict['Pump_freq'] = np.loadtxt(self.device_folder+temp_folder+'pump_freq.txt')*1e9
            g4_IMD = np.zeros(len(data_dict['current']))
            CW_freqs = np.zeros(len(data_dict['current']))
            freq_c = np.zeros(len(data_dict['current']))
            kappa_c = np.zeros(len(data_dict['current']))
            kappa_t = np.zeros(len(data_dict['current']))
            P_1dB = np.zeros(len(data_dict['current']))
            IIP3 = np.zeros(len(data_dict['current']))
            max_Gain = np.zeros(len(data_dict['current']))
            bandwidth = np.zeros(len(data_dict['current']))
            NVR = np.zeros(len(data_dict['current']))
            NVR_spike = np.zeros(len(data_dict['current']))
            Gains = np.zeros((len(data_dict['current']),201))
            Signal_Powers_dBm = np.zeros((len(data_dict['current']),201))
            # TODO change this number to the size of signal powers aray
            hdf5 = h5py.File(self.data_file)
            try:
                for index, current in enumerate(data_dict['current']):
                    # define all data folders (dates/times)
                    IMD_time = float_to_DateTime( data_dict['IMD_time'][index] )
                    IMD_date = float_to_DateTime( data_dict['IMD_date'][index] )
                    time_linear = float_to_DateTime( data_dict['time_linear'][index] )
                    date_linear = float_to_DateTime( data_dict['date_linear'][index] )
                    if log_g:
                        date = float_to_DateTime( data_dict['date'][index] )
                        time = float_to_DateTime( data_dict['time'][index] )

                    # load linear fits
                    freq_c[index] = float( np.asarray(hdf5[date_linear][time_linear]['LIN'][meas_name]['fits'].get('f0')) )
                    kappa_c[index] = float( np.asarray(hdf5[date_linear][time_linear]['LIN'][meas_name]['fits'].get('kc')) )
                    kappa_i = float( np.asarray(hdf5[date_linear][time_linear]['LIN'][meas_name]['fits'].get('ki') ))
                    kappa_t[index] = kappa_c[index] + kappa_i

                    # continuous wave frequency for signal
                    if log_g:
                        freq_CW = hdf5[date][time]['POW']['powers'].attrs.get('freqINP')
                    else:
                        freq_CW = freq_c[index] + 100e6
                    CW_freqs[index] = freq_CW

                    if log_g:
                        # signal powers
                        Signal_Powers_dBm[index] = np.asarray( hdf5[date][time]['POW'].get('powers') )
                        Signal_Powers_dBm[index] = Signal_Powers_dBm[index] + attenuation(freq_CW, self.data_file, 'signal_attenuation')

                        # Gains
                        ind_CW = np.abs(np.asarray(hdf5[date][time]['memory'].get('frequencies'))-freq_CW).argmin() + 2           # index of approximately CW_frequency in the memory trace
                        a_out = np.asarray( hdf5[date][time]['memory'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['memory'][meas_name].get('imag') )
                        logmag_pump_off = complex_to_PLOG(a_out)[0]
                        logmag_pump_off_ref= logmag_pump_off[ind_CW]
                        a_out = np.asarray( hdf5[date][time]['POW'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['POW'][meas_name].get('imag') )
                        logmag_pump_on = complex_to_PLOG(a_out)[0]
                        logmag_pump_on_ref = logmag_pump_on[0]      # logmag at the lowest CW power where amplifier is presumably not saturated yet
                        gain_ref = logmag_pump_on_ref-logmag_pump_off_ref
                        Gains[index] = logmag_pump_on - logmag_pump_on_ref + gain_ref
                        max_Gain[index] = gain_ref

                        # Figuring out gain profile in frequency
                        a_out = np.asarray( hdf5[date][time]['memory'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['memory'][meas_name].get('imag') )
                        logmag_pump_off = complex_to_PLOG(a_out)[0]
                        a_out = np.asarray( hdf5[date][time]['LIN'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['LIN'][meas_name].get('imag') )
                        logmag_pump_on = complex_to_PLOG(a_out)[0]
                        gain_profile = logmag_pump_on - logmag_pump_off

                        # Finding the 3 dB bandwidth relative to CW frequency
                        ind_left, ind_right = find_3dB_bandwidth(gain_profile, ind_CW)
                        bandwidth[index] = np.asarray(hdf5[date][time]['memory'].get('frequencies'))[ind_right]-np.asarray(hdf5[date][time]['memory'].get('frequencies'))[ind_left]

                        # P_1dB and IIP3 points
                        Saturation_power_ind = find_1dB_compression_point(Gains[index], gain_ref)
                        P_1dB[index] = Signal_Powers_dBm[index][Saturation_power_ind]

                        if 'noise' in hdf5[date][time]['fits'].keys():
                            noise_flag = True
                            key1 = list(hdf5[date][time]['fits']['noise'].keys())[0]
                            key2 = list(hdf5[date][time]['fits']['noise'][key1].keys())[0]

                            noise_raise = np.asarray(hdf5[date][time]['noise'][key1][key2].get('logmag')) - np.asarray(hdf5[date][time]['noise'][key1]['memory'][key2].get('logmag'))
                            NVR[index] = noise_raise[ind_CW]
                            NVR_spike[index] = max(noise_raise) - NVR[index]
                        else:
                            noise_flag = False

                    data = np.asarray(hdf5[IMD_date][IMD_time]['POW']['"CH5_IIP3_11"'].get('logmag'))

                    derivative = derivative_smooth(data, N_smooth=15)
                    Ind = np.argmin( np.abs(derivative) )

                    IIP3[index] =np.mean(data[max(Ind-5,0):min(Ind+5,len(data)-1)])
                    IIP3[index] = IIP3[index] + attenuation( float(hdf5[IMD_date][IMD_time]['POW']['xs'].attrs.get('imd_f1_cw')), self.data_file, 'signal_attenuation' )
                    IIP_3 = dBm_to_Watts( IIP3[index] )
                    if log_g:
                        G = np.power(10, max_Gain[index]/10 )
                        g4_IMD[index] = float(kappa_c[index]**2*freq_c[index]/12/IIP_3*(2*np.pi*h)*( (np.sqrt(G)-1)/(G-1) )**3 )
                    else:
                        G=1
                        g4_IMD[index] = float(kappa_c[index]**2*freq_c[index]/12/IIP_3*(2*np.pi*h)/8 )

                if 'pumping_data_'+str(int(log_g)) in hdf5[sweep_name].keys():
                    del hdf5[sweep_name]['pumping_data_'+str(int(log_g))]
                grp = hdf5[sweep_name].create_group('pumping_data_'+str(int(log_g)))
                grp.create_dataset('freq_c', data = freq_c)
                grp.create_dataset('kappa_c', data = kappa_c)
                grp.create_dataset('kappa_t', data = kappa_t)
                grp.create_dataset('CW_freq', data = CW_freqs)              # CW frequencies
                grp.create_dataset('current', data = data_dict['current'])
                grp.create_dataset('date_linear', data = data_dict['date_linear'])
                grp.create_dataset('time_linear', data = data_dict['time_linear'])
                grp.create_dataset('IIP3', data = IIP3)
                grp.create_dataset('IMD_time', data = data_dict['IMD_time'])
                grp.create_dataset('IMD_date', data = data_dict['IMD_date'])
                grp.create_dataset('g4_IMD', data = g4_IMD)
                if log_g:
                    if noise_flag:
                        grp.create_dataset('NVR', data = NVR)
                        grp.create_dataset('NVR_spike', data = NVR_spike)
                    grp.create_dataset('P_1dB', data = P_1dB)
                    grp.create_dataset('Bandwidth', data = bandwidth)
                    grp.create_dataset('Gain', data = max_Gain)                 # gain at CW frequency
                    grp.create_dataset('Pump_power', data = data_dict['Pump_power'] + attenuation( data_dict['Pump_freq'], self.data_file, 'pump_attenuation') )
                    grp.create_dataset('Gain_array', data = Gains)
                    grp.create_dataset('Signal_pow_array', data = Signal_Powers_dBm)
                    grp.create_dataset('Pump_freq', data = data_dict['Pump_freq'])
                    grp.create_dataset('time', data = data_dict['time'])
                    grp.create_dataset('date', data = data_dict['date'])
            finally:
                hdf5.close()

    def load_pump_power_sweep(self):

        new_folder = self.device_folder+'pump_power_sweep'+SLASH
        gains = np.loadtxt(new_folder+'gain_sweep.txt')
        currents = np.loadtxt(new_folder+'curr.txt')
        pump_powers = np.loadtxt(new_folder+'pump_powers.txt')

        attributes = [a for a in dir(self) if not a.startswith('__') and not callable(getattr(self,a)) ]
        flux_calibrated = True if ('a' in attributes) and ('b' in attributes) else False

        hdf5 = h5py.File(self.data_file)
        try:
            if 'pump_power_sweep' in hdf5.keys():
                del hdf5['pump_power_sweep']
            grp = hdf5.create_group('pump_power_sweep')
            grp.create_dataset('current', data = currents)
            grp.create_dataset('pump_powers', data = pump_powers)
            grp.create_dataset('gains', data = gains)
            if flux_calibrated:
                flux = Flux(currents,self.a,self.b)
                grp.create_dataset('flux', data = flux)
        finally:
            hdf5.close()

    def add_two_tone_spectroscopy_data(self):

        new_path = self.device_folder + SLASH + 'two_tone_spectroscopy' + SLASH

        currents = np.loadtxt(new_path + 'current.txt')
        ag_freqs = np.loadtxt(new_path + 'ag_freqs.txt')
        real = np.loadtxt(new_path + 'real.txt')
        imag = np.loadtxt(new_path + 'imag.txt')
        ag_powers = np.loadtxt(new_path + 'ag_pows.txt')
        res_freqs = np.loadtxt(new_path + 'res_freq.txt')
        # normalize the phase to have 0 for no response
        data = np.vectorize(complex)(real, imag)
        z = data / np.abs(data)
        for ind,I in enumerate(currents):
            z0 = z[ind][0]
            z[ind] = z[ind]/z0
        z = np.angle(z,deg=True)

        hdf5 = h5py.File(self.data_file)
        try:
            if 'two_tone_spectroscopy_data' in hdf5.keys():
                del hdf5['two_tone_spectroscopy_data']
            grp = hdf5.create_group('two_tone_spectroscopy_data')
            grp.create_dataset('currents', data = currents)
            grp.create_dataset('ag_freqs', data = ag_freqs)
            grp.create_dataset('real', data = real)
            grp.create_dataset('imag', data = imag)
            grp.create_dataset('ag_powers', data = ag_powers)
            grp.create_dataset('res_freqs', data = res_freqs)
            grp.create_dataset('phase_normalized', data = z)
        finally:
            hdf5.close()

        self.plot_two_tone_spectroscopy()

    def plot_two_tone_spectroscopy(self):

        hdf5 = h5py.File(self.data_file)
        try:
            z = np.asarray(hdf5['two_tone_spectroscopy_data'].get('phase_normalized'))
            ag_freqs = np.asarray(hdf5['two_tone_spectroscopy_data'].get('ag_freqs'))
            currents = np.asarray(hdf5['two_tone_spectroscopy_data'].get('currents'))
        finally:
            hdf5.close()
        # colorplot the two-tone spectroscopy data
        dpi = 150
        fig1, ax1 = plt.subplots(1,dpi=dpi)
        p1 = ax1.pcolormesh(currents*1e3, ag_freqs*1e-9, np.transpose(z), cmap='RdBu',vmin=-180,vmax=180)
        fig1.colorbar(p1,ax=ax1, label=r'$\rm Phase \,(deg)$')
        ax1.set_ylabel(r'${\rm Frequency \,(GHz)}$')
        ax1.set_xlabel(r'${\rm Current\,(mA)}$')
        fig1.savefig(self.device_folder + 'phase_colormap.png',dpi=dpi)

    def load_pump_params_sweep(self, meas_name, sweep_name='pump_params_sweep', IMD_flag=False, NVR_flag=True):

        self.pumping_meas_name = meas_name
        nominal_gain_array = np.loadtxt(self.device_folder+sweep_name+SLASH+'gains.txt')

        nominal_gain_array = [nominal_gain_array]

        hdf5 = h5py.File(self.data_file)
        try:
            if sweep_name in hdf5.keys():
                del hdf5[sweep_name]
            grp = hdf5.create_group(sweep_name)
            grp.create_dataset('gains', data = nominal_gain_array)

            for log_g in nominal_gain_array:
                temp_folder = sweep_name+SLASH+'Gain_'+str(int(log_g))+SLASH
                data_dict = {}
                print(np.loadtxt(self.device_folder+temp_folder+'date_linear.txt'))
                date_linear = float_to_DateTime(np.loadtxt(self.device_folder+temp_folder+'date_linear.txt'))
                time_linear = float_to_DateTime(np.loadtxt(self.device_folder+temp_folder+'time_linear.txt'))
                current = np.loadtxt(self.device_folder+temp_folder+'current.txt')

                # load linear fits
                freq_c = float( np.asarray(hdf5[date_linear][time_linear]['LIN'][meas_name]['fits'].get('f0')) )
                kappa_c = float( np.asarray(hdf5[date_linear][time_linear]['LIN'][meas_name]['fits'].get('kc')) )
                kappa_i = float( np.asarray(hdf5[date_linear][time_linear]['LIN'][meas_name]['fits'].get('ki') ))
                kappa_t = kappa_c + kappa_i

                data_dict['date'] = np.loadtxt(self.device_folder+temp_folder+'date.txt')
                data_dict['time'] = np.loadtxt(self.device_folder+temp_folder+'time.txt')
                if IMD_flag:
                    data_dict['IMD_time'] = np.loadtxt(self.device_folder+temp_folder+'IMD_time.txt')
                    data_dict['IMD_date'] = np.loadtxt(self.device_folder+temp_folder+'IMD_date.txt')

                data_dict['Pump_power'] = np.loadtxt(self.device_folder+temp_folder+'pump_power.txt')
                data_dict['Pump_freq'] = np.loadtxt(self.device_folder+temp_folder+'pump_freq.txt')*1e9
                CW_freqs = np.zeros(len(data_dict['Pump_freq']))
                P_1dB = np.zeros(len(data_dict['Pump_freq']))
                NVR = np.zeros(len(data_dict['Pump_freq']))
                period_doubling_peak = np.zeros(len(data_dict['Pump_freq']))
                IIP3 = np.zeros(len(data_dict['Pump_freq']))
                max_Gain = np.zeros(len(data_dict['Pump_freq']))
                bandwidth = np.zeros(len(data_dict['Pump_freq']))
                Gains = np.zeros((len(data_dict['Pump_freq']),201))
                Signal_Powers_dBm = np.zeros((len(data_dict['Pump_freq']),201))
                # TODO change this number to the size of signal powers array

                for index, pump_freq in enumerate(data_dict['Pump_freq']):

                    grp.attrs['freq_c'] = freq_c
                    grp.attrs['kappa_i'] = kappa_i
                    grp.attrs['kappa_c'] = kappa_c
                    grp.attrs['kappa_t'] = kappa_t
                    grp.attrs['current'] = current

                    # define all data folders (dates/times)
                    date = float_to_DateTime( data_dict['date'][index] )
                    time = float_to_DateTime( data_dict['time'][index] )

                    if IMD_flag:
                        IMD_time = float_to_DateTime( data_dict['IMD_time'][index] )
                        IMD_date = float_to_DateTime( data_dict['IMD_date'][index] )

                    # continuous wave frequency for signal
                    freq_CW = hdf5[date][time]['POW']['powers'].attrs.get('freqINP')
                    CW_freqs[index] = freq_CW

                    # signal powers
                    Signal_Powers_dBm[index] = np.asarray( hdf5[date][time]['POW'].get('powers') )
                    Signal_Powers_dBm[index] = Signal_Powers_dBm[index] + attenuation(freq_CW, self.data_file, 'signal_attenuation')

                    # Gains
                    ind_CW = np.abs(np.asarray(hdf5[date][time]['memory'].get('frequencies'))-freq_CW).argmin() + 2            # index of approximately CW_frequency in the memory trace
                    a_out = np.asarray( hdf5[date][time]['memory'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['memory'][meas_name].get('imag') )
                    logmag_pump_off = complex_to_PLOG(a_out)[0]
                    logmag_pump_off_ref= logmag_pump_off[ind_CW]
                    a_out = np.asarray( hdf5[date][time]['POW'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['POW'][meas_name].get('imag') )
                    logmag_pump_on = complex_to_PLOG(a_out)[0]
                    logmag_pump_on_ref = logmag_pump_on[0]      # logmag at the lowest CW power where amplifier is presumably not saturated yet
                    gain_ref = logmag_pump_on_ref-logmag_pump_off_ref
                    Gains[index] = logmag_pump_on - logmag_pump_on_ref + gain_ref
                    max_Gain[index] = gain_ref

                    # Figuring out gain profile in frequency
                    a_out = np.asarray( hdf5[date][time]['memory'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['memory'][meas_name].get('imag') )
                    logmag_pump_off = complex_to_PLOG(a_out)[0]
                    a_out = np.asarray( hdf5[date][time]['LIN'][meas_name].get('real') )+ 1j*np.asarray( hdf5[date][time]['LIN'][meas_name].get('imag') )
                    logmag_pump_on = complex_to_PLOG(a_out)[0]
                    gain_profile = logmag_pump_on - logmag_pump_off

                    # Finding the 3 dB bandwidth relative to CW frequency
                    ind_left, ind_right = find_3dB_bandwidth(gain_profile, ind_CW)
                    bandwidth[index] = np.asarray(hdf5[date][time]['memory'].get('frequencies'))[ind_right]-np.asarray(hdf5[date][time]['memory'].get('frequencies'))[ind_left]

                    # P_1dB and IIP3 points
                    Saturation_power_ind = find_1dB_compression_point(Gains[index], gain_ref)
                    P_1dB[index] = Signal_Powers_dBm[index][Saturation_power_ind]
                    if IMD_flag:
                        data = np.asarray(hdf5[IMD_date][IMD_time]['POW']['"CH5_IIP3_11"'].get('logmag'))
                        derivative = derivative_smooth(data, N_smooth=15)
                        Ind = np.argmin( np.abs(derivative) )

                        IIP3[index] =np.mean(data[max(Ind-10,0):min(Ind+10,len(data)-1)])
                        IIP3[index] = IIP3[index] + attenuation( float(hdf5[IMD_date][IMD_time]['POW']['xs'].attrs.get('imd_f1_cw')), self.data_file, 'signal_attenuation' )
                        IIP_3 = dBm_to_Watts( IIP3[index] )

                    if NVR_flag:
                        # NVR
                        key1 = list(hdf5[date][time]['fits']['noise'].keys())[0]
                        key2 = list(hdf5[date][time]['fits']['noise'][key1].keys())[0]
    #                        NVR[index] = np.asarray(hdf5[date][time]['fits']['noise'][key1][key2].get('nvr'))
                        noise_raise = np.asarray(hdf5[date][time]['noise'][key1][key2].get('logmag')) - np.asarray(hdf5[date][time]['noise'][key1]['memory'][key2].get('logmag'))
                        ind_CW = np.abs(np.asarray(hdf5[date][time]['noise'][key1].get('frequencies'))-freq_CW).argmin() + 2
                        NVR[index] = noise_raise[ind_CW]
#                        period_doubling_peak[index] = max(noise_raise) - NVR[index]
#                        period_doubling_peak[index] = noise_raise[ind_CW+3] - NVR[index]

#                if 'pump_params_sweep_'+str(int(log_g)) in grp.keys():
#                    del grp['pump_params_sweep_'+str(int(log_g))]
                grp1 = grp.create_group('pump_params_sweep_'+str(int(log_g)))

                grp1.create_dataset('CW_freq', data = CW_freqs)              # CW frequencies
                grp1.create_dataset('P_1dB', data = P_1dB)
                grp1.create_dataset('Bandwidth', data = bandwidth)
                grp1.create_dataset('Gain', data = max_Gain)                 # gain at CW frequency
                grp1.create_dataset('Pump_power', data = data_dict['Pump_power'] + attenuation( data_dict['Pump_freq'], self.data_file, 'pump_attenuation') )
                grp1.create_dataset('Gain_array', data = Gains)
                grp1.create_dataset('Signal_pow_array', data = Signal_Powers_dBm)
                grp1.create_dataset('Pump_freq', data = data_dict['Pump_freq'])
                grp1.create_dataset('time', data = data_dict['time'])
                grp1.create_dataset('date', data = data_dict['date'])
                if NVR_flag:
                    grp1.create_dataset('NVR', data = NVR)
                    grp1.create_dataset('period_doubling_peak', data = period_doubling_peak)
                if IMD_flag:
                    grp1.create_dataset('IIP3', data = IIP3)
                    grp1.create_dataset('IMD_time', data = data_dict['IMD_time'])
                    grp1.create_dataset('IMD_date', data = data_dict['IMD_date'])
        finally:
            hdf5.close()

    def add_array_modes_v2(self, array):
        if array != {}:
            hdf5 = h5py.File(self.data_file)
            try:
                if 'array_modes' in hdf5.keys():
                    del hdf5['array_modes']
                grp = hdf5.create_group('array_modes')
                for n in array:
                    i,f = array[n]
                    mode_grp = grp.create_group(n)
                    mode_grp.create_dataset('currents', data = i)
                    mode_grp.create_dataset('res_freqs', data = f)
            finally:
                hdf5.close()

    def load_array_modes_v2(self):
        hdf5 = h5py.File(self.data_file)
        try:
            if 'array_modes' in hdf5.keys():
                array = {}
                for x in hdf5['array_modes'].keys():
                    currents = np.asarray(hdf5['array_modes'][x].get('currents'))
                    freqs = np.asarray(hdf5['array_modes'][x].get('res_freqs'))
                    array[x] = (currents, freqs)
                flag = True
            else:
                flag = False
        finally:
            hdf5.close()
        return array if flag else None

    def mode_detector_flux_sweep(self, date_time=None, meas_name='"CH1_S11_1"'):

        date, time = date_time if date_time else [self.flux_sweep_date, self.flux_sweep_time]
        array_modes = mode_detector_flux_sweep(self.data_file, date, time, meas_name = meas_name,
                                               savepath = self.device_folder, ftype='.png')
        self.add_array_modes_v2(array_modes)


### Machine-Learning Library - Datasets
### MLLIB template
### DataType - Vectors dense

from __future__ import print_function
from pyspark.ml.regression import LinearRegression
from pyspark.sql import SparkSession
from pyspark.ml.linalg import Vectors

def main():
    lines   = spark.sparkContext.textFile("/Users/atul/Google Drive/2.atul/Atul-work/spark/regression.txt")
    data    = lines.map(lambda x: x.split(",")).map(lambda x: (float(x[0]), Vectors.dense(float(x[1]))))

    ## Convert this RDD to a DataFrame
    colNames    = ["label", "features"]
    df          = data.toDF(colNames)

    ## Let's split our data into training data and testing data
    trainTest   = df.randomSplit([0.5, 0.5])
    trainingDF  = trainTest[0]
    testDF      = trainTest[1]

    ## Create Linear Regression model
    lir     = LinearRegression(maxIter=10, regParam=0.3, elasticNetParam=0.8)

    ## Train model using our training data
    model   = lir.fit(trainingDF)

    ## See if we can predict values in our test data.
    ## Generate predictions using our linear regression model for all features in our
    ## test dataframe:
    fullPredictions = model.transform(testDF).cache()

    ## Extract the predictions and the "known" correct labels.
    predictions     = fullPredictions.select("prediction").rdd.map(lambda x: x[0])
    labels          = fullPredictions.select("label").rdd.map(lambda x: x[0])

    ## Zip them together
    predictionAndLabel = predictions.zip(labels).collect()

    # Print out the predicted and actual values for each point
    for prediction in predictionAndLabel:
      print(prediction)

if __name__=='__main__':
    spark = SparkSession.builder.appName("LinearRegression").getOrCreate()
    main()
    spark.stop()
    print("\nDone")

spark.stop()


from django.utils import timezone
from enum import Enum
from django.contrib.auth import get_user_model
from django.core.validators import MinLengthValidator, MinValueValidator, MaxValueValidator
from django.db import models
from django_cleanup import cleanup
from imagekit.processors import ResizeToFill
from imagekit.models import ImageSpecField

User = get_user_model()

class CarMake(models.Model):
    MAX_MAKE_LENGTH = 16

    make = models.CharField(
        verbose_name='Make',
        max_length=MAX_MAKE_LENGTH,
        unique=True,
        choices=()
    )

    def __str__(self):
        return f"{self.make}"

    class Meta:
        ordering = ['make']
        verbose_name_plural = "Manufacturers"

# Create your models here.
class CarModel(models.Model):
    MAX_MODEL_LENGTH = 50
    MIN_MODEL_LENGTH = 1

    make = models.ForeignKey(
        CarMake,
        on_delete=models.CASCADE,
        related_name='models'
    )
    model = models.CharField(
        verbose_name='Car Model',
        max_length=MAX_MODEL_LENGTH,
        choices=(),
    )

    def __str__(self):
        return f"{self.model}"

    class Meta:
        ordering = ['make', 'model']
        unique_together = ('make', 'model')
        verbose_name_plural = 'Models'

class ChoicesEnum(Enum):
    @classmethod
    def choices(cls):
        return [(c.name, c.value) for c in cls]

    @classmethod
    def max_length(cls):
        return max(len(c.value) for c in cls)

class CarListingBodyStyleChoices(ChoicesEnum):
    Sedan = 'Sedan'
    Coupe = 'Coupe'
    SUV = 'SUV'
    Hatchback = 'Hatchback'
    Sports = 'Sports Car'
    Wagon = 'Wagon'

class CarListingTransmissionChoices(ChoicesEnum):
    Manual = 'Manual'
    Automatic = 'Automatic'

class CarListingDriveTypeChoices(ChoicesEnum):
    FWD = 'FWD'
    RWD = 'RWD'
    AWD = 'AWD'

class CarListingEngineTypeChoices(ChoicesEnum):
    Petrol = 'Petrol'
    Diesel = 'Diesel'
    Hybrid = 'Hybrid'
    Electric = 'Electric'

class CarListingEuroChoices(ChoicesEnum):
    Euro_1 = 'Euro 1'
    Euro_2 = 'Euro 2'
    Euro_3 = 'Euro 3'
    Euro_4 = 'Euro 4'
    Euro_5 = 'Euro 5'
    Euro_6 = 'Euro 6'

@cleanup.select
class CarListing(models.Model):
    MAX_CAR_TITLE_LENGTH = 50
    MIN_CAR_TITLE_LENGTH = 5
    MIN_CAR_TITLE_LENGTH_MESSAGE = f"Your car title should be at least {MIN_CAR_TITLE_LENGTH} characters long."

    MIN_PRICE = 1

    MIN_BODY_STYLE_LENGTH = 4
    MAX_BODY_STYLE_LENGTH = 10

    MIN_YEAR = 1886

    MIN_HORSE_POWER = 1

    MIN_MILEAGE = 1

    MIN_COLOR_LENGTH = 2
    MAX_COLOR_LENGTH = 16

    MIN_TRANSMISSION_LENGTH = 4
    MAX_TRANSMISSION_LENGTH = 16

    MIN_DRIVE_TYPE_LENGTH = 1
    MAX_DRIVE_TYPE_LENGTH = 8

    MIN_FUEL_TYPE_LENGTH = 2
    MAX_FUEL_TYPE_LENGTH = 12

    MAX_EURO_LENGTH = 10

    MAX_ENGINE_TYPE_LENGTH = 10

    MIN_ENGINE_LITRES = 0.0
    MAX_ENGINE_LITRES = 8
    MAX_ENGINE_LITRES_MESSAGE = f'You cannot have an engine above {MAX_ENGINE_LITRES} litres.'
    MIN_ENGINE_LITRES_MESSAGE = f'You cannot have an engine below {MIN_ENGINE_LITRES} litres.'

    LOW_KMS_MAX = 50000

    car_title = models.CharField(
        null=False,
        blank=False,
        max_length=MAX_CAR_TITLE_LENGTH,
        validators=(
            MinLengthValidator(MIN_CAR_TITLE_LENGTH, message=MIN_CAR_TITLE_LENGTH_MESSAGE),
        )
    )

    make = models.ForeignKey(
        CarMake,
        on_delete=models.CASCADE,
        related_name="Manafacturer",
    )
    model = models.ForeignKey(
        CarModel,
        on_delete=models.CASCADE,
        related_name='Car_Model',
    )

    year = models.IntegerField(
        verbose_name='Year',
        null=False,
        blank=False,
        validators=(
            MinValueValidator(MIN_YEAR),
        )
    )

    price = models.PositiveIntegerField(
        verbose_name='$/Day',
        null=False,
        blank=False,
    )

    transmission = models.CharField(
        verbose_name='Transmission',
        null=False,
        blank=False,
        max_length=CarListingTransmissionChoices.max_length(),
        choices=CarListingTransmissionChoices.choices()
    )

    engine_type = models.CharField(
        verbose_name='Engine Type',
        null=False,
        blank=False,
        max_length=CarListingEngineTypeChoices.max_length(),
        choices=CarListingEngineTypeChoices.choices()
    )
    engine_litres = models.FloatField(
        verbose_name='Engine Litres',
        null=False,
        blank=False,
        validators=(
            MinValueValidator(MIN_ENGINE_LITRES,
                              message=MIN_ENGINE_LITRES_MESSAGE),
            MaxValueValidator(MAX_ENGINE_LITRES,
                              message=MAX_ENGINE_LITRES_MESSAGE),
        )

    )
    horse_power = models.IntegerField(
        verbose_name='Horse Power',
        null=False,
        blank=False,
        validators=(
            MinValueValidator(MIN_HORSE_POWER),
        )
    )

    mileage = models.IntegerField(
        verbose_name='Mileage',
        null=False,
        blank=False,
        validators=(
            MinValueValidator(MIN_MILEAGE),
        )
    )

    body_style = models.CharField(
        verbose_name='Body Style',
        null=False,
        blank=False,
        max_length=CarListingBodyStyleChoices.max_length(),
        choices=CarListingBodyStyleChoices.choices()
    )

    euro = models.CharField(
        verbose_name='Euro',
        null=False,
        blank=False,
        max_length=CarListingEuroChoices.max_length(),
        choices=(
            ('1', 'Euro 1'),
            ('2', 'Euro 2'),
            ('3', 'Euro 3'),
            ('4', 'Euro 4'),
            ('5', 'Euro 5'),
            ('6', 'Euro 6'),
        ),
    )

    drive_type = models.CharField(
        verbose_name='Drive Type',
        null=False,
        blank=False,
        max_length=CarListingDriveTypeChoices.max_length(),
        choices=CarListingDriveTypeChoices.choices()
    )

    created = models.DateTimeField(
        auto_now_add=True
    )

    modified = models.DateTimeField(
        auto_now=True
    )

    attached_user = models.ForeignKey(
        User,
        null=True,
        on_delete=models.CASCADE,
        related_name='cars_attached',
    )
    is_available = models.BooleanField(
        default=True
    )

    def __str__(self):
        return f"{self.make} {self.model} {self.year}"

    @property
    def car_make_and_model(self):
        return f"{self.make} {self.model}"

    @property
    def low_kms(self):
        return True if self.mileage <= self.LOW_KMS_MAX else False

    def delta(self):
        delta = timezone.now() - self.created
        seconds = int(delta.total_seconds())
        minutes = seconds // 60
        hours = minutes // 60
        days = hours // 24
        if days >= 1:
            return f"{days} days"
        elif minutes > 60:
            return f"{hours} hours"
        elif 1 <= minutes < 60:
            return f"{minutes} minutes"
        elif minutes < 1:
            return f"{seconds} seconds"

@cleanup.select
class PhotoCarModel(models.Model):
    MAX_IMAGES_ALLOWED = 3

    image = models.ImageField(upload_to='car_photos/', blank=True, null=True)
    thumbnail = ImageSpecField(
        source='image',
        processors=[ResizeToFill(200, 200)],
        format='JPEG',
        options={"quality": 60},
    )

    car = models.ForeignKey(
        to=CarListing,
        on_delete=models.CASCADE,
        related_name='car_pics'
    )

    @property
    def image_url(self):
        if self.image and hasattr(self.image, 'url'):
            return self.image.url


#!/usr/bin/env python
# Utility for predicting labels based on pre-trained RF model
import sys
import pandas as pd
from joblib import load
from sklearn.metrics import classification_report

if __name__ == "__main__":
    features = pd.read_csv(sys.argv[1], header=0, index_col=0, sep="\t")
    outName = sys.argv[2]
    model = load(sys.argv[3])
    metadata = None
    if len(sys.argv) == 5:
        metadata = pd.read_csv(sys.argv[4], sep="\t", header=None, index_col=0, dtype=str)
        metadata.index = metadata.index.astype(str)

    M = features.shape[0]  # features count
    N = features.shape[1]  # samples  count

    X = features.T
    y_pred = model.predict(X)

    outFile = open(outName+".tsv", "w")
    for sam, pred in zip(X.index, y_pred):
        print(sam, pred, sep="\t", file=outFile)
    outFile.close()

    if metadata is not None:
        y = [metadata.loc[i, 1] for i in X.index]
        print("Predictions accuracy compared with given labels:")
        print(classification_report(y, y_pred, zero_division=0))


import streamlit as st
import pickle
import string
from nltk.corpus import stopwords
import nltk
nltk.download('all')
from nltk.stem.porter import PorterStemmer
import base64

ps = PorterStemmer()

def transform_text(text):
    text = text.lower()
    text = nltk.word_tokenize(text)

    y = []
    for i in text:
        if i.isalnum():
            y.append(i)

    text = y[:]
    y.clear()

    for i in text:
        if i not in stopwords.words('english') and i not in string.punctuation:
            y.append(i)

    text = y[:]
    y.clear()

    for i in text:
        y.append(ps.stem(i))

    return " ".join(y)

tfidf = pickle.load(open('C:/Users/user/PycharmProjects/ai-2/venv/vectorizer.pkl','rb'))
model = pickle.load(open('C:/Users/user/PycharmProjects/ai-2/venv/model.pkl','rb'))

main_bg = "C:/Users/user/PycharmProjects/ai-2/venv/img_16.png"
main_bg_ext = "png"

st.markdown(
    f"""

    """,
    unsafe_allow_html=True
)

st.markdown("
Tweet Spam Classifier
",unsafe_allow_html=True)
input_sms = st.text_area("",value='Enter a Tweet')

if st.button('Predict'):

        # 1. preprocess
        transformed_sms = transform_text(input_sms)
        # 2. vectorize
        vector_input = tfidf.transform([transformed_sms])
        # 3. predict
        result = model.predict(vector_input)[0]
        # 4. Display
        if result == 1:
            st.markdown("
Spam
",unsafe_allow_html=True)
        else:
            st.markdown("
Not Spam
",unsafe_allow_html=True)


import webapp2
import handlers

PAGE_RE = r'(/(?:[a-zA-Z0-9_-]+/?)*)'
app = webapp2.WSGIApplication([('/signup', handlers.SignUpHandler),
                               ('/login', handlers.LoginHandler),
                               ('/logout', handlers.LogoutHandler),
                               ('/_edit' + PAGE_RE, handlers.EditPage),
                               ('/_history' + PAGE_RE, handlers.HistoryHandler),
                               (PAGE_RE, handlers.WikiPage),
                               ],
                              debug=True)


from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton

from looping import pg
from text.language.main import Text_main
from text.function.function import TextFunc
from text.language.ru import Ru_language as Model
Txt = Text_main()
func = TextFunc()

class InlineDriverData:
    def __init__(self, language: str, spots: list = None, delivery: bool = False, truck: bool = False,
                 pay_id: int = None, car_type: str = None, car_subtype: str = None, order_id: int = None):
        self.__markup = None
        self.__language = language
        self.__spots = spots
        self.__delivery = delivery
        self.__truck = truck
        self.__pay_id = pay_id
        self.__order_id = order_id
        self.__car_type = car_type
        self.__car_subtype = car_subtype
        self.__Lang: Model = Txt.language[language]
        self.__back = InlineKeyboardButton(text=self.__Lang.buttons.driver.back, callback_data="back")

    async def menu_auto(self):
        markup = InlineKeyboardMarkup(row_width=1)
        b1 = InlineKeyboardButton(text=self.__Lang.buttons.personal_cabinet.model, callback_data=f"model")
        b2 = InlineKeyboardButton(text=self.__Lang.buttons.personal_cabinet.number, callback_data=f"number")
        b3 = InlineKeyboardButton(text=self.__Lang.buttons.personal_cabinet.color, callback_data=f"colors")
        markup.add(b1, b2, b3, self.__back)
        return markup

    async def menu_taxi(self):
        self.__markup = InlineKeyboardMarkup(row_width=3)
        b1 = InlineKeyboardButton(text=self.__Lang.buttons.personal_cabinet.truck, callback_data=f"truck")
        self.__markup.add(b1)
        await self._taxi()
        self.__markup.add(self.__back)
        return self.__markup

    async def _taxi(self):
        cars = await pg.id_and_model(car_type="taxi")
        for index, car in enumerate(cars):
            b = InlineKeyboardButton(text=car[1], callback_data=f"car_{car[0]}")
            if index == 0:
                self.__markup.add(b)
            else:
                self.__markup.insert(b)

    async def menu_car_subtype(self):
        self.__markup = InlineKeyboardMarkup(row_width=1)
        await self._car_subtype()
        self.__markup.add(self.__back)
        return self.__markup

    async def _car_subtype(self):
        elements = await pg.id_and_car_subtype(car_type="truck", language=self.__language)
        for element in elements:
            b = InlineKeyboardButton(text=element[1], callback_data=f"subtype_{element[0]}")
            self.__markup.insert(b)

    async def menu_trucks(self):
        self.__markup = InlineKeyboardMarkup(row_width=3)
        await self._trucks()
        self.__markup.add(self.__back)
        return self.__markup

    async def _trucks(self):
        print(self.__car_type, self.__car_subtype)
        cars = await pg.id_and_model2(car_type=self.__car_type, car_subtype=self.__car_subtype)
        for index, car in enumerate(cars):
            b = InlineKeyboardButton(text=car[1], callback_data=f"truck_{car[0]}")
            self.__markup.insert(b)

    async def menu_color(self):
        self.__markup = InlineKeyboardMarkup(row_width=3)
        await self._color()
        self.__markup.add(self.__back)
        return self.__markup

    async def _color(self):
        colors = await pg.id_and_color(language=self.__language)
        for color in colors:
            b = InlineKeyboardButton(text=color[1], callback_data=f"color_{color[0]}")
            self.__markup.insert(b)

    async def menu_back(self):
        self.__markup = InlineKeyboardMarkup(row_width=1)
        self.__markup.add(self.__back)
        return self.__markup

    async def menu_agreement(self):
        self.__markup = InlineKeyboardMarkup(row_width=2)
        b_no = InlineKeyboardButton(text=self.__Lang.buttons.driver.no, callback_data='back')
        b_yes = InlineKeyboardButton(text=self.__Lang.buttons.driver.yes, callback_data='agree')
        self.__markup.add(b_no, b_yes, self.__back)
        return self.__markup

    async def menu_personal_data(self):
        markup = InlineKeyboardMarkup(row_width=1)
        b1 = InlineKeyboardButton(text=self.__Lang.buttons.personal_cabinet.name, callback_data=f"name")
        b2 = InlineKeyboardButton(text=self.__Lang.buttons.personal_cabinet.phone, callback_data=f"phone")
        b3 = InlineKeyboardButton(text=self.__Lang.buttons.personal_cabinet.car, callback_data=f"auto")
        markup.add(b1, b2, b3)
        return markup

    async def menu_spots(self):
        self.__markup = InlineKeyboardMarkup(row_width=3)
        # await self._delivery()
        await self._spots()
        accept = InlineKeyboardButton(text=self.__Lang.buttons.driver.accept, callback_data="accept")
        self.__markup.add(accept)
        return self.__markup

    async def _delivery(self):
        text = self.__Lang.buttons.driver.delivery
        text = text if self.__delivery is False else "✅" + text
        b = InlineKeyboardButton(text=text, callback_data=f"delivery")
        self.__markup.insert(b)

    async def _truck(self):
        text = self.__Lang.buttons.driver.truck
        text = text if self.__truck is False else "✅" + text
        b = InlineKeyboardButton(text=text, callback_data=f"truck")
        self.__markup.insert(b)

    async def _spots(self):
        button_list = [1, 15, 17, 19, 24, 29]
        spots = await pg.id_and_spot(language=self.__language)
        # i = 1
        for index, spot in enumerate(spots):

            b = InlineKeyboardButton(text=await self._check_spot(spot), callback_data=f"spot_{spot[0]}")

            if index == 0:
                self.__markup.add(b)
                await self._delivery()
                await self._truck()

            elif index in button_list:

                self.__markup.add(b)
            else:
                self.__markup.insert(b)
            # i += 1
            #
            # i = 1 if i == 4 else i
            # i = 2 if i == 5 else i

    async def _check_spot(self, spot: list):
        if spot[0] in self.__spots:
            return "✅" + spot[1]
        else:
            return spot[1]

    async def menu_accept(self):
        markup = InlineKeyboardMarkup(row_width=1)
        accept = InlineKeyboardButton(text=self.__Lang.buttons.driver.accept, callback_data=f"paymentAccept_{self.__pay_id}")
        reject = InlineKeyboardButton(text=self.__Lang.buttons.driver.reject, callback_data=f"paymentReject_{self.__pay_id}")
        markup.add(accept, reject)
        return markup

    async def menu_cancel_active_order(self):
        Lang: Model = Txt.language[self.__language]
        markup = InlineKeyboardMarkup(row_width=1)
        b1 = InlineKeyboardButton(text=Lang.buttons.active_order.cancel, callback_data=f"cancel_{self.__order_id}")
        markup.add(b1)
        return markup

    async def menu_delete_active_order(self):
        Lang: Model = Txt.language[self.__language]
        markup = InlineKeyboardMarkup(row_width=1)
        b1 = InlineKeyboardButton(text=Lang.buttons.active_order.delete, callback_data=f"delete_{self.__order_id}")
        markup.add(b1)
        return markup

    async def menu_balance(self):
        markup = InlineKeyboardMarkup(row_width=1)
        b = InlineKeyboardButton(text=self.__Lang.buttons.wallet.balance, callback_data="balance")
        markup.add(b)
        return markup

    async def menu_cash(self):
        self.__markup = InlineKeyboardMarkup(row_width=2)
        await self._cash()
        self.__markup.add(self.__back)
        return self.__markup

    async def _cash(self):
        for cash in Txt.money.wallet.price:
            text = await func.int_to_str(num=cash)
            b = InlineKeyboardButton(text=text, callback_data=f"cash_{cash}")
            self.__markup.insert(b)

    async def menu_pay_way(self):
        markup = InlineKeyboardMarkup(row_width=3)
        b1 = InlineKeyboardButton(text=self.__Lang.buttons.wallet.payme, callback_data="Payme")
        b2 = InlineKeyboardButton(text=self.__Lang.buttons.wallet.click, callback_data="Click")
        b3 = InlineKeyboardButton(text=self.__Lang.buttons.wallet.paynet, callback_data="Paynet")
        markup.add(b1, b2, b3).add(self.__back)
        return markup

    async def menu_payment(self):
        markup = InlineKeyboardMarkup(row_width=1)
        b = InlineKeyboardButton(text=self.__Lang.buttons.wallet.pay, callback_data="pay")
        markup.add(b, self.__back)
        return markup

    async def payme_url(self):
        markup = InlineKeyboardMarkup(row_width=1)
        url_button = InlineKeyboardButton(text=self.__Lang.buttons.wallet.pay, callback_data="payed")
        markup.add(url_button, self.__back)
        return markup

    async def click_url(self):
        markup = InlineKeyboardMarkup(row_width=1)
        url_button = InlineKeyboardButton(text=self.__Lang.buttons.wallet.pay,  callback_data="payed")
        markup.add(url_button, self.__back)
        return markup


from flask import Flask, render_template
from flask import abort, request
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.restful import Api

app = Flask(__name__)
app.config.from_object('config')
db = SQLAlchemy(app)
api = Api(app)

@app.teardown_appcontext
def shutdown_session(exception=None):
    db.session.remove()

@app.errorhandler(404)
def not_found(error):
    return render_template('404.html'), 404

from web_hooks import OrderHook, SmartCollection

api.add_resource(OrderHook, '/wh_orders/', endpoint='wh_orders_ep')
api.add_resource(SmartCollection, '/wh_smart_cols/', endpoint='wh_smart_cols_ep')


from django.contrib import admin
from django.core.exceptions import ValidationError
from django.forms import BaseInlineFormSet
from .models import Article, Scope, Tag

class RelationshipInlineFormset(BaseInlineFormSet):
    def clean(self):
        i = 0
        for form in self.forms:
            dictionary = form.cleaned_data
            if dictionary.get('is_main'):
                i += 1
            else:
                continue
        if i == 0:
            raise ValidationError('Выберите главную тему')
        elif i > 1:
            raise ValidationError('Главной темой может быть только одна')
        return super().clean()
    # def clean(self):
    #     count = 0
    #     # for form in self.forms:
    #     #     if 'is_main' in form.cleaned_data:
    #     #         if form.cleaned_data['is_main']:
    #     #             count += 1
    #     #     if count == 0:
    #     #         raise ValidationError('Какая-то ошибка')
    #     #     else:
    #     #         pass

    #     for self.deleted_forms in self._should_delete_form(form):
    #         continue
    #     elif self.form.cleaned_data.get(‘is_main’):
    #         count += 1
    #     if count > 1:
    #         raise ValidationError(‘Is_main should be only one tag’)
    #     elif count == 0:
    #         raise ValidationError(‘Assign is_main’)

    #     return super().clean()

class RelationshipInline(admin.TabularInline):
    model = Scope
    formset = RelationshipInlineFormset

@admin.register(Article)
class ObjectAdmin(admin.ModelAdmin):
    inlines = [RelationshipInline]

@admin.register(Tag)
class ObjectAdmin(admin.ModelAdmin):
    pass


from flask import Flask, render_template
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///market.db'
db = SQLAlchemy(app)

db.create_all()

#local database that will hold items
class Item(db.Model):
    #each of these lines will be elements in our db table
    id = db.Column(db.Integer(), primary_key=True)
    name = db.Column(db.String(length=30), nullable=False, unique=True)
    price = db.Column(db.Integer(), nullable=False)
    barcode = db.Column(db.String(length=12), nullable=False, unique=True)
    description = db.Column(db.String(length=1024), nullable=False, unique=True)

    def __repr__(self):
        return f'Item {self.name}'

@app.route('/')
@app.route('/home')
def home_page():
    return render_template('home.html')

@app.route('/market')
def market_page():  #market page with items

      items = [
        {'id': 1, 'name': 'Phone', 'barcode': '893212299897', 'price': 500},
        {'id': 2, 'name': 'Laptop', 'barcode': '123985473165', 'price': 900},
        {'id': 3, 'name': 'Keyboard', 'barcode': '231985128446', 'price': 150}
      ]

      return render_template('market.html', items=items)

if __name__== "__main__":
     app.run(debug = True)

     #xml
     #javascript / bootstrap