import os
import yaml
import argparse
import subprocess
import logging
import json
from PIL import Image
from tqdm import tqdm

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

parser = argparse.ArgumentParser(description='Configure benchmark for Faster R-CNN')
parser.add_argument('--img-dir', help='Image directory name', type=str, required=True)
# parser.add_argument('--benchmark-type', help='Benchmark name, i.e. baseline, fsrcnn_x2, etc.', type=str, required=True)
# parser.add_argument('--sr-multiplier', type=int, help='Resolution upscaling multiplier, i.e. 1 for baseline', required=True)
# parser.add_argument('--device', default='0', help='Cuda device, i.e. 0 or 0,1,2,3 or cpu')
# parser.add_argument('--workers', type=int, default=8, help='Number of data loading workers')
# parser.add_argument('--batch-size', type=int, default=16, help='Batch size')
# parser.add_argument('--total-iterations', type=int, default=5000, help='Total training iterations')
parser.add_argument('--dataset-path', type=str, default='dataset', help='Path to root dataset directory')
args = parser.parse_args()

dataset_configs = {
    'apple_detection_drone_brazil': {
        'img_res': 256,
        'total_train_imgs': 483
    },
    'apple_detection_spain': {
        'img_res': 512,
        'total_train_imgs': 677
    },
    'apple_detection_usa': {
        'img_res': 640,
        'total_train_imgs': 1598
    },
    'fruit_detection_worldwide': {
        'img_res': 800,
        'total_train_imgs': 395
    },
    'grape_detection_californianight': {
        'img_res': 416,
        'total_train_imgs': 106
    },
    'mango_detection_australia': {
        'img_res': 512,
        'total_train_imgs': 870
    },
    'wheat_head_counting': {
        'img_res': 512,
        'total_train_imgs': 4558
    },
}

def run_if_exists(path, func):
    if os.path.exists(path):
        func(path)

def create_coco_annotations(img_lbl_path, img_dir, config_yaml):

    img_path = os.path.join(img_lbl_path, img_dir)
    lbl_path = os.path.join(img_lbl_path, 'labels')

    result = {
        'info': {
            'year': '2023',
            'description': 'Autogenerated COCO annotations',
        },
        'licenses': [{
            'id': 1,
            'url': 'https://creativecommons.org/licenses/by/4.0/',
            'name': 'CC BY 4.0'
        }],
        'categories': [
            {
                'id': i + 1,
                'name': config_yaml['names'][i],
                'supercategory': config_yaml['names'][i],
            }
            for i in range(len(config_yaml['names']))
        ],
        'images': [],
        'annotations': []
    }

    assert os.path.exists(img_path), f'Image path {img_path} does not exist'
    assert os.path.exists(lbl_path), f'Label path {lbl_path} does not exist'

    img_id = 0
    annotation_id = 0
    img_files = next(os.walk(img_path))[2]
    for img in tqdm(img_files):
        img_name = os.path.splitext(img)[0]
        label_file = os.path.join(lbl_path, f'{img_name}.txt')

        assert os.path.exists(label_file), f'Label file {label_file} does not exist'

        img_file = Image.open(os.path.join(img_path, img))
        img_width, img_height = img_file.size

        with open(label_file, 'r') as f:
            for line in f.readlines():
                line = line.strip()
                if line == '':
                    continue
                line = line.split(' ')
                category_id = int(line[0]) + 1
                x = int(float(line[1]) * img_width)
                y = int(float(line[2]) * img_height)
                w = int(float(line[3]) * img_width)
                h = int(float(line[4]) * img_height)

                result['annotations'].append({
                    'id': annotation_id,
                    'image_id': img_id,
                    'category_id': category_id,
                    'bbox': [x, y, w, h],
                    'area': w * h,
                    'segmentation': [],
                    'iscrowd': 0,
                })
                annotation_id += 1

        result['images'].append({
            'id': img_id,
            'license': 1,
            'file_name': img,
            'height': img_height,
            'width': img_width,
        })
        img_id += 1
    json.dump(result, open(os.path.join(img_lbl_path, 'annotations.json'), 'w'))

dataset_path = args.dataset_path
datasets = next(os.walk(dataset_path))[1]
for dataset in datasets:
    logger.info(f'Preparing {dataset}')
    d_train_path = os.path.join(os.path.abspath(dataset_path), dataset, 'train', args.img_dir)
    d_val_path = os.path.join(os.path.abspath(dataset_path), dataset, 'val', args.img_dir)
    d_test_path = os.path.join(os.path.abspath(dataset_path), dataset, 'test', args.img_dir)

    train_path = os.path.join(os.path.abspath(dataset_path), dataset, 'train', 'images')
    val_path = os.path.join(os.path.abspath(dataset_path), dataset, 'val', 'images')
    test_path = os.path.join(os.path.abspath(dataset_path), dataset, 'test', 'images')

    assert os.path.exists(d_train_path), f'Train path {d_train_path} does not exist'
    assert os.path.exists(d_val_path), f'Val path {d_val_path} does not exist'
    assert os.path.exists(d_test_path), f'Test path {d_test_path} does not exist'

    run_if_exists(train_path, os.unlink)
    run_if_exists(val_path, os.unlink)
    run_if_exists(test_path, os.unlink)
    os.symlink(d_train_path, train_path)
    os.symlink(d_val_path, val_path)
    os.symlink(d_test_path, test_path)

    with open(os.path.join(dataset_path, dataset, 'base.yaml'), 'r') as f:
        train_config = yaml.safe_load(f)

    base_dataset_path = os.path.join(os.path.abspath(dataset_path), dataset)
    run_if_exists(os.path.join(base_dataset_path, 'train', 'annotations.json'), os.remove)
    run_if_exists(os.path.join(base_dataset_path, 'val', 'annotations.json'), os.remove)
    run_if_exists(os.path.join(base_dataset_path, 'test', 'annotations.json'), os.remove)
    logger.info(f'Creating COCO annotations for {dataset}')

    create_coco_annotations(os.path.join(base_dataset_path, 'train'), args.img_dir, train_config)
    create_coco_annotations(os.path.join(base_dataset_path, 'val'), args.img_dir, train_config)
    create_coco_annotations(os.path.join(base_dataset_path, 'test'), args.img_dir, train_config)

    # total_train_imgs = dataset_configs[dataset]['total_train_imgs']
    # img_res = dataset_configs[dataset]['img_res']
    # epoch = args.total_iterations // (total_train_imgs // args.batch_size)

    # train_command = [
    #     f'python train.py --workers {args.workers} --device {args.device} --batch-size {args.batch_size} --epoch {epoch}',
    #     '--cfg cfg/training/yolov7.yaml --weights yolov7_training.pt --hyp data/hyp.scratch.custom.yaml',
    #     f'--data ../../dataset/{dataset}/config.yaml',
    #     f'--img-size {img_res * args.sr_multiplier} {img_res * args.sr_multiplier}',
    #     f'--name {dataset}',
    #     f'--project ../../results/{args.benchmark_type}/yolov7/train'
    # ]
    # train_command = ' '.join(train_command)
    # logger.info(f'Running training command: {train_command}')
    # subprocess.call(train_command.split(' '), cwd='./object_detectors/yolov7')

    # test_command = [
    #     f'python test.py --device 0 --weights ../../results/{args.benchmark_type}/yolov7/train/{dataset}/weights/best.pt',
    #     f'--data ../../dataset/{dataset}/config.yaml',
    #     f'--img-size {img_res * args.sr_multiplier} --batch 16',
    #     f'--task test --name {dataset}',
    #     f'--project ../../results/{args.benchmark_type}/yolov7/test',
    # ]
    # test_command = ' '.join(test_command)
    # logger.info(f'Running test command: {test_command}')
    # subprocess.call(test_command.split(' '), cwd='./object_detectors/yolov7')

    logger.info(f'Cleaning up {dataset}')
    run_if_exists(os.path.join(base_dataset_path, 'train', 'annotations.json'), os.remove)
    run_if_exists(os.path.join(base_dataset_path, 'val', 'annotations.json'), os.remove)
    run_if_exists(os.path.join(base_dataset_path, 'test', 'annotations.json'), os.remove)
    run_if_exists(train_path, os.unlink)
    run_if_exists(val_path, os.unlink)
    run_if_exists(test_path, os.unlink)
    logger.info(f'Finished {dataset}\n')

# logger.info(f'Benchmark {args.benchmark_type} completed')


# -*- coding: utf-8 -*-
# pylint: skip-file
# by Roman Golev

__doc__ = "Returns element Volume for a selected elements. Calculate totals in square meters"
__author__ = 'Roman Golev'
__title__ = "Volume\nQuery"

from types import NoneType
import clr
clr.AddReference("RevitAPI")
import Autodesk
from Autodesk.Revit.DB import *
from Autodesk.Revit.UI import *
from Autodesk.Revit.UI import Selection

import sys

clr.AddReference('System')
clr.AddReference('RevitAPIUI')
import pyrevit
from pyrevit import forms
from System.Collections.Generic import List
from core.selectionhelpers import CustomISelectionFilterByIdInclude, ID_WALLS
from Autodesk.Revit.UI.Selection import ObjectType

doc = __revit__.ActiveUIDocument.Document
uidoc = __revit__.ActiveUIDocument
uiapp = __revit__
app = uiapp.Application
t = Autodesk.Revit.DB.Transaction(doc)

versionNumber = uiapp.Application.VersionNumber
if "2019" in versionNumber or "2020" in versionNumber:
        from Autodesk.Revit.DB import UnitUtils
elif "2021" in versionNumber or "2022" in versionNumber or "2023" in versionNumber or "2024" in versionNumber:
        from Autodesk.Revit.DB import UnitUtils, UnitTypeId
else:
        from Autodesk.Revit.DB import UnitUtils, UnitTypeId

categories = doc.Settings.Categories
n = categories.Size

# Get unput: selected by user elements
def get_selection():
     selobject = uidoc.Selection.GetElementIds()
     if selobject.Count == 0:
          try:
               selection = uidoc.Selection.PickObjects(ObjectType.Element, CustomISelectionFilterByIdInclude(ID_WALLS), "Selection Objects")
          except:
               sys.exit()
     elif selobject.Count != 0:
          selection = selobject
     return selection

selobject = get_selection()
volumes = []

def get_host_volume(elem):
        vl = elem.get_Parameter(BuiltInParameter.HOST_VOLUME_COMPUTED).AsDouble()
        if "2019" in versionNumber or "2020" in versionNumber:
                return UnitUtils.ConvertToInternalUnits(vl, DisplayUnitType.DUT_CUBIC_METERS)
        if "2021" in versionNumber or "2022" in versionNumber or "2023" in versionNumber:
                return UnitUtils.ConvertFromInternalUnits(vl, UnitTypeId.CubicMeters)

def get_ifc_volume(el):
        geo = el.get_Geometry(Options())
        vl = 0
        for i in geo:
                if "Solid" in i.ToString():
                        if "2019" in versionNumber or "2020" in versionNumber:
                                vl += UnitUtils.ConvertToInternalUnits(i.Volume, DisplayUnitType.DUT_CUBIC_METERS)
                        if "2021" in versionNumber or "2022" in versionNumber or "2023" in versionNumber:
                                vl += UnitUtils.ConvertFromInternalUnits(i.Volume, UnitTypeId.CubicMeters)
        return vl

if selobject.Count == 0:
        print('Nothing selected')
elif selobject.Count != 0:
        for element in selobject:
                el = doc.GetElement(element)
                try:
                        vol = get_host_volume(el)
                        if float(vol) == 0.0:
                                raise ValueError
                        volumes.append(vol)
                        print(vol)
                except ValueError:
                        try:
                                vol = get_ifc_volume(el)
                                print(vol)
                                volumes.append(vol)
                        except:
                                print("Couldn't get the volume of element {}".format(el.Id))
                except:
                        print("Couldn't get the volume of element {}".format(el.Id))

print("Summary : {}".format(sum(volumes)) + " m³")

#TODO: Add option to calculate volume in case if it is no Volume property in model
# Based on Geometry


from django.shortcuts import get_object_or_404, redirect, render

from .forms import CommentForm

from blog.models import Post

from .models import Comment

# Create your views here.

def post_comment(request, post_pk):
    # 这里先过去被评论的文章，因为后面需要把评论和文章关联起来
    # 这里我们使用了django内置的函数get_objects_or_404
    # 从而获取post类 也就是文章详情
    post = get_object_or_404(Post, pk=post_pk)

    # http协议中只有post和get方法，2而表单的数据传输都是用过 post 请求，
    # 所以这里我们选择当 请求 为post是才处理表单数据
    if request.method == 'POST':
        # 用户评论时提交的数据 存在 request.POST
        # 我们利用这个数据结构 CommentForm 的实例，这样生成了field字段的（name email texe url）
        form = CommentForm(request.POST)
        # 生成评论的实例
        comment = Comment()

        # 通过is_vaild()方法，django 会帮我们检测数据内容是否合法
        if form.is_valid():
            # 当数据合法时候，我们调表单的save方法 将其存入数据库
            # commit = False 的作用是 利用表单数据，成成 Comment 模型的实例
            # 意思是 目前不把form里的内容提交到数据库
            comment = form.save(commit=False)
            comment.parent = request.POST.get('comment_parent')
            # 将评论和文关联起来
            comment.post = post
            # 最后将commen模型的数据保存到数据库
            comment.save()

            # 保存之后 重定向网页 重新渲染
            return redirect(post)

        else:
            # 检查到数据不合法，重新渲染本页面，并渲染出表单的错误
            # 因此我们穿了三个变量给 detail 模板，
            # Post （文章） 平路列表 表单Form
            comment_list = post.comment_set.all()
            context = {
                'post': post,
                'form': form,
                'comment_list':comment_list,
            }
            return render(request, 'blog/detail.html', context=context)

    # 不是 psot 请求，说名用户没有提交数据，重定向到文章详情页
    return redirect(post)

def comment_reply(request, post_pk, comment_pk):
    '''
    回复评论
    comment_pk： 父级评论id
    post_pk: 文章post id
    '''
    post = get_object_or_404(Post, pk=post_pk)
    last_comment = Comment.objects.filter(pk=comment_pk)[0]
    form = CommentForm()
    context = {
        'post': post,
        'last_comment': last_comment,
        'form': form,
    }

    return render(request, 'comments/reply.html', context=context)


# -*- coding: utf-8 -*-
"""
Created on Fri Nov 16 10:13:40 2018

@author: Alexandre
"""
###############################################################################
import numpy as np

###############################################################################
from pyro.dynamic  import integrator
from pyro.analysis import costfunction
###################################
# Simple integrator
###################################

# Simple integrator
sys      = integrator.SimpleIntegrator()

# Default input signal
sys.ubar = np.array([1])

###################################
# Analysis
###################################

# Phase plane behavior
sys.plot_phase_plane(0,0) # only one state for two axis!

# Cost function with unit weights
sys.cost_function = costfunction.QuadraticCostFunction(1,1,1)

sys.cost_function.Q[0,0] = 0.1

# Simulation
sys.x0 = np.array([2])

# Plot output
sys.plot_trajectory('xuj')
sys.plot_phase_plane_trajectory(0,0)


from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.urls import reverse
from courses.models import Review
from util.models import Vote
from professors.models import Professor
from .course_util import *
from util.views import error404, util_like_review, util_dislike_review
from django.utils import timezone
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger

LOGGER = logging.getLogger("project")

def course_detail(request, course_id: str):
    LOGGER.debug(f"course_detail: {course_id}")
    try:
        if request.method == "GET":
            rev_sorting = request.GET.get('rev-sorting')
            if rev_sorting is None:
                return load_course_detail(request, course_id)
            else:
                return load_course_detail(request, course_id, rev_sorting=rev_sorting)
        elif request.method == "POST" and "submit" in request.POST:
            LOGGER.debug(request.POST)
            review, message = add_review_from_details(request)
            return load_course_detail(request, course_id, review, message)
        else:
            LOGGER.error(request.POST)
            return error404(request, error="Invalid request")
    except Exception as e:
        return error404(request, error=e)

def load_course_detail(
    request, course_id: str, review: bool = None, review_message: str = "",
        rev_sorting: str = "RevDateDesc"
) -> HttpResponse:
    try:
        course = Course.objects.get(course_id=course_id)
        classes = Class.objects.filter(course=course)
        professors_list = [cl.professor for cl in classes]
        reviews_list = create_review_objects(classes, rev_sorting)
        reviews_avg = calculate_rating_avg(reviews_list)
        likes = []
        dislikes = []
        if request.user.id is not None:
            user = User.objects.get(username=request.user)
            likes = [vote.review.id for vote in Vote.objects.filter(user=user) if vote.vote == "L"]
            dislikes = [vote.review.id for vote in Vote.objects.filter(user=user) if vote.vote == "D"]

        paginator = Paginator(reviews_list, 10)
        page_number = request.GET.get('page')

        for rev in reviews_list:
            rev["like"] = len(Vote.objects.filter(review=rev['review_obj'], vote="L"))
            rev["dislike"] = len(Vote.objects.filter(review=rev['review_obj'], vote="D"))

        try:
            page_obj = paginator.get_page(page_number)
        except PageNotAnInteger:
            page_obj = paginator.page(1)
        except EmptyPage:
            page_obj = paginator.page(paginator.num_pages)

        if review is not None:
            context = {
                "classes": classes,
                "course": course,
                "reviews_list": reviews_list,
                "reviews_avg": reviews_avg,
                "professors_list": professors_list,
                "review_saved": review,
                "review_message": review_message,
                "page_obj": page_obj,
                "likes": likes,
                "dislikes": dislikes,
                "sorting_reviews": rev_sorting
            }
        else:
            context = {
                "classes": classes,
                "course": course,
                "reviews_list": reviews_list,
                "reviews_avg": reviews_avg,
                "professors_list": professors_list,
                "page_obj": page_obj,
                "likes": likes,
                "dislikes": dislikes,
                "sorting_reviews": rev_sorting,
            }
        LOGGER.debug(context)
        return render(request, "courses/detail.html", context)
    except Exception as e:
        LOGGER.exception(f"Could not load course detail, encountered error: {e}")
        return error404(request, error=e)

def add_review(request):
    context = {
        "courses": Course.objects.all(),
        "professors": Professor.objects.only("professor_id", "name"),
        "course_ids": [f"{course.course_subject_code} {course.catalog_number}" for course in Course.objects.all()],
        "courses_json": get_courses_data_json(),
        "professors_json": get_professors_data_json()
    }
    if request.method == "GET":
        if request.user.id is None:
            return redirect(reverse("search:index"))
        return render(request, "courses/add_review.html", context)
    elif request.method == "POST":
        user = User.objects.get(username=request.user)
        try:
            if not text_is_valid(request.POST["review_text"]):
                context["review_text_invalid"] = True
                add_redirect_message(
                    request=request,
                    message="Review not saved. Review text failed to meet RateNYU standards.",
                    success=False,
                )
                return redirect("courses:add_review")

            new_review = save_new_review(
                user=user,
                user_entered_course_id=request.POST["add_review_course_id"],
                professor_name=request.POST["add_review_professor_name"],
                review_rating=request.POST["review_rating"],
                review_text=request.POST["review_text"],
            )
            if new_review == "Already wrote review for this course":
                add_redirect_message(
                    request=request,
                    message="You already wrote a review for this course.",
                    success=False,
                )
            else:
                LOGGER.info(f"Created new Review: {new_review}")
                add_redirect_message(
                    request=request,
                    message="Your review was saved!",
                    success=True,
                )
            return redirect("courses:add_review")
        except Exception as e:
            LOGGER.exception(f"Could not create review, encountered error: {e}")
            add_redirect_message(
                request=request,
                message="Uh Oh, something went wrong. Your review could not be saved.",
                success=False,
            )
            return redirect("courses:add_review")

def delete_review(request, review_id: str):
    try:
        r = Review.objects.get(pk=review_id)
        r.delete()
        add_redirect_message(
            request=request, message="Your review was deleted.", success=True
        )
        return redirect("users:profile", user_name=request.user)
    except Exception as e:
        return error404(request, error=e)

def edit_review(request):
    if request.method == 'POST':
        r = Review.objects.get(pk=request.POST.get('review_id'))
        r.review_text = request.POST.get('new_review_text')
        if not text_is_valid(r.review_text):
            add_redirect_message(request=request,
                                 message="Review not saved. Review text failed to meet RateNYU standards.",
                                 success=False)
            return redirect('users:profile', user_name=request.user)

        r.rating = request.POST['review_rating']
        r.pub_date = timezone.now()
        r.save()
    return redirect('users:profile', user_name=request.user)

def like_review(request, review_id: str):
    return util_like_review(request=request, review_id=review_id)

def dislike_review(request, review_id: str):
    return util_dislike_review(request=request, review_id=review_id)


import subprocess
import os
import io
from flask import Flask
from flask import request, jsonify

app = Flask(__name__)

MY_AUTH_TOKEN = os.getenv('CI_TOKEN', None)
PATH_TO_SCRIPT = os.getenv('PATH_TO_SCRIPT', None)

@app.route('/', methods=['POST'])
def MainHandler():
    if request.headers.get('Authorization') != MY_AUTH_TOKEN:
        return jsonify({'message': 'Bad token'}), 401
    else:
        logs = []
        errors = []
        logs.append('run \'' + PATH_TO_SCRIPT + '\'')

        shellscript = subprocess.Popen(["sh", PATH_TO_SCRIPT], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = shellscript.communicate()

        for line in stdout.decode("utf-8").split('\n'):
            logs.append(line)
        for line in stderr.decode("utf-8").split('\n'):
            errors.append(line)

        logs.append('done!')
        return jsonify({'message': 'Any work', 'logs': logs, 'errors': errors}), 200

def main():
    app.run(host='127.0.0.1', port=1114)

if __name__ == '__main__':
    main()


from timeit import default_timer
from typing import List, Dict, Any, Callable

__all__ = ["time_this"]

# A module-level store of all the evaluation times of things you ran with the
# @time_this decorator
runtime_table: List[Dict[str, Any]] = list()

def time_this(f):
    def timed_function(*args, **kwargs):
        ts = default_timer()
        print(f'Running {f.__name__} ...')

        result = f(*args, **kwargs)
        te = default_timer()
        t = 1000 * (te - ts)
        print(f'Completed {f.__name__} in {round(t, 3)} milliseconds')
        print()
        return result

    return timed_function


from setuptools import find_packages, setup

with open("README.md", "r") as fh:
    LONG_DESC = fh.read()
    setup(
        name="hydra-range-sweeper",
        version="0.1.0",
        author="Jan-Matthis Lueckmann, Omry Yadan",
        author_email="mail@jan-matthis.de, omry@fb.com",
        description="Ray Launcher for Hydra apps",
        long_description=LONG_DESC,
        long_description_content_type="text/markdown",
        url="https://github.com/facebookresearch/hydra/",
        packages=find_packages(exclude=["tests", "example"]),
        classifiers=[
            "License :: OSI Approved :: MIT License",
            "Programming Language :: Python :: 3.6",
            "Programming Language :: Python :: 3.7",
            "Programming Language :: Python :: 3.8",
            "Operating System :: OS Independent",
        ],
        install_requires=["hydra-core>=1.0.0rc1"],
        include_package_data=True,
    )


import jsonschema
import unittest
import pytest
from pathlib import Path
import os
import json

# Load all JSON files in RAM
path_function_dictionaries = os.path.join(
    os.path.dirname(__file__), "..", "sql_translate", "function_dictionaries"
)
path_jsonschema = os.path.join(
    os.path.dirname(__file__), "samples", "jsonschema", "function_dictionaries.json"
)
files_to_review = {}
for path_file in Path(os.path.join(path_function_dictionaries)).rglob("*.json"):
    print(f"Loading {path_file} in memory")
    with open(path_file) as f:
        files_to_review[path_file] = json.load(f)
assert files_to_review  # If empty list, file must have moved

def test_dictionaries_against_jsonschema() -> None:
    with open(path_jsonschema) as f:
        schema = json.load(f)
    for file_to_review, content in files_to_review.items():
        print(f"Validating {file_to_review} against jsonschema")
        assert jsonschema.validate(content, schema) is None

def test_dictionaries_are_sorted() -> None:
    for file_to_review, content in files_to_review.items():
        print(f"Validating {file_to_review} has sorted keys")
        assert list(content.keys()) == sorted(content.keys())


import os
import time
import json
from collections import namedtuple, deque
import numpy as np
import pandas as pd
from scipy import stats
import scipy
from operator import itemgetter, attrgetter
from scipy.special import factorial
import math
#from gmplot import gmplot

Point = namedtuple('Point', ('x', 'y'))

class TaxiZone:
    def __init__(self, locID, polygons, name="", boro=""):
        self.locID = locID
        self.polygons = polygons
        self.name = name
        self.boro = boro
        self.count = 0

def haversine_distance(lat1, lng1, lat2, lng2):
    """function to calculate haversine distance between two co-ordinates"""
    lat1, lng1, lat2, lng2 = map(np.radians, (lat1, lng1, lat2, lng2))
    earth_radius = 6371  #  km
    lat = lat2 - lat1
    lng = lng2 - lng1
    d = np.sin(lat * 0.5) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(lng * 0.5) ** 2
    h = 2 * earth_radius * np.arcsin(np.sqrt(d))
    return h

def manhattan_distance(lat1, lng1, lat2, lng2):
    """function to calculate manhatten distance between pick->drop"""
    a = haversine_distance(lat1, lng1, lat1, lng2)
    b = haversine_distance(lat1, lng1, lat2, lng1)
    return a + b

def is_point_in_zone(point, zone):
    """Checks if coordinates of a Point are in the zone.
        Very slow and not efficient algo, may be we can use code from OpenCV for that."""
    for poly in zone.polygons:
        inside = False
        for i in range(len(poly)):
            p1 = poly[i]
            p2 = poly[i - 1]
            if ((p1.y > point.y) != (p2.y > point.y)) and (point.x < (p2.x - p1.x) * (point.y - p1.y) / (p2.y - p1.y) + p1.x):
                inside = not inside
        if inside:
            return True
    return False

def get_zone_index(zones, point):
    for i in range(len(zones)):
        zone = zones[i]
        if is_point_in_zone(point, zone):
            return i
    return -1

def get_zone_locid(zones, point):
    for i in range(len(zones)):
        zone = zones[i]
        if is_point_in_zone(point, zone):
            return zone.locID
    return 400

def get_zone_index_by_name(zones, name):
    for i in range(len(zones)):
        zone = zones[i]
        if name == zone.name:
            return i
    return -1

def process_zones(zones_path):
    with open(zones_path, 'r') as f:
        zones_json = json.load(f)

    zones = []
    for feature in zones_json['features']:
        geom_type = feature['geometry']['type']
        zone_name = feature['properties']['zone']
        area_name = feature['properties']['borough']
        if geom_type == 'Polygon':
            polygons_json = [feature['geometry']['coordinates']]
        elif geom_type == 'MultiPolygon':
            polygons_json = feature['geometry']['coordinates']
        else:
            raise ValueError('unknown geometry type {}'.format(geom_type))

        polygons = [[Point(*p) for p in poly[0]] for poly in polygons_json]
        zones.append(TaxiZone(feature['properties']['LocationID'], polygons, zone_name, area_name))
    return zones

def getCleanTripData():
    trip = pd.read_csv('./data/clean_1_trip_data_4_2013.csv', skipinitialspace=True, dtype={"rate_code": int, "passenger_count": int,"store_and_fwd_flag": str})
    lat_min = 40.5
    lat_max = 41
    long_min = -74.25
    long_max = -73.70
    location_p_long_filter = ((trip.pickup_longitude >= long_min) & (trip.pickup_longitude <= long_max))
    location_p_lat_filter = ((trip.pickup_latitude >= lat_min) & (trip.pickup_latitude <= lat_max))
    location_d_lat_filter = ((trip.dropoff_latitude >= lat_min) & (trip.dropoff_latitude <= lat_max))
    location_d_long_filter = ((trip.dropoff_longitude >= long_min) & (trip.dropoff_longitude <= long_max))
    filtered_trips = trip[location_p_long_filter & location_p_lat_filter & location_d_long_filter & location_d_lat_filter]
    return filtered_trips

if __name__ == "__main__":
    """Here we just map trips to Taxi zones"""
    filtered_trips = getCleanTripData()
    zones = process_zones('./data/taxi_zones/taxi_zones.json')

    pickups_lat = np.array(filtered_trips.pickup_latitude)
    pickups_long = np.array(filtered_trips.pickup_longitude)
    dropoff_lat = np.array(filtered_trips.dropoff_latitude)
    dropoff_long = np.array(filtered_trips.dropoff_longitude)
    pickups = []
    dropoffs = []

    l = len(filtered_trips.index)
    print("Total: {}".format(l))
    s = time.time()
    for i in range(l):
        pp = Point(pickups_long[i], pickups_lat[i])
        dp = Point(dropoff_long[i], dropoff_lat[i])
        pickup_locid = get_zone_locid(zones, pp)
        dropoff_locid = get_zone_locid(zones, dp)
        pickups.append(pickup_locid)
        dropoffs.append(dropoff_locid)
        if i % 1000 == 0:
            if i > 0:
                e = time.time()
                d = (e - s)
                speed = i / d
                print("Done: {} / {} Time Spent: {} mins. Left Time: {} mins. Speed: {}".format(i,l,d/60,(l-i)/speed/60,speed))

    trips = filtered_trips.assign(pickup_locid=pd.Series(pickups).values)
    trips = trips.assign(dropoff_locid=pd.Series(dropoffs).values)
    trips.to_csv('./data/1_clean_trip_data_plus_locations_4_2013.csv', sep=',')


from __future__ import annotations

import logging
from typing import Optional

from langchain_core.callbacks import CallbackManagerForToolRun

from langchain_community.tools.edenai.edenai_base_tool import EdenaiTool

logger = logging.getLogger(__name__)

class EdenAiParsingInvoiceTool(EdenaiTool):
    """Tool that queries the Eden AI Invoice parsing API.

    for api reference check edenai documentation:
    https://docs.edenai.co/reference/ocr_invoice_parser_create.

    To use, you should have
    the environment variable ``EDENAI_API_KEY`` set with your API token.
    You can find your token here: https://app.edenai.run/admin/account/settings

    """

    name = "edenai_invoice_parsing"

    description = (
        "A wrapper around edenai Services invoice parsing. "
        """Useful for when you have to extract information from
        an image it enables to take invoices
        in a variety of formats and returns the data in contains
        (items, prices, addresses, vendor name, etc.)
        in a structured format to automate the invoice processing """
        "Input should be the string url of the document to parse."
    )

    language: Optional[str] = None
    """
    language of the image passed to the model.
    """

    feature = "ocr"
    subfeature = "invoice_parser"

    def _parse_response(self, response: list) -> str:
        formatted_list: list = []

        if len(response) == 1:
            self._parse_json_multilevel(
                response[0]["extracted_data"][0], formatted_list
            )
        else:
            for entry in response:
                if entry.get("provider") == "eden-ai":
                    self._parse_json_multilevel(
                        entry["extracted_data"][0], formatted_list
                    )

        return "\n".join(formatted_list)

    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        query_params = {
            "file_url": query,
            "language": self.language,
            "attributes_as_list": False,
        }

        return self._call_eden_ai(query_params)


from operator import itemgetter
from datetime import datetime
from sklearn.metrics import roc_auc_score, classification_report
from sklearn.metrics import recall_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import f1_score
from sklearn.metrics import r2_score, explained_variance_score, mean_squared_error
import numpy as np
import pandas as pd
import pickle
import json
import os

def timer(start_time=None):
    if not start_time:
        start_time = datetime.now()
        return start_time
    elif start_time:
        thour, temp_sec = divmod((datetime.now() - start_time).total_seconds(), 3600)
        tmin, tsec = divmod(temp_sec, 60)
        print('\n Time taken: %i hours %i minutes and %s seconds.' % (thour, tmin, round(tsec, 2)))

# Utility function to report best scores
def grid_search_report(grid_scores, n_top):
    top_scores = sorted(grid_scores, key=itemgetter(1), reverse=True)[:n_top]
    for i, score in enumerate(top_scores):
        print("Model with rank: {0}".format(i + 1))
        print("Mean validation score: {0:.4f})".format(
            score.mean_validation_score,
            np.std(score.cv_validation_scores)))
        print("Parameters: {0}".format(score.parameters))
        print("")

def classification_performance_report(Y_test, Y_pred, Y_pred_prob, savefile):
    accuracy_score_ = accuracy_score(Y_test, Y_pred)
    roc_auc_score_ = roc_auc_score(Y_test, Y_pred_prob[:, 1])
    report_dict = {"accuracy_score": accuracy_score_,
                   "roc_auc_score": roc_auc_score_,
                   "classification_report": classification_report(Y_test, Y_pred)}

    print("*" * 40 + "classfication report" + "*" * 40)
    for key, value in report_dict.items():
        print("{0} : {1}".format(key, value))

    with open(savefile, 'w') as file:
        file.write(json.dumps(report_dict))

def regression_performance_report(y_true_, y_pred_, savefile):
    r2_score_ = r2_score(y_true=y_true_, y_pred=y_pred_)
    explained_variance_score_ = explained_variance_score(y_true=y_true_, y_pred=y_pred_)
    mean_squared_error_ = mean_squared_error(y_true=y_true_, y_pred=y_pred_)
    report_dict = {"r2_score": r2_score_,
                   "explained_variance_score": explained_variance_score_,
                   "mean_square_error": mean_squared_error_}

    print("*" * 40 + "regression report" + "*" * 40)
    for key, value in report_dict.items():
        print("{0} : {1}".format(key, value))

    with open(savefile, 'w') as file:
        file.write(json.dumps(report_dict))

def read_performance_report(report_dict_file):
    if os.path.isfile(report_dict_file):
        return json.loads(report_dict_file)
    print("{} does not exist".format(report_dict_file))
    return None

def report(cv_results, savefile):
    save_cv_results = {"params": cv_results["params"],
                       "rank_test_score": cv_results["rank_test_score"],
                       "mean_test_score": cv_results["mean_test_score"],
                       "std_test_score": cv_results["std_test_score"]}

    df_cv_results = pd.DataFrame(save_cv_results)
    df_cv_results.sort_values(by="rank_test_score", inplace=True)
    df_cv_results.to_csv(savefile, index=None)

def select_low_importance_feature(feature_names_, feature_importances_, threshold_=0.01):
    """return a list of feature names that has feature_importances smaller than threthold
    :param feature_names_: a list of feature_names
    :param feature_importances: a list/array of feature importances (which has the same length as feature_names)
    :param threshold_: threshold of selection
    """
    return np.array(feature_names_)[np.where(feature_importances_ < threshold_)[0]].tolist()


# https://diagrams.mingrammer.com/docs/nodes/aws
# https://www.graphviz.org/doc/info/attrs.html
from diagrams import Cluster, Diagram, Edge
from diagrams.aws.compute import Lambda
from diagrams.aws.storage import S3
from diagrams.aws.network import APIGateway
from diagrams.aws.database import DynamodbTable
from diagrams.aws.security import IdentityAndAccessManagementIam
from diagrams.aws.devtools import Codebuild
from diagrams.aws.devtools import Codecommit
from diagrams.aws.devtools import Codedeploy
from diagrams.aws.devtools import Codepipeline
from diagrams.aws.management import Cloudformation
from diagrams.aws.devtools import CommandLineInterface
from diagrams.aws.general import User
from diagrams.aws.general import General
from diagrams.onprem.client import Client

graph_attr = {
    "fontsize": "45"
}

cluster_attr = {
    "bgcolor": "lightgrey"
}

with Diagram("Canary Deployments", show=False, direction="LR", filename="canary-deployments", graph_attr=graph_attr):

    with Cluster("Launch API URL", direction="LR"):
        user = User("User")
        console = Client("Browser")
        user >> console

    with Cluster("Cloud9", direction="LR"):
        builder = User("Builder")
        cli = CommandLineInterface("AWS CLI")
        builder >> cli

    with Cluster("CloudFormation", direction="LR"):
        cloudformation = Cloudformation("Stack")
        cli >> cloudformation

    with Cluster("CodePipeline"):
        codepipeline = Codepipeline("Pipeline")
        cfn  = Cloudformation("Deploy")
        codepipeline >> cfn

    with Cluster("Serverless Application Model (SAM) Stack"):
        sam = Cloudformation("SAM Template")
        apigateway = APIGateway("API Gateway")
        mylambda = Lambda("Lambda")
        ddb  = DynamodbTable("DynamoDB")
        codedeploy  = Codedeploy("CodeDeploy")
        mylambda >> codedeploy
        sam >> apigateway
        sam >> mylambda
        sam >> ddb
        mylambda >> ddb

        cloudformation >> codepipeline
        cfn >> sam
        console >> apigateway


from vyper.compiler import (
    compile_codes,
    mk_full_signature,
)

def test_only_init_function():
    code = """
x: int128

@public
def __init__():
    self.x = 1
    """
    code_init_empty = """
x: int128

@public
def __init__():
    pass
    """

    empty_sig = [{
        'outputs': [],
        'inputs': [],
        'constant': False,
        'payable': False,
        'type': 'constructor'
    }]

    assert mk_full_signature(code) == empty_sig
    assert mk_full_signature(code_init_empty) == empty_sig

def test_default_abi():
    default_code = """
@payable
@public
def __default__():
    pass
    """

    assert mk_full_signature(default_code) == [{
        'constant': False,
        'payable': True,
        'type': 'fallback'
    }]

def test_method_identifiers():
    code = """
x: public(int128)

@public
def foo(x: uint256) -> bytes[100]:
    return b"hello"
    """

    out = compile_codes(
        codes={'t.vy': code},
        output_formats=['method_identifiers'],
        output_type='list'
    )[0]

    assert out['method_identifiers'] == {
        'foo(uint256)': '0x2fbebd38',
        'x()': '0xc55699c'
    }


import math

from Cryptodome.Random import random

from schemes import (
    precompute_both_scheme,
    precompute_gm_scheme,
    precompute_gnr_scheme,
    scheme1,
    scheme3,
)
from schemes.config import POWER

def testScheme1(m1, m2):
    ps = scheme1.PaillierScheme()

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testScheme3(m1, m2):
    ps = scheme3.PaillierScheme()

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testPrecomputeGm(m1, m2):
    ps = precompute_gm_scheme.PaillierScheme.constructFromJsonFile(
        "precompute_gm-2022-01-06_22:16:03.993283.json"
    )

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testPrecomputeGnr(m1, m2):
    ps = precompute_gnr_scheme.PaillierScheme.constructFromJsonFile(
        "precompute_both-2022-01-07_14.47.27.047353.json"
    )

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testPrecomputeBoth(m1, m2):
    ps = precompute_both_scheme.PaillierScheme.constructFromJsonFile(
        "precompute_both-2022-01-07_14.47.27.047353.json"
    )

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

if __name__ == "__main__":
    m1 = random.getrandbits(int(math.log2(POWER)) * 2)
    m2 = random.getrandbits(int(math.log2(POWER)) * 2)

    print("Testing scheme1")
    testScheme1(m1, m2)

    print("Testing scheme3")
    testScheme3(m1, m2)

    print("Testing precompute_gm")
    testPrecomputeGm(m1, m2)

    print("Testing precompute_gnr")
    testPrecomputeGnr(m1, m2)

    print("Testing precompute_both")
    testPrecomputeBoth(m1, m2)

    print("Finished successfully")


import calendar
import time
import datetime
from datetime import datetime, timedelta
from datetime import datetime
import os
import os.path as op
import zipfile
import random

# calendar: 获取一年的日历字符串
# 参数
# w = 每个日期之间的间隔字符数
# l = 每周所占用的行数
# c = 每个月之间的间隔字符数
cal = calendar.calendar(2017, l=0, c=3)

# isleap：判断某一年是否闰年
isleap = calendar.isleap(2007)

# leapdays：获取指定年份之间的闰年的个数
leapdays = calendar.leapdays(1998, 2018)

# mouth() 获取某个月的日历字符串
# 格式：calendar.mouth(年，月)
# 返回值：月日历的字符串
m3 = calendar.month(2018, 2)

# monthrange() 获取一个月的周几开始及总天数
# 格式：calendar.monthrange（年，月）
# 返回值：元组（周几开始，总天数）
# 注意：周默认 0-6，表示周一到周天
w, t = calendar.monthrange(2017, 3)

# monthcalendar() 返回一个月每天的矩阵列表
# 格式：calendar.monthcalendar(年，月)
# 返回值：二级列表
# 注意：矩阵中没有天数用0表示
m = calendar.monthcalendar(2018, 3)

# prcal：直接打印日历
# calendar.prcal(2018)
# help(calendar.prcal)

# prmonth() 直接打印整个月的日历
# 格式：calendar.permonth(年，月)
# 返回值：无
# calendar.prmonth(2018, 3)

# weekday() 获取周几
# 格式：calendar.weekday(年，月，日)
# 返回值：周几对应的数字
weekday = calendar.weekday(2018, 2, 2)

# 时间模块
# timezone：当期时区和UTC时间差的描述，在没有夏令时的情况下的间隔，东八区的是-28800
# altzone：获取当前时区与UTC时间差的秒数，在有夏令时的情况下
# daylight 测试当前是否是夏令时时间状态，
# print(time.timezone)

# 得到时间戳
t = time.time()

# localtime，得到当前时间的时间结构
# 可以通过点号操作符得到相应的属性元素的内容
t = time.localtime()

# asctime() 返回元组的正常字符串化之后的时间格式
# 格式：time.asctime（时间元组）
# 返回值：字符串 Tue Jun 6 11:11:00 2017
t = time.asctime()

# ctime：获取字符串化的当前时间
t = time.ctime()

# mktime() 使用时间元组获取对应的时间戳
# 格式：time.mktime(时间元组)
# 返回值：浮点数时间戳
lt = time.localtime()
ts = time.mktime(lt)

# clock：获取cpu时间，3.0 - 3.3版本直接使用，3.6调用有问题

# sleep：是程序进入睡眠，n秒后继续
time.sleep(1)

# 把时间表示称，2018年3月26日 21:05
t = time.localtime()
ft = time.strftime("%Y %m %d %H:%M", t)

# datetime常见属性
# datetime.date：一个理想的日期，提供year，mouth，day属性
# print(datetime.date(2018, 3, 26))

# datetime.time：提供一个理想的时间，提供hour，minute，sec，microsec等内容
# datetime.datetime：提供日期跟时间的组合
# datetime.timedelta：提供一个时间差，时间长度

# datetime.datetime

# 常用方法
# today：
# now
# utcnow
# fromtimestamp：从时间戳中返回本地时间
# dt = datetime(2018, 9, 26)
# print(dt.today())
# print(dt.now())
# print(dt.fromtimestamp(time.time()))

t1 = datetime.now()
# print(t1)
# print(t1.strftime("%Y-%m-%d %H:%M:%S"))
# td表示一小时的时间长度
td = timedelta(hours=1)

# 当前时间加上时间间隔后，得到一个小时后的时间
# print((t1 + td).strftime("%Y-%m-%d %H:%M:%S"))

# timeit-时间测量工具
# 测量测序运行时间间隔实验
def p():
    time.sleep(3.6)

t1 = time.time()
# p()
# print(time.time() - t1)
# 生成列表两种方法的比较
# 如果单纯比较生成一个列表的时间，可能很难实现
c = '''
sum = []
for i in range(1000):
    sum.append(i)
'''
import timeit

# 利用timeit调用代码，执行100000，查看运行时间
# t1 = timeit.timeit(stmt="[i for i in range(1000)]", number=100000)
# 测量代码c执行100000次运行结果
# t2 = timeit.timeit(stmt=c, number=100000)
# print(t1)
# print(t2)
# timeit 可以执行一个函数，来测量一个函数的执行时间
# def doIt():
#     num = 3
#     for i in range(num):
#         print("Repeat for {0}".format(i))
#
#
# t = timeit.timeit(stmt=doIt, number=10)
# print(t)
s = """
def doIt(num):
    for i in range(num):
        print("Repeat for {0}".format(i))
"""
# 执行doIt(num)
# setup负责吧环境变量准备好
# 实际相当于给timi创再来个一个小环境
# 在创建的小环境中，代码执行的顺序大致是
#
"""
def doIt(num):
    ....
num = 3
doIt(num)
"""
# t = timeit.timeit("doIt(num)", setup=s + "num=3", number=10)

# getcwd() 获取当期的工作目录
# 格式： os.getcwd()
# 返回值：当前工作目录的字符串
# 当前工作目录就是程序在进行文件相关操作，默认查找文件的目录
mydir = os.getcwd()

# chdir() 改变当前的工作目录
# change directory
# 格式： os.chdir(路径)
# 返回值：无
# os.chdir('G:\Python\marketdown')
# mydir = os.getcwd()

# listdiir() 获取一个目录中所有子目录和文件的名称列表
# 格式： os.listdir（路径）
# 返回值：所有子目录和文件名称的列表
ld = os.listdir()

# makedir() 递归创建文件夹
# 格式：os.makedirs(递归路径)
# 返回值：无
# 递归路径：多个文件夹层层包换的路径就是递归路径，例如 a/b/c
# rst = os.makedirs('test')

# system() 运行系统shell命令
# 格式： os.system(系统命令)
# 返回值：打开一个shell或者终端界面
# 一般推荐使用subprocess代替
# rst = os.system("dir")

# getenv() 获取指定的系统环境变量值
# 格式：os.getenv('环境变量名')
# 返回值：指定环境变量名对应的值
rst = os.getenv('PATH')

# exit() 退出当前程序
# 格式：exit()
# 返回值：无

# abspath() 将路径转换成绝对路径
# abselute 绝对
# 格式：os.path.abspath('路径')
# 返回值：路径的绝对路径形式
# 在linux中
# . 点号，代表当前目录
# .. 双点，代表父目录
absp = op.abspath(".")

# basename() 获取路径中的文件名部分
# 格式：os.path.basename('路径')
# 返回值：文件名字符串

# join() 将多个路径拼合成一个路径
# 格式：os.path.join(路径1，路径2)
# 返回值：组合之后的新路径字符串

# split() 将路径切割成文件夹部分和当期文件部分
# 格式：os.path.split(路径)
# 返回值：路径和文件名组成的元祖

# isdir() 检测是否是目录
# 格式： os.path.isdir(路径)
# 返回值：布尔值

# exists() 检测文件或者目录是否存在
# 格式：os.path.exists(路径)
# 返回值：布尔值

# copy() 复制文件
# 格式： shutil.copy(来源路径，目标路径)
# 返回值：返回目标路径
# 拷贝的同时，可以给文件重命名

# copy2() 复制文件，保留元数据（文件信息）
# 格式：shutil.copy2(来源路径，目标路径)
# 返回值：返回目标路径
# 注意：copy和copy2的唯一区别在于copy2复制文件时尽量保留元数据

# copyfile() 将一个文件中的内容复制到另一个文件当中
# 格式：shutil.copyfile（'源路径','目标路径'）
# 返回值：无

# move() 移动文件/文件夹
# 格式： shutil.move('源路径','目标路径')
# 返回值：目标路径

# make_archive() 归档操作
# 格式：shutil.make_archive('归档之后的目录和文件名','后缀','需要归档的文件夹')
# 返回值：归档之后的地址

# unpack_archive() 解包操作
# 格式：shutil.unpack_archive('归档文件地址','解包之后的地址')
# 返回值：解包之后的地址

# zipfile.ZipFile(file[.mode[,compression[,allowZip64]]])
# 创建一个ZipFile对象，表示一个zip文件。参数file表示文件的路径或者类文件对象

# zipfile.getinfo(name)
# 获取zip文档内指定文件的信息。返回一个zipfile.ZipInfo对象，它包括文件的详细信息。

# zipfile.namelist()
# 获取zip文档内所有文件的名称列表。

# ZipFile.extractall([path.[,members[,pwd]]])
# 解压zip文档中的所有文件当当前目录。参数members的默认值为zip文档内的所有文件名称...

# random() 获取0-1之间的随机小数
# 格式：random.random()
# 返回值：随机0-1之间的小数

# choice() 随机返回序列中的某个值
# 格式：random.choice(序列)
# 返回值：序列中的某个值
l = [str(i) + "haha" for i in range(10)]
rst = random.choice(l)

# shuffle() 随机打乱列表
# 格式：random.shuffle(列表)
# 返回值：None，打乱源列表

# randint() 随机产生一个证书
# 格式：random.randint(a,b)
# 返回值：返回一个a到b之间的随机整数，包含a和b


#!/usr/bin/python3
"""
Contains class Airacft
"""

import models
import copy
import json
from models.airport import Airport
from math import radians, cos, sin, asin, sqrt, pow, atan2, pi
from datetime import date, datetime, timedelta

class Aircraft(object):
	"Class model for all aircrafts"

	#class needed information
	instancecount = 0
	plane_list = []

	#default safety measures and rate of refresh of the data (amount of refresh's in a minute)
	safety_vertical = 1000
	safety_horizontal = 4828.03
	refresh_rate = 3
	descent_rate = 1000 / refresh_rate
	permited_altitudes = {
		"truck0": [2000, 3900, 5900, 7900,
				   9800, 11800, 13800, 15700,
				   17700, 19700, 21700, 23600,
				   25600, 28200, 31500, 34800,
				   38100, 43000, 49500],
				   #and evey 2000 meters after
		"truck1": [3000, 4900, 6900, 8900,
				   10800, 12800, 14800, 16700,
				   18700, 20700, 22600, 24600,
				   26600, 29900, 33100, 36400,
				   39700, 46300]
				   #and evey 2000 meters after
	}

	def __init__(self, args):
		"""method to assign all variables when creating an instance of Aircraft"""
		#Query for all data
		aircraft_data = models.storage.aircraft_query_id(args)
		self.FlightID = aircraft_data["FlightID"]
		self.IATA = aircraft_data["IATA"]
		self.ICAO = aircraft_data["ICAO"]

		#Load departure information
		self.departure_IATA = aircraft_data["departure_IATA"]
		self.departure_ICAO = aircraft_data["departure_ICAO"]
		self.departure_time = aircraft_data["departure_time"]
		self.departure_airport = aircraft_data["departure_airport"]

		#Load arrival information
		self.arrival_IATA = aircraft_data["arrival_IATA"]
		self.arrival_ICAO = aircraft_data["arrival_ICAO"]
		self.arrival_time = aircraft_data["arrival_time"]
		self.arrival_airport = aircraft_data["arrival_airport"]

		#load info regarding aircraft
		self.type = aircraft_data["type"]
		self.registration = aircraft_data["registration"]
		self.airline = aircraft_data["airline"]
		self.working_altitude = 0

		#Queries for dinamic information
		self.path = models.storage.aircraft_query_update(args)
		self.manifesto = False
		self.current_path = 0
		wpa = 0
		while (datetime.strptime(self.path[wpa]["time"], '%Y-%m-%dt%H:%M:%Sz') -
				datetime.now() > timedelta(0, 60 / Aircraft.refresh_rate)):
			try:
				self.path[self.current_path]["time"]
				self.status = "On air"
			except:
				self.status = "Outside Airspace"
				break
			wpa += 1
		self.status = "Outside Airspace"

		#collisions information
		self.collision_l = []
		self.suggested_flightpath = []
		self.estimated_flightpath = []

		#adds to list of instances
		Aircraft.plane_list.append(self)
		Aircraft.instancecount += 1

	def __str__(self):
		"""String representation of the Aircraft class"""
		return "[{:s}] ({:s}) {}".format(self.__class__.__name__, self.FlightID,
										 self.__dict__)

	def to_dict(self):
		"""returns a dictionary containing all keys/values of the instance"""
		new_dict = self.__dict__.copy()
		return new_dict

	def update(self):
		"""update all information of aircraft"""
		pos = 0
		if (self.manifesto):
			path1 = self.path
			self.current_path += 1
			starting_point1 = self.current_path
		else:
			path1 = self.estimated_flightpath
			starting_point1 = 0
		coso = path1[starting_point1]
		next_location = self.point_ahead(coso["latitude"], coso["longitude"], coso["truck"],
								 		 coso["speed"], coso["time"], coso["altitude"])
		if (((next_location["latitude"] > -35.15 and next_location["latitude"] < -34.65) and (next_location["longitude"] > -57.20 and next_location["longitude"] < -53.25))
			  or ((next_location["latitude"] > -34.65 and next_location["latitude"] < -34.25) and (next_location["longitude"] > -58.15 and next_location["longitude"] < -53.25))
			  or ((next_location["latitude"] > -34.25 and next_location["latitude"] < -33.05) and (next_location["longitude"] > -58.75 and next_location["longitude"] < -53.25))
			  or ((next_location["latitude"] > -33.05 and next_location["latitude"] < -31.40) and (next_location["longitude"] > -58.50 and next_location["longitude"] < -54.30))
			  or ((next_location["latitude"] > -33.05 and next_location["latitude"] < -32.50) and (next_location["longitude"] > -54.30 and next_location["longitude"] < -52.95))
			  or ((next_location["latitude"] > -32.50 and next_location["latitude"] < -31.75) and (next_location["longitude"] > -54.30 and next_location["longitude"] < -53.50))
			  or ((next_location["latitude"] > -31.40 and next_location["latitude"] < -30.00) and (next_location["longitude"] > -58.15 and next_location["longitude"] < -56.00))
			  or ((next_location["latitude"] > -31.40 and next_location["latitude"] < -30.75) and (next_location["longitude"] > -56.00 and next_location["longitude"] < -54.80))):
			self.create_estimated_flightpath()
			if (path1[starting_point1]["truck"] > 180):
				side = "truck0"
			else:
				side = "truck1"
			for elem in Aircraft.permited_altitudes[side]:
				if (abs(elem - path1[starting_point1]["altitude"]) < 500):
					self.working_altitude = pos
					break
				pos += 1
			self.path = models.storage.aircraft_query_update(self.FlightID)
			return 1
		else:
			Airport.mapped_planes.remove(self.FlightID)
			Aircraft.plane_list.remove(self)
			return 0

	#Collisons needs at least the current position of the aircraft loaded into the path
	def collision(self, avion2):
		"""detects a colision between 2 aircrafts"""
		pos = 0
		if (self.status == "Landed" or avion2.status == "Landed"):
			return Airport.map_collisions
		if (self.manifesto):
			path1 = self.path
			starting_point1 = self.current_path
		else:
			path1 = self.estimated_flightpath
			starting_point1 = 0
		if (avion2.manifesto):
			path2 = avion2.path
			starting_point2 = avion2.current_path
		else:
			path2 = avion2.estimated_flightpath
			starting_point2 = 0
		for element in path1[starting_point1:]:
			if (len(path2) <= (starting_point2 + pos)):
				break
			#Formula to calculate point in a determined distance and angle
			lat1 = radians(element["latitude"])
			lat2 = radians(path2[(starting_point2) + pos]["latitude"])
			lon1 = radians(element["longitude"])
			lon2 = radians(path2[(starting_point2) + pos]["longitude"])
			dlon = lon2 - lon1
			dlat = lat2 - lat1
			a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
			c = 2 * asin(sqrt(a))
			r = 6371
			horizontal_distance = c * r * 1000
			vertical_distance = abs(element["altitude"] - path2[(starting_point2) + pos]["altitude"])
			if (horizontal_distance < Aircraft.safety_horizontal) and (vertical_distance < Aircraft.safety_vertical):
				dic = {
					"ID1": self.FlightID,
					"ID2": avion2.FlightID,
					"crash_time": element["time"],
					"crash_latitude": (element["latitude"] + (element["latitude"] - path2[(starting_point2)  + pos]["latitude"]) / 2),
					"crash_longitude": (element["longitude"] + (element["longitude"] - path2[(starting_point2) + pos]["longitude"]) / 2),
					"crash_altitude": (element["altitude"] - (element["altitude"] - path2[(starting_point2) + pos]["altitude"]) / 2),
					"crash_radious": horizontal_distance / 2,
					"crash_id": self.FlightID + avion2.FlightID + element["time"],
					"crash_id2": avion2.FlightID + self.FlightID + element["time"]
				}
				flag = 0
				for elements in Airport.map_collisions:
					if (dic["crash_id"] == elements["crash_id"]) or (dic["crash_id2"] == elements["crash_id"]):
						flag = 1
						break
				if flag == 0:
					Airport.map_collisions.append(copy.deepcopy(dic))
					self.collision_l.append(copy.deepcopy(dic))
					avion2.collision_l.append(copy.deepcopy(dic))
					self.new_route()
			pos += 1
		return self.collision_l

	def all_collision(obj_list):
		"""checks collision between all aircrafts"""
		pos = 0
		Airport.map_collisions = []
		for plane in obj_list:
			pos+=1
			for plane2 in obj_list[pos:]:
				plane.collision_l = []
				plane2.collision_l = []
				plane.collision(plane2)
		return Airport.map_collisions

	def new_route(self, target=None):
		"""Preliminar suggestion of deviating route based on altitude"""
		self.suggested_flightpath = self.create_estimated_flightpath(-2000)
		#self.collision(target)

	#To create an estimated flightpath, an initial path list with current location and time is needed
	def create_estimated_flightpath(self, altitude_to_change=0):
		"""Method that creates a preliminar route for a plane"""
		flightpath = []
		current_time = datetime.strptime(self.path[self.current_path]["time"], '%Y-%m-%dt%H:%M:%Sz')
		current_time = current_time + timedelta(0, 60 / Aircraft.refresh_rate)
		if altitude_to_change == 0:
			delta_altitude = 0
		else:
			if altitude_to_change > 0:
				delta_altitude = 1000 / Aircraft.refresh_rate
			else:
				delta_altitude = -1000 / Aircraft.refresh_rate

		altitude_to_change -= delta_altitude
		next_location = self.point_ahead(self.path[self.current_path]["latitude"],
										 self.path[self.current_path]["longitude"],
										 self.path[self.current_path]["truck"],
										 self.path[self.current_path]["speed"],
										 current_time.strftime('%Y-%m-%dt%H:%M:%Sz'),
										 self.path[self.current_path]["altitude"] + delta_altitude)
		flightpath.append(copy.deepcopy(next_location))
		#This while needs to be corrected to fit critera inside airspace
		while(((next_location["latitude"] > -35.15 and next_location["latitude"] < -34.65) and (next_location["longitude"] > -57.20 and next_location["longitude"] < -53.25))
				or ((next_location["latitude"] > -34.65 and next_location["latitude"] < -34.25) and (next_location["longitude"] > -58.15 and next_location["longitude"] < -53.25))
				or ((next_location["latitude"] > -34.25 and next_location["latitude"] < -33.05) and (next_location["longitude"] > -58.75 and next_location["longitude"] < -53.25))
				or ((next_location["latitude"] > -33.05 and next_location["latitude"] < -31.40) and (next_location["longitude"] > -58.50 and next_location["longitude"] < -54.30))
				or ((next_location["latitude"] > -33.05 and next_location["latitude"] < -32.50) and (next_location["longitude"] > -54.30 and next_location["longitude"] < -52.95))
				or ((next_location["latitude"] > -32.50 and next_location["latitude"] < -31.75) and (next_location["longitude"] > -54.30 and next_location["longitude"] < -53.50))
				or ((next_location["latitude"] > -31.40 and next_location["latitude"] < -30.00) and (next_location["longitude"] > -58.15 and next_location["longitude"] < -56.00))
				or ((next_location["latitude"] > -31.40 and next_location["latitude"] < -30.75) and (next_location["longitude"] > -56.00 and next_location["longitude"] < -54.80))):

			if (altitude_to_change > -333) and (altitude_to_change < 333):
				delta_altitude = altitude_to_change
			else:
				altitude_to_change -= delta_altitude
			current_time = current_time + timedelta(0, 60 / Aircraft.refresh_rate)
			next_location = self.point_ahead(next_location["latitude"],
											 next_location["longitude"],
											 self.path[self.current_path]["truck"],
											 self.path[self.current_path]["speed"],
											 current_time.strftime('%Y-%m-%dT%H:%M:%SZ'),
											 next_location["altitude"] + delta_altitude)
			flightpath.append(copy.deepcopy(next_location))
		self.estimated_flightpath = flightpath
		return flightpath

	def switch_manifesto(self):
		"""switches between manifestos"""
		#self.manifesto = False

	def point_ahead(self, lat, lon, truck, speed, time, altitude):
		"""
			Given a start point, initial bearing, and distance,
			this will calculate the destination point and final bearing
			travelling along a (shortest distance) great circle arc.
		"""
		#Earth radious
		r = 6371
		d = speed / 120

		#Calculation of point in radians
		final_lat = asin(sin(radians(lat)) * cos(d/r) + cos(radians(lat)) * sin(d/r) * cos(radians(truck)))
		final_long = radians(lon) + atan2(sin(radians(truck)) * sin(d/r) * cos(radians(lat)), cos(d/r) - sin(radians(lat)) * sin(final_lat))

		#Conversiion to decimal degrees
		final_lat = final_lat * 180 / pi
		final_long = final_long * 180 / pi

		#Load into dictionary for return
		final_point = {
			"latitude": final_lat,
			"longitude": final_long,
			"altitude": altitude,
			"speed": speed,
			"truck": truck,
			"time": time
		}
		return final_point

	def accept_route(self):
		"""accepts routes and deploys thems to testing jsons"""
		self.suggested_flightpath = self.create_estimated_flightpath(-1000)
		data = json.dumps(self.suggested_flightpath)
		with open("{:}.json".format(self.FlightID), "w") as f:
			json.dump(data, f)


import pytest
import datetime
from snekdate.snekdate import SnekDate

############offset
def test_offset_type_returned():
    assert isinstance(SnekDate().offset(days=1), SnekDate)

@pytest.mark.parametrize(
    'end_date,offset_date,days,hours,minutes,seconds',
    [
        (
                datetime.datetime(day=20, year=2020, month=7),
                datetime.datetime(day=19, year=2020, month=7),
                1,
                0,
                0,
                0,
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=5, second=20),
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=5, second=15),
                0,
                0,
                0,
                5,
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=5, second=20),
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=3, second=50),
                0,
                0,
                0,
                90,
        ),
    ]
)
def test_offset_value_returned(end_date, offset_date, days, hours, minutes, seconds):
    assert SnekDate(end_date).offset(
        days=days,
        hours=hours,
        minutes=minutes,
        seconds=seconds,
    ).start == offset_date

############round_down_date
def test_round_down_type_returned():
    assert isinstance(SnekDate().round_down(increment='minute').start, datetime.datetime)

def test_round_down_exception():
    with pytest.raises(Exception):
        SnekDate.round_down_date(SnekDate().round_down(increment='minute'), increment='min')

@pytest.mark.parametrize(
    'date_raw,date_rounded,increment',
    [
        (
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=15),
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=0),
                'minute',
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=15),
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=0, second=0),
                'hour',
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=15),
                datetime.datetime(day=20, year=2020, month=7, hour=0, minute=0, second=0),
                'day',
        ),
    ]
)
def test_round_down_value_returned(date_raw, date_rounded, increment):
    assert SnekDate(date_raw).round_down(increment=increment).start == date_rounded


#!/usr/bin/env python
# coding: utf-8

# In[1]:

from sklearn import tree
import numpy as np
import pydot
import pandas as pd
import graphviz
import seaborn as sns
from tensorflow import keras
import matplotlib.pyplot as plt
import tensorflow as tf
from pylab import rcParams
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn import preprocessing
from sklearn.metrics import r2_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.metrics import plot_confusion_matrix

# In[4]:
# Initializing dataset

heart =pd.read_csv('D:/University Modules/Machine Learning/heart.csv')
# In[5]:
#Checking the dataset
heart

# In[6]:
#Exploring the data

#describing the data

heart.describe()
# In[7]:

#Checking the column types and selecting particular columns
heart_exp = heart.select_dtypes(include=["float64","int64"])
heart_exp.head()

# In[8]:
#Checking the scale of the dataset

heart.shape
# In[9]:
# EXPLORATORY DATA ANALYSIS------------------------------------------------------

#created a box plot for the dataset with respect to each column values

f, axes = plt.subplots(len(heart_exp.columns), 1, constrained_layout = True, figsize=([13,9]))

for i in range(len(heart_exp.columns)):

    sns.boxplot(x=heart_exp[heart_exp.columns[i]], ax=axes[i])

# In[10]:
heart.columns
# In[11]:

#Plotted a bar graph for patient with heart disease and no heart disease using the heartdisease as a target column
f = sns.countplot(x='HeartDisease', data=heart)
f.set_title("Heart disease presence distribution")
f.set_xticklabels(['No Heart disease', 'Heart Disease'])
plt.xlabel("");

# In[12]:
#plotting a bar graph for patient having heart diease with respect to their sex( M or F)

f = sns.countplot(x='HeartDisease', data=heart, hue='Sex')
plt.legend(['Female', 'Male'])
f.set_title("Heart disease presence by gender")
f.set_xticklabels(['No Heart disease', 'Heart Disease'])
plt.xlabel("");

# In[13]:
# Created a correlation matrix with respect to all the columns in the heart dataset.

heat_map = sns.heatmap(heart.corr(method='pearson'), annot=True, fmt='.2f', linewidths=2)
heat_map.set_xticklabels(heat_map.get_xticklabels());

sns.set() #

corr = heart.corr()
print(corr)

# In[14]:

corr['Age'].sort_values(ascending=False)

# In[15]:

plt.scatter(x=heart.Age[heart.HeartDisease==1], y=heart.RestingBP[(heart.HeartDisease==1)], c="red", s=60)
plt.scatter(x=heart.Age[heart.HeartDisease==0], y=heart.RestingBP[(heart.HeartDisease==0)], s=60)
plt.legend(["Disease", "No Disease"])
plt.xlabel("Age")
plt.ylabel("Maximum Heart Rate");

# In[16]:
#Created a new column to compare the data of people having heart disease and no heart disease with respect to Age
heart['Age_Category'] = pd.cut(heart['Age'],bins=list(np.arange(25, 85, 5)))

# In[17]:
#Created a bar plot and compared the both with repect to age group of people having heart disease and no heart disease.

plt.subplot(121)
heart[heart['HeartDisease']==1].groupby('Age_Category')['Age'].count().plot(kind='bar')
plt.title('Age Distribution of Patients with +ve Heart Diagonsis')

plt.subplot(122)
heart[heart['HeartDisease']==0].groupby('Age_Category')['Age'].count().plot(kind='bar')
plt.title('Age Distribution of Patients with -ve Heart Diagonsis')

# In[18]:
#Deleting the column created after the comparison as a part of EDA.

del heart['Age_Category']

# In[25]:
#Created a bar plot for the type chest pain people are having comparing the category of chestpaintype
#Comparing with respect to heart disease
f = sns.countplot(x='ChestPainType', data=heart, hue='HeartDisease')
f.set_xticklabels(['ATA', 'NAP', 'TSY','NA']);
f.set_title('Disease presence by chest pain type')
plt.ylabel('Chest Pain Type')
plt.xlabel('')
plt.legend(['No Disease', 'Disease']);

# In[20]:
#Comparing the Exersice Angina to the Sex Column for both male and females.

heart['Sex'] = np.where(heart['Sex'] == "F", 0, 1)
heart['ExerciseAngina'] = np.where(heart['ExerciseAngina'] == "N", 0, 1)

# In[31]:

#Plotted a comparison barplot for patients with different category of chestpain and checking the total amount of people having the type of pain.
sns.countplot(heart['ChestPainType'])
plt.xticks([0,1,2,3],["atypical angina", "non-anginal pain", "asymptomatic", "typical angina" ])
plt.xticks()
plt.show()

# In[32]:

# Showing Fasting Blood Sugar Distribution According To HeartDisease as Variable
sns.countplot(x="FastingBS", hue="HeartDisease", data=heart)
plt.legend(labels = ['No-Disease','Disease'])
plt.show()

# In[34]:

heart['RestingBP'].hist() #Check Resting Blood Pressure Distribution

# In[36]:

#Checking the blood pressure range for the patients with respect to their sex.
g = sns.FacetGrid(heart,hue="Sex",aspect=4)
g.map(sns.kdeplot, 'RestingBP', shade=True)
plt.legend(labels=['Male', 'Female'])

# EXPLORATORY DATA ANALYSIS------------------------------------------------------END
# In[32]:

#MODEL CLASSIFICATION------------------------------------------------------------------------START

# Manipulating data and creating dummy variables for Model processing.

heart = pd.get_dummies(heart)
heart.head().T

# In[81]:

#creating dataset to pass the value for test_train_split

y=heart["HeartDisease"]
x=heart.drop(["HeartDisease"], axis=1)

x_train, x_test , y_train, y_test = train_test_split(x,y,test_size = 0.25, random_state=42)

# In[35]:
# 1st MODEL DECISION TREE CLASSIFIER

decision=DecisionTreeClassifier().fit(x_train,y_train)

y_pred = decision.predict(x_test)
np.sqrt(mean_squared_error(y_test,y_pred))

# In[36]:
#Calcuating the model accuracy for Test Set and Training Set

print('Training set score: {:.4f}'.format(decision.score(x_train, y_train)*100))
print('Test set score: {:.4f}'.format(decision.score(x_test, y_test)*100))

# In[37]:
#Printed the classification report using the classification_report module from skelearn

print(classification_report(y_pred,y_test))

# In[38]:
#plotting the confusion matrix for decsion tree model

plot_confusion_matrix(decision, x_test, y_test)

#Getting Accuracy Score
decision.score(x_test, y_test)

accuracy_score_dt = round(decision.score(x_test, y_test)*100,2)

accuracy_score_dt

print("The accuracy score achieved using Neural Network is: "+str(accuracy_score_dt)+" %")

# In[85]: #Reference [3] from kaggle mentioned in Report
# HPYER PARAMETER TUNING FOR DECISION TREE CLASSIFIER

#creating a grid search CV classifier using decision tree

def evaluate_model(decision):
    print("Train Accuracy :", accuracy_score(y_train, decision.predict(x_train)))
    print("Train Confusion Matrix:")
    print(confusion_matrix(y_train, decision.predict(x_train)))
    print("-"*50)
    print("Test Accuracy :", accuracy_score(y_test, decision.predict(x_test)))
    print("Test Confusion Matrix:")
    print(confusion_matrix(y_test, decision.predict(x_test)))

# In[86]:
#evaluating the model accuracy
evaluate_model(decision)

# In[88]:

# Create the parameter grid based on the results of random search
params = {
    'max_depth': [2, 3, 5, 10, 20],
    'min_samples_leaf': [5, 10, 20, 50, 100],
    'criterion': ["gini", "entropy"]
}

# In[90]:

# Instantiate the grid search model
grid_search = GridSearchCV(estimator=decision,
                           param_grid=params,
                           cv=4, n_jobs=-1, verbose=1, scoring = "accuracy")

# In[91]:

#Running the Grid Search CV (Cross Validation for 4 folds on decision tree classifier under hyper parameters grid)
get_ipython().run_cell_magic('time', '', 'grid_search.fit(x_train, y_train)')

# In[92]:

#Calculating the accracy for the gridsearchcv model run on decision tree

score_df = pd.DataFrame(grid_search.cv_results_)
score_df.head()

# In[93]:
#Calculating the best score out of the all the results

heart_best=grid_search.best_estimator_

evaluate_model(heart_best)

# In[95]:

#created the classification report for the GridsearchCV classifier for hyper parameter tuning done on decision tree classifer.
print(classification_report(y_test, heart_best.predict(x_test)))
#Created a confusion matrix for gridsearchcv
plot_confusion_matrix(heart_best, x_train, y_train)
# In[46]:
## SECOND MODEL RANDOM FOREST CLASSIFIER

#Building up the random forest model.

rf_model = RandomForestClassifier().fit(x_train, y_train)

# In[47]:
#Creating a function and calculating the mean squared error for random forest

y_pred_rf = rf_model.predict(x_test)
np.sqrt(mean_squared_error(y_test, y_pred_rf))

# In[48]:
#Calculating the accuracy for test set and train for random forest classifier

print('Training set score: {:.4f}'.format(rf_model.score(x_train, y_train)*100))
print('Test set score: {:.4f}'.format(rf_model.score(x_test, y_test)*100))
# In[49]:

#Created a classification report for random forest model accuracy
print(classification_report(y_pred_rf,y_test))

#Getting the accuracy score for random forest classifier

rf_accuracy=accuracy_score(y_pred_rf,y_test)*100
rf_accuracy

print("The accuracy score achieved using Neural Network is: "+str(rf_accuracy)+" %")

# In[50]:
#plotting the confusion matrix for random forest

plot_confusion_matrix(rf_model, x_test, y_test)

# In[3]: # Reference [4] from Towards Data Science mentioned in the report
## Third MODEL USING KERAS BACKED BY TENSORFLOW

#for creating graph on white plots

sns.set(style='whitegrid', palette='muted', font_scale=1.5)

rcParams['figure.figsize'] = 14, 8

#For classifier in tensorflow

RANDOM_SEED = 42

np.random.seed(RANDOM_SEED)

# In[54]:
#Data PreProcessing
# Tensorflow feature columns are used here as the data contains a mixture of categorial and numerical data.
#so i have created functions for Numeric columns, bucketed other columns having float values and created embedded columns with the string and float values which are amended in the feature column
feature_columns = []

# numeric cols
for header in ['Age', 'RestingECG_Normal' ,'Cholesterol', 'RestingBP', 'Oldpeak']:
      feature_columns.append(tf.feature_column.numeric_column(header))

# bucketized cols
age = tf.feature_column.numeric_column("Age")
age_buckets = tf.feature_column.bucketized_column(age, boundaries=[25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85])
feature_columns.append(age_buckets)

# indicator cols
heart["FastingBS"] = heart["FastingBS"].apply(str)
FBS = tf.feature_column.categorical_column_with_vocabulary_list(
      'FastingBS', ['0', '1', '2'])
FBS_one_hot = tf.feature_column.indicator_column(FBS)
feature_columns.append(FBS_one_hot)

heart["Sex_F"] = heart["Sex_F"].apply(str)
sex_F = tf.feature_column.categorical_column_with_vocabulary_list(
      'Sex_F', ['0', '1'])
sex_F_one_hot = tf.feature_column.indicator_column(sex_F)
feature_columns.append(sex_F_one_hot)

heart["Sex_M"] = heart["Sex_M"].apply(str)
sex_M = tf.feature_column.categorical_column_with_vocabulary_list(
      'Sex_M', ['0', '1'])
sex_M_one_hot = tf.feature_column.indicator_column(sex_M)
feature_columns.append(sex_M_one_hot)

heart["ST_Slope_Up"] = heart["ST_Slope_Up"].apply(str)
slope_Up = tf.feature_column.categorical_column_with_vocabulary_list(
      'ST_Slope_Up', ['0', '1', '2'])
slope_Up_one_hot = tf.feature_column.indicator_column(slope_Up)
feature_columns.append(slope_Up_one_hot)

heart["ST_Slope_Down"] = heart["ST_Slope_Down"].apply(str)
slope_Down = tf.feature_column.categorical_column_with_vocabulary_list(
      'ST_Slope_Down', ['0', '1', '2'])
slope_down_one_hot = tf.feature_column.indicator_column(slope_Down)
feature_columns.append(slope_down_one_hot)

heart["ChestPainType_TA"] = heart["ChestPainType_TA"].apply(str)
cp = tf.feature_column.categorical_column_with_vocabulary_list(
      'ChestPainType_TA', ['0', '1', '2'])
cp_one_hot = tf.feature_column.indicator_column(cp)
feature_columns.append(cp_one_hot)

# embedding cols
thal_embedding = tf.feature_column.embedding_column(FBS, dimension=8)
feature_columns.append(thal_embedding)

# crossed cols
age_FBS_crossed = tf.feature_column.crossed_column([age_buckets, FBS], hash_bucket_size=1000)
age_FBS_crossed = tf.feature_column.indicator_column(age_FBS_crossed)
feature_columns.append(age_FBS_crossed)

cp_slope_crossed = tf.feature_column.crossed_column([cp, slope_Up], hash_bucket_size=1000)
cp_slope_crossed = tf.feature_column.indicator_column(cp_slope_crossed)
feature_columns.append(cp_slope_crossed)

# In[55]:

# Created a pandas dataframe and converted it into a Tensorflow dataset

def create_dataset(dataframe, batch_size=32):
    dataframe = dataframe.copy()
    labels = dataframe.pop('HeartDisease')
    return tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))     .shuffle(buffer_size=len(dataframe))     .batch(batch_size)

# In[56]:

#created a test trian split for the model
train, test = train_test_split(heart, test_size=0.2, random_state=RANDOM_SEED)

# In[57]:
#Created train and test dataset to fit in the model

train_ds = create_dataset(train)
test_ds = create_dataset(test)

# In[58]:
#Creating a binary classifier using tensorflow

model = tf.keras.models.Sequential([
        tf.keras.layers.DenseFeatures(feature_columns=feature_columns),
        tf.keras.layers.Dense(units=128, activation='relu'),
        tf.keras.layers.Dropout(rate=0.2),
        tf.keras.layers.Dense(units=128, activation='relu'),
        tf.keras.layers.Dense(units=1, activation='sigmoid')
])

# In[59]:

#This model will run the classifier build above for the batch size created and will show the accuracy and loss values for each iteration.

model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_ds, validation_data=test_ds, epochs=100, use_multiprocessing=True)

# In[60]:

#finded the model accuracy for the test set
model.evaluate(test_ds)

# In[61]:
#finded the model accuracy for train set

model.evaluate(train_ds)

# In[62]:

#plotted a graph for the model accuracy
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.ylim((0, 1))
plt.legend(['train','test'], loc='upper left');

# In[63]:
#plotted a graph for the model loss.

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

# In[66]:
#Predicting the heart disease
#here the model is making a binary decsion, so we are taking the maximum binary probability of the output layer

predictions = model.predict(test_ds)
bin_predictions = tf.round(predictions).numpy().flatten()
y_test_keras = tf.round(model.predict(test_ds)).numpy().flatten()

# In[69]:

#Build up a classification report stating the accuracy and other scores.
print(classification_report(y_test_keras,bin_predictions))

#Getting the accuracy score for Keras Model:

keras_accuracy=accuracy_score(y_test_keras, bin_predictions)*100
keras_accuracy

print("The accuracy score achieved using Neural Network is: "+str(keras_accuracy)+" %")

# In[70]:

#Created a function for plotting the confusion matrix using the predicted values and test_labels
cnf_matrix = confusion_matrix(y_test_keras, bin_predictions)
cnf_matrix

# In[71]:
#Plotted the confusion matrix for the model.

class_names = [0,1]
fig,ax = plt.subplots()
tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks,class_names)
plt.yticks(tick_marks,class_names)

sns.heatmap(pd.DataFrame(cnf_matrix),annot=True,cmap="Blues",fmt="d",cbar=False)
ax.xaxis.set_label_position('top')
plt.tight_layout()
plt.ylabel('Actual label')
plt.xlabel('Predicted label');

##MODEL CLASSIFICATION -----------------------------------------------------------------------------------END

#MODEL ACCURACY COMPARISON # Reference [1] from Github Mentioned in Report

# Created a list of scores and Models

scores=[accuracy_score_dt,rf_accuracy,keras_accuracy]
models= ['DecisionTreeClassifier','Random Forest Classifier', 'Sequnetial Model-Keras']

#Printing the accuracy of each models:

for i in range(len(models)):
    print("The accuracy score achieved using "+models[i]+" is: "+str(scores[i])+" %")

# Comapring the accuracy of each model

sns.set(rc={'figure.figsize':(15,8)})
plt.xlabel("Models")
plt.ylabel("Accuracy score")

sns.barplot(models,scores)


"""initial migration

Revision ID: 282408d6d0f2
Revises:
Create Date: 2017-11-17 12:01:28.240094

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '282408d6d0f2'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('login',
    sa.Column('id', sa.INTEGER(), nullable=False),
    sa.Column('user_id', sa.INTEGER(), nullable=True),
    sa.Column('passwd', sa.VARCHAR(length=20), nullable=True),
    sa.Column('identity', sa.VARCHAR(length=10), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.drop_table('suggestion')
    op.drop_table('goods_common')
    op.drop_table('card_info')
    op.drop_table('django_migrations')
    op.alter_column('user', 'username',
               existing_type=mysql.VARCHAR(length=30),
               nullable=True)
    op.create_unique_constraint(None, 'user', ['email'])
    op.create_unique_constraint(None, 'user', ['username'])
    op.drop_column('user', 'department_number')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('department_number', mysql.INTEGER(display_width=10), autoincrement=False, nullable=True))
    op.drop_constraint(None, 'user', type_='unique')
    op.drop_constraint(None, 'user', type_='unique')
    op.alter_column('user', 'username',
               existing_type=mysql.VARCHAR(length=30),
               nullable=False)
    op.create_table('django_migrations',
    sa.Column('id', mysql.INTEGER(display_width=11), nullable=False),
    sa.Column('app', mysql.VARCHAR(length=255), nullable=False),
    sa.Column('name', mysql.VARCHAR(length=255), nullable=False),
    sa.Column('applied', mysql.DATETIME(fsp=6), nullable=False),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.create_table('card_info',
    sa.Column('id', mysql.INTEGER(display_width=50), nullable=False),
    sa.Column('student_id', mysql.INTEGER(display_width=20), autoincrement=False, nullable=True),
    sa.Column('kinds', mysql.VARCHAR(length=30), nullable=True),
    sa.Column('location', mysql.VARCHAR(length=20), nullable=True),
    sa.Column('pay_type', mysql.VARCHAR(length=20), nullable=True),
    sa.Column('pay_time', mysql.DATETIME(), nullable=True),
    sa.Column('money', mysql.VARCHAR(length=10), nullable=True),
    sa.Column('left_money', mysql.VARCHAR(length=10), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.create_table('goods_common',
    sa.Column('id', mysql.INTEGER(display_width=11), nullable=False),
    sa.Column('title', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('vardate', mysql.DATETIME(), nullable=True),
    sa.Column('linkman', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('contact', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('pulldate', mysql.DATETIME(), nullable=True),
    sa.Column('validdate', mysql.DATETIME(), nullable=True),
    sa.Column('haulway', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('ps', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('websource', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('websourcelink', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('longterm', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('linkaddress', mysql.VARCHAR(length=255), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.create_table('suggestion',
    sa.Column('id', mysql.INTEGER(display_width=50), nullable=False),
    sa.Column('student_id', mysql.INTEGER(display_width=20), autoincrement=False, nullable=True),
    sa.Column('type', mysql.VARCHAR(length=10), nullable=True),
    sa.Column('mean', mysql.DOUBLE(precision=20, scale=0, asdecimal=True), nullable=True),
    sa.Column('amount', mysql.INTEGER(display_width=20), autoincrement=False, nullable=True),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.drop_table('login')
    # ### end Alembic commands ###