import errno
import socket
import subprocess
import warnings
from typing import cast

import numpy as np
from pyquil.api import WavefunctionSimulator, get_qc
from qeforest.conversions import export_to_pyquil, qubitop_to_pyquilpauli
from zquantum.core.circuits import Circuit
from zquantum.core.interfaces.backend import QuantumSimulator, StateVector
from zquantum.core.measurement import ExpectationValues, Measurements
from zquantum.core.openfermion.ops import QubitOperator, SymbolicOperator
from zquantum.core.wavefunction import flip_amplitudes

class ForestSimulator(QuantumSimulator):
    supports_batching = False

    def __init__(self, device_name: str, seed: int = None, nthreads: int = 1):
        super().__init__()
        self.nthreads = nthreads
        self.device_name = device_name
        self.seed = seed

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        try:
            s.bind(("127.0.0.1", 5000))
            subprocess.Popen(["qvm", "-S"])
        except socket.error as e:
            if e.errno == errno.EADDRINUSE:
                warnings.warn(
                    "Port 5000 which is used to run QVM is already "
                    "running a process."
                )
            else:
                raise e
        try:
            s.bind(("127.0.0.1", 5555))
            subprocess.Popen(["quilc", "-S"])
        except socket.error as e:
            """TODO: figure out why quilc sometimes throws EINVAL error"""
            if e.errno == errno.EADDRINUSE or e.errno == errno.EINVAL:
                warnings.warn(
                    "Port 5555 which is used to run QUILC is already "
                    "running a process."
                )
            else:
                raise e

    def run_circuit_and_measure(self, circuit: Circuit, n_samples: int) -> Measurements:
        """Run a circuit and measure a certain number of bitstrings. Note: the number
        of bitstrings measured is derived from self.n_samples

        Args:
            circuit: the circuit to prepare the state
            n_samples: The number of samples to measure.
        Returns:
            a list of bitstrings (a list of tuples)
        """
        super().run_circuit_and_measure(circuit, n_samples=n_samples)
        cxn = get_forest_connection(self.device_name, self.seed)
        bitstrings = cxn.run_and_measure(export_to_pyquil(circuit), trials=n_samples)
        if isinstance(bitstrings, dict):
            bitstrings = np.vstack([bitstrings[q] for q in sorted(cxn.qubits())]).T

        bitstrings = [tuple(b) for b in bitstrings.tolist()]
        return Measurements(bitstrings)

    def get_exact_expectation_values(
        self, circuit: Circuit, qubit_operator: SymbolicOperator
    ) -> ExpectationValues:
        self.number_of_jobs_run += 1
        self.number_of_circuits_run += 1
        if self.device_name != "wavefunction-simulator":
            raise RuntimeError(
                "To compute exact expectation values, the device name must be "
                '"wavefunction-simulator". The device name is currently '
                f"{self.device_name}."
            )
        cxn = get_forest_connection(self.device_name, self.seed)

        # Pyquil does not support PauliSums with no terms.
        if len(qubit_operator.terms) == 0:
            return ExpectationValues(np.zeros((0,)))

        pauli_sum = qubitop_to_pyquilpauli(cast(QubitOperator, qubit_operator))
        expectation_values = np.real(
            cxn.expectation(export_to_pyquil(circuit), pauli_sum.terms)
        )

        if expectation_values.shape[0] != len(pauli_sum):
            raise (
                RuntimeError(
                    f"Expected {len(pauli_sum)} expectation values but received "
                    f"{expectation_values.shape[0]}."
                )
            )
        return ExpectationValues(expectation_values)

    def _get_wavefunction_from_native_circuit(
        self, circuit: Circuit, initial_state: StateVector
    ) -> StateVector:
        if not np.array_equal(
            np.array(initial_state), [1] + [0] * (len(initial_state) - 1)
        ):
            raise ValueError(
                "ForestSimulator does not support starting simulations from state "
                "other than |0>. In particular, it currently does not support "
                "non-native circuit components."
            )

        cxn = get_forest_connection(self.device_name, self.seed)
        wavefunction = cxn.wavefunction(export_to_pyquil(circuit))
        return flip_amplitudes(wavefunction.amplitudes)

def kill_subprocesses():
    """Kills all instances of QVM and QUILC using pkill command."""
    subprocess.run(["pkill", "qvm"])
    subprocess.run(["pkill", "quilc"])

def get_forest_connection(device_name: str, seed=None):
    """Get a connection to a forest backend

    Args:
        device_name: the device to connect to

    Returns:
        A connection to either a pyquil simulator or a QPU
    """
    if device_name == "wavefunction-simulator":
        return WavefunctionSimulator(random_seed=seed)
    else:
        return get_qc(device_name)


import pint
import time
import datetime

from . import ureg
from .constants import std_temperature, std_pressure
from .error import OpgeeException, AbstractMethodError
from .log import getLogger
from .utils import coercible, getBooleanXML

_logger = getLogger(__name__)

def magnitude(value, units=None):
    """
    Return the magnitude of ``value``. If ``value`` is a ``pint.Quantity`` and
    ``units`` is not None, check that ``value`` has the expected units and
    return the magnitude of ``value``. If ``value`` is not a ``pint.Quantity``,
    just return it.

    :param value: (float or pint.Quantity) the value for which we return the magnitude.
    :param units: (None or pint.Unit) the expected units
    :return: the magnitude of `value`
    """
    if isinstance(value, ureg.Quantity):
        # if optional units are provided, validate them
        if units:
            if not isinstance(units, pint.Unit):
                units = ureg.Unit(units)
            if value.units != units:
                raise OpgeeException(f"magnitude: value {value} units are not {units}")

        return value.m
    else:
        return value

def name_of(obj):
    return obj.name

def elt_name(elt):
    return elt.attrib.get('name')

def instantiate_subelts(elt, cls, parent=None, as_dict=False, include_names=None):
    """
    Return a list of instances of ``cls`` (or of its indicated subclass of ``Process``).

    :param elt: (lxml.etree.Element) the parent element
    :param cls: (type) the class to instantiate. If cls is Process, the class will
        be that indicated instead in the element's "class" attribute.
    :param parent: (XmlInstantiable) the parent to record in each object instantiated
    :param as_dict: (bool) if True, return a dictionary of subelements, keyed by name
    :param include_names: (list of str) Names of elements to include (i.e., the element's
       attrib dict must have a "name" item, whose value is compared to the list). If
       ``include_names`` is not None, then elements with names not in the list are
       ignored.
    :return: (list) instantiated objects
    """
    tag = cls.__name__  # class name matches element name

    include = None if include_names is None else set(include_names)
    objs = [cls.from_xml(e, parent=parent) for e in elt.findall(tag) if include is None or e.attrib.get('name') in include]

    if as_dict:
        d = {obj.name: obj for obj in objs}
        return d
    else:
        return objs

def dict_from_list(objs):
    """
    Create a dictionary of ``XMLInstantiable`` objects by their name attribute, but
    raise an error if any name is repeated.

    :param objs: (list of XMLInstantiable instances) the object to create dict from.
    :return: (dict) objects keyed by name
    :raises: OpgeeException if a any name is repeated
    """
    d = dict()
    for obj in objs:
        name = obj.name
        if d.get(name):
            classname = obj.__class__.__name__
            raise OpgeeException(f"{classname} instances must have unique names: {name} is not unique.")

        d[name] = obj

    return d

CLASS_DELIMITER = '.'

def split_attr_name(attr_name):
    splits = attr_name.split(CLASS_DELIMITER)
    count = len(splits)

    if count == 0:
        raise OpgeeException(f"Attribute name is empty")

    if count == 1:
        class_name, attr_name = None, splits[0]

    elif count == 2:
        class_name, attr_name = splits

    else:
        raise OpgeeException(f"Attribute name '{attr_name}' has more than 2 dot-delimited parts")

    return class_name, attr_name

# Top of hierarchy, because it's useful to know which classes are "ours"
class OpgeeObject():
    @classmethod
    def clear(cls):
        # Clear state stored in class variables
        pass

class XmlInstantiable(OpgeeObject):
    """
    This is the superclass for all classes that are instantiable from XML. The requirements of
    such classes are:

    1. They subclass from XmlInstantiable or its subclasses
    2. They define ``__init__(self, name, **kwargs)`` and call ``super().__init__(name)``
    3. They define ``@classmethod from_xml(cls, element, parent=None)`` to create an instance from XML.
    4. Subclasses of Container and Process implement ``run(self)`` to perform any required operations.

    """
    def __init__(self, name, parent=None):
        super().__init__()
        self.name = name
        self.parent = parent
        self.enabled = True

    def set_parent(self, parent):
        self.parent = parent

    @classmethod
    def from_xml(cls, elt, parent=None):
        raise AbstractMethodError(cls, 'XmlInstantiable.from_xml')

    def __str__(self):
        type_str = type(self).__name__
        name_str = f' name="{self.name}"' if self.name else ''
        return f'<{type_str}{name_str} enabled={self.enabled}>'

    def is_enabled(self):
        return self.enabled

    def set_enabled(self, value):
        self.enabled = getBooleanXML(value)

    def adopt(self, objs, asDict=False):
        """
        Set the `parent` of each object to self. This is used to create back pointers
        up the hieararchy so Processes and Streams can find their Field and Analysis
        containers. Return the objects either as a list or dict.

        :param objs: (None or list of XmlInstantiable)
        :param asDict: (bool) if True, return a dict of objects keyed by their name,
            otherwise return a list of the objects.
        :return: (list) If objs is None, return an empty list or dict (per `asDict`),
            otherwise return the objs either in a list or dict.
        """
        objs = [] if objs is None else objs

        for obj in objs:
            obj.set_parent(self)

        return {obj.name: obj for obj in objs} if asDict else objs

    def find_container(self, cls):
        """
        Ascend the parent links in the object hierarchy until an object of class `cls`
        is found, or an object with a parent that is None.

        :param cls: (type or str name of type) the class of the parent sought
        :return: (XmlInstantiable or None) the desired parent instance or None
        """
        if type(self) == cls or self.__class__.__name__ == cls:
            return self

        if self.parent is None:
            return None

        return self.parent.find_container(cls)  # recursively ascend the graph

# to avoid redundantly reporting bad units
_undefined_units = {}

def validate_unit(unit):
    """
    Return the ``pint.Unit`` associated with the string ``unit``, or ``None``
    if ``unit`` is ``None`` or not in the unit registry.

    :param unit: (str) a string representation of a ``pint.Unit``

    :return: (pint.Unit or None)
    """
    if not unit:
        return None

    if unit in ureg:
        return ureg.Unit(unit)

    if unit not in _undefined_units:
        _logger.warning(f"Unit '{unit}' is not in the UnitRegistry")
        _undefined_units[unit] = 1

    return None

class A(OpgeeObject):
    """
    The ``<A>`` element represents the value of an attribute previously defined in
    `attributes.xml` or a user-provided file. Note that this class is not instantiated
    using ``from_xml()`` approach since values are merged with metadata from `attributes.xml`.
    """

    def __init__(self, name, value=None, pytype=None, unit=None, explicit=False):
        """
        Creates an attribute instance.

        :param name: (str) the attribute name
        :param value: (str) string representation of attribute value
        :param pytype: (type) the type the value should be coerced to (plus unit)
        :param unit: (str) the attributes units
        :param explicit: (bool) whether the value was explicit in the file; if not,
          it implies the value was set from the default in the attribute definition.
        """
        super().__init__()
        self.name = name
        self.unit = unit
        self.pytype = pytype
        self.explicit = explicit
        self.value = self.set_value(value)

    def set_value(self, value):
        """
        Sets the instances' value to the value given, using the stored ``pytype`` for type
        conversion and ``unit`` to define a ``pint.Quantity``, if given.

        :param value: (str, numerical, or pint.Quantity) the value to possibly convert
        :return: the value, converted to ``pytype``, and with ``unit``, if specified.
        """
        if value is None:
            return None

        unit_obj = validate_unit(self.unit)

        if isinstance(value, pint.Quantity):
            if value.units == unit_obj:
                self.value = value
                return value
            else:
                value = value.magnitude

        if self.pytype:
            value = coercible(value, self.pytype)

        if unit_obj is not None:
            value = ureg.Quantity(value, unit_obj)

        self.value = value
        return value

    def str_value(self):
        """
        Return the value of an attribute as a string.

        :return: (str) string representation of the attribute value
        """
        return str(self.value)

    def __str__(self):
        type_str = type(self).__name__
        attrs = f"name='{self.name}' type='{self.pytype}' value='{self.value}'"

        return f"<{type_str} {attrs}>"

class TemperaturePressure(OpgeeObject):
    """
    Stores temperature and pressure together for convenience.
    """
    __slots__ = ('T', 'P')      # keeps instances small and fast

    def __init__(self, T, P):
        self.T = None
        self.P = None
        self.set(T=T, P=P)

    def __str__(self):
        return f"<T={self.T} P={self.P}>"

    def set(self, T=None, P=None):
        if T is None and P is None:
            #_logger.warning("Tried to set TemperaturePressure with both values None")
            return

        if T is not None:
            self.T = T if isinstance(T, pint.Quantity) else ureg.Quantity(float(T), "degF")

        if P is not None:
            self.P = P if isinstance(P, pint.Quantity) else ureg.Quantity(float(P), "psia")

    def get(self):
        return (self.T, self.P)

    def copy_from(self, tp):
        self.set(T=tp.T, P=tp.P)

# Standard temperature and pressure
STP = TemperaturePressure(std_temperature, std_pressure)

class Timer:
    def __init__(self, feature_name):
        self.feature_name = feature_name
        self.start_time = None
        self.stop_time  = None

    def start(self):
        self.start_time = time.time()
        return self

    def stop(self):
        self.stop_time = time.time()
        return self

    def duration(self):
        seconds = self.stop_time - self.start_time
        return datetime.timedelta(seconds=int(seconds))

    def __str__(self):
        if self.start_time is None:
            status = "is uninitialized"
        elif self.stop_time is None:
            status = "is running"
        else:
            d = self.duration()
            status = f"completed in {d}"

        return f"<Timer '{self.feature_name}' {status}>"


from __future__ import absolute_import, division, print_function, unicode_literals
import os
import torch
import argparse
from scipy.io.wavfile import write
import numpy as np
from model.generator import Generator
import json
from utils.hparams import HParam, load_hparam_str
from denoiser import Denoiser

MAX_WAV_VALUE = 32768.0

def load_checkpoint(filepath, device):
    assert os.path.isfile(filepath)
    print("Loading '{}'".format(filepath))
    checkpoint_dict = torch.load(filepath, map_location=device)
    print("Complete.")
    return checkpoint_dict

def main(args):
    checkpoint = torch.load(args.checkpoint_path)
    if args.config is not None:
        hp = HParam(args.config)
    else:
        hp = load_hparam_str(checkpoint['hp_str'])

    model = Generator(hp.audio.n_mel_channels).cuda()

    model.load_state_dict(checkpoint['model_g'])
    model.eval()
    #model.remove_weight_norm()

    with torch.no_grad():
        mel = torch.from_numpy(np.load(args.input))
        if len(mel.shape) == 2:
            mel = mel.unsqueeze(0)
        mel = mel.cuda()
        #zero = torch.full((1, 80, 10), -11.5129).to(mel.device)
        #mel = torch.cat((mel, zero), dim=2)
        hifigan_trace = torch.jit.trace(model, mel)
        #print(state_dict_g.keys())
        hifigan_trace.save("{}/hifigan_{}.pt".format(args.out, args.name))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', type=str, default=None, required=True,
                        help="yaml file for config. will use hp_str from checkpoint if not given.")
    parser.add_argument('-p', '--checkpoint_path', type=str, required=True,
                        help="path of checkpoint pt file for evaluation")
    parser.add_argument('-i', '--input', type=str, required=True,
                        help="directory of mel-spectrograms to invert into raw audio. ")
    parser.add_argument('-o', '--out', type=str, required=True,
                        help="path of output pt file")
    parser.add_argument('-n', '--name', type=str, required=True,
                        help="name of the output file")
    args = parser.parse_args()

    main(args)


import scrapy

OUTPUT_DIR = 'icons'

HTTPS = 0

class FaviconSpider(scrapy.Spider):
    name = 'favicon'

    def start_requests(self):
        with open('urls.txt') as urls:
            last_basename = None
            for url in urls:
                url = url.strip()
                basename = url.split('.')[0]
                if basename == last_basename:
                    continue
                last_basename = basename
                req = scrapy.Request('http{}://www.'.format('s' if HTTPS else '') + url + '/favicon.ico', callback=self.parse)
                req.meta['url'] = url
                yield req

    def parse(self, response):
        name = response.request.meta['url']
        if response.status != 200:
            return
        with open(OUTPUT_DIR + '/' + name + '.favicon.ico', 'wb') as f:
            f.write(response.body)


import cv2
import numpy as np

img = cv2.imread(
    r"D:\FCI\The 3 Level\The Second Term\New\Digital Image Processing\Tasks\imags\input_image1.png")

hist = np.zeros((256, 1), dtype=np.uint8)

for row in range(img.shape[0]):
    for col in range(img.shape[1]):

        pixel_value = img[row, col]
        hist[pixel_value] += 1

print(hist)


import pytest

from lightprop.propagation.params import ParamsValidationException, PropagationParams

class TestPropagationParams:
    @pytest.fixture()
    def params(self):
        return PropagationParams.get_example_propagation_data()

    def test_int_params(self, params):
        int_params = ["focal_length", "distance"]
        proper_value = 5
        proper_negative_value = -1
        proper_float_value = 0.1
        xfail_value_not_convertible = " - 1 2 test 3"

        for int_param in int_params:
            setattr(params, int_param, proper_value)
            assert getattr(params, int_param) == proper_value
            setattr(params, int_param, proper_negative_value)
            assert getattr(params, int_param) == proper_negative_value
            setattr(params, int_param, proper_float_value)
            assert getattr(params, int_param) == int(proper_float_value)
            with pytest.raises(ParamsValidationException, match=f"cannot be converted to {int}"):
                setattr(params, int_param, xfail_value_not_convertible)

    def test_positive_int_params(self, params):
        positive_int_params = ["matrix_size"]
        proper_value = 5
        xfail_zero = 0
        xfail_negative_value = -1

        for int_param in positive_int_params:
            setattr(params, int_param, proper_value)
            assert getattr(params, int_param) == proper_value
            with pytest.raises(ParamsValidationException, match=f"{int} greater than 0"):
                setattr(params, int_param, xfail_negative_value)
            with pytest.raises(ParamsValidationException, match=f"{int} greater than 0"):
                setattr(params, int_param, xfail_zero)

    def test_positive_float_params(self, params):
        positive_float_params = ["wavelength", "pixel_size", "nu", "beam_diameter"]
        proper_value = 5
        proper_value2 = "5.3"
        xfail_negative_value = -1

        for float_param in positive_float_params:
            setattr(params, float_param, proper_value)
            assert getattr(params, float_param) == proper_value
            setattr(params, float_param, proper_value2)
            assert getattr(params, float_param) == float(proper_value2)
            with pytest.raises(ParamsValidationException, match=f"{float} greater than 0"):
                setattr(params, float_param, xfail_negative_value)


# -*- coding: utf-8 -*-
# @Time    : 2020/4/9 12:45
# @Author  : DforceL
import requests
from bs4 import BeautifulSoup
import os
URL="https://www.msgao.com/xinggan/"
HTML=requests.get(URL).text
# print(HTML)
soup=BeautifulSoup(HTML,features="lxml")
# print(soup)
imgullist=soup.find_all("div",{"class":"listmain left"})
# print(imgullist)
counter=0
for iul in imgullist:
    imgusl=iul.find_all("img")
    for imgul in imgusl:
        ul=imgul["src"]
        counter+=1
        r=requests.get(ul)
        imgname=str(counter)+".jpg"
        pathname="./floder/sexylady/"+imgname
        with open(pathname,"wb") as f:
            f.write(r.content)
        print("saved as:{}".format(imgname))


"""
https://towardsdatascience.com/work-with-sql-in-python-using-sqlalchemy-and-pandas-cd7693def708
"""

from sqlalchemy import create_engine
from sqlalchemy.orm import Session, aliased
from sqlalchemy.sql import text
import pandas as pd
from decouple import config             # pip install python-decouple

# Create an engine to the Database
# url = 'postgresql+psycopg2://username:password@host:port/database'
db_url_1 = config("DB_POWERWORKS_1")   # from .env
engine_1 = create_engine(db_url_1, echo=False, future=True)

# Run a SQL Query using SQLAlchemy.orm (Read data from database 1)
with Session(engine_1) as session:
    sql = '''
        SELECT * FROM person;
    '''
    results = session.execute(sql)
df = pd.DataFrame(results.fetchall())
# print("data from database_1")
# print(df)

# Read SQL Table into a Pandas Data Frame (using an engine connection)
# with engine_1.connect().execution_options(autocommit=True) as conn:
#     df = pd.read_sql_table("person", conn)

# Connenction to Database #2
db_url_2 = config("DB_SQLALCHEMY_PANDAS_1")   # from .env
engine_2 = create_engine(db_url_2, echo=False, future=False)   # << Set future=False

# Go to DBeaver to create database table Person in the "DB_SQLALCHEMY_PANDAS_1"

# Insert DataFrame into an existing SQL Database Table (Database 2)
with engine_2.connect().execution_options(autocommit=True) as conn:
    # Create a New SQL Database
    df.to_sql('person', con=conn, schema='public',if_exists='append', index=False)

# # Run a SQL Query using SQLAlchemy.orm (Read data from Databse 2)
with Session(engine_2) as session:
    sql = '''
        SELECT * FROM person;
    '''
    results = session.execute(sql)
    df2 = pd.DataFrame(results.fetchall())
    print("data from database_2")
    print(df2)


from django.shortcuts import render
from django.views.generic import ListView, DetailView
from django.views.generic.base import View

from .models import Artist, Album, Track

class HomeView(View):
    def get(self, request):
        context = {'artist_list': Artist.objects.all(), 'track_list': Track.objects.all()}
        return render(request, 'index.html', context=context)

class Artist_detail(DetailView):
    model = Artist
    slug_field = "url"
    context_object_name = 'artists'
    template_name = "main/artist.html"

class Artist_list(View):
    pass

class Album_detail(DetailView):
    model = Album
    slug_field = "url"
    context_object_name = 'releases'
    template_name = "main/release.html"

class Album_list(ListView):
    def get(self, request):
        album = Album.objects.all()
        return render(request, 'main/releases.html', {"album_list": album})

class Tracks_detail(DetailView):
    model = Track
    slug_field = "url"
    context_object_name = 'tracks'
    template_name = "include/test.html"


import json
import yaml
import logging.config
from typing import Tuple
import click

# DATA
from source.data import read_data, split_data, construct_features
from source.data.construct_features import build_transformer, get_target, dump_transformer
# MODEL
from source.models import train_model, predict_model, evaluate_model, dump_model
# PIPELINE + LOGGER
from source.entities import StreamLoggingParams, TrainPipelineParams, read_train_pipeline_params

# LOGGER CONFIGURING
with open(StreamLoggingParams.config_path, "r") as stream:
    config = yaml.load(stream, Loader=yaml.FullLoader)
logging.config.dictConfig(config)
logger = logging.getLogger(StreamLoggingParams.logger)
splitter = StreamLoggingParams.field_splitter

def train_pipeline(train_pipeline_params: TrainPipelineParams) -> Tuple[str, str, dict]:
    # READING DATASET
    # logger.info(f"training pipeline params:{splitter}{train_pipeline_params}{splitter}")
    data = read_data(train_pipeline_params.input_data_path)
    logger.debug(f"data.shape: {data.shape}")
    # SPLITTING DATA
    train_df, test_df = split_data(data, train_pipeline_params.split_params)
    logger.debug(f"train_df.shape: {train_df.shape}")
    logger.debug(f"test_df.shape: {test_df.shape}")

    transformer = build_transformer(train_pipeline_params.feature_params)
    transformer.fit(train_df)

    # dump fitted transformer
    try:
        path_to_transformer = dump_transformer(transformer, train_pipeline_params.output_transformer_path)
        logger.info(f"transformer serialization: {train_pipeline_params.output_transformer_path}")
    except FileNotFoundError:
        path_to_transformer = None
        logger.warning(f"invalid path for transformer serialization: {train_pipeline_params.output_transformer_path}")

    # FEATURES CONSTRUCTION
    train_features = construct_features(transformer, train_df)
    train_target = get_target(train_df, train_pipeline_params.feature_params)
    logger.debug(f"train_features.shape: {train_features.shape}")

    test_features = construct_features(transformer, test_df)
    val_target = get_target(test_df, train_pipeline_params.feature_params)
    logger.debug(f"test_features.shape: {test_features.shape}")

    model = train_model(train_features, train_target, train_pipeline_params.model_params)

    preds = predict_model(model, test_features)
    metrics = evaluate_model(preds, val_target)

    # dump metrics
    with open(train_pipeline_params.metric_path, "w") as metric_file:
        json.dump(metrics, metric_file)
    logger.info(f"training results: {metrics}")

    # dump model
    try:
        path_to_model = dump_model(model, train_pipeline_params.output_model_path)
        logger.info(f"model serialization: {train_pipeline_params.output_model_path}")
    except FileNotFoundError:
        path_to_model = None
        logger.warning(f"invalid path for model serialization: {train_pipeline_params.output_model_path}")

    return path_to_model, path_to_transformer, metrics

# CALL FROM COMMAND LINE
@click.command()
@click.argument("config_path")
def train_pipeline_command(config_path: str):
    params = read_train_pipeline_params(config_path)
    train_pipeline(params)

def main():
    train_pipeline_command()

if __name__ == "__main__":
    main()


import datetime
import json
import os
from uuid import uuid4

import numpy as np
import pytest

from whylogs.core.datasetprofile import DatasetProfile, array_profile, dataframe_profile
from whylogs.core.model_profile import ModelProfile
from whylogs.util import time
from whylogs.util.protobuf import message_to_dict, message_to_json
from whylogs.util.time import to_utc_ms

def test_all_zeros_returns_summary_with_stats():
    stats = ("min", "max", "stddev", "mean")
    array = np.zeros([100, 1])

    prof = array_profile(array)
    msg = prof.to_summary()
    d = message_to_dict(msg)
    d1 = json.loads(message_to_json(msg))
    number_summary = d["columns"]["0"]["numberSummary"]
    missing_stats = [k for k in stats if k not in number_summary]
    if len(missing_stats) > 0:
        raise RuntimeError(f"Stats missing from number summary: {missing_stats}")

    assert d == d1

def test_empty_valid_datasetprofiles_empty():
    now = datetime.datetime.now(datetime.timezone.utc)
    shared_session_id = uuid4().hex
    x1 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    x2 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )

    merged = x1.merge(x2)
    assert merged.name == "test"
    assert merged.session_id == shared_session_id
    assert merged.session_timestamp == now
    assert merged.columns == {}

def test_merge_different_columns():
    now = datetime.datetime.now(datetime.timezone.utc)
    shared_session_id = uuid4().hex
    x1 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "x1"},
    )
    x1.track("col1", "value")
    x2 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "x2"},
    )
    x2.track("col2", "value")

    merged = x1.merge(x2)

    assert merged.name == "test"
    assert merged.session_id == shared_session_id
    assert merged.session_timestamp == now
    assert set(list(merged.columns.keys())) == {"col1", "col2"}
    assert merged.columns["col1"].counters.count == 1
    assert merged.columns["col2"].counters.count == 1
    assert merged.tags == dict({"name": "test", "key": "value"})
    assert merged.metadata == dict({"key": "x1"})

def test_merge_lhs_no_profile():
    now = datetime.datetime.now(datetime.timezone.utc)
    shared_session_id = uuid4().hex
    x1 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    x2 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
        model_profile=ModelProfile(),
    )

    merged = x1.merge(x2)
    assert merged.name == "test"
    assert merged.session_id == shared_session_id
    assert merged.session_timestamp == now
    assert merged.columns == {}
    assert merged.model_profile is not None

def test_merge_rhs_no_profile():
    now = datetime.datetime.now(datetime.timezone.utc)
    shared_session_id = uuid4().hex
    x1 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
        model_profile=ModelProfile(),
    )
    x2 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )

    merged = x1.merge(x2)
    assert merged.name == "test"
    assert merged.session_id == shared_session_id
    assert merged.session_timestamp == now
    assert merged.columns == {}
    assert merged.model_profile is not None

def test_merge_same_columns():
    now = datetime.datetime.now(datetime.timezone.utc)
    shared_session_id = uuid4().hex
    x1 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    x1.track("col1", "value1")
    x2 = DatasetProfile(
        name="test",
        session_id=shared_session_id,
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    x2.track("col1", "value1")
    x2.track("col2", "value")

    merged = x1.merge(x2)
    assert merged.name == "test"
    assert merged.session_id == shared_session_id
    assert merged.session_timestamp == now
    assert set(list(merged.columns.keys())) == {"col1", "col2"}
    assert merged.columns["col1"].counters.count == 2
    assert merged.columns["col2"].counters.count == 1

def test_protobuf_round_trip():
    now = datetime.datetime.now(datetime.timezone.utc)
    tags = {"k1": "rock", "k2": "scissors", "k3": "paper"}
    original = DatasetProfile(
        name="test",
        dataset_timestamp=now,
        tags=tags,
    )
    original.track("col1", "value")
    original.track("col2", "value")

    msg = original.to_protobuf()
    roundtrip = DatasetProfile.from_protobuf(msg)

    assert roundtrip.name == "test"
    assert roundtrip.session_id == original.session_id
    assert to_utc_ms(roundtrip.session_timestamp) == to_utc_ms(original.session_timestamp)
    assert set(list(roundtrip.columns.keys())) == {"col1", "col2"}
    assert roundtrip.columns["col1"].counters.count == 1
    assert roundtrip.columns["col2"].counters.count == 1

    tags["name"] = "test"
    assert set(roundtrip.tags) == set(tags)
    assert roundtrip.metadata == original.metadata

def test_non_string_tag_raises_assert_error():
    now = datetime.datetime.now(datetime.timezone.utc)
    tags = {"key": "value"}
    x = DatasetProfile("test", now, tags=tags)
    x.validate()
    # Include a non-string tag
    x._tags["number"] = 1

    try:
        x.validate()
        raise RuntimeError("validate should raise an AssertionError")
    except AssertionError:
        pass

def test_mismatched_tags_raises_assertion_error():
    now = datetime.datetime.now(datetime.timezone.utc)
    x1 = DatasetProfile("test", now, tags={"key": "foo"})
    x2 = DatasetProfile("test", now, tags={"key": "bar"})
    try:
        x1.merge_strict(x2)
        raise RuntimeError("Assertion error not raised")
    except AssertionError:
        pass

def test_mismatched_tags_merge_succeeds():
    now = datetime.datetime.now(datetime.timezone.utc)
    x1 = DatasetProfile("test", now, tags={"key": "foo"})
    x2 = DatasetProfile("test2", now, tags={"key": "bar"})

    result = x1.merge(x2)
    assert result.tags.get("key") == "foo"

def test_name_always_appear_in_tags():
    x1 = DatasetProfile(name="test")
    assert x1.tags["name"] == "test"

def test_parse_delimited_from_java_single():
    dir_path = os.path.dirname(os.path.realpath(__file__))

    with open(os.path.join(dir_path, "output_from_java_08242020.bin"), "rb") as f:
        data = f.read()
        assert DatasetProfile.parse_delimited_single(data) is not None

    with open(os.path.join(dir_path, "output_from_java_01212021.bin"), "rb") as f:
        data = f.read()
        assert DatasetProfile.parse_delimited_single(data) is not None

def test_parse_from_protobuf():
    dir_path = os.path.dirname(os.path.realpath(__file__))
    DatasetProfile.read_protobuf(os.path.join(dir_path, "output_from_java_08242020.bin"))

def test_parse_delimited_from_java_multiple():
    dir_path = os.path.dirname(os.path.realpath(__file__))

    with open(os.path.join(dir_path, "output_from_java_08242020.bin"), "rb") as f:
        data = f.read()
        multiple = data + data
        result = DatasetProfile.parse_delimited(multiple)
        assert len(result) == 2

def test_write_delimited_single():
    now = datetime.datetime.now(datetime.timezone.utc)

    original = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    original.track("col1", "value")

    output_bytes = original.serialize_delimited()
    pos, roundtrip = DatasetProfile.parse_delimited_single(output_bytes)

    assert roundtrip.session_id == original.session_id
    # Python time precision includes nanoseconds
    assert time.to_utc_ms(roundtrip.session_timestamp) == time.to_utc_ms(original.session_timestamp)
    assert roundtrip.tags == original.tags
    assert roundtrip.metadata == original.metadata

def test_write_delimited_multiple():
    now = datetime.datetime.now(datetime.timezone.utc)

    original = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    original.track("col1", "value")

    output_bytes = original.serialize_delimited()

    multiple_entries = output_bytes
    for i in range(1, 5):
        multiple_entries += output_bytes

    entries = DatasetProfile.parse_delimited(multiple_entries)
    assert len(entries) == 5

    for entry in entries:
        assert entry.session_id == original.session_id
        # Python time precisions are different
        assert time.to_utc_ms(entry.session_timestamp) == time.to_utc_ms(original.session_timestamp)
        assert entry.tags == original.tags
        assert entry.metadata == original.metadata

def test_verify_schema_version():
    dp = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=datetime.datetime.now(),
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    props = dp.to_properties()
    assert props.schema_major_version == 1
    assert props.schema_minor_version == 2

def tests_timestamp():
    time = datetime.datetime.now()
    dp = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=datetime.datetime.now(),
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    time_2 = dp.session_timestamp_ms
    assert time_2 == int(time.replace(tzinfo=datetime.timezone.utc).timestamp() * 1000.0)

def test_dataframe_profile(df):
    time = datetime.datetime.now()

    profile = DatasetProfile("test", time)
    profile.track_dataframe(df)

    profile_factory = dataframe_profile(df, name="test", timestamp=time)

    assert profile_factory.columns["A"].number_tracker.variance.mean == profile.columns["A"].number_tracker.variance.mean

    profile_factory_2 = dataframe_profile(df)
    assert profile_factory_2.columns["A"].number_tracker.variance.mean == profile.columns["A"].number_tracker.variance.mean
    profile_factory_3 = dataframe_profile(df, timestamp=103433)

    assert profile_factory_3.columns["A"].number_tracker.variance.mean == profile.columns["A"].number_tracker.variance.mean

def test_track():
    now = datetime.datetime.now(datetime.timezone.utc)

    original = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )

    data = {
        "rows": 1,
        "names": "roger roger",
    }
    original.track(columns=data)

def test_errors():
    now = datetime.datetime.now(datetime.timezone.utc)

    original = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    with pytest.raises(TypeError):
        original.track(columns=1, data=34)

def test_flat_summary():
    now = datetime.datetime.now(datetime.timezone.utc)

    original = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    flat_summary = original.flat_summary()
    assert flat_summary is not None
    assert len(original.flat_summary()) == 3

def test_chunk_iterator():
    now = datetime.datetime.now(datetime.timezone.utc)

    original = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    data = {
        "rows": 1,
        "names": "roger roger",
    }
    original.track(columns=data)

    for each_chuck in original.chunk_iterator():
        assert each_chuck is not None

def test_array():
    now = datetime.datetime.now(datetime.timezone.utc)

    original = DatasetProfile(
        name="test",
        session_id="test.session.id",
        session_timestamp=now,
        tags={"key": "value"},
        metadata={"key": "value"},
    )
    with pytest.raises(ValueError):
        original.track_array(np.random.rand(3))


import pygame # 导入 Pygame 模块
from plane import * # 导入本地 plane 模块
import time # 导入 time 模块
from pygame.locals import *

def start(): # 定义一个 start 类
    screen = pygame.display.set_mode((480, 700))
    background = pygame.image.load("./Resources/background.png").convert()
    hero_plane = HeroPlane(screen)
    enemy_plane = EnemyPlane(screen)
    while True:
        screen.blit(background, (0, 0))
        hero_plane.display()
        enemy_plane.move()
        enemy_plane.launch_bullet()
        enemy_plane.display()
        time.sleep(0.01)
        pygame.display.update()
        for event in pygame.event.get():
            if event.type == QUIT:
                print("再见")
                exit()
            elif event.type == KEYDOWN:
                if event.key == K_LEFT or event.key == K_a:
                    print("←")
                    hero_plane.move_left()
                if event.key == K_RIGHT or event.key == K_d:
                    print("→")
                    hero_plane.move_right()
                if event.key == K_SPACE:
                    print("你按了空格")
                    hero_plane.launch_bullet()

if __name__ == '__main__':
    start()


# -*- coding: utf-8 -*-
"""
@File: logger
@Time: 2020-07-01 14:37:16
@Auth: tang
"""
import datetime
from gli.logger import Logger
from gli.connect.mongodb import MongoDB
from datetime import datetime, timedelta

class ProxyIP(object):
    """Proxy IP
    """
    def __init__(self):
        self.logger = Logger(folder="zhima")
        client = MongoDB()
        if "local_rw" in client:
            self.reader = client["local_rw"]["proxies"]
        else:
            raise RuntimeError("The specified configuration item could not be found.")
        if "local_rw" in client:
            self.writer = client["local_rw"]["proxies"]
        else:
            raise RuntimeError("The specified configuration item could not be found.")

    def inc_used(self, id: object, field: str):
        """ Increase the number of used.
        Args:
            id: Proxy ID.
            field: Field name in proxy information.
        """
        try:
            self.writer["proxy"].update_one(
                { "_id": id },
                { "$set": { "update_time": datetime.now() }, "$inc": { "used": int(1), "spider.%s.used" % field: int(1) } })
        except Exception as err:
            self.logger.error(err)

    def inc_fail(self, id:object, field:str):
        """ Increase the number of fail.
        Args:
            id: Proxy ID.
            field: Field name in proxy information.
        """
        try:
            self.writer["proxy"].update_one(
                { "_id": id },
                { "$set": { "update_time": datetime.now() }, "$inc": { "fail": int(1), "spider.%s.fail" % field: int(1) } })
        except Exception as err:
            self.logger.error(err)

    def modify_update_time(self):
        """ Modify update time.
        Args:
            id: Proxy ID.
            field: Field name in proxy information.
        """
        try:
            self.reader["proxy"].find_one_and_update(
                {},
                { "$set": { "update_time": datetime.now() } },
                sort = [("update_time", -1)])
        except Exception as err:
            self.logger.error(err)

    def run(self, field:str, fail_weight: int = 20, margin: int = 10):
        """Get proxy IP address.
        Args:
            field: Field name in proxy information.
            fail_weight: Reduce selection probability after failure.
            margin: Expiration time margin of proxy IP.
        Returns:
            Success returns { "_id", "ip", "port" }, otherwise None.
        """
        try:
            cursor = self.reader["proxy"].aggregate([
                { "$match": { "expire_time": { "$gt": datetime.now() + timedelta(seconds=margin) } } },
                { "$addFields": { "count": { "$add": [ "$spider.%s.used" % field, { "$multiply": [ "$spider.%s.fail" % field, fail_weight ] } ] } } },
                { "$sort": { "count": 1 } },
                { "$project": { "ip": 1, "port": 1 } },
                { "$limit": 1 } ],
                allowDiskUse=False)
            for doc in cursor:
                # Increase the number of used.
                self.inc_used(doc["_id"], field)
                return doc
            # Modify update time.
            self.modify_update_time()
        except Exception as err:
            self.logger.error(err)


'''
File dependencies are in requirements.txt, install using pip install -r requirements.txt .
Please include positive_words.txt in the working directory as well.

Project Quote Generator
The program takes user input of a book title, scrapes that book's description for Goodreads, takes the longest
word from that description, and returns a Goodreads quote to the user that's tagged with that word. If
the user does not provide a title, the script will take a random word from a text file and return
a random tagged quote.

The program moves slowly as it's a lot of Selenium execution. It will default
to showing the Selenium web browser, but this can be turned off by answering
the first prompt to the user.

Thank you!
'''

from selenium import webdriver
import random
from bs4 import BeautifulSoup
import time
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def init_browser(no_show):
    '''
    Helper function for setting up a webdriver.
    Parameters:
        no_show: a boolean representing whether or not the user wants to see the web scraper run
    '''
    if no_show:
        # if the user doesn't want to watch the scraper run, we set the driver to headless, so no window pops up.
        chrome_options = Options()
        chrome_options.add_argument("--headless")
        browser = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)
    else:
        # otherwise, invoke webdriver which defaults to showing the window as it scrapes.
        browser = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
    return browser

def soupify(browser):
    '''
    Helper function for converting page to beautiful soup object.
    Parameters:
        browser: a selenium webdriver instance
    '''
    html = browser.page_source
    soup = BeautifulSoup(html, features = 'lxml')
    return soup

def get_recent_read(no_show):
    '''
    Asks for user input of a book title and scrapes the book description from Goodreads.
    Returns a tuple of the title the user entered and the scraped description, if the user provides
    a book title, else it returns an empty string.
    Parameters:
        no_show: a boolean representing whether or not the user wants to see the web scraper run
    '''
    book_title = input("Enter the title of a book you've enjoyed recently, or hit enter for a random quote: ")
    if not book_title:
       return ''

    # initialize chrome webdriver
    browser = init_browser(no_show)
    print('Fetching your book from goodreads!')
    browser.get('https://www.goodreads.com/quotes')

    # Find search elements
    search_input = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH, "//input[@name='q']")))
    submit_button = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH, "//button[@aria-label='Search']")))

    # Input book title and click search icon
    search_input.send_keys(book_title)
    submit_button.click()

    # Once on new page, click on the first book title
    page_title = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH, "//a[@class='bookTitle']")))
    page_title.click()
    time.sleep(5) # ensure page loads all elements

    # turn browser html into a Beautiful soup object for further parsing.
    soup = soupify(browser)
    # get page contents and find description.
    div = soup.find('div', id = 'description')
    if div:
        spans = div.find_all('span') # the book description lives in a series of spans
        desc_text = spans[1].text # the second one has the information I want.
    browser.close()
    return (book_title.title(), desc_text)

def get_word_from_desc(desc):
    '''
    Takes a book description as scraped by get_recent_read. Removes a portion of stopwords, sorts in descending order,
    and returns the longest word from the description. Choosing to return the longest under the surface assumption
    that a longer word will be more interesting/relevant for later steps.
    '''
    # NLTK would be better for this purpose, but it's a hassle to install, so hard-coding a list of some stopwords instead.
    stop_words = [ 'we', 'her','of', 'she', 'he', 'stop', 'the', 'to', 'and', 'a', 'in', 'it', 'is', 'i', 'that', 'had', 'on', 'for', 'were', 'was','an','at']
    desc_list = desc.split()
    possible_words = [word.lower() for word in desc_list if word.lower() not in stop_words and word.lower().isalpha()]
    # throw out duplicates and sort
    possible_words = set(possible_words)
    sorted_words = sorted(possible_words, key = len, reverse=True)
    print("Picked a word out from your book's description.")
    return sorted_words[0]

def pick_random_word():
    '''
    In the case that the user doesn't enter a book title, this function is called instead.
    It returns a random word from a file of positive words.

    This list of positive sentiment words was created by  Minqing Hu and Bing Liu. "Mining and Summarizing Customer Reviews."
    Proceedings of the ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD-2004), Aug 22-25, 2004, Seattle,
    Washington, USA. They asked that this paper be cited.
    '''
    word_list = []
    with open('positive_words.txt','r') as fp:
        for line in fp:
            word_list.append(line.strip())
    word_list = word_list[35:] #remove documentation at beginning of file
    return random.choice(word_list)

def pick_quote(random_word, no_show):
    '''
    Takes a word either generated at random or parsed from the user's book selection.
    Searches Goodreads Quotes for this word and returns the top quote that's tagged with this word.
    Parameters:
        no_show: a boolean representing whether or not the user wants to see the web scraper run
    Returns: string representing a quote scraped from Goodreads.
    '''
    # intialize chrome webdriver
    browser = init_browser(no_show)
    print("Picking a quote from Goodreads!")
    browser.get('https://www.goodreads.com/quotes')

    # find search elements
    search_input = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH, "//input[@id='explore_search_query']")))
    submit_button = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH, "//input[@name='commit']")))

    # send word and click submit button
    search_input.send_keys(random_word)
    submit_button.click()
    time.sleep(4) # ensure page loads all elements

    # get page html and extract quote
    soup = soupify(browser)
    browser.close()
    quotes = soup.find_all('div', class_='quoteText')
    quote = quotes[0].text.strip()
    return(quote)

def main():
    specify_no_show = input("Would you like to watch the web scraper run? Type 'y' if you do and 'n' if not. ")
    if specify_no_show.lower() in ['n', 'no']:
        no_show = True
    else:
        no_show = False # if users says yes or nothing, the scraper will display windows as it scrapes

    user_selection = get_recent_read(no_show) # tuple of book title and desc, if given
    if not user_selection:
        # if get_recent_read() returns an empty string, go ahead with generating a random quote
        random_word = pick_random_word()
        quote = pick_quote(random_word, no_show)
        print(f"\nYour random word is {random_word}.")
        print(f"\nAnd the random quote generated by {random_word} is:\n")
        print(quote)

    else:
        desc_word = get_word_from_desc(user_selection[1])
        quote = pick_quote(desc_word, no_show)
        print(f"\nThe word selected from your latest read {user_selection[0]} is {desc_word}.")
        print(f"\nAnd the random quote generated by {desc_word} is:\n")
        print(quote)

if __name__ == "__main__":
    main()


import numpy as np
import matplotlib.pyplot as plt

WL0 = 580e-9
nUp = 2
nDn = 2

# Initialize the coordinate for each layer
# The dipole should be better in the coordinate z=0
dl = np.zeros((nUp + nDn, 1))
dis = 10e-9
dl[3] = 600e-9
dl[2] = 400e-9
dl[1] = 200e-9
dl[0] = 0

# The position of the dipole
POSD = dl[nDn - 1] + dis

# Dipole direction
# phi = np.pi / 2 + np.pi / 6
# dx = np.sin(16 / 180 * np.pi) * np.cos(phi)
# dy = np.sin(16 / 180 * np.pi) * np.sin(phi)
# dz = np.cos(16 / 180 * np.pi)
# p0 = np.array([dx, dy, dz])
p0 = np.array([0, 0, 1])

# Initialize the permitivity
Eplist = np.zeros((nUp + nDn + 1, 1), dtype=complex)

Eplist[4] = 1.78**2
Eplist[3] = 1.5**2
Eplist[2] = 1.78**2
Eplist[1] = 1.5**2
Eplist[0] = 1

def Show_Structure(Eplist, dl, WL0):

    x = np.ones(10)

    # Obtain the thickness
    dis_ext = (dl[-1] - dl[0]) / (nUp + nDn + 1)
    dlext = np.ravel(np.vstack((dl[0] - dis_ext, dl[:], dl[-1] + dis_ext)))

    fig1 = plt.figure()
    for l in range(nUp + nDn):
        plt.plot(x * dl[l] * 1e9)
    plt.ylim((dlext[0] * 1e9, dlext[-1] * 1e9))

    for l in range(nUp + nDn + 1):
        strname = 'permitivity=' + str(Eplist[l, 0])
        plt.text(4, (dlext[l] + dlext[l + 1]) * 1e9 / 2, strname)
    plt.ylabel('Z (nm)')
    plt.xlabel('X Direction')
    plt.scatter(4, POSD * 1e9, s=5, marker="o")
    titlestr = 'Structure with dipole wavelength ' + str(WL0 * 1e9) + 'nm'
    plt.title(titlestr)
    filename = './Figures/Structure' + '.png'
    plt.savefig(filename)
    plt.show()


#!/usr/bin/python3

import argparse
import logging
import subprocess
import os
from collections import defaultdict
from sys import exit, stdout
from signal import signal, SIGINT, SIGTERM
import threading
import time
import importlib
import importlib.util
import inspect

logger = logging.getLogger()

class FuncNameFilter(logging.Filter):
    def __init__(self, func_name):
        self.func_names = set()
        self.add_func_name(func_name)
        super().__init__()

    def add_func_name(self, func_name):
        self.func_names.add(func_name)

    def filter(self, record):
        # True will propagate the record, False will filter out the record
        return record.funcName in self.func_names

def main():
    _init_logger()
    signal(SIGINT, __sigterm_handler)
    signal(SIGTERM, __sigterm_handler)

    parser = argparse.ArgumentParser()
    parser.add_argument('--verbose',
                        dest='verbose',
                        metavar='namespace',
                        nargs='*',
                        help='Sets the logging level to INFO. '
                             'It\'s a shortcut for "--set-logging-level INFO". See also there for details.')
    parser.add_argument('--debug',
                        dest='debug',
                        metavar='namespace',
                        nargs='*',
                        help='Sets the logging level to DEBUG. '
                             'It\'s a shortcut for "--set-logging-level DEBUG". See also there for details.')
    parser.add_argument('--set_logging_level',
                        dest='set_logging_level',
                        metavar='<LEVEL> [namespace,[namespace,...]]',
                        nargs='+',
                        help='Sets the logging level to <LEVEL> which can be chosen from [CRITICAL, ERROR, WARNING, '
                             'INFO, DEBUG, NOTSET]. If namespaces are given behind a <LEVEL> only these specific '
                             'modules or packages are affective with this <LEVEL>. '
                             'A namespace represents a dotted module hierarchy like "<a>.<b>.<c>" whereat the last '
                             'part <c> could be either the bottom-module of the hierarchy or the function <c> within '
                             'the module <b>. '
                             'Multiple sets are possible as "<LEVEL> [namespace,...] <LEVEL> [namespace,...]". '
                             'Namespace notations like "<a>.[<b>,<c>.[<d>,<e>]]" will be resolved to "<a.b> <a.c.d> '
                             '<a.c.e>".')
    args = parser.parse_args()

    if args.verbose is not None:
        _apply_logging_level(logging.INFO, args.verbose)
    if args.debug is not None:
        _apply_logging_level(logging.DEBUG, args.debug)
    if args.set_logging_level is not None:
        logging_level_name = args.set_logging_level[0]
        if not logging_level_name.isupper():
            raise ValueError(f'<LEVEL>="{logging_level_name}" needs to be a upper case string!')
        logging_level = logging.getLevelName(logging_level_name)  # also recovers the LevelNumber for a given LevelName

        namespaces_by_level = defaultdict(list)
        namespaces_by_level[logging_level] = []  # in case no optional namespaces are given
        for arg in args.set_logging_level[1:]:
            if arg.isupper():
                logging_level = logging.getLevelName(arg)  # also recovers the LevelNumber for a given LevelName
            else:
                namespaces_by_level[logging_level].append(arg)
        for lvl, namespaces in namespaces_by_level.items():
            _apply_logging_level(lvl, namespaces)

def _init_logger():
    sh = logging.StreamHandler(stdout)

    class FrameNumberFormatter(logging.Formatter):
        def format(self, record):
            global event_distributor
            if event_distributor is not None:
                return f'{event_distributor.frame_number}. {super().format(record)}'
            else:
                return super().format(record)
    sh.setFormatter(FrameNumberFormatter('%(levelname)s, %(module)s@%(lineno)d - %(funcName)s(), %(asctime)s:\n'
                                         '  %(message)s'))
    logger.addHandler(sh)
    logger.setLevel(logging.WARNING)

def _apply_logging_level(logging_level, namespaces=None):
    logging_level_name = logging.getLevelName(logging_level)
    if not namespaces:
        print(f'{logging_level_name} logging level is set.')
        logger.setLevel(logging_level)
    else:
        print(f'{logging_level_name} logging level is set for:')
        indent = '  '

        if not type(namespaces) in (list, tuple):
            namespaces = [namespaces]
        root_module_name = 'capore_tuio_server'
        for namespace in namespaces:
            resolved_namespaces = _resolve_bracketed_logging_namespace(namespace)
            if resolved_namespaces != namespace:
                namespaces.extend(resolved_namespaces)
                continue

            namespace = root_module_name + '.' + namespace

            module_obj = None
            module_name = None
            function_name = None
            try:
                module_obj = importlib.import_module(namespace)
            except ModuleNotFoundError:  # check if ending is a function within namespace
                module_name, function_name = namespace.rsplit('.', maxsplit=1)
                try:
                    module_obj = importlib.import_module(module_name)
                except ModuleNotFoundError:
                    function_name = None
                namespace = module_name

            if not module_obj:
                print(f'{indent}! MODULE NOT FOUND "{namespace}"')
                continue

            # check if function_name exists
            if function_name:
                all_classes_within_module = []
                for _, value in inspect.getmembers(module_obj):
                    if inspect.isclass(value) and value.__module__ == module_name:
                        all_classes_within_module.append(value)
                if not all_classes_within_module:
                    raise ValueError(f'"{module_name}" has no classes!')

                for cls in all_classes_within_module:
                    try:
                        function_obj = getattr(cls, function_name)
                        is_existing = False
                        if inspect.isfunction(function_obj):
                            is_existing = True
                        elif isinstance(function_obj, property):
                            is_existing = True
                            f_get = function_obj.fget
                            f_set = function_obj.fset
                            if f_get is not None:
                                function_name = f_get.__name__
                            elif f_set is not None:
                                function_name = f_set.__name__
                            else:
                                is_existing = False
                        if not is_existing:
                            raise ValueError(f'"{function_name}" should be a function within "{namespace}!"')
                        break
                    except AttributeError:
                        continue
                else:
                    raise ValueError(f'Can\'t find "{function_name}" within "{namespace}"!')

            child_logger = logger.getChild(namespace)
            prev_level = child_logger.getEffectiveLevel()
            child_logger.setLevel(logging_level)

            if prev_level == logging.WARNING:
                print(f'{indent}{namespace}')
            elif prev_level != logging_level:
                print(f'{indent}{namespace} '
                      f'(overrides previous level {logging.getLevelName(prev_level)})')

            if function_name:
                existing_filters = child_logger.filters
                if not existing_filters:
                    child_logger.addFilter(FuncNameFilter(function_name))
                    print(f'{2*indent}|  (filtering following function(s):)')
                else:
                    existing_filters[0].add_func_name(function_name)
                print(f'{2*indent}|- {function_name}')

def _resolve_bracketed_logging_namespace(namespace):
    # namespaces like "<a>.[<b>,<c>.[<d>,<e>]]" will resolved to ["<a.b>", "<a.c.d>", "<a.c.e>"]
    if '[' not in namespace:
        return namespace

    prefix, suffix = namespace.split('[', maxsplit=1)
    if not prefix.endswith('.'):
        raise ValueError(f'"{namespace}" looks like a bracketed namespace declaration but '
                         f'the package separator "." between "{prefix}" and "[{suffix}" is missing!')
    if not suffix.endswith(']'):
        raise ValueError(f'"{namespace}" looks like a bracketed namespace declaration but '
                         f'the trailing "]" after "{prefix}[{suffix}" is missing!')
    suffix = suffix[:-1]
    result = []
    while suffix:
        idx = suffix.find(',')
        if idx == -1:
            idx = len(suffix)
        a = suffix[:idx]
        b = suffix[idx+1:]
        if '[' in a:
            idx = suffix.rfind(']')
            if idx == -1:
                idx = len(suffix)
            c = suffix[:idx+1]
            d = suffix[idx+1:]
            for n in _resolve_bracketed_logging_namespace(c):
                result.append(prefix + n)
            if d and d[0] not in [',', ']']:
                raise ValueError(f'"{namespace}" looks like a bracketed namespace declaration but '
                                 f'there is a syntax problem between "{namespace[0:namespace.find(d)]}" and "{d}"!')
            suffix = d[1:]
        else:
            result.append(prefix + a)
            suffix = b
    return result


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 24 01:45:57 2023

@author: noob
"""
from testdomain import testdomain
obj = testdomain("http://127.0.0.1/mytest/x.php")
X = obj.data
from joblib import load
clf = load('DTM.pkl')
Y = clf.predict([X])
print('# Decission Tree Classification : ')
if Y == -1 :
    print("RESULT : NOT PHISHING")
else:
    print("RESULT : PHISHING")
model = load('LOM.pkl')
Ya = model.predict([X])
print('# Logistic Regression : ')
if Ya == -1 :
    print("RESULT : NOT PHISHING")
else:
    print("RESULT : PHISHING")


"""
Для valid-выборки сами jpg-файлы не копируются.
Вместо этого используются png - к ним добавляется фон и сохраняются в виде jpg.
Чтобы проверить как изменится точность распознавания.
"""
import os
import sys
import random
import numpy as np
from PIL import Image
import time

if os.path.exists('.local'):
	src_dir = '/mnt/lin2/ineru/detector_dataset/test_input'
	dst_dir = '/mnt/lin2/ineru/detector_dataset/test_output'
	bg_dir  = '/mnt/lin2/ineru/detector_dataset/background/'
else:
	src_dir = '/home/andrei/Data/Datasets/ScalesDetector/dataset-splited/'
	dst_dir = '/home/andrei/Data/Datasets/ScalesDetector/output/'
	bg_dir  = '/home/andrei/Data/Datasets/ScalesDetector/background/'

parts = ['train', 'valid']

PARTS_COPY_JPG = ['train','valid']  # select dataset which will be just copied
#PARTS_COPY_JPG = []  #
PARTS_ADD_BG = ['train','valid'] # select dataset part for adding background
MAX_NUM_BG_ON_PNG = 10

BG_SCALE_ALPHA = 230

def merge_with_background(im, bg, alpha_value=220):
	"""
	alpha_value = 220  - чем значение меньше,
	тем сильнее "отпечатывается" фон на весах
	"""

	#bg = bg.rotate(180).resize(im.size)
	bg = bg.transpose(Image.FLIP_TOP_BOTTOM).resize(im.size)
	bg.putalpha(200)

	a = np.array(im)
	b = np.array(bg)

	im_alpha = a[:,:,3]
	bg_alpha = b[:,:,3]

	bg_alpha = np.where(im_alpha==255, bg_alpha, 0)
	im_alpha = np.where(im_alpha==255, alpha_value, 0)

	a[:,:,3] = im_alpha
	b[:,:,3] = bg_alpha

	a0 = Image.fromarray(a)
	b0 = Image.fromarray(b)

	# Image.paste(im, box=None, mask=None)
	b0.paste(a0, (0, 0), a0)

	b = np.array(b0)
	bg_alpha = np.where(im_alpha==alpha_value, 255, 0)
	b[:,:,3] = bg_alpha
	b_img = Image.fromarray(b)

	return b_img

def add_background_to_image(foreground, background):

	if foreground.size != background.size:
		background = background.resize(foreground.size)
	background.paste(foreground, (0, 0), foreground)
	return background

def copy_files(src_dir, dst_dir, bg_dir, parts):

	count = {'valid': 0, 'train': 0}

	src_dir = src_dir.rstrip('/')
	dst_dir = dst_dir.rstrip('/')
	os.system('mkdir -p {}'.format(dst_dir))

	for part in parts:

		bg_subdir  = bg_dir + '/' + part
		src_subdir = src_dir + '/' + part
		dst_subdir = dst_dir + '/' + part

		os.system('mkdir -p {}'.format(dst_subdir))

		filenames = os.listdir(src_subdir)
		if len(filenames) == 0:
			print('{0} - empty subdir'.format(src_subdir))
			continue

		all_bg_filenames = os.listdir(bg_subdir)
		print(part, 'bg_filenames(all):', all_bg_filenames)
		if len(all_bg_filenames) == 0:
			print('{0} - empty subdir'.format(bg_subdir))
			raise Exception()

		#file_names = list(filter(lambda s: s[-4:]=='.jpg', file_names))

		for filename in filenames:

			basename = os.path.splitext(filename)[0]
			ext = os.path.splitext(filename)[1]

			if ext == '.jpg':

				if part in PARTS_COPY_JPG:

					cmd = 'cp {} {}'.format(src_subdir + '/' + basename + '.txt',
											dst_subdir + '/' + basename + '.txt')
					print(cmd)
					os.system(cmd)

					# It just copies jpg files. It works fast but size of files is also stay large
					cmd = 'cp {} {}'.format(src_subdir + '/' + filename,
											dst_subdir + '/' + filename)
					print(cmd)
					os.system(cmd)
					count[part] += 1

					# that makes size less
					#img = Image.open(src_subdir + '/' + filename)
					#img.save(dst_subdir + '/' + filename, quality=90, optimize=True,
					#		progressive=True)
					# defalut quality=80 ?

			if ext == '.png':

				if part in PARTS_ADD_BG:

					"""
					# if require copy also JPG file
					cmd = 'cp {} {}'.format(src_subdir + '/' + basename + '.txt',
											dst_subdir + '/' + basename + '.txt')
					print(cmd)
					os.system(cmd)

					# It just copies jpg files. It works fast but size of files is too large
					cmd = 'cp {} {}'.format(src_subdir + '/' + basename + '.jpg',
											dst_subdir + '/' + basename + '.jpg')
					print(cmd)
					os.system(cmd)
					"""

					# add background to png
					#src_path_base = src_subdir + '/' + filename

					if len(all_bg_filenames) > MAX_NUM_BG_ON_PNG:
						bg_filenames = random.sample(all_bg_filenames, MAX_NUM_BG_ON_PNG)
					else:
						bg_filenames = all_bg_filenames

					img_fg_path = src_subdir + '/' + filename
					img_fg = Image.open(img_fg_path)
					json_file_path = src_subdir + '/' + basename + '.jpg.json'
					print('json_file_path:', json_file_path)

					for i, bg_filename in enumerate(bg_filenames):

						bg_path = bg_subdir + '/' + bg_filename
						img_bg = Image.open(bg_path)

						try:
							if not os.path.isfile(json_file_path):
								# if the scale is empty
								img_fg_new = merge_with_background(img_fg, img_bg, alpha_value=BG_SCALE_ALPHA)
								print('merge_with_background')
							else:
								img_fg_new = img_fg
							img = add_background_to_image(img_fg_new, img_bg)

						except Exception as ex:
							print('filename:', filename)
							print('bg_path:', bg_path)
							print('*** error in add_background_to_image ***')
							os.system('echo "add_background_to_image: {} {}" | cat >> _ERRORS.log'.\
								format(filename, bg_path))
							print(ex)
							time.sleep(10)
							break

						dst_path_base = dst_subdir + '/' + basename

						dst_path_jpg = dst_path_base + '_bg' + str(i) + '.jpg'
						print(i, ':', dst_path_jpg)
						img.save(dst_path_jpg, quality=85, optimize=True, progressive=True)
						count[part] += 1

						dst_path_txt = dst_path_base + '_bg' + str(i) + '.txt'
						cmd = 'cp {} {}'.format(src_subdir + '/' + basename + '.txt',
												dst_path_txt)
						os.system(cmd)

	print(count)

if __name__ == '__main__':

	copy_files(src_dir, dst_dir, bg_dir, parts)
	#split_copy_files(src_dir, dst_dir, parts, ratio=[1,1,0])


import open3d as o3d
import numpy as np
from matplotlib import pyplot as plt

source = o3d.io.read_point_cloud("E:/University/Thesis/color_pd30_8bit_small_filtered.ply")
source_np = np.asarray(source.points)
for i in range(3):
    print(source_np[:, i].min(),
          source_np[:, i].max(),
          source_np[:, i].min() - source_np[:, i].max()
          )

plt.plot(source_np[:, 0], source_np[:, 1])
plt.show()
o3d.visualization.draw_geometries([target])

source_new = source.select_by_index(np.where(source_np[:, 2] < 70)[0])
source_new.translate([0, 0, 0], relative=False)
source_new.scale(1, center=source_new.get_center())

o3d.io.write_point_cloud("E:/University/Thesis/color_pd30_8bit_small_filtered.ply", source_new)

target = o3d.io.read_point_cloud("D:/sfm_dense/street_2_0/dense/0/fused_filtered_down_aligned.ply")
target_np = np.asarray(target.points)
for i in range(3):
    print(target_np[:, i].min(), target_np[:, i].max(), target_np[:, i].min() - target_np[:, i].max())

target.translate([0, 0, 0], relative=False)
target.scale(20, center=target.get_center())
o3d.io.write_point_cloud("D:/sfm_dense/street_2_0/dense/0/fused_filtered.ply", target)

target.sample_points_uniformly(number_of_points=1000)
target.uniform_down_sample(100)
target_down = target.voxel_down_sample(voxel_size=5)

o3d.visualization.draw_geometries([target])

def get_plane(ply):
    plane_model, inliers = ply.segment_plane(
        distance_threshold=5,
        ransac_n=1000,
        num_iterations=1000
    )
    [a, b, c, d] = plane_model
    print(f"Plane equation: {a:.2f}x + {b:.2f}y + {c:.2f}z + {d:.2f} = 0")

    inlier_cloud = ply.select_by_index(inliers)
    inlier_cloud.paint_uniform_color([1.0, 1.0, 0])

    outlier_cloud = ply.select_by_index(inliers, invert=True)
    outlier_cloud.paint_uniform_color([1.0, 0, 0])

    o3d.visualization.draw_geometries([inlier_cloud])
    o3d.visualization.draw_geometries([outlier_cloud])

get_plane(source_scaled_uniformed)

def align_ply_to_z(ply):
    plane_model, inliers = ply.segment_plane(
        distance_threshold=5,
        ransac_n=1000,
        num_iterations=1000
    )
    [a, b, c, d] = plane_model
    plane_n = np.array([a, b, c])
    plane_z = np.array([0, 0, -1])

    vec1 = np.reshape(plane_n, (1, -1))
    vec2 = np.reshape(plane_z, (1, -1))

    from scipy.spatial.transform import Rotation as R
    r = R.align_vectors(vec2, vec1)
    rot = r[0].as_matrix()

    ply_aligned = copy.deepcopy(ply)
    ply_aligned.rotate(rot, center=ply_aligned.get_center())
    o3d.visualization.draw_geometries([ply, ply_aligned])
    return ply_aligned

source_scaled_uniformed_aligned = align_ply_to_z(source_scaled_uniformed)

# street_4_0
import copy
import open3d as o3d
import numpy as np
from matplotlib import pyplot as plt

source = "D:/sfm_dense/street_4_0_1/dense/0/fused.ply"
target = "D:/sfm_dense/color_pd30_8bit_street_4_0.ply"

source = o3d.io.read_point_cloud(source)
target = o3d.io.read_point_cloud(target)

source.translate([0, 0, 0], relative=False)
target.translate([0, 0, 0], relative=False)

def ply_show_range_coords(ply):
    ply_np = np.asarray(ply.points)
    print()
    for i in range(3):
        print(
            ply_np[:, i].min(),
            ply_np[:, i].max(),
            "range:",
            ply_np[:, i].max() - ply_np[:, i].min()
        )

ply_show_range_coords(source)
ply_show_range_coords(target)

# target
target_sliced = target.select_by_index(np.where(np.asarray(target.points)[:, 2] < 0)[0])
o3d.visualization.draw_geometries([target_sliced])

target_sliced_uniformed = target_sliced.voxel_down_sample(voxel_size=1)
o3d.visualization.draw_geometries([target_sliced_uniformed])

# source
o3d.visualization.draw_geometries([source])

source_scaled = source.scale(20, center=source.get_center())

source_scaled_uniformed = source_scaled.voxel_down_sample(voxel_size=1)
o3d.visualization.draw_geometries([source_scaled_uniformed])

# scaling
def draw_registration_result(source, target):
    source_temp = copy.deepcopy(source)
    target_temp = copy.deepcopy(target)
    source_temp.paint_uniform_color([1, 0.706, 0])
    target_temp.paint_uniform_color([0, 0.651, 0.929])
    o3d.visualization.draw_geometries([source_temp, target_temp])

ply_show_range_coords(source_scaled_uniformed)
ply_show_range_coords(target_sliced_uniformed)
draw_registration_result(source_scaled_uniformed_aligned, target_sliced_uniformed)

# save

o3d.io.write_point_cloud("D:/sfm_dense/street_4_0_1/dense/0/fused_scaled_uniformed.ply", source_scaled_uniformed)
o3d.io.write_point_cloud("D:/sfm_dense/color_pd30_8bit_street_4_0_sliced_uniformed.ply", target_sliced_uniformed)

# Test
# This function is only used to make the keypoints look better on the rendering
def keypoints_to_spheres(keypoints):
    spheres = o3d.geometry.TriangleMesh()
    for keypoint in keypoints.points:
        sphere = o3d.geometry.TriangleMesh.create_sphere(radius=0.001)
        sphere.translate(keypoint)
        spheres += sphere
    spheres.paint_uniform_color([1.0, 0.75, 0.0])
    return spheres

source = "D:/sfm_dense/street_4_0_1/dense/0/fused.ply"
target = "D:/sfm_dense/color_pd30_8bit_street_4_0.ply"

source = o3d.io.read_point_cloud(source)
keypoints = o3d.geometry.keypoint.compute_iss_keypoints(source,
                                                        salient_radius=0.5,
                                                        non_max_radius=1.0,
                                                        gamma_21=0.5,
                                                        gamma_32=0.5)

source.paint_uniform_color([0.5, 0.5, 0.5])
o3d.visualization.draw_geometries([keypoints_to_spheres(keypoints), source])


#!/usr/bin/env python
"""Functions to optimally fit one 'shape' into another."""
from scipy import array, optimize, eye, c_, r_, ones, sqrt
from scipy import linalg, sum, size
import numpy
import random
from conclasses import ConSet
from convertfuns import con2vert
from auxfuns import qhull, splitAb

def tryvol(A, b, cs):
    """
    Try to determine volume of feasible region, otherwise impose box
    constraint.
    """
    try:
        V = con2vert(A, b)[0] # get vertices for constraint set iteration
    except NameError: #catch unbound shapes
        #create large box around original shape
        fixA = r_[eye(cs.nd), -eye(cs.nd)]
        fixb = r_[[numpy.max(cs.vert[:, x]) for x in range(cs.vert.shape[1])],
                  [-numpy.min(cs.vert[:, x]) for x in range(cs.vert.shape[1])]]
        fixb = array([fixb]).T
        fA = r_[fixA, A]
        fb = 2. * r_[fixb, b]  # double the box size
        V = con2vert(fA, fb)[0]
    #return vol (normal or fixed) and vertices (normal or fixed)
    return qhull(V, "FS"), V
    #TODO: check box fitting

def genstart(shapetype, *args):
    """
    Generate a starting shape (for optimisation) with given number of faces.
        shapetype : [string] (r)ectangle, (a)rbitrary
        args -> (initial constraint set, number of faces)
    """
    if shapetype in 'rR':  # Rectangle
        cs = args[0]
        ncon = cs.nd*2
    elif shapetype in 'aA':  # Arbitrary shape
        cs = args[0]
        ncon = args[1]

    #1. Determine centre of initial region, cscent
    cscent = cs.cscent  # assuming the region is convex
    def spherefn(srad, cent, svecs):
        """
        Function to create points on a sphere (of radius, srad), convert them to
        halfspaces and convert the halfspaces to a constraint set.
        """
        spherepts = svecs.T*(srad/sqrt(sum((svecs.T)**2)))
        spherepts = spherepts.T + cent  # move to center of constraint set
        #4. Generate tangent planes on sphere at points, convert to inequalities
        A = -(spherepts - cent)
        b = array([(sum((A*spherepts).T, axis=0))]).T
        s = array(ones(b.shape))
        return ConSet(A, s, b)  # constraints around sphere

    if shapetype in 'rR':  # Rectangle
        Astart = r_[eye(cs.nd), -eye(cs.nd)]
        sstart = -ones((cs.nd*2, 1))
        bstart = lambda k: array(r_[cscent.T + k, -(cscent.T - k)]).reshape(size(cscent)*2,1)
        kstart = 1.0
        if ConSet(Astart, sstart, bstart(kstart)).allinside(cs)[0]:
            while ConSet(Astart, sstart, bstart(kstart)).allinside(cs)[0]:
                kstart = kstart * 1.01
            kfin = kstart / 1.01
        elif not ConSet(Astart, sstart, bstart(kstart)).allinside(cs)[0]:
            while not ConSet(Astart, sstart, bstart(kstart)).allinside(cs)[0]:
                kstart = kstart / 1.01
            kfin = kstart
        return ConSet(Astart, sstart, bstart(kfin))
    elif shapetype in 'aA':  # Arbitrary shape
        #2. Generate ncon-1 Gaussian vectors
        spherevecs = ones((ncon, cs.vert.shape[1]))
        for rows in range(spherevecs.shape[0] - 1):
            for cols in range(spherevecs.shape[1]):
                spherevecs[rows, cols] = random.gauss(0, 0.33)  # TODO: check sigma
        #3. Determine resultant of vectors, add last vector as mirror of resultant
        spherevecs[-1, :] = -sum(spherevecs[:-1, :], axis=0)
        # points on sphere
        #6. Optimise sphere-radius, r, to have all points within initial shape
        startrad = cs.vol()**(1./cs.nd)
        if spherefn(startrad, cscent, spherevecs).allinside(cs)[0]:
            while spherefn(startrad, cscent, spherevecs).allinside(cs)[0]:
                startrad = startrad * 1.01
            finrad = startrad / 1.01
        elif not spherefn(startrad, cscent, spherevecs).allinside(cs)[0]:
            while not spherefn(startrad, cscent, spherevecs).allinside(cs)[0]:
                startrad = startrad / 1.01
            finrad = startrad
        return spherefn(finrad, cscent, spherevecs)

def fitshape(cset, spset, solver):
    """
    Fit a constraint set (specified by the number of constraints) within an existing
    constraint set.
     cset - [ConSet] existing constraint set
     ncon - [int] number of constraints to fit
    """
    #### State problem
    # ncongiven > ncon >= nD+1
    # Constraints are bounding
    # All vertices within constraint set
    #### Define parameters
    snorm = linalg.norm(spset.b)
    sp = c_[spset.A, spset.b]/snorm # starting point - combined Ab matrix to optimise

    #### Objective fn (FMIN)
    def objfn(Ab, *args):
        """Volume objective function for fmin (Simplex)."""
        initcs = args[0]
        A, b = splitAb(Ab, initcs.nd)
        vol, V = tryvol(A, b, initcs)
        Pv = 200.
        Pn = 100.
        #Penalties
        # large b norm
        bnorm = abs(linalg.norm(b) - 1)
        # points outside of init space
        iterset = ConSet(V)
        outnorm = linalg.norm(iterset.allinside(initcs)[1])
        #outnorm = iterset.allinside(initcs)[2]
        # open shape
        cl = con2vert(A, b)[1]
        if cl:
            closed = 1
        else:
            closed = -1
        vol = tryvol(A, b, initcs)[0]
        return (-vol*closed) + Pn*(bnorm**initcs.nd) + Pv*(outnorm**(initcs.nd+3))

    #### Objective fn (SLSQP)
    def objfn2(Ab, *args):
        """Volume objective function for SLSQP."""
        initcs = args[0]
        A, b = splitAb(Ab, initcs.nd)
        cl = con2vert(A, b)[1]
        if cl:
            closed = 1
        else:
            closed = -1
        vol = tryvol(A, b, initcs)[0]
        return closed * -vol
    def eqconsfn(Ab, *args):
        """Optimiser equality constraint function."""
        initcs = args[0]
        b = splitAb(Ab, initcs.nd)[1]
        # get vertices
        bn = linalg.norm(b) - 1
        return array([bn])
    def ieqconsfn(Ab, *args):
        """Optimiser inequality constraint function."""
        initcs = args[0]
        A, b = splitAb(Ab, initcs.nd)
        # get vertices
        V = tryvol(A, b, initcs)[1]
        iterset = ConSet(V)
        # constraint checking
        if iterset.allinside(initcs):
            return 0
        else:
            return -1

    #### Maximise volume
    if solver in 'aA':
        optAb = optimize.fmin_slsqp(objfn2, sp, f_eqcons=eqconsfn,
                                    f_ieqcons=ieqconsfn, args=[cset], iprint=0)
    elif solver in 'bB':
        optAb = optimize.fmin(objfn, sp, args=[cset], maxiter=20000, disp=False)
    elif solver in 'cC':
        optAb = optimize.fmin_cobyla(objfn2, sp, ieqconsfn, args=[cset], iprint=0)
        optAb = optAb.ravel()
    tA, tb = splitAb(optAb, cset.nd)
    ts = -ones(tb.shape)
    optsol = ConSet(tA, ts, tb)
    if solver in 'bB':
        optcent = sum(optsol.vert, axis=0)/len(optsol.vert)
        itersol = ConSet(optsol.vert)
        while not itersol.allinside(cset)[0]:
            vi = (itersol.vert - optcent)*0.9999 + optcent
            itersol = ConSet(vi)
        optsol = itersol
    return optsol

def fitcube(cset, spset, solver):
    """
    Fit a rectangle (high/low limits on outputs) within an existing
    constraint set.
     cset - [ConSet] existing constraint set
    """
    #### State problem
    # nvar = cset.nd
    # Constraints are bounding
    # All vertices within constraint set
    #### Define parameters
    sp = spset.b.ravel() # starting point - b matrix to optimise
    #### Objective fn (SLSQP)
    def objfn(Ab, *args):
        """Volume objective function for SLSQP."""
        initcs = args[0]
        A = r_[eye(initcs.nd), -eye(initcs.nd)]
        b = array([Ab]).T
        cl = con2vert(A, b)[1]
        if cl:
            closed = 1
        else:
            closed = -1
        vol = tryvol(A, b, initcs)[0]
        return closed * -vol
    #### Constraints
    def ieqconsfn(Ab, *args):
        """Optimiser inequality constraint function."""
        initcs = args[0]
        A = r_[eye(initcs.nd), -eye(initcs.nd)]
        b = array([Ab]).T
        # get vertices for constraint set iteration
        V = tryvol(A, b, initcs)[1]
        iterset = ConSet(V)
        #constraint checking for vertices
        ineqs = iterset.allinside(initcs)[1]
        return array([-linalg.norm(ineqs)])

    #### Objective fn (FMIN)
    def objfn2(Ab, *args):
        """Volume objective function for fmin (Simplex)."""
        initcs = args[0]
        A = r_[eye(initcs.nd), -eye(initcs.nd)]
        b = array([Ab]).T
        vol, V = tryvol(A, b, initcs)
        Pv = 200.
        #Penalties
        # points outside of init space
        iterset = ConSet(V)
        outnorm = linalg.norm(iterset.allinside(initcs)[1])
        # open shape
        cl = con2vert(A, b)[1]
        if cl:
            closed = 1
        else:
            closed = -1
        vol = tryvol(A, b, initcs)[0]
        return (-vol*closed) + Pv*(outnorm**3)

    #### Maximise volume
    if solver in 'aA':
        optAb = optimize.fmin_slsqp(objfn, sp, f_ieqcons=ieqconsfn, args=[cset],
                                iprint=0)
    elif solver in 'bB':
        optAb = optimize.fmin(objfn2, sp, args=[cset], maxiter=50000, disp=False)
    if solver in 'cC':
        optAb = optimize.fmin_cobyla(objfn, sp, ieqconsfn, args=[cset],
                                iprint=0)
    tA = r_[eye(cset.nd), -eye(cset.nd)]
    tb = array([optAb]).T
    ts = -ones(tb.shape)
    optsol = ConSet(tA, ts, tb)
    if solver in 'bBcC':
        optcent = sum(optsol.vert, axis=0)/len(optsol.vert)
        itersol = ConSet(optsol.vert)
        while not itersol.allinside(cset)[0]:
            vi = (itersol.vert - optcent)*0.9999 + optcent
            itersol = ConSet(vi)
        optsol = itersol
    return optsol

def fitset(cset, *args):
    """
    Return a fitted constraint set within cset.
      cset - [ConSet] to fit within
      *args (in this order) - (stype)[string] 'a'rbitrary / 'r'ectangular
                            - (solver)[string] 'a' SLSQP / 'b' fmin / 'c' Cobyla
                            - (ncon)[integer] number of constraints to fit
                                cset.nd + 1 < ncon < cset.A.shape[0]
    """
    #Get args
    stype = args[0]
    solver = args[1]
    if stype in 'aA':
        ncon = args[2]
        nostarts = 1
    else:
        ncon = 0
        nostarts = 1
    #Starting point
    refvol = 0
    for k in range(nostarts):
        spshape = genstart(stype, cset, ncon)
        if spshape.vol() > refvol:
            finalsp = spshape
            refvol = finalsp.vol()
    #Fit set
    if stype in 'aA':
        optsol = fitshape(cset, finalsp, solver)
    elif stype in 'rR':
        optsol = fitcube(cset, finalsp, solver)
    #Return
    return optsol

def fitmaxbox(cset, sf):
    """
    Return high/low constraint set around the given constraint set.
      cset - [ConSet] to fit over
      sf - [integer] safety factor (fraction 0-1)
    """
    #Get args
    dev = sf*((cset.vert.max(0))-cset.cscent)
    maxvals = cset.vert.max(0) + dev#-cset.cscent) + cset.cscent
    minvals = cset.vert.min(0) - dev#-cset.cscent) + cset.cscent
    Abox = r_[eye(cset.nd), -eye(cset.nd)]
    bbox = c_[maxvals, -minvals].T
    sbox = -ones((Abox.shape[0], 1))
    return ConSet(Abox, sbox, bbox)


from azure.cognitiveservices.vision.customvision.training.models import ImageFileCreateEntry, Region
from pycocotools.coco import COCO
import mPyPl as mp
from os.path import join

def bbox_to_dict(bbox, tag):
    x, y, w, h = bbox
    return {
        'x1' : x,
        'y1' : y,
        'width' : w,
        'height' : h,
        'tag' : tag
    }

def most_common(lst):
    return max(lst, key=lst.count)

def get_coco_stream(tags, ann_file, data_dir, threshold=0.1):
    coco = COCO(ann_file)
    catIds = coco.getCatIds(catNms=tags)
    imgIds = sum([coco.getImgIds(catIds=catId) for catId in catIds], [])
    stream = (
        coco.loadImgs(imgIds)
        | mp.as_field('meta')
        | mp.apply('meta', 'width', lambda x: x['width'])
        | mp.apply('meta', 'height', lambda x: x['height'])
        | mp.apply('meta', 'url', lambda x: x['coco_url'])
        | mp.apply('meta', 'filename', lambda x: x['file_name'])
        | mp.apply('meta', 'anns_ids', lambda x: coco.getAnnIds(imgIds=x['id'], catIds=catIds, iscrowd=None))
        | mp.apply('anns_ids', 'anns', lambda x: coco.loadAnns(x))
        | mp.apply('anns', 'ground_truth', lambda x: x
            | mp.select(lambda m: bbox_to_dict(m['bbox'], coco.cats[m['category_id']]['name']))
            | mp.as_list
         )
        | mp.apply('ground_truth', 'class_id', lambda x: most_common(x
            | mp.select(lambda m: m['tag'])
            | mp.as_list
        ))
        | mp.iter('meta', lambda x: coco.download(data_dir, [x['id']]))
        | mp.delfield(['meta', 'anns_ids', 'anns'])
    )
    return stream

def coco_to_custom_vision(stream, project_id, trainer, data_dir):
    stream = stream | mp.as_list
    tags = stream | mp.select_field('class_id') | mp.dedup() | mp.as_list
    cv_tags = { tag : trainer.create_tag(project_id, tag) for tag in tags }

    stream = (
        stream
        | mp.apply(['width', 'height', 'ground_truth'], 'ground_truth', lambda x:
            x[2]
            | mp.where(lambda box: (box['width'] >= x[0] * 0.1) and (box['height'] >= x[1] * 0.1))
            | mp.as_list
        )
        | mp.filter('ground_truth', lambda x: len(x) > 0)
        | mp.apply(['width', 'height', 'ground_truth'], 'regions', lambda x:
            x[2]
            | mp.select(lambda box:
                Region(
                    tag_id=cv_tags[box['tag']].id,
                    left=box['x1'] / x[0],
                    top=box['y1'] / x[1],
                    width=box['width'] / x[0],
                    height=box['height'] / x[1])
                )
            | mp.as_list
        )
        | mp.apply(['filename', 'regions'], 'tagged_img', lambda x:
            ImageFileCreateEntry(
                name=x[0],
                contents=open(join(data_dir, x[0]), mode="rb").read(),
                regions=x[1])
            )
        | mp.as_list
    )
    tagged_images_with_regions = stream | mp.select_field('tagged_img') | mp.as_list
    for i in range(0, len(tagged_images_with_regions), 50):
        trainer.create_images_from_files(project_id, images=tagged_images_with_regions[i:i + 50])


import multiprocessing
import os
import warnings

import feets
import hydra
import pandas as pd
from base import BaseG2NetFeatureEngineeringDataset, G2NetFeatureEngineering
from omegaconf import DictConfig

class FeatsFeatures(BaseG2NetFeatureEngineeringDataset):
    def _engineer_features(self, signals):
        fs = feets.FeatureSpace(data=["magnitude"])

        features = {}
        for i, x in enumerate(signals):
            feature_names, values = fs.extract(magnitude=x)
            for name, value in zip(feature_names, values):
                features[f"{name}_channel_{i}"] = value

        return features

@hydra.main(config_path="../../../config", config_name="default")
def main(config: DictConfig) -> None:
    filename = __file__.split("/")[-1][:-3]
    input_dir = config.input_dir
    features_dir = config.features_dir
    os.makedirs(features_dir, exist_ok=True)

    train = pd.read_csv(config.competition.train_path)
    test = pd.read_csv(config.competition.test_path)

    train["path"] = train["id"].apply(
        lambda x: f"{input_dir}/train/{x[0]}/{x[1]}/{x[2]}/{x}.npy"
    )
    test["path"] = test["id"].apply(
        lambda x: f"{input_dir}/test/{x[0]}/{x[1]}/{x[2]}/{x}.npy"
    )

    num_workers = multiprocessing.cpu_count()
    transformer = G2NetFeatureEngineering(
        FeatsFeatures, batch_size=num_workers, num_workers=num_workers
    )

    X_train = transformer.fit_transform(train["path"])
    X_test = transformer.transform(test["path"])

    print(X_train.info())

    X_train.to_pickle(os.path.join(features_dir, f"{filename}_train.pkl"))
    X_test.to_pickle(os.path.join(features_dir, f"{filename}_test.pkl"))

if __name__ == "__main__":
    warnings.filterwarnings("ignore")
    main()


import pygame

from enum import Enum

class ControlType(Enum):
    PLAYER1 = 0
    PLAYER2 = 1

class Player:
    def __init__(self, rect, color, controls):
        self.rect = rect
        self.speed = 0.8
        self.set_control_keys(controls)
        self.color = color

    def update(self, keys, dt, w, h):
        if keys[self.K_UP]:
            self.rect.top -= self.speed * dt

        if keys[self.K_DOWN]:
            self.rect.top += self.speed * dt

        self.rect = self.rect.clamp(pygame.Rect(0, 0, w, h))

    def draw(self, window):
        pygame.draw.rect(window, self.color, self.rect)

    def set_control_keys(self, controls):
        if controls == ControlType.PLAYER1:
            self.K_UP = pygame.K_w
            self.K_DOWN = pygame.K_s
        elif controls == ControlType.PLAYER2:
            self.K_UP = pygame.K_UP
            self.K_DOWN = pygame.K_DOWN


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import sys
import logging
import argparse

LOG = logging.getLogger(__name__)

__version__ = "1.0.0"
__author__ = ("Xingguo Zhang",)
__email__ = "invicoun@foxmail.com"
__all__ = []

def read_tsv(file, sep=None):

    LOG.info("reading message from %r" % file)

    for line in open(file):
        line = line.strip()

        if not line or line.startswith("#"):
            continue

        yield line.split(sep)

def read_gfa(file):

    segment = {}
    link = {}

    for line in read_tsv(file, "\t"):
        if line[0] =='S':
            depth = "%.2f" % float(line[4].split(":")[-1])
            segment[line[1]] = (line[2], depth)
        if line[0] =='L':
            if line[1] not in link:
                link[line[1]] = []
            if line[3] not in link:
                link[line[3]] = []

            values = (line[2], line[3], line[4])
            if values not in link[line[1]]:
                link[line[1]].append(values)

            qs = "+"
            rs = "+"
            if line[4]=="+":
                qs = "-"
            if line[2]=="+":
                rs = "-"
            values = (qs, line[1], rs)
            if values not in link[line[3]]:
                link[line[3]].append(values)
    return segment, link

def get_id(file):

    r = []

    for line in read_tsv(file, "\t"):
        r.append(line[0])

    return r

def get_gfa(gfa, seqids):

    ids = get_id(seqids)
    segment, link = read_gfa(gfa)

    seqid = set()
    for i in link:
        if i not in ids:
            continue
        seqid.add(i)
        for qs, rid, rs in link[i]:
            seqid.add(rid)
            print("L\t%s\t%s\t%s\t%s\t0M" % (i, qs, rid, rs))

    for i in segment:
        if i not in seqid:
            continue
        seq, depth = segment[i]
        print("S\t%s\t%s\tLN:i:%s\tdp:f:%s" % (i, seq, len(seq), depth))

#    for i in link:
#        if i not in seqid:
#            continue
#        for qs, rid, rs in link[i]:
#            print("L\t%s\t%s\t%s\t%s\t0M" % (i, qs, rid, rs))

    return 0

def add_help(parser):

    parser.add_argument('gfa', metavar='FILE', type=str,
        help='Input the assembled gfa file.')
    parser.add_argument('-i', '--seqids', metavar='FILE', type=str, required=True,
        help='Input the id file of the extracted sequence.')

    return parser

def main():

    logging.basicConfig(
        stream=sys.stderr,
        level=logging.INFO,
        format="[%(levelname)s] %(message)s"
    )
    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
        description='''
name:
    get_gfa.py Extract gfa file
attention:
    get_gfa.py assembly.gfa -i id >assembly_new.gfa
version: %s
contact:  %s <%s>\
        ''' % (__version__, ' '.join(__author__), __email__))

    args = add_help(parser).parse_args()

    get_gfa(args.gfa, args.seqids)

if __name__ == "__main__":

    main()