import numpy as np
import pandas as pd
import igraph as ig
import multires_consensus_clustering as mcc
import time

def multiresolution_graph(clustering_data, settings_data, list_resolutions, neighbour_based, single_resolution):
    """
    Creates a multi-resolution graph based on the resolutions given in the list_resolutions.
    Can either create a graph where all resolution vertices are connected or only the neighbouring resolutions are connected.

    @param settings_data: Settings data about the clusters.
    @param clustering_data: The cluster data.
    @param neighbour_based: Boolean to decided on the way to connect the vertices across resolutions.
    @param list_resolutions: The list containing the different resolutions, e.g. [3,5,9,20, ... ] or "all"
    @param single_resolution: Resolution Parameter for the meta-graph (leiden community detection resolution parameter)
    @return: The mulit-graph as a iGraph graph.
    """

    # set maximum level for hierarchy plot
    len_list_resolutions = len(list_resolutions)
    level_count = len_list_resolutions + 1

    # check if list resolution contains less the two resolutions
    if len_list_resolutions <= 1:
        print("More then one resolution needed for multi resolution graph.")

        return mcc.meta_graph(clustering_data, settings_data, list_resolutions[0], single_resolution)

    # create the multi graph
    else:
        if list_resolutions == "all":
            bins_clusterings = mcc.bin_n_clusters(settings_data["n_clusters"])
            list_resolutions = [int(first_number_clusters[0]) for first_number_clusters in bins_clusterings]
        else:
            # sort resolution list in cases not sorted
            list_resolutions.sort()

        # create first graph and assign the level
        resolution_1 = mcc.meta_graph(clustering_data, settings_data, list_resolutions[0], single_resolution)
        resolution_1.vs["level"] = [level_count] * resolution_1.vcount()

        # create new attribute to save the cell probabilities in a meta node
        probability_df = mcc.graph_nodes_cells_to_df(resolution_1, clustering_data)
        resolution_1.vs["probability_df"] = [probability_df[column].values for column in probability_df.columns]

        resolution_1.vs[0]["cell_index"] = probability_df.index.tolist()

        # delete all edges of the old graph
        mcc.delete_edges_single_resolution(resolution_1)

        # change level count
        level_resolution_2 = level_count - 1

        # select all resolutions except the first
        list_resolutions = list_resolutions[1:]

        # create multi-graph using the rest of the list_resolutions
        for resolution in list_resolutions:

            # create graph and assign the level
            resolution_2 = mcc.meta_graph(clustering_data, settings_data, resolution, single_resolution)
            resolution_2.vs["level"] = [level_resolution_2] * resolution_2.vcount()

            # delete all edges of the old graph
            mcc.delete_edges_single_resolution(resolution_2)

            # create multi graph based on neighbours or connect all vertices
            if neighbour_based:
                resolution_1 = mcc.merge_two_resolution_graphs(resolution_1, resolution_2,
                                                               current_level=level_resolution_2 + 1,
                                                               neighbours=True, clustering_data=clustering_data)
            else:
                # connect all vertices
                resolution_1 = mcc.merge_two_resolution_graphs(resolution_1, resolution_2, current_level=None,
                                                               neighbours=False, clustering_data=clustering_data)

            # set level for next graph
            level_resolution_2 -= 1

        # return the final multi-graph
        return resolution_1

def delete_edges_single_resolution(graph):
    """
    Deletes all edges with an edge weight above 0 -> these are all edges of the graph.
    @param graph: The given graph, iGraph graph.
    @return: The graph without edges; only vertices with attributes
    """

    return graph.delete_edges()

def merge_two_resolution_graphs(graph_1, graph_2, current_level, neighbours, clustering_data):
    """
    Merges two graphs;
    Either connects all vertices or only the bins neighbouring each other. All edges are based on the jaccard-index.

    @param clustering_data: The clustering data from which the meta graphs are build.
    @param neighbours: Connects all vertices or only neighbouring resolutions; Boolean
    @param current_level: The level of the last added vertices.
        Need so the new vertices are only connected to the latest vertices and not all.
    @param graph_1: The first graph, iGraph graph.
    @param graph_2: The second graph, iGraph graph.
    @return: The merged graph.
    """

    # create edge lists
    edge_list, edge_weights = [], []

    # to check the graph merger visually
    graph_1.vs["graph"] = [1] * graph_1.vcount()
    graph_2.vs["graph"] = [2] * graph_2.vcount()

    # add cell probability to the second graph with is added in the new layer (level)
    probability_df = mcc.graph_nodes_cells_to_df(graph_2, clustering_data)
    graph_2.vs["probability_df"] = [probability_df[column].values for column in probability_df.columns]

    # creates a graph based on the two given resolutions
    graph = graph_1.disjoint_union(graph_2)

    # connects vertices based on neighbouring resolutions.
    if neighbours:
        for vertex_1 in graph.vs.select(graph=1):
            for vertex_2 in graph.vs.select(graph=2):
                # connects only bins next to each other
                if vertex_1["level"] == current_level and vertex_2["level"] == current_level - 1:
                    # calculate edge weight
                    edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                    # if the edge_weight is greater 0 the edge is added
                    if edge_weight != 0:
                        edge_list.append((vertex_1, vertex_2))
                        edge_weights.append(edge_weight)

    # connects all vertices
    else:
        for vertex_1 in graph.vs.select(graph=1):
            for vertex_2 in graph.vs.select(graph=2):
                # calculate edge weight
                # edge_weight = mcc.jaccard_index_two_vertices(vertex_1, vertex_2)
                edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                # if the edge_weight is greater 0 the edge is added
                if edge_weight != 0:
                    edge_list.append((vertex_1, vertex_2))
                    edge_weights.append(edge_weight)

    # add edges to the graph
    graph.add_edges(edge_list)

    # add edge weights to the graph
    graph.es["weight"] = edge_weights

    return graph

def reconnect_graph(graph):
    """
    Reconnects the graph. Useful if after merging the graph is just a set of separated nodes.

    @param graph: The graph, in the ideal case a set of discrete nodes.
    @return: The connected graph. Edges are chosen that in the end the graph has a tree structure,
        based on the level of the nodes.
    """

    # check if graph has more than one level
    if len(set(graph.vs["level"])) != 1:
        # if the graph has more then one level a tree structure can be created, using the resolution of the graphs
        for vertex_1 in graph.vs:
            level_vertex_1 = vertex_1["level"]
            for vertex_2 in graph.vs:
                if vertex_1 != vertex_2:
                    # connects only bins next to each other
                    if vertex_2["level"] < level_vertex_1:
                        # calculate edge weight
                        edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                        # if the edge_weight is greater 0 the edge is added
                        if edge_weight != 0:
                            index_1 = vertex_1.index
                            index_2 = vertex_2.index
                            graph.add_edge(index_1, index_2, weight=edge_weight)

                        # check if better edge is available for tree structure
                        edge_list_vertex_2 = vertex_2.all_edges()

                        # if the vertex has only one edge no better is available
                        if len(edge_list_vertex_2) > 1:
                            edges_to_delete = []
                            current_best_weight = 0
                            current_best_edge = None

                            for edge in edge_list_vertex_2:
                                edges_to_delete.append(edge)
                                edge_weight = edge["weight"]

                                # only check edge going from a lower resolution to a higher resolution -> tree structure
                                if graph.vs[edge.target]["level"] < vertex_2["level"]:
                                    edges_to_delete.pop(len(edges_to_delete) - 1)
                                else:
                                    # choose the edge with the best edge weight
                                    if edge_weight > current_best_weight:
                                        edges_to_delete.pop(len(edges_to_delete) - 1)
                                        if current_best_edge is not None:
                                            edges_to_delete.append(current_best_edge)
                                        current_best_weight = edge_weight
                                        current_best_edge = edge

                                    # if the edges have the same edge weight choose the higher resolution one
                                    elif edge_weight == current_best_weight:
                                        last_edge = edges_to_delete.pop(len(edges_to_delete) - 1)
                                        if graph.vs[last_edge.target]["level"] > graph.vs[edge.target]["level"]:
                                            edges_to_delete.append(last_edge)
                                        else:
                                            edges_to_delete.append(edge)

                            graph.delete_edges(edges_to_delete)

    # otherwise returns the maximum spanning tree
    else:
        # calculate all edge weights for every vertex in the graph
        for vertex_1 in graph.vs:
            for vertex_2 in graph.vs:
                if vertex_1 != vertex_2:
                    # calculate edge weight
                    edge_weight = mcc.weighted_jaccard(vertex_1["probability_df"], vertex_2["probability_df"])

                    # if the edge_weight is greater 0 the edge is added
                    if edge_weight != 0:
                        index_1 = vertex_1.index
                        index_2 = vertex_2.index
                        graph.add_edge(index_1, index_2, weight=edge_weight)

        # invert edge weights
        inverted_weights = [1 - edge_weight for edge_weight in graph.es["weight"]]

        # create the minimum spanning tree using the inverted edge weight -> maximum spanning tree
        graph = graph.spanning_tree(weights=inverted_weights, return_tree=True)

    return graph

def multires_community_detection(graph, clustering_data, community_detection, multi_resolution):
    """
    Uses louvain community detection on the multi-resolution graph and creates a clustering tree with reconnect_graph.
    Optional clean up of the clustering tree with edge merging.

    @param multi_resolution: Resolution parameter for the community detection function.
    @param clustering_data: The clustering data on which the graph is based
    @param community_detection: "leiden", "hdbscan" or else automatically louvain,
        detects community with the named algorithms
    @param graph: The multi resolution graph, iGraph graph.
    @return: A clustering tree, igraph Graph.
    """

    # apply probability outlier detection to the graph
    graph = mcc.filter_by_node_probability(graph)

    # community detection
    if community_detection == "leiden":
        # uses the leiden algorithm for community detection
        vertex_clustering = ig.Graph.community_leiden(graph, weights="weight", resolution_parameter=multi_resolution)
    elif community_detection == "hdbscan" and graph.vcount() > 1 and graph.ecount() > 0:
        # use hdbscan for community detection
        vertex_clustering = mcc.hdbscan_community_detection(graph)
    else:
        # if nothing is selected uses louvain community detection
        vertex_clustering = ig.Graph.community_multilevel(graph, weights="weight")

    # combines attributes to a list
    graph = mcc.merge_by_list(vertex_clustering)

    # recalculate the probabilities based on the new merged nodes
    new_probabilities = mcc.graph_nodes_cells_to_df(graph, clustering_data)
    graph.vs["probability_df"] = [new_probabilities[column].values for column in new_probabilities.columns]

    # delete edges and create graph tree structure if there is more than one node
    if graph.vcount() != 1:
        graph.delete_edges()
        graph = reconnect_graph(graph)

        if graph.ecount() > 0:
            # calculate max and min edge weights
            max_edge = max(graph.es["weight"])
            min_edge = min(graph.es["weight"])

            # calculate max-min average
            max_min_average = (max_edge + min_edge) / 2

            # set the upper quantile for the merge edges threshold
            merge_edges_threshold = (max_edge + max_min_average) / 2

            # clean up graph by merging some edges
            graph = mcc.merge_edges_weight_above_threshold(graph, threshold=merge_edges_threshold)

            # recalculate the probabilities based on the new merged nodes
            new_probabilities = mcc.graph_nodes_cells_to_df(graph, clustering_data)
            graph.vs["probability_df"] = [new_probabilities[column].values for column in new_probabilities.columns]

            # check if graph has more then vertices, if so recalculates the edges
            if graph.vcount() != 1:
                graph.delete_edges()
                graph = reconnect_graph(graph)

    return graph


import os.path
import requests
import csv
import time
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
import asyncio
from random import uniform
from fake_useragent import UserAgent

ua = UserAgent()

headers = {
    'User-Agent': ua.random
}

DIR_PATH = os.path.abspath(os.path.dirname(__file__))
folder_name = "_Output"
file_name = 'all_job_data.csv'

def fetch_job_ids(url, max_pages=2):
    try:
        all_job_ids = []
        page = 1

        while page <= max_pages:
            response = goto_next_page(url, page)

            if response is not None:
                soup = BeautifulSoup(response, 'html.parser')
                job_elements = soup.find_all('li', class_='has-pointer-d')
                if not job_elements:
                    print("No job IDs found.")
                    break  # No more job IDs to fetch
                for job_element in job_elements:
                    job_id = job_element.get("data-job-id")
                    if job_id:
                        all_job_ids.append(job_id)

                page += 1
                time.sleep(uniform(1, 2))  # Add a random delay (1-2 seconds) to avoid overloading the website
            else:
                print(f"Failed to fetch data from the next page. Exiting the loop.")
                break

        return all_job_ids

    except Exception as e:
        print(f"Error occurred: {str(e)}")
        return []

def goto_next_page(url, page, retries=3, backoff_factor=2):
    try:
        headers['User-Agent'] = ua.random  # Rotate user-agent
        response = requests.get(url + f'?page={page}', headers=headers, timeout=30)  # Increase timeout to 30 seconds

        if response.status_code == 200:
            return response.content
        elif response.status_code == 429 and retries > 0:
            retry_after = int(response.headers.get('Retry-After', 5))
            print(f"Rate limited. Retrying after {retry_after} seconds...")
            time.sleep(retry_after)
            return goto_next_page(url, page, retries - 1, backoff_factor * 2)

        print(f"Failed to retrieve data from {url} (Page: {page}). Status code: {response.status_code}")
        return None

    except requests.Timeout:
        print(f"Request timed out while fetching data from {url} (Page: {page}).")
        return None

    except Exception as e:
        print(f"Error occurred while fetching data from {url} (Page: {page}). {str(e)}")
        return None

def fetch_data_for_job_id(job_id, retries=3, backoff_factor=2):
    try:
        headers['User-Agent'] = ua.random  # Rotate user-agent
        url = f'https://www.bayt.com/en/job/{job_id}/'
        with requests.Session() as session:
            response = session.get(url, headers=headers)

        if response.status_code == 200:
            soup = BeautifulSoup(response.content, 'html.parser')
            details_desc_mapping = {'Job ID': job_id}

            job_elements = soup.find_all('dl', class_='dlist is-spaced is-fitted t-small')

            for job_element in job_elements:
                job_attributes = job_element.find_all('dt')
                job_desc = job_element.find_all('dd')

                for title, data in zip(job_attributes, job_desc):
                    title_name = title.text.strip()
                    data_text = data.text.strip()
                    details_desc_mapping[title_name] = data_text

            return details_desc_mapping
        elif response.status_code == 429 and retries > 0:
            retry_after = int(response.headers.get('Retry-After', 5))
            print(f"Rate limited. Retrying after {retry_after} seconds...")
            time.sleep(retry_after)
            return fetch_data_for_job_id(job_id, retries - 1, backoff_factor * 2)

        print(f"Failed to retrieve data for Job ID: {job_id}. Status code: {response.status_code}")
        return {}

    except Exception as e:
        print(f"Error occurred while fetching data for Job ID: {job_id}. {str(e)}")
        return {}

def save_to_csv(all_data, csv_filename):
    try:
        with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
            fieldnames = list(all_data[0].keys())
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(all_data)
        print(f"Data has been successfully saved to '{csv_filename}'.")

    except Exception as e:
        print(f"Error occurred while saving to CSV: {str(e)}")

async def main():
    url = 'https://www.bayt.com/en/saudi-arabia/jobs/'
    job_ids = fetch_job_ids(url)

    if job_ids:
        all_data = []
        field_names = set()  # Set to store all unique field names

        with ThreadPoolExecutor() as executor:
            loop = asyncio.get_event_loop()
            futures = [loop.run_in_executor(executor, fetch_data_for_job_id, job_id) for job_id in job_ids]

            for result in await asyncio.gather(*futures):
                all_data.append(result)
                field_names.update(result.keys())  # Update field names set

        # Add missing fields with empty values to all_data
        for details_desc_mapping in all_data:
            for field_name in field_names:
                if field_name not in details_desc_mapping:
                    details_desc_mapping[field_name] = ''

        folder_name = "_Output"
        path = os.path.join(DIR_PATH, folder_name)
        try:
            os.mkdir(path)
        except OSError as error:
            print(error)

        csv_filename = "all_job_data.csv"
        save_to_csv(all_data, os.path.join(path, csv_filename))
    else:
        print("No job IDs found.")

if __name__ == '__main__':
    asyncio.run(main())


import pandas as pd
from models.MCMC_fourier_windfield import RandomFourierFeatures
# from models.quadratic_fourier_windfield import FourierWindfield
from models.averaging_windfield import AveragingWindfield
from error_estimation.windfield_prediction import do_parallel_5fold, split_and_save_data
from framework.data_loader import load_parquet
import itertools
import numpy as np
from pathos.multiprocessing import freeze_support
from sys import path
from os.path import dirname as dir

# Prevent matrix inversion from using multiprocessing,
# Only parallelize the prediction process based on the time slices.
let_numpy_use_multiproc = False
if not let_numpy_use_multiproc:
    import os
    os.environ["OMP_NUM_THREADS"] = "1" # export OMP_NUM_THREADS=4
    os.environ["OPENBLAS_NUM_THREADS"] = "1" # export OPENBLAS_NUM_THREADS=4
    os.environ["MKL_NUM_THREADS"] = "1" # export MKL_NUM_THREADS=6
    os.environ["VECLIB_MAXIMUM_THREADS"] = "1" # export VECLIB_MAXIMUM_THREADS=4
    os.environ["NUMEXPR_NUM_THREADS"] = "1" # export NUMEXPR_NUM_THREADS=6

if __name__ == "__main__" and __package__ is None:
    freeze_support()
    path.append(dir(path[0]))

    # Folder to put the data in
    data_folder = "D:/python_runs/windmodelling/Data/"

    # Folder to save the runs in
    save_folder = "D:/python_runs/windmodelling/Runs/Hyper_optimisation_run/"

    # Load data
    data = load_parquet("Data/wind_data_2018.parquet")
    data = data[data['date'].apply(lambda f: f[0:7]) != '2018-09']      # Remove high outlier data from a storm in september.

    # Divide into time slices and save locally for reference later
    #split_and_save_data(data, data_folder)      # Comment this out if you already ran the code once

    # Specify parameter grid.
    # Make sure to copy the grid setup as the filing system does not explicitely keep track of this for you
    n_terms = 20
    n_steps = [10]
    reg_params = np.logspace(-3, 0, 2)
    div_params = np.logspace(-3, 0, 2)
    gammas = [1.25]
    sigmas = [2.25]
    params = itertools.product(n_steps, reg_params, div_params, gammas, sigmas)

    # Generate windfields corresponding to the parameters
    # Generate save directories for each windfield.
    # Preallocate lists
    windfields = []
    save_folders = []
    for p in params:
        n = p[0]
        r = p[1]
        d = p[2]
        g = p[3]
        s = p[4]
        windfields.append(RandomFourierFeatures(n_terms=n_terms, n_steps=n, reg_param=r, div_param=d, gamma=g, sigma=s, seed=100))
        save_folders.append(save_folder + f"RandomFeatures/{n}/{r}/{d}/{g}/{s}/")

    do_parallel_5fold(windfields, data_folder, save_folders, processes=7)

# ============ DISCLAIMER ==============================================================
# The data is now ready for post processing, see multiproc_fourier_features_analysis.py.


import functools
import itertools
from typing import (
    Any,
    Dict,
    Generator,
    Iterable,
    Iterator,
    List,
    Optional,
    Sequence,
    Tuple,
    TypeVar,
    Union,
)

from eth_typing import Hash32
from eth_utils import to_dict, to_tuple
from eth_utils.toolz import groupby, partition, pipe
from pyrsistent import pvector
from pyrsistent._transformations import transform
from pyrsistent.typing import PVector

from ssz.abc import (
    HashableStructureAPI,
    HashableStructureEvolverAPI,
    ResizableHashableStructureAPI,
    ResizableHashableStructureEvolverAPI,
)
from ssz.constants import CHUNK_SIZE, ZERO_BYTES32
from ssz.hash_tree import HashTree
from ssz.sedes.base import BaseProperCompositeSedes

TStructure = TypeVar("TStructure", bound="BaseHashableStructure")
TResizableStructure = TypeVar(
    "TResizableStructure", bound="BaseResizableHashableStructure"
)
TElement = TypeVar("TElement")

def update_element_in_chunk(
    original_chunk: Hash32, index: int, element: bytes
) -> Hash32:
    """
    Replace part of a chunk with a given element.

    The chunk is interpreted as a concatenated sequence of equally sized elements. This function
    replaces the element given by its index in the chunk with the given data.

    If the length of the element is zero or not a divisor of the chunk size, a `ValueError` is
    raised. If the index is out of range, an `IndexError` is raised.

    .. doctest::

        >>> from ssz.hashable_structure import update_element_in_chunk
        >>> update_element_in_chunk(b"aabbcc", 1, b"xx")
        b'aaxxcc'
    """
    element_size = len(element)
    chunk_size = len(original_chunk)

    if element_size == 0:
        raise ValueError(f"Element size is zero")
    if chunk_size % element_size != 0:
        raise ValueError(f"Element size is not a divisor of chunk size: {element_size}")
    if not 0 <= index < chunk_size // element_size:
        raise IndexError(f"Index out of range for element size {element_size}: {index}")

    first_byte_index = index * element_size
    last_byte_index = first_byte_index + element_size

    prefix = original_chunk[:first_byte_index]
    suffix = original_chunk[last_byte_index:]
    return Hash32(prefix + element + suffix)

def update_elements_in_chunk(
    original_chunk: Hash32, updated_elements: Dict[int, bytes]
) -> Hash32:
    """
    Update multiple elements in a chunk.

    The set of updates is given by a dictionary mapping indices to elements. The items of the
    dictionary will be passed one by one to `update_element_in_chunk`.
    """
    return pipe(
        original_chunk,
        *(
            functools.partial(update_element_in_chunk, index=index, element=element)
            for index, element in updated_elements.items()
        ),
    )

def get_num_padding_elements(
    *, num_original_elements: int, num_original_chunks: int, element_size: int
) -> int:
    """Compute the number of elements that would still fit in the empty space of the last chunk."""
    total_size = num_original_chunks * CHUNK_SIZE
    used_size = num_original_elements * element_size
    padding_size = total_size - used_size
    num_elements_in_padding = padding_size // element_size
    return num_elements_in_padding

@to_dict
def get_updated_chunks(
    *,
    updated_elements: Dict[int, bytes],
    appended_elements: Sequence[bytes],
    original_chunks: Sequence[Hash32],
    element_size: int,
    num_original_elements: int,
    num_padding_elements: int,
) -> Generator[Tuple[int, Hash32], None, None]:
    """
    For an element changeset, compute the updates that have to be applied to the existing chunks.

    The changeset is given as a dictionary of element indices to updated elements and a sequence of
    appended elements. Note that appended elements that do not affect existing chunks are ignored.

    The pre-existing state is given by the sequence of original chunks and the number of elements
    represented by these chunks.

    The return value is a dictionary mapping chunk indices to chunks.
    """
    effective_appended_elements = appended_elements[:num_padding_elements]
    elements_per_chunk = CHUNK_SIZE // element_size

    padding_elements_with_indices = dict(
        enumerate(effective_appended_elements, start=num_original_elements)
    )
    effective_updated_elements = {**updated_elements, **padding_elements_with_indices}

    element_indices = effective_updated_elements.keys()
    element_indices_by_chunk = groupby(
        lambda element_index: element_index // elements_per_chunk, element_indices
    )

    for chunk_index, element_indices in element_indices_by_chunk.items():
        chunk_updates = {
            element_index
            % elements_per_chunk: effective_updated_elements[element_index]
            for element_index in element_indices
        }
        updated_chunk = update_elements_in_chunk(
            original_chunks[chunk_index], chunk_updates
        )
        yield chunk_index, updated_chunk

@to_tuple
def get_appended_chunks(
    *, appended_elements: Sequence[bytes], element_size: int, num_padding_elements: int
) -> Generator[Hash32, None, None]:
    """Get the sequence of appended chunks."""
    if len(appended_elements) <= num_padding_elements:
        return

    elements_per_chunk = CHUNK_SIZE // element_size

    chunk_partitioned_elements = partition(
        elements_per_chunk,
        appended_elements[num_padding_elements:],
        pad=b"\x00" * element_size,
    )
    for elements_in_chunk in chunk_partitioned_elements:
        yield Hash32(b"".join(elements_in_chunk))

class BaseHashableStructure(HashableStructureAPI[TElement]):
    def __init__(
        self,
        elements: PVector[TElement],
        hash_tree: HashTree,
        sedes: BaseProperCompositeSedes,
        max_length: Optional[int] = None,
    ) -> None:
        self._elements = elements
        self._hash_tree = hash_tree
        self._sedes = sedes
        self._max_length = max_length

    @classmethod
    def from_iterable_and_sedes(
        cls,
        iterable: Iterable[TElement],
        sedes: BaseProperCompositeSedes,
        max_length: Optional[int] = None,
    ):
        elements = pvector(iterable)
        if max_length and len(elements) > max_length:
            raise ValueError(
                f"Number of elements {len(elements)} exceeds maximum length {max_length}"
            )

        serialized_elements = [
            sedes.serialize_element_for_tree(index, element)
            for index, element in enumerate(elements)
        ]
        appended_chunks = get_appended_chunks(
            appended_elements=serialized_elements,
            element_size=sedes.element_size_in_tree,
            num_padding_elements=0,
        )
        hash_tree = HashTree.compute(
            appended_chunks or [ZERO_BYTES32], sedes.chunk_count
        )
        return cls(elements, hash_tree, sedes, max_length)

    @property
    def elements(self) -> PVector[TElement]:
        return self._elements

    @property
    def hash_tree(self) -> HashTree:
        return self._hash_tree

    @property
    def chunks(self) -> PVector[Hash32]:
        return self.hash_tree.chunks

    @property
    def max_length(self) -> Optional[int]:
        return self._max_length

    @property
    def raw_root(self) -> Hash32:
        return self.hash_tree.root

    @property
    def sedes(self) -> BaseProperCompositeSedes:
        return self._sedes

    #
    # Hash and equality
    #
    def __hash__(self) -> int:
        # hashable structures have the same hash if they share both sedes and root
        return hash((self.sedes, self.hash_tree_root))

    def __eq__(self, other: Any) -> bool:
        # hashable structures are equal if they use the same sedes and have the same root
        if isinstance(other, BaseHashableStructure):
            sedes_equal = self.sedes == other.sedes
            roots_equal = self.hash_tree_root == other.hash_tree_root
            return sedes_equal and roots_equal
        else:
            return False

    #
    # PVector interface
    #
    def __len__(self) -> int:
        return len(self.elements)

    def __getitem__(self, index: int) -> TElement:
        return self.elements[index]

    def __iter__(self) -> Iterator[TElement]:
        return iter(self.elements)

    def transform(self, *transformations):
        return transform(self, transformations)

    def mset(self: TStructure, *args: Union[int, TElement]) -> TStructure:
        if len(args) % 2 != 0:
            raise TypeError(
                f"mset must be called with an even number of arguments, got {len(args)}"
            )

        evolver = self.evolver()
        for index, value in partition(2, args):
            evolver[index] = value
        return evolver.persistent()

    def set(self: TStructure, index: int, value: TElement) -> TStructure:
        return self.mset(index, value)

    def evolver(
        self: TStructure,
    ) -> "HashableStructureEvolverAPI[TStructure, TElement]":
        return HashableStructureEvolver(self)

class HashableStructureEvolver(HashableStructureEvolverAPI[TStructure, TElement]):
    def __init__(self, hashable_structure: TStructure) -> None:
        self._original_structure = hashable_structure
        self._updated_elements: Dict[int, TElement] = {}
        # `self._appended_elements` is only used in the subclass ResizableHashableStructureEvolver,
        # but the implementation of `persistent` already processes it so that it does not have to
        # be implemented twice.
        self._appended_elements: List[TElement] = []

    def __getitem__(self, index: int) -> TElement:
        if index < 0:
            index += len(self)

        if index in self._updated_elements:
            return self._updated_elements[index]
        elif 0 <= index < len(self._original_structure):
            return self._original_structure[index]
        elif 0 <= index < len(self):
            return self._appended_elements[index - len(self._original_structure)]
        else:
            raise IndexError(f"Index out of bounds: {index}")

    def set(self, index: int, element: TElement) -> None:
        self[index] = element

    def __setitem__(self, index: int, element: TElement) -> None:
        if index < 0:
            index += len(self)

        if 0 <= index < len(self._original_structure):
            self._updated_elements[index] = element
        elif 0 <= index < len(self):
            self._appended_elements[index - len(self._original_structure)] = element
        else:
            raise IndexError(f"Index out of bounds: {index}")

    def __len__(self) -> int:
        return len(self._original_structure) + len(self._appended_elements)

    def is_dirty(self) -> bool:
        return bool(self._updated_elements or self._appended_elements)

    def persistent(self) -> TStructure:
        if not self.is_dirty():
            return self._original_structure

        sedes = self._original_structure.sedes

        num_original_elements = len(self._original_structure)
        num_original_chunks = len(self._original_structure.chunks)
        num_padding_elements = get_num_padding_elements(
            num_original_elements=num_original_elements,
            num_original_chunks=num_original_chunks,
            element_size=sedes.element_size_in_tree,
        )

        updated_elements = {
            index: sedes.serialize_element_for_tree(index, element)
            for index, element in self._updated_elements.items()
        }
        appended_elements = [
            sedes.serialize_element_for_tree(index, element)
            for index, element in enumerate(
                self._appended_elements, start=num_original_elements
            )
        ]

        updated_chunks = get_updated_chunks(
            updated_elements=updated_elements,
            appended_elements=appended_elements,
            original_chunks=self._original_structure.chunks,
            num_original_elements=num_original_elements,
            num_padding_elements=num_padding_elements,
            element_size=sedes.element_size_in_tree,
        )
        appended_chunks = get_appended_chunks(
            appended_elements=appended_elements,
            element_size=sedes.element_size_in_tree,
            num_padding_elements=num_padding_elements,
        )

        elements = self._original_structure.elements.mset(
            *itertools.chain.from_iterable(  # type: ignore
                self._updated_elements.items()
            )
        ).extend(self._appended_elements)
        hash_tree = self._original_structure.hash_tree.mset(
            *itertools.chain.from_iterable(updated_chunks.items())  # type: ignore
        ).extend(appended_chunks)

        return self._original_structure.__class__(
            elements, hash_tree, self._original_structure.sedes
        )

class BaseResizableHashableStructure(
    BaseHashableStructure, ResizableHashableStructureAPI[TElement]
):
    def append(self: TResizableStructure, value: TElement) -> TResizableStructure:
        evolver = self.evolver()
        evolver.append(value)
        return evolver.persistent()

    def extend(
        self: TResizableStructure, values: Iterable[TElement]
    ) -> TResizableStructure:
        evolver = self.evolver()
        evolver.extend(values)
        return evolver.persistent()

    def __add__(
        self: TResizableStructure, values: Iterable[TElement]
    ) -> TResizableStructure:
        return self.extend(values)

    def __mul__(self: TResizableStructure, times: int) -> TResizableStructure:
        if times <= 0:
            raise ValueError(f"Multiplication factor must be positive: {times}")
        elif times == 1:
            return self
        else:
            return (self + self) * (times - 1)

    def evolver(
        self: TResizableStructure,
    ) -> "ResizableHashableStructureEvolverAPI[TResizableStructure, TElement]":
        return ResizableHashableStructureEvolver(self)

class ResizableHashableStructureEvolver(
    HashableStructureEvolver, ResizableHashableStructureEvolverAPI[TStructure, TElement]
):
    def append(self, element: TElement) -> None:
        max_length = self._original_structure.max_length
        if max_length is not None and len(self) + 1 > max_length:
            raise ValueError(f"Structure would exceed maximum length {max_length}")
        self._appended_elements.append(element)

    def extend(self, elements: Iterable[TElement]) -> None:
        extension = list(elements)

        max_length = self._original_structure.max_length
        if max_length is not None and len(self) + len(extension) > max_length:
            raise ValueError(f"Structure would exceed maximum length {max_length}")

        self._appended_elements.extend(extension)


from collections import defaultdict
from contextlib import contextmanager
import csv
from itertools import tee
import json
import math
import os
import tempfile
import time

import folium
import joblib
from selenium import webdriver
from shapely.geometry import shape, Point

SEASONS = ["winter", "spring", "summer", "autumn"]

@contextmanager
def _tmp_html(data):
    """Yields the path of a temporary HTML file containing data."""
    filepath = ''
    try:
        fid, filepath = tempfile.mkstemp(suffix=".html", prefix="folium_")
        os.write(fid, data.encode("utf8"))
        os.close(fid)
        yield filepath
    finally:
        if os.path.isfile(filepath):
            os.remove(filepath)

def dump_geojson(data, path, **dump_kwargs):
    with open(path, "w") as f:
        json.dump(data, f, separators=(",", ":"), **dump_kwargs)

def _is_point_in_multigon(point, polygons):
    for polygon in polygons:
        if polygon.contains(point):
            return True
    return False

def _parse_sswi_row(row, karst_polygons):
    try:
        point, lat, lng, _, horizon, season, sswi, _ = row
    except ValueError:
        return
    if point.startswith("#"):
        return

    season = SEASONS[int(season) - 1]
    lat = float(lat)
    lng = float(lng)
    sswi = float(sswi)

    in_karst = _is_point_in_multigon(Point(lng, lat), karst_polygons)
    risk_level = 0
    if sswi < -1.4 and in_karst:
        risk_level = 3
    elif sswi < -1.4 and not in_karst:
        risk_level = 2
    elif sswi < -0.7:
        risk_level = 1

    return (horizon, season, {
        "type": "Feature",
        "geometry": {
            "type": "Point",
            "coordinates": [lng, lat],
        },
        "properties": {
            "riskLevel": risk_level,
            "sswi": sswi,
            "inKarst": in_karst,
        }
    })

def compute_risks(sswi_file, karst_file, n_jobs=-1, batch_size="auto", verbose=50, logger=None):
    if logger is not None:
        logger.info("Loading data...")
        sswi_file, sswi_file2 = tee(sswi_file)
        logger.info("Number of points:", sum(1 for line in sswi_file2))

    sswi_reader = csv.reader(sswi_file, delimiter=";")
    karst_polygons = []
    for feature in json.load(karst_file)["features"]:
        karst_polygons.append(shape(feature["geometry"]).buffer(0))

    if logger is not None:
        logger.info("Number of karstic polygons:", len(karst_polygons))
        logger.info("Parsing data...")

    points = joblib.Parallel(
        n_jobs=n_jobs,
        verbose=verbose,
        backend="multiprocessing",
        batch_size=batch_size,
    )(
        joblib.delayed(_parse_sswi_row)(row, karst_polygons)
        for row in sswi_reader
    )

    min_sswi = math.inf
    max_sswi = -math.inf
    features = defaultdict(  # Horizon
        lambda: defaultdict(list)  # Season
    )
    for point in points:
        if point is None:
            continue
        h, s, p = point
        features[h][s].append(p)
        min_sswi = min(p["properties"]["sswi"], min_sswi)
        max_sswi = max(p["properties"]["sswi"], max_sswi)

    geojson = defaultdict(dict)
    for horizon, horizon_data in features.items():
        horizon_points = defaultdict(list)
        for season, points in horizon_data.items():
            geojson[horizon][season] = {
                "type": "FeatureCollection",
                "features": points,
            }
            for p in points:
                horizon_points[tuple(p["geometry"]["coordinates"])].append(p)

        worst_horizon_points = []
        best_horizon_points = []
        for coords, points in horizon_points.items():
            if len(points) != len(horizon_data): # Not all seasons have this point
                continue
            worst_p = best_p = None
            for p in points:
                risk = p["properties"]["riskLevel"]
                if worst_p is None or risk > worst_p["properties"]["riskLevel"]:
                    worst_p = p
                if best_p is None or risk < best_p["properties"]["riskLevel"]:
                    best_p = p
            worst_horizon_points.append(worst_p)
            best_horizon_points.append(best_p)

        geojson[horizon]["more_risky"] = {
            "type": "FeatureCollection",
            "features": worst_horizon_points,
        }
        geojson[horizon]["less_risky"] = {
            "type": "FeatureCollection",
            "features": best_horizon_points,
        }

    metadata = {
        "sswi": dict(min=min_sswi, max=max_sswi),
        "riskLevels": [
            "",
            "Limitations de tous les prélèvements d'eau",
            "Interdiction d'utiliser l'eau pour des usages non prioritaires",
            "Menaces de pénuries en eau potable"
        ],
    }

    return geojson, metadata

def filter_karst_by_type(f, type_=1, logger=None):
    if logger is not None:
        logger.info("Loading karst data")
    data = json.load(f)

    if logger is not None:
        logger.info("Filtering karst data")
    features = []
    for feature in data["features"]:
        if feature["properties"]["TypeZK"] == type_:
            features.append(feature)

    data["features"] = features
    return data

def display(risk, colorscale, karst=None, map_location=[46.2, 2.4], map_zoom=5, width=350, height=350, **map_kwargs):
    m = folium.Map(
        location=map_location,
        zoom_start=map_zoom,
        width=width,
        height=height,
        prefer_canvas=True,
        **map_kwargs
    )

    risk_layer = folium.FeatureGroup(name="Risk")
    for p in risk["features"]:
        color = colorscale[p["properties"]["riskLevel"]]
        lng, lat = p["geometry"]["coordinates"]
        folium.CircleMarker(
            location=[lat, lng],
            radius=0.5,
            fill=True,
            fill_color=color,
            color=color,
        ).add_to(risk_layer)
    risk_layer.add_to(m)

    if karst is not None:
        folium.GeoJson(karst, name="Karst").add_to(m)
        folium.LayerControl().add_to(m)

    return m

def map_to_png(m, path, delay=3):
    # https://github.com/python-visualization/folium/blob/master/folium/folium.py#L296
    # geckodriver must be installed:
    # https://selenium-python.readthedocs.io/installation.html#drivers
    options = webdriver.firefox.options.Options()
    options.add_argument("--headless")
    driver = webdriver.Firefox(options=options)
    html = m.get_root().render()
    with _tmp_html(html) as fname:
        # We need the tempfile to avoid JS security issues.
        driver.get("file:///{path}".format(path=fname))
        driver.maximize_window()
        time.sleep(delay)
        el = driver.find_elements_by_class_name("folium-map")[0]
        el.screenshot(path)
        driver.quit()


#!/usr/bin/env python3
import argparse
# import collections.abc
import os
import time
import warnings
import math
import numpy as np
import yaml

import analysis_utils as au
import plot_utils as pu

from ROOT import TH1D, TFile, gROOT, TDatabasePDG, TF1

# avoid pandas warning
warnings.simplefilter(action='ignore', category=FutureWarning)
gROOT.SetBatch()

###############################################################################
parser = argparse.ArgumentParser()
parser.add_argument('config', help='Path to the YAML configuration file')
parser.add_argument('-m', '--merged', help='Run on the merged histograms', action='store_true')
parser.add_argument('-p', '--peak', help='Take signal from the gaussian fit', action='store_true')
args = parser.parse_args()

with open(os.path.expandvars(args.config), 'r') as stream:
    try:
        params = yaml.full_load(stream)
    except yaml.YAMLError as exc:
        print(exc)
###############################################################################

###############################################################################
# define analysis global variables
N_BODY = params['NBODY']
PDG_CODE = params['PDG']
FILE_PREFIX = params['FILE_PREFIX']
MULTIPLICITY = params['MULTIPLICITY']
BRATIO = params['BRATIO']
SIGMA = params['SIGMA']
EINT = pu.get_sNN(params['EINT'])
GAUSS = params['GAUSS']
PT_BINS = params['PT_BINS']
T = params['T']

EFF_MIN, EFF_MAX, EFF_STEP = params['BDT_EFFICIENCY']
FIX_EFF_ARRAY = np.arange(EFF_MIN, EFF_MAX, EFF_STEP)
BKG_MODELS = params['BKG_MODELS']

PEAK_MODE = args.peak
MERGED = args.merged

LABELS = [f'{x:.2f}_{y}' for x in FIX_EFF_ARRAY for y in BKG_MODELS]

###############################################################################
# define paths for loading results
results_dir = os.environ['RESULTS']

input_file_name = results_dir + '/' + FILE_PREFIX + f'/{FILE_PREFIX}_results_merged.root' if MERGED else results_dir + '/' + FILE_PREFIX + f'/{FILE_PREFIX}_results.root'
input_file = TFile(input_file_name, 'read')

output_file_name = results_dir + '/' + FILE_PREFIX + f'/{FILE_PREFIX}_results_BS.root'
output_file = TFile(output_file_name, 'recreate')

###############################################################################

###############################################################################
# start the actual signal extraction
mass = TDatabasePDG.Instance().GetParticle(PDG_CODE).Mass()
pt_spectrum = TF1("fpt", "x*exp(-TMath::Sqrt(x**2+[0]**2)/[1])", 0, 100)
pt_spectrum.FixParameter(0, mass)
pt_spectrum.FixParameter(1, T)
n_events = 0
for index in range(0, len(params['EVENT_PATH'])):
    event_path = os.path.expandvars(params['EVENT_PATH'][index])
    background_file = TFile(event_path)
    hist_ev = background_file.Get('hNevents')
    n_events += hist_ev.GetBinContent(1)
nsigma = 3
cent_dir_name = '0-5'
#cent_dir = output_file.mkdir(cent_dir_name)
#cent_dir.cd()

h1_eff = input_file.Get(cent_dir_name + '/PreselEff')
for ptbin in zip(PT_BINS[:-1], PT_BINS[1:]):
    ptbin_index = au.get_ptbin_index(h1_eff, ptbin)
    # get the dir where the inv mass histo are
    subdir_name = f'pt_{ptbin[0]}{ptbin[1]}'
    input_subdir = input_file.Get(f'{cent_dir_name}/{subdir_name}')
    eff_presel = h1_eff.GetBinContent(ptbin_index)
    # create the subdir in the output file
    #output_subdir = cent_dir.mkdir(subdir_name)
    #output_subdir.cd()
    nbins = int((EFF_MAX-EFF_MIN)/EFF_STEP)
    hist_BS = TH1D("hist_BS_" + subdir_name, ";BDT efficiency;B/S", nbins, EFF_MAX, EFF_MIN)
    #hist_BS.GetXaxis().SetNdivisions(10)
    hist_BS_efftot = TH1D("hist_BS_efftot_" + subdir_name, ";BDT efficiency;B/S", nbins, EFF_MAX*eff_presel, EFF_MIN*eff_presel)
    #hist_BS_efftot.GetXaxis().SetNdivisions(10)
    hist_Sgn = TH1D("hist_Sgn_" + subdir_name, ";BDT efficiency;Sign/#sqrt{n_{ev}}", nbins, EFF_MAX , EFF_MIN)
    #hist_Sgn.GetXaxis().SetNdivisions(10)
    hist_Sgn_efftot = TH1D("hist_Sgn_efftot_" + subdir_name, ";BDT efficiency;Sign/#sqrt{n_{ev}}", nbins, EFF_MAX*eff_presel, EFF_MIN*eff_presel)
    #hist_BS_efftot.GetXaxis().SetNdivisions(10)

    print("eff presel: ", eff_presel)
    print("multiplicity: ", MULTIPLICITY)
    print("b-ratio: ", BRATIO)
    # loop over all the histo in the dir
    for key in input_subdir.GetListOfKeys():
        keff = key.GetName()[-4:]
        #for latter in reversed(key.GetName()[-4:]):
        hist = TH1D(key.ReadObj())
        mass_range = [mass - nsigma*SIGMA, mass + nsigma*SIGMA]
        bkg_counts = 0
        for index in np.arange(mass_range[0], mass_range[1], hist.GetBinWidth(1)):
            bkg_counts += hist.GetBinContent(hist.GetXaxis().FindBin(index))

        pt_frac = 1# pt_spectrum.Integral(ptbin[0], ptbin[1], 1e-8) / pt_spectrum.Integral(0, 100, 1e-8)
        #ct_frac = ct_spectrum.Integral(ctbin[0], ctbin[1], 1e-8) / ct_spectrum.Integral(0, 100, 1e-8)

        print("pt_frac: ", eff_presel)
        sig_counts = MULTIPLICITY*BRATIO*n_events*0.701#*pt_frac#*float(keff)#*ct_frac TODO: fix this stuff
        print("eff: ",keff," sig: ",sig_counts," bkg: ",bkg_counts," B/S",round(bkg_counts/sig_counts,1)," bin: ",hist_BS.GetXaxis().FindBin(keff))
        hist_BS.SetBinContent(hist_BS.GetXaxis().FindBin(keff),bkg_counts/sig_counts)
        hist_BS_efftot.SetBinContent(hist_BS.GetXaxis().FindBin(keff),bkg_counts/sig_counts)
        hist_Sgn.SetBinContent(hist_BS.GetXaxis().FindBin(keff),sig_counts/math.sqrt(sig_counts+bkg_counts)/math.sqrt(n_events))
        hist_Sgn_efftot.SetBinContent(hist_BS.GetXaxis().FindBin(keff),sig_counts/math.sqrt(sig_counts+bkg_counts)/math.sqrt(n_events))

    output_file.cd()
    hist_BS.Write()
    hist_BS_efftot.Write()
    hist_Sgn.Write()
    hist_Sgn_efftot.Write()

output_file.Close()


from utility import Config, make_request, get_server_time, make_binance_request
from json import loads
from tabulate import tabulate
from logic import be_get_token_defi_value
from matplotlib import use
from matplotlib.pyplot import pie, legend, suptitle, axis, figure, close, cla
from matplotlib.backends.backend_pdf import PdfPages
from numpy import array
from threading import Thread
from datetime import datetime
from logging import info

class BinanceThread (Thread):

    def __init__(self, key):
        Thread.__init__(self)
        self.key = key
        self.order_list = []
        self.buy_sell_orders = {}

    def run(self):
        orders = []
        for stable in Config.settings['binance']['symbols'][self.key]['stablecoin']:
            orders += make_binance_request("api/v3/myTrades", f"timestamp={Config.settings['binance']['time']}&symbol={self.key}{stable}")
        medium, qty_total, medium_sell, qty_total_sell = 0, 0, 0, 0
        if self.key + "BUSD" in Config.orders:
            for order in Config.orders[self.key + "BUSD"]:
                if order['type'] == 'BUY':
                    qty_total += order['amount']
                    medium += order['amount'] * order['value']
                else:
                    qty_total_sell += order['amount']
                    medium_sell += order['amount'] * order['value']
                self.order_list.append([order['date'], order['type'], self.key, order['value'], order['amount'], "", order['amount'] * order['value']])
        for order in orders:
            type_order = "SELL"
            if order['isBuyer']:
                type_order = "BUY"
                medium += float(order['price']) * float(order['qty'])
                qty_total += float(order['qty'])
            else:
                medium_sell += float(order['price']) * float(order['qty'])
                qty_total_sell += float(order['qty'])
            self.order_list.append([datetime.fromtimestamp(order['time'] / 1000).strftime('%Y-%m-%d'), type_order, order['symbol'], order['price'], order['qty'], order['commission'] + " " + order['commissionAsset'], order['quoteQty']])
        self.order_list.sort(key=lambda x: x[0])
        self.buy_sell_orders[self.key] = {
            'buy': {'medium': 0, 'qty_total': 0},
            'sell': {'medium': 0, 'qty_total': 0}
        }
        if qty_total > 0:
            self.buy_sell_orders[self.key]['buy'] = {'medium': medium / qty_total, 'qty_total': qty_total}
        if qty_total_sell > 0:
            self.buy_sell_orders[self.key]['sell'] = {'medium': medium_sell / qty_total_sell, 'qty_total': qty_total_sell}

'''
make_binance_request("sapi/v1/lending/daily/token/position", "asset={}&timestamp={}".format(asset, str(get_server_time())))
make_binance_request("sapi/v1/lending/project/position/list", "asset={}&status=HOLDING&timestamp={}".format(asset, str(get_server_time())))
make_binance_request("sapi/v1/lending/union/interestHistory", "lendingType=CUSTOMIZED_FIXED&timestamp={}".format(str(get_server_time())))
'''

def get_open_orders(filename):
    f = open(filename, "w")
    f.write("")
    f.close()
    orders = make_binance_request("api/v3/openOrders", f"timestamp={Config.settings['binance']['time']}")
    order_list = []
    for order in orders:
        order_list.append([order['side'], order['symbol'], order['price'], order['origQty'], str(float(order['price']) * float(order['origQty']))])
    f = open(filename, "a")
    f.write(tabulate(order_list, headers=['TYPE', 'ASSET', 'PRICE', 'QTY', 'TOTAL'], tablefmt='orgtbl', floatfmt=".8f") + "\n\n\n")
    f.close()

def get_order_history(filename):
    order_list = []
    buy_sell_orders = {}
    thread_list = []
    for key in Config.settings['binance']['symbols'].keys():
        thread = BinanceThread(key)
        thread.setDaemon(True)
        thread_list.append(thread)
        thread.start()
    for t in thread_list:
        t.join()
        order_list = order_list + t.order_list
        buy_sell_orders |= t.buy_sell_orders
    f = open(filename, "a")
    f.write(tabulate(order_list, headers=['DATE', 'TYPE', 'ASSET', 'PRICE', 'QTY', 'FEE', 'TOTAL'], tablefmt='orgtbl', floatfmt=".8f") + "\n\n\n")
    f.close()
    return buy_sell_orders

def calculate_budget_coin(buy_sell_orders, total_wallet):
    to_ret = {'mining': 0, 'fee': 0}
    diff_order = buy_sell_orders['buy']['qty_total'] - buy_sell_orders['sell']['qty_total']
    if total_wallet > diff_order:
        to_ret['mining'] = total_wallet - diff_order
    else:
        to_ret['fee'] = diff_order - total_wallet
    to_ret['budget'] = total_wallet
    return to_ret

def prepare_url_coinmarketcap(buy_sell_orders, coin):
    url = f'{Config.settings["coinmarketcap"]["price_url"]}{Config.settings["coinmarketcap"]["key"]}&convert={coin}&slug='
    for key in buy_sell_orders.keys():
        if Config.settings['binance']["symbols"][key]['coinmarketcap'][:2] != '0x' and buy_sell_orders[key]:
            url += Config.settings['binance']["symbols"][key]['coinmarketcap'] + ","
    return url[:-1]

def get_ath_and_value(res_conv, key, coin, ath=False):
    cry = {}
    to_ret = {}
    name = key
    if key == "EUR":
        name = "EURC"
    if Config.settings['binance']["symbols"][key]['coinmarketcap'][:2] != '0x':
        for cry in res_conv:
            if cry['symbol'] == name:
                break
        if ath:
            res = str(make_request("https://coinmarketcap.com/currencies/" + cry['slug'])['response'])
            to_ret['ath'] = res.split("
All Time High
")[1].split("")[1].split("")[0][1:]
        to_ret['actual_value'] = cry['quote'][coin]['price']
    else:
        addr = Config.settings['binance']["symbols"]['coinmarketcap'][key].split("-")[0]
        chain = Config.settings['binance']["symbols"]['coinmarketcap'][key].split("-")[1]
        res = be_get_token_defi_value({chain: {addr: name}})
        to_ret['actual_value'] = 0
        if name in res:
            to_ret['actual_value'] = res[name]
        if ath:
            to_ret['ath'] = 'N.D.'
    return to_ret

def prepare_output(output_data, file_name_order, file_name_pdf):
    gain = round(output_data['total_eur'] - output_data['total_deposit_eur'], 2)
    gain_perc = round(gain / output_data['total_deposit_eur'] * 100, 2)
    fig1 = figure(figsize=(7, 5))
    fig1.subplots_adjust(0.3, 0, 1, 0.9)
    patch, text = pie(array([p['perc'] for p in output_data['percs_wall_eur']]))
    legend(patch, [p['label'] for p in output_data['percs_wall_eur']], loc="upper left", prop={'size': 12}, bbox_to_anchor=(0.0, 0.65), bbox_transform=fig1.transFigure)
    axis('equal')
    suptitle(f"\nDEPOSIT: {round(output_data['total_deposit_eur'], 2)}€    NOW: {round(output_data['total_eur'], 2)}€    GAIN: {gain}€ ({gain_perc}%)\n")
    fig2 = figure(figsize=(7, 5))
    fig2.subplots_adjust(0.3, 0, 1, 0.9)
    patch, text = pie(array([p['perc'] for p in output_data['percs_wall']]))
    legend(patch, [p['label'] for p in output_data['percs_wall']], loc="upper left", prop={'size': 9}, bbox_to_anchor=(0.0, 0.9), bbox_transform=fig2.transFigure)
    axis('equal')
    suptitle(f"\nTOTAL INVEST CRYPTO: {round(output_data['total_total_invest_eur'], 2)}€  - TOTAL CRYPTO: {str(round(output_data['total_balance_crypto_eur'], 2))}€")
    pdf = PdfPages(file_name_pdf)
    for fig in range(1, figure().number):
        pdf.savefig(fig)
    pdf.close()
    fig1.clear()
    fig2.clear()
    cla()
    close("all")
    head_asset_list = ['ASSET', 'MINED/FEE', 'TOT BUY', 'TOT SELL', 'AVG BUY', 'AVG SELL', 'TOT INVEST', 'TOT RETURN', 'TOT MARGIN', 'SELL NOW']
    head_actual_list = ['ASSET', 'ACT INVEST', 'REAL AVG BUY', 'ACT AVG BUY', 'ACT PRICE', 'BUDGET', 'SELL NOW', 'MARGIN', 'FINAL MARGIN']
    head_usd = ["COIN", "TOT BUY", "TOT SELL", "AVG BUY", "AVG SELL", "TOT EUR INVEST", "TOT EUR RETURN", "TOTAL MARGIN", "SELL NOW"]
    head_actual_usd = ["COIN", "ACTUAL PRICE", "BUDGET", "SELL NOW", "FINAL MARGIN"]
    f = open(file_name_order, "a")
    f.write(tabulate(output_data['assets_list'], headers=head_asset_list, tablefmt='orgtbl', floatfmt=".6f") + "\n\n\n" +
            tabulate(output_data['actual_list'], headers=head_actual_list, tablefmt='orgtbl', floatfmt=".6f") + "\n\n\n" +
            tabulate(output_data['eur_gain_total'], headers=head_usd, tablefmt='orgtbl', floatfmt=".6f") + "\n\n\n" +
            tabulate(output_data['eur_gain_actual'], headers=head_actual_usd, tablefmt='orgtbl', floatfmt=".6f") + "\n\n\n" +
            "TOTAL CRYPTO INVEST: " + str(round(output_data['total_total_invest_eur'], 2)) + "€ - " + str(round(output_data['total_total_invest'], 2)) + "$\n\n" +
            "TOTAL CRYPTO MARGIN: " + str(round(output_data['total_total_margin_eur'], 2)) + "€ - " + str(round(output_data['total_total_margin'], 2)) + "$\n\n" +
            "TOTAL CRYPTO BALANCE: " + str(round(output_data['total_balance_crypto_eur'], 2)) + "€ - " + str(round(output_data['total_balance'], 2)) + "$\n\n" +
            "TOTAL STABLECOIN: " + str(round(output_data['total_balance_stable_eur'], 2)) + "€ - " + str(round(output_data['total_balance_stable'], 2)) + "$\n\n" +
            "TOTAL EUR DEPOSIT: " + str(round(output_data['total_deposit_eur'], 2)) + "€\n\n" +
            "TOTAL EUR BALANCE: " + str(round(output_data['total_balance_eur'], 2)) + "€\n\n" +
            "TOTAL EUR IF SELL ALL NOW: " + str(round(output_data['total_eur'], 2)) + "€\n\n" +
            "TOTAL EUR MARGIN: " + str(gain) + "€\n")
    f.close()
    assets_list_tg = []
    for asset in output_data['actual_list']:
        if asset[5] > 0:
            assets_list_tg.append([asset[0], asset[3], asset[4], asset[5], asset[8]])
    return f"{tabulate(assets_list_tg, headers=['ASSET', 'AVG BUY', 'ACTUAL', 'BUDGET', 'FINAL MARGIN'], tablefmt='orgtbl', floatfmt='.4f')}\n\n\nDEP: {round(output_data['total_deposit_eur'], 2)}€   NOW: {round(output_data['total_eur'], 2)}€   MARG: {str(gain)}€"

def get_wallet(buy_sell_orders, total_wallet, file_name_order, file_name_pdf):
    output_data = {
        'assets_list': [],
        'actual_list': [],
        'percs_wall': [],
        'percs_wall_eur': [],
        'total_total_invest': 0,
        'total_total_margin': 0,
        'total_balance': 0,
        'total_balance_eur': 0
    }
    coin = 'USD'
    my_fiat = 'EUR'
    res_conv = list(loads(make_request(prepare_url_coinmarketcap(buy_sell_orders, coin))['response'])['data'].values())
    coins = list(buy_sell_orders.keys())
    coins.remove(my_fiat)
    for key in coins:
        total_invest = buy_sell_orders[key]['buy']['qty_total'] * buy_sell_orders[key]['buy']['medium']
        total_return = buy_sell_orders[key]['sell']['qty_total'] * buy_sell_orders[key]['sell']['medium']
        total_margin, actual_margin, sell_mining, sell_now, sell_now_mining, total_margin_perc = 0, 0, 0, 0, 0, 0
        my_actual_mid_buy, real_actual_mid_buy, my_actual_invest = 0, 0, 0
        actual_budget = calculate_budget_coin(buy_sell_orders[key], total_wallet[key] if key in total_wallet.keys() else 0)
        if buy_sell_orders[key]['sell']['qty_total'] > 0 or actual_budget['fee'] > 0:
            if buy_sell_orders[key]['sell']['qty_total'] > buy_sell_orders[key]['buy']['qty_total']:
                total_margin = total_return - total_invest - (actual_budget['fee'] * buy_sell_orders[key]['buy']['medium'])
            else:
                total_margin = total_return - ((buy_sell_orders[key]['sell']['qty_total'] + actual_budget['fee']) * buy_sell_orders[key]['buy']['medium'])
        total_margin = round(total_margin, 2)
        if total_invest > 0:
            total_margin_perc = f' ({str(round((total_return - total_invest) / total_invest * 100, 2))}%)'
        else:
            total_margin_perc = ' (∞)'
        final_margin = total_margin
        if actual_budget['budget'] > 0:
            total_margin_perc = ' (N.D.)'
            value_and_ath = get_ath_and_value(res_conv, key, coin)
            sell_now = actual_budget['budget'] * value_and_ath['actual_value']
            sell_now_mining = (actual_budget['mining'] if actual_budget['budget'] >= actual_budget['mining'] else actual_budget['budget']) * value_and_ath['actual_value']
            if total_invest - total_return > 0:
                my_actual_invest = total_invest - total_return
                my_actual_mid_buy = my_actual_invest / actual_budget['budget']
                real_actual_mid_buy = my_actual_invest / (actual_budget['budget'] - actual_budget['mining'])
                output_data['total_total_invest'] += my_actual_invest
            actual_margin = sell_now - sell_now_mining - my_actual_invest
            final_margin = round(actual_margin + sell_now_mining, 2)
            final_margin_perc = ' (∞)'
            if my_actual_invest > 0:
                final_margin_perc = f' ({str(round((sell_now - my_actual_invest) / my_actual_invest * 100, 2))}%)'
            output_data['total_balance'] += sell_now
            output_data['percs_wall'].append({'perc': sell_now, 'label': key + " - " + str(round(sell_now, 2)) + "$ "})
            output_data['actual_list'].append([key, my_actual_invest, real_actual_mid_buy, my_actual_mid_buy, value_and_ath['actual_value'], actual_budget['budget'], sell_now, actual_margin, str(final_margin) + final_margin_perc])
        output_data['total_total_margin'] += final_margin
        output_data['assets_list'].append([key, actual_budget['mining'] - actual_budget['fee'], buy_sell_orders[key]['buy']['qty_total'], buy_sell_orders[key]['sell']['qty_total'], buy_sell_orders[key]['buy']['medium'], buy_sell_orders[key]['sell']['medium'], total_invest, total_return, str(total_margin) + total_margin_perc, sell_now])
    i = 0
    while i < len(output_data['percs_wall']):
        output_data['percs_wall'][i]['perc'] = (output_data['percs_wall'][i]['perc'] * 100) / output_data['total_balance']
        output_data['percs_wall'][i]['label'] += f"({round(output_data['percs_wall'][i]['perc'], 2)}%)"
        i += 1
    eur_value = get_ath_and_value(res_conv, my_fiat, coin)['actual_value']
    total_invest = buy_sell_orders[my_fiat]['sell']['qty_total'] * buy_sell_orders[my_fiat]['sell']['medium']
    total_return = buy_sell_orders[my_fiat]['buy']['qty_total'] * buy_sell_orders[my_fiat]['buy']['medium']
    total_margin = (buy_sell_orders[my_fiat]['buy']['qty_total'] * buy_sell_orders[my_fiat]['sell']['medium']) - total_return
    sell_now = total_wallet[coin] / eur_value
    m = sell_now - (total_wallet[coin] / buy_sell_orders[my_fiat]['sell']['medium'])
    output_data['eur_gain_total'] = [[coin, total_invest, total_return, 1 / buy_sell_orders[my_fiat]['sell']['medium'], 1 / buy_sell_orders[my_fiat]['buy']['medium'], buy_sell_orders[my_fiat]['sell']['qty_total'], buy_sell_orders[my_fiat]['buy']['qty_total'], total_margin, sell_now]]
    output_data['eur_gain_actual'] = [[coin, 1 / eur_value, total_wallet[coin], sell_now, m]]
    output_data['total_total_margin_eur'] = output_data['total_total_margin'] / eur_value
    output_data['total_total_invest_eur'] = output_data['total_total_invest'] / eur_value
    output_data['total_balance_stable'] = total_wallet[coin]
    output_data['total_balance_crypto_eur'] = output_data['total_balance'] / eur_value
    output_data['total_balance_stable_eur'] = output_data['total_balance_stable'] / eur_value
    output_data['total_deposit_eur'] = sum(Config.settings["binance"]["deposits"]) - sum(Config.settings["binance"]["card"]) + total_wallet['card_eur']
    if my_fiat in total_wallet:
        output_data['total_balance_eur'] = total_wallet[my_fiat]
    output_data['total_eur'] = output_data['total_balance_eur'] + output_data['total_balance_stable_eur'] + output_data['total_balance_crypto_eur']
    output_data['percs_wall_eur'].append({"perc": (output_data['total_balance_crypto_eur'] * 100) / output_data['total_eur'], "label": f"CRYPTO perc\n{round(output_data['total_balance_crypto_eur'], 2)}€"})
    output_data['percs_wall_eur'].append({"perc": (output_data['total_balance_stable_eur'] * 100) / output_data['total_eur'], "label": f"STABLE perc\n{round(output_data['total_balance_stable_eur'], 2)}€"})
    output_data['percs_wall_eur'].append({"perc": (output_data['total_balance_eur'] * 100) / output_data['total_eur'], "label": f"EUR perc\n{round(output_data['total_balance_eur'], 2)}€"})
    i = 0
    while i < len(output_data['percs_wall_eur']):
        output_data['percs_wall_eur'][i]['label'] = output_data['percs_wall_eur'][i]['label'].replace("perc", f"({round(output_data['percs_wall_eur'][i]['perc'], 2)}%)")
        i += 1
    output_data['assets_list'].sort(key=lambda x: x[9], reverse=True)
    output_data['actual_list'].sort(key=lambda x: x[6], reverse=True)
    output_data['percs_wall'].sort(key=lambda x: x['perc'], reverse=True)
    output_data['percs_wall_eur'].sort(key=lambda x: x['perc'], reverse=True)
    use('Agg')
    return prepare_output(output_data, file_name_order, file_name_pdf)


import networkx as nx

#iterate through a trip string and map the weight of the total trip
def get_weight_of_trip(graph,trip):
    weight = 0
    counter = 0
    while counter < len(trip)-1:
        eD = graph.get_edge_data(trip[counter],trip[counter+1])
        if eD:
            weight += int(eD[0]['weight'])
            counter+=1
        else:
            return 'NO SUCH ROUTE'
    return weight

def get_steps_of_trip(graph,trip):
    weight = 0
    counter = 0
    while counter < len(trip)-1:
        eD = graph.get_edge_data(trip[counter],trip[counter+1])
        if eD:
            weight += int(eD[0]['weight'])
            counter+=1
        else:
            return None
    return weight

def chart_trip_weight(graph,steps):
    weight = 0
    counter = 0
    while counter < len(steps)-1:
        trip = steps[counter]+steps[counter+1]
        weight += get_weight_of_trip(graph,trip)
        counter+=1
    return weight

def chart_trip_with_stop_cap(graph,start,end,stops,depthLimit):
    trips = []
    if start==end:
        trips = chart_trip_allow_loops(graph,start,end,depthLimit,"")
    else:
        trips = nx.all_simple_edge_paths(graph, start, end)

    counter = 0
    for trip in trips:
        if len(trip) <= stops:
            counter+=1
    return counter

def chart_trip_with_stop_amt(graph,start,end,stops,depthLimit):
    trips = []
    if start==end:
        trips = chart_trip_allow_loops(graph,start,end,depthLimit,"")
    else:
        trips = nx.all_simple_edge_paths(graph, start, end)

    counter = 0
    for trip in trips:
        if len(trip)%stops:
            counter+=1
    return counter

def chart_trip_with_weight_cap(graph,start,end,weightCap,depthLimit):
    trips = []
    if start==end:
        trips = chart_trip_allow_loops(graph,start,end,depthLimit,start)
    else:
        tripOptions = list(nx.all_simple_edge_paths(graph, start, end))
        for trip in tripOptions:
            tripDescription = [start]
            for t in trip:
                tripDescription.append(t[1])
            trips.append(tripDescription)

    validTrips = []
    for trip in trips:
        wt = chart_trip_weight(graph,trip)
        if wt and wt < weightCap:
            validTrips.append(trip)
    return validTrips

def chart_shortest_trip(graph,start,end,depthLimit):
    trips = []
    if start==end:
        trips = chart_trip_allow_loops(graph,start,end,depthLimit,"")
    else:
        tripOptions = list(nx.all_simple_edge_paths(graph, start, end))
        for trip in tripOptions:
            tripDescription = [start]
            for t in trip:
                tripDescription.append(t[1])
            trips.append(tripDescription)

    shortest_trip = None
    for trip in trips:
        wt = chart_trip_weight(graph,trip)
        if wt and shortest_trip == None or wt < shortest_trip[0]:
            shortest_trip = [wt,trip]
    return shortest_trip

def chart_trip_allow_loops(graph,start,end,depthLimit,startPath):
    paths = []
    if depthLimit > 0:
        edges = graph.edges(data=True)
        for e in edges:
            if e[0] == start:
                if e[1] == end:
                    paths.append(startPath+e[1])

                subPaths = chart_trip_allow_loops(graph,e[1],end,depthLimit-1,startPath+e[1])
                for subPath in subPaths:
                    paths.append(subPath)
    return paths

def main():
    graph = nx.MultiDiGraph()
    mapDefinition = "AB5, BC4, CD8, DC8, DE6, AD5, CE2, EB3, AE7"

    #build map from mapDefinition string
    mapElements = mapDefinition.split(", ")
    for el in mapElements:
        graph.add_weighted_edges_from([(el[0], el[1], int(el[2]))])

    print('Output #1:'+str(get_weight_of_trip(graph,'ABC')))
    print('Output #2:'+str(get_weight_of_trip(graph,'AD')))
    print('Output #3:'+str(get_weight_of_trip(graph,'ADC')))
    print('Output #4:'+str(get_weight_of_trip(graph,'AEBCD')))
    print('Output #5:'+str(get_weight_of_trip(graph,'AED')))

    print('Output #6:'+str(chart_trip_with_stop_cap(graph,'C','C',3,5)))
    print('Output #7:'+str(chart_trip_with_stop_amt(graph,'A','C',4,5)))

    print('Output #8:'+str(get_weight_of_trip(graph,nx.shortest_path(graph, 'A', 'C', weight='weight', method='dijkstra'))))
    print('Output #9:'+str(chart_shortest_trip(graph,'A','C',5)[0]))
    print('Output #10:'+str(len(chart_trip_with_weight_cap(graph,'C','C',30,10))))

if __name__ == "__main__":
    main()


from __future__ import annotations

import logging
from itertools import combinations
from typing import Any
from typing import Optional

import numpy as np
from nptyping import Float
from nptyping import NDArray
from scipy import linalg
from sklearn.base import BaseEstimator
from sklearn.base import TransformerMixin
from sklearn.utils.validation import check_array
from sklearn.utils.validation import check_is_fitted

logger = logging.getLogger(__name__)

def _jade(
    arr: NDArray[(Any, ...), Float], n_components: int = 1
) -> NDArray[(Any, ...), Float]:
    """Blind separation of real signals with JADE.

    jadeR implements JADE, an Independent Component Analysis (ICA) algorithm developed
    by Jean-Francois Cardoso. See http://www.tsi.enst.fr/~cardoso/guidesepsou.html, and
    papers cited at the end of the source file.

    Translated into NumPy from the original Matlab Version 1.8 (May 2005) by Gabriel
    Beckers, http://gbeckers.nl .

    Parameters
    ----------
    arr : NDArray
        a data matrix (n_features, n_samples)
    n_components : int
        output matrix B has size mxn so that only m sources are extracted.  This is done
        by restricting the operation of jadeR to the m first principal components.
        Defaults to None, in which case :math:`m = n`.

    Returns
    -------
    NDArray
        An m*n matrix B (NumPy matrix type), such that Y=B*X are separated sources
        extracted from the n*T data matrix X. If m is omitted, B is a square n*n matrix
        (as many sources as sensors). The rows of B are ordered such that the columns of
        :math:`pinv(B)` are in order of decreasing norm; this has the effect that the
        `most energetically significant` components appear first in the rows of
        :math:`Y = B * X`.

    Notes
    -----
    Quick notes (more at the end of this file):

    - This code is for REAL-valued signals.  A MATLAB implementation of JADE for both
      real and complex signals is also available from
      http://sig.enst.fr/~cardoso/stuff.html

    - This algorithm differs from the first released implementations of JADE in that it
      has been optimized to deal more efficiently
        1) with real signals (as opposed to complex)
        2) with the case when the ICA model does not necessarily hold.

    - There is a practical limit to the number of independent components that can be
      extracted with this implementation.  Note that the first step of JADE amounts to a
      PCA with dimensionality reduction from `n` to `m` (which defaults to n).  In
      practice m cannot be *very large* (more than 40, 50, 60... depending on available
      memory)

    - See more notes, references and revision history at the end of this file and more
      stuff at http://sig.enst.fr/~cardoso/stuff.html

    - For more info on NumPy translation, see the end of this file.

    - This code is supposed to do a good job!  Please report any problem relating to
      the NumPY code gabriel@gbeckers.nl

    Copyright original Matlab code : Jean-Francois Cardoso <cardoso@sig.enst.fr>
    Copyright Numpy translation : Gabriel Beckers <gabriel@gbeckers.nl>
    """
    logger.info("jade -> Looking for %d sources", n_components)
    logger.info("jade -> Removing the mean value")
    n_samples, _ = arr.shape
    arr -= arr.mean(axis=0)

    # whitening & projection onto signal subspace
    # ===========================================
    logger.info("jade -> Whitening the data")

    # --- PCA  ----------------------------------------------------------
    u, s, vh = linalg.svd(arr, full_matrices=False)
    u, s, vh = u[:, :n_components], s[:n_components], vh[:n_components]
    B = linalg.inv(np.diag(s)) @ vh * np.sqrt(n_samples)
    arr = u * np.sqrt(n_samples)

    del u, s, vh
    # NOTE: At this stage, X is a PCA analysis in m components of the real data, except
    # that all its entries now have unit variance.  Any further rotation of X will
    # preserve the property that X is a vector of uncorrelated components.  It remains
    # to find the rotation matrix such that the entries of X are not only uncorrelated
    # but also `as independent as possible".  This independence is measured by
    # correlations of order higher than 2.  We have defined such a measure of
    # independence which
    #   1) is a reasonable approximation of the mutual information
    #   2) can be optimized by a `fast algorithm"
    # This measure of independence also corresponds to the `diagonality" of a set of
    # cumulant matrices.  The code below finds the `missing rotation " as the matrix
    # which best diagonalizes a particular set of cumulant matrices.

    # Estimation of the cumulant matrices.
    # ====================================
    logger.info("jade -> Estimating cumulant matrices")

    # Dim. of the space of real symm matrices
    dimsymm = n_components * (n_components + 1) // 2
    nbcm = dimsymm  # number of cumulant matrices
    # Storage for cumulant matrices
    CM = np.zeros((n_components, n_components * nbcm))
    R = np.eye(n_components)

    # I am using a symmetry trick to save storage.  I should write a short note one of
    # these days explaining what is going on here.
    # will index the columns of CM where to store the cum. mats.
    Range = np.arange(n_components)
    sqrt2 = np.sqrt(2)

    for im in range(n_components):
        Xim = np.c_[arr[:, im]]
        Xijm = Xim ** 2
        # Note to myself: the -R on next line can be removed: it does not affect
        # the joint diagonalization criterion
        Rim = np.c_[R[:, im]]
        CM[:, Range] = (Xijm * arr).T @ arr / n_samples - R - 2 * (Rim @ Rim.T)
        Range += n_components
        for jm in range(im):
            Xijm = Xim * np.c_[arr[:, jm]]
            Rjm = np.c_[R[:, jm]]
            CM[:, Range] = (
                sqrt2 * (Xijm * arr).T @ arr / n_samples - Rim @ Rjm.T - Rjm @ Rim.T
            )
            Range = Range + n_components

    # Now we have nbcm = m(m+1)/2 cumulants matrices stored in a big m x m*nbcm array.

    V = np.eye(n_components)

    On = np.zeros(CM.shape[0])
    Range = np.arange(n_components)
    for _ in range(nbcm):
        diag = np.diag(CM[:, Range])
        On += np.sum(diag ** 2, axis=0)
        Range += n_components
    Off = np.sum(CM ** 2) - On

    # % A statistically scaled threshold on `small" angles
    seuil = 1e-6 / np.sqrt(n_samples)
    encore = True
    sweep = 0  # % sweep number
    updates = 0  # % Total number of rotations

    # Joint diagonalization proper
    logger.info("jade -> Contrast optimization by joint diagonalization")

    counters = list(combinations(range(n_components), 2))
    while encore:
        encore = False
        logger.info("jade -> Sweep #%3d", sweep)
        sweep += 1
        upds = 0  # % Number of rotations in a given seep

        for p, q in counters:
            Ip = np.arange(p, n_components * nbcm, n_components)
            Iq = np.arange(q, n_components * nbcm, n_components)

            # computation of Givens angle
            g = np.c_[[CM[p, Ip] - CM[q, Iq], CM[p, Iq] + CM[q, Ip]]]
            gg = g @ g.T
            ton = gg[0, 0] - gg[1, 1]
            toff = gg[0, 1] + gg[1, 0]
            theta = 0.5 * np.arctan2(toff, ton + np.sqrt(ton * ton + toff * toff))
            Gain = (np.sqrt(ton * ton + toff * toff) - ton) / 4.0

            # Givens update
            if abs(theta) > seuil:
                encore = True
                upds += 1
                c = np.cos(theta)
                s = np.sin(theta)
                G = np.array([[c, -s], [s, c]])
                pair = np.array([p, q])
                V[:, pair] = V[:, pair] @ G
                CM[pair, :] = G.T @ CM[pair, :]
                cIp = np.c_[CM[:, Ip]]
                cIq = np.c_[CM[:, Iq]]
                CM[:, np.r_[Ip, Iq]] = np.c_[c * cIp + s * cIq, -s * cIp + c * cIq]
                On += Gain
                Off -= Gain

        logger.info("completed in %d rotations", upds)
        updates = updates + upds
    logger.info("jade -> Total of %d Givens rotations", updates)

    # A separating matrix
    # ===================
    B = V.T @ B

    # Permute the rows of the separating matrix B to get the most energetic components
    # first. Here the **signals** are normalized to unit variance.  Therefore, the sort
    # is according to the norm of the columns of A = pinv(B)

    logger.info("jade -> Sorting the components")

    A = linalg.pinv(B)
    keys = np.argsort((A * A).sum(axis=0))[::-1]
    B = B[keys, :]

    logger.info("jade -> Fixing the signs")
    b = B[:, 0]
    signs = np.sign(np.sign(b) + 0.1)  # just a trick to deal with sign=0
    B = np.diag(signs) @ B

    return B

    # Revision history of MATLAB code:
    #
    # - V1.8, May 2005
    #  - Added some commented code to explain the cumulant computation tricks.
    #  - Added reference to the Neural Comp. paper.
    #
    # -  V1.7, Nov. 16, 2002
    #   - Reverted the mean removal code to an earlier version (not using
    #     repmat) to keep the code octave-compatible.  Now less efficient,
    #     but does not make any significant difference wrt the total
    #     computing cost.
    #   - Remove some cruft (some debugging figures were created.  What
    #     was this stuff doing there???)
    #
    #
    # -  V1.6, Feb. 24, 1997
    #   - Mean removal is better implemented.
    #   - Transposing X before computing the cumulants: small speed-up
    #   - Still more comments to emphasize the relationship to PCA
    #
    # -  V1.5, Dec. 24 1997
    #   - The sign of each row of B is determined by letting the first element be
    #     positive.
    #
    # -  V1.4, Dec. 23 1997
    #   - Minor clean up.
    #   - Added a verbose switch
    #   - Added the sorting of the rows of B in order to fix in some reasonable way the
    #     permutation indetermination.  See note 2) below.
    #
    # -  V1.3, Nov.  2 1997
    #   - Some clean up.  Released in the public domain.
    #
    # -  V1.2, Oct.  5 1997
    #   - Changed random picking of the cumulant matrix used for initialization to a
    #     deterministic choice.  This is not because of a better rationale but to make
    #     the ouput (almost surely) deterministic.
    #   - Rewrote the joint diag. to take more advantage of Matlab"s tricks.
    #   - Created more dummy variables to combat Matlab"s loose memory management.
    #
    # -  V1.1, Oct. 29 1997.
    #    Made the estimation of the cumulant matrices more regular. This also corrects a
    #    buglet...
    #
    # -  V1.0, Sept. 9 1997. Created.
    #
    # Main references:
    # @article{CS-iee-94,
    #  title 	= "Blind beamforming for non {G}aussian signals",
    #  author       = "Jean-Fran\c{c}ois Cardoso and Antoine Souloumiac",
    #  HTML 	= "ftp://sig.enst.fr/pub/jfc/Papers/iee.ps.gz",
    #  journal      = "IEE Proceedings-F",
    #  month = dec, number = 6, pages = {362-370}, volume = 140, year = 1993}
    #
    #
    # @article{JADE:NC,
    #  author  = "Jean-Fran\c{c}ois Cardoso",
    #  journal = "Neural Computation",
    #  title   = "High-order contrasts for independent component analysis",
    #  HTML    = "http://www.tsi.enst.fr/~cardoso/Papers.PS/neuralcomp_2ppf.ps",
    #  year    = 1999, month =	jan,  volume =	 11,  number =	 1,  pages =  "157-192"}
    #
    #
    #  Notes:
    #  ======
    #
    #  Note 1) The original Jade algorithm/code deals with complex signals in Gaussian
    #  noise white and exploits an underlying assumption that the model of independent
    #  components actually holds.  This is a reasonable assumption when dealing with
    #  some narrowband signals.  In this context, one may i) seriously consider dealing
    #  precisely with the noise in the whitening process and ii) expect to use the small
    #  number of significant eigenmatrices to efficiently summarize all the 4th-order
    #  information.  All this is done in the JADE algorithm.
    #
    #  In *this* implementation, we deal with real-valued signals and we do NOT expect
    #  the ICA model to hold exactly.  Therefore, it is pointless to try to deal
    #  precisely with the additive noise and it is very unlikely that the cumulant
    #  tensor can be accurately summarized by its first n eigen-matrices.  Therefore,
    #  we consider the joint diagonalization of the *whole* set of eigen-matrices.
    #  However, in such a case, it is not necessary to compute the eigenmatrices at all
    #  because one may equivalently use `parallel slices" of the cumulant tensor.  This
    #  part (computing the eigen-matrices) of the computation can be saved: it suffices
    #  to jointly diagonalize a set of cumulant matrices.  Also, since we are dealing
    #  with reals signals, it becomes easier to exploit the symmetries of the cumulants
    #  to further reduce the number of matrices to be diagonalized. These
    #  considerations, together with other cheap tricks lead to this version of JADE
    #  which is optimized (again) to deal with real mixtures and to work `outside the
    #  model'.  As the original JADE algorithm, it works by minimizing a `good set' of
    #  cumulants.
    #
    #  Note 2) The rows of the separating matrix B are resorted in such a way that the
    #  columns of the corresponding mixing matrix A=pinv(B) are in decreasing order of
    #  (Euclidian) norm.  This is a simple, `almost canonical" way of fixing the
    #  indetermination of permutation.  It has the effect that the first rows of the
    #  recovered signals (ie the first rows of B*X) correspond to the most energetic
    #  *components*.  Recall however that the source signals in S=B*X have unit
    #  variance.  Therefore, when we say that the observations are unmixed in order of
    #  decreasing energy, this energetic signature is to be found as the norm of the
    #  columns of A=pinv(B) and not as the variances of the separated source signals.
    #
    #  Note 3) In experiments where JADE is run as B=jadeR(X,m) with m varying in range
    #  of values, it is nice to be able to test the stability of the decomposition.  In
    #  order to help in such a test, the rows of B can be sorted as described above. We
    #  have also decided to fix the sign of each row in some arbitrary but fixed way.
    #  The convention is that the first element of each row of B is positive.
    #
    #  Note 4) Contrary to many other ICA algorithms, JADE (or least this version) does
    #  not operate on the data themselves but on a statistic (the full set of 4th order
    #  cumulant). This is represented by the matrix CM below, whose size grows as m^2 x
    #  m^2 where m is the number of sources to be extracted (m could be much smaller
    #  than n).  As a consequence, (this version of) JADE will probably choke on a
    #  `large' number of sources. Here `large' depends mainly on the available memory
    #  and could be something like 40 or so.  One of these days, I will prepare a
    #  version of JADE taking the `data' option rather than the `statistic' option.

    # Notes on translation (GB):
    # =========================
    #
    # Note 1) The function jadeR is a relatively literal translation from the original
    # MATLABcode. I haven't really looked into optimizing it for NumPy. If you have any
    # time to look at this and good ideas, let me know.
    #
    # Note 2) A test module that compares NumPy output with Octave (MATLAB
    # clone) output of the original MATLAB script is available

class JadeICA(TransformerMixin, BaseEstimator):
    """Perform blind source separation using joint diagonalization."""

    def __init__(self, *, n_components: Optional[int] = None) -> None:
        """Perform a blind signal separation using joint diagonalization.

        Parameters
        ----------
        n_components : int, default=None
            Number of signals to extract. `None` assumes all components
        """
        super().__init__()

        self.n_components: Optional[int] = n_components
        self.mean_: NDArray[(Any, ...), Float]
        self.components_: NDArray[(Any, ...), Float]

    def fit(
        self,
        arr: NDArray[(Any, ...), Float],
        y: Optional[NDArray[(Any, ...), Float]] = None,
    ) -> JadeICA:
        """Calculate the unmixing matrix.

        Parameters
        ----------
        arr : NDArray
            mixed signal array
        y : NDArray, optional
            unused

        Returns
        -------
        self

        Raises
        ------
        IndexError
            if :math:`m > n_features`
        """
        # GB: we do some checking of the input arguments and copy data to new variables to
        # avoid messing with the original input. We also require double precision (float64)
        # and a numpy matrix type for X.
        origtype = arr.dtype  # remember to return matrix B of the same type
        arr = check_array(arr, dtype=float)

        # GB: n is number of input signals, T is number of samples
        _, n_features = arr.shape

        if self.n_components is None:
            self.n_components = n_features  # Number of sources defaults to # of sensors
        elif self.n_components > n_features:
            raise IndexError(
                f"More sources ({self.n_components}) than sensors ({n_features})"
            )

        self.mean_ = arr.mean(axis=0)
        self.components_ = _jade(arr, n_components=self.n_components).astype(origtype)
        return self

    def transform(self, arr: NDArray[(Any, ...), Float]) -> NDArray[(Any, ...), Float]:
        """Project the unmixing matrix onto `arr` to give independent signals.

        Parameters
        ----------
        arr : NDArray
            Unmixed signals

        Returns
        -------
        NDArray
            Unmixed signal array
        """
        check_is_fitted(self)

        arr -= self.mean_
        signal: NDArray[(Any, ...), Float] = arr @ self.components_.T
        return signal

    def fit_transform(
        self,
        arr: NDArray[(Any, ...), Float],
        y: Optional[NDArray[(Any, ...), Float]] = None,
        **fit_params: str,
    ) -> NDArray[(Any, ...), Float]:
        """Determine the independent signals using joint diagonalization.

        Parameters
        ----------
        arr : NDArray
            Mixed signal array
        y : NDArray, optional
            unused
        fit_params : dict
            unused

        Returns
        -------
        NDArray
            Unmixed signal array

        Raises
        ------
        IndexError
            if :math:`m > n_features`
        """
        # GB: we do some checking of the input arguments and copy data to new variables to
        # avoid messing with the original input. We also require double precision (float64)
        # and a numpy matrix type for X.
        origtype = arr.dtype  # remember to return matrix B of the same type
        arr1 = check_array(arr, dtype=float)

        # GB: n is number of input signals, T is number of samples
        _, n_features = arr.shape

        if self.n_components is None:
            self.n_components = n_features  # Number of sources defaults to # of sensors
        elif self.n_components > n_features:
            raise IndexError(
                f"More sources ({self.n_components}) than sensors ({n_features})"
            )

        self.mean_ = arr.mean(axis=0)
        self.components_ = _jade(arr1, n_components=self.n_components).astype(origtype)

        arr -= self.mean_
        signal: NDArray[(Any, ...), Float] = arr @ self.components_.T
        return signal

    def inverse_transform(
        self, arr: NDArray[(Any, ...), Float]
    ) -> NDArray[(Any, ...), Float]:
        """Find the original mixed signals.

        Parameters
        ----------
        arr : NDArray
            Unmixed signal array

        Raises
        ------
        NotImplementedError
            If called because Jade does not provide inversion
        """
        raise NotImplementedError(
            "Inverse transformation is currently not implemented."
        )


import requests

#baseinfo
#requests默认使用session对象，是为了在多次和服务器端交互中保留绘画的信息，例如cookies
ua = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
region_url = ['https://bbs.hupu.com/','https://nba.hupu.com/']
session = requests.Session()

with session:
    for url in region_url:
        response = session.get(url,headers={'User-Agent':ua})
        with response:
            print(type(response))
            print(response.url)
            print(response.status_code)
            print(response.request.headers)
            print(response.cookies)


# Logistic Regression with Prior.

# ============= Imports =============
from Numpy_and_mnist.data.data import *
import autograd.numpy as np
from autograd.scipy.misc import logsumexp
from tqdm import tqdm

# ============= Data =============
N_data, train_images, train_labels, test_images, test_labels = load_mnist()
train_digits = train_images[:1000]
train_labels = train_labels[:1000]
test_digits = test_images[:100]
test_labels= test_labels[:100]

# ============= Code =============

def gradient(theta, inputs, labels):
    probabilities = np.exp(logprobabilities(theta, inputs))
    repor = labels - probabilities
    return (np.dot(repor.T, inputs))

def predicted_ratio(theta, inputs, classes):
    pred_true = np.argmax(logprobabilities(theta, inputs), axis=1)
    rate = sum(np.array(classes == pred_true, dtype=int))/pred_true.shape[0]
    return rate

def logprobabilities(theta, inputs, use_prior=False):
    sigma_squared = 1e2
    probability = np.dot(inputs, theta.T)
    prior = 0.0 if not use_prior else np.sum(np.square(theta)/(2*sigma_squared))
    return probability-logsumexp(probability, axis=1, keepdims=True)-prior

def class_labels(labels):
    return np.argmax(labels, axis=1)

def main():
    inputs, labels = np.array(train_digits), np.array(train_labels)
    theta = np.zeros((10, 784), dtype=np.float32)
    loglikelihoods, ratios = [], []
    optimal_i, optimal_thetas = None, None

    for i in tqdm(range(1000)):
        theta += 0.1 * gradient(theta, inputs, labels)
        ratios.append(predicted_ratio(theta, inputs, class_labels(train_labels)))
        lgl = -np.sum(logprobabilities(theta, inputs, False) * labels)
        loglikelihoods.append(lgl)
        if max(loglikelihoods) == lgl:
            optimal_i, optimal_thetas = i, theta

    for idx, (x, y) in enumerate([(train_digits, train_labels), (test_digits, test_labels)]):
        procedure = ['Train', 'Test'][idx]
        lgl = np.sum(logprobabilities(optimal_thetas, x) * y)
        print('{} Accuracy is at {}'.format(procedure, predicted_ratio(optimal_thetas, x, class_labels(y))))
        print('{} Average predictive log-likelihood :{}'.format(procedure, lgl/x.shape[0]))
        save_images(optimal_thetas, "weights.png")
if __name__ == '__main__':
    main()