import csv
import traceback
from datetime import datetime, timezone

from django.db import DatabaseError, connection
from django.utils.timezone import is_aware, make_aware
from git.exc import GitCommandError
from github import Repository
from gitlab.v4.objects import projects
from pydriller import Repository as PyDrillerRepository
from pydriller.domain.commit import Commit as PyDrillerCommit

from .models import Author, Commit, CommittedFile, MergeRequest, ensure_repository
from .sql import QUERY_SQL, STATS_SQL
from .utils import (
    display_url,
    gitlab_ts_to_datetime,
    log,
    normalize_branches,
    redact_http_url,
    should_exclude_from_stats,
)

GITLAB_TIMETSAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"

#
# notes about timezone handling
#
#  1. USE_TZ is set to True, which means DateTime field in models are required to have a tzinfo component
#  2. For uniformity, we use timezone.utc for all datetime objects, which means some conversion is needed here
#  3. Github APIs returns native datetime object and the timezone is assumed to be in UTC
#  4. Gitlab APIs returns timetime as stringm with "Z" at the end, also in UTC
#  5. In model Repository, last_commit_at value is from Commit object, which is a tz aware datetime object
#     (need to validate accuracy here?). The last_indexed_at created in this module, UTC is used.
#

def index_commits(
    clone_url: str, git_repo_type: str = "", show_progress: bool = False, index_all: bool = False, timeout: int = 28800
) -> int:
    n_branch_updates, n_new_commits = 0, 0
    log_url = display_url(redact_http_url(clone_url))

    try:
        repo = ensure_repository(clone_url, git_repo_type)
        if repo is None:
            log(f"### cannot create repostitory object for {log_url}")
            return 0

        if repo.is_active is False:
            log(f"### skipping inactive repository {log_url}")
            return 0

        log(f"starting to index {log_url}")
        start_t = datetime.now()

        # use list comprehension to force loading of commits
        old_commits = {}
        for commit in repo.commits.all():
            old_commits[commit.sha] = commit

        if repo.last_commit_at and not index_all:
            index_since = repo.last_commit_at
        else:
            index_since = datetime.min

        for git_commit in PyDrillerRepository(
            clone_url,
            include_refs=True,
            include_remotes=True,
            since=index_since,
        ).traverse_commits():
            # impose some timeout to avoid spending tons of time on very large repositories
            if (datetime.now() - start_t).seconds > timeout:  # pragma: no cover
                print(f"### indexing not done after {timeout} seconds, aborting {log_url}")
                break

            if git_commit.hash in old_commits:
                # we've seen this commit before, just compare branches and update
                # if needed
                commit = old_commits[git_commit.hash]
                new_branches = normalize_branches(git_commit.branches)
                if new_branches != commit.branches:
                    commit.branches = new_branches
                    commit.save()
                    n_branch_updates += 1
            else:
                # check if the same repo is already linked to another repo
                try:
                    commit = Commit.objects.get(sha=git_commit.hash)
                except Commit.DoesNotExist:
                    commit = _new_commit_(git_commit)
                repo.commits.add(commit)

                if repo.last_commit_at is None or (commit.created_at and commit.created_at > repo.last_commit_at):
                    repo.last_commit_at = commit.created_at

                n_new_commits += 1

            nn = n_new_commits + n_branch_updates
            if nn > 0 and nn % 200 == 0 and show_progress:
                log(f"indexed {n_new_commits:5,} new commits and {n_branch_updates:5,} branch updates")

        if (n_new_commits + n_branch_updates) > 0:
            log(f"indexed {n_new_commits:5,} new commits and {n_branch_updates:5,} branch updates in the repository")

        repo.last_indexed_at = datetime.utcnow().replace(tzinfo=timezone.utc)
        repo.save()

        return n_new_commits + n_branch_updates

    except GitCommandError as e:
        print(f"{e._cmdline} returned {e.stderr} for {log_url}")
    except DatabaseError as e:
        exc = traceback.format_exc()
        print(f"DatabaseError indexing repository {log_url} => {str(e)}\n{exc}")
    except Exception as e:  # pragma: no cover
        exc = traceback.format_exc()
        print(f"Exception indexing repository {log_url} => {str(e)}\n{exc}")

    return 0

def index_gitlab_merge_requests(project: projects.Project, show_progress: bool = False) -> int:
    n_requests = 0
    log_url = display_url(project.http_url_to_repo)

    try:
        repo = ensure_repository(project.http_url_to_repo, "gitlab")
        if repo is None:
            log(f"### cannot create repostitory object for {log_url}")
            return 0

        if repo.is_active is False:
            log(f"### skipping inactive repository {log_url}")
            return 0

        log(f"starting to index merge requests for {log_url}")

        merge_requests = project.mergerequests.list(all=True)
        for mr in merge_requests:
            mr_id = mr.get_id()

            if mr.state not in ["closed", "merged"]:
                # index only closed or merged merge requests
                continue

            db_obj = MergeRequest.objects.filter(request_id=mr_id, repo=repo).first()
            if db_obj is not None:
                # merge request already indexed
                continue

            is_merged, merged_by_username, merge_commit_sha = False, None, None
            if mr.state == "merged":
                is_merged, merged_by_username = True, mr.merge_user["username"]
                merge_commit_sha = mr.squash_commit_sha if mr.squash else mr.merge_commit_sha

            MergeRequest.objects.create(
                repo=repo,
                request_id=mr_id,
                state=mr.state,
                source_branch=mr.source_branch,
                target_branch=mr.target_branch,
                source_sha=mr.sha,
                merge_sha=merge_commit_sha,
                created_at=gitlab_ts_to_datetime(mr.created_at),
                merged_at=gitlab_ts_to_datetime(mr.merged_at),
                updated_at=gitlab_ts_to_datetime(mr.updated_at),
                # first_comment_at = models.CharField(max_length=32)
                is_merged=is_merged,
                merged_by_username=merged_by_username,
            ).save()

            n_requests += 1

            if n_requests > 0 and n_requests % 50 == 0 and show_progress:
                log(f"indexed {n_requests:3,} merge requests ")

        if n_requests > 0:
            log(f"indexed {n_requests:3,} merge requests in the repository")

        return n_requests

    except GitCommandError as e:
        print(f"{e._cmdline} returned {e.stderr} for {log_url}")
    except DatabaseError as e:
        exc = traceback.format_exc()
        print(f"DatabaseError indexing repository {log_url} => {str(e)}\n{exc}")
    except Exception as e:  # pragma: no cover
        exc = traceback.format_exc()
        print(f"Exception indexing repository {log_url} => {str(e)}\n{exc}")

    return 0

def index_github_pull_requests(git_repo: Repository.Repository, show_progress: bool = False) -> int:
    n_requests = 0
    log_url = display_url(git_repo.clone_url)

    try:
        repo = ensure_repository(git_repo.clone_url, "github")
        if repo is None:
            log(f"### cannot create repostitory object for {log_url}")
            return 0

        if repo.is_active is False:
            log(f"### skipping inactive repository {log_url}")
            return 0

        log(f"starting to index merge requests for {log_url}")

        pull_requests = git_repo.get_pulls(state="closed")
        for pr in pull_requests:
            db_obj = MergeRequest.objects.filter(request_id=pr.number, repo=repo).first()
            if db_obj is not None:
                # merge request already indexed
                continue

            # TODO: pr.created_at is a naive datetime object, timezone is assumed to be UTC
            MergeRequest.objects.create(
                repo=repo,
                request_id=pr.number,
                state=pr.state,
                source_branch=pr.head.ref,
                target_branch=pr.base.ref,
                source_sha=pr.head.sha,  # does this value change when new commits are added to source branch?
                merge_sha=pr.merge_commit_sha,
                created_at=pr.created_at.replace(tzinfo=timezone.utc),
                merged_at=pr.merged_at.replace(tzinfo=timezone.utc),
                updated_at=pr.updated_at.replace(tzinfo=timezone.utc),
                # first_comment_at = ???
                is_merged=pr.merged,
                merged_by_username=pr.merged_by.login if pr.merged else None,
            ).save()

            n_requests += 1

            if n_requests > 0 and n_requests % 50 == 0 and show_progress:
                log(f"indexed {n_requests:3,} merge requests ")

        if n_requests > 0:
            log(f"indexed {n_requests:3,} merge requests in the repository")

        return n_requests

    except GitCommandError as e:
        print(f"{e._cmdline} returned {e.stderr} for {log_url}")
    except DatabaseError as e:
        exc = traceback.format_exc()
        print(f"DatabaseError indexing repository {log_url} => {str(e)}\n{exc}")
    except Exception as e:  # pragma: no cover
        exc = traceback.format_exc()
        print(f"Exception indexing repository {log_url} => {str(e)}\n{exc}")

    return 0

def update_commit_stats() -> None:
    """update stats at commit level"""
    log("updating commit stats")
    cursor = connection.cursor()
    for statement in STATS_SQL:
        try:
            cursor.execute(statement)
        except DatabaseError as e:
            exc = traceback.format_exc()
            print(f"Exception execute statement {statement} => {str(e)}\n{exc}")

def _new_commit_(git_commit: PyDrillerCommit) -> Commit:
    author, created = Author.objects.get_or_create(
        name=git_commit.committer.name.lower(),
        email=git_commit.committer.email.lower(),
    )
    if created:
        author.real_name = author.name
        author.real_email = author.email
        author.save()

    if is_aware(git_commit.committer_date):
        commit_dt = git_commit.committer_date
    else:
        commit_dt = make_aware(git_commit.committer_date)

    commit = Commit(
        sha=git_commit.hash,
        message=git_commit.msg[:2048],  # some commits has super long message, e.g. squash merge
        author=author,
        is_merge=git_commit.merge,
        branches=normalize_branches(git_commit.branches),
        n_lines=git_commit.lines,
        n_files=git_commit.files,
        n_insertions=git_commit.insertions,
        n_deletions=git_commit.deletions,
        # comment to save some time. metrics not used for now
        # dmm_unit_size=git_commit.dmm_unit_size,
        # dmm_unit_complexity=git_commit.dmm_unit_complexity,
        # dmm_unit_interfacing=git_commit.dmm_unit_interfacing,
        created_at=commit_dt,
    )
    commit.save()

    for mod in git_commit.modified_files:
        file_path = mod.new_path or mod.old_path
        flag = should_exclude_from_stats(file_path)
        new_file = CommittedFile(
            commit_sha=git_commit.hash,
            change_type=str(mod.change_type).split(".")[1],  # enum ModificationType.ADD => "ADD"
            file_path=file_path,
            file_name=mod.filename,
            n_lines_added=mod.added_lines,
            n_lines_deleted=mod.deleted_lines,
            n_lines_changed=mod.added_lines + mod.deleted_lines,
            n_lines_of_code=mod.nloc if mod.nloc else 0,
            n_methods=len(mod.methods),
            n_methods_changed=len(mod.changed_methods),
            is_on_exclude_list=flag,
            is_superfluous=flag,
            commit=commit,
        )
        new_file.save()

    return commit

def export_all_data(csv_file: str) -> None:
    cursor = connection.cursor()
    result = cursor.execute(QUERY_SQL["all_commit_data"])
    columns = [col[0] for col in cursor.description]

    n_rows = 0
    with open(csv_file, "w", newline="") as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(columns)
        for row in result.fetchall():
            n_rows += 1
            writer.writerow(row)
        log(f"exported {n_rows} rows to {csv_file}")


"""Transition and sequence analysis of neighborhood change."""

from time import time
from warnings import warn

import geopandas as gpd
import numpy as np
import pandas as pd
from giddy.markov import Markov, Spatial_Markov
from giddy.sequence import Sequence
from libpysal.weights import Voronoi, lag_categorical
from libpysal.weights.contiguity import Queen, Rook
from libpysal.weights.distance import KNN, DistanceBand, Kernel
from numpy.random import PCG64, SeedSequence, default_rng
from sklearn.cluster import AgglomerativeClustering
from tqdm.auto import tqdm

Ws = {
    "queen": Queen,
    "rook": Rook,
    "voronoi": Voronoi,
    "knn": KNN,
    "kernel": Kernel,
    "distanceband": DistanceBand,
}

def transition(
    gdf,
    cluster_col,
    temporal_index="year",
    unit_index="geoid",
    w_type="rook",
    w_options=None,
    permutations=0,
):
    """
    Model neighborhood change as a discrete spatial Markov process.

    Parameters
    ----------
    gdf : geopandas.GeoDataFrame or pandas.DataFrame
        Long-form geopandas.GeoDataFrame or pandas.DataFrame containing neighborhood
        attributes with a column defining neighborhood clusters.
    cluster_col : string or int
        Column name for the neighborhood segmentation, such as
        "ward", "kmeans", etc.
    temporal_index : string, optional
        Column defining time and or sequencing of the long-form data.
        Default is "year".
    unit_index : string, optional
        Column identifying the unique id of spatial units.
        Default is "geoid".
    w_type : string, optional
        Type of spatial weights type ("rook", "queen", "knn" or
        "kernel") to be used for spatial structure. Default is
        None, if non-spatial Markov transition rates are desired.
    w_options : dict
        additional options passed to a libpysal weights constructor
        (e.g. `k` for a KNN weights matrix)
    permutations : int, optional
        number of permutations for use in randomization based
        inference (the default is 0).

    Returns
    --------
    mar : giddy.markov.Markov instance or giddy.markov.Spatial_Markov
        if w_type=None, a classic Markov instance is returned.
        if w_type is given, a Spatial_Markov instance is returned.

    Examples
    --------
    >>> from geosnap import Community
    >>> columbus = Community.from_ltdb(msa_fips="18140")
    >>> columbus1 = columbus.cluster(columns=['median_household_income',
    ... 'p_poverty_rate', 'p_edu_college_greater', 'p_unemployment_rate'],
    ... method='ward', n_clusters=6)
    >>> gdf = columbus1.gdf
    >>> a = transition(gdf, "ward", w_type="rook")
    >>> a.p
    array([[0.79189189, 0.00540541, 0.0027027 , 0.13243243, 0.06216216,
        0.00540541],
       [0.0203252 , 0.75609756, 0.10569106, 0.11382114, 0.        ,
        0.00406504],
       [0.00917431, 0.20183486, 0.75229358, 0.01834862, 0.        ,
        0.01834862],
       [0.1959799 , 0.18341709, 0.00251256, 0.61809045, 0.        ,
        0.        ],
       [0.32307692, 0.        , 0.        , 0.        , 0.66153846,
        0.01538462],
       [0.09375   , 0.0625    , 0.        , 0.        , 0.        ,
        0.84375   ]])
    >>> a.P[0]
    array([[0.82119205, 0.        , 0.        , 0.10927152, 0.06622517,
        0.00331126],
       [0.14285714, 0.57142857, 0.14285714, 0.14285714, 0.        ,
        0.        ],
       [0.5       , 0.        , 0.5       , 0.        , 0.        ,
        0.        ],
       [0.21428571, 0.14285714, 0.        , 0.64285714, 0.        ,
        0.        ],
       [0.18918919, 0.        , 0.        , 0.        , 0.78378378,
        0.02702703],
       [0.28571429, 0.        , 0.        , 0.        , 0.        ,
        0.71428571]])
    """
    if not w_options:
        w_options = {}
    assert (
        unit_index in gdf.columns
    ), f"The unit_index ({unit_index}) column is not in the geodataframe"
    gdf_temp = gdf.copy().reset_index()
    df = gdf_temp[[unit_index, temporal_index, cluster_col]]
    df_wide = df.pivot(
        index=unit_index, columns=temporal_index, values=cluster_col
    ).dropna()
    y = df_wide.values
    if w_type is None:
        mar = Markov(y)  # class markov modeling
    else:
        geoms = gdf_temp.groupby(unit_index).first()[gdf_temp.geometry.name]
        gdf_wide = df_wide.merge(geoms, left_index=True, right_index=True)
        w = Ws[w_type].from_dataframe(gpd.GeoDataFrame(gdf_wide), **w_options)
        w.transform = "r"
        mar = Spatial_Markov(
            y, w, permutations=permutations, discrete=True, variable_name=cluster_col
        )
    return mar

def sequence(
    gdf,
    cluster_col,
    seq_clusters=5,
    subs_mat=None,
    dist_type=None,
    indel=None,
    temporal_index="year",
    unit_index="geoid",
):
    """
    Pairwise sequence analysis and sequence clustering.

    Dynamic programming if optimal matching.

    Parameters
    ----------
    gdf             : geopandas.GeoDataFrame or pandas.DataFrame
                      Long-form geopandas.GeoDataFrame or pandas.DataFrame containing neighborhood
                      attributes with a column defining neighborhood clusters.
    cluster_col     : string or int
                      Column name for the neighborhood segmentation, such as
                      "ward", "kmeans", etc.
    seq_clusters    : int, optional
                      Number of neighborhood sequence clusters. Agglomerative
                      Clustering with Ward linkage is now used for clustering
                      the sequences. Default is 5.
    dist_type       : string
                      "hamming": hamming distance (substitution only
                      and its cost is constant 1) from sklearn.metrics;
                      "markov": utilize empirical transition
                      probabilities to define substitution costs;
                      "interval": differences between states are used
                      to define substitution costs, and indel=k-1;
                      "arbitrary": arbitrary distance if there is not a
                      strong theory guidance: substitution=0.5, indel=1.
                      "tran": transition-oriented optimal matching. Sequence of
                      transitions. Based on :cite:`Biemann:2011`.
    subs_mat        : array
                      (k,k), substitution cost matrix. Should be hollow (
                      0 cost between the same type), symmetric and non-negative.
    indel           : float, optional
                      insertion/deletion cost.
    temporal_index        : string, optional
                      Column defining time and or sequencing of the long-form data.
                      Default is "year".
    unit_index          : string, optional
                      Column identifying the unique id of spatial units.
                      Default is "geoid".

    Returns
    --------
    gdf_temp        : geopandas.GeoDataFrame or pandas.DataFrame
                      geopandas.GeoDataFrame or pandas.DataFrame with a new column for sequence
                      labels.
    df_wide         : pandas.DataFrame
                      Wide-form DataFrame with k (k is the number of periods)
                      columns of neighborhood types and 1 column of sequence
                      labels.
    seq_dis_mat     : array
                      (n,n), distance/dissimilarity matrix for each pair of
                      sequences

    Examples
    --------
    >>> from geosnap.data import Community
    >>> columbus = Community.from_ltdb(msa_fips="18140")
    >>> columbus1 = columbus.cluster(columns=['median_household_income',
    ... 'p_poverty_rate', 'p_edu_college_greater', 'p_unemployment_rate'],
    ... method='ward', n_clusters=6)
    >>> gdf = columbus1.gdf
    >>> gdf_new, df_wide, seq_hamming = Sequence(gdf, dist_type="hamming")
    >>> seq_hamming.seq_dis_mat[:5, :5]
    array([[0., 3., 4., 5., 5.],
           [3., 0., 3., 3., 3.],
           [4., 3., 0., 2., 2.],
           [5., 3., 2., 0., 0.],
           [5., 3., 2., 0., 0.]])

    """
    assert (
        unit_index in gdf.columns
    ), f"The unit_index ({unit_index}) column is not in the geodataframe"
    gdf_temp = gdf.copy().reset_index()
    df = gdf_temp[[unit_index, temporal_index, cluster_col]]
    df_wide = (
        df.pivot(index=unit_index, columns=temporal_index, values=cluster_col)
        .dropna()
        .astype("int")
    )
    y = df_wide.values
    seq_dis_mat = Sequence(
        y, subs_mat=subs_mat, dist_type=dist_type, indel=indel, cluster_type=cluster_col
    ).seq_dis_mat
    model = AgglomerativeClustering(n_clusters=seq_clusters).fit(seq_dis_mat)
    name_seq = dist_type + "_%d" % (seq_clusters)
    df_wide[name_seq] = model.labels_
    gdf_temp = gdf_temp.merge(df_wide[[name_seq]], left_on=unit_index, right_index=True)
    gdf_temp = gdf_temp.reset_index(drop=True)

    return gdf_temp, df_wide, seq_dis_mat

def predict_markov_labels(
    gdf,
    unit_index="geoid",
    temporal_index="year",
    cluster_col=None,
    w_type="rook",
    w_options=None,
    base_year=None,
    new_colname=None,
    time_steps=1,
    increment=None,
    seed=None,
    verbose=True,
):
    """Predict neighborhood labels based on spatial Markov transition model

    Parameters
    ----------
    gdf : geopandas.GeoDataFrame
        a long-form geodataframe with a column of labels to be simulated with a spatial Markov model
    unit_index : str,
        column on dataframe that identifies unique geographic units, by default "geoid"
    temporal_index : str
        column on dataframe that identifies unique time periods, by default "year"
    cluster_col : str
        column on the dataframe that stores cluster or other labels to be simulated
    w_type : str, optional
        type of spatial weights matrix to include in the transition model, by default "queen"
    w_options : dict, optional
        additional keyword arguments passed to the libpysal weights constructor
    base_year : int or str, optional
        the year from which to begin simulation (i.e. the set of labels to define the first
        period of the Markov sequence)
    new_colname : str, optional
        new column name to store predicted labels under. Defaults to "predicted"
    time_steps : int, optional
        the number of time-steps to simulate, by default 1
    increment : str or int, optional
        styled increment each time-step referrs to. For example, for a model fitted to decadal
        Census data, each time-step refers to a period of ten years, so an increment of 10 ensures
        that the temporal index aligns appropriately with the time steps being simulated
    verbose: bool
        if true, print warnings from the label sampling process

    Returns
    -------
    geopandas.GeoDataFrame
        long-form geodataframe with predicted cluster labels stored in the `new_colname` column
    """
    crs = gdf.crs
    np.random.seed(seed)
    if not new_colname:
        new_colname = "predicted"
    if not w_options:
        w_options = {}

    assert (
        cluster_col and cluster_col in gdf.columns
    ), f"The input dataframe has no column named {cluster_col}"

    assert (
        base_year
    ), "Missing `base_year`. You must provide an initial time point with labels to begin simulation"
    assert (
        base_year in gdf[temporal_index].unique().tolist()
    ), "A set of observations with `temporal_index`==`base_year` must be included in the gdf"

    gdf = gdf.copy()
    gdf = gdf.dropna(subset=[cluster_col]).reset_index(drop=True)
    t = transition(
        gdf,
        cluster_col,
        w_type=w_type,
        unit_index=unit_index,
        temporal_index=temporal_index,
        w_options=w_options,
    )

    if time_steps == 1:

        gdf = gdf[gdf[temporal_index] == base_year].reset_index(drop=True)
        w = Ws[w_type].from_dataframe(gdf, **w_options)
        predicted = _draw_labels(w, gdf, cluster_col, t, unit_index, verbose)
        if new_colname:
            predicted = predicted.rename(columns={cluster_col: new_colname})
        return predicted

    else:
        assert (
            increment
        ), "You must set the `increment` argument to simulate multiple time steps"
        predictions = []
        gdf = gdf[gdf[temporal_index] == base_year]
        gdf = gdf[[unit_index, cluster_col, temporal_index, gdf.geometry.name]]
        current_time = base_year + increment
        gdf = gdf.dropna(subset=[cluster_col]).reset_index(drop=True)
        w = Ws[w_type].from_dataframe(gdf, **w_options)
        predictions.append(gdf)

        for step in range(1, time_steps + 1):
            # use the last known set of labels  to get the spatial context for each geog unit
            gdf = predictions[step - 1].copy()

            predicted = _draw_labels(w, gdf, cluster_col, t, unit_index, verbose)
            predicted[temporal_index] = current_time
            predictions.append(predicted)
            current_time += increment
        gdf = gpd.GeoDataFrame(pd.concat(predictions), crs=crs)
        if new_colname:
            gdf = gdf.rename(columns={cluster_col: new_colname})
        return gdf

def _draw_labels(w, gdf, cluster_col, markov, unit_index, verbose):
    """Draw a random class label from the spatially-conditioned transition rates.

    Parameters
    ----------
    w : libpysal.weights.W
        spatial weights object
    gdf : geopandas.GeoDataFrame
        geodataframe of observations with class/cluster labels as a column
    cluster_col : string
        the column on the dataframe that holds class labels
    markov : giddy.Spatial_Markov
        an instance of a Spatial_Markov class
    unit_index : string
        the column on the dataframe that identifies unique spatial units

    Returns
    -------
    geopandas.GeoDataFrame
        long-form geodataframe with predicted cluster labels stored in the `new_colname` column
    """
    gdf = gdf.copy()
    gdf = gdf.dropna(subset=[cluster_col])
    lags = lag_categorical(w, gdf[cluster_col].values)
    clusters = gdf.reset_index()[cluster_col].astype(str).values
    classes = markov.classes.astype(str)
    cluster_idx = dict(zip(classes, list(range(len(classes)))))

    labels = {}
    for i, lag in enumerate(lags):
        #  select the transition matrix using the label of unit's spatial lag
        spatial_context = np.nan_to_num(markov.P, posinf=0.0, neginf=0.0)[
            cluster_idx[lag]
        ]
        #  select the class row from the transition matrix using the unit's label
        probs = spatial_context[cluster_idx[clusters[i]]]
        probs /= (
            probs.sum()
        )  # correct for tolerance, see https://stackoverflow.com/questions/25985120/numpy-1-9-0-valueerror-probabilities-do-not-sum-to-1
        probs = np.nan_to_num(probs.flatten())
        if sum(probs) == 0:
            # in case obs have a modal neighbor never before seen in the model
            # (so all transition probs are 0)
            # fall back to the aspatial transition matrix
            if verbose:
                warn(
                    f"Falling back to aspatial transition rule for unit "
                    f"{gdf[unit_index][i]}"
                )
            probs = markov.p[cluster_idx[clusters[i]]].flatten()

        labels[i] = np.random.choice(classes, p=probs)

    labels = pd.Series(labels, name=cluster_col, index=gdf.index)
    out = gdf[[unit_index, gdf.geometry.name]]
    predicted = gpd.GeoDataFrame(pd.concat([labels, out], axis=1))
    return predicted

def draw_sequence_from_gdf(
    gdf,
    w,
    label_column,
    smk,
    time_column,
    start_time=None,
    time_steps=1,
    increment=None,
    seed=None,
):
    """Draw a set of class labels for each unit in a geodataframe using transition
    probabilities defined by a giddy.Spatial_Markov model and the spatial lag of each
    unit.

    Parameters
    ----------
    gdf : geopandas.GeoDataFrame
        geodataframe of observations with class/cluster labels as a column
    w : libpysal.weights.W
        spatial weights object that defines the neigbhbor graph for each unit.
    label_column : str
        the column on the dataframe that holds class labels
    smk : giddy.Spatial_Markov
        an instance of a Spatial_Markov class created from the giddy package
        or `geosnap.analyze.transition`
    time_column : str
        column on dataframe that identifies unique time periods, by default "year"
    start_time : str, int, or float, optional
        Time period to begin drawing a sequence of labels (must be present in
        `gdf[label_col]`). If None, use the most recent time period given by
        max(gdf[label_column].unique()). By default None
    time_steps : int, optional
        the number of time-steps to simulate (i.e. the number of labels to draw in a
        sequence for each unit), by default 1
    increment : itn, required
        styled increment each time-step referrs to. For example, for a model fitted to
        decadal Census data, each time-step refers to a period of ten years, so an
        increment of 10 ensures that the temporal index aligns appropriately with the
        time steps being simulated
    seed: int
        seed for  reproducible pseudo-random results. Used to create a SeedSequence and
        spawn a set of Generators using PCG64. If None, uses the current time

    Returns
    -------
    geopandas.GeoDataFrame
        long-form geodataframe with the same index as the geodataframe in the
        time period equal the start time (i.e. `gdf[gdf[time_column]==start_time]`).

    """
    assert increment

    steps = [start_time + (increment * step) for step in range(time_steps + 1)]
    steps = steps[1:]
    gdf = gdf.copy()
    gdf = gdf[[label_column, time_column, gdf.geometry.name]].copy()

    dfs = list()

    current_df = gdf[gdf[time_column] == start_time]

    if seed is None:
        seed = int(time())

    base_seq = SeedSequence(seed)
    child_seqs = base_seq.spawn(time_steps)
    generators = [PCG64(seq) for seq in child_seqs]

    dfs.append(current_df)
    # must run in sequence because we need the prior time's spatial lag
    for i, step in enumerate(tqdm(steps)):
        # `steps` and `dfs` are off by one so the ith in dfs is the previous time period
        current_df = dfs[i].copy()
        predicted_labels = _draw_labels_from_gdf(
            current_df, w, label_column, smk, generators[i]
        )
        predicted_df = gdf[gdf[time_column] == start_time]
        # overwrite labels with predictions for the new time period
        predicted_df[label_column] = predicted_labels
        predicted_df[time_column] = step

        dfs.append(predicted_df)
    simulated = pd.concat(dfs)

    return simulated

def _draw_labels_from_gdf(gdf, w, label_column, smk, seed):
    """Draw set of new labels given a geodataframe and a spatial Markov transition model

    Parameters
    ----------
    gdf : geopandas.GeoDataFrame
        a geodataframe defining the study region, including a column holding class labels
    w : libpysal.Weights
        a spatial weights matrix relating units to one another. This should be the same
        W object that was used to fit the Spatial_Markov instance
    label_column : str
        the column in gdf holding class labels
    smk : giddy.Spatial_Markov
        a spatial Markov transition model created by the pysal giddy package
        or geosnap.analyze.transition
    seed: int or nmnumpy.random.Generator instance
        seed passed to np.random.default_rng for reproducible pseudo-random results

    Returns
    -------
    numpy.array
        an array of simulated class labels drawn from the conditional probabilities
        provided in the Spatial_Markov object
    """

    labels = gdf[label_column].astype(str).values
    lags = lag_categorical(w, labels)
    probs = _conditional_probs_from_smk(labels, lags, smk)
    assert len(lags) == len(
        probs
    ), "Lag values and probability vectors are different lengths"
    simulated_labels = _draw_labels_from_probs(
        smk.classes.astype(str), probs=probs, seed=seed
    )

    return simulated_labels

def _draw_labels_from_probs(classes, probs, seed):
    """Draw from a fized set of classes using a vector of probabilities

    Parameters
    ----------
    classes : list-like
        set of class labels
    probs : list-like
        list of probabilities
    seed: int or numpy.random.Generator instance
        seed passed to np.random.default_rng for reproducible pseudo-random results
    verbose : bool, optional
        if true, print a warning when a label cannot be drawn from the
        probability vector, by default True

    Returns
    -------
    list
        list of labels drawn from `classes` with size n_probs
    """
    labels = list()
    rng = default_rng(seed=seed)
    # for each set of probabilities in the array, draw a label
    # this could also be done in parallel. We could also take multiple draws per
    # unit by passing a `size` argument to rng.choice
    for p in probs:
        try:
            label = rng.choice(classes, p=p)
        except ValueError as e:
            raise ValueError from e
        labels.append(label)
    return np.array(labels)

def _conditional_probs_from_smk(labels, lags, smk, fill_null_probs=True):
    """Given a set of existing labels and associated lags, return a vector of
    transition probabilities from a giddy.Spatial_Markov model

    Parameters
    ----------
    labels : list
        array of categorical labels that define a class for each unit
    lags : list
        array of categorical labels that define the context for each unit
    smk : giddy.Spatial_Markov
        a spatial Markov transition model from giddy or geosnap.analyze.transition

    Returns
    -------
    list
        list of lists, where each element of the outer list is
        a set of transition probabilities, and each element of the innerr
        list(s) is the probability of transitioning into class with index i
    """
    classes = smk.classes.astype(str)

    # mapping back to the order each class is given in the smk object
    class_idx = dict(zip(classes, list(range(len(classes)))))

    probs = list()
    # this piece could be parallelized as long as it gets concatenated back in order
    for i in range(len(labels)):
        current_label = labels[i]
        current_class_idx = class_idx[current_label]
        aspatial_p = np.nan_to_num(smk.p[current_class_idx]).flatten()
        aspatial_p /= aspatial_p.sum()

        lag_idx = class_idx[lags[i]]
        conditional_matrix = smk.P[lag_idx]
        p = conditional_matrix[current_class_idx].flatten()
        # correct for tolerance, see https://stackoverflow.com/questions/25985120/numpy-1-9-0-valueerror-probabilities-do-not-sum-to-1
        p = np.nan_to_num(p)
        if p.sum() == 0:
            if fill_null_probs:
                warn(
                    f"No spatial transition rules for {current_label} with conrtext {lags[i]}"
                    "falling back to aspatial transition rules"
                )
                probs.append(aspatial_p)
            else:
                raise ValueError(
                    f"No spatial transition rules for {current_label} with conrtext {lags[i]}"
                )
        else:
            p /= p.sum()

            probs.append(np.nan_to_num(p))

    return probs


from django.shortcuts import render
from django.http import HttpResponse, Http404

from preference.models import Choice
from preference.forms import AddingForm
from preference.forms import TestForm
# Create your views here.

def preferencehomepage(request):
    #http = 'this page should have two choices on it. At some point.'
    #return HttpResponse(http)
    return render(request, 'main.html', {})
def add_choice(request):
    if request.method == 'POST':
        form = AddingForm(request.POST)
        if form.is_valid():
            option1 = form.cleaned_data['option1']
            option2 = form.cleaned_data['option2']
            number = form.cleaned_data['number']
            return HttpResponse('Thank you for submitting your choices')
            number = form.cleaned_data['number']
            C1 = Choice(text = option1)
            C1.save()
            C2 = Choice(text = option2)
            C2.save()
            return HttpResponse('Thank you for submitting your form, your second option was %s' %C2.text )
    else:
        form = AddingForm()
    return render(request,'addchoice.html',{'AddingForm':form})

def contact(request):
    return render(request,'contact.html',{})


# type: ignore
from typing import List

from sqlalchemy.dialects.postgresql import UUID

from .. import db
from .base import BaseModel

class DatasetMeta(BaseModel):
    project_id = db.Column(
        UUID(as_uuid=True), db.ForeignKey("project.id"), nullable=False
    )
    key = db.Column(db.String, nullable=False)
    fullname = db.Column(db.String, nullable=False)
    unit = db.Column(db.String)
    short = db.Column(db.String, nullable=False)
    data_type = db.Column(db.String, nullable=False)

    _options = db.Column(db.String)

    @property
    def options(self):
        if self._options:
            return self._options.split(",")
        return self._options

    @options.setter
    def options(self, options):
        self._options = options

    _default_fields = ["key", "fullname", "unit", "short", "data_type", "options"]
    _hidden_fields = []
    _readonly_fields = []

    @classmethod
    def add_category(
        cls, fullname: str, short: str, unit: str = None, options: List[str] = []
    ):
        ob = cls(
            fullname=fullname,
            short=short,
            unit=unit,
            data_type="categorical",
            options=",".join(options),
        )
        db.session.add(ob)
        db.session.commit()


import sqlite3
import os
from sqlite3.dbapi2 import Cursor, Error

def criarBanco():
    global conexao, Cursor
    if not os.path.exists('estacionamento.db'):
        conexao = sqlite3.connect('estacionamento.db')
        Cursor = conexao.cursor()

        sql_cliente =   'create table cliente('\
                        'id integer primary key autoincrement,'\
                        'nome varchar(50),'\
                        'cpf varchar(20),'\
                        'endereco varchar(40),'\
                        'contato varchar(15))'

        sql_veiculo =   'create table veiculo('\
                        'id_veiculo integer primary key autoincrement,'\
                        'modelo varchar(20),'\
                        'placa varchar(10),'\
                        'cor varchar(20),'\
                        'proprietario varchar(40),'\
                        'id_cliente integer not null,'\
                        'FOREING KEY (id_cliente) REFERENCES cliente(id))'

        Cursor.execute(sql_cliente)
        Cursor.execute(sql_veiculo)
        conexao.commit()

def adicionar_cliente(nome): # parametro
    global conexao, Cursor
    sql_cliente = 'insert into contato(nome, cpf, endereco, contato) values(?,?,?,?)'
    Cursor.execute(sql_cliente, [nome ]) #passamos o comando sql e logo seguindo do parametro em forma de lista

def adicionar_veiculo(id_cliente, veiculos): #id_contato chave estrageira na tabela telefones fones?
    global conexao, Cursor
    sql_veiculo = 'insert into veiculos(modelo, placa, cor, proprietario id_cliente) values(?,?,?)' #id_contato foreid key
    for i in veiculos:
        Cursor.execute(sql_veiculo, [i[0], i[1], id_cliente])

def adicionar_veiculos():
    veiculos = []
    while True:
        modelo = input(f'modelo do carro: ')
        placa = input(f'numero da placa : ')
        cor = input(f'cor do veiculo : ')
        if modelo  == "":
            modelo = None
        if placa == "":
            placa = None
        if cor == "":
            cor = None

        veiculos.append((modelo, placa, cor))
        print(veiculos)
        conf = input('deseja cadastrar outro veiculo para esse cliente? (S/N)')
        if conf in 'Ss':
            pass
        if conf in 'Nn':
            break
    return veiculos

def cadastro():
    global conexao, Cursor
    if conexao == None:
        conexao = sqlite3.connect('estacionamento.db')
        Cursor = conexao.cursor()
    nome = input('entre com o nome do cliente : ')
    if nome == '':
        nome = None
    else:
        res = consultar('cliente', 'id,nome','nome', nome)
        print(res)
        if len(res)>0:
            id = res[0][0]
            res = consultar('veiculos', 'modelo, placa', 'id_cliente', id)
            if len(res)>0:
                print('veiculos cadastrados')
                for i in res:
                    print(i)
            else:
                print('não ha veiculos cadastrados para esse cliente!')
        else:
            fones = adicionar_veiculos()
                ## sequencia sql parar adicionar registros
            try:
                adicionar_cliente(nome)
                id_cliente = Cursor.lastrowid
                adicionar_veiculos(id_cliente)
                conexao.commit()
            except:
                    print('erro de integridade do banco de dados!')

def atualizar_cliente():
    global conexao, Cursor
    if conexao == None:
        conexao = sqlite3.connect('estacionamento.db')
        Cursor = conexao.cursor()
    print('---------------------------------------------------------------')
    print('consultar/atualizar cliente')
    print('---------------------------------------------------------------')
    nome = input('digite o nome do cliente : ')
    sql = f"select * from cliente where nome like '%{nome}%'"## like busca aproximada
    Cursor.execute(sql)
    res = Cursor.fetchall()
    if len(res)>0:
        for i in res:
            print(i)
        resp = int(input('digite o nome do cliente a ser atualizado ou 0 para sair : '))
        if resp != 0:
            id_cliente = resp
            res = consultar('cliente', 'id,cliente', 'id', id_cliente)
            if len(res)>0:
                id_cliente = res[0][0]
                print(f'nomes : {res[0][1]}')
                res2 = consultar('veiculos', 'id_veiculo, modelo,placa', 'id_cliente', id_cliente)
                if len(res2)>0:
                    print('---------------------------------------------')
                    print('                  telefones                  ')
                    print('---------------------------------------------')
                    for i in res2:
                        print(i)
                    menu ('menu',['1-alterar cliente', '2-excluir cliente', '3-incluir veiculo', '4-alterar veiculo', '5-excluir veiculo', '6-sair']) ## tratar opção 2
                    opc = int(input('entre com uma opção : '))
                    if opc == 1:
                        novo_cliente = input('digite o novo cliente : ')
                        sql = "UPTADE cliente SET nome = ? where id = ?"
                        Cursor.execute(sql, [novo_cliente, id_cliente])
                    elif opc == 2:
                        sql = "DELETE FROM cliente WHERE id = ? "
                        Cursor.execute(sql, [id_cliente])
                    elif opc == 3:
                        veiculos = adicionar_cliente()
                        adicionar_veiculos(id_cliente, veiculos )
                    elif opc == 4:
                        id_veiculo = int(input('digite o id do veiculo a ser alterado : '))
                        novo_modelo = input('digite o novo numero : ')
                        nova_placa = input('digite a nova descrição : ')
                        sql = "UPDATE telefones SET numero = ?, descricao = ? WHERE id_fone = ?"
                        Cursor.execute(sql, [novo_modelo, nova_placa, id_veiculo])
                    elif opc == 5:
                        id_veiculo = int(input('digite o id do telefone a ser excluido : '))
                        sql = "DELETE from telefones WHERE id_fone = ?"
                        Cursor.execute(sql, [id_veiculo])
                    elif opc == 6:
                        exit
                else:
                    print('sem veiculos cadastrados!')
                    resp = input('deseja incluir um novo veiculo? (S/N)')
                    if resp in 'Ss':
                        veiculos = adicionar_veiculos()
                        adicionar_cliente(id_cliente, veiculos)
                conexao.commit()
            else:
                print('cliente não cadastrado!')
    else:
        print('cliente não cadastrado!')

def consultar(tabela, campos, campo_cond, valor):
    sql = f'select {campos} from {tabela} where {campo_cond} = ?'
    Cursor = conexao.cursor()
    Cursor.execute(sql, [valor])
    res = Cursor.fetchall()
    return res

def menu(cliente, veiculos):
    print(cliente)
    for i in veiculos:
        print(i)

def main():
    global conexao, Cursor
    conexao = None
    Cursor = None
    criarBanco()
    while True:
        menu ('menu_principal', ['1 - novo cliente', '2 - atualizar_cliente', '3 - sair'])
        opc = int(input('escolha uma opção : '))
        if opc == 1:
            cadastro()
        elif opc == 2:
            atualizar_cliente()
        elif opc == 3:
            break
    if Cursor != None:
        Cursor.close()
    if conexao != None:
        conexao.close()
main()


"""
条形码/二维码的生成：
1.EAN-13产品条形码批量生成
2.产品服务二维码批量生成
"""
import os
import random
from pystrich.ean13 import EAN13Encoder
import qrcode
from securitycodegen import inputbox

number='1234567890'
file_EAN="barcode"
file_QR="qrcode"
url="www.mingrisoft.com/book/"

'''
1.EAN-13产品条形码批量生成
条形码可以获得商品的生产国家、生产企业、商品名称、价格等信息。EAN-13是欧洲物品编码的简写，共计13位。
1~3位为国家代码，4~7位为企业代码，8~12位为商品代码，13位是校验码。
13位校验码计算方法：c3=3*(前12位的偶数位之和)+前12位奇数位之和；取c3的个位数；10-个位数；将得到的结果再取个位数，即为校验位
'''
def scode7():
    countryId=inputbox("\033[1;32m  请输入EN13的国家代码(3位)：\33[0m",3,3)
    while int(countryId) == 0:
        countryId=inputbox("\033[1;32m  请输入EN13的国家代码(3位)：\33[0m",3,3)
    compId=inputbox("\033[1;32m  请输入EN13的企业代码(4位)：\33[0m",3,4)
    while int(compId)==0:
        compId=inputbox("\033[1;32m  请输入EN13的企业代码(4位)：\33[0m",3,4)
    incount=inputbox("\033[1;32m  请输入要生成的条形码数量：\33[0m",1,0)
    while int(incount)==0:
        incount=inputbox("\033[1;32m  请输入要生成的条形码数量：\33[0m",1,0)
    if not os.path.exists(file_EAN):
        os.mkdir(file_EAN)          #判断保存条形码的文件是否存在，不存在则创建
    for j in range(int(incount)):    #批量生成条形码
        strone=''                    #清空保存单条条形码的变量
        for i in range(5):           #生成条形码的5位数字
            strone=strone + str(random.choice(number))
        barcode=countryId + compId + strone    #组合代码
        #计算条形码的校验位
        evensum=int(barcode[1])+int(barcode[3])+int(barcode[5])+int(barcode[7])+int(barcode[9])+int(barcode[11])
        oddsum=int(barcode[0])+int(barcode[2])+int(barcode[4])+int(barcode[6])+int(barcode[8])+int(barcode[10])
        checkbit=int((10-(evensum*3+oddsum)%10)%10)
        barcode=barcode+str(checkbit)   #组成完整的EAN13条形码的13位数字
        encoder=EAN13Encoder(barcode)
        encoder.save("barcode\\" + barcode + ".png")

'''
2.二维码批量生成
二维码是用某种特定的几何图形按一定规律在平面(二维方向上)分布的黑白相间记录数据的图形符号信息
黑色方块表示1，白色方块表示0，相应的组合表示了一系列的信息，QR码是目前使用最广的二维码
'''
def scode8():           #本函数生成固定的12位二维码
    #输入要生成的二维码数量
    incount=inputbox("\033[1;32m  请输入要生成的12位数字二维码数量：\33[0m",1,0)
    while int(incount)==0:
        incount=inputbox("\033[1;32m  请输入要生成的12位数字二维码数量：\33[0m",1,0)
    if not os.path.exists(file_QR):
        os.mkdir(file_QR)       #判断保存二维码的文件夹是否存在
    for j in range(int(incount)):
        strone=""               #清空存储单个二维码的变量
        for i in range(12):     #生成单条二维码数字
            strone=strone + str(random.choice(number))
            strone1=url + strone + ".html"
            encoder=qrcode.make(strone1)      #生成二维码
            encoder.save("qrcode\\" + strone + ".png")  #保存二维码图片到文件


from http import client as http_client

from ironic_lib import metrics_utils
from oslo_utils import uuidutils
from pecan import rest

from ironic import api
from ironic.api.controllers import link
from ironic.api.controllers.v1 import collection
from ironic.api.controllers.v1 import notification_utils as notify
from ironic.api.controllers.v1 import utils as api_utils
from ironic.api import method
from ironic.common import args
from ironic.common import exception
from ironic.common.i18n import _
from ironic.common import policy
from ironic import objects

METRICS = metrics_utils.get_metrics_logger(__name__)

_DEFAULT_RETURN_FIELDS = ['uuid', 'node_uuid', 'volume_type',
                          'boot_index', 'volume_id']

TARGET_SCHEMA = {
    'type': 'object',
    'properties': {
        'boot_index': {'type': 'integer'},
        'extra': {'type': ['object', 'null']},
        'node_uuid': {'type': 'string'},
        'properties': {'type': ['object', 'null']},
        'volume_id': {'type': 'string'},
        'volume_type': {'type': 'string'},
        'uuid': {'type': ['string', 'null']},
    },
    'required': ['boot_index', 'node_uuid', 'volume_id', 'volume_type'],
    'additionalProperties': False,
}

TARGET_VALIDATOR_EXTRA = args.dict_valid(
    node_uuid=args.uuid,
    uuid=args.uuid,
)

TARGET_VALIDATOR = args.and_valid(
    args.schema(TARGET_SCHEMA),
    TARGET_VALIDATOR_EXTRA
)

PATCH_ALLOWED_FIELDS = [
    'boot_index',
    'extra',
    'node_uuid',
    'properties',
    'volume_id',
    'volume_type'
]

def convert_with_links(rpc_target, fields=None, sanitize=True):
    target = api_utils.object_to_dict(
        rpc_target,
        link_resource='volume/targets',
        fields=(
            'boot_index',
            'extra',
            'properties',
            'volume_id',
            'volume_type'
        )
    )
    api_utils.populate_node_uuid(rpc_target, target)

    if fields is not None:
        api_utils.check_for_invalid_fields(fields, target)

    if not sanitize:
        return target

    api_utils.sanitize_dict(target, fields)

    return target

def list_convert_with_links(rpc_targets, limit, url, fields=None,
                            detail=None, **kwargs):
    if detail:
        kwargs['detail'] = detail
    return collection.list_convert_with_links(
        items=[convert_with_links(p, fields=fields, sanitize=False)
               for p in rpc_targets],
        item_name='targets',
        limit=limit,
        url=url,
        fields=fields,
        sanitize_func=api_utils.sanitize_dict,
        **kwargs
    )

class VolumeTargetsController(rest.RestController):
    """REST controller for VolumeTargets."""

    invalid_sort_key_list = ['extra', 'properties']

    def __init__(self, node_ident=None):
        super(VolumeTargetsController, self).__init__()
        self.parent_node_ident = node_ident

    def _redact_target_properties(self, target):
        # Filters what could contain sensitive information. For iSCSI
        # volumes this can include iscsi connection details which may
        # be sensitive.
        redacted = ('** Value redacted: Requires permission '
                    'baremetal:volume:view_target_properties '
                    'access. Permission denied. **')
        redacted_message = {
            'redacted_contents': redacted
        }
        target.properties = redacted_message

    def _get_volume_targets_collection(self, node_ident, marker, limit,
                                       sort_key, sort_dir, resource_url=None,
                                       fields=None, detail=None,
                                       project=None):
        limit = api_utils.validate_limit(limit)
        sort_dir = api_utils.validate_sort_dir(sort_dir)

        marker_obj = None
        if marker:
            marker_obj = objects.VolumeTarget.get_by_uuid(
                api.request.context, marker)

        if sort_key in self.invalid_sort_key_list:
            raise exception.InvalidParameterValue(
                _("The sort_key value %(key)s is an invalid field for "
                  "sorting") % {'key': sort_key})
        node_ident = self.parent_node_ident or node_ident

        if node_ident:
            # FIXME(comstud): Since all we need is the node ID, we can
            #                 make this more efficient by only querying
            #                 for that column. This will get cleaned up
            #                 as we move to the object interface.
            node = api_utils.get_rpc_node(node_ident)
            targets = objects.VolumeTarget.list_by_node_id(
                api.request.context, node.id, limit, marker_obj,
                sort_key=sort_key, sort_dir=sort_dir, project=project)
        else:
            targets = objects.VolumeTarget.list(api.request.context,
                                                limit, marker_obj,
                                                sort_key=sort_key,
                                                sort_dir=sort_dir,
                                                project=project)
        cdict = api.request.context.to_policy_values()
        if not policy.check_policy('baremetal:volume:view_target_properties',
                                   cdict, cdict):
            for target in targets:
                self._redact_target_properties(target)

        return list_convert_with_links(targets, limit,
                                       url=resource_url,
                                       fields=fields,
                                       sort_key=sort_key,
                                       sort_dir=sort_dir,
                                       detail=detail)

    @METRICS.timer('VolumeTargetsController.get_all')
    @method.expose()
    @args.validate(node=args.uuid_or_name, marker=args.uuid,
                   limit=args.integer, sort_key=args.string,
                   sort_dir=args.string, fields=args.string_list,
                   detail=args.boolean)
    def get_all(self, node=None, marker=None, limit=None, sort_key='id',
                sort_dir='asc', fields=None, detail=None, project=None):
        """Retrieve a list of volume targets.

        :param node: UUID or name of a node, to get only volume targets
                     for that node.
        :param marker: pagination marker for large data sets.
        :param limit: maximum number of resources to return in a single result.
                      This value cannot be larger than the value of max_limit
                      in the [api] section of the ironic configuration, or only
                      max_limit resources will be returned.
        :param sort_key: column to sort results by. Default: id.
        :param sort_dir: direction to sort. "asc" or "desc". Default: "asc".
        :param fields: Optional, a list with a specified set of fields
                       of the resource to be returned.
        :param detail: Optional, whether to retrieve with detail.
        :param project: Optional, an associated node project (owner,
                        or lessee) to filter the query upon.

        :returns: a list of volume targets, or an empty list if no volume
                  target is found.

        :raises: InvalidParameterValue if sort_key does not exist
        :raises: InvalidParameterValue if sort key is invalid for sorting.
        :raises: InvalidParameterValue if both fields and detail are specified.
        """
        project = api_utils.check_volume_list_policy(
            parent_node=self.parent_node_ident)
        if fields is None and not detail:
            fields = _DEFAULT_RETURN_FIELDS

        if fields and detail:
            raise exception.InvalidParameterValue(
                _("Can't fetch a subset of fields with 'detail' set"))

        resource_url = 'volume/targets'
        return self._get_volume_targets_collection(node, marker, limit,
                                                   sort_key, sort_dir,
                                                   resource_url=resource_url,
                                                   fields=fields,
                                                   detail=detail,
                                                   project=project)

    @METRICS.timer('VolumeTargetsController.get_one')
    @method.expose()
    @args.validate(target_uuid=args.uuid, fields=args.string_list)
    def get_one(self, target_uuid, fields=None):
        """Retrieve information about the given volume target.

        :param target_uuid: UUID of a volume target.
        :param fields: Optional, a list with a specified set of fields
               of the resource to be returned.

        :returns: API-serializable volume target object.

        :raises: OperationNotPermitted if accessed with specifying a parent
                 node.
        :raises: VolumeTargetNotFound if no volume target with this UUID exists
        """

        rpc_target, _ = api_utils.check_volume_policy_and_retrieve(
            'baremetal:volume:get',
            target_uuid,
            target=True)

        if self.parent_node_ident:
            raise exception.OperationNotPermitted()

        cdict = api.request.context.to_policy_values()
        if not policy.check_policy('baremetal:volume:view_target_properties',
                                   cdict, cdict):
            self._redact_target_properties(rpc_target)

        return convert_with_links(rpc_target, fields=fields)

    @METRICS.timer('VolumeTargetsController.post')
    @method.expose(status_code=http_client.CREATED)
    @method.body('target')
    @args.validate(target=TARGET_VALIDATOR)
    def post(self, target):
        """Create a new volume target.

        :param target: a volume target within the request body.

        :returns: API-serializable volume target object.

        :raises: OperationNotPermitted if accessed with specifying a parent
                 node.
        :raises: VolumeTargetBootIndexAlreadyExists if a volume target already
                 exists with the same node ID and boot index
        :raises: VolumeTargetAlreadyExists if a volume target with the same
                 UUID exists
        """
        context = api.request.context
        raise_node_not_found = False
        node = None
        owner = None
        lessee = None
        node_uuid = target.get('node_uuid')
        try:
            node = api_utils.replace_node_uuid_with_id(target)
            owner = node.owner
            lessee = node.lessee
        except exception.NotFound:
            raise_node_not_found = True
        api_utils.check_owner_policy('node', 'baremetal:volume:create',
                                     owner, lessee=lessee,
                                     conceal_node=False)
        if raise_node_not_found:
            raise exception.InvalidInput(fieldname='node_uuid',
                                         value=node_uuid)

        if self.parent_node_ident:
            raise exception.OperationNotPermitted()

        # NOTE(hshiina): UUID is mandatory for notification payload
        if not target.get('uuid'):
            target['uuid'] = uuidutils.generate_uuid()
        new_target = objects.VolumeTarget(context, **target)

        notify.emit_start_notification(context, new_target, 'create',
                                       node_uuid=node.uuid)
        with notify.handle_error_notification(context, new_target, 'create',
                                              node_uuid=node.uuid):
            new_target.create()
        notify.emit_end_notification(context, new_target, 'create',
                                     node_uuid=node.uuid)
        # Set the HTTP Location Header
        api.response.location = link.build_url('volume/targets',
                                               new_target.uuid)
        return convert_with_links(new_target)

    @METRICS.timer('VolumeTargetsController.patch')
    @method.expose()
    @method.body('patch')
    @args.validate(target_uuid=args.uuid, patch=args.patch)
    def patch(self, target_uuid, patch):
        """Update an existing volume target.

        :param target_uuid: UUID of a volume target.
        :param patch: a json PATCH document to apply to this volume target.

        :returns: API-serializable volume target object.

        :raises: OperationNotPermitted if accessed with specifying a
                 parent node.
        :raises: PatchError if a given patch can not be applied.
        :raises: InvalidParameterValue if the volume target's UUID is being
                 changed
        :raises: NodeLocked if the node is already locked
        :raises: NodeNotFound if the node associated with the volume target
                 does not exist
        :raises: VolumeTargetNotFound if the volume target cannot be found
        :raises: VolumeTargetBootIndexAlreadyExists if a volume target already
                 exists with the same node ID and boot index values
        :raises: InvalidUUID if invalid node UUID is passed in the patch.
        :raises: InvalidStateRequested If a node associated with the
                 volume target is not powered off.
        """
        context = api.request.context

        api_utils.check_volume_policy_and_retrieve('baremetal:volume:update',
                                                   target_uuid,
                                                   target=True)

        if self.parent_node_ident:
            raise exception.OperationNotPermitted()

        api_utils.patch_validate_allowed_fields(patch, PATCH_ALLOWED_FIELDS)

        values = api_utils.get_patch_values(patch, '/node_uuid')
        for value in values:
            if not uuidutils.is_uuid_like(value):
                message = _("Expected a UUID for node_uuid, but received "
                            "%(uuid)s.") % {'uuid': str(value)}
                raise exception.InvalidUUID(message=message)

        rpc_target = objects.VolumeTarget.get_by_uuid(context, target_uuid)
        target_dict = rpc_target.as_dict()
        # NOTE(smoriya):
        # 1) Remove node_id because it's an internal value and
        #    not present in the API object
        # 2) Add node_uuid
        rpc_node = api_utils.replace_node_id_with_uuid(target_dict)

        target_dict = api_utils.apply_jsonpatch(target_dict, patch)

        try:
            if target_dict['node_uuid'] != rpc_node.uuid:

                # TODO(TheJulia): I guess the intention is to
                # permit the mapping to be changed
                # should we even allow this at all?
                rpc_node = objects.Node.get(
                    api.request.context, target_dict['node_uuid'])
        except exception.NodeNotFound as e:
            # Change error code because 404 (NotFound) is inappropriate
            # response for a PATCH request to change a volume target
            e.code = http_client.BAD_REQUEST  # BadRequest
            raise

        api_utils.patched_validate_with_schema(
            target_dict, TARGET_SCHEMA, TARGET_VALIDATOR)

        api_utils.patch_update_changed_fields(
            target_dict, rpc_target, fields=objects.VolumeTarget.fields,
            schema=TARGET_SCHEMA, id_map={'node_id': rpc_node.id}
        )

        notify.emit_start_notification(context, rpc_target, 'update',
                                       node_uuid=rpc_node.uuid)
        with notify.handle_error_notification(context, rpc_target, 'update',
                                              node_uuid=rpc_node.uuid):
            topic = api.request.rpcapi.get_topic_for(rpc_node)
            new_target = api.request.rpcapi.update_volume_target(
                context, rpc_target, topic)

        api_target = convert_with_links(new_target)
        notify.emit_end_notification(context, new_target, 'update',
                                     node_uuid=rpc_node.uuid)
        return api_target

    @METRICS.timer('VolumeTargetsController.delete')
    @method.expose(status_code=http_client.NO_CONTENT)
    @args.validate(target_uuid=args.uuid)
    def delete(self, target_uuid):
        """Delete a volume target.

        :param target_uuid: UUID of a volume target.

        :raises: OperationNotPermitted if accessed with specifying a
                 parent node.
        :raises: NodeLocked if node is locked by another conductor
        :raises: NodeNotFound if the node associated with the target does
                 not exist
        :raises: VolumeTargetNotFound if the volume target cannot be found
        :raises: InvalidStateRequested If a node associated with the
                 volume target is not powered off.
        """
        context = api.request.context

        api_utils.check_volume_policy_and_retrieve('baremetal:volume:delete',
                                                   target_uuid,
                                                   target=True)

        if self.parent_node_ident:
            raise exception.OperationNotPermitted()

        rpc_target = objects.VolumeTarget.get_by_uuid(context, target_uuid)
        rpc_node = objects.Node.get_by_id(context, rpc_target.node_id)
        notify.emit_start_notification(context, rpc_target, 'delete',
                                       node_uuid=rpc_node.uuid)
        with notify.handle_error_notification(context, rpc_target, 'delete',
                                              node_uuid=rpc_node.uuid):
            topic = api.request.rpcapi.get_topic_for(rpc_node)
            api.request.rpcapi.destroy_volume_target(context,
                                                     rpc_target, topic)
        notify.emit_end_notification(context, rpc_target, 'delete',
                                     node_uuid=rpc_node.uuid)


import traceback
import urllib2
import zipfile
import datetime
import os

releases = [
    'NOV',
    'AUG',
    'MAY',
    'FEB'
]

year = datetime.datetime.now().year

file = None
filename = None
while file == None:
    for release in releases:
        url = "https://geoportal.statistics.gov.uk/Docs/PostCodes/ONSPD_{0}_{1}_csv.zip".format(release,year)
        try:
            release = "ONSPD_{0}_{1}".format(release,year)
            file = urllib2.urlopen(url)
            break
        except:
            None
    year -= 1

output = open(release + "_csv.zip",'wb')
output.write(file.read())
output.close()

with zipfile.ZipFile(release + "_csv.zip", 'r') as datazip:
    file = datazip.open("Data/{0}_UK.csv".format(release))
    output = open(release + ".csv",'wb')
    output.write(file.read())
    output.close()

os.remove(release + "_csv.zip")


from collections.abc import Iterator

class PartitionIterator(Iterator):
    """
    Iterator on partitioned data.

    Parameters
    ----------
    df : modin.pandas.DataFrame
        The dataframe to iterate over.
    axis : {0, 1}
        Axis to iterate over.
    func : callable
        The function to get inner iterables from each partition.
    """

    def __init__(self, df, axis, func):
        self.df = df
        self.axis = axis
        self.index_iter = (
            zip(
                iter(slice(None) for _ in range(len(self.df.columns))),
                range(len(self.df.columns)),
            )
            if axis
            else zip(
                range(len(self.df.index)),
                iter(slice(None) for _ in range(len(self.df.index))),
            )
        )
        self.func = func

    def __iter__(self):
        """
        Implement iterator interface.

        Returns
        -------
        PartitionIterator
            Iterator object.
        """
        return self

    def __next__(self):
        """
        Implement iterator interface.

        Returns
        -------
        PartitionIterator
            Incremented iterator object.
        """
        key = next(self.index_iter)
        df = self.df.iloc[key]
        return self.func(df)


#
# @lc app=leetcode id=740 lang=python3
#
# [740] Delete and Earn
#

# @lc code=start
from collections import Counter

# Build the counter.
# Build the num * counter[num] sorted dicts `earns``.
#
# Then this problems becomes similar to the house robber problem.
#
# dp[i] means the max earn of the subproblem earns[:i].
# dp[i]:
#   - nums[i - 1] + 1 = nums[i]: max(dp[i - 1], earn[i] + dp[i - 2])
#   - nums[i - 1] + 1 <  nums[i]: cur + dp[i - 1]
class Solution:
    def deleteAndEarn(self, nums: list[int]) -> int:
        counter = dict((k, k * v) for k, v in Counter(nums).items())
        prev, cur, val_prev = 0, 0, None
        for num, earn in sorted(counter.items()):
            prev, cur, val_prev = (
                cur,
                max(cur, earn + prev) if val_prev == num - 1 else cur + earn,
                num,
            )

        return cur

# @lc code=end


""" This module includes the unittest class for the fcn8 model and a decorator
class for prittfy the console output."""

from __future__ import absolute_import
import unittest
import warnings

import torch
# from torch.nn import parameter
# from torch.autograd.gradcheck import gradcheck
# from torch.autograd import Variable
# import torch.nn as nn
from SemanticSegmentation.models.fcn8 import FCN8

class LineInfo():
    """ Deco Class for Lines """
    def __init__(self, lineformat:str = '-', linelength:int = 80,
                 info:tuple = ('', '')) -> None:
        self.lineformat = lineformat
        self.linelength = linelength
        self.info = info

    def __call__(self, func, *args, **kwargs):
        """ Prints Lines above and after the decorated func """
        def inner(*args, **kwargs):
            print(self.linelength * self.lineformat)
            print(self.info[0])
            func(*args, **kwargs)
            print(self.info[1])
            print(self.linelength * self.lineformat)
        return inner

    def __repr__(self) -> str:
        return 'LineInfo(lineformat=%r, linelength=%r, info=%r)' \
            % (self.lineformat, self.linelength, self.info)

class TestFCN8(unittest.TestCase):
    """ Unittest class for the fcn8 modell """
    def setUp(self):
        self.model = FCN8(8, pretrained=True)
        torch.manual_seed(0)
        # instead of zero init for score tensors use random init
        self.x = torch.rand((4, 3, 45, 45))

    @LineInfo(linelength=70, info=('Test Forward startet','Test Forward ended'))
    def testForward(self):
        """ Tests the FCN8.forward(x) function
        what is tested:
            - number of elements in output
            - shape of output
            - value at a specific index
            - max value if score is init zero_
        """

        self.changeWeights('random')
        output = self.model.forward(self.x)

        self.assertEqual(output.shape.numel(), 64800)
        self.assertEqual(list(output.shape), [4, 8, 45, 45])
        self.assertEqual(float(output[3][4][44][4]), 2362961408.0)

        self.changeWeights('zero')
        output = self.model.forward(self.x)

        self.assertEqual(torch.max(output), 0.0)

    def changeWeights(self, value:str = 'zero'):
        """ function to change the init value of score layers """
        if value == 'zero':
            self.model.score_fr[6].weight.data.zero_()
            self.model.score_fr[6].bias.data.zero_()
            self.model.score_pool3.weight.data.zero_()
            self.model.score_pool3.bias.data.zero_()
            self.model.score_pool4.weight.data.zero_()
            self.model.score_pool4.bias.data.zero_()
        elif value == 'random':
            self.model.score_fr[6].weight.data.random_()
            self.model.score_fr[6].bias.data.random_()
            self.model.score_pool3.weight.data.random_()
            self.model.score_pool3.bias.data.random_()
            self.model.score_pool4.weight.data.random_()
            self.model.score_pool4.bias.data.random_()
        else:
            warnings.warn(
                "weights stays unchanged use value == 'random' or 'zero'")

    '''def atest_sanity(self):
        input = (Variable(torch.randn(20, 20).double(), requires_grad=True), )
        model = nn.Linear(20, 1).double()
        test = gradcheck(model, input, eps=1e-6, atol=1e-4)
        print(test)
    '''

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/python3

# 用途    : サーバにリモートでアクセスしている時に、そこで生成したrustdocを見るためにhttpサーバを建てる。
# 使用方法: このgitリポジトリのルートディレクトリにて
#          $ ./view_cargo_doc.py
# 注意事項: http通信なので、秘密にしたいcreateに対してはしない方が良さそう。

import re
import http.server
import socketserver

IP_ADDRESS = "127.0.0.1" # localhost
CANDIDATE_PORTS = range(8000, 8010) # 連続で使用すると、エラー("OSError: [Errno 98] Address already in use")が出たので対策した。
TARGET_DIR = "project/target" # project/.cargo/config.tomlのtarget-dirに対応
SETTINGS_HTML = TARGET_DIR + "/doc/settings.html"

# extract </button>
with open(SETTINGS_HTML, "r") as f:
    match = re.search('</button>', f.read())
    stripped_match = match.group().replace('</button>', '')
    INDEX_HTML = TARGET_DIR + "/doc/" + stripped_match

# http server
Handler = http.server.SimpleHTTPRequestHandler
for port in CANDIDATE_PORTS:
    try:
        with socketserver.TCPServer((IP_ADDRESS, port), Handler) as httpd:
            print("Serving at port", "http://" + IP_ADDRESS + ":" + str(port) + "/" + INDEX_HTML + ".")
            httpd.serve_forever()
    except OSError as e: # OSEエラーが出たら、次のポート候補で再度挑戦する。
        continue
    except KeyboardInterrupt:
        print('\nCatch KeyboardInterrupt.')
        break
    else: # httpd.serve_forever() は終了しないので、このelseには到達しない。
        print("Error: このprint()関数が実行されることはない。")
        raise


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 26 11:56:40 2022

@author: pdavid

This script has been created as part of the virgin code to have something with
the basic tools I'm commonly using, so everytime I wanna start some simulations
I have a script somewhat ready'
"""

#%% - First set the proper directory
import os
directory='/home/pdavid/Bureau/Code/SS_auto57/2D_cartesian/Validated_2D_Code/FV_metab_dimensional/Updated_BCs/Code'
os.chdir(directory)

#%%

import pdb
import numpy as np
import matplotlib.pyplot as plt
from module_2D_coupling_FV_nogrid import *
import reconst_and_test_module as post
import random
import scipy as sp
from scipy import sparse
import scipy.sparse.linalg
import matplotlib.pylab as pylab
params = {'legend.fontsize': 'x-large',
          'figure.figsize': (6,6 ),
         'axes.labelsize': 'x-large',
         'axes.titlesize':'x-large',
         'xtick.labelsize':'x-large',
         'ytick.labelsize':'x-large',
         'lines.linewidth' : 2
         }
pylab.rcParams.update(params)

#separation of scales
alpha=500

#physical parameters

D=1
K0=1

#Geometry
L=240
cells=5
h_ss=L/cells
ratio=int(50/cells)*2 #ratio for the reconstruction and validation
print("ratio: ", ratio)
#ratio=12
#Rv=np.exp(-2*np.pi)*h_ss

print("R: ", 1/(1/K0 + np.log(alpha/(5*cells*ratio))/(2*np.pi*D)))
validation=True

#%%- Construction of the mesh
x_ss=np.linspace(h_ss/2, L-h_ss/2, int(np.around(L/h_ss)))
y_ss=x_ss
directness=1
print("directness=", directness)

#%% - Definition of the source problem
pos_s=np.array([[0.75,0.75]])*L
#pos_s=np.array([[0.5,0.5]])*L

#pos_s=np.array([[0.77,0.77]])*L
S=len(pos_s)

C_v_array=np.ones(S)

Rv=L/alpha+np.zeros(S)

print("alpha: {} must be greater than {} \n for the Peaceman validation".format(alpha, 5*ratio*cells))
print("h coarse:",h_ss)
K_eff=K0/(np.pi*Rv**2)

#Position image

vline=(y_ss[1:]+x_ss[:-1])/2
plt.scatter(pos_s[:,0], pos_s[:,1])
plt.title("Position of the point sources")
for xc in vline:
    plt.axvline(x=xc, color='k', linestyle='--')
for xc in vline:
    plt.axhline(y=xc, color='k', linestyle='--')
plt.xlim([0,L])
plt.ylim([0,L])
plt.ylabel("y ($\mu m$)")
plt.xlabel("x ($\mu m$)")
plt.show()

#%% - 1st solve the system through the hybrid model
from module_2D_coupling_FV_nogrid import *

T=assemble_SS_2D_FD(pos_s, Rv, h_ss,L, K_eff, D,directness)
T.pos_arrays()
T.initialize_matrices()

#Set Dirichlet BC:
dirichlet_array=np.array([0,0,0,0])
BC_type=np.array(['Periodic', 'Periodic','Dirichlet','Dirichlet'])

T.solve_problem(BC_type, dirichlet_array, C_v_array)

#%%
pepa=real_NN_rec(T.x,T.y, T.phi_FV,pos_s, T.s_blocks, T.phi_q, ratio, T.h, directness, Rv, np.zeros(S)+K0, C_v_array)
plt.imshow(pepa.rec_plus_phibar,origin='lower')
plt.title("Slow potential piece-wise constant \n functions")
plt.colorbar()
plt.show()
plt.imshow(pepa.add_singular(directness)+pepa.rec,origin='lower'); plt.colorbar()
plt.title("$\phi$-field using piece wise constant functions \n for the slow potential")
plt.show()

#%% 3 - Bilinear Reconstruction
import reconst_and_test_module as post

b=post.reconstruction_sans_flux(np.concatenate((np.ndarray.flatten(T.phi_FV), T.phi_q)), T, L,ratio, directness)
p=b.reconstruction()
plt.imshow(b.rec_final, origin='lower')
plt.title("$\phi$-field using bilinear reconstruction \n for the slow potential")
plt.colorbar(); plt.show()

#%%

BC_values=np.array([0,0,0,0])
b.reconstruction_boundaries_short(BC_type, BC_values)
plt.imshow(b.rec_final, origin='lower')
plt.title("bilinear reconstruction \n coupling model Steady State ")
plt.colorbar(); plt.show()
#%%
b.rec_corners()
plt.imshow(b.rec_final, origin='lower')
plt.title("$\phi$-field using bilinear reconstruction \n for the slow potential")
plt.colorbar(); plt.show()

#%%

s,r = b.get_u_pot(C_v_array)
plt.imshow(r, origin='lower')
plt.colorbar()
plt.title('Rapid potential')
plt.show()

plt.imshow(s, origin='lower')
plt.colorbar()
plt.title('Slow potential')
plt.show()

#%%

pp=np.argmax(r)//len(b.x)

plt.plot(b.x,b.rec_final[pp])
plt.plot(b.x, s[pp])
plt.plot(b.x, r[pp])

#%% - 1st - FV resolution as a reference

FV=FV_validation(L, cells*ratio, pos_s, C_v_array, D, K_eff, Rv, dirichlet_array)
FV_linear=FV.solve_linear_system()
FV_linear_mat=FV_linear.reshape(cells*ratio, cells*ratio)

#%% Plots FV reference solution - Peaceman Coupling

plt.imshow(FV_linear_mat, origin='lower')
plt.colorbar()
plt.title("FV reference solution, linear system\n mesh:{}x{}".format(ratio*cells, ratio*cells))
plt.show()

#%%
print("relative error in the flux esimation: ", get_MRE(FV.get_q(FV_linear), T.phi_q))

#%% - Peyrounette like figure
"""This section is to produce the single source figure. Skip is there
are multiple sources"""

length=len(x_ss)*ratio

pos_s_i=coord_to_pos(b.x, b.y, pos_s[0])//length
l2=int(coord_to_pos(b.x, b.y, pos_s[0])%length)

plot_array=np.zeros(length)

#recover the values at the FV cells' center:
S=post.coarse_cell_center_rec(x_ss, y_ss, T.phi_FV, pos_s, T.s_blocks, T.phi_q, ratio, h_ss, directness, Rv)
S[np.argmax(S)//cells, np.argmax(S)%cells]=np.sum(C_v_array)-np.sum(T.phi_q/T.K_0)

A=post.tool_piece_wise_constant_ratio(ratio, S)
reg=post.tool_piece_wise_constant_ratio(ratio, T.phi_FV)

plot_array=np.concatenate((A[pos_s_i, :l2], b.rec_final[pos_s_i, l2:]))

plt.scatter(np.arange(length), plot_array)

u,pot=b.get_u_pot(C_v_array)

u-=np.sum(C_v_array)-np.sum(T.phi_q)/T.K_0
Slow=reg-np.sum(C_v_array)+np.sum(T.phi_q)/T.K_0

plt.figure()
for xc in vline:
    plt.axvline(x=xc, color='k', linestyle='--')

#right side
#FV averaged concentration
plt.plot(b.x[l2:3*ratio], A[pos_s_i,l2:3*ratio], color='darkcyan')
plt.plot(b.x[3*ratio:4*ratio], A[pos_s_i,3*ratio:4*ratio], color='darkcyan')
plt.plot(b.x[4*ratio:5*ratio], A[pos_s_i,4*ratio:5*ratio], color='darkcyan')
plt.scatter(x_ss[2:], S[2,2:], label='$\phi$ cell center', color='darkcyan')
#Reference solution
plt.plot(b.x[l2:], FV_linear_mat[pos_s_i, l2:], label='reference', color='red')

#plt.plot(b.x[l2:],b.rec_final[pos_s_i, l2:], label='reconstructed')
plt.plot(b.x[ratio:l2+1],Slow[pos_s_i, ratio:l2+1], color='green')
plt.scatter(x_ss[1:3], T.phi_FV[2,1:3]-np.sum(C_v_array)+np.sum(T.phi_q)/T.K_0 ,label='$\mathcal{s}$ cell center', color='green')
plt.plot(b.x[:ratio],Slow[pos_s_i, :ratio],linestyle='--' ,color='green')
plt.plot(b.x[ratio:l2+1],pot[pos_s_i, ratio:l2+1], label='$\mathcal{r}$', color='slategrey')
plt.plot(b.x[:ratio],pot[pos_s_i, :ratio], linestyle='--' ,color='slategrey')
#plt.plot(b.x[:ratio], A[pos_s_i, :ratio], color='darkcyan')
#plt.scatter(x_ss[0], S[2,0], color='darkcyan')
#plt.legend()
plt.ylim(-0.1,0.6)

#%%
if len(pos_s) > 1:
    plt.plot(T.phi_q, label="MyCode")
    plt.plot(FV.get_q(FV_linear), label="FV reference")
    plt.legend()
    plt.show()

#%% - Plots
print("MRE steady state system", get_MRE(n.phi_q, FV.get_q(FV_linear)))

#%%
for i in pos_s:
    pos=coord_to_pos(FV.x, FV.y, i)

    plt.plot(FV_linear_mat[pos//len(FV.x),:], label="FV")
    plt.plot(b.rec_final[pos//len(FV.x),:],label="SS no metab")
    plt.legend()
    plt.title("Linear solution")
    plt.show()

for i in pos_s:
    pos=coord_to_pos(FV.x, FV.y, i)

    plt.plot(phi_FV[pos//len(FV.x),:], label="FV")
    plt.plot(a.rec_final[pos//len(FV.x),:],label="SS")
    plt.plot(b.rec_final[pos//len(FV.x),:],label="SS no metab")
    plt.legend()
    plt.show()

# =============================================================================
# c=0
# for i in pos_s:
#     pos=coord_to_pos(FV.x, FV.y, i)
#
#     plt.plot(FV.x,phi_FV[pos//len(FV.x),:], label="FV")
#     plt.scatter(n.x,(n.rec_sing+n.u).reshape(cells, cells)[n.s_blocks[c]//cells,:],label="SS")
#     plt.plot(b.x,b.rec_final[pos//len(FV.x),:],label="SS no metab")
#     plt.legend()
#     plt.show()
#     c+=1
# =============================================================================

print("relative errors")
print(np.abs(n.phi[-1,-S:]-FV.get_q(FV.phi[-1]))/FV.get_q(FV.phi[-1]))

print("absolute error")
print(np.abs(n.phi[-1,-S:]-FV.get_q(FV.phi[-1])))

print("L2_error")
print(get_L2(n.phi[-1,-S:], FV.get_q(FV.phi[-1])))

print("MRE", get_MRE(n.phi[-1,-S:], FV.get_q(FV.phi[-1])))

u,pot=b.get_u_pot(C_v_array)

plt.subplots(2,2, figsize=(12,12))

plt.subplot(2,2,1)
plt.imshow(pot, origin='lower')
plt.title("SL + DL in $\overline{\Omega}$")
plt.colorbar()

plt.subplot(2,2,2)
plt.imshow(b.rec_potentials, origin="lower")
plt.title("SL in $\overline{\Omega} (old singular term)$")
plt.colorbar()

plt.subplot(2,2,3)
plt.imshow(u, origin="lower")
plt.title("u")
plt.colorbar()

plt.subplot(2,2,4)
plt.imshow(b.rec_final, origin="lower")
plt.title("$\phi$")
plt.colorbar()

#%%


from qabot.lib.jenkinslib import JenkinsLib
from qabot.lib.githublib import GithubLib
import json
import time
import os
import logging

logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
log = logging.getLogger(__name__)

class JenkinsJobInvoker:
    def run_test_on_environment(self, jenkins_instance, target_environment, test_suite):
        """
        Prepares a py request with url and basic auth based on details provided by the user
        """
        params_str = '{{ "TARGET_ENVIRONMENT": "{}", "TEST_SUITE": "{}" }}'.format(
            target_environment, test_suite
        )
        bot_response = self.invoke_jenkins_job(
            "run-tests-on-environment", jenkins_instance, params_str
        )
        return bot_response

    def invoke_jenkins_job(self, job_name, jenkins_instance, params_str=""):
        """
        Invoke any job
        """
        params = json.loads(params_str)
        jl = JenkinsLib(jenkins_instance)
        err, id_of_triggered_job = jl.prepare_request_and_invoke(job_name, params)
        if err == None:
            bot_response = "The job has been triggered, here's its URL: \n  {}".format(
                "https://{}.planx-pla.net/job/{}/{}/console".format(
                    jenkins_instance,
                    job_name,
                    id_of_triggered_job,
                )
            )
        else:
            bot_response = "Something wrong happened :facepalm:. Deets: {}".format(err)
        return bot_response

    def replay_pr(self, repo_name, pr_number, labels=[]):
        """
        Replay a Pull Request like a boss
        """
        githublib = GithubLib(repo=repo_name)
        try:
            if " " in labels:
                raise Exception("Whitespace found in comma-separated list of labels")
            if len(labels) > 0:
                labels = labels.split(",")
                log.info("applying labels...")
                for i, label in enumerate(labels):
                    # only override all labels on the first iteration
                    override_all = i == 0
                    githublib.set_label_to_pr(
                        int(pr_number), label.replace("*", ""), override_all
                    )
                    log.debug("applied label: {}".format(label))
            else:
                log.warn("Replaying PR without labels...")
        except Exception as err:
            return "Something wrong happened :facepalm:. Deets: {}".format(err)

        jl = JenkinsLib("jenkins")
        log.info("find the number of the last build...")
        job_num = jl.get_number_of_last_build(repo_name, pr_number)

        err, url_from_replayed_pr = jl.send_blueocean_request(
            repo_name, pr_number, job_num
        )
        if err == None:
            bot_response = "Your PR has been labeled and replayed successfully :tada: \n Czech it out :muscle: {}".format(
                url_from_replayed_pr
            )
        else:
            bot_response = "Something wrong happened :facepalm:. Deets: {}".format(err)
        return bot_response

    def get_status_of_job(self, job_name, job_id, jenkins_instance):
        """
        Return status of a given job based on its id
        """
        return "not implemented yet"

    def roll_service(self, service_name, cluster_name, environment):
        """
        Roll a service in one of our gen3 environments
        """
        return "not implemented yet"

    def fetch_list_of_environments(self, cluster_name):
        # TODO: Refactor and move this to a dictionary lookup in Jenkinslib
        jenkins_instance = None
        if cluster_name == "qaplanetv1":
            jenkins_instance = "jenkins"
        elif cluster_name == "qaplanetv2":
            jenkins_instance = "jenkins2"
        else:
            return "This cluster does not exist :wat:"
        jl = JenkinsLib(jenkins_instance)
        # Just fetch the latest archived artifact (the job runs on a schedule)
        err, list_of_environments = jl.fetch_archived_artifact(
            "list-namespaces-in-this-cluster", "ls_environments.txt"
        )
        if err == None:
            bot_response = "Here is the list of environments in this cluster: \n"
            bot_response += "```{}```".format(list_of_environments)
        else:
            bot_response = "Something wrong happened :facepalm:. Deets: {}".format(err)
        return bot_response

    def fetch_selenium_status(self, cluster_name):
        # TODO: Refactor and move this to a dictionary lookup in Jenkinslib
        jenkins_instance = None
        if cluster_name == "qaplanetv1":
            jenkins_instance = "jenkins"
        elif cluster_name == "qaplanetv2":
            jenkins_instance = "jenkins2"
        else:
            return "This cluster does not exist :wat:"

        jl = JenkinsLib(jenkins_instance)
        err, id_of_triggered_job = jl.prepare_request_and_invoke(
            "selenium-check-status", None
        )

        if err == None:
            selenium_status_check_result = jl.get_status_of_job(
                "selenium-check-status", id_of_triggered_job
            )
            log.info("checking the result of the selenium-status-check job...")
            log.info(f"result: {selenium_status_check_result}")

            err, selenium_status = jl.fetch_archived_artifact(
                "selenium-check-status", "selenium-status.txt"
            )
            if err == None:
                bot_response = "Here is the status of the Selenium hub :selenium: \n"
                bot_response += "```{}```".format(selenium_status)
            else:
                bot_response = "Could not fetch archived artifacts from the Jenkins job :facepalm:. Deets: {}".format(
                    err
                )
        else:
            bot_response = "Could not run the Jenkins job :facepalm:. Deets: {}".format(
                err
            )

        return bot_response

if __name__ == "__main__":
    # jji = JenkinsJobInvoker()
    # result = jji.run_test_on_environment(
    #    "run-tests-on-environment", "jenkins2", "ci-env-1", "test-portal-homepageTest"
    # )
    # print(result)
    # result2 = jji.invoke_jenkins_job(
    #    "self-service-qa-gen3-roll",
    #    "jenkins2",
    #    '{ "SERVICE_NAME": "all", "TARGET_ENVIRONMENT": "ci-env-1" }',
    # )
    jji = JenkinsJobInvoker()
    # jji.replay_pr('gen3-qa', '549', 'test-portal-homepageTest,test-portal-dataUploadTest')
    jji.replay_pr(
        "gen3-qa", "549", "test-portal-homepageTest, test-portal-dataUploadTest"
    )


"""Support NWS VTEC encoding"""
import re
from datetime import datetime, timedelta, timezone

from pyiem.util import LOG

VTEC_RE = (
    r"(/([A-Z])\.([A-Z]+)\.([A-Z]+)\.([A-Z]+)\.([A-Z])\."
    r"([0-9]+)\.([0-9TZ]+)-([0-9TZ]+)/)"
)

VTEC_CLASS = {
    "O": "Operational",
    "T": "Test",
    "E": "Experimental",
    "X": "Experimental VTEC",
}

VTEC_ACTION = {
    "NEW": "issues",
    "CON": "continues",
    "EXA": "expands area to include",
    "EXT": "extends time of",
    "EXB": "extends time and expands area to include",
    "UPG": "issues upgrade to",
    "CAN": "cancels",
    "EXP": "expires",
    "ROU": "routine",
    "COR": "corrects",
}

VTEC_SIGNIFICANCE = {
    "W": "Warning",
    "Y": "Advisory",
    "A": "Watch",
    "S": "Statement",
    "O": "Outlook",
    "N": "Synopsis",
    "F": "Forecast",
}

# https://www.nws.noaa.gov/directives/sym/pd01017003curr.pdf
VTEC_PHENOMENA = {
    "AF": "Ashfall",
    "AS": "Air Stagnation",
    "BH": "Beach Hazard",
    "BS": "Blowing Snow",
    "BW": "Brisk Wind",
    "BZ": "Blizzard",
    "CF": "Coastal Flood",
    "CW": "Cold Weather",
    "DF": "Debris Flow",
    "DS": "Dust Storm",
    "DU": "Blowing Dust",
    "EC": "Extreme Cold",
    "EH": "Excessive Heat",
    "EW": "Extreme Wind",
    "FA": "Flood",
    "FF": "Flash Flood",
    "FG": "Dense Fog",
    "FL": "Flood",
    "FR": "Frost",
    "FW": "Red Flag",
    "FZ": "Freeze",
    "UP": "Freezing Spray",
    "GL": "Gale",
    "HF": "Hurricane Force Wind",
    "HI": "Inland Hurricane",
    "HS": "Heavy Snow",
    "HT": "Heat",
    "HU": "Hurricane",
    "HW": "High Wind",
    "HY": "Hydrologic",
    "HZ": "Hard Freeze",
    "IP": "Sleet",
    "IS": "Ice Storm",
    "LB": "Lake Effect Snow and Blowing Snow",
    "LE": "Lake Effect Snow",
    "LO": "Low Water",
    "LS": "Lakeshore Flood",
    "LW": "Lake Wind",
    "MA": "Marine",
    "MF": "Marine Dense Fog",
    "MH": "Marine Ashfall",
    "MS": "Marine Dense Smoke",
    "RB": "Small Craft for Rough",
    "RP": "Rip Currents",
    "SB": "Snow and Blowing",
    "SC": "Small Craft",
    "SE": "Hazardous Seas",
    "SI": "Small Craft for Winds",
    "SM": "Dense Smoke",
    "SN": "Snow",
    "SQ": "Snow Squall",
    "SR": "Storm",
    "SS": "Storm Surge",
    "SU": "High Surf",
    "SV": "Severe Thunderstorm",
    "SW": "Small Craft for Hazardous Seas",
    "TI": "Inland Tropical Storm",
    "TO": "Tornado",
    "TR": "Tropical Storm",
    "TS": "Tsunami",
    "TY": "Typhoon",
    "WC": "Wind Chill",
    "WI": "Wind",
    "WS": "Winter Storm",
    "WW": "Winter Weather",
    "ZF": "Freezing Fog",
    "ZR": "Freezing Rain",
}

# Taken from http://www.weather.gov/help-map
# Not all of these are an exact match.
NWS_COLORS = {
    "AF.W": "#A9A9A9",
    "AF.Y": "#696969",
    "AS.Y": "#808080",
    "BH.S": "#40E0D0",
    "BW.Y": "#D8BFD8",
    "BZ.A": "#ADFF2F",
    "BZ.W": "#FF4500",
    "CF.A": "#66CDAA",
    "CF.S": "#6B8E23",
    "CF.W": "#228B22",
    "CF.Y": "#7CFC00",
    "DS.W": "#FFE4C4",
    "DS.Y": "#BDB76B",
    "DU.W": "#FFE4C4",
    "DU.Y": "#BDB76B",
    "EC.A": "#0000FF",
    "EC.W": "#0000FF",
    "EH.A": "#800000",
    "EH.W": "#C71585",
    "EH.Y": "#800000",
    "EW.W": "#FF8C00",
    "FA.A": "#2E8B57",
    "FA.W": "#00FF00",
    "FA.Y": "#00FF7F",
    "FF.A": "#2E8B57",
    "FF.S": "#8B0000",
    "FF.W": "#8B0000",
    "FG.Y": "#708090",
    "FL.A": "#2E8B57",
    "FL.S": "#00FF00",
    "FL.W": "#00FF00",
    "FL.Y": "#00FF7F",
    "FR.Y": "#6495ED",
    "FW.A": "#FFDEAD",
    "FW.W": "#FF1493",
    "FZ.A": "#00FFFF",
    "FZ.W": "#483D8B",
    "GL.A": "#FFC0CB",
    "GL.W": "#DDA0DD",
    "HF.A": "#9932CC",
    "HF.W": "#CD5C5C",
    "HT.Y": "#FF7F50",
    "HU.A": "#FF00FF",
    "HU.S": "#FFE4B5",
    "HU.W": "#DC143C",
    "HW.A": "#B8860B",
    "HW.W": "#DAA520",
    "HY.Y": "#00FF7F",
    "HZ.A": "#4169E1",
    "HZ.W": "#9400D3",
    "IS.W": "#8B008B",
    "LE.A": "#87CEFA",
    "LE.W": "#008B8B",
    "LE.Y": "#48D1CC",
    "LO.Y": "#A52A2A",
    "LS.A": "#66CDAA",
    "LS.S": "#6B8E23",
    "LS.W": "#228B22",
    "LS.Y": "#7CFC00",
    "LW.Y": "#D2B48C",
    "MA.S": "#FFDAB9",
    "MA.W": "#FFA500",
    "MF.Y": "#708090",
    "RB.Y": "#D8BFD8",
    "RP.S": "#40E0D0",
    "SC.Y": "#D8BFD8",
    "SE.A": "#483D8B",
    "SE.W": "#D8BFD8",
    "SI.Y": "#D8BFD8",
    "SM.Y": "#F0E68C",
    "SQ.W": "#C71585",
    "SR.A": "#FFE4B5",
    "SR.W": "#9400D3",
    "SS.A": "#DB7FF7",
    "SS.W": "#C0C0C0",
    "SU.W": "#228B22",
    "SU.Y": "#BA55D3",
    "SV.A": "#DB7093",
    "SV.W": "#FFA500",
    "SW.Y": "#D8BFD8",
    "TO.A": "#FFFF00",
    "TO.W": "#FF0000",
    "TR.A": "#F08080",
    "TR.S": "#FFE4B5",
    "TR.W": "#B22222",
    "TS.A": "#FF00FF",
    "TS.W": "#FD6347",
    "TS.Y": "#D2691E",
    "TY.A": "#FF00FF",
    "TY.W": "#DC143C",
    "UP.A": "#4682B4",
    "UP.W": "#8B008B",
    "UP.Y": "#8B008B",
    "WC.A": "#5F9EA0",
    "WC.W": "#B0C4DE",
    "WC.Y": "#AFEEEE",
    "WI.Y": "#D2B48C",
    "WS.A": "#4682B4",
    "WS.W": "#FF69B4",
    "WW.Y": "#7B68EE",
    "ZF.Y": "#008080",
    "ZR.Y": "#DA70D6",
}

def parse(text):
    """I look for and return vtec objects as I find them"""
    vtec = []
    tokens = re.findall(VTEC_RE, text)
    for token in tokens:
        vtec.append(VTEC(token))
    return vtec

def contime(text):
    """Convert text into a UTC datetime."""
    # The 0000 is the standard VTEC undefined time
    if text.startswith("0000"):
        return None
    try:
        ts = datetime.strptime(text, "%y%m%dT%H%MZ")
    except Exception as err:
        LOG.exception(err)
        return None
    # NWS has a bug sometimes whereby 1969 or 1970s timestamps are emitted
    if ts.year < 1971:
        return None
    return ts.replace(tzinfo=timezone.utc)

def get_ps_string(phenomena, significance):
    """Return the combination of Phenomena + Significance as string"""
    pstr = VTEC_PHENOMENA.get(phenomena, f"Unknown {phenomena}")
    astr = VTEC_SIGNIFICANCE.get(significance, f"Unknown {significance}")
    # Hack for special FW case
    if significance == "A" and phenomena == "FW":
        pstr = "Fire Weather"
    return f"{pstr} {astr}"

def get_action_string(action):
    """Return the action string"""
    return VTEC_ACTION.get(action, f"unknown {action}")

class VTEC:
    """A single VTEC encoding instance"""

    def __init__(self, tokens):
        self.line = tokens[0]
        self.status = tokens[1]
        self.action = tokens[2]
        self.office = tokens[3][1:]
        self.office4 = tokens[3]
        self.phenomena = tokens[4]
        self.significance = tokens[5]
        self.etn = int(tokens[6])
        self.begints = contime(tokens[7])
        self.endts = contime(tokens[8])
        # Not explicitly defined, but set later by product parsing logic
        self.year = None

    def s3(self):
        """Return a commonly used string representation."""
        return f"{self.phenomena}.{self.significance}.{self.etn}"

    def s2(self):
        """Return a commonly used string representation."""
        return f"{self.phenomena}.{self.significance}"

    def get_end_string(self, prod):
        """Return an appropriate end string for this VTEC"""
        if self.action in ["CAN", "EXP"]:
            return ""
        if self.endts is None:
            return "until further notice"
        fmt = "%b %-d, %-I:%M %p"
        if self.endts < (prod.valid + timedelta(hours=1)):
            fmt = "%-I:%M %p"
        if prod.tz is None:
            fmt = "%b %-d, %-H:%M"
        localts = self.endts
        if prod.tz is not None:
            localts = self.endts.astimezone(prod.tz)
        # A bit of complexity as offices may not implement daylight saving
        if prod.z is not None and prod.z.endswith("ST") and localts.dst():
            localts -= timedelta(hours=1)
        tt = prod.z if prod.z is not None else "UTC"
        return f"till {localts.strftime(fmt)} {tt}"

    def get_begin_string(self, prod):
        """Return an appropriate beginning string for this VTEC"""
        if self.begints is None:
            return ""
        fmt = "%b %-d, %-I:%M %p"
        if self.begints < (prod.valid + timedelta(hours=1)):
            fmt = "%-I:%M %p"
        localts = self.begints.astimezone(prod.tz)
        # A bit of complexity as offices may not implement daylight saving
        if prod.z.endswith("ST") and localts.dst():
            localts -= timedelta(hours=1)
        return f"valid at {localts.strftime(fmt)} {prod.z}"

    def url(self, year):
        """Generate a VTEC url string needed"""
        tt = year if self.year is None else self.year
        return (
            f"{tt}-{self.status}-{self.action}-{self.office4}-"
            f"{self.phenomena}-{self.significance}-{self.etn:04.0f}"
        )

    def get_id(self, year):
        """Return a custom string identifier for this VTEC product

        This is used by the Live client
        """
        tt = year if self.year is None else self.year
        return (
            f"{tt}-{self.office4}-{self.phenomena}-{self.significance}-"
            f"{self.etn:04.0f}"
        )

    def __str__(self):
        """Return string representation"""
        return self.line

    def get_ps_string(self):
        """Return the combination of Phenomena + Significance as string"""
        return get_ps_string(self.phenomena, self.significance)

    def get_action_string(self):
        """Return the action string"""
        return get_action_string(self.action)

    def product_string(self):
        """Return the combination of action and phenomena+significance"""
        return f"{self.get_action_string()} {self.get_ps_string()}"