text:
import argparse, csv, pickle
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import dataframe_image as dfi
import seaborn as sn
from script.feature_extraction.character_length import CharacterLength
from script.feature_extraction.feature_collector import FeatureCollector
from script.util import COLUMN_TWEET, COLUMN_LABEL

# setting up CLI
parser = argparse.ArgumentParser(description="Visualization")
parser.add_argument("input_file", help="path to the input csv file")
parser.add_argument("output_file", help="path to the output folder where images are saved")
parser.add_argument("-d", "--default_feat_visualizations", action="store_true", help="generates all png features visualization graphs")
parser.add_argument("-v", "--features_variance", action="store_true", help = "generates the csv data variance as png")
parser.add_argument("-des", "--data_description", action="store_true", help = "generates the csv description of the data as png")
parser.add_argument("-mg", "--group_means", action="store_true", help = "generates png image of the features group means")
parser.add_argument("-fmv", "--feature_mean_var", action="store_true", help = "generates png image of the features mean and variance")
parser.add_argument("-fc", "--feature_correlations", action="store_true", help = "generates png image of the features correlation before and after feature selection")
parser.add_argument("-p", "--pairplot_correlations", action="store_true", help = "generates png image of the pairwise relationship of the features distributions in the dataset")
parser.add_argument("-vir", "--tweets_virality", action="store_true", help = "generates png images of the virality of tweets by feature")
parser.add_argument("-t", "--time_virality", action="store_true", help = "generates png images of the virality of tweets by time features")
parser.add_argument("-dv", "--describe_virality", action="store_true", help = "generates png images of the virality description of tweets by features")

args = parser.parse_args()

# load data
df = pd.read_csv(args.input_file, quoting=csv.QUOTE_NONNUMERIC, lineterminator="\n")

def describe(data):
    """
    :param data: a csv file.
    :return: the csv general description of the data as png.
    """
    description = pd.DataFrame(data.describe())
    dfi.export(description, args.output_file + "/description_data.png")

if args.data_description:
    describe(df)

# clean, process, and group data features
df_clean = df[["likes_count", "replies_count", "retweets_count", "language", "video", "label", "date", "time"]]
# extract the amount of photos, urls, hashtags, and the hours when tweets were made
df_clean["photos"] = df["photos"].map(lambda x: len(x[1:-1].split(', ')))
df_clean["urls"] = df["urls"].map(lambda x: len(x[1:-1].split(', ')))
df_clean["hashtags"] = df["hashtags"].map(lambda x: len(x[1:-1].split(', ')))
df_clean["tweet_hour"] = pd.DataFrame(pd.DatetimeIndex(df['time']).hour.values)

# Variance can help us identify which features values change the most. Those that have
# a low variance means numbers stay pretty much same and will probably not tell us much.
# On the contrary, those with high variance can tell us what us going on with the data.
def variance(data, data2):
    """
    :param data: uncleaned .csv file
    :param data2: cleaned .csv file
    :return: the csv data variance as png for cleaned and uncleaned data.
    """
    var_img1 = pd.DataFrame({"Variance": data.var()})
    var_img2 = pd.DataFrame({"Variance cleaned": data2.var()})

    dfi.export(var_img1, args.output_file + "/features_variance1.png")
    dfi.export(var_img2, args.output_file + "/features_variance_cleaned.png")

# check the variance of initial df and df_clean
if args.features_variance:
    variance(df, df_clean)

# group the cleaned data by groups
groups = df_clean.groupby('label')

def groups_means(data):
    """
    :param data: a DataFrame file.
    :return: an image of the grouped means by label
    """
    fig, ax = plt.subplots()
    # groups means by language
    grouped_mean_lang = data.groupby('label').mean()
    img = np.log10(grouped_mean_lang).plot(kind='bar')
    plt.xticks(rotation=0, ha='right')
    plt.title("Features means grouped by label")
    plt.savefig(args.output_file + "/features_means_grouped_by_label.png")

if args.group_means:
    groups_means(df_clean)

def feature_mean_var_by_label(data):
    """
    :param data: a DataFrame file
    :return: generates png image of the features mean and variance
    """
    # group mean and variance by label which show interesting features to be those to the left
    feature_means_label = data.groupby('label').mean()
    feature_var_label = data.groupby('label').var()

    # means
    np.log10(feature_means_label).plot(kind='box')
    plt.xticks(rotation=18, ha='right')
    plt.title("Feature mean by label")
    plt.savefig(args.output_file + "/features_means_by_label.png")
    # variances
    np.log10(feature_var_label).plot(kind='box')
    plt.xticks(rotation=18, ha='right')
    plt.title("Feature variance by label")
    plt.savefig(args.output_file + "/features_variance_by_label.png")

if args.feature_mean_var:
    feature_mean_var_by_label(df_clean)

# heat map correlation of features after selection (fig. bottom)
def feature_correlation(data, data_cleaned):
    """
    :param data: a Dataframe file.
    :param data_cleaned: a DataFrame file after feature selection.
    :return: generates png image of the features correlation before and after feature selection.
    """

    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12))
    # Heatmap help us find the correlation between the different data features (fig. top)
    sn.heatmap(data.corr(), ax=ax1)
    ax1.set_title("Feature correlation initial tweets data")
    # Heat map correlation of features after selection (fig. bottom)
    sn.heatmap(data_cleaned.corr(), annot=True, ax=ax2)
    ax2.set_title("Feature correlation after selection")
    fig.tight_layout()
    plt.savefig(args.output_file + "/feature_selection_by_correlation.png")

if args.feature_correlations:
    feature_correlation(df, df_clean)

def pairplot_correlations(data, sample=30000):
    """
    :param data: a DataFrame file.
    :param sample: the amount of data points to sample from (default = 30000).
    :return: generates a pairwise plot relationship of the data set.
    """
    # pairwise relationship of the features distributions in the dataset.
    sn.pairplot(data.sample(sample), hue="label")
    plt.savefig(args.output_file + "/feature_selection_by_correlation_pairplot.png")

if args.pairplot_correlations:
    pairplot_correlations(df_clean)

# explore tweets virality by features relations
def scatter_viral(data, x, y, xlog=False, ylog=False, plot_title="", save=""):
    """
    :param data: a DataFrame file.
    :param x: feature to project on the x-axis.
    :param y: feature to project on the y-axis.
    :param xlog: bool. When xlog=True, projects the logarithm of values on the respective axis.
    :param ylog: bool. When xlog=True, projects the logarithm of values on the respective axis.
    :param plot_title: string sentence to set as plot's title.
    :param save: string non-spaced sentence to save the image followed by .png
    :return: generates a scatter plot of the virality relationship between the x and y features specified
    """
    fig, ax = plt.subplots(figsize=(7, 5))
    ax.margins(0.05)  # Optional, just adds 5% padding to the autoscaling
    if xlog:
        ax.set_xscale('log')
    if ylog:
        ax.set_yscale('log')

    for name, group in data:
        ax.plot(group[x], group[y], marker='o', linestyle='', ms=1, label=name, alpha=0.3)
    ax.set_xlabel(x)
    ax.set_ylabel(y)
    ax.title.set_text(plot_title)
    # reset the legend text name
    L = ax.legend()
    L.get_texts()[0].set_text("Not-viral")
    L.get_texts()[1].set_text("Viral")

    # save images to the visualization folder as indicated by the parser
    plt.savefig(args.output_file + save)

def tweets_virality_features(groups):
    scatter_viral(groups, "likes_count", "retweets_count", True, True,
                  plot_title="Feature selection for virality: retweets as a function of likes",
                  save="/retweets_likes.png")
    scatter_viral(groups, "likes_count", "replies_count", True, True,
                  plot_title="Feature selection for virality: replies as a function of likes",
                  save="/replies_likes.png")
    scatter_viral(groups, "retweets_count", "replies_count", True, True,
                  plot_title="Feature selection for virality: replies as a function of retweets",
                  save="/replies_retweets.png")
    scatter_viral(groups, "likes_count", "language", True, False,
                  plot_title="Feature selection for virality: language as a function of likes",
                  save="/language_likes.png")
    scatter_viral(groups, "likes_count", "hashtags", True, False,
                  plot_title="Feature selection for virality: hashtags as a function of likes",
                  save="/hashtags_likes.png")
    scatter_viral(groups, "likes_count", "photos", True, False,
                  plot_title="Feature selection for virality: photos as a function of likes", save="/photos_likes.png")
    scatter_viral(groups, "tweet_hour", "likes_count", False, True,
                  plot_title="likes as a function of tweet's hour", save="/likes_tweet_hour.png")

if args.tweets_virality:
    tweets_virality_features(groups)

# explore tweets virality by date and time features.
def count_tweets_per_creation_date(data):
    """
    :param data: a DataFrame file
    :return: generates histograms of the amount of tweets made per date and time and their virality.
    """
    data['tweets_per_year'] = pd.DataFrame(pd.DatetimeIndex(data['date']).year.values)
    data['tweets_per_month'] = pd.DataFrame(pd.DatetimeIndex(data['date']).month.values)
    data['tweets_per_day'] = pd.DataFrame(pd.DatetimeIndex(data['date']).day.values)
    data['tweets_per_hour'] = pd.DataFrame(pd.DatetimeIndex(data['time']).hour.values)

    column_dates = ["tweets_per_year", "tweets_per_month","tweets_per_day","tweets_per_hour"]
    plt.figure(figsize=(16, 10))

    # amount of tweets per date
    for i, name in enumerate(column_dates):
        plt.subplot(2,2, i+1)
        sn.histplot(data, x = name, discrete = True)
    plt.suptitle("Amount of tweets per creation date", fontsize=20)
    plt.savefig(args.output_file + "/tweets_amount_per_creation_date.png")

    # virality of tweets per date
    plt.figure(figsize=(16, 10))
    for i, name in enumerate(column_dates):
        plt.subplot(2, 2, i + 1)
        sn.countplot(name, data=data, hue='label')
    plt.suptitle("Virality of tweets per creation date", fontsize=20)
    plt.savefig(args.output_file + "/tweets_virality_per_creation_date.png")

if args.time_virality:
    count_tweets_per_creation_date(df_clean)

# explore tweets virality by describing the data to confirm the visualizations information
def describe_features_virality(df):
    """
    :param df: a DataFrame file.
    :return: generates the dataFrame image of the features description.
    """
    # select and group the features by label
    likes_to_viral = df[["likes_count", "label"]]
    retweets_to_viral = df[["retweets_count", "label"]]
    replies_to_viral = df[["replies_count", "label"]]

    likes_groups = likes_to_viral.groupby('label')
    retweets_groups = retweets_to_viral.groupby('label')
    replies_groups = replies_to_viral.groupby('label')

    # Learning: Likely not viral if likes < 50
    desc1 = likes_groups.describe()
    dfi.export(desc1, args.output_file + "/likes_description.png")

    # Learning: Likely not viral if retweets < 47
    desc2 = retweets_groups.describe()
    dfi.export(desc2, args.output_file + "/retweets_description.png")

    # Learning: Does not explain virality as well, given percentile distributions
    # are flat between true and false labeled tweets
    desc3 = replies_groups.describe()
    dfi.export(desc3, args.output_file + "/replies_description.png")

if args.describe_virality:
    describe_features_virality(df_clean)

# default (-d) image generation for features visualization
if args.default_feat_visualizations:
    variance(df, df_clean)
    describe(df)
    groups_means(df_clean)
    feature_mean_var_by_label(df_clean)
    feature_correlation(df, df_clean)
    pairplot_correlations(df_clean)
    tweets_virality_features(groups)
    count_tweets_per_creation_date(df_clean)

max_stars_repo_name: yesidc/MLinPractice

max_issues_repo_name: yesidc/MLinPractice

max_forks_repo_name: yesidc/MLinPractice


text:
"""
script to generate training data for the SVM classifying with training
images in the image_data/ diretcory with a folder for each person.

MIT License

Copyright (c) 2019 JinJie Chen

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

"""

import argparse
import numpy as np
import os
from edgetpu.basic.basic_engine import BasicEngine
from PIL import Image
import csv

def main(user_id, output_file='training_data.txt'):
    # initial the facenet TFLite model
    engine = BasicEngine("../src/models/facenet_edgetpu.tflite")
    # list of people (subdirectory folder names)
    people = [person for person in os.listdir("image_data/")] if user_id == "-1" else [str(user_id)]
    with open(output_file, 'a+') as f:
        writer = csv.writer(f)
        for person in people:
            image_names = [image for image in os.listdir("image_data/" + person)]
            # run inferece on each mage in the directory
            for image_name in image_names:
                image = Image.open("image_data/" + person + '/' + image_name)
                print("\t->" + person + '/' + image_name)
                # run inference
                engine.RunInference(np.array(image).flatten())
                value = np.zeros(513).astype(object)
                value[0] = str(person).replace('_', ' ')
                value[1:] = engine.get_raw_output()
                # append new label and face embedding pair of the image to the output file
                writer.writerow(value)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--user', help='user id, -1 for all')
    parser.add_argument('--output', help='output file file')
    args = parser.parse_args()
    main(args.user, args.output)

ext: py

reason: import numpy


text:
#!/usr/bin/env python3
#------------------------------------------------------------
# Programmer(s):  Daniel R. Reynolds @ SMU
#------------------------------------------------------------
# Copyright (c) 2019, Southern Methodist University.
# All rights reserved.
# For details, see the LICENSE file.
#------------------------------------------------------------
# matplotlib-based plotting utility function for
# hurricane test problem in the xy-plane

# imports
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
import matplotlib.pyplot as plt
from utilities_euler3D import *

# determine if running interactively
if __name__=="__main__":
  showplots = False
else:
  showplots = True

# set view for surface plots
elevation = 15
angle = 20

# set test constants
rho0  = 1.0
v0    = 10.0
Amp   = 25.0
gamma = 2.0
xl    = -1.0
xr    =  1.0
yl    = -1.0
yr    =  1.0

# utility function to create analytical solution
def analytical_solution(t,nx,ny):
  if (t == 0):
    t = 1e-14
  p0prime = Amp*gamma*rho0**(gamma-1.0)
  rthresh = 2.0*t*np.sqrt(p0prime)
  rho = np.zeros((nx,ny), dtype=float)
  mx  = np.zeros((nx,ny), dtype=float)
  my  = np.zeros((nx,ny), dtype=float)
  dx  = (xr-xl)/nx
  dy  = (yr-yl)/ny
  for j in range(ny):
    for i in range(nx):
      x = (i+0.5)*dx + xl
      y = (j+0.5)*dy + yl
      r = np.sqrt(x*x + y*y)
      if (r == 0.0):  # protect against division by zero
        r = 1e-14
      costheta = x/r
      sintheta = y/r
      if (r < rthresh):
        rho[i,j] = r*r / (8*Amp*t*t)
        mx[i,j] = rho[i,j] * (x + y) / (2*t)
        my[i,j] = rho[i,j] * (y - x) / (2*t)
      else:
        rho[i,j] = rho0
        mx[i,j] = rho0 * ( 2*t*p0prime*costheta +
                           np.sqrt(2*p0prime)*np.sqrt(r*r-2*t*t*p0prime)*sintheta )/r
        my[i,j] = rho0 * ( 2*t*p0prime*sintheta -
                           np.sqrt(2*p0prime)*np.sqrt(r*r-2*t*t*p0prime)*costheta )/r
  return [rho, mx, my]

# load solution data
nx, ny, nz, nchem, nt, xgrid, ygrid, zgrid, tgrid, rho, mx, my, mz, et, chem = load_data()

# output general information to screen
print('Generating plots for data set:')
print('  nx: ', nx)
print('  ny: ', ny)
print('  nt: ', nt)

# determine extents of plots
minmaxrho = [0.9*rho.min(), 1.1*rho.max()]
if (rho.min() == rho.max()):
    minmaxrho = [rho.min()-0.1, rho.max()+0.1]
minmaxmx  = [0.9*mx.min(), 1.1*mx.max()]
if (mx.min() == mx.max()):
    minmaxmx = [mx.min()-0.1, mx.max()+0.1]
minmaxmy  = [0.9*my.min(), 1.1*my.max()]
if (my.min() == my.max()):
    minmaxmy = [my.min()-0.1, my.max()+0.1]
minmaxet  = [0.9*et.min(), 1.1*et.max()]
if (et.min() == et.max()):
    minmaxet = [et.min()-0.1, et.max()+0.1]

# generate plots of solution
for tstep in range(nt):
    numfigs = 0

    print('time step', tstep+1, 'out of', nt)

    # get true solutions
    rhotrue, mxtrue, mytrue = analytical_solution(tgrid[tstep],nx,ny)

    # set string constants for current time, mesh sizes
    tstr  = repr(tstep)
    nxstr = repr(nx)
    nystr = repr(ny)

    # extract 2D velocity fields (computed and true)
    U = mx[:,:,nz//2,tstep]/rho[:,:,nz//2,tstep]
    Utrue = mxtrue/rhotrue
    V = my[:,:,nz//2,tstep]/rho[:,:,nz//2,tstep]
    Vtrue = mytrue/rhotrue
    speed = np.sqrt(U**2 + V**2)
    speedtrue = np.sqrt(Utrue**2 + Vtrue**2)

    # set filenames for graphics
    rhosurf  = 'rho_surface.'   + repr(tstep).zfill(4) + '.png'
    etsurf   = 'et_surface.'    + repr(tstep).zfill(4) + '.png'
    vstr     = 'velocity.'      + repr(tstep).zfill(4) + '.png'
    rhocont  = 'rho_contour.'   + repr(tstep).zfill(4) + '.png'
    etcont   = 'et_contour.'    + repr(tstep).zfill(4) + '.png'
    rho1dout = 'rho1d.'         + repr(tstep).zfill(4) + '.png'
    mx1dout  = 'mx1d.'          + repr(tstep).zfill(4) + '.png'
    my1dout  = 'my1d.'          + repr(tstep).zfill(4) + '.png'
    sp1dout  = 'speed1d.'       + repr(tstep).zfill(4) + '.png'

    # set x and z meshgrid objects
    X,Y = np.meshgrid(xgrid,ygrid)

    # surface plots
    numfigs += 1
    fig = plt.figure(numfigs)
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_surface(X, Y, rho[:,:,nz//2,tstep], rstride=1, cstride=1,
                    cmap=cm.jet, linewidth=0, antialiased=True, shade=True)
    ax.set_xlabel('x'); ax.set_ylabel('y'); ax.set_zlim((minmaxrho[0], minmaxrho[1]))
    ax.view_init(elevation,angle);
    plt.title(r'$\rho(x,y)$ at output ' + tstr + ', mesh = ' + nxstr + 'x' + nystr)
    plt.savefig(rhosurf)

    numfigs += 1
    fig = plt.figure(numfigs)
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_surface(X, Y, et[:,:,nz//2,tstep], rstride=1, cstride=1,
                    cmap=cm.jet, linewidth=0, antialiased=True, shade=True)
    ax.set_xlabel('x'); ax.set_ylabel('y'); ax.set_zlim((minmaxet[0], minmaxet[1]))
    ax.view_init(elevation,angle);
    plt.title(r'$e_t(x,y)$ at output ' + tstr + ', mesh = ' + nxstr + 'x' + nystr)
    plt.savefig(etsurf)

    # stream plots
    numfigs += 1
    fig = plt.figure(numfigs,figsize=(12,4))
    ax1 = fig.add_subplot(121)
    lw = speed / speed.max()
    ax1.streamplot(X, Y, U, V, color='b', linewidth=lw)
    ax1.set_xlabel('x'); ax1.set_ylabel('y'); ax1.set_aspect('equal')
    ax2 = fig.add_subplot(122)
    lw = speedtrue / speedtrue.max()
    ax2.streamplot(X, Y, Utrue, Vtrue, color='k', linewidth=lw)
    ax2.set_xlabel('z'); ax2.set_ylabel('x'); ax2.set_aspect('equal')
    plt.suptitle(r'$\mathbf{v}(x,y)$ (left) vs $\mathbf{v}_{true}(x,y)$ (right) at output ' + tstr + ', mesh = ' + nxstr + 'x' + nystr)
    plt.savefig(vstr)

    # contour plots
    # numfigs += 1
    # fig = plt.figure(numfigs,figsize=(12,4))
    # ax1 = fig.add_subplot(121)
    # ax1.contourf(X, Y, rho[:,:,nz//2,tstep])
    # ax1.colorbar();  ax1.set_xlabel('x'); ax1.set_ylabel('y'); ax1.set_axis('equal')
    # ax2 = fig.add_subplot(122)
    # ax2.contourf(X, Y, rhotrue)
    # ax2.colorbar();  ax2.set_xlabel('x'); ax2.set_ylabel('y'); ax2.set_axis('equal')
    # plt.suptitle(r'$\rho(x,y)$ (left) vs $\rho_{true}(x,y)$ (right) at output ' + tstr + ', mesh = ' + nxstr + 'x' + nystr)
    # plt.savefig(rhotr)

    # numfigs += 1
    # fig = plt.figure(numfigs)
    # plt.contourf(X, Y, et[:,:,nz//2,tstep])
    # plt.colorbar();  plt.xlabel('x'); plt.ylabel('y'); plt.axis('equal')
    # plt.title(r'$e_t(x,y)$ at output ' + tstr + ', mesh = ' + nxstr + 'x' + nystr)
    # plt.savefig(etcont)

    # line/error plots
    rho1d = rho[:,ny//2,nz//2,tstep]
    mx1d  = mx[:,ny//2,nz//2,tstep]
    my1d  = my[:,ny//2,nz//2,tstep]
    sp1d  = speed[:,ny//2]
    rhotrue1d = rhotrue[:,ny//2]
    mxtrue1d  = mxtrue[:,ny//2]
    mytrue1d  = mytrue[:,ny//2]
    sptrue1d  = speedtrue[:,ny//2]

    numfigs += 1
    fig = plt.figure(numfigs,figsize=(12,4))
    ax1 = fig.add_subplot(121)
    ax1.plot(xgrid,rho1d,'b--',xgrid,rhotrue1d,'k-')
    ax1.legend(('computed','analytical'))
    ax1.set_xlabel('x'); ax1.set_ylabel(r'$\rho(x)$')
    ax2 = fig.add_subplot(122)
    ax2.semilogy(xgrid,np.abs(rho1d-rhotrue1d)+1e-16)
    ax1.set_xlabel('x'); ax1.set_ylabel(r'$|\rho-\rho_{true}|$')
    plt.suptitle(r'$\rho(x)$ and error at output ' + tstr + ', mesh = ' + nxstr)
    plt.savefig(rho1dout)

    numfigs += 1
    fig = plt.figure(numfigs,figsize=(12,4))
    ax1 = fig.add_subplot(121)
    ax1.plot(xgrid,mx1d,'b--',xgrid,mxtrue1d,'k-')
    ax1.legend(('computed','analytical'))
    ax1.set_xlabel('x'); ax1.set_ylabel(r'$m_x(x)$')
    ax2 = fig.add_subplot(122)
    ax2.semilogy(xgrid,np.abs(mx1d-mxtrue1d)+1e-16)
    ax2.set_xlabel('x'); ax2.set_ylabel(r'$|m_x-m_{x,true}|$')
    plt.suptitle(r'$m_x(x)$ and error at output ' + tstr + ', mesh = ' + nxstr)
    plt.savefig(mx1dout)

    numfigs += 1
    fig = plt.figure(numfigs,figsize=(12,4))
    ax1 = fig.add_subplot(121)
    ax1.plot(xgrid,my1d,'b--',xgrid,mytrue1d,'k-')
    ax1.legend(('computed','analytical'))
    ax1.set_xlabel('x'); ax1.set_ylabel(r'$m_y(x)$')
    ax2 = fig.add_subplot(122)
    ax2.semilogy(xgrid,np.abs(my1d-mytrue1d)+1e-16)
    ax2.set_xlabel('x'); ax2.set_ylabel(r'$|m_y-m_{y,true}|$')
    plt.suptitle(r'$m_y(x)$ and error at output ' + tstr + ', mesh = ' + nxstr)
    plt.savefig(my1dout)

    numfigs += 1
    fig = plt.figure(numfigs,figsize=(12,4))
    ax1 = fig.add_subplot(121)
    ax1.plot(xgrid,sp1d,'b--',xgrid,sptrue1d,'k-')
    ax1.legend(('computed','analytical'))
    ax1.set_xlabel('x'); ax1.set_ylabel('s(x)')
    ax2 = fig.add_subplot(122)
    ax2.semilogy(xgrid,np.abs(sp1d-sptrue1d)+1e-16)
    ax2.set_xlabel('x'); ax2.set_ylabel(r'$|s-s_{true}|$')
    plt.suptitle(r'$s(x)$ and error at output ' + tstr + ', mesh = ' + nxstr)
    plt.savefig(sp1dout)

    if (showplots):
      plt.show()
    for i in range(1,numfigs+1):
      plt.figure(i), plt.close()

##### end of script #####


text:
import matplotlib.pyplot as plt
from matplotlib import gridspec
from RhythmCount import helpers as hlp
from RhythmCount import data_processing as dproc
import copy
import random
import numpy as np
import matplotlib.dates as mdates
import matplotlib.dates as md
from collections import OrderedDict

plt.rc('font', size=15)

def plot_model(df, model_type, n_components, title='', plot_CIs=True, repetitions=20, save_file_to='model.pdf',
               maxiter=5000, maxfun=5000, method='nm', period=24):
    rows, cols = hlp.get_factors(1)
    fig = plt.figure(figsize=(8 * cols, 8 * rows))
    gs = gridspec.GridSpec(rows, cols)

    results, df_result, _ = dproc.fit_to_model(df, n_components, model_type, period, maxiter, maxfun, method, 0)

    # plot
    ax = fig.add_subplot(gs[0])
    if plot_CIs:
        CIs = subplot_confidence_intervals(df, n_components, model_type, ax, repetitions=repetitions, maxiter=maxiter,
                                           maxfun=maxfun, period=period, method=method)
    subplot_model(df['X'], df['Y'], df_result['X_test'], df_result['Y_test'], ax, color='blue', title=title,
                  fit_label='fitted curve')

    ax_list = fig.axes
    for ax in ax_list:
        ax.legend(loc='upper left')
    fig.tight_layout()
    plt.show()

    # save
    try:
        hlp.make_results_dir()
        fig.savefig(r'results\/' + save_file_to)
    except:
        print("Can not save plot.")

    if plot_CIs:
        return CIs

def plot_models(dfs, model_type, n_components, title=[''], rows=1, cols=1, period=24,maxiter=5000,maxfun=5000,method='nm',disp=0,plot_CIs=True, repetitions=50,
                save_file_to='win.pdf'):
    fig = plt.figure(figsize=(8 * cols, 8 * rows))
    gs = gridspec.GridSpec(rows, cols)

    i = 0
    for df in dfs:
        _, df_result, _ = dproc.fit_to_model(df, n_components[i], model_type[i], period, maxiter, maxfun, method, disp)

        # plot
        ax = fig.add_subplot(gs[i])
        if plot_CIs:
            subplot_confidence_intervals(df, n_components[i], model_type[i], ax, repetitions=repetitions,period=period,maxiter=maxiter,maxfun=maxfun,method=method)

        subplot_model(df['X'], df['Y'], df_result['X_test'], df_result['Y_test'], ax, color='blue', title=title[i], fit_label='prilagojena krivulja',raw_label='izvorni podatki')
        i = i + 1

    ax_list = fig.axes
    for ax in ax_list:
        ax.legend(loc='upper left')
    fig.tight_layout()
    plt.show()

    # save
    try:
        hlp.make_results_dir()
        fig.savefig(r'results\/' + save_file_to)
    except:
        print("Can not save plot.")

def plot_confidence_intervals(df, model_type, n_components, title='', repetitions=20, maxiter=5000, maxfun=5000,
                              period=24, method='nm', save_file_to='CIs.pdf'):
    rows, cols = hlp.get_factors(1)
    fig = plt.figure(figsize=(8 * cols, 8 * rows))
    gs = gridspec.GridSpec(rows, cols)

    ax = fig.add_subplot(gs[0])
    Y = df['Y']
    results, df_result, X_fit_test = dproc.fit_to_model(df, n_components, model_type, period, maxiter, maxfun, method,
                                                        0)

    # CI
    res2 = copy.deepcopy(results)
    params = res2.params
    CIs = dproc.calculate_confidence_intervals(df, n_components, model_type, repetitions, maxiter, maxfun, method,
                                               period)

    N2 = round(10 * (0.7 ** n_components) + 4)
    P = np.zeros((len(params), N2))

    i = 0
    for index, CI in CIs.iterrows():
        P[i, :] = np.linspace(CI[0], CI[1], N2)
        i = i + 1

    param_samples = hlp.lazy_cartesian_product(P)
    size = param_samples.max_size
    N = round(df.shape[0] - df.shape[0] / 3)

    for i in range(0, N):
        j = random.randint(0, size)
        p = param_samples.entry_at(j)
        res2.initialize(results.model, p)
        if model_type == 'zero_nb' or model_type == "zero_poisson":
            Y_test_CI = res2.predict(X_fit_test, exog_infl=X_fit_test)
        else:
            Y_test_CI = res2.predict(X_fit_test)
        if i == 0:
            ax.plot(df_result['X_test'], Y_test_CI, color='tomato', alpha=0.05, linewidth=0.1,
                    label='confidence intervals')
        else:
            ax.plot(df_result['X_test'], Y_test_CI, color='tomato', alpha=0.05, linewidth=0.1)

    subplot_model(df['X'], Y, df_result['X_test'], df_result['Y_test'], ax, title=title, plot_model=False)

    ax_list = fig.axes
    for ax in ax_list:
        ax.legend(loc='upper left')
    fig.tight_layout()
    plt.show()

    # save
    try:
        hlp.make_results_dir()
        fig.savefig(r'results\/' + save_file_to)
    except:
        print("Can not save plot.")

    return CIs

def subplot_confidence_intervals(df, n_components, model_type, ax, repetitions=20, maxiter=5000, maxfun=5000, period=24,
                                 method='nm'):
    results, df_result, X_fit_test = dproc.fit_to_model(df, n_components, model_type, period, maxiter, maxfun, method,
                                                        0)

    # CI
    res2 = copy.deepcopy(results)
    params = res2.params
    CIs = dproc.calculate_confidence_intervals(df, n_components, model_type, repetitions, maxiter, maxfun, method,
                                               period)

    N2 = round(10 * (0.7 ** n_components) + 4)
    P = np.zeros((len(params), N2))

    i = 0
    for index, CI in CIs.iterrows():
        P[i, :] = np.linspace(CI[0], CI[1], N2)
        i = i + 1

    param_samples = hlp.lazy_cartesian_product(P)
    size = param_samples.max_size
    N = round(df.shape[0] - df.shape[0] / 3)

    for i in range(0, N):
        j = random.randint(0, size)
        p = param_samples.entry_at(j)
        res2.initialize(results.model, p)
        if model_type == 'zero_nb' or model_type == "zero_poisson":
            Y_test_CI = res2.predict(X_fit_test, exog_infl=X_fit_test)
        else:
            Y_test_CI = res2.predict(X_fit_test)
        if i == 0:
            ax.plot(df_result['X_test'], Y_test_CI, color='tomato', alpha=0.3, linewidth=0.15)
        else:
            ax.plot(df_result['X_test'], Y_test_CI, color='tomato', alpha=0.3, linewidth=0.15)

    return CIs

def subplot_model(X, Y, X_test, Y_test, ax, plot_measurements=True, plot_measurements_with_color=False, plot_model=True,
                  title='', color='black', fit_label='', raw_label='raw data'):
    ax.set_title(title)
    ax.set_xlabel('Dan')
    ax.set_ylabel('Število pozitivnih testov')

    if plot_measurements:
        if plot_measurements_with_color:
            ax.plot(X, Y,'.', markersize=1, color=color, label=raw_label)
        else:
            ax.plot(X, Y, '.', markersize=1, color='black', label=raw_label)
    if plot_model:
        ax.plot(X_test, Y_test, label=fit_label, color=color)

    ax.set_xlim(0, 6)
    ax.set_xticks([0,1,2,3,4,5,6])
    ax.set_xticklabels(['Ponedeljek','Torek','Sreda','Četrtek','Petek','Sobota','Nedelja'])

    return ax

def plot_raw_data(df, title, hour_intervals, save_file_to='raw.pdf'):
    rows, cols = hlp.get_factors(1)
    fig = plt.figure(figsize=(8 * cols, 8 * rows))
    gs = gridspec.GridSpec(rows, cols)

    var = df[['Y']].to_numpy().var()
    mean = df[['Y']].to_numpy().mean()
    print(title, " Varianca: ", var, " Povprečje: ", mean)

    ax = fig.add_subplot(gs[0])
    ax.scatter(df.date.head(500), df.Y.head(500), c='blue', s=1)

    date_form = md.DateFormatter("%d.%m.%Y")
    ax.xaxis.set_major_formatter(date_form)
    ax.xaxis.set_major_locator(mdates.HourLocator(interval=hour_intervals))
    plt.xticks(rotation=45)
    plt.xlabel('Dan')
    plt.ylabel('Število pozitivnih testov')
    plt.title(title)

    fig.tight_layout()
    plt.show()

    # save
    try:
        hlp.make_results_dir()
        fig.savefig(r'results\/' + save_file_to)
    except:
        print("Can not save plot.")


text:
# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np
import torch

from mmocr.models.textdet.dense_heads import DRRGHead

def test_drrg_head():
    in_channels = 10
    drrg_head = DRRGHead(in_channels)
    assert drrg_head.in_channels == in_channels
    assert drrg_head.k_at_hops == (8, 4)
    assert drrg_head.num_adjacent_linkages == 3
    assert drrg_head.node_geo_feat_len == 120
    assert np.allclose(drrg_head.pooling_scale, 1.0)
    assert drrg_head.pooling_output_size == (4, 3)
    assert np.allclose(drrg_head.nms_thr, 0.3)
    assert np.allclose(drrg_head.min_width, 8.0)
    assert np.allclose(drrg_head.max_width, 24.0)
    assert np.allclose(drrg_head.comp_shrink_ratio, 1.03)
    assert np.allclose(drrg_head.comp_ratio, 0.4)
    assert np.allclose(drrg_head.comp_score_thr, 0.3)
    assert np.allclose(drrg_head.text_region_thr, 0.2)
    assert np.allclose(drrg_head.center_region_thr, 0.2)
    assert drrg_head.center_region_area_thr == 50
    assert np.allclose(drrg_head.local_graph_thr, 0.7)

    # test forward train
    num_rois = 16
    feature_maps = torch.randn((2, 10, 128, 128), dtype=torch.float)
    x = np.random.randint(4, 124, (num_rois, 1))
    y = np.random.randint(4, 124, (num_rois, 1))
    h = 4 * np.ones((num_rois, 1))
    w = 4 * np.ones((num_rois, 1))
    angle = (np.random.random_sample((num_rois, 1)) * 2 - 1) * np.pi / 2
    cos, sin = np.cos(angle), np.sin(angle)
    comp_labels = np.random.randint(1, 3, (num_rois, 1))
    num_rois = num_rois * np.ones((num_rois, 1))
    comp_attribs = np.hstack([num_rois, x, y, h, w, cos, sin, comp_labels])
    comp_attribs = comp_attribs.astype(np.float32)
    comp_attribs_ = comp_attribs.copy()
    comp_attribs = np.stack([comp_attribs, comp_attribs_])
    pred_maps, gcn_data = drrg_head(feature_maps, comp_attribs)
    pred_labels, gt_labels = gcn_data
    assert pred_maps.size() == (2, 6, 128, 128)
    assert pred_labels.ndim == gt_labels.ndim == 2
    assert gt_labels.size()[0] * gt_labels.size()[1] == pred_labels.size()[0]
    assert pred_labels.size()[1] == 2

    # test forward test
    with torch.no_grad():
        feat_maps = torch.zeros((1, 10, 128, 128))
        drrg_head.out_conv.bias.data.fill_(-10)
        preds = drrg_head.single_test(feat_maps)
        assert all([pred is None for pred in preds])

    # test get_boundary
    edges = np.stack([np.arange(0, 10), np.arange(1, 11)]).transpose()
    edges = np.vstack([edges, np.array([1, 0])])
    scores = np.ones(11, dtype=np.float32) * 0.9
    x1 = np.arange(2, 22, 2)
    x2 = x1 + 2
    y1 = np.ones(10) * 2
    y2 = y1 + 2
    comp_scores = np.ones(10, dtype=np.float32) * 0.9
    text_comps = np.stack([x1, y1, x2, y1, x2, y2, x1, y2,
                           comp_scores]).transpose()
    outlier = np.array([50, 50, 52, 50, 52, 52, 50, 52, 0.9])
    text_comps = np.vstack([text_comps, outlier])

    (C, H, W) = (10, 128, 128)
    img_metas = [{
        'img_shape': (H, W, C),
        'ori_shape': (H, W, C),
        'pad_shape': (H, W, C),
        'filename': '<demo>.png',
        'scale_factor': np.array([1, 1, 1, 1]),
        'flip': False,
    }]
    results = drrg_head.get_boundary(
        edges, scores, text_comps, img_metas, rescale=True)
    assert 'boundary_result' in results.keys()

max_stars_repo_name: yuexy/mmocr

max_issues_repo_name: yuexy/mmocr

max_forks_repo_name: yuexy/mmocr


text:
#!/usr/bin/env python3
from __future__ import print_function
from __future__ import division
import rospy
import rosbag
import numpy as np
import matplotlib.pyplot as plt
import matplotlib; matplotlib.use('TkAgg')
from waypoint_generation_library import WaypointGen
from scipy import linalg
from nav_msgs.msg import Odometry
from geometry_msgs.msg import Quaternion
from sensor_msgs.msg import Imu
from tf.transformations import euler_from_quaternion, quaternion_from_euler
from mpl_toolkits import mplot3d

WaypointGeneration = WaypointGen()
waypoints, desVel, desAcc, timeVec = WaypointGeneration.waypoint_calculation()
desiredPos = WaypointGeneration.desiredPos

# plot the waypoints
figPos = plt.figure()
axPos = plt.axes(projection = '3d')
axPos.plot3D(desiredPos[:,0], desiredPos[:,1], desiredPos[:,2], 'ro')

pnt3d = axPos.scatter(waypoints[:,0], waypoints[:,1], waypoints[:,2], c = timeVec)
cbar = plt.colorbar(pnt3d)
cbar.set_label("Time [sec]")
# label the axes and give title
axPos.set_xlabel('X-Axis [m]')
axPos.set_ylabel('Y-Axis [m]')
axPos.set_zlabel('Z-Axis [m]')
axPos.set_title('Minimum Jerk Position Waypoints')

# plot the desired kinematics
figOtherKinematics = plt.figure()
figOtherKinematics.suptitle('Desired Kinematics in Inertial Frame')
# desired position waypoints
axPos = plt.subplot(311)
axPos.plot(timeVec, waypoints[:,0], '-r', label = '$x_b$')
axPos.plot(timeVec, waypoints[:,1], '-k', label = '$y_b$')
axPos.plot(timeVec, waypoints[:,2], '-b', label = '$z_b$')
# add the yaw legend
axPos.plot(np.nan, '-g', label = 'yaw')
axPos.legend(loc = 0)
plt.grid()
plt.xlabel('Time [sec]')
plt.ylabel('Position [m]')
# plt.title('Desired Position in Inertial Frame')
# desired yaw
axYaw = axPos.twinx()
axYaw.plot(timeVec, waypoints[:,3], '-g')
axYaw.set_ylabel('Yaw [rad]')

# desired velocity waypoints
axVel = plt.subplot(312)
axVel.plot(timeVec, desVel[:,0], '-r', label = '$v_{x,b}$')
axVel.plot(timeVec, desVel[:,1], '-k', label = '$v_{y,b}$')
axVel.plot(timeVec, desVel[:,2], '-b', label = '$v_{z,b}$')
# add the yaw legend
axVel.plot(np.nan, '-g', label = '$yaw_{rate}$')
axVel.legend(loc = 0)
plt.grid()
plt.xlabel('Time [sec]')
plt.ylabel('Velocity [m/s]')
# plt.title('Desired Velocity in Inertial Frame')
# desired yaw
axYawRate = axVel.twinx()
axYawRate.plot(timeVec, desVel[:,3], '-g')
axYawRate.set_ylabel('Yaw [rad/s]')

# desired acceleration waypoints
axAcc = plt.subplot(313)
axAcc.plot(timeVec, desAcc[:,0], '-r', label = '$a_{x,b}$')
axAcc.plot(timeVec, desAcc[:,1], '-k', label = '$a_{y,b}$')
axAcc.plot(timeVec, desAcc[:,2], '-b', label = '$a_{z,b}$')
# add the yaw legend
axAcc.plot(np.nan, '-g', label = '$yaw_{acc}$')
axAcc.legend(loc = 0)
plt.grid()
plt.xlabel('Time [sec]')
plt.ylabel('Acceleration [$m/s^2$]')
# plt.title('Desired Acceleration in Inertial Frame')
# desired yaw
axYawRate = axAcc.twinx()
axYawRate.plot(timeVec, desAcc[:,3], '-g')
axYawRate.set_ylabel('Yaw [$rad/s^2$]')
plt.show()

reason: import numpy,from scipy


text:
import numpy as np
from copy import deepcopy

class SymbolicRegressionFitness:

	def __init__( self, X_train, y_train ):
		self.X_train = X_train
		self.y_train = y_train
		self.elite = None
		self.evaluations = 0

	def Evaluate( self, individual ):

		self.evaluations = self.evaluations + 1

		output = individual.GetOutput( self.X_train )

		mean_squared_error = np.mean ( np.square( self.y_train - output ) )
		individual.fitness = mean_squared_error

		if not self.elite or individual.fitness < self.elite.fitness:
			del self.elite
			self.elite = deepcopy(individual)

		return mean_squared_error

max_stars_repo_name: thbolijn/SimpleGP

max_issues_repo_name: thbolijn/SimpleGP

max_forks_repo_name: thbolijn/SimpleGP


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Sep  9 13:37:25 2020

@author: Riedel
"""

import sys
from readreflex.readreflex import radargram
import numpy as np
import matplotlib.pyplot as plt

#for reading seismic formats:
import os
from tqdm import tqdm

filepath=r'exampledata\testdata.DAT'
calibdata=radargram()
if os.path.isfile(filepath+'.hdf5'):
    calibdata.load_hdf5(filepath)
else:
    calibdata.read_data_file(filepath,version=9)
    calibdata.save(filepath)

    calibdata.header['xoffset']=0

#copy the data
calibcopy=calibdata.copy()
#upsample it
#calibcopy.time_resampling(16*512)
#plot it
calibcopy.radarplot()

calibcopy.freq_times_powersum(2)
test=[]
for i in tqdm(range(calibcopy.traces.shape[0])):
    test.append(calibcopy.spectrofuncs(i))

max_stars_repo_name: GGDRriedel/readreflexw

max_issues_repo_name: GGDRriedel/readreflexw

max_forks_repo_name: GGDRriedel/readreflexw


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 21 20:24:16 2020

@author: Hossameldin Mohammed & Mohamed Kamel
"""
import matplotlib.pyplot as plt
from matplotlib import animation, rc
rc('animation', html='jshtml')
from IPython.display import HTML
#from IPython.display import display, clear_output
import PIL
import zarr
from pathlib import Path
import numpy as np
import pandas as pd
pd.set_option('display.max_columns', None)
#import scipy as sp
import itertools as it
import seaborn as sns
from l5kit.data import ChunkedDataset, LocalDataManager
from l5kit.dataset import EgoDataset, AgentDataset
from l5kit.rasterization import build_rasterizer
from l5kit.configs import load_config_data
from l5kit.visualization import draw_trajectory, TARGET_POINTS_COLOR
from l5kit.geometry import transform_points
#from tqdm import tqdm
#from collections import Counter
from l5kit.data import PERCEPTION_LABELS
#from prettytable import PrettyTable

from IPython.display import display, clear_output

import os

# import gc
# import os
# from pathlib import Path
# import random
# import sys

# from tqdm.notebook import tqdm
# import numpy as np
# import pandas as pd
# import scipy as sp

# import matplotlib.pyplot as plt
# import seaborn as sns

# from IPython.core.display import display, HTML

# # --- plotly ---
# from plotly import tools, subplots
# import plotly.offline as py
# py.init_notebook_mode(connected=True)
# import plotly.graph_objs as go
# import plotly.express as px
# import plotly.figure_factory as ff
# import plotly.io as pio
# pio.templates.default = "plotly_dark"

# # --- models ---
# from sklearn import preprocessing
# from sklearn.model_selection import KFold
# import lightgbm as lgb
# import xgboost as xgb
# import catboost as cb

# pd.set_option('max_columns', 50)

os.environ["L5KIT_DATA_FOLDER"] = "data"

cfg = load_config_data("examples/visualisation/visualisation_config.yaml")
print(cfg)

# Loading sample data for EDA
# set env variable for data

dm = LocalDataManager()
dataset_path = dm.require('scenes/sample.zarr')
zarr_dataset = ChunkedDataset(dataset_path)
zarr_dataset.open()
print(zarr_dataset)

frames = zarr_dataset.frames
agents = zarr_dataset.agents
scenes = zarr_dataset.scenes
# tl_faces = zarr_dataset.tl_faces

# Visualization Functions

def animate_solution(images, timestamps=None):
    def animate(i):
        changed_artifacts = [im]
        im.set_data(images[i])
        if timestamps is not None:
            time_text.set_text(timestamps[i])
            changed_artifacts.append(im)
        return tuple(changed_artifacts)

    fig, ax = plt.subplots()
    im = ax.imshow(images[0])
    if timestamps is not None:
        time_text = ax.text(0.02, 0.95, "", transform=ax.transAxes)

    anim = animation.FuncAnimation(fig, animate, frames=len(images), interval=60, blit=True)

    # To prevent plotting image inline.
    plt.close()
    return anim

def visualize_rgb_image(dataset, index, title="", ax=None):
    """Visualizes Rasterizer's RGB image"""
    data = dataset[index]
    im = data["image"].transpose(1, 2, 0)
    im = dataset.rasterizer.to_rgb(im)

    if ax is None:
        fig, ax = plt.subplots()
    if title:
        ax.set_title(title)
    ax.imshow(im[::-1])

def create_animate_for_indexes(dataset, indexes):
    images = []
    timestamps = []

    for idx in indexes:
        data = dataset[idx]
        im = data["image"].transpose(1, 2, 0)
        im = dataset.rasterizer.to_rgb(im)
        target_positions_pixels = transform_points(data["target_positions"] + data["centroid"][:2], data["world_to_image"])
        center_in_pixels = np.asarray(cfg["raster_params"]["ego_center"]) * cfg["raster_params"]["raster_size"]
        draw_trajectory(im, target_positions_pixels, data["target_yaws"], TARGET_POINTS_COLOR)
        clear_output(wait=True)
        images.append(PIL.Image.fromarray(im[::-1]))
        timestamps.append(data["timestamp"])

    anim = animate_solution(images, timestamps)
    return anim

def create_animate_for_scene(dataset, scene_idx):
    indexes = dataset.get_scene_indices(scene_idx)
    return create_animate_for_indexes(dataset, indexes)

# Prepare all rasterizer and EgoDataset for each rasterizer
rasterizer_dict = {}
dataset_dict = {}

rasterizer_type_list = ["py_satellite", "satellite_debug", "py_semantic", "semantic_debug", "box_debug", "stub_debug"]

for i, key in enumerate(rasterizer_type_list):
    # print("key", key)
    cfg["raster_params"]["map_type"] = key
    rasterizer_dict[key] = build_rasterizer(cfg, dm)
    dataset_dict[key] = EgoDataset(cfg, zarr_dataset, rasterizer_dict[key])

# default lane color is "light yellow" (255, 217, 82).
# green, yellow, red color on lane is to show trafic light condition.
# orange box represents crosswalk

fig, axes = plt.subplots(2, 3, figsize=(15, 10))
axes = axes.flatten()
for i, key in enumerate(["stub_debug", "satellite_debug", "semantic_debug", "box_debug", "py_satellite", "py_semantic"]):
    visualize_rgb_image(dataset_dict[key], index=0, title=f"{key}: {type(rasterizer_dict[key]).__name__}", ax=axes[i])
fig.show()

# Scenes animations
# That will work just fine in the notebook
dataset = dataset_dict["py_semantic"]
plt.rcParams['animation.embed_limit'] = 4**128
SceneIndex = 10
print("scene_idx", SceneIndex)
anim = create_animate_for_scene(dataset, SceneIndex)
display(HTML(anim.to_jshtml()))

# Agents EDA

agent = agents[0]
PERCEPTION_LABELS = [
    "NOT_SET",
    "UNKNOWN",
    "DONTCARE",
    "CAR",
    "VAN",
    "TRAM",
    "BUS",
    "TRUCK",
    "EMERGENCY_VEHICLE",
    "OTHER_VEHICLE",
    "BICYCLE",
    "MOTORCYCLE",
    "CYCLIST",
    "MOTORCYCLIST",
    "PEDESTRIAN",
    "ANIMAL",
    "DONTCARE",
]
DATA_ROOT = Path("C:/Users/Omar/Documents/GitHub/Kaggle-Lyft/data")
#A robust and fast interface to load l5kit data into  Pandas dataframes
class BaseParser:

    field = "scenes"
    dtypes = {}

    def __init__(self, start=0, end=None, chunk_size=1000, max_chunks=1000, root=DATA_ROOT,
                 zarr_path="scenes/sample.zarr"):

        self.start = start
        self.end = end
        self.chunk_size = chunk_size
        self.max_chunks = max_chunks

        self.root = Path(root)
        assert self.root.exists(), "There is nothing at {}!".format(self.root)
        self.zarr_path = Path(zarr_path)

    def parse(self):
        raise NotImplementedError

    def to_pandas(self, start=0, end=None, chunk_size=None, max_chunks=None):
        start = start or self.start
        end = end or self.end
        chunk_size = chunk_size or self.chunk_size
        max_chunks = max_chunks or self.max_chunks

        if not chunk_size or not max_chunks:  # One shot load, suitable for small zarr files
            df = zarr.load(self.root.joinpath(self.zarr_path).as_posix()).get(self.field)
            df = df[start:end]
            df = map(self.parse, df)
        else:  # Chunked load, suitable for large zarr files
            df = []
            with zarr.open(self.root.joinpath(self.zarr_path).as_posix(), "r") as zf:
                end = start + max_chunks * chunk_size if end is None else min(end, start + max_chunks * chunk_size)
                for i_start in range(start, end, chunk_size):
                    items = zf[self.field][i_start: min(i_start + chunk_size, end)]
                    items = map(self.parse, items)
                    df.append(items)
            df = it.chain(*df)

        df = pd.DataFrame.from_records(df)
        for col, col_dtype in self.dtypes.items():
            df[col] = df[col].astype(col_dtype, copy=False)
        return df

class AgentParser(BaseParser):
    field = "agents"

    @staticmethod
    def parse(agent):
        frame_dict = {
            'centroid_x': agent[0][0],
            'centroid_y': agent[0][1],
            'extent_x': agent[1][0],
            'extent_y': agent[1][1],
            'extent_z': agent[1][2],
            'yaw': agent[2],
            "velocity_x": agent[3][0],
            "velocity_y": agent[3][1],
            "track_id": agent[4],
        }
        for p_label, p in zip(PERCEPTION_LABELS, agent[5]):
            frame_dict["{}".format(p_label)] = p
        return frame_dict

    def to_pandas(self, start=0, end=None, chunk_size=None, max_chunks=None, frame=None):
        if frame is not None:
            start = int(frame.agent_index_interval_start)
            end = int(frame.agent_index_interval_end)

        df = super().to_pandas(start=start, end=end, chunk_size=chunk_size, max_chunks=max_chunks)
        return df

ap = AgentParser()
agents_df = ap.to_pandas(frame=None)
agents_df.head()
agents_df.columns
# Agents EDA
agents_df.describe()
agents_df.shape
agents_df.info()
agents_labels = [agents_df.CAR.sum(), agents_df.PEDESTRIAN.sum(), agents_df.CYCLIST.sum()]
colormap = plt.cm.magma
corr_matrix = ["centroid_x", "centroid_y", "extent_x", "extent_y", "extent_z", "yaw", 'velocity_x', 'velocity_y', "CAR","PEDESTRIAN","CYCLIST"  ]
plt.figure(figsize=(20,20));
plt.title('Pearson correlation of features', y=1.0, size=14);
sns.heatmap(agents_df[corr_matrix].corr(),linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True)
sns.scatterplot(agents_df["centroid_x"],agents_df["centroid_y"])

agents_CPC_df = agents_df.loc[((agents_df.CAR>= 0.5)|(agents_df.PEDESTRIAN>= 0.5)|(agents_df.CYCLIST>= 0.5))]
agents_CPC_df.info()

agents_CPC_df.centroid_x.idxmax()

colormap = plt.cm.magma
corr_matrix = ["centroid_x", "centroid_y", "extent_x", "extent_y", "extent_z", "yaw", 'velocity_x', 'velocity_y', "CAR","PEDESTRIAN","CYCLIST"  ]
plt.figure(figsize=(20,20));
plt.title('Pearson correlation of features', y=1.0, size=14);
sns.heatmap(agents_CPC_df[corr_matrix].corr(),linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True)

agents_CAR_df = agents_df.loc[((agents_df.CAR>= 0.5))]
agents_CAR_df.info()

colormap = plt.cm.magma
corr_matrix = ["centroid_x", "centroid_y", "extent_x", "extent_y", "extent_z", "yaw", 'velocity_x', 'velocity_y']
plt.figure(figsize=(20,20));
plt.title('Pearson correlation of features', y=1.0, size=14);
sns.heatmap(agents_CAR_df[corr_matrix].corr(),linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True)

reason: import numpy,import scipy


text:
import os

import numpy as np
import torch
import torch.nn as nn

from .abs_model import AbsModel

class CTC(nn.Module, AbsModel):
    def __init__(self, model):
        super().__init__()
        self.model = model

    def forward(self, x):
        return torch.log_softmax(self.model(x), dim=2)

    def get_dummy_inputs(self, enc_size):
        enc_out = torch.randn(1, 100, enc_size)
        return (enc_out,)

    def get_input_names(self):
        return ['x']

    def get_output_names(self):
        return ['ctc_out']

    def get_dynamic_axes(self):
        return {
            "x": {1: "ctc_in_length"},
            "ctc_out": {1: "ctc_out_length"}
        }

    def get_model_config(self, path):
        return {
            "model_path": os.path.join(path, "ctc.onnx")
        }


text:
"""
To plot densities from the weighted importance samples, the visualization
routines employ a kernel density estimate. Note that this can "over-smoothen"
so that local structure is lost. If this could be the case, it makes sense
to in the argument `kde` reduce the `scaling` in the default
MultivariateNormalTransition(), or to replace it by a GridSearchCV() to
automatically find a visually good level of smoothness.
"""

import numpy as np
from ..transition import MultivariateNormalTransition
import matplotlib.pyplot as plt
import pandas as pd

def kde_1d(df, w, x, xmin=None, xmax=None, numx=50, kde=None):
    """
    Calculates a 1 dimensional histogram from a Dataframe and weights.

    For example, a results distribution might be obtained from the history
    class and plotted as follows::

        df, w = history.get_distribution(0)
        x, pdf = hist_2d(df, w, "x")
        plt.plot(x, pdf)

    Parameters
    ----------
    df: Pandas Dataframe
        The rows are the observations, the columns the variables
    w: np.ndarray
        The corresponding weights
    x: str
        The variable for the x-axis
    xmin: float, optional
        The lower limit in x for the histogram.
        If left empty, it is set to the minimum of the ovbservations of the
        variable to be plotted as x.
    xmax: float, optional
        The upper limit in x for the histogram.
        If left empty, it is set to the maximum of the ovbservations of the
        variable to be plotted as x.
    numx: int, optional
        The number of bins in x direction.
        Defaults to 50.
    kde: pyabc.Transition, optional
        The kernel density estimator to use for creating a smooth density
        from the sample. If None, a multivariate normal kde with
        cross-validated scaling is used.

    Returns
    -------

    x, pdf: (np.ndarray, np.ndarray)
        The x and the densities at these points.
        These can be passed for plotting, for example as
        plt.plot(x, pdf)

    """
    if kde is None:
        kde = MultivariateNormalTransition(scaling=1)
    kde.fit(df[[x]], w)

    if xmin is None:
        xmin = df[x].min()
    if xmax is None:
        xmax = df[x].max()
    x_vals = np.linspace(xmin, xmax, num=numx)
    test = pd.DataFrame({x: x_vals})
    pdf = kde.pdf(test)
    return x_vals, pdf

def plot_kde_1d(df, w, x, xmin=None, xmax=None,
                numx=50, ax=None, title: str = None,
                refval=None, kde=None, **kwargs):
    """
    Plots a 1d histogram.

    Parameters
    ----------
    df: Pandas Dataframe
        The rows are the observations, the columns the variables
    w: The corresponding weights
    x: str
        The variable for the x-axis

    xmin: float, optional
        The lower limit in x for the histogram.
        If left empty, it is set to the minimum of the ovbservations of the
        variable to be plotted as x.
    xmax: float, optional
        The upper limit in x for the histogram.
        If left empty, it is set to the maximum of the ovbservations of the
        variable to be plotted as x.
    numx: int, optional
        The number of bins in x direction.
        Defaults to 50.
    ax: matplotlib.axes.Axes, optional
        The axis object to use.
    title: str, optional
        Title for the plot. Defaults to None.
    refval: dict, optional
        A reference value for x (as refval[x]: float).
        If not None, the value will be highlighted in the plot.
        Default: None.
    kde: pyabc.Transition, optional
        The kernel density estimator to use for creating a smooth density
        from the sample. If None, a multivariate normal kde with
        cross-validated scaling is used.

    Returns
    -------

    ax: matplotlib axis
        axis of the plot

    """
    x_vals, pdf = kde_1d(df, w, x, xmin=xmin, xmax=xmax,  numx=numx, kde=kde)
    if ax is None:
        _, ax = plt.subplots()
    ax.plot(x_vals, pdf, **kwargs)
    ax.set_xlabel(x)
    ax.set_ylabel("Posterior")
    ax.set_xlim(xmin, xmax)
    if title is not None:
        ax.set_title(title)
    if refval is not None:
        ax.axvline(refval[x], color='C1', linestyle='dashed')
    return ax

def kde_2d(df, w, x, y, xmin=None, xmax=None, ymin=None, ymax=None,
           numx=50, numy=50, kde=None):
    """
    Calculates a 2 dimensional histogram from a Dataframe and weights.

    For example, a results distribution might be obtained from the history
    class and plotted as follows::

        df, w = history.get_distribution(0)
        X, Y, PDF = hist_2d(df, w, "x", "y")
        plt.pcolormesh(X, Y, PDF)

    Parameters
    ----------
    df: Pandas Dataframe
        The rows are the observations, the columns the variables
    w: The corresponding weights
    x: str
        The variable for the x-axis
    y: str
        The variable for the y-axis
    xmin: float, optional
        The lower limit in x for the histogram.
        If left empty, it is set to the minimum of the ovbservations of the
        variable to be plotted as x.
    xmax: float, optional
        The upper limit in x for the histogram.
        If left empty, it is set to the maximum of the ovbservations of the
        variable to be plotted as x.
    ymin: float, optional
        The lower limit in y for the histogram.
        If left empty, it is set to the minimum of the ovbservations of the
        variable to be plotted as y
    ymax: float, optional
        The upper limit in y for the histogram.
        If left empty, it is set to the maximum of the ovbservations of the
        variable to be plotted as y.
    numx: int, optional
        The number of bins in x direction.
        Defaults to 50.
    numy int, optional
        The number of bins in y direction.
        Defaults to 50.
    kde: pyabc.Transition, optional
        The kernel density estimator to use for creating a smooth density
        from the sample. If None, a multivariate normal kde with
        cross-validated scaling is used.

    Returns
    -------

    X, Y, PDF: (np.ndarray, np.ndarray, np.ndarray)
        The X, the Y and the densities at these points.
        These can be passed for plotting, for example as
        plt.pcolormesh(X, Y, PDF)

    """
    if kde is None:
        kde = MultivariateNormalTransition(scaling=1)
    kde.fit(df[[x, y]], w)

    if xmin is None:
        xmin = df[x].min()
    if xmax is None:
        xmax = df[x].max()
    if ymin is None:
        ymin = df[y].min()
    if ymax is None:
        ymax = df[y].max()
    X, Y = np.meshgrid(np.linspace(xmin, xmax, num=numx),
                       np.linspace(ymin, ymax, num=numy))
    test = pd.DataFrame({x: X.flatten(), y: Y.flatten()})
    pdf = kde.pdf(test)
    PDF = pdf.reshape(X.shape)
    return X, Y, PDF

def plot_kde_2d(df, w, x, y, xmin=None, xmax=None, ymin=None, ymax=None,
                numx=50, numy=50, ax=None, colorbar=True,
                title: str = None, refval=None, kde=None, **kwargs):
    """
    Plots a 2d histogram.

    Parameters
    ----------
    df: Pandas Dataframe
        The rows are the observations, the columns the variables
    w: The corresponding weights.
    x: str
        The variable for the x-axis.
    y: str
        The variable for the y-axis.
    xmin: float, optional
        The lower limit in x for the histogram.
        If left empty, it is set to the minimum of the ovbservations of the
        variable to be plotted as x.
    xmax: float, optional
        The upper limit in x for the histogram.
        If left empty, it is set to the maximum of the ovbservations of the
        variable to be plotted as x.
    ymin: float, optional
        The lower limit in y for the histogram.
        If left empty, it is set to the minimum of the ovbservations of the
        variable to be plotted as y.
    ymax: float, optional
        The upper limit in y for the histogram.
        If left empty, it is set to the maximum of the ovbservations of the
        variable to be plotted as y.
    numx: int, optional
        The number of bins in x direction.
        Defaults to 50.
    numy int, optional
        The number of bins in y direction.
        Defaults tp 50.
    ax: matplotlib.axes.Axes, optional
        The axis object to use.
    colorbar: bool, optional
        Whether to plot a colorbar. Defaults to True.
    title: str, optional
        Title for the plot. Defaults to None.
    refval: dict, optional
        A reference parameter to be shown in the plots. Default: None.
    kde: pyabc.Transition, optional
        The kernel density estimator to use for creating a smooth density
        from the sample. If None, a multivariate normal kde with
        cross-validated scaling is used.

    Returns
    -------

    ax: matplotlib axis
        axis of the plot

    """
    X, Y, PDF = kde_2d(df, w, x, y,
                       xmin=xmin, xmax=xmax,
                       ymin=ymin, ymax=ymax, numx=numx, numy=numy,
                       kde=kde)
    if ax is None:
        _, ax = plt.subplots()
    mesh = ax.pcolormesh(X, Y, PDF, **kwargs)
    ax.set_xlabel(x)
    ax.set_ylabel(y)
    if title is not None:
        ax.set_title(title)
    if colorbar:
        plt.colorbar(mesh, ax=ax)
        # cbar.set_label("PDF")
    if refval is not None:
        ax.scatter([refval[x]], [refval[y]], color='C1')
    return ax

def plot_kde_matrix(df, w,
                    limits=None,
                    colorbar=True,
                    height=2.5,
                    numx=50,
                    numy=50,
                    refval=None,
                    kde=None):
    """
    Plot a KDE matrix.

    Parameters
    ----------

    df: Pandas Dataframe
        The rows are the observations, the columns the variables.
    w: np.narray
        The corresponding weights.
    colorbar: bool
        Whether to plot the colorbars or not.
    limits: dictionary, optional
        Dictionary of the form ``{"name": (lower_limit, upper_limit)}``.
    height: float, optional
        Height of each subplot in inches. Default: 2.5.
    numx: int, optional
        The number of bins in x direction.
        Defaults to 50.
    numy: int, optional
        The number of bins in y direction.
        Defaults to 50.
    refval: dict, optional
        A reference parameter to be shown in the plots (e.g. the
        underlying ground truth parameter used to simulate the data
        for testing purposes). Default: None.
    kde: pyabc.Transition, optional
        The kernel density estimator to use for creating a smooth density
        from the sample. If None, a multivariate normal kde with
        cross-validated scaling is used.
    """

    n_par = df.shape[1]
    par_names = list(df.columns.values)
    fig, arr_ax = plt.subplots(nrows=n_par, ncols=n_par,
                               sharex=False, sharey=False,
                               figsize=(height * n_par, height * n_par))

    if limits is None:
        limits = {}
    default = (None, None)

    def hist_2d(x, y, ax):
        df = pd.concat((x, y), axis=1)
        plot_kde_2d(df, w,
                    x.name, y.name,
                    xmin=limits.get(x.name, default)[0],
                    xmax=limits.get(x.name, default)[1],
                    ymin=limits.get(y.name, default)[0],
                    ymax=limits.get(y.name, default)[1],
                    numx=numx, numy=numy,
                    ax=ax, title=None, colorbar=colorbar,
                    refval=refval, kde=kde)

    def scatter(x, y, ax):
        alpha = w / w.max()
        colors = np.zeros((alpha.size, 4))
        colors[:, 3] = alpha
        ax.scatter(x, y, color="k")
        if refval is not None:
            ax.scatter([refval[x.name]], [refval[y.name]], color='C1')
        ax.set_xlim(*limits.get(x.name, default))
        ax.set_ylim(*limits.get(y.name, default))

    def hist_1d(x, ax):
        df = pd.concat((x,), axis=1)
        plot_kde_1d(df, w, x.name,
                    xmin=limits.get(x.name, default)[0],
                    xmax=limits.get(x.name, default)[1],
                    numx=numx,
                    ax=ax, refval=refval, kde=kde)

    # fill all subplots
    for i in range(0, n_par):
        y_name = par_names[i]
        y = df[y_name]

        # diagonal
        ax = arr_ax[i, i]
        hist_1d(y, ax)

        for j in range(0, i):
            x_name = par_names[j]
            x = df[x_name]

            # lower
            ax = arr_ax[i, j]
            hist_2d(x, y, ax)

            # upper
            ax = arr_ax[j, i]
            scatter(y, x, ax)

    # format
    _format_kde_matrix(arr_ax, par_names)

    # adjust subplots to fit
    fig.tight_layout()

    return arr_ax

def _format_kde_matrix(arr_ax, par_names):
    """
    Clear all labels and legends, and set the left-most and bottom-most
    labels to the parameter names.

    Parameters
    ----------

    arr_ax: array of matplotlib.axes.Axes
        Shape (n_par, n_par) where len(par_names) == n_par.
    par_names: list of str
        Parameter names to be used as labels.
    """
    n_par = len(par_names)

    for i in range(0, n_par):
        for j in range(0, n_par):
            # clear labels
            arr_ax[i, j].set_xlabel("")
            arr_ax[i, j].set_ylabel("")

            # clear legends
            arr_ax[i, j].legend = None

            # remove spines
            arr_ax[i, j].spines['right'].set_visible(False)
            arr_ax[i, j].spines['top'].set_visible(False)

    # set left-most and bottom-most labels to parameter names
    for ax, label in zip(arr_ax[-1, :], par_names):
        ax.set_xlabel(label)
    for ax, label in zip(arr_ax[:, 0], par_names):
        ax.set_ylabel(label)

max_stars_repo_name: j-zimmermann/pyABC

max_issues_repo_name: j-zimmermann/pyABC

max_forks_repo_name: j-zimmermann/pyABC


text:
import numpy
import itertools

from training.dataprep.airbnb_example import AirbnbExample
from training.trainer import Trainer
from training.model import DocLabelingModel

example = AirbnbExample(file_path="/path/to/file.csv")
texts = numpy.array([i for i in example.texts()])
labels = numpy.array([i for i in example.labels()])

eval_train_index = len(texts)-50

train_texts = texts[:eval_train_index]
eval_texts = texts[eval_train_index:]
train_labels = labels[:eval_train_index]
eval_labels = labels[eval_train_index:]

model = DocLabelingModel(len(labels[0]))
trainer = Trainer(model.model)

model = trainer.train(train_texts, train_labels)
print(trainer.evaluate(eval_texts, eval_labels))

# simple experiment
predictions = model.predict(eval_texts)
results = [(example.read_prediction(i), example.read_prediction(j)) for (i, j) in zip(predictions, eval_labels)]
for result in results:
    print(result, "\n")

max_stars_repo_name: pierrel/document-labeling

max_issues_repo_name: pierrel/document-labeling

max_forks_repo_name: pierrel/document-labeling


text:
# -*- coding: utf-8 -*-
#
# Copyright © 2009-2010 CEA
# Pierre Raybaut
# Licensed under the terms of the CECILL License
# (see guiqwt/__init__.py for details)

# pylint: disable=C0103

"""
guiqwt.histogram
----------------

The `histogram` module provides histogram related objects:
    * :py:class:`guiqwt.histogram.HistogramItem`: an histogram plot item
    * :py:class:`guiqwt.histogram.ContrastAdjustment`: the `contrast
      adjustment panel`
    * :py:class:`guiqwt.histogram.LevelsHistogram`: a curve plotting widget
      used by the `contrast adjustment panel` to compute, manipulate and
      display the image levels histogram

``HistogramItem`` objects are plot items (derived from QwtPlotItem) that may
be displayed on a 2D plotting widget like :py:class:`guiqwt.curve.CurvePlot`
or :py:class:`guiqwt.image.ImagePlot`.

Example
~~~~~~~

Simple histogram plotting example:

.. literalinclude:: /../guiqwt/tests/histogram.py

Reference
~~~~~~~~~

.. autoclass:: HistogramItem
   :members:
   :inherited-members:
.. autoclass:: ContrastAdjustment
   :members:
   :inherited-members:
.. autoclass:: LevelsHistogram
   :members:
   :inherited-members:
"""

import weakref
import numpy as np
from qtpy.QtCore import Qt, Signal
from qtpy.QtWidgets import QHBoxLayout, QVBoxLayout, QToolBar

from guidata.dataset.datatypes import DataSet
from guidata.dataset.dataitems import FloatItem
from guidata.utils import assert_interfaces_valid, update_dataset
from guidata.configtools import get_icon, get_image_layout
from guidata.qthelpers import add_actions, create_action

# Local imports
from guiqwt.transitional import QwtPlotCurve
from guiqwt.config import CONF, _
from guiqwt.interfaces import IBasePlotItem, IHistDataSource, IVoiImageItemType, IPanel
from guiqwt.panels import PanelWidget, ID_CONTRAST
from guiqwt.curve import CurveItem, CurvePlot
from guiqwt.image import ImagePlot
from guiqwt.styles import HistogramParam, CurveParam
from guiqwt.shapes import XRangeSelection
from guiqwt.tools import SelectTool, BasePlotMenuTool, SelectPointTool, AntiAliasingTool
from guiqwt.plot import PlotManager

class HistDataSource(object):
    """
    An objects that provides an Histogram data source interface
    to a simple numpy array of data
    """

    __implements__ = (IHistDataSource,)

    def __init__(self, data):
        self.data = data

    def get_histogram(self, nbins):
        """Returns the histogram computed for nbins bins"""
        return np.histogram(self.data, nbins)

assert_interfaces_valid(HistDataSource)

def hist_range_threshold(hist, bin_edges, percent):
    hist = np.concatenate((hist, [0]))
    threshold = 0.5 * percent / 100 * hist.sum()
    i_bin_min = np.cumsum(hist).searchsorted(threshold)
    i_bin_max = -1 - np.cumsum(np.flipud(hist)).searchsorted(threshold)
    return bin_edges[i_bin_min], bin_edges[i_bin_max]

def lut_range_threshold(item, bins, percent):
    hist, bin_edges = item.get_histogram(bins)
    return hist_range_threshold(hist, bin_edges, percent)

class HistogramItem(CurveItem):
    """A Qwt item representing histogram data"""

    __implements__ = (IBasePlotItem,)

    def __init__(self, curveparam=None, histparam=None):
        self.hist_count = None
        self.hist_bins = None
        self.bins = None
        self.old_bins = None
        self.source = None
        self.logscale = None
        self.old_logscale = None
        if curveparam is None:
            curveparam = CurveParam(_("Curve"), icon="curve.png")
            curveparam.curvestyle = "Steps"
        if histparam is None:
            self.histparam = HistogramParam(title=_("Histogram"), icon="histogram.png")
        else:
            self.histparam = histparam
        CurveItem.__init__(self, curveparam)
        self.setCurveAttribute(QwtPlotCurve.Inverted)

    def set_hist_source(self, src):
        """
        Set histogram source

        *source*:

            Object with method `get_histogram`, e.g. objects derived from
            :py:data:`guiqwt.image.ImageItem`
        """
        self.source = weakref.ref(src)
        self.update_histogram()

    def get_hist_source(self):
        """
        Return histogram source

        *source*:

            Object with method `get_histogram`, e.g. objects derived from
            :py:data:`guiqwt.image.ImageItem`
        """
        if self.source is not None:
            return self.source()

    def set_hist_data(self, data):
        """Set histogram data"""
        self.set_hist_source(HistDataSource(data))

    def set_logscale(self, state):
        """Sets whether we use a logarithm or linear scale
        for the histogram counts"""
        self.logscale = state
        self.update_histogram()

    def get_logscale(self):
        """Returns the status of the scale"""
        return self.logscale

    def set_bins(self, n_bins):
        self.bins = n_bins
        self.update_histogram()

    def get_bins(self):
        return self.bins

    def compute_histogram(self):
        return self.get_hist_source().get_histogram(self.bins)

    def update_histogram(self):
        if self.get_hist_source() is None:
            return
        hist, bin_edges = self.compute_histogram()
        hist = np.concatenate((hist, [0]))
        if self.logscale:
            hist = np.log(hist + 1)

        self.set_data(bin_edges, hist)
        # Autoscale only if logscale/bins have changed
        if self.bins != self.old_bins or self.logscale != self.old_logscale:
            if self.plot():
                self.plot().do_autoscale()
        self.old_bins = self.bins
        self.old_logscale = self.logscale

        plot = self.plot()
        if plot is not None:
            plot.do_autoscale(replot=True)

    def update_params(self):
        self.histparam.update_hist(self)
        CurveItem.update_params(self)

    def get_item_parameters(self, itemparams):
        CurveItem.get_item_parameters(self, itemparams)
        itemparams.add("HistogramParam", self, self.histparam)

    def set_item_parameters(self, itemparams):
        update_dataset(
            self.histparam, itemparams.get("HistogramParam"), visible_only=True
        )
        self.histparam.update_hist(self)
        CurveItem.set_item_parameters(self, itemparams)

assert_interfaces_valid(HistogramItem)

class LevelsHistogram(CurvePlot):
    """Image levels histogram widget"""

    #: Signal emitted by LevelsHistogram when LUT range was changed
    SIG_VOI_CHANGED = Signal()

    def __init__(self, parent=None):
        super(LevelsHistogram, self).__init__(
            parent=parent, title="", section="histogram"
        )
        self.antialiased = False

        # a dict of dict : plot -> selected items -> HistogramItem
        self._tracked_items = {}
        self.curveparam = CurveParam(_("Curve"), icon="curve.png")
        self.curveparam.read_config(CONF, "histogram", "curve")

        self.histparam = HistogramParam(_("Histogram"), icon="histogram.png")
        self.histparam.logscale = False
        self.histparam.n_bins = 256

        self.range = XRangeSelection(0, 1)
        self.range_mono_color = self.range.shapeparam.sel_line.color
        self.range_multi_color = CONF.get("histogram", "range/multi/color", "red")

        self.add_item(self.range, z=5)
        self.SIG_RANGE_CHANGED.connect(self.range_changed)
        self.set_active_item(self.range)

        self.setMinimumHeight(80)
        self.setAxisMaxMajor(self.Y_LEFT, 5)
        self.setAxisMaxMinor(self.Y_LEFT, 0)

        if parent is None:
            self.set_axis_title("bottom", "Levels")

    def connect_plot(self, plot):
        if not isinstance(plot, ImagePlot):
            # Connecting only to image plot widgets (allow mixing image and
            # curve widgets for the same plot manager -- e.g. in pyplot)
            return
        self.SIG_VOI_CHANGED.connect(plot.notify_colormap_changed)
        plot.SIG_ITEM_SELECTION_CHANGED.connect(self.selection_changed)
        plot.SIG_ITEM_REMOVED.connect(self.item_removed)
        plot.SIG_ACTIVE_ITEM_CHANGED.connect(self.active_item_changed)

    def tracked_items_gen(self):
        for plot, items in list(self._tracked_items.items()):
            for item in list(items.items()):
                yield item  # tuple item,curve

    def __del_known_items(self, known_items, items):
        del_curves = []
        for item in list(known_items.keys()):
            if item not in items:
                curve = known_items.pop(item)
                del_curves.append(curve)
        self.del_items(del_curves)

    def selection_changed(self, plot):
        items = plot.get_selected_items(item_type=IVoiImageItemType)
        known_items = self._tracked_items.setdefault(plot, {})

        if items:
            self.__del_known_items(known_items, items)
            if len(items) == 1:
                # Removing any cached item for other plots
                for other_plot, _items in list(self._tracked_items.items()):
                    if other_plot is not plot:
                        if not other_plot.get_selected_items(
                            item_type=IVoiImageItemType
                        ):
                            other_known_items = self._tracked_items[other_plot]
                            self.__del_known_items(other_known_items, [])
        else:
            # if all items are deselected we keep the last known
            # selection (for one plot only)
            for other_plot, _items in list(self._tracked_items.items()):
                if other_plot.get_selected_items(item_type=IVoiImageItemType):
                    self.__del_known_items(known_items, [])
                    break

        for item in items:
            if item not in known_items:
                curve = HistogramItem(self.curveparam, self.histparam)
                curve.set_hist_source(item)
                self.add_item(curve, z=0)
                known_items[item] = curve

        nb_selected = len(list(self.tracked_items_gen()))
        if not nb_selected:
            self.replot()
            return
        self.curveparam.shade = 1.0 / nb_selected
        for item, curve in self.tracked_items_gen():
            self.curveparam.update_curve(curve)
            self.histparam.update_hist(curve)

        self.active_item_changed(plot)

        # Rescaling histogram plot axes for better visibility
        ymax = None
        for item in known_items:
            curve = known_items[item]
            _x, y = curve.get_data()
            ymax0 = y.mean() + 3 * y.std()
            if ymax is None or ymax0 > ymax:
                ymax = ymax0
        ymin, _ymax = self.get_axis_limits("left")
        if ymax is not None:
            self.set_axis_limits("left", ymin, ymax)
            self.replot()

    def item_removed(self, item):
        for plot, items in list(self._tracked_items.items()):
            if item in items:
                curve = items.pop(item)
                self.del_items([curve])
                self.replot()
                break

    def active_item_changed(self, plot):
        items = plot.get_selected_items(item_type=IVoiImageItemType)
        if not items:
            # XXX: workaround
            return

        active = plot.get_last_active_item(IVoiImageItemType)
        if active:
            active_range = active.get_lut_range()
        else:
            active_range = None

        multiple_ranges = False
        for item, curve in self.tracked_items_gen():
            if active_range != item.get_lut_range():
                multiple_ranges = True
        if active_range is not None:
            _m, _M = active_range
            self.set_range_style(multiple_ranges)
            self.range.set_range(_m, _M, dosignal=False)
        self.replot()

    def set_range_style(self, multiple_ranges):
        if multiple_ranges:
            self.range.shapeparam.sel_line.color = self.range_multi_color
        else:
            self.range.shapeparam.sel_line.color = self.range_mono_color
        self.range.shapeparam.update_range(self.range)

    def set_range(self, _min, _max):
        if _min < _max:
            self.set_range_style(False)
            self.range.set_range(_min, _max)
            self.replot()
            return True
        else:
            # Range was not changed
            return False

    def range_changed(self, _rangesel, _min, _max):
        for item, curve in self.tracked_items_gen():
            item.set_lut_range([_min, _max])
        self.SIG_VOI_CHANGED.emit()

    def set_full_range(self):
        """Set range bounds to image min/max levels"""
        _min = _max = None
        for item, curve in self.tracked_items_gen():
            imin, imax = item.get_lut_range_full()
            if _min is None or _min > imin:
                _min = imin
            if _max is None or _max < imax:
                _max = imax
        if _min is not None:
            self.set_range(_min, _max)

    def apply_min_func(self, item, curve, min):
        _min, _max = item.get_lut_range()
        return min, _max

    def apply_max_func(self, item, curve, max):
        _min, _max = item.get_lut_range()
        return _min, max

    def reduce_range_func(self, item, curve, percent):
        return lut_range_threshold(item, curve.bins, percent)

    def apply_range_function(self, func, *args, **kwargs):
        item = None
        for item, curve in self.tracked_items_gen():
            _min, _max = func(item, curve, *args, **kwargs)
            item.set_lut_range([_min, _max])
        self.SIG_VOI_CHANGED.emit()
        if item is not None:
            self.active_item_changed(item.plot())

    def eliminate_outliers(self, percent):
        """
        Eliminate outliers:
        eliminate percent/2*N counts on each side of the histogram
        (where N is the total count number)
        """
        self.apply_range_function(self.reduce_range_func, percent)

    def set_min(self, _min):
        self.apply_range_function(self.apply_min_func, _min)

    def set_max(self, _max):
        self.apply_range_function(self.apply_max_func, _max)

class EliminateOutliersParam(DataSet):
    percent = FloatItem(
        _("Eliminate outliers") + " (%)", default=2.0, min=0.0, max=100.0 - 1e-6
    )

class ContrastAdjustment(PanelWidget):
    """Contrast adjustment tool"""

    __implements__ = (IPanel,)
    PANEL_ID = ID_CONTRAST
    PANEL_TITLE = _("Contrast adjustment tool")
    PANEL_ICON = "contrast.png"

    def __init__(self, parent=None):
        super(ContrastAdjustment, self).__init__(parent)

        self.local_manager = None  # local manager for the histogram plot
        self.manager = None  # manager for the associated image plot

        # Storing min/max markers for each active image
        self.min_markers = {}
        self.max_markers = {}

        # Select point tools
        self.min_select_tool = None
        self.max_select_tool = None

        style = "%s"
        layout, _label = get_image_layout(
            self.PANEL_ICON, style % self.PANEL_TITLE, alignment=Qt.AlignCenter
        )
        layout.setAlignment(Qt.AlignCenter)
        vlayout = QVBoxLayout()
        vlayout.addLayout(layout)
        self.local_manager = PlotManager(self)
        self.histogram = LevelsHistogram(parent)
        vlayout.addWidget(self.histogram)
        self.local_manager.add_plot(self.histogram)
        hlayout = QHBoxLayout()
        self.setLayout(hlayout)
        hlayout.addLayout(vlayout)

        self.toolbar = toolbar = QToolBar(self)
        toolbar.setOrientation(Qt.Vertical)
        #        toolbar.setToolButtonStyle(Qt.ToolButtonTextBesideIcon)
        hlayout.addWidget(toolbar)

        # Add standard plot-related tools to the local manager
        lman = self.local_manager
        lman.add_tool(SelectTool)
        lman.add_tool(BasePlotMenuTool, "item")
        lman.add_tool(BasePlotMenuTool, "axes")
        lman.add_tool(BasePlotMenuTool, "grid")
        lman.add_tool(AntiAliasingTool)
        lman.get_default_tool().activate()

        self.outliers_param = EliminateOutliersParam(self.PANEL_TITLE)

    def register_panel(self, manager):
        """Register panel to plot manager"""
        self.manager = manager
        default_toolbar = self.manager.get_default_toolbar()
        self.manager.add_toolbar(self.toolbar, "contrast")
        self.manager.set_default_toolbar(default_toolbar)
        self.setup_actions()
        for plot in manager.get_plots():
            self.histogram.connect_plot(plot)

    def configure_panel(self):
        """Configure panel"""
        self.min_select_tool = self.manager.add_tool(
            SelectPointTool,
            title=_("Minimum level"),
            on_active_item=True,
            mode="create",
            tip=_("Select minimum level on image"),
            toolbar_id="contrast",
            end_callback=self.apply_min_selection,
        )
        self.max_select_tool = self.manager.add_tool(
            SelectPointTool,
            title=_("Maximum level"),
            on_active_item=True,
            mode="create",
            tip=_("Select maximum level on image"),
            toolbar_id="contrast",
            end_callback=self.apply_max_selection,
        )

    def get_plot(self):
        return self.manager.get_active_plot()

    def closeEvent(self, event):
        self.hide()
        event.ignore()

    def setup_actions(self):
        fullrange_ac = create_action(
            self,
            _("Full range"),
            icon=get_icon("full_range.png"),
            triggered=self.histogram.set_full_range,
            tip=_("Scale the image's display range " "according to data range"),
        )
        autorange_ac = create_action(
            self,
            _("Eliminate outliers"),
            icon=get_icon("eliminate_outliers.png"),
            triggered=self.eliminate_outliers,
            tip=_(
                "Eliminate levels histogram "
                "outliers and scale the image's "
                "display range accordingly"
            ),
        )
        add_actions(self.toolbar, [fullrange_ac, autorange_ac])

    def eliminate_outliers(self):
        def apply(param):
            self.histogram.eliminate_outliers(param.percent)

        if self.outliers_param.edit(self, apply=apply):
            apply(self.outliers_param)

    def apply_min_selection(self, tool):
        item = self.get_plot().get_last_active_item(IVoiImageItemType)
        point = self.min_select_tool.get_coordinates()
        z = item.get_data(*point)
        self.histogram.set_min(z)

    def apply_max_selection(self, tool):
        item = self.get_plot().get_last_active_item(IVoiImageItemType)
        point = self.max_select_tool.get_coordinates()
        z = item.get_data(*point)
        self.histogram.set_max(z)

    def set_range(self, _min, _max):
        """Set contrast panel's histogram range"""
        self.histogram.set_range(_min, _max)
        # Update the levels histogram in case active item data has changed:
        self.histogram.selection_changed(self.get_plot())

assert_interfaces_valid(ContrastAdjustment)

max_stars_repo_name: fochoao/cpython

max_issues_repo_name: fochoao/cpython

max_forks_repo_name: fochoao/cpython


text:
import logging
import sys
import time

import numpy as np
from multiprocessing import cpu_count

from joblib import Parallel, delayed
from scipy.spatial.qhull import Voronoi, ConvexHull
from sklearn.neighbors.ball_tree import BallTree
from tqdm import tqdm

from amset import amset_defaults
from amset.misc.constants import output_width
from amset.misc.log import log_time_taken
from pymatgen.core.lattice import Lattice
from pymatgen.util.coord import lattice_points_in_supercell

__author__ = "Alex Ganose"
__maintainer__ = "Alex Ganose"
__email__ = "aganose@lbl.gov"
__date__ = "June 21, 2019"

pdefaults = amset_defaults["performance"]
logger = logging.getLogger(__name__)

class PeriodicVoronoi(object):
    """

    Say no existing packages (scipy.qhull, tess, pyvoro) can handle 250,000+
    points in a reasonable amount of time with reasonable memory requirements.
    Note this class works because most of the points are on a regular grid so
    it is valid to calculate the Voronoi diagram in blocks.
    """

    def __init__(self,
                 reciprocal_lattice: Lattice,
                 original_points: np.ndarray,
                 original_dim: np.ndarray,
                 extra_points: np.ndarray,
                 nworkers: int = pdefaults["nworkers"]):
        """

        Args:
            original_points:
            nworkers:
        """
        self._nworkers = nworkers if nworkers != -1 else cpu_count()

        supercell_points = get_supercell_points(
            [2, 2, 2], original_points)

        # want points in cartesian space so we can define a regular spherical
        # cutoff even if reciprocal lattice is not cubic. If we used a
        # fractional cutoff, the cutoff regions would not be spherical
        cart_points = reciprocal_lattice.get_cartesian_coords(
            supercell_points)

        cart_extra_points = reciprocal_lattice.get_cartesian_coords(
            extra_points)

        # small cutoff is slighly larger than the max regular grid spacing
        # means at least 1 neighbour point will always be included in each
        # direction
        dim_lengths = np.dot(1 / original_dim, reciprocal_lattice.matrix)
        small_cutoff = np.max(dim_lengths) * 1.01
        big_cutoff = small_cutoff * 2

        # use BallTree for quickly evaluating which points are within cutoffs
        tree = BallTree(cart_points)

        # big cutoff points are those which surround the extra points within
        # the big cutoff (it does not include the extra points themselves)
        big_cutoff_points_idx = np.concatenate(
            tree.query_radius(cart_extra_points, big_cutoff), axis=0)

        # Voronoi points are those we actually calculate in the Voronoi diagram
        # e.g. the big points + extra points
        voronoi_points = supercell_points[big_cutoff_points_idx]
        self._voronoi_points = np.concatenate((
            voronoi_points, extra_points))

        # small points are the points in original_points for which we want to
        # calculate the Voronoi volumes. Note this does not include the
        # indices of the extra points. Outside the small cutoff, the weights
        # will just be the regular grid weight.
        small_cutoff_points_idx = np.concatenate(
            tree.query_radius(cart_extra_points, small_cutoff), axis=0)

        # get the indices of small_cutoff_points in voronoi_points
        small_in_voronoi_idx = _get_loc(
            big_cutoff_points_idx, small_cutoff_points_idx)

        # get the indices of the small cutoff points + extra points
        # in voronoi points that we want the volumes for. The extra points
        # were just added at the end of big_cutoff_points, so getting their
        # indices is simple
        self._volume_points_idx = np.concatenate(
            (small_in_voronoi_idx,
             np.arange(len(extra_points)) + len(big_cutoff_points_idx)))

        # get the indices of the small_cutoff_points (not including the extra
        # points) in the original mesh. this works because the supercell
        # points are in the same order as the original mesh, just repeated for
        # each cell in the supercell
        small_in_original_idx = (
                small_cutoff_points_idx % len(original_points))

        # get the indices of the small cutoff points + extra points in the
        # final volume array. Note that the final volume array has the same
        # order as original_mesh + extra_points
        self._volume_in_final_idx = np.concatenate(
            (small_in_original_idx,
             np.arange(len(extra_points)) + len(original_points)))

        # prepopulate the final volumes array. By default, each point has the
        # volume of the original mesh. Note: at this point, the extra points
        # will have zero volume. This will array will be updated by
        # compute_volumes
        self._final_volumes = np.full(
            len(original_points) + len(extra_points),
            1 / len(original_points))
        self._final_volumes[len(original_points):] = 0

    def compute_volumes(self):
        logger.info("Calculating k-point Voronoi diagram:")
        logger.debug("  ├── num k-points near extra points: {}".format(
            len(self._voronoi_points)))
        t0 = time.perf_counter()

        # after some testing it seems like sorting the points before calculating
        # the Voronoi diagram can speed things up by > 1000x when there are many
        # points
        sorted_idx = np.argsort(self._voronoi_points, axis=0)[:, 1]

        # add the QJ option to qhull, necessary to slightly jiggle the points
        voro = Voronoi(self._voronoi_points[sorted_idx],
                       qhull_options="Qbb Qc Qz QJ")

        # need to unsort regions to get correct points
        inv_sorted_idx = np.argsort(sorted_idx)
        regions = voro.point_region[inv_sorted_idx][self._volume_points_idx]
        indices = np.array(voro.regions)[regions]
        vertices = [voro.vertices[i] for i in indices]

        log_time_taken(t0)

        volumes = self._final_volumes.copy()
        volumes[self._volume_in_final_idx] = self._get_voronoi_volumes(
            indices, vertices)

        zero_vols = volumes == 0
        if any(zero_vols):
            logger.warning("{} volumes are zero".format(np.sum(zero_vols)))

        inf_vols: np.ndarray = volumes == np.inf
        if any(inf_vols):
            logger.warning("{} volumes are infinite".format(np.sum(inf_vols)))

        sum_volumes = volumes.sum()
        vol_diff = abs(sum_volumes - 1)
        if vol_diff > 0.01:
            logger.warning("Sum of weights does not equal 1 (diff = {:.1f})... "
                           "renormalising weights".format(vol_diff * 100))
            volumes = volumes / sum_volumes

        return volumes

    def _get_voronoi_volumes(self, indices, vertices) -> np.ndarray:
        logger.info("Calculating k-point weights:")

        voronoi_info = tqdm(
            list(zip(indices, vertices)),
            total=len(indices),
            ncols=output_width,
            desc="    ├── progress",
            file=sys.stdout,
            bar_format='{l_bar}{bar}| {elapsed}<{remaining}{postfix}')

        t0 = time.perf_counter()
        volumes = Parallel(n_jobs=self._nworkers, prefer="processes")(
            delayed(_get_volume)(idx, verts) for idx, verts in voronoi_info)
        log_time_taken(t0)
        return np.array(volumes)

def _get_volume(indices, vertices):
    if -1 in indices:
        # some regions can be open
        return np.inf

    else:
        return ConvexHull(vertices).volume

def _get_loc(x, y):
    """
    Based on https://stackoverflow.com/questions/8251541

    Args:
        x:
        y:

    Returns:

    """
    # len(x) > len(y)
    index = np.argsort(x)
    sorted_x = x[index]
    sorted_index = np.searchsorted(sorted_x, y)
    yindex = np.take(index, sorted_index, mode="clip")
    mask = x[yindex] == y
    return yindex[mask]

def get_supercell_points(supercell_dim, points,
                         replicate_backwards=True):
    scale_matrix = np.array(supercell_dim, np.int16)
    if scale_matrix.shape != (3, 3):
        scale_matrix = np.array(scale_matrix * np.eye(3), np.int16)

    f_lat = lattice_points_in_supercell(scale_matrix)

    # get a list of supercell images, e.g. [[1, 0, 0], [2, 0, 0]]
    images = np.dot(f_lat, scale_matrix)

    if replicate_backwards:
        images = np.unique(np.concatenate((images, -images)), axis=0)

    repeated_points = np.tile(points, (len(images), 1))
    repeated_images = np.repeat(images, len(points), axis=0)

    return repeated_images + repeated_points

max_stars_repo_name: Navolo/amset

max_issues_repo_name: Navolo/amset

max_forks_repo_name: Navolo/amset