text:
#!/eecs/research/asr/mingbin/python-workspace/hopeless/bin/python

import numpy, os, codecs, itertools, logging
from gigaword2feature import *
from scipy.sparse import csr_matrix
from sklearn import preprocessing

logger = logging.getLogger( __name__ )

def LoadED( rspecifier, language = 'eng' ):

    entity2cls = {  # KBP2015 label
                    'PER_NAM' : 0,
                    'PER_NOM' : 5,
                    'ORG_NAM' : 1,
                    'GPE_NAM' : 2,
                    'LOC_NAM' : 3,
                    'FAC_NAM' : 4,
                    'TTL_NAM' : 5,

                    # KBP2016 label
                    'ORG_NOM' : 6,
                    'GPE_NOM' : 7,
                    'LOC_NOM' : 8,
                    'FAC_NOM' : 9,

                    # iflytek label
                    'PER_NAME' : 0,
                    'ORG_NAME' : 1,
                    'GPE_NAME' : 2,
                    'LOC_NAME' : 3,
                    'FAC_NAME' : 4,
                    'PER_NOMINAL' : 5,
                    'ORG_NOMINAL' : 6,
                    'GPE_NOMINAL' : 7,
                    'LOC_NOMINAL' : 8,
                    'FAC_NOMINAL' : 9,
                    'TITLE_NAME' : 5,
                    'TITLE_NOMINAL' : 5
                }

    if os.path.isfile( rspecifier ):
        with codecs.open( rspecifier, 'rb', 'utf8' ) as fp:
            processed, original = fp.read().split( u'=' * 128, 1 )
            original = original.strip()

            # texts, tags, failures = processed.split( u'\n\n\n', 2 )
            texts = processed.split( u'\n\n\n' )[0]
            for text in texts.split( u'\n\n' ):
                parts = text.split( u'\n' )
                # assert len(parts) in [2, 3], 'sentence, offsets, labels(optional)'
                if len( parts ) not in [2, 3]:
                    logger.exception( text )
                    continue

                sent, boe, eoe, target, mids, spelling = parts[0].split(u' '), [], [], [], [], []
                offsets = map( lambda x : (int(x[0]), int(x[1])),
                               [ offsets[1:-1].split(u',') for offsets in parts[1].split() ] )
                assert len(offsets) == len(sent), rspecifier + '\n' + \
                        str( offsets ) + '\n' + str( sent ) + '\n%d vs %d' % (len(offsets), len(sent))

                if len(parts) == 3:
                    for ans in parts[-1].split():
                        try:
                            begin_idx, end_idx, mid, mention1, mention2 = ans[1:-1].split(u',')
                            target.append( entity2cls[str(mention1 + u'_' + mention2)] )
                            boe.append( int(begin_idx) )
                            eoe.append( int(end_idx) )
                            mids.append( mid )
                            spelling.append( original[ offsets[boe[-1]][0] : offsets[eoe[-1] - 1][1] ] )
                        except ValueError as ex1:
                            logger.exception( rspecifier )
                            logger.exception( ans )
                        except KeyError as ex2:
                            logger.exception( rspecifier )
                            logger.exception( ans )

                        try:
                            assert 0 <= boe[-1] < eoe[-1] <= len(sent), \
                                    '%s  %d  ' % (rspecifier.split('/')[-1], len(sent)) + \
                                    '  '.join( str(x) for x in [sent, boe, eoe, target, mids] )
                        except IndexError as ex:
                            logger.exception( rspecifier )
                            logger.exception( str(boe) + '   ' + str(eoe) )
                            continue
                    assert( len(boe) == len(eoe) == len(target) == len(mids) )

                # move this part to processed_sentence
                # if language == 'eng':
                #     for i,w in enumerate( sent ):
                #         sent[i] = u''.join( c if 0 <= ord(c) < 128 else chr(0) for c in list(w) )
                yield sent, boe, eoe, target, mids, spelling

    else:
        for filename in os.listdir( rspecifier ):
            for X in LoadED( os.path.join( rspecifier, filename ), language ):
                yield X

def LoadEL( rspecifier, language = 'eng', window = 1 ):
    if os.path.isfile( rspecifier ):
        data = list( LoadED( rspecifier, language ) )
        for i,(sent,boe,eoe,label,mid,spelling) in enumerate(data):
            if len(label) > 0:
                previous, next = [], []
                for s,_,_,_,_,_ in data[i - window: i]:
                    previous.extend( s )
                for s,_,_,_,_,_ in data[i + 1: i + 1 + window]:
                    next.extend( s )
                yield previous + sent + next, \
                      [ len(previous) + b for b in boe ], \
                      [ len(previous) + e for e in eoe ], \
                      label, mid, spelling

    else:
        for filename in os.listdir( rspecifier ):
            for X in LoadEL( os.path.join( rspecifier, filename ), language ):
                yield X

def PositiveEL( embedding_basename,
                rspecifier, language = 'eng', window = 1 ):

    raw_data = list( LoadEL( rspecifier, language, window ) )

    # with open( embedding_basename + '.word2vec', 'rb' ) as fp:
    #   shape = numpy.fromfile( fp, dtype = numpy.int32, count = 2 )
    #   projection = numpy.fromfile( fp, dtype = numpy.float32 ).reshape( shape )
    # logger.debug( 'embedding loaded' )

    with codecs.open( embedding_basename + '.wordlist', 'rb', 'utf8' ) as fp:
        n_word = len( fp.read().strip().split() )
    logger.debug( 'a vocabulary of %d words is used' % n_word )

    numericizer = vocabulary( embedding_basename + '.wordlist',
                              case_sensitive = False )

    bc = batch_constructor( [ rd[:4] for rd in raw_data ],
                              numericizer, numericizer,
                              window = 1024, n_label_type = 7 )
    logger.debug( bc )

    index_filter = set([2, 3, 6, 7, 8])

    mid_itr = itertools.chain.from_iterable( rd[-2] for rd in raw_data )

    mention = itertools.chain.from_iterable( rd[-1] for rd in raw_data )

    # for sent, boe, eoe, _, _ in raw_data:
    #   for b,e in zip( boe, eoe ):
    #       mention.append( sent[b:e] )

    # feature_itr = bc.mini_batch( 1,
    #                            shuffle_needed = False,
    #                            overlap_rate = 0, disjoint_rate = 0,
    #                            feature_choice = 7  )
    # # assert( len(list(mid_itr)) == len(list(feature_itr)) )

    # for mid, feature in zip( mid_itr, feature_itr ):
    #   yield mid, \
    #         [ f.reshape([-1])[1::2] if i in index_filter else f.reshape([-1]) \
    #           for i,f in enumerate(feature[:9]) ]

    l1v, r1v, l1i, r1i, l2v, r2v, l2i, r2i, bow = \
            bc.mini_batch( len(bc.positive),
                           shuffle_needed = False,
                           overlap_rate = 0,
                           disjoint_rate = 0,
                           feature_choice = 7  ).next()[:9]
    l1 = csr_matrix( ( l1v, ( l1i[:,0].reshape([-1]), l1i[:,1].reshape([-1]) ) ),
                     shape = [len(bc.positive), n_word] ).astype( numpy.float32 )
    l2 = csr_matrix( ( l2v, ( l2i[:,0].reshape([-1]), l2i[:,1].reshape([-1]) ) ),
                     shape = [len(bc.positive), n_word] ).astype( numpy.float32 )
    r1 = csr_matrix( ( r1v, ( r1i[:,0].reshape([-1]), r1i[:,1].reshape([-1]) ) ),
                     shape = [len(bc.positive), n_word] ).astype( numpy.float32 )
    r2 = csr_matrix( ( r2v, ( r2i[:,0].reshape([-1]), r2i[:,1].reshape([-1]) ) ),
                     shape = [len(bc.positive), n_word] ).astype( numpy.float32 )
    bow = csr_matrix( ( numpy.ones( bow.shape[0] ),
                        ( bow[:,0].reshape([-1]), bow[:,1].reshape([-1]) ) ),
                      shape = [len(bc.positive), n_word] ).astype( numpy.float32 )
    return list(mid_itr), mention, l1, l2, r1, r2, bow

def LoadTfidf( tfidf_basename, col ):
    indices = numpy.fromfile( tfidf_basename + '.indices', dtype = numpy.int32 )
    data = numpy.fromfile( tfidf_basename + '.data', dtype = numpy.float32 )
    indptr = numpy.fromfile( tfidf_basename + '.indptr', dtype = numpy.int32 )
    assert indices.shape == data.shape

    mid2tfidf = csr_matrix( (data, indices, indptr),
                            shape = (indptr.shape[0] - 1, col) )
    del data, indices, indptr
    mid2tfidf = mid2tfidf.astype( numpy.float32 )

    with open( tfidf_basename + '.list' ) as fp:
        idx2mid = [ mid[1:-1] for mid in fp.readlines() ]
        mid2idx = { m:i for i,m in enumerate( idx2mid ) }

    return mid2tfidf, idx2mid, mid2idx

if __name__ == '__main__':
    logging.basicConfig( format = '%(asctime)s : %(levelname)s : %(message)s',
                         level = logging.DEBUG )

    embedding_basename = 'word2vec/gigaword128-case-insensitive'
    tfidf_basename = '/eecs/research/asr/Shared/Entity_Linking_training_data_from_Freebase/mid2tfidf'

    with open( embedding_basename + '.word2vec', 'rb' ) as fp:
        shape = numpy.fromfile( fp, dtype = numpy.int32, count = 2 )
        projection = numpy.fromfile( fp, dtype = numpy.float32 ).reshape( shape )
    logger.info( 'embedding loaded' )

    solution, mention, l1, l2, r1, r2, bow = PositiveEL( embedding_basename,
                                                         'kbp-raw-data/eng-train-parsed' )
    logger.info( 'fofe loaded' )

    mid2tfidf, idx2mid, mid2idx = LoadTfidf( tfidf_basename, projection.shape[0] )
    logger.info( 'tfidf loaded' )

    l1p = l1.dot( projection )
    l2p = l2.dot( projection )
    r1p = r1.dot( projection )
    r2p = r2.dot( projection )
    bowp = bow.dot( projection )
    mid2tfidfp = mid2tfidf.dot( projection )
    logger.info( 'projection done' )
    del l1, l2, r1, r2, bow, mid2tfidf

    bow_coef = 0.5

    feature = bow_coef * bowp + (1. - bowp) * (l2p + r2p) / 2.
    del l1p, l2p, r1p, r2p, bowp

    normalized_feature = preprocessing.normalize(feature, norm = 'l2')
    logger.info( 'feature computed & normalized' )
    del feature

    normalized_mid2tfidfp = preprocessing.normalize(mid2tfidfp, norm = 'l2')
    logger.info( 'tfidf normalized' )
    del mid2tfidfp

    for i,(s,m) in enumerate( zip( solution, mention ) ):
        print s, m
        # similarity = numpy.dot( normalized_feature[i:i + 1], normalized_mid2tfidfp.T )
        # top = numpy.argsort( similarity, axis = 1, kind = 'heapsort' )
        # print m, s, idx2mid[top[0,-1]]

max_stars_repo_name: xmb-cipher/fofe-ner

max_issues_repo_name: xmb-cipher/fofe-ner

max_forks_repo_name: xmb-cipher/fofe-ner

reason: import numpy,from scipy


text:
"""
python gen_clustered_matrices_main.py offline

"""
from utils.sparse import inplace_set_rows_zero
from scipy.sparse import csr_matrix
from utils.datareader import Datareader
from utils.definitions import ROOT_DIR
import numpy as np
import scipy.sparse as sps
import os
import sys

mode = sys.argv[1] # online or offline

if __name__ == '__main__':

    dr = Datareader(mode=mode, only_load=True, verbose=False)

    track_to_artist_dict = dr.get_track_to_artist_dict()
    track_to_album_dict = dr.get_track_to_album_dict()

    test_pids = dr.get_test_pids()
    urm = dr.get_urm()
    popularity = np.diff(urm.tocsc().indptr)

    cluster_1_artist = list()
    cluster_2_artist = list()
    cluster_3_artist = list()
    cluster_4_artist = list()

    ## clustering the matrices
    for prog, pid in enumerate(test_pids):
        songs = urm[pid].nonzero()[1]

        if len(songs) > 0:
            artists_array = [track_to_artist_dict[song] for song in songs]

            unique_songs_l = len(np.unique(songs))
            unique_artists_l = len(np.unique(artists_array))

            tracks_div_artists = np.log2(unique_songs_l / unique_artists_l)

            if tracks_div_artists == 0:
                cluster_1_artist.append(prog)
            elif tracks_div_artists < 1:
                cluster_2_artist.append(prog)
            elif tracks_div_artists < 2:
                cluster_3_artist.append(prog)
            else:
                cluster_4_artist.append(prog)

    directory_npz = ROOT_DIR + '/recommenders/script/main/'+mode+'_npz/'

    ## folder creation
    ar1 = ROOT_DIR+'/recommenders/script/main/'+mode+'_npz/npz_ar1/'
    ar2 = ROOT_DIR+'/recommenders/script/main/'+mode+'_npz/npz_ar2/'
    ar3 = ROOT_DIR+'/recommenders/script/main/'+mode+'_npz/npz_ar3/'
    ar4 = ROOT_DIR+'/recommenders/script/main/'+mode+'_npz/npz_ar4/'

    folders = [ar1,ar2,ar3,ar4]

    for folder in folders:
        if not os.path.exists(folder):
            os.makedirs(folder)

    filenames = list(
        map(lambda x: directory_npz + x, list(filter(lambda x: mode in x, os.listdir(directory_npz)))))

    ## selecting lines for each cluster
    all_lines = np.array([x for x in range(10000)])

    cluster_1_artist = np.array(cluster_1_artist).ravel()
    cluster_2_artist = np.array(cluster_2_artist).ravel()
    cluster_3_artist = np.array(cluster_3_artist).ravel()
    cluster_4_artist = np.array(cluster_4_artist).ravel()

    cluster_1_artist_skip_lines = np.setdiff1d(all_lines, cluster_1_artist)
    cluster_2_artist_skip_lines = np.setdiff1d(all_lines, cluster_2_artist)
    cluster_3_artist_skip_lines = np.setdiff1d(all_lines, cluster_3_artist)
    cluster_4_artist_skip_lines = np.setdiff1d(all_lines, cluster_4_artist)

    ## writing the clustered matrices

    for path_eurm in filenames:

        eurm = sps.load_npz(path_eurm)
        print(path_eurm)

        eurm_cluster_1_art = csr_matrix(eurm.copy())
        inplace_set_rows_zero(eurm_cluster_1_art, cluster_1_artist_skip_lines)
        sps.save_npz(ar1 + path_eurm.split('/')[-1], eurm_cluster_1_art.tocsr())

        eurm_cluster_2_art = csr_matrix(eurm.copy())
        inplace_set_rows_zero(eurm_cluster_2_art, cluster_2_artist_skip_lines)
        sps.save_npz(ar2 + path_eurm.split('/')[-1], eurm_cluster_2_art.tocsr())

        eurm_cluster_3_art = csr_matrix(eurm.copy())
        inplace_set_rows_zero(eurm_cluster_3_art, cluster_3_artist_skip_lines)
        sps.save_npz(ar3 + path_eurm.split('/')[-1], eurm_cluster_3_art.tocsr())

        eurm_cluster_4_art = csr_matrix(eurm.copy())
        inplace_set_rows_zero(eurm_cluster_4_art, cluster_4_artist_skip_lines)
        sps.save_npz(ar4 + path_eurm.split('/')[-1], eurm_cluster_4_art.tocsr())

ext: py

max_stars_repo_name: edervishaj/spotify-recsys-challenge

max_issues_repo_name: kiminh/spotify-recsys-challenge

max_forks_repo_name: kiminh/spotify-recsys-challenge

reason: import numpy,import scipy,from scipy


text:
"""
This module implements tools to visualise and analyse the results of a
fuzzy c-means (FCM) classification.
-----------------------------------------------------------------------
The distance checking tool checks the distances between cluster
centroids against the distances between data points to verify how
cluster centroids are distributed across the feature space.
The FCM classification is run for a wide range of fuzzifier values and
number of clusters. The first step of the analysis of FCM results is to
identify interesting clustering solutions. Clustering solutions which
are stable over a wide range of fuzzifier values are considered
representative of an underlying structure.
To further characterise the fuzzy partitions, this module propose the
computation of the typicality of samples - a quantitative measure of
whether samples are much closer to one cluster than the others or how
they lie between categories - and a visualisation method for
typicality.
In addition, triangular plots, with the membership to cluster A on the
x-axis and membership to cluster B on the y-axis, display where graded
samples lie between clusters.
Moreover, if another (crisp) partition of the dataset is available (as
ground truth or for method comparison), this module proposes to compare
the partitions, by assigning each sample to the cluster to which it has
the highest membership score and displaying a correspondance matrix.
Finally, this module can represent high-dimensional datasets in a 2D
space, using either a PCA or t-SNE approach, to visualise the
distribution of fuzzy clusters and samples in the feature space.

-----------------------------------------------------------------------
FUNCTIONS
---------
distance_check
identify_stable_solutions
typicality
plot_typicality
triangular_gradation_plot
partition_comparison
PCA_plot
tSNE_plot
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse
import matplotlib.transforms as transforms
import lib.algorithms as al
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA
import pylotwhale.utils.plotTools as pT

def distance_check(FC, colours=['k', 'b'], YLIM=None,
                   fileName=None, res=150
                   ):
    """
    Plots an histogram to compare the pairwise distances between data
    points and the pairwise distances between cluster centroids.

    Parameters
    ----------
    FC (FuzzyClustering instance)
        Fuzzy clustering results.
    colours (2-element list)
        Colours to use for distances between data points (1st element)
        and cluster centroids (2nd element). Default is black and blue.
    YLIM (float)
        Maximum value to display on the y-axis. Allows zooming in if
        the number of clusters is much smaller than the number of
        datapoints. Default is None and does not zoom in.
    fileName (path)
    The location where to save the figure. Default is None and does
        not save the figure.
    res (integer)
        Resolution (in dpi) to use when saving the figure.
        Default is 150.

    Returns
    -------
    An histogram of the pairwise distances between data points (in
    colours[0]) and pairwise distances between cluster centroids (in
    colours[1]).
    """

    dist_data = np.array([[al.euclidian_distance(i, j)
                           for i in FC.data
                           ]
                          for j in FC.data
                          ]
                         )
    dist_clusters = np.array([[al.euclidian_distance(c1, c2)
                               for c1 in FC.clusters[-1]
                               ]
                              for c2 in FC.clusters[-1]
                              ]
                             )
    plt.hist((dist_data.flat, dist_clusters.flat), histtype='bar',
             color=colours, label=('data points', 'cluster centroids')
             )
    plt.xlabel('Euclidian distances')
    plt.ylabel('Number of occurrences')
    plt.legend('data points', 'clusters')
    if YLIM is not None:
        plt.ylim(0, YLIM)
    if fileName is not None:
        plt.savefig(fileName, dpi=res, bbox_inches='tight')
    plt.show()

def identify_stable_solutions(dict_FC, plot=False, fileName=None, res=150):
    """
    Finds the optimal number of clusters for each value of fuzzifier,
    and optionally plots the optimal number of clusters as a function
    of the fuzzifier value.
    Clustering solutions which are stable over a large range of
    fuzzifier values are considered to reflect an underlying structure
    in the dataset.

    Arguments
    ---------
    dict_FC (dict)
        The results of a fuzzy c-means classification arranged in a
        dictionary. See module algorithms for details.
    plot (boolean)
        Whether or not to show the plot. Default is False and does not
        display the plot.
    fileName (path)
        Location to save the figure in. Default is None and does not
        save the figure.
    res (integer)
        Resolution, in dpi, of the saved figure. Default is 150.

    Returns
    -------
    stable_solutions (2d array)
        A three column array containing the fuzzifer values (1st col),
        the optimal number of clusters (2nd col) and the quality index
        of the optimal solution (3rd col).
    Plots the number of clusters in the best solution as a function of
    the fuzzifier value (optional).
    """

    stable_solutions = np.empty(shape=(len(dict_FC.keys()), 4))
    # The keys of dict_FC are the fuzzifier values.

    # Identify the best clustering solution for each value of fuzzifier.
    for i, p in enumerate(dict_FC.keys()):
        lst_FC = [dict_FC[p][key] for key in dict_FC[p].keys()]  # Same fuzzifier, different number of clusters.
        quality, idx = al.compare_quality(lst_FC)  # Uses the same quality index as the fuzzy c-means process.
        best = lst_FC[idx[0]]  # Best fuzzy partition.
        stable_solutions[i] = [p, list(dict_FC[p].keys())[idx[0]], best.n_clusters, quality[idx[0]]]

    # Plot the optimal number of clusters relative to the fuzzifier value.
    fig, ax = plt.subplots(figsize=(8, 8))
    plt.plot(stable_solutions[:, 0], stable_solutions[:, 2], 'kx')
    plt.xlabel('Fuzzifier value')
    plt.ylabel('Optimal number of clusters')
    plt.ylim(0, max(stable_solutions[:, 2] + 1))
    if fileName is not None:
        plt.savefig(fileName, dpi=res, bbox_inches='tight')
    if plot:
        plt.show()
    else:
        plt.close()
    return stable_solutions

def typicality(FC):
    """
    Measures the typicality of all samples of the dataset. The
    typicality of a sample is the difference between its two highest
    membership scores.

    Arguments
    ---------
    FC (FuzzyClustering instance)
        Fuzzy partition

    Returns
    -------
    typicality (2d array)
        Samples in rows. 1st column: typicality, 2nd column: cluster
        with the highest membership score, 3rd column: cluster with the
        second highest membership score.
    """

    mb = FC.memberships
    typicality = np.zeros(shape=(len(mb), 3))
    for i in range(FC.n_samples):
        idx = np.argsort(mb[i])
        typicality[i] = (mb[i, idx[-1]] - mb[i, idx[-2]],
                         idx[-1],
                         idx[-2]
                         )
    return typicality

def plot_typicality(FC, grouping=None, colour_set=None,
                    fileName=None, res=150
                    ):
    """
    Plots a stacked histogram of typicality values, coloured by main
    cluster or by another provided partition.

    Arguments
    ---------
    FC (FuzzyClustering instance)
        The fuzzy partition to plot.
    grouping (list)
        The grouping categories used to colour the samples in the
        histogram. Default is None and colour the samples by fuzzy
        cluster with the highest membership score.
    colour_set (list)
        The list of colours to use in the histogram. Default is None
        and uses a default set of 13 colourblind-suitable colours.
    fileName (path)
        Location to save the figure. Default is None and does not save
        the figure.
    res (integer)
        Resolution (in dpi) of the save figure. Default is 150 dpi.

    Returns
    -------
    An histogram of typicality values. The samples are coloured by
    category and categories are stacked.
    """

    mb = FC.memberships
    typ = typicality(FC)
    if grouping is None:  # Use cluster with highest score to colour samples.
        grouping = typ[:, 1]

    # Sort typicality values by grouping category for colouring.
    typ_lst = [[typ[i, 0] for i in range(len(typ))
                if grouping[i] == group
                ]
               for group in set(grouping)
               ]
    if colour_set is None:
        colour_set = np.array(["#000000","#004949","#009292","#ff6db6","#ffb6db",
                               "#490092","#006ddb","#b66dff","#6db6ff","#b6dbff",
                               "#920000","#924900","#db6d00","#24ff24","#ffff6d"
                               ]
                              )
    plt.hist(typ_lst, bins=20, histtype='barstacked', color=colour_set[0:len(set(grouping))])
    plt.xlim(0, 1)
    if fileName is not None:
        plt.savefig(fileName, dpi=res, bbox_inches='tight')
    plt.show()

def triangular_gradation_plots(FC, c1, c2, fileName=None, res=150):
    """
    Makes a triangular plot with the membership scores to a first
    cluster on the x-axis and the membership to a second cluster on the
    y-axis.

    Arguments
    ---------
    FC (FuzzyClustering instance)
        The fuzzy partition.
    c1 (integer)
        The index of the first cluster.
    c2 (integer)
        The index of the second cluster.
    fileName (path)
        Where to save the figure. Default is None and does not save
        the figure.
    res (integer)
        The resolution (in dpi) to use when saving the figure.
        Default is 150.

     Returns
    -------
    A triangular gradation plot.
    """

    x_mb = FC.memberships[:, c1]
    y_mb = FC.memberships[:, c2]
    plt.plot(x_mb, y_mb, 'b+')
    plt.xlabel('Membership to cluster {}'.format(c1))
    plt.ylabel('Membership to cluster {}'.format(c2))
    plt.xlim(0, 1)
    plt.ylim(0, 1)
    plt.title('Gradation between clusters {} and {}'.format(c1, c2))
    if fileName is not None:
        plt.savefig(fileName, dpi=res, bbox_inches='tight')
    plt.show()

def partition_comparison(FC, partition, fileName=None, res=150):
    """
    Constructs a matrix of correspondence between the fuzzy partition
    (as clusters with the highest membership scores) and another crisp
    partition of the dataset.

    Arguments
    ---------
    FC (FuzzyClustering instance)
        The fuzzy partition of the dataset.
    partition (list)
        The second (crisp) parition of the dataset. A list of the
        categories to which each sample belongs. Categories should
        be coded by integer values.
    fileName (path)
        Location to save the figure. Default is None and does not
        save the figure.
    res (integer)
        The resolution (in dpi) of the saved figure.
        Default is 150 dpi.

    Returns
    -------
    corr_matrix (2d array)
        A correspondence matrix between the two partition. Each
        element of the matrix is the count of samples which belong
        to a given pair of fuzzy cluster and partition category.
    Draws the correspondence matrix.
    """

    main_cluster = np.argmax(FC.memberships, axis=1)
    corr_matrix = np.zeros(shape=(FC.n_clusters, len(set(partition))))
    for i in range(FC.n_samples):
        corr_matrix[main_cluster[i], partition[i]] += 1
        # Adds each sample to the count of the corresponding category.
    column_sums = np.sum(corr_matrix, axis=0)
    norm_matrix = corr_matrix / column_sums
    fig, ax = plt.subplots()
    ax.imshow(norm_matrix, cmap=plt.cm.Blues, alpha=0.5,
              interpolation='nearest'
              )
    # Tick labels
    ax.set_xticks(range(len(set(partition))))
    ax.set_xticklabels(set(partition), rotation=90)
    ax.set_yticks(range(FC.n_clusters))
    ax.set_yticklabels(range(FC.n_clusters))
    # Axis labels
    ax.set_xlabel('Reference partition')
    ax.set_ylabel('Fuzzy clusters')
    # Display sample counts in the image.
    pT.display_numbers(fig, ax, corr_matrix, fontSz=8
                       )
    if fileName is not None:
        plt.savefig(fileName, dpi=res, bbox_inches='tight')
    plt.show()
    return corr_matrix

def PCA_plot(FC, grouping=None, n_std=1, colour_set=None,
             fileName=None, res=150
             ):
    """
    Plots the dataset using the first two dimensions of a PCA. Adds a
    confidence ellipse for each fuzzy cluster (or another partition).

    Arguments:
    ----------
    FC (FuzzyClustering instance):
        Contains the dataset and the fuzzy partition of the data.
    grouping (list):
        The grouping categories to plot the confidence ellipses.
    n_std (integer):
        The number of standard deviations to use in the ellipses.
        1 corresponds roughly to a 68% confidence interval.
        2 corresponds roughly to a 95% confidence interval.
        3 corresponds roughly to a 99.7% confidence interval.
    colour_set (list):
        The colours used to represent the different categories in the
        dataset and their confidence ellipse. Default is None and uses
        a predefined colourblind-friendly set of 15 colours.
    fileName (path):
        Location to save the figure. Default is None and does not save
        the figure.
    res (integer)
        Resolution (in dpi) of the saved figure. Default is 150.

    Plots a two-dimensional representation of the data using the first
    two principal components and confidence ellipses corresponding to
    each fuzzy cluster (or other input partition).

    Notes:
    ------
    This function uses code from https://carstenschelp.github.io/2018/09/14/Plot_Confidence_Ellipse_001.html
    The figures given for the confidence interval do not account for
    covariance. A larger number of outliers should be expected.
    """
    if colour_set is None:  # Default colour palette.
        colour_set = np.array(["#000000","#004949","#009292","#ff6db6","#ffb6db",
                               "#490092","#006ddb","#b66dff","#6db6ff","#b6dbff",
                               "#920000","#924900","#db6d00","#24ff24","#ffff6d"
                               ]
                              )
    if grouping is None:  # Uses the fuzzy clusters.
        grouping = np.argmax(FC.memberships, axis=1)
    # Run the PCA and compute the two first principal components.
    pca = PCA(n_components=2, svd_solver='full')
    princ_comp = pca.fit_transform(FC.data)
    # Make the plot.
    fig, ax = plt.subplots(figsize=(8, 8))
    for i, grp in enumerate(sorted(list(set(grouping)))):
        x = princ_comp[grouping == grp, 0]
        y = princ_comp[grouping == grp, 1]
        if len(x) > 0:  # Make scatter plot.
            ax.scatter(x, y, c=colour_set[i], marker='+')
        if len(x) >= 2:  # Draw confidence ellipse.
            cov = np.cov(x, y)
            pearson = cov[0, 1]/np.sqrt(cov[0, 0] * cov[1, 1])
            # Get the ellipse radius.
            radius_x = np.sqrt(1 + pearson)
            radius_y = np.sqrt(1 - pearson)
            ellipse = Ellipse((0, 0),
                              width=radius_x * 2,
                              height=radius_y * 2,
                              facecolor=colour_set[i],
                              alpha=0.3,
                              edgecolor=colour_set[i],
                              linewidth=2
                              )
            # Scale to n_std standard deviations and center on the means.
            scale_x = np.sqrt(cov[0, 0]) * n_std
            scale_y = np.sqrt(cov[1, 1]) * n_std
            # Rotate and scale the ellipse.
            transf = transforms.Affine2D() \
                     .rotate_deg(45) \
                     .scale(scale_x, scale_y) \
                     .translate(np.mean(x), np.mean(y))
            ellipse.set_transform(transf + ax.transData)
            ax.add_patch(ellipse)
    plt.legend(sorted(list(set(grouping))), bbox_to_anchor=(1.05, 1), loc='upper left')
    ax.set_xlabel('1st principal component - {}% of variance'.format(int(100*pca.explained_variance_ratio_[0])))
    ax.set_ylabel('2nd principal component - {}% of variance'.format(int(100*pca.explained_variance_ratio_[1])))
    if fileName is not None:
        plt.savefig(fileName, dpi=res, bbox_inches='tight')
    # plt.show()
    plt.close('all')

def tSNE_plot(FC, p, n, rs=None, ellipse=False,
              n_cp=None,
              grouping=None, colour_set=None,
              fileName=None, res=150
              ):
    """
    Plots the dataset in 2D using the t-SNE algorithm. Colours data
    points according to fuzzy cluster (or other grouping categories).

    Arguments:
    ----------
    FC (FuzzyClustering instance):
        Contains the dataset and the fuzzy partition of the data.
    p (integer):
        Perplexity. Number of closest neighbours to use in the t-SNE.
        Lower values focus on local structure, larger values on overall
        structure. Usual values in [5, 50].
        Should not exceed the number of data points.
    n (integer):
        Number of iterations to perform for the t-SNE.
        Usual values in [1000-5000].
    rs (object):
        Random state to use for the t-SNE algorithm. Default is None and
        chooses a random state based on system and current time.
    ellipse (bool):
        Whether to draw confidence ellipses for each grouping category.
        Default if False and does not draw confidence ellipses.
    n_cp (integer):
        Number of principal components to keep before performing the
        t-SNE.
        Default is None and does not perform the PCA prior to the t-SNE.
        -1 keeps all components.
    grouping (list):
        The grouping categories to colour the data points.
    colour_set (list):
        The colours used to represent the different categories in the
        dataset and their confidence ellipse. Default is None and uses
        a predefined colourblind-friendly set of 15 colours.
    fileName (path):
        Location to save the figure. Default is None and does not save
        the figure.
    res (integer)
        Resolution (in dpi) of the saved figure. Default is 150.

    Plots a two-dimensional representation of the data using the t-SNE
    algorithm. The data points are coloured according to fuzzy cluster
    (or another partition).

    Notes:
    ------
    It may be necessary to use several values of perplexity to have a
    complete view of the dataset.
    """
    if colour_set is None:
        colour_set = np.array(["#000000","#004949","#009292","#ff6db6","#ffb6db",
                               "#490092","#006ddb","#b66dff","#6db6ff","#b6dbff",
                               "#920000","#924900","#db6d00","#24ff24","#ffff6d"
                               ]
                              )
    if grouping is None:  # Uses the fuzzy clusters.
        grouping = np.argmax(FC.memberships, axis=1)
    if n_cp is None:  # No data transformation.
        tsne_data = FC.data
    else:  # PCA transformation.
        pca = PCA(n_components=n_cp, svd_solver='full')
        tsne_data = pca.fit_transform(FC.data)
    tsne = TSNE(perplexity=p, n_iter=n, random_state=rs, method='exact')
    plot_data = tsne.fit_transform(tsne_data)
    fig, ax = plt.subplots(figsize=(8, 8))
    for i, grp in enumerate(sorted(list(set(grouping)))):
        x = plot_data[grouping==grp, 0]
        y = plot_data[grouping==grp, 1]
        if len(x)>0:
            ax.scatter(x, y, c=colour_set[i], marker='+')
        if len(x)>2:  # Draw confidence ellipse.
            cov = np.cov(x, y)
            pearson = cov[0, 1]/np.sqrt(cov[0, 0] * cov[1, 1])
            # Get the ellipse radius.
            radius_x = np.sqrt(1 + pearson)
            radius_y = np.sqrt(1 - pearson)
            ellipse = Ellipse((0, 0),
                              width=radius_x * 2,
                              height=radius_y * 2,
                              facecolor=colour_set[i],
                              alpha=0.3,
                              edgecolor=colour_set[i],
                              linewidth=2
                              )
            # Scale to 1 standard deviations and center on the means.
            scale_x = np.sqrt(cov[0, 0]) * 1
            scale_y = np.sqrt(cov[1, 1]) * 1
            # Rotate and scale the ellipse.
            transf = transforms.Affine2D() \
                               .rotate_deg(45) \
                               .scale(scale_x, scale_y) \
                               .translate(np.mean(x), np.mean(y))
            ellipse.set_transform(transf + ax.transData)
            ax.add_patch(ellipse)
    plt.legend(sorted(list(set(grouping))), bbox_to_anchor=(1.05, 1), loc='upper left')
    if fileName is not None:
        plt.savefig(fileName, dpi=res, bbox_inches='tight')
    # plt.show()
    plt.close('all')

max_stars_repo_name: benbenti/fuzzyclustering

max_issues_repo_name: benbenti/fuzzyclustering

max_forks_repo_name: benbenti/fuzzyclustering

reason: import numpy


text:
import os
from PIL import Image
import numpy as np
import cv2
import pickle

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
image_dir = os.path.join(BASE_DIR,"images")
cascPath = "C:/Users/Andrew Cao/Desktop/test image/main/haarcascade_frontalface_default.xml"

faceCascade = cv2.CascadeClassifier(cascPath)
recognizer = cv2.face.LBPHFaceRecognizer_create()
current_id = 0
label_ids = {}
x_train = []
y_labels = []

for root, dirs, files in os.walk(image_dir):
    for file in files:
        if file.endswith("JPG") or file.endswith("jpg"):
            path = os.path.join(root, file)
            print(path)
            label = os.path.basename(root)
            print(label)
            if not label in label_ids:
                label_ids[label] = current_id
                current_id += 1
            id_ = label_ids[label]
            print id_
            pil_image = Image.open(path).convert("L")#grayscale
            image_array = np.array(pil_image,"uint8")
            faces = faceCascade.detectMultiScale(image_array, scaleFactor =1.5,minNeighbors=5)

            for (x,y,w,h) in faces:
                roi = image_array[y:y+h, x:x+w]
                x_train.append(roi)
                y_labels.append(id_)
recognizer.train(x_train, np.array(y_labels))
recognizer.save("trainer.yml")


text:
from __future__ import division

from numpy import zeros, float64, array

from pynurbs.config import Settings
from pynurbs.geometry.methods.compare_floats import CompareFloats as CmpFlt
from pynurbs.geometry.methods.evaluate import find_span_mult, deCasteljau1
from pynurbs.geometry.methods.modify import curve_knot_ins, surface_knot_ins

def split_bezier_curve(cpw, n, u, a=0., b=1.):
    """
    Split Bezier curve into two segments at *u*.

    :param int n: Number of control points - 1.
    :param ndarray cpw: Control points.
    :param float u: Parametric point (0 <= u <= 1).
    :param float a: Lower domain of curve (0 <= a < b).
    :param float b: Upper domain of curve (a < b <= 1).

    :return: New control points and domain for two curves
        (qw1, a1, b1, qw2, a2, b2).
    :rtype: tuple
    """
    # Split using deCasteljau algorithm, tracking first and last points in
    # each column.
    qw1 = zeros((n + 1, 4), dtype=float64)
    qw2 = array(cpw, copy=True)
    qw1[0] = qw2[0]
    for k in range(1, n + 1):
        for i in range(0, n - k + 1):
            qw2[i] = (1. - u) * qw2[i] + u * qw2[i + 1]
        qw1[k] = qw2[0]
    # Adjust domain.
    a1 = a
    b1 = a + u * (b - a)
    a2 = b1
    b2 = b
    return qw1, a1, b1, qw2, a2, b2

def extract_bezier_curve(cpw, n, u0, u1, a=0., b=1.):
    """
    Extract a Bezier curve between parameters *u0* and *u1*.

    :param ndarray cpw: Control points.
    :param int n: Number of control points - 1 (degree).
    :param float u0: Starting parameter (0 <= u0 < u1).
    :param float u1: Ending parameter (u0 < u1 <= 1).
    :param float a: Lower domain of curve (0 <= a < b).
    :param float b: Upper domain of curve (a < b <= 1).

    :return: Control points of Bezier curve between *u0* and *u1* and new
        domain (qw, a1, b1).
    :rtype: ndarray
    """
    # Split at u0 keeping the right part.
    _, _, _, qw2, a2, b2 = split_bezier_curve(cpw, n, u0, a, b)
    # Split at u1 keeping the left part. Adjust u1 so it considers new curve.
    u1b = (u1 - u0) / (1. - u0)
    qw1, a1, b1, _, _, _ = split_bezier_curve(qw2, n, u1b, a2, b2)
    return qw1, a1, b1

def split_nurbs_curve(n, p, uk, cpw, u):
    """
    Split NURBS curve into two segments at *u*.

    :param int n: Number of control points - 1.
    :param int p: Degree.
    :param ndarray uk: Knot vector.
    :param ndarray cpw: Control points.
    :param float u: Parametric point to split curve at.

    :return: New knot vector and control points for two curves
        (uk1, qw1, uk2, qw2).
    :rtype: tuple
    """
    # Special case if a >= u >= b.
    # if u <= uk[p]:
    ptol = Settings.ptol
    if CmpFlt.le(u, uk[p], ptol):
        qw1 = zeros((n + 1, 4), dtype=float64)
        qw1[:] = cpw[0]
        uk1 = array(uk, dtype=float64)
        uk1[:] = uk[p]
        qw2 = array(cpw, dtype=float64)
        uk2 = array(uk, dtype=float64)
        return uk1, qw1, uk2, qw2
    # elif u >= uk[n + 1]:
    elif CmpFlt.ge(u, uk[n + 1], ptol):
        qw1 = array(cpw, dtype=float64)
        uk1 = array(uk, dtype=float64)
        qw2 = zeros((n + 1, 4), dtype=float64)
        qw2[:] = cpw[n]
        uk2 = array(uk, dtype=float64)
        uk2[:] = uk[n + 1]
        return uk1, qw1, uk2, qw2
    # Find multiplicity of knot.
    k, s = find_span_mult(n, p, u, uk)
    # Insert knot p - s times and update knot vector and control points.
    if s >= p:
        uq, qw = array(uk, dtype=float64), array(cpw, dtype=float64)
    else:
        uq, qw = curve_knot_ins(n, p, uk, cpw, u, p - s)
    # Control points and knot vectors.
    qw1 = qw[:k - s + 1]
    qw2 = qw[k - s:]
    m = n + p + 1
    uk1 = zeros((k + 1) + (p - s + 1), dtype=float64)
    uk2 = zeros((m - k) + p + 1, dtype=float64)
    uk1[:-1] = uq[:k + (p - s) + 1]
    uk1[-1] = u
    uk2[1:] = uq[k - s + 1:]
    uk2[0] = u
    return uk1, qw1, uk2, qw2

def extract_nurbs_curve(n, p, uk, cpw, u0, u1):
    """
    Extract a NURBS curve between parameters *u0* and *u1* where (u0 < u1).

    :param int n: Number of control points - 1.
    :param int p: Degree.
    :param ndarray uk: Knot vector.
    :param ndarray cpw: Control points.
    :param float u0: Starting parameter.
    :param float u1: Ending parameter.

    :return: Knot vector and control points of NURBS curve between *u0* and
        *u1* (uq, qw).
    :rtype: tuple
    """
    if u0 > u1:
        u0, u1 = u1, u0
    # Split at u0 keeping the right part.
    _, _, bk, bw = split_nurbs_curve(n, p, uk, cpw, u0)
    # Split at u1 keeping the left part.
    n = bw.shape[0] - 1
    uq, qw, _, _ = split_nurbs_curve(n, p, bk, bw, u1)
    return uq, qw

def split_bezier_surface(cpw, n, m, u=None, v=None,
                         au=0., bu=1., av=0., bv=1.):
    """
    Split Bezier surface.

    :param ndarray cpw: Control points.
    :param int n: Number of control points - 1 in u-direction.
    :param int m: Number of control points - 1 in v-direction.
    :param u: Location of *u* split (in v-direction).
    :type u: float or None
    :param float v: Location of *v* split (in u-direction).
    :type v: float or None
    :param float au: Lower domain in u-direction.
    :param float bu: Upper domain in u-direction.
    :param float av: Lower domain in v-direction.
    :param float bv: Upper domain in v-direction.

    :return: Control points of two Bezier surfaces after split and domain
        data after split (qw1, qw2, ab1, ab2).
    :rtype: tuple

    **Note**: The domain after split for each surface are contained in 2 x 2
    matrices in the form:

         Data         Index
        au | bu     0,0 | 0,1
        ------- ---> ---------
        av | bv     1,0 | 1,1

    """
    # Use deCasteljau algorithm.
    qw2 = array(cpw, copy=True)
    if u is not None:
        # Split at u in v-direction.
        qw1 = zeros((n + 1, m + 1, 4), dtype=float64)
        qw1[0, :] = qw2[0, :]
        for j in range(0, m + 1):
            for k in range(1, n + 1):
                for i in range(0, n - k + 1):
                    qw2[i, j] = (1. - u) * qw2[i, j] + u * qw2[i + 1, j]
                qw1[k, j] = qw2[0, j]
        # Adjust domain.
        au1 = au
        bu1 = au + u * (bu - au)
        av1, bv1 = av, bv
        au2, bu2 = bu1, bu
        av2, bv2 = av, bv
        ab1 = array([[au1, bu1], [av1, bv1]], dtype=float64)
        ab2 = array([[au2, bu2], [av2, bv2]], dtype=float64)
        return qw1, qw2, ab1, ab2
    elif v is not None:
        # Split at v in u-direction.
        qw1 = zeros((n + 1, m + 1, 4), dtype=float64)
        qw1[:, 0] = qw2[:, 0]
        for i in range(0, n + 1):
            for k in range(1, m + 1):
                for j in range(0, m - k + 1):
                    qw2[i, j] = (1. - v) * qw2[i, j] + v * qw2[i, j + 1]
                qw1[i, k] = qw2[i, 0]
        # Adjust domain.
        au1, bu1 = au, bu
        av1 = av
        bv1 = av + v * (bv - av)
        au2, bu2 = au, bu
        av2, bv2 = bv1, bv
        ab1 = array([[au1, bu1], [av1, bv1]], dtype=float64)
        ab2 = array([[au2, bu2], [av2, bv2]], dtype=float64)
        return qw1, qw2, ab1, ab2

def extract_bezier_isocurve(n, m, cpw, u=None, v=None):
    """
    Extract iso-curve from Bezier surface.

    :param int n: Number of control points - 1 in u-direction.
    :param int m: Number of control points - 1 in v-direction.
    :param ndarray cpw: Control points.
    :param float u: Parameter to extract curve at (in v-direction).
    :param float v: Parameter to extract curve at (in u-direction).

    :return: Control points of Bezier iso-curve.
    :rtype: ndarray
    """
    if u is not None:
        qw = zeros((m + 1, 4), dtype=float64)
        for j in range(0, m + 1):
            qw[j] = deCasteljau1(cpw[:, j], n, u)
    else:
        qw = zeros((n + 1, 4), dtype=float64)
        for i in range(0, n + 1):
            qw[i] = deCasteljau1(cpw[i, :], m, v)
    return qw

def split_nurbs_surface(n, p, uk, m, q, vk, cpw, u=None, v=None):
    """
    Split NURBS surface into two segments in specified direction.

    :param int n: Number of control points - 1 in u-direction.
    :param int p: Degree in u-direction.
    :param ndarray uk: Knot vector in u-direction.
    :param int m: Number of control points - 1 in v-direction.
    :param int q: Degree in v-direction.
    :param ndarray vk: Knot vector in v-direction.
    :param ndarray cpw: Control points.
    :param u: Location of *u* split (in v-direction).
    :type u: float or None
    :param v: Location of *v* split (in u-direction).
    :type v: float or None

    :return: New knot vectors and control points for two surfaces
        (uk1, vk1, qw1, uk2, vk2, qw2).
    :rtype: tuple
    """
    if u is not None:
        # Split at u in v-direction.
        vk12 = array(vk, dtype=float64)
        if u <= uk[p]:
            qw1 = zeros((n + 1, m + 1, 4), dtype=float64)
            for i in range(0, n + 1):
                qw1[i, :] = cpw[0, :]
            qw2 = array(cpw, dtype=float64)
            uk1 = array(uk, dtype=float64)
            uk1[:] = uk[p]
            uk2 = array(uk, dtype=float64)
            return uk1, vk12, qw1, uk2, vk12, qw2
        if u >= uk[n + 1]:
            qw1 = array(cpw, dtype=float64)
            qw2 = zeros((n + 1, m + 1, 4), dtype=float64)
            for i in range(0, n + 1):
                qw2[i, :] = cpw[n, :]
            uk1 = array(uk, dtype=float64)
            uk2 = array(uk, dtype=float64)
            uk2[:] = uk[n + 1]
            return uk1, vk12, qw1, uk2, vk12, qw2
        # Find multiplicity of knot.
        k, s = find_span_mult(n, p, u, uk)
        # Insert knot p - s times and update knot vector and control points.
        if s >= p:
            uq, qw = array(uk, dtype=float64), array(cpw, dtype=float64)
        else:
            uq, _, qw = surface_knot_ins(n, p, uk, m, q, vk, cpw,
                                         'u', u, p - s)
        qw1 = qw[:k - s + 1, :]
        qw2 = qw[k - s:, :]
        r = n + p + 1
        uk1 = zeros((k + 1) + (p - s + 1), dtype=float64)
        uk2 = zeros((r - k) + p + 1, dtype=float64)
        uk1[:-1] = uq[:k + (p - s) + 1]
        uk1[-1] = u
        uk2[1:] = uq[k - s + 1:]
        uk2[0] = u
        return uk1, vk12, qw1, uk2, vk12, qw2
    elif v is not None:
        # Split at v in u-direction.
        uk12 = array(uk, dtype=float64)
        if v <= vk[q]:
            qw1 = zeros((n + 1, m + 1, 4), dtype=float64)
            for j in range(0, m + 1):
                qw1[:, j] = cpw[:, 0]
            qw2 = array(cpw, dtype=float64)
            vk1 = array(vk, dtype=float64)
            vk1[:] = vk[q]
            vk2 = array(vk, dtype=float64)
            return uk12, vk1, qw1, uk12, vk2, qw2
        if v >= vk[m + 1]:
            qw1 = array(cpw, dtype=float64)
            qw2 = zeros((n + 1, m + 1, 4), dtype=float64)
            for j in range(0, m + 1):
                qw2[:, j] = cpw[:, m]
            vk1 = array(vk, dtype=float64)
            vk2 = array(vk, dtype=float64)
            vk2[:] = vk[m + 1]
            return uk12, vk1, qw1, uk12, vk2, qw2
        # Find multiplicity of knot.
        k, s = find_span_mult(m, q, v, vk)
        # Insert knot q - s times and update knot vector and control points.
        if s >= q:
            vq, qw = array(vk, dtype=float64), array(cpw, dtype=float64)
        else:
            _, vq, qw = surface_knot_ins(n, p, uk, m, q, vk, cpw,
                                         'v', v, q - s)
        qw1 = qw[:, :k - s + 1]
        qw2 = qw[:, k - s:]
        r = m + q + 1
        vk1 = zeros((k + 1) + (q - s + 1), dtype=float64)
        vk2 = zeros((r - k) + q + 1, dtype=float64)
        vk1[:-1] = vq[:k + (q - s) + 1]
        vk1[-1] = v
        vk2[1:] = vq[k - s + 1:]
        vk2[0] = v
        return uk12, vk1, qw1, uk12, vk2, qw2

def extract_nurbs_isocurve(n, p, uk, m, q, vk, cpw, u=None, v=None):
    """
    Extract iso-curve from NURBS surface.

    :param int n: Number of control points - 1 in u-direction.
    :param int p: Degree in u-direction.
    :param ndarray uk: Knot vector in u-direction.
    :param int m: Number of control points - 1 in v-direction.
    :param int q: Degree in v-direction.
    :param ndarray vk: Knot vector in v-direction.
    :param ndarray cpw: Control points.
    :param float u: Location of *u* split (in v-direction).
    :param float v: Location of *v* split (in u-direction).

    :return: Knot vector and control points for NURBS iso-curve in specified
        direction (uq, qw).
    :rtype: tuple
    """
    if u is not None:
        # Split at u in v-direction.
        uq = array(vk, dtype=float64)
        # Special case if a >= u >= b.
        if u <= uk[p]:
            return uq, cpw[0, :]
        if u >= uk[n + 1]:
            return uq, cpw[n, :]
        # Determine multiplicity.
        k, s = find_span_mult(n, p, u, uk)
        qw = zeros((m + 1, 4), dtype=float64)
        for j in range(0, m + 1):
            _, qwi = curve_knot_ins(n, p, uk, cpw[:, j], u, p - s)
            qw[j] = qwi[k - s]
        return uq, qw
    elif v is not None:
        # Split at v in u-direction.
        uq = array(uk, dtype=float64)
        # Special case if a >= v >= b.
        if v <= vk[q]:
            return uq, cpw[:, 0]
        if v >= vk[m + 1]:
            return uq, cpw[:, m]
        # Determine multiplicity.
        k, s = find_span_mult(m, q, v, vk)
        qw = zeros((n + 1, 4), dtype=float64)
        for i in range(0, n + 1):
            _, qwi = curve_knot_ins(m, q, vk, cpw[i, :], v, q - s)
            qw[i] = qwi[k - s]
        return uq, qw

def extract_bezier_surface(n, m, cpw, u0, u1, v0, v1, au, bu, av, bv):
    """
    Extract Bezier surface bounded by parameters.

    :param int n: Number of control points - 1 in u-direction.
    :param int m: Number of control points - 1 in v-direction.
    :param ndarray cpw: Control points.
    :param float u0: Starting parameter in u-direction.
    :param float u1: Ending parameter in u-direction.
    :param float v0: Starting parameter in v-direction.
    :param float v1: Ending parameter in v-direction.
    :param float au: Lower domain in u-direction.
    :param float bu: Upper domain in u-direction.
    :param float av: Lower domain in v-direction.
    :param float bv: Upper domain in v-direction.

    :return: Extracted Bezier surface.
    :rtype: :class:`.BezierSurface`
    """
    # Split at u0 keeping upper part.
    _, qw1, _, ab1 = split_bezier_surface(cpw, n, m, u0, None, au, bu, av, bv)
    # Split at u1 keeping lower part.
    u1b = (u1 - u0) / (1. - u0)
    qw2, _, ab2, _ = split_bezier_surface(qw1, n, m, u1b, None,
                                          ab1[0, 0], ab1[0, 1],
                                          ab1[1, 0], ab1[1, 1])
    # Split at v0 keeping upper part.
    _, qw3, _, ab3 = split_bezier_surface(qw2, n, m, None, v0,
                                          ab2[0, 0], ab2[0, 1],
                                          ab2[1, 0], ab2[1, 1])
    # Split at v1 keeping lower part.
    v1b = (v1 - v0) / (1. - v0)
    qw4, _, ab4, _ = split_bezier_surface(qw3, n, m, None, v1b,
                                          ab3[0, 0], ab3[0, 1],
                                          ab3[1, 0], ab3[1, 1])
    # Return control points and domain.
    return qw4, ab4

def extract_nurbs_surface(n, p, uk, m, q, vk, cpw, u0, u1, v0, v1):
    """
    Extract NURBS surface bounded by parameters.

    :param int n: Number of control points - 1 in u-direction.
    :param int p: Degree in u-direction.
    :param ndarray uk: Knot vector in u-direction.
    :param int m: Number of control points - 1 in v-direction.
    :param int q: Degree in v-direction.
    :param ndarray vk: Knot vector in v-direction.
    :param ndarray cpw: Control points.
    :param float u0: Starting parameter in u-direction.
    :param float u1: Ending parameter in u-direction.
    :param float v0: Starting parameter in v-direction.
    :param float v1: Ending parameter in v-direction.

    :return: Extracted NURBS surface (nq, mq, uq, vq, qw).
    :rtype: :class:`.NurbsSurface`
    """
    # Split at u0 keeping upper part.
    _, _, _, uk1, vk1, qw1 = split_nurbs_surface(n, p, uk, m, q, vk, cpw,
                                                 u0, None)
    # Split at u1 keeping lower part.
    nm = qw1.shape[:-1]
    n1, m1 = nm[0] - 1, nm[1] - 1
    uk2, vk2, qw2, _, _, _ = split_nurbs_surface(n1, p, uk1, m1, q, vk1, qw1,
                                                 u1, None)
    # Split at v0 keeping upper part.
    nm = qw2.shape[:-1]
    n2, m2 = nm[0] - 1, nm[1] - 1
    _, _, _, uk3, vk3, qw3 = split_nurbs_surface(n2, p, uk2, m2, q, vk2, qw2,
                                                 None, v0)
    # Split at v1 keeping lower part.
    nm = qw3.shape[:-1]
    n3, m3 = nm[0] - 1, nm[1] - 1
    uk4, vk4, qw4, _, _, _ = split_nurbs_surface(n3, p, uk3, m3, q, vk3, qw3,
                                                 None, v1)
    nm = qw4.shape[:-1]
    n4, m4 = nm[0] - 1, nm[1] - 1
    return n4, m4, uk4, vk4, qw4

max_stars_repo_name: trelau/pyNURBS

max_issues_repo_name: trelau/pyNURBS

max_forks_repo_name: trelau/pyNURBS

reason: from numpy


text:
"""
    Save to .bin from .pth

    Run after
        download_data.py
        split_data.py
        prepare_data.py
"""

import torch
import numpy as np
import pptk
import math
import os

# ------ Configurations ------

# path to ScanNet dataset
data_dir = "/home/dtc/Data/ScanNet"

# path to the processed pth folder
pth_dir = "/home/dtc/MyGit/dtc-sparseconvnet"

# path to save .bin
save_dir = "/home/dtc/Data/ScanNet/Bin/"

# --- end of Configurations ---

def pth_to_bin(pth_filename, bin_filename):
    data = torch.load(pth_filename)
    coords, colors, labels = data
    m = np.eye(3)
    theta = 0
    m = np.matmul(m, [[math.cos(theta), math.sin(theta), 0], [-math.sin(theta), math.cos(theta), 0], [0, 0, 1]])
    coords = np.matmul(coords, m)
    bin_data = np.c_[coords, colors, labels]
    with open(bin_filename, "wb") as f_bin:
        bin_data.astype("float32").tofile(f_bin)

def read_bin(bin_filename):
    bin_data = np.fromfile(bin_filename, "<f4")
    bin_data = np.reshape(bin_data, (-1, 7))
    return bin_data

if __name__ == "__main__":

    data_train_txt = os.path.join(data_dir, "scannetv2_train.txt")
    data_valid_txt = os.path.join(data_dir, "scannetv2_val.txt")

    pth_train_dir = os.path.join(pth_dir, "train")
    pth_valid_dir = os.path.join(pth_dir, "val")

    with open(data_train_txt, "r") as f_in:
        lines = f_in.readlines()
        for line in lines:
            line = line.strip()
            print(line)
            pth_filename = os.path.join(pth_train_dir, line + "_vh_clean_2.pth")
            bin_filename = os.path.join(os.path.join(save_dir, "train"), line + "_vh_clean_2.bin")
            pth_to_bin(pth_filename, bin_filename)

    with open(data_valid_txt, "r") as f_in:
        lines = f_in.readlines()
        for line in lines:
            line = line.strip()
            print(line)
            pth_filename = os.path.join(pth_valid_dir, line + "_vh_clean_2.pth")
            bin_filename = os.path.join(os.path.join(save_dir, "val"), line + "_vh_clean_2.bin")
            pth_to_bin(pth_filename, bin_filename)

max_stars_repo_name: dtczhl/Slimmer

max_issues_repo_name: dtczhl/Slimmer

max_forks_repo_name: dtczhl/Slimmer


text:
"""@author: Bryan Silverthorn <bcs@cargo-cult.org>"""

import contextlib
import numpy
import borg

logger = borg.get_logger(__name__, default_level = "DETAIL")

class FakeSolverProcess(object):
    """Provide a solver interface to stored run data."""

    def __init__(self, run):
        """Initialize."""

        self._run = run
        self._elapsed = 0.0
        self._terminated = False

    def run_then_stop(self, budget):
        """Unpause the solver for the specified duration."""

        try:
            return self.run_then_pause(budget)
        finally:
            self.stop()

    def run_then_pause(self, budget):
        """Unpause the solver for the specified duration."""

        assert not self._terminated

        position = self._elapsed + budget

        logger.detail(
            "moving %s run to %.0f of %.0f (from %.0f)",
            self._run.solver,
            position,
            self._run.cost,
            self._elapsed,
            )

        if position >= self._run.cost:
            borg.get_accountant().charge_cpu(self._run.cost - self._elapsed)

            self._elapsed = self._run.cost
            self._terminated = True

            return self._run.success
        else:
            borg.get_accountant().charge_cpu(budget)

            self._elapsed = position

            return None

    def stop(self):
        """Terminate the solver."""

        self._position = None

    @property
    def name(self):
        """Name of the running solver."""

        return self._run.solver

    @property
    def elapsed(self):
        """Number of CPU seconds used by this process (or processes)."""

        return self._elapsed

    @property
    def terminated(self):
        """Has this process terminated?"""

        return self._terminated

class FakeSolverFactory(object):
    def __init__(self, solver_name, runs_data):
        self._solver_name = solver_name
        self._runs_data = runs_data

    def start(self, task):
        """Return a fake solver process."""

        all_runs = self._runs_data.run_lists[task]
        our_runs = filter(lambda run: run.solver == self._solver_name, all_runs)

        if len(our_runs) == 0:
            raise Exception("no runs of solver \"{0}\" are recorded".format(self._solver_name))

        run = our_runs[numpy.random.randint(len(our_runs))]

        return FakeSolverProcess(run)

class FakeDomain(object):
    name = "fake"

    def __init__(self, suite):
        """Initialize."""

        self._suite = suite

    @contextlib.contextmanager
    def task_from_path(self, task_path):
        yield task_path

    def compute_features(self, instance):
        """Return static features of an instance."""

        # get features with cost
        with_cost = self._suite.run_data.get_feature_vector(instance)

        borg.get_accountant().charge_cpu(with_cost["cpu_cost"])

        # return the features
        features = dict(with_cost.iteritems())

        del features["cpu_cost"]

        return (features.keys(), features.values())

    def is_final(self, task, answer):
        """Does this answer imply success?"""

        return bool(answer)

    @property
    def extensions(self):
        """Not applicable."""

        raise NotImplementedError()

class FakeSuite(object):
    """Mimic a solver suite using simulated solvers."""

    def __init__(self, run_data):
        """Initialize."""

        self.run_data = run_data
        self.domain = FakeDomain(self)
        self.solvers = dict((k, FakeSolverFactory(k, self.run_data)) for k in self.run_data.solver_names)

max_stars_repo_name: borg-project/borg

max_issues_repo_name: borg-project/borg

max_forks_repo_name: borg-project/borg


text:
import math
import numpy as np

# TODO comments needed :)

"""
Args:
	x: numpy array
	mid: x-value of sigmoid's midpoint
	k: steepness
	L: max value of curve
Returns:
	sigmoid numpy array
"""
def sigmoid(x, mid, k, L):
	y = k*(x-mid)
	return L / (1 + np.exp(-y))

"""
"""
def stretch_linear(array, k):
	stretched = []
	for i in range(len(array)-1): # exclude last element
		step = (array[i+1] - array[i])/k
		for j in range(k):
			stretched.append(array[i] + j*step)
	stretched.append(array[-1])
	return stretched

"""
"""
def stretch_geom(array, k):
	stretched = []
	for i in range(len(array)-1): # exclude last element
		step = math.pow(array[i+1]/array[i], 1/k)
		for j in range(k):
			stretched.append(array[i]*step**j)
	stretched.append(array[-1])
	return stretched

"""
"""
def arange_geom(a, b, k, inverted=False):
	x = b-a+1
	k -= 1 # for simplicity of proceeding calculations
	r = math.pow(x, 1/k)
	v = [1]
	for i in range(1, k+1):
		exp = k-i if inverted else i-1
		delta = (r**exp)*(r-1)
		v.append(v[i-1] + delta)
	return np.array(v) + a - 1

"""
"""
def scale_growth_rates(growth_rates, scale_factor):
	return [1 + (r-1)*scale_factor for r in growth_rates]

"""
We need a model that takes in a CAGR and a decay parameter
and returns a list of growth rates.
"""

max_stars_repo_name: terra-project/research

max_issues_repo_name: terra-money/research

max_forks_repo_name: terra-project/research


text:
import pint

from handcalcs.handcalcs import (
    CalcLine, round_and_render_line_objects_to_latex
)

ureg = pint.UnitRegistry(auto_reduce_dimensions=True)
ureg.default_format = '~'
ft = ureg.ft
kip = ureg.kip

def test_pint_rounding():
    L = (1.23456789 * kip)
    d = (2 * ft)
    M = L * d

    assert round_and_render_line_objects_to_latex(
        CalcLine([L], '', ''), precision=2, dec_sep='.'
    ).latex == '1.23\\ \\mathrm{kip}'

    assert round_and_render_line_objects_to_latex(
        CalcLine([d], '', ''), precision=2, dec_sep='.'
    ).latex == '2\\ \\mathrm{ft}'

    assert round_and_render_line_objects_to_latex(
        CalcLine([M], '', ''), precision=2, dec_sep='.'
    ).latex == '2.47\\ \\mathrm{ft} \\cdot \\mathrm{kip}'

def test_pint_with_sympy():
    import sympy
    pint.quantity._Quantity._sympy_ = lambda s: sympy.sympify(f'({s.m})*{s.u}')
    pint.quantity._Quantity._repr_latex_ = lambda s: (
        s.m._repr_latex_() + r'\ ' + s.u._repr_latex_()
        if hasattr(s.m, '_repr_latex_') else '${:~L}$'.format(s)
    )
    L = 1.23456789 * sympy.symbols('a') * kip
    assert round_and_render_line_objects_to_latex(
        CalcLine([L], '', ''), precision=3, dec_sep='.'
    ).latex == r'\displaystyle 1.235 a\ \mathrm{kip}'

max_stars_repo_name: daradib/handcalcs

max_issues_repo_name: daradib/handcalcs

max_forks_repo_name: daradib/handcalcs

reason: import sympy


text:
import numpy as np
from datetime import timedelta
from random import randrange
from os import getenv

chain_codes = [["A", "B", "C", "D"], ["E", "F", "G", "H"], ["I", "J", "K", "L"]]
desktop_location = getenv('HOME') + '/Desktop/'

# Determine which experiment number a chain belongs to
def determine_experiment_number(chain):
  for experiment in range(0, len(chain_codes)):
    if chain in chain_codes[experiment]:
      return experiment + 1
  return None

# Get the words for a given set
def getWords(experiment, chain, generation, set_type):
  if set_type == "c":
    data = load(experiment, chain, generation, "d") + load(experiment, chain, generation, "s")
  else:
    data = load(experiment, chain, generation, set_type)
  return [data[x][0] for x in range(0,len(data))]

# Get the triangles for a given set
def getTriangles(experiment, chain, generation, set_type):
  if set_type == "c":
    data = load(experiment, chain, generation, "d") + load(experiment, chain, generation, "s")
  else:
    data = load(experiment, chain, generation, set_type)
  triangles = []
  for row in data:
    x1, y1 = row[1].split(',')
    x2, y2 = row[2].split(',')
    x3, y3 = row[3].split(',')
    triangles.append(np.array([[x1,y1],[x2,y2],[x3,y3]], dtype=float))
  return triangles

# Return a count of the number of unique strings in a given set
def uniqueStrings(experiment, chain, generation, set_type):
  if set_type == 's':
    words = getWords(experiment, chain, generation, 's')
  elif set_type == 'd':
    words = getWords(experiment, chain, generation, 'd')
  elif set_type == 'c':
    words = getWords(experiment, chain, generation, 's') + getWords(experiment, chain, generation, 'd')
  else:
    raise ValueError('Invalid set type. Should be "s", "d", or "c".')
  return len(set(words))

def allTrainingErrors(experiment):
  results = []
  for chain in basics.chain_codes[experiment-1]:
    scores = []
    for generation in range(1, 11):
      try:
        score = basics.trainingError(experiment, chain, generation)
        scores.append(score)
      except:
        scores.append("N/A")
    results.append(scores)
  return results

# Return the amount of training error made by a participant
def trainingError(experiment, chain, generation, subject=False):
  if subject == 'A' or subject == 'B':
    filename = 'logSub' + subject
  else:
    filename = 'log'
  data = load(experiment, chain, generation, filename)
  words_A = [data[x][0] for x in range(5,53)]
  words_B = [data[x][1] for x in range(5,53)]
  x = meanNormLevenshtein(words_A, words_B)
  return x

# Calculate the average amount of time spent on each test item
def timePerItem(experiment, chain, generation):
  set_d = load(experiment, chain, generation, "d")
  timestamp_1 = stringToTimeStamp(set_d[0][4])
  timestamp_50 = stringToTimeStamp(set_d[47][4])
  difference = timestamp_50 - timestamp_1
  time_per_item_set_d = difference.total_seconds() / 94.0
  return time_per_item_set_d

# Calcualte total time spent on the experiment
def timeSpent(experiment, chain, generation, subject=False):
  print experiment, chain, generation
  if subject == False:
    log_file = load(experiment, chain, generation, "log")
  else:
    log_file = load(experiment, chain, generation, 'logSub' + subject)
  if experiment == 3:
    start_time = stringToTimeStamp(log_file[1][4].split(" ")[1])
  else:
    start_time = stringToTimeStamp(log_file[1][3].split(" ")[1])
  if experiment == 2:
    end_time = stringToTimeStamp(log_file[56][0].split(" ")[3])
  else:
    end_time = stringToTimeStamp(log_file[54][0].split(" ")[3])
  difference = end_time - start_time
  return difference.total_seconds()

# Convert string to timestamp
def stringToTimeStamp(string):
  time = string.split(":")
  timestamp = timedelta(hours=int(time[0]), minutes=int(time[1]), seconds=int(time[2]))
  return timestamp

# Return number of times a participant was asked to choose new word (experiment 2 only)
def overuseCount(chain, generation, subject):
  if subject == 'A' or subject == 'B':
    filename = 'logSub' + subject
  else:
    filename = 'log'
  data = load(2, chain, generation, filename)
  line = str(data[54])
  split1 = line.split("overuse count = ")
  split2 = split1[1].split("'")
  return int(split2[0])

# Take a list of strings, conpute the distance between every pair, and return condensed distance matrix
def stringDistances(strings):
  distances = []
  for i in range(0,len(strings)):
    for j in range(i+1,len(strings)):
      distances.append(LevenshteinDistance(strings[i], strings[j]))
  return distances

# Compute mean normalized Levenshtein distance between consecutive static sets
def meanNormLevenshtein(strings1, strings2):
  total = 0.0
  for i in range(0, len(strings1)):
    total += LevenshteinDistance(strings1[i], strings2[i])
  return total/float(len(strings1))

# Calculate the normalized Levenshtein distance between two strings
def LevenshteinDistance(s1, s2):
  if len(s1) > len(s2):
    s1, s2 = s2, s1
  distances = range(len(s1) + 1)
  for index2, char2 in enumerate(s2):
    newDistances = [index2 + 1]
    for index1, char1 in enumerate(s1):
      if char1 == char2:
        newDistances.append(distances[index1])
      else:
        newDistances.append(1 + min((distances[index1], distances[index1+1], newDistances[-1])))
    distances = newDistances
  return float(distances[-1]) / max(len(s1), len(s2))

# Load in a data file
def load(experiment, chain, generation, set_type):
  filename = "../data/experiment_" + str(experiment) + "/" + chain + "/" + str(generation) + set_type
  f = open(filename, 'r')
  data = f.read()
  f.close()
  return [row.split('\t') for row in data.split('\n')]

# Write out a matrix to a file on the desktop
def writeOut(matrix, filename='file'):
  data = ""
  for row in matrix:
    row = [str(x) for x in row]
    data = data + "\t".join(row) + "\n"
  data = data[0:-1]
  f = open(desktop_location + filename, 'w')
  f.write(data)
  f.close()

# Read in a previously saved data file to a matrix
def readIn(filename):
  f = open(filename, 'r')
  data = f.read()
  f.close()
  lines = data.split("\n")
  matrix = []
  for line in lines:
    cells = line.split("\t")
    row = []
    for cell in cells:
      try:
        cell = float(cell)
      except:
        cell = None
      row.append(cell)
    matrix.append(row)
  return matrix

# Given a list of chain codes and a start and end generation, this outputs
# a random order in which to run participants, such that generation i comes
# after generation i-1. This is used to assign participants to a chain at
# random.
def assign_to_chain(chain_codes, start_gen, end_gen):
  chain_gens = {}
  for chain in chain_codes:
    chain_gens[chain] = range(start_gen, end_gen+1)
  random_assignment = []
  for i in range(0, len(chain_codes)*(end_gen-start_gen+1)):
    code = chain_codes[randrange(0, len(chain_codes))]
    gen = chain_gens[code][0]
    del chain_gens[code][0]
    if len(chain_gens[code]) == 0:
      chain_codes.remove(code)
    random_assignment.append( code+str(gen) )
  return random_assignment

max_stars_repo_name: jwcarr/Infinity

max_issues_repo_name: jwcarr/flatlanders

max_forks_repo_name: jwcarr/flatlanders


text:
# -*- coding: utf-8 -*-
# pylint: disable=wrong-import-position, invalid-name

"""
Inverse transform sampling for a PDF that can be described by a polynomial.
"""

from __future__ import absolute_import, division, print_function

__all__ = ['InvPolySampler']

__author__ = 'J.L. Lanfranchi'
__license__ = '''Copyright 2018 Justin L. Lanfranchi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.'''

import numpy as np
from scipy.optimize import brentq

class InvPolySampler(object):
    """
    Inverse transform sampling for a probability density function that can be
    described by a polynomial.

    Parameters
    ----------
    n_samp : int
        Linear interpolation will be performed between this many points sampled
        from the inverse CDF. Note that both endpoints of the range (i.e., 0 and
        1) are included.

    power : float
        The range of the CDF--which is in [0, 1]--is transformed by this power
        in order to sample the CDF more accurately.

    pdf : numpy.polynomial.Polynomial or array-like of coeffs
        If an array-like of coeffs is provided, note that these must be in
        _ascending_ order, as these are passed to
        :class:`numpy.polynomial.Polynomial`.

    domain : 2-seq of float; optional if `pdf` is np.polynomial.Polynomial
        The domain of the PDF (and hence also of the CDF). This is unnecessary
        if `pdf` is a :class:`numpy.polynomial.Polynomial` since this object
        already has a `domain` attribute defined. If both are specified, both
        domains must be equal.

    """
    def __init__(self, n_samp, power, pdf, domain=None):
        self.n_samp = n_samp
        self.prob_binwidth = 1 / (n_samp - 1)
        self.power = power
        self.inv_power = 1 / power

        self.cdf_samples = np.linspace(0, 1, n_samp)**power
        """Sample CDF at y-values regularly spaced in ``y**power``"""

        if domain is None:
            assert isinstance(pdf, np.polynomial.Polynomial)
            domain = pdf.domain
        else:
            if len(domain) != 2:
                raise ValueError('`domain` must be a length-2 sequence; got {}'
                                 .format(domain))
            if isinstance(pdf, np.polynomial.Polynomial):
                assert pdf.domain[0] == domain[0]
                assert pdf.domain[1] == domain[1]

        if domain[0] >= domain[1]:
            raise ValueError(
                '`domain[0]` (lower edge) is greater than or equal to'
                ' `domain[1]` (upper edge): {}'.format(domain)
            )
        self.domain = min(domain), max(domain)

        if not isinstance(pdf, np.polynomial.Polynomial):
            pdf = np.polynomial.Polynomial(pdf, domain=self.domain)

        # To be a valid PDF, must integrate to 1
        pdf_int = pdf.integ()
        pdf = pdf / (pdf_int(self.domain[1]) - pdf_int(self.domain[0]))

        cdf_poly = pdf.integ()
        cdf_poly -= cdf_poly(self.domain[0])
        cdf_poly /= cdf_poly(self.domain[1])

        # Make sure we have a valid CDF
        assert np.isclose(cdf_poly(self.domain[0]), 0)
        assert np.isclose(cdf_poly(self.domain[1]), 1)

        self.pdf_poly = pdf
        self.cdf_poly = cdf_poly

        # Numerically invert the CDF: Find domain locations (x-values) that
        # correspond to the CDF samples (y-values)
        domain_samples = []
        for cdf_y_val in self.cdf_samples:
            try:
                domain_x_val = brentq(
                    f=lambda x: self.cdf_poly(x) - cdf_y_val,
                    a=self.domain[0],
                    b=self.domain[1]
                )
                domain_samples.append(domain_x_val)
            except ValueError:
                print('Failed to find location in domain where CDF = {}'
                      .format(cdf_y_val))
                raise

        self.domain_samples = np.array(domain_samples)
        self.inv_cdf_slopes = (
            np.diff(self.domain_samples)
            / np.diff(self.cdf_samples**self.inv_power)
        )

    def __call__(self, val):
        """Perform linear interpolation on `val`, sample(s) in [0, 1]"""
        scalar = np.isscalar(val)
        val = np.atleast_1d(val)
        prob_bin_idx, dx = np.divmod(val**self.inv_power, self.prob_binwidth)
        exceed_max_mask = prob_bin_idx > self.n_samp - 2
        prob_bin_idx[exceed_max_mask] = self.n_samp - 2
        dx[exceed_max_mask] = self.prob_binwidth
        prob_bin_idx = prob_bin_idx.astype(int)
        y0 = self.domain_samples[prob_bin_idx]
        m = self.inv_cdf_slopes[prob_bin_idx]
        y = m*dx + y0
        if scalar:
            y = y[0]
        return y

max_stars_repo_name: ellohfin/retro

max_issues_repo_name: ellohfin/retro

max_forks_repo_name: ellohfin/retro


text:
import pandas as pd
import seaborn as sns
import numpy as np

def addpairs(
    taxon, csv_file, subject, category, groups,
    ax=None, figsize=None, width=0.8, **kwargs
):
    """
    Add paired lines in a plot created by `dokdo.taxa_abundance_box_plot`.

    Parameters
    ----------
    taxon : str
        Target taxon name.
    csv_file : str
        Path to the .csv file from the `taxa_abundance_box_plot` method.
    subject : str
        Column name to indicate pair information.
    category : str
        Column name to be studied.
    groups : list
        Groups in the category column.
    ax : matplotlib.axes.Axes, optional
        Axes object to draw the plot onto, otherwise uses the current Axes.
    figsize : tuple, optional
        Width, height in inches. Format: (float, float).
    width : float, default: 0.8
        Width of all the elements for one level of the grouping variable.
    kwargs : other keyword arguments
        All other keyword arguments are passed to `seaborn.lineplot`.

    Returns
    -------
    matplotlib.axes.Axes
        Axes object with the plot drawn onto it.

    See Also
    --------
    dokdo.api.taxa_abundance_box_plot

    Examples
    --------
    Below is a simple example where we pretend we only have the samples
    shown below and they are from a single subject. We are interested in
    comparing the relative abundance of the phylum Preteobacteria between
    the left palm and right palm. We also want to perofrm the comparison
    in the context of ``days-since-experiment-start`` (i.e. paired
    comparison).

    .. code:: python3

        import dokdo
        from qiime2 import Metadata
        import matplotlib.pyplot as plt
        %matplotlib inline
        import seaborn as sns
        sns.set()

        qzv_file = '/Users/sbslee/Desktop/dokdo/data/moving-pictures-tutorial/taxa-bar-plots.qzv'
        metadata_file = '/Users/sbslee/Desktop/dokdo/data/moving-pictures-tutorial/sample-metadata.tsv'

        sample_names = ['L2S240', 'L3S242', 'L2S155', 'L4S63', 'L2S175', 'L3S313', 'L2S204', 'L4S112', 'L2S222', 'L4S137']

        metadata = Metadata.load(metadata_file)
        metadata = metadata.filter_ids(sample_names)
        mf = dokdo.get_mf(metadata)
        mf = mf[['body-site', 'days-since-experiment-start']]

        ax = dokdo.taxa_abundance_box_plot(
            qzv_file,
            level=2,
            hue='body-site',
            taxa_names=['k__Bacteria;p__Proteobacteria'],
            show_others=False,
            figsize=(8, 7),
            sample_names=sample_names,
            pretty_taxa=True,
            include_samples={'body-site': ['left palm', 'right palm']},
            csv_file='addpairs.csv'
        )

        dokdo.addpairs(
            'k__Bacteria;p__Proteobacteria',
            'addpairs.csv',
            'days-since-experiment-start',
            'body-site',
            ['left palm', 'right palm'],
            ax=ax
        )

        plt.tight_layout()

    .. image:: images/addpairs-1.png

    Note that the method suppors more than two groups.

    .. code:: python3

        metadata = Metadata.load(metadata_file)
        mf = dokdo.get_mf(metadata)
        mf = mf[mf['subject'] == 'subject-1']
        mf = mf[['body-site', 'days-since-experiment-start']]
        mf = mf.drop_duplicates()

        ax = dokdo.taxa_abundance_box_plot(
            qzv_file,
            metadata=Metadata(mf),
            level=2,
            hue='body-site',
            taxa_names=['k__Bacteria;p__Proteobacteria'],
            show_others=False,
            figsize=(8, 7),
            pretty_taxa=True,
            csv_file='addpairs.csv'
        )

        dokdo.addpairs(
            'k__Bacteria;p__Proteobacteria',
            'addpairs.csv',
            'days-since-experiment-start',
            'body-site',
            ['gut', 'left palm', 'right palm', 'tongue'],
            ax=ax
        )

        plt.tight_layout()

    .. image:: images/addpairs-2.png
    """
    if ax is None:
        fig, ax = plt.subplots(figsize=figsize)

    df = pd.read_csv(csv_file)
    df = df[[subject, category, taxon]]
    df = df.rename(columns={taxon: 'perc'})

    n = len(groups)
    half = width / n / 2
    positions = np.linspace(0, width - half * 2, num=n) - half * (n-1)

    d = dict(zip(groups, positions))
    df[category] = df[category].map(d)
    df = df.pivot(index=category, columns=subject, values='perc')
    ax = sns.lineplot(data=df, dashes=False, legend=False, ax=ax, **kwargs)
    ax.set_xlabel('')

    return ax

max_stars_repo_name: sbslee/dokdo

max_issues_repo_name: sbslee/dokdo

max_forks_repo_name: sbslee/dokdo


text:
"""
Copyright (R) @huawei.com, all rights reserved
-*- coding:utf-8 -*-
CREATED:  2020-12-11 10:12:13
MODIFIED: 2020-12-11 14:04:45
"""
import sys
import numpy as np

def txt_contrast(txtname1, txtname2):
    """
    Verify that the txt are the same
    """
    with open(txtname1, "r", encoding="utf-8") as fp:
        file1 = fp.read()
    with open(txtname2, "r", encoding="utf-8") as fp:
        file2 = fp.read()

    if(file1 == file2):
        return 1
    else:
        return 0

if __name__ == '__main__':
    txt1 = sys.argv[1]
    txt2 = sys.argv[2]
    result = txt_contrast(txt1, txt2)
    if (result != 1):
        sys.exit(1)
    else:
        sys.exit(0)


text:
# This script is only used to create figures which are used in the documentation
# The script is hardcoded and not a good example how to develop :-)

# Dependencies
import numpy as np
import cv2
import matplotlib
matplotlib.use('Agg') # Do not display (no display in console at udacity)
import matplotlib.pyplot as plt

# Example data from next './next_data/driving_log_next.csv'
data = [
    ['./next_data/IMG/center_2019_11_30_02_59_02_499.jpg', './next_data/IMG/left_2019_11_30_02_59_02_499.jpg', './next_data/IMG/right_2019_11_30_02_59_02_499.jpg', 0.0],
    ['./next_data/IMG/center_2019_11_30_03_01_49_365.jpg', './next_data/IMG/left_2019_11_30_03_01_49_365.jpg', './next_data/IMG/right_2019_11_30_03_01_49_365.jpg', 0.5022397]
]

# Get images to display (also convert from BGR to RGB)
img1_center = cv2.cvtColor(cv2.imread(data[0][0]), cv2.COLOR_BGR2RGB)
img1_left = cv2.cvtColor(cv2.imread(data[0][1]), cv2.COLOR_BGR2RGB)
img1_right = cv2.cvtColor(cv2.imread(data[0][2]), cv2.COLOR_BGR2RGB)

img2_center = cv2.cvtColor(cv2.imread(data[1][0]), cv2.COLOR_BGR2RGB)
img2_left = cv2.cvtColor(cv2.imread(data[1][1]), cv2.COLOR_BGR2RGB)
img2_right = cv2.cvtColor(cv2.imread(data[1][2]), cv2.COLOR_BGR2RGB)

# Horizontally flip images
img1_center_flipped = np.fliplr(img1_center)
img1_left_flipped = np.fliplr(img1_left)
img1_right_flipped = np.fliplr(img1_right)

img2_center_flipped = np.fliplr(img2_center)
img2_left_flipped = np.fliplr(img2_left)
img2_right_flipped = np.fliplr(img2_right)

# Visualize images in figure and save it afterwards
fig, subplots = plt.subplots(2,3, figsize=(20, 8))
subplots[0][0].imshow(img1_center)
subplots[0][0].set_title("Centered image #1")
subplots[0][1].imshow(img1_left)
subplots[0][1].set_title("Left image #1 (+2.0 steering angle)")
subplots[0][2].imshow(img1_right)
subplots[0][2].set_title("Right image #1 (-2.0 steering angle)")
subplots[1][0].imshow(img2_center)
subplots[1][0].set_title("Centered image #2")
subplots[1][1].imshow(img2_left)
subplots[1][1].set_title("Left image #2 (+2.0 steering angle)")
subplots[1][2].imshow(img2_right)
subplots[1][2].set_title("Right image #2 (-2.0 steering angle)")

fig.savefig("./example_figures/example-images.png")

# Visualize flipped images in figure and save it afterwards
fig, subplots = plt.subplots(2,3, figsize=(20, 8))
subplots[0][0].imshow(img1_center_flipped)
subplots[0][0].set_title("Centered image #1 flipped")
subplots[0][1].imshow(img1_left_flipped)
subplots[0][1].set_title("Left image #1 (flipped)")
subplots[0][2].imshow(img1_right_flipped)
subplots[0][2].set_title("Right image #1 (flipped)")
subplots[1][0].imshow(img2_center_flipped)
subplots[1][0].set_title("Centered image #2")
subplots[1][1].imshow(img2_left_flipped)
subplots[1][1].set_title("Left image #2 (flipped)")
subplots[1][2].imshow(img2_right_flipped)
subplots[1][2].set_title("Right image #2 (flipped)")

fig.savefig("./example_figures/example-images-flipped.png")

max_stars_repo_name: KevinHubert-Dev/AI-BehavioralCloning

max_issues_repo_name: KevinHubert-Dev/AI-BehavioralCloning

max_forks_repo_name: KevinHubert-Dev/AI-BehavioralCloning


text:
from numpy import mean
from core.sdlxliff import SdlXliff
from collections import namedtuple
from math import floor

DiscProblem = namedtuple('DiscProblem', 'disc mid source target'.split(' '))

def check_length(sdlxliff, min=0.2, max=1.3, source_min=10):

    def _get_disc(segment):
        return round(len(segment.target) / len(segment.source), 2)

    def _get_average_discrepancy(sdlxliff):

        discrepancies = [
            _get_disc(segment) for segment in sdlxliff.segments
        ]

        return mean(discrepancies)

    sdlxliff = SdlXliff(sdlxliff)
    average_disc = _get_average_discrepancy(sdlxliff)

    min = average_disc * min
    max = average_disc * max

    result = [
        DiscProblem(_get_disc(segment), segment.mid, segment.source, segment.target)
        for segment in sdlxliff.segments
        if min < _get_disc(segment) > max and len(segment.source) >= source_min
        ]

    return result

max_stars_repo_name: elderdk/pyxliff

max_issues_repo_name: elderdk/pyxliff

max_forks_repo_name: elderdk/pyxliff


text:
"""
Credit: kyubyong park(kbpark.linguist@gmail.com) and Jongseok Kim(https://github.com/ozmig77)
"""
import typing
from enum import Enum
from pathlib import Path

import grapheme
import numpy as np

# -----------------------------------------------------------------------------

class FixedSymbols(str, Enum):
    """Pseudo-graphemes with fixed indexes"""

    PAD = "<pad>"
    SOS = "<s>"
    EOS = "</s>"

def _sigmoid(x):
    return 1 / (1 + np.exp(-x))

def _grucell(x, h, w_ih, w_hh, b_ih, b_hh):
    rzn_ih = np.matmul(x, w_ih.T) + b_ih
    rzn_hh = np.matmul(h, w_hh.T) + b_hh

    rz_ih, n_ih = (
        rzn_ih[:, : rzn_ih.shape[-1] * 2 // 3],
        rzn_ih[:, rzn_ih.shape[-1] * 2 // 3 :],
    )
    rz_hh, n_hh = (
        rzn_hh[:, : rzn_hh.shape[-1] * 2 // 3],
        rzn_hh[:, rzn_hh.shape[-1] * 2 // 3 :],
    )

    rz = _sigmoid(rz_ih + rz_hh)
    r, z = np.split(rz, 2, -1)  # pylint: disable=unbalanced-tuple-unpacking

    n = np.tanh(n_ih + r * n_hh)
    h = (1 - z) * n + z * h

    return h

def _gru(x, steps, w_ih, w_hh, b_ih, b_hh, h0=None):
    if h0 is None:
        h0 = np.zeros((x.shape[0], w_hh.shape[1]), np.float32)
    h = h0  # initial hidden state
    outputs = np.zeros((x.shape[0], steps, w_hh.shape[1]), np.float32)
    for t in range(steps):
        h = _grucell(x[:, t, :], h, w_ih, w_hh, b_ih, b_hh)  # (b, h)
        outputs[:, t, ::] = h
    return outputs

# -----------------------------------------------------------------------------

class GeepersG2P:
    """Phoneme prediction from a pre-trained model using pure numpy"""

    def __init__(
        self,
        graphemes: typing.List[str],
        phonemes: typing.List[str],
        dec_maxlen: int = 20,
    ):
        self.graphemes = [FixedSymbols.PAD.value, FixedSymbols.EOS.value] + graphemes
        self.phonemes = [
            FixedSymbols.PAD.value,
            FixedSymbols.EOS.value,
            FixedSymbols.SOS.value,
        ] + phonemes

        self.g2idx = {g: idx for idx, g in enumerate(self.graphemes)}
        self.p2idx = {p: idx for idx, p in enumerate(self.phonemes)}

        self.pad_idx = 0
        self.eos_idx = 1
        self.sos_idx = 2

        self.dec_maxlen = dec_maxlen

        # Empty until load_variables is called
        self.enc_emb = np.empty((1,))
        self.enc_w_ih = np.empty((1,))
        self.enc_w_hh = np.empty((1,))
        self.enc_b_ih = np.empty((1,))
        self.enc_b_hh = np.empty((1,))

        self.dec_emb = np.empty((1,))
        self.dec_w_ih = np.empty((1,))
        self.dec_w_hh = np.empty((1,))
        self.dec_b_ih = np.empty((1,))
        self.dec_b_hh = np.empty((1,))
        self.fc_w = np.empty((1,))
        self.fc_b = np.empty((1,))

    def load_variables(self, npz_path: typing.Union[str, Path]):
        """Load encoder/decoder weights from .npz file"""
        variables = np.load(str(npz_path))
        self.enc_emb = variables["encoder.emb.weight"]  # (len(graphemes), emb)
        self.enc_w_ih = variables["encoder.rnn.weight_ih_l0"]  # (3*128, 64)
        self.enc_w_hh = variables["encoder.rnn.weight_hh_l0"]  # (3*128, 128)
        self.enc_b_ih = variables["encoder.rnn.bias_ih_l0"]  # (3*128,)
        self.enc_b_hh = variables["encoder.rnn.bias_hh_l0"]  # (3*128,)

        self.dec_emb = variables["decoder.emb.weight"]  # (len(phonemes), emb)
        self.dec_w_ih = variables["decoder.rnn.weight_ih_l0"]  # (3*128, 64)
        self.dec_w_hh = variables["decoder.rnn.weight_hh_l0"]  # (3*128, 128)
        self.dec_b_ih = variables["decoder.rnn.bias_ih_l0"]  # (3*128,)
        self.dec_b_hh = variables["decoder.rnn.bias_hh_l0"]  # (3*128,)
        self.fc_w = variables["decoder.fc.weight"]  # (74, 128)
        self.fc_b = variables["decoder.fc.bias"]  # (74,)

    def _encode(self, graphemes: typing.List[str]) -> np.ndarray:
        x = [self.g2idx[g] for g in graphemes] + [self.eos_idx]
        return np.take(self.enc_emb, np.expand_dims(x, 0), axis=0)

    def predict(self, word: str) -> typing.List[str]:
        """Predict phonemes for the given word"""
        # encoder
        graphemes = list(grapheme.graphemes(word))
        enc = self._encode(graphemes)
        enc = _gru(
            enc,
            len(graphemes) + 1,
            self.enc_w_ih,
            self.enc_w_hh,
            self.enc_b_ih,
            self.enc_b_hh,
            h0=np.zeros((1, self.enc_w_hh.shape[-1]), np.float32),
        )
        last_hidden = enc[:, -1, :]

        # decoder
        dec = np.take(self.dec_emb, [2], axis=0)  # 2: <s>
        h = last_hidden

        preds = []
        for _ in range(self.dec_maxlen):
            h = _grucell(
                dec, h, self.dec_w_ih, self.dec_w_hh, self.dec_b_ih, self.dec_b_hh
            )  # (b, h)
            logits = np.matmul(h, self.fc_w.T) + self.fc_b
            pred = logits.argmax()
            if pred == self.eos_idx:
                break  # </s>
            preds.append(pred)
            dec = np.take(self.dec_emb, [pred], axis=0)

        preds = [self.phonemes[idx] for idx in preds]
        return preds

max_stars_repo_name: daniele-athome/rhasspy-nlu

max_issues_repo_name: daniele-athome/rhasspy-nlu

max_forks_repo_name: daniele-athome/rhasspy-nlu


text:
from __future__ import print_function
from benchpress.benchmarks import util
import numpy as np

bench = util.Benchmark("Rosenbrock", "size*iterations")

def rosen(x):
    return np.sum(
        100.0 * (x[1:] - x[:-1] ** 2.0) ** 2.0 + \
        (1 - x[:-1]) ** 2.0
    )

def main():
    N, T = bench.args.size  # Grab command-line arguments
    dataset = np.arange(N, dtype=bench.dtype) / bench.dtype(N)

    bench.start()  # Sample wall-clock start
    res = 0.0
    for _ in range(0, T):  # Do T trials of..
        res += rosen(dataset)  # ..executing rosenbrock.
        bench.flush()
    res /= T
    bench.stop()  # Sample wall-clock stop
    bench.pprint()  # Print elapsed wall-clock etc.

if __name__ == "__main__":
    main()


text:
#!/usr/bin/env python

"""Calculate a score for a square matrix."""

import random
import numpy as np

random.seed(0)

def calculate_score(cm, weights):
    """
    Calculate a score how close big elements of cm are to the diagonal.

    Examples
    --------
    >>> cm = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])
    >>> weights = calculate_weight_matrix(3)
    >>> calculate_score(cm, weights)
    32
    """
    return int(np.tensordot(cm, weights, axes=((0, 1), (0, 1))))

def calculate_weight_matrix(n):
    """
    Calculate the weights for each position.

    The weight is the distance to the diagonal.
    """
    weights = np.abs(np.arange(n) - np.arange(n)[:, None])
    return weights

def measure_time(n):
    """Measure the time of calculate_score for n x n matrices."""
    import time
    import numpy as np
    score_calculations = 10**3

    t = 0
    weights = calculate_weight_matrix(n)
    for step in range(score_calculations):
        cm = np.random.randint(0, 150000, size=(n, n))
        t0 = time.time()
        calculate_score(cm, weights)
        t1 = time.time()
        t += (t1 - t0)
    print("{:0.2f} scores / sec".format(score_calculations / t))

if __name__ == '__main__':
    import doctest
    doctest.testmod()
    measure_time(369)

max_stars_repo_name: PepSalehi/algorithms

max_issues_repo_name: PepSalehi/algorithms

max_forks_repo_name: PepSalehi/algorithms