text:
#!/usr/bin/env python
# coding: utf-8

# In[29]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pylab import *

# In[4]:

import csv
#import numpy
from os import listdir
from os.path import isfile, join

# ### Load dataset

# In[3]:

users = pd.read_csv('datasets/User.csv')
symptom = pd.read_csv('datasets/Symptom.csv')
period = pd.read_csv('datasets/Period.csv')

# In[4]:

users.info()

# In[5]:

users.isnull().sum()

# ## 2 . Preprocessing Data

# ## We found many NaN values in dob columns. It can be explained that users avoid to inputting their personal inforlmation. So dropping this column is better for the exploratory data

# In[6]:

df_user = users.drop(columns = ['dob'])

# In[7]:

df_user.info()

# In[8]:

df_user.describe()

# ### we can see some nonsense Max value in period_length_initial column. It is not normal if the period last more than 7 days

# In[9]:

# we split users into 3 groups:
# group 1: length <= 7
# group 2: 7 < length <= 15
# group 3: 15 < length <= 51

criteria  = df_user['period_length_initial'] <= 7
df_user1 = df_user[criteria]

criteria1  = df_user['period_length_initial'] > 7
criteria2 = df_user['period_length_initial'] <= 15
criteria = criteria1 & criteria2

df_user2 = df_user[criteria]

criteria = df_user['period_length_initial'] > 15
df_user3 = df_user[criteria]

# In[10]:

# show some lines of each dataframe
print(df_user1.head())
print(df_user2.head())
print(df_user3.head())

# ### Lets explore period dataset

# In[11]:

period.info()

# In[14]:

#df_period = period.dropna()
#df_period.info()

#convert date object into date format
df_period = period
df_period['start_date'] = pd.to_datetime(df_period['start_date'], errors='coerce' )
df_period['end_date'] = pd.to_datetime(df_period['end_date'], errors='coerce' )
df_period = df_period.dropna()

#period['start'] = pd.to_datetime(period['start_date'])
#period['end'] = pd.to_datetime(period['end_date'])

# ### Lets see how many missing records deleted

# In[15]:

print(len(period.values) - len(df_period.values))

# ### Lets explore symptom dataset

# In[16]:

symptom.info()

# In[17]:

symptom.isnull().sum()

# ### There is no NaN values found in Symptom dataset.
# ### Convert date object into date format

# In[18]:

df_symptom = symptom
df_symptom['date'] = pd.to_datetime(df_symptom['date'], errors='coerce')

# In[19]:

df_symptom.info()

# In[20]:

df_symptom.describe()

# ### To easy to know how much painful users suffer before, after and during period, We add one more column that is total of value of symptoms

# In[21]:

df_sum = df_symptom.drop(columns = ['id', 'user_id', 'date'])
df_symptom['total'] = np.sum(df_sum, axis=1)

df_symptom.head()

# ## 3. Exploratory Data Analysis

# In[22]:

import matplotlib.pyplot as  plt

# In[23]:

print(df_user1.head())
print(df_symptom.head())

# ### We know that period length is not valid if it is greater than 7. Let plot 3 group of periods to see how many invalid data

# In[35]:

data = {'2-7':len(df_user1), '8-15':len(df_user2), '>15':len(df_user3)}
cols = list(data.keys())
vals = list(data.values())

plt.bar(cols, vals)

# ### Next, we will discover if there is any relation between period length and symptom

# In[36]:

df_tmp = df_user1.set_index('id').join(df_symptom.set_index('user_id'))
val1 = df_tmp['total'].median()

df_tmp = df_user2.set_index('id').join(df_symptom.set_index('user_id'))
val2 = df_tmp['total'].median()

df_tmp = df_user3.set_index('id').join(df_symptom.set_index('user_id'))
val3 = df_tmp['total'].median()

data = {'2-7':val1, '8-15':val2, '>15':val3}
cols = list(data.keys())
vals = list(data.values())

plt.bar(cols, vals)

# ### Users with normal period (7 days maximum) tend to suffer less pain than users with long period

# In[73]:

df_tmp = df_symptom.set_index('user_id').join(df_user.set_index('id'), how='outer')

data = {'registration':len(df_user['id']), 'observation':len(df_symptom.groupby('user_id'))}
keys = list(data.keys())
vals = list(data.values())

plt.bar(keys, vals)

# ### The number of people who record their data are just a little bit more than half of registration. Maybe, they forget to track their activity after creating an account on application. There should be an automatic notification to remind users keeping to record their data

# In[13]:

#def __init__(self):
#    self.list_tuple=1

def readCSV(file_name):
    csv_file = open(file_name, mode='rU')
    data = csv.DictReader(csv_file)
    return data
def date2days(date):
    if len(date) and '/' in date:
        date=date.strip('')
        date=date.split("/")
        #print int(date[0]) + int(date[1])*30 + int(date[2])*365
        return int(date[0]) + int(date[1])*30 + int(date[2])*365
    else:
        return len(date)

def totalcoloumn(matrix):
    temp=[]
    for row in matrix:
        sum=0
        for i in range(len(row)):
            sum+=int(row[i])
        temp.append(sum)
    return temp

def coloumn(matrix, i):
    temp=[]
    for row in matrix:
        if i<len(row):
            temp.append(row[i])
    return temp

# In[35]:

def parseVal():
    #onlyfiles = [f for f in listdir("/Users/abhishekmishra/Downloads/Hackathon/ECG_RR/bidmc_csv") if isfile(join("/Users/abhishekmishra/Downloads/Hackathon/ECG_RR/bidmc_csv", f))]
    #mypath = "/Users/abhishekmishra/Downloads/Hackathon/pslovedata/"$
    mypath = "datasets/"
    #period = pslove()
    #user = pslove()
    #symptom = pslove()
    period_data = readCSV(mypath+"Period.csv")
    user_data = readCSV(mypath+"User.csv")
    symptom_data = readCSV(mypath+"Symptom.csv")
    user_id_array=set()
    user_dict={}
    for row in user_data:
        user_id_array.add(row["id"])
    for i in range(len(list(user_id_array))):
        user_dict[list(user_id_array)[i]]=[]
    user_data = readCSV(mypath+"User.csv")
    for row in user_data:
        user_dict[row["id"]].append(row)
    period_data = readCSV(mypath+"Period.csv")
    for row in period_data:
        user_dict[row["User_id"]].append(row)
    symptom_data = readCSV(mypath+"Symptom.csv")
    for row in symptom_data:
        user_dict[row["user_id"]].append(row)
    analyze_list=[]
    for key in user_dict.keys():
        flag=1
        index=1
        final_index=1
        if len(user_dict[key])>1:
            while(flag):
                if "User_id" in user_dict[key][index].keys():
                    index+=1
                else:
                    flag=0
                    final_index=index
                    index=final_index
                if index==len(user_dict[key]):
                    break
            while(index<len(user_dict[key])):
                temp=[]
                temp.append(user_dict[key][0]['cycle_length_initial'])
                temp.append(user_dict[key][0]['period_length_initial'])
                val=date2days(user_dict[key][index]["date"])
                flag=0
                for i in range(1,final_index):
                    #print self.date2days(user_dict[key][i]["end_date"]) - self.date2days(user_dict[key][i]["start_date"])
                    if (val>=date2days(user_dict[key][i]["start_date"]) and val<=date2days(user_dict[key][i]["end_date"])):
                        if (val-date2days(user_dict[key][i]["start_date"]))>0 and (val-date2days(user_dict[key][i]["start_date"]))<15:
                            temp.append(val-date2days(user_dict[key][i]["start_date"]))
                            #print val-self.date2days(user_dict[key][i]["start_date"])
                        else:
                            temp.append(0)
                        flag=1
                        break
                if flag==0:
                    temp.append(flag)
                index+=1
                if 'backache' in user_dict[key][index-1].keys():
                    temp.append(user_dict[key][index-1]['backache'])
                    temp.append(user_dict[key][index-1]['cramp'])
                    temp.append(user_dict[key][index-1]['mood'])
                    temp.append(user_dict[key][index-1]['bloating'])
                    temp.append(user_dict[key][index-1]['acne'])
                    temp.append(user_dict[key][index-1]['diarrhea'])
                    temp.append(user_dict[key][index-1]['sore'])
                    temp.append(user_dict[key][index-1]['dizzy'])
                    temp.append(user_dict[key][index-1]['nausea'])
                    temp.append(user_dict[key][index-1]['headache'])
                analyze_list.append(temp)
        else:
            temp=[]
            temp.append(user_dict[key][0]['cycle_length_initial'])
            temp.append(user_dict[key][0]['period_length_initial'])
            analyze_list.append(temp)
    pain_dict={}
    pain_dict["1"]='backache'
    pain_dict["2"]='cramp'
    pain_dict["3"]='mood'
    pain_dict["4"]='bloating'
    pain_dict["5"]='acne'
    pain_dict["6"]='diarrhea'
    pain_dict["7"]='sore'
    pain_dict["8"]='dizzy'
    pain_dict["9"]='nausea'
    pain_dict["10"]='headache'
    for i in range(10):
        string=pain_dict[str(i+1)]
        plt.figure(i+1)
        colors = (0,0,0)
        area = np.pi*3
        plt.scatter(coloumn(analyze_list,2),coloumn(analyze_list,3+i),s=area, c=colors, alpha=0.7)
        plt.xlabel("Days passed from start of period")
        plt.ylabel(string+" Score")
        plt.title(string + " variation with period onset")
        plt.yticks(np.arange(1,100,20))
        plt.savefig(string+".png")
        plt.show()

        #for i in range(1):
    plt.figure(11)
    colors = (0,0,0)
    area = np.pi*3
    plt.scatter(coloumn(analyze_list,1),totalcoloumn(analyze_list),s=area, c=colors, alpha=0.7)
    plt.xlabel("Initial period length")
    plt.ylabel("Frequency of user's record")
    plt.title("Pain variation with initial period length")
    plt.yticks(np.arange(1,1500,100))
    #plt.savefig("Pain Initial Period.png")
    plt.show()
#for i in range(1):
    plt.figure(12)
    colors = (0,0,0)
    area = np.pi*3
    plt.scatter(coloumn(analyze_list,0),totalcoloumn(analyze_list),s=area, c=colors, alpha=0.7)
    plt.xlabel("Initial cycle length")
    plt.ylabel("Frequency of user's record")
    plt.title("Pain variation with initial cycle length")
    plt.yticks(np.arange(1,1500,100))
    #plt.savefig("Pain Initial cycle.png")
    plt.show()#

# ### We will plot the frequency distribution of symptoms according to cycle length and period length

# In[36]:

parseVal()

# ### The output clearly shows that all the pain scores follow a general trend of decreasing with increase of days passed from the onset of period.
# ### After 6th day, we notice that pain is almost minimal. We can conclude that this sort of data distribution is not normal and can be considered as outliners
# ### Based on type of symptoms happens before and on initial days of period, we can recommend medicine , herbs and pregnant pills to users.

# In[ ]:


text:
#!/usr/bin/env python
""" atlasextraction.py: extract data for each ROI in an atlas image
USAGE: atlasextraction.py <featdir> <descriptor> <atlasimage>
"""
import nibabel as nib
import numpy as N
import os,sys
from run_shell_cmd import *
#from mvpa.misc.fsl.base import read_fsl_design

def mk_native_atlas(featdir,atlasfile,atlas_descriptor):

    # make the atlas in native space if it doesn't exist
    # just use linear reg, as fnirt does goofy stuff
    nativeatlas=featdir+'/%s_native.nii.gz'%atlas_descriptor
    if not os.path.exists(nativeatlas):
        cmd='flirt -in %s -out %s -ref %s -applyxfm -init %s/reg/standard2example_func.mat -interp nearestneighbour'%(atlasfile,nativeatlas,featdir+'/example_func.nii.gz',featdir)
        run_shell_cmd(cmd)
    return nativeatlas

def extract_roi_means(datafile,atlasdata,outfilename):

    atlasrois=N.unique(atlasdata[atlasdata>0])
    nroi=N.max(atlasrois)+1

    img=nib.load(datafile)
    data=img.get_data()
    ntp=data.shape[3]

    roidata=N.zeros((nroi,ntp))
    for r in atlasrois:
        roivox=(atlasdata==r)
        roidata[r,:]=N.mean(data[roivox,:],0)
    roidata=roidata[1:,:].T

    N.savetxt(outfilename,roidata,delimiter='\t')
    return roidata

def main():
#if 1==1:

    #atlasfile='/work/01329/poldrack/code/poldrack/roi_atlas/sc_HO_atlas.nii.gz'
    #atlas_descriptor='sc_HO'
    bsmethod='lsone'
    featdir=sys.argv[1]
    atlas_descriptor=sys.argv[2]
    atlasfile=sys.argv[3]

    nativeatlas=mk_native_atlas(featdir,atlasfile,atlas_descriptor)
    atlas=nib.load(nativeatlas)
    atlasdata=atlas.get_data()

    # extract roi data from betaseries

    if os.path.exists(featdir+'/betaseries'):

        bsfiles_all=os.listdir(featdir+'/betaseries')
        bsfiles=['%s/betaseries/%s'%(featdir,f) for f in bsfiles_all if f.find(bsmethod)>-1 and f.find('nii.gz')>-1]
    #    print "files:"
    #    print bsfiles

        for bsf in bsfiles:
            roidata=extract_roi_means(bsf,atlasdata,bsf.replace('.nii.gz','_%s.txt'%atlas_descriptor))

    # now extract for residual
    residfile=featdir+'/stats/res4d.nii.gz'
    roidata_res4d=extract_roi_means(residfile,atlasdata,residfile.replace('.nii.gz','_%s.txt'%atlas_descriptor))

if __name__=='__main__':
    main()

ext: py

max_stars_repo_name: lbconner/openfMRI

max_issues_repo_name: lbconner/openfMRI

max_forks_repo_name: lbconner/openfMRI

reason: import numpy


text:
r"""
Crystals

Let `T` be a CartanType with index set `I`, and
`W` be a realization of the type `T` weight
lattice.

A type `T` crystal `C` is a colored oriented graph
equipped with a weight function from the nodes to some realization
of the type `T` weight lattice such that:

-  Each edge is colored with a label in `i \in I`.

-  For each `i\in I`, each node `x` has:

   -  at most one `i`-successor `f_i(x)`;

   -  at most one `i`-predecessor `e_i(x)`.

   Furthermore, when they exist,

   -  `f_i(x)`.weight() = x.weight() - `\alpha_i`;

   -  `e_i(x)`.weight() = x.weight() + `\alpha_i`.

This crystal actually models a representation of a Lie algebra if
it satisfies some further local conditions due to Stembridge [St2003]_.

REFERENCES:

.. [St2003] J. Stembridge, *A local characterization of simply-laced crystals*,
   Trans. Amer. Math. Soc. 355 (2003), no. 12, 4807-4823.

EXAMPLES:

We construct the type `A_5` crystal on letters (or in representation
theoretic terms, the highest weight crystal of type `A_5`
corresponding to the highest weight `\Lambda_1`)::

    sage: C = crystals.Letters(['A',5]); C
    The crystal of letters for type ['A', 5]

It has a single highest weight element::

    sage: C.highest_weight_vectors()
    [1]

A crystal is an enumerated set (see :class:`EnumeratedSets`); and we
can count and list its elements in the usual way::

    sage: C.cardinality()
    6
    sage: C.list()
    [1, 2, 3, 4, 5, 6]

as well as use it in for loops::

    sage: [x for x in C]
    [1, 2, 3, 4, 5, 6]

Here are some more elaborate crystals (see their respective
documentations)::

    sage: Tens = crystals.TensorProduct(C, C)
    sage: Spin = crystals.Spins(['B', 3])
    sage: Tab  = crystals.Tableaux(['A', 3], shape = [2,1,1])
    sage: Fast = crystals.FastRankTwo(['B', 2], shape = [3/2, 1/2])
    sage: KR = crystals.KirillovReshetikhin(['A',2,1],1,1)

One can get (currently) crude plotting via::

    sage: Tab.plot()

If dot2tex is installed, one can obtain nice latex pictures via::

    sage: K = crystals.KirillovReshetikhin(['A',3,1], 1,1)
    sage: view(K, pdflatex=True, tightpage=True) #optional - dot2tex graphviz

or with colored edges::

    sage: K = crystals.KirillovReshetikhin(['A',3,1], 1,1)
    sage: G = K.digraph()
    sage: G.set_latex_options(color_by_label = {0:"black", 1:"red", 2:"blue", 3:"green"}) #optional - dot2tex graphviz
    sage: view(G, pdflatex=True, tightpage=True) #optional - dot2tex graphviz

For rank two crystals, there is an alternative method of getting
metapost pictures. For more information see C.metapost?

See also the categories :class:`Crystals`, :class:`ClassicalCrystals`,
:class:`FiniteCrystals`, :class:`HighestWeightCrystals`.

.. TODO::

    -  Vocabulary and conventions:

       -  For a classical crystal: connected / highest weight /
          irreducible

       -  ...

    -  Layout instructions for plot() for rank 2 types

    -  RestrictionOfCrystal

Most of the above features (except Littelmann/alcove paths) are in
MuPAD-Combinat (see lib/COMBINAT/crystals.mu), which could provide
inspiration.
"""

#*****************************************************************************
#       Copyright (C) 2007 Anne Schilling <anne at math.ucdavis.edu>
#                          Nicolas Thiery <nthiery at users.sf.net>
#
#  Distributed under the terms of the GNU General Public License (GPL)
#
#    This code is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    General Public License for more details.
#
#  The full text of the GPL is available at:
#
#                  http://www.gnu.org/licenses/
#****************************************************************************
# Acknowledgment: most of the design and implementation of this
# library is heavily inspired from MuPAD-Combinat.
#****************************************************************************

#from sage.structure.unique_representation import UniqueRepresentation
#from sage.structure.parent import Parent
#from sage.structure.element import Element
#from sage.categories.finite_enumerated_sets import FiniteEnumeratedSets
#from sage.graphs.all import DiGraph
#from sage.combinat import ranker
#from sage.combinat.root_system.weyl_characters import WeylCharacter
from sage.combinat.backtrack import GenericBacktracker

class CrystalBacktracker(GenericBacktracker):
    def __init__(self, crystal, index_set=None):
        r"""
        Time complexity: `O(nF)` amortized for each produced
        element, where `n` is the size of the index set, and `F` is
        the cost of computing `e` and `f` operators.

        Memory complexity: `O(D)` where `D` is the depth of the crystal.

        Principle of the algorithm:

        Let `C` be a classical crystal. It's an acyclic graph where all
        connected component has a unique element without predecessors (the
        highest weight element for this component). Let's assume for
        simplicity that `C` is irreducible (i.e. connected) with highest
        weight element `u`.

        One can define a natural spanning tree of `C` by taking
        `u` as the root of the tree, and for any other element
        `y` taking as ancestor the element `x` such that
        there is an `i`-arrow from `x` to `y` with
        `i` minimal. Then, a path from `u` to `y`
        describes the lexicographically smallest sequence
        `i_1,\dots,i_k` such that
        `(f_{i_k} \circ f_{i_1})(u)=y`.

        Morally, the iterator implemented below just does a depth first
        search walk through this spanning tree. In practice, this can be
        achieved recursively as follow: take an element `x`, and
        consider in turn each successor `y = f_i(x)`, ignoring
        those such that `y = f_j(x^{\prime})` for some `x^{\prime}` and
        `j<i` (this can be tested by computing `e_j(y)`
        for `j<i`).

        EXAMPLES::

            sage: from sage.combinat.crystals.crystals import CrystalBacktracker
            sage: C = crystals.Tableaux(['B',3],shape=[3,2,1])
            sage: CB = CrystalBacktracker(C)
            sage: len(list(CB))
            1617
            sage: CB = CrystalBacktracker(C, [1,2])
            sage: len(list(CB))
            8
        """
        GenericBacktracker.__init__(self, None, None)
        self._crystal = crystal
        if index_set is None:
            self._index_set = crystal.index_set()
        else:
            self._index_set = index_set

    def _rec(self, x, state):
        """
        Return an iterator for the (immediate) children of ``x`` in the search
        tree.

        EXAMPLES::

            sage: from sage.combinat.crystals.crystals import CrystalBacktracker
            sage: C = crystals.Letters(['A', 5])
            sage: CB = CrystalBacktracker(C)
            sage: list(CB._rec(C(1), 'n/a'))
            [(2, 'n/a', True)]
        """
        #We will signal the initial case by having a object and state
        #of None and consider it separately.
        if x is None and state is None:
            for gen in self._crystal.highest_weight_vectors():
                yield gen, "n/a", True
            return

        # Run through the children y of x
        for i in self._index_set:
            y = x.f(i)
            if y is None:
                continue
            # Ignore those which can be reached by an arrow with smaller label
            hasParent = False
            for j in self._index_set:
                if j == i:
                    break
                if not y.e(j) is None:
                    hasParent = True
                    break
            if hasParent:
                continue

            # yield y and all elements further below
            yield y, "n/a", True

reason: from sage


text:
import numpy as np
try:
    import tensorflow as tf
except ImportError:
    tf = None

def custom_mse(y_true, y_pred, sample_weight=None):
    y_pred = tf.split(y_pred, num_or_size_splits=2, axis=1)[0]
    se = tf.square(tf.subtract(y_true, y_pred))
    se_red = tf.reduce_mean(se)
    return se_red

def custom_mean_squared_logp1_error(y_true, y_pred, sample_weight=None):
    y_pred = tf.split(y_pred, num_or_size_splits=2, axis=1)[0]
    y_true = tf.math.log(y_true + 1.)
    y_pred = tf.math.log(y_pred + 1.)
    se = tf.square(tf.subtract(y_true, y_pred))
    se_red = tf.reduce_mean(se)
    return se_red

def custom_negll_nb(y_true, y_pred, sample_weight=None):
    x = y_true
    loc, scale = tf.split(y_pred, num_or_size_splits=2, axis=1)

    eta_loc = tf.math.log(loc)
    eta_scale = tf.math.log(scale)

    log_r_plus_mu = tf.math.log(scale + loc)

    ll = tf.math.lgamma(scale + x)
    ll = ll - tf.math.lgamma(x + tf.ones_like(x))
    ll = ll - tf.math.lgamma(scale)
    ll = ll + tf.multiply(x, eta_loc - log_r_plus_mu) + tf.multiply(scale, eta_scale - log_r_plus_mu)

    ll = tf.clip_by_value(ll, -300, 300, "log_probs")
    neg_ll = -ll
    neg_ll = tf.reduce_mean(neg_ll)
    return neg_ll

def custom_negll_gaussian(y_true, y_pred, sample_weight=None):
    loc, scale = tf.split(y_pred, num_or_size_splits=2, axis=1)

    ll = -tf.math.log(scale * tf.math.sqrt(2. * np.pi)) - 0.5 * tf.math.square((y_true - loc) / scale)
    ll = tf.clip_by_value(ll, -300, 300, "log_probs")
    neg_ll = -ll
    neg_ll = tf.reduce_mean(neg_ll)
    return neg_ll

def custom_kl(y_true, y_pred, sample_weight=None):
    expected_logqz_x, expected_logpz = tf.split(y_pred, num_or_size_splits=2, axis=1)
    kl_loss = tf.reduce_mean(expected_logqz_x - expected_logpz)
    return kl_loss

def custom_cce_agg(y_true, y_pred, sample_weight=None):
    y_pred = tf.clip_by_value(y_pred, 1e-10, 1., "y_pred")
    ll_cce_agg = -tf.math.log(tf.reduce_mean(y_true * y_pred, axis=1, keepdims=False))
    return ll_cce_agg

class CustomAccAgg(tf.keras.metrics.Metric):

    def __init__(self, name='acc_agg', **kwargs):
        super(CustomAccAgg, self).__init__(name=name, **kwargs)
        self.acc_agg = self.add_weight(name='acc', initializer='zeros')
        self.count = self.add_weight(name='count', initializer='zeros')

    def update_state(self, y_true, y_pred, sample_weight=None):
        phat_pos_agg = tf.reduce_sum(y_true * y_pred, axis=1, keepdims=True)
        acc_agg = tf.cast(
            phat_pos_agg > tf.reduce_max((tf.ones_like(y_true) - y_true) * y_pred, axis=1),
            dtype=y_true.dtype
        )
        # Do not use weighting for accuracy.
        self.acc_agg.assign_add(tf.reduce_mean(acc_agg))
        self.count.assign_add(1.)

    def result(self):
        return tf.divide(self.acc_agg, self.count)

    def reset_states(self):
        self.acc_agg.assign(0.)
        self.count.assign(0.)

class CustomTprClasswise(tf.keras.metrics.Metric):

    def __init__(self, k: int, name='tpr', **kwargs):
        super(CustomTprClasswise, self).__init__(name=name, **kwargs)
        self.tp = self.add_weight(shape=(k,), name='tp', initializer='zeros')
        self.fn = self.add_weight(shape=(k,), name='fn', initializer='zeros')

    def update_state(self, y_true, y_pred, sample_weight=None):
        tp_by_class = tf.reduce_sum(tf.cast(
            y_pred == tf.reduce_max(y_pred, axis=1, keepdims=True),
            dtype=y_true.dtype
        ) * y_true, axis=0)
        fn_by_class = tf.reduce_sum(tf.cast(
            y_pred < tf.reduce_max(y_pred, axis=1, keepdims=True),
            dtype=y_true.dtype
        ) * y_true, axis=0)
        self.tp.assign_add(tp_by_class)
        self.fn.assign_add(fn_by_class)

    def result(self):
        # Catch division by zero, in that case tpr becomes zero after clipping the divisor to 1.
        divisor = tf.clip_by_value(self.tp + self.fn, 1, np.inf, "divisor")
        tpr = tf.reduce_mean(self.tp / divisor)
        return tpr

    def reset_states(self):
        self.tp.assign(tf.zeros_like(self.tp))
        self.fn.assign(tf.zeros_like(self.fn))

class CustomFprClasswise(tf.keras.metrics.Metric):

    def __init__(self, k: int, name='fpr', **kwargs):
        super(CustomFprClasswise, self).__init__(name=name, **kwargs)
        self.fp = self.add_weight(shape=(k,), name='fp', initializer='zeros')
        self.tn = self.add_weight(shape=(k,), name='tn', initializer='zeros')

    def update_state(self, y_true, y_pred, sample_weight=None):
        fp_by_class = tf.reduce_sum(tf.cast(
            y_pred == tf.reduce_max(y_pred, axis=1, keepdims=True),
            dtype=y_true.dtype
        ) * (1. - y_true), axis=0)
        tn_by_class = tf.reduce_sum(tf.cast(
            y_pred < tf.reduce_max(y_pred, axis=1, keepdims=True),
            dtype=y_true.dtype
        ) * (1. - y_true), axis=0)
        self.fp.assign_add(fp_by_class)
        self.tn.assign_add(tn_by_class)

    def result(self):
        # Catch division by zero, in that case fpr becomes zero after clipping the divisor to 1.
        divisor = tf.clip_by_value(self.fp + self.tn, 1, np.inf, "divisor")
        fpr = tf.reduce_mean(self.fp / divisor)
        return fpr

    def reset_states(self):
        self.fp.assign(tf.zeros_like(self.fp))
        self.tn.assign(tf.zeros_like(self.tn))

class CustomF1Classwise(tf.keras.metrics.Metric):

    def __init__(self, k: int, name='f1', **kwargs):
        super(CustomF1Classwise, self).__init__(name=name, **kwargs)
        self.tp = self.add_weight(shape=(k,), name='tp', initializer='zeros')
        self.fp = self.add_weight(shape=(k,), name='fp', initializer='zeros')
        self.fn = self.add_weight(shape=(k,), name='fn', initializer='zeros')

    def update_state(self, y_true, y_pred, sample_weight=None):
        tp_by_class = tf.reduce_sum(tf.cast(
            y_pred == tf.reduce_max(y_pred, axis=1, keepdims=True),
            dtype=y_true.dtype
        ) * y_true, axis=0)
        fp_by_class = tf.reduce_sum(tf.cast(
            y_pred == tf.reduce_max(y_pred, axis=1, keepdims=True),
            dtype=y_true.dtype
        ) * (1. - y_true), axis=0)
        fn_by_class = tf.reduce_sum(tf.cast(
            y_pred < tf.reduce_max(y_pred, axis=1, keepdims=True),
            dtype=y_true.dtype
        ) * y_true, axis=0)
        self.tp.assign_add(tp_by_class)
        self.fp.assign_add(fp_by_class)
        self.fn.assign_add(fn_by_class)

    def result(self):
        # Catch divisions by zero, in that case precision or recall become zero after clipping the divisor to 1.
        divisor_precision = tf.clip_by_value(self.tp + self.fp, 1, np.inf, "divisor")
        divisor_recall = tf.clip_by_value(self.tp + self.fn, 1, np.inf, "divisor")
        precision = self.tp / divisor_precision
        recall = self.tp / divisor_recall
        precision = tf.clip_by_value(precision, 1e-100, np.inf, "divisor")
        recall = tf.clip_by_value(recall, 1e-100, np.inf, "divisor")
        f1 = tf.reduce_mean(2 * 1 / (1 / precision + 1 / recall))
        return f1

    def reset_states(self):
        self.tp.assign(tf.zeros_like(self.tp))
        self.fp.assign(tf.zeros_like(self.fp))
        self.fn.assign(tf.zeros_like(self.fn))

max_stars_repo_name: johnmous/sfaira

max_issues_repo_name: johnmous/sfaira

max_forks_repo_name: johnmous/sfaira


text:
from . import PythonModule
from .join_feature import JoinFeature

class JuPyMake(JoinFeature):
    r"""
    A :class:`sage.features.Feature` describing the presence of the ``JuPyMake``
    module, a Python interface to the polymake library.

    EXAMPLES::

        sage: from sage.features.polymake import JuPyMake
        sage: JuPyMake().is_present()  # optional: jupymake
        FeatureTestResult('jupymake', True)
    """
    def __init__(self):
        r"""
        TESTS::

            sage: from sage.features.polymake import JuPyMake
            sage: isinstance(JuPyMake(), JuPyMake)
            True
        """
        JoinFeature.__init__(self, "jupymake",
                             [PythonModule("JuPyMake", spkg="jupymake")])


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 25 08:58:42 2019

@author: iamav
"""

import numpy as np
from matplotlib import pyplot as plt
from PIL import Image
import os
import cv2
import math

path = "MPIIGaze//Data//Original//p00//day01"
def load_images_from_folder(folder):
    images = []
    for filename in os.listdir(folder):
        img = cv2.imread(os.path.join(folder,filename), cv2.IMREAD_GRAYSCALE)
        if img is not None:
            images.append(img)

    return images

images = load_images_from_folder(path)
#plt.imshow(images[0])
#plt.show

#img= images[0]

#px = img[300,600]
#print(np.mean(img))

mean_img=[]
for i in range(len(images)):
    mean_img.append(np.mean(images[i]))
print(math.ceil(min(mean_img)))

bins = np.linspace(math.ceil(min(mean_img)),
                   math.floor(max(mean_img)), 20)

plt.hist(mean_img, bins=bins)
plt.title("Mean image histogram for p00 day1")
plt.ylabel("no. of images")
plt.xlabel("mean value")
plt.show()


text:
from distutils.core import setup, Extension
from Cython.Build import cythonize
from Cython.Distutils import build_ext
import numpy

setup(
	name = 'render_core',
    cmdclass={'build_ext': build_ext},
    ext_modules=[Extension("render_core_cython",
                 sources=["render_core_cython.pyx", "render_core.cpp"],
                 language='c++',
                 include_dirs=[numpy.get_include()])],
)


text:
## python lasso.py train.txt test.txt 5 pred.txt coef.txt
import sys

import numpy as np
from sklearn.linear_model import LassoCV

train = np.genfromtxt(sys.argv[1], delimiter=",")
test = np.genfromtxt(sys.argv[2], delimiter=",")
model = LassoCV(
    cv=int(sys.argv[3]), fit_intercept=False).fit(train[:, 1:], train[:, 1])
Ypred = model.predict(test[:, 1:])
np.savetxt(sys.argv[4], Ypred)
np.savetxt(sys.argv[5], model.coef_)

max_stars_repo_name: BoPeng/sos

max_issues_repo_name: BoPeng/sos

max_forks_repo_name: BoPeng/sos


text:
import numpy as np

from hls4ml.model.optimizer import OptimizerPass

from hls4ml.model.hls_model import Layer, register_layer
from hls4ml.templates import templates

class Repack(Layer):
    ''' Inserted between layers with different packing factors.'''

    def initialize(self):
        shape = self.attributes['target_shape']
        if shape[0] is None:
            shape = shape[1:]
        dims = ['N_SIZE_{}_{}'.format(i, self.index) for i in range(1, len(shape) + 1)]

        self.add_output_variable(shape, dims)

    def function_cpp(self):
        params = self._default_function_params()
        params['size'] = np.prod(self.get_output_variable().shape)
        return [self._function_template.format(**params)]

    def config_cpp(self):
        return None

class Broadcast(Layer):
    ''' Inserted between layers for broadcasting.'''

    def initialize(self):
        shape = self.attributes['target_shape']
        if shape[0] is None:
            shape = shape[1:]
        dims = ['N_SIZE_{}_{}'.format(i, self.index) for i in range(1, len(shape) + 1)]
        self.add_output_variable(shape, dims)

    def function_cpp(self):
        params = self._default_function_params()
        return [self._function_template.format(**params)]

    def config_cpp(self):
        params = self._default_config_params()
        params['in_height'] = self.get_input_variable().shape[0]
        params['in_width'] = self.get_input_variable().shape[1]
        params['n_chan'] = self.get_input_variable().shape[2]
        params['n_dupl'] = int(np.prod(self.get_output_variable().shape)/np.prod(self.get_input_variable().shape))
        return self._config_template.format(**params)

repack_function_template = 'nnet::repack_stream<{input_t}, {output_t}, {size}>({input}, {output});'
repack_include_list = ['nnet_utils/nnet_stream.h']

broadcast_function_template = 'nnet::broadcast_stream<{input_t}, {output_t}, {config}>({input}, {output});'
broadcast_config_template = """struct config{index} : nnet::broadcast_config {{
static const unsigned in_width = {in_width};
static const unsigned in_height = {in_height};
static const unsigned n_chan = {n_chan};
static const unsigned n_dupl = {n_dupl};
}};\n"""
broadcast_include_list = ['nnet_utils/nnet_stream.h']

# Register the layer types to the layer map
register_layer('Repack', Repack)
register_layer('Broadcast', Broadcast)

# Register the templates for config and function
templates.get_backend('Vivado').register_templates('Repack', repack_function_template, None, repack_include_list)
templates.get_backend('Vivado').register_templates('Broadcast', broadcast_function_template, broadcast_config_template, broadcast_include_list)

class ReshapeStream(OptimizerPass):
    ''' Repacks stream for Reshape layer '''
    def match(self, node):
        return node.__class__.__name__ == 'Reshape'

    def transform(self, model, node):
        if model.config.backend.name != 'Vivado' or \
            model.config.get_config_value('IOType') != 'io_stream':
            return False

        attrs = {
            'target_shape': node.get_attr('target_shape')
        }

        # Insert new Repack node instead of Reshape
        repack_layer = model.make_node('Repack', 'repack_' + node.name, attrs, node.inputs.copy())
        model.replace_node(node, repack_layer)

        return True

class BroadcastStream(OptimizerPass):
    def match(self, node):
        if node.__class__.__name__ == 'Merge':
            inp1 = node.get_input_variable(node.inputs[0])
            inp2 = node.get_input_variable(node.inputs[1])
            return inp1.shape != inp2.shape
        else:
            return False

    def transform(self, model, node):
        if model.config.backend.name not in ['Vivado'] or \
            model.config.get_config_value('IOType') != 'io_stream':
            return False

        inp1 = node.get_input_variable(node.inputs[0])
        inp2 = node.get_input_variable(node.inputs[1])
        if np.prod(inp1.shape) > np.prod(inp2.shape):
            idx = 1
            attrs = {
                'target_shape': inp1.shape
            }
        else:
            idx = 0
            attrs = {
                'target_shape': inp2.shape
            }
        brdcst_layer = model.make_node('Broadcast', 'broadcast_' + node.inputs[idx], attrs, [node.inputs[idx]].copy())
        model.insert_node(brdcst_layer)
        node.inputs[idx] = 'broadcast_' + node.inputs[idx]

        return True


text:
import matplotlib.pyplot as plt
import numpy as np
from sklearn.preprocessing import scale, MinMaxScaler, normalize, robust_scale, maxabs_scale, minmax_scale
from sklearn.metrics import r2_score, mean_squared_error as mse, mean_absolute_error as mae

name = './records/results-2017-05-15 10:15:23'
Y = np.genfromtxt('{}.txt'.format(name), delimiter=',', usecols=(0))
predict = np.genfromtxt('{}.txt'.format(name), delimiter=',', usecols=(1))
# Y = scale(Y, with_std=True)
# predict = scale(predict, with_std=True)

# Y = minmax_scale(Y, feature_range=(-1, 1))
# predict = minmax_scale(predict, feature_range=(-1, 1))

Y = Y[0:100]
predict = predict[0:100]
Y_prev = 0
predict_prev = 0
point_to_griphindor = 0
point_to_slizerin = 0

for index, val in enumerate(Y):
    if Y_prev >= Y[index] and predict_prev >= predict[index]:
        point_to_griphindor+=1
    elif Y_prev <= Y[index] and predict_prev <= predict[index]:
        point_to_griphindor+=1
    else:
        point_to_slizerin+=1

    Y_prev = Y[index]
    predict_prev = predict[index]
print(point_to_slizerin, point_to_griphindor)

print('R2 SCORE:', r2_score(Y, predict))
print('MSE SCORE:', mse(Y, predict))
print('MAE SCORE:', mae(Y, predict))
# print('MAPE SCORE:', r2_score(Y, predict))

plt.plot(Y, c='blue')
plt.plot(predict, c="red")
plt.show()

max_stars_repo_name: Metnew/neural-finance

max_issues_repo_name: Deeptradingfx/neural-finance

max_forks_repo_name: Deeptradingfx/neural-finance


text:
# Dotenv
from dotenv import load_dotenv
# Environment Variables
from pathlib import Path
env_path = Path('.') / '.env'
load_dotenv(dotenv_path=env_path)

# Dependencies
import pandas as pd
import numpy as np
import datetime
import pytz
import os
import tweepy
import requests
import time
import aqi_tweet
import random

# Locations Dataset
locations_data = pd.read_csv(os.path.join('../input', 'locations.csv'))

# Air Quality Dataset
air_quality = pd.read_csv(os.path.join('../output', 'air_quality.csv'))

# Local Datetime
#LOCAL_UTC = datetime.datetime.utcnow().isoformat()
#print(LOCAL_UTC)
#tz_co = pytz.timezone('America/Bogota')
#local_dt = datetime.datetime.utcnow() - datetime.timedelta(hours=5)
#LOCAL_DATETIME = local_dt.astimezone(tz_co).strftime('%Y-%m-%dT%H:%M:%S')
LOCAL_DATETIME = datetime.datetime.utcnow().astimezone(pytz.timezone('America/Bogota')).isoformat()
print(LOCAL_DATETIME)

# Cities
CITIES = list(locations_data['label'].values)
CITY = random.choice(CITIES)
#print(CITIES)
#print(CITY)
print('Tweet City:', CITY)

# BreezoMeter API Request
def breezometer_api_request(LAT, LON):
    URL_BREEZOMETER = "https://api.breezometer.com/air-quality/v2/current-conditions?"
    API_KEY = os.getenv('BREEZOMETER_API_KEY')
    URL_BREEZOMETER += "lat={lat}&lon={lon}&key={api_key}".format(lat=LAT, lon=LON, api_key=API_KEY)
    # Request
    with requests.get(URL_BREEZOMETER) as bm_aq:
        bm_aq = bm_aq.json()
        return bm_aq

def get_text_tweet(aqi_data, lang='eng'):
    # aqi_data = [ID, LABEL, LAT, LON, BM_UTC, AQI, CATEGORY, COLOR]
    ctg_emoji = {
        "Excellent air quality": "🤩",
        "Good air quality": "🙂",
        "Moderate air quality": "😷",
        "Low air quality": "🥵",
        "Poor air quality": "☠"
    }
    ctg_esp = {
        "Excellent air quality": "Excelente calidad de aire",
        "Good air quality": "Buena calidad de aire",
        "Moderate air quality": "Moderada calidad de aire",
        "Low air quality": "Baja calidad de aire",
        "Poor air quality": "Pobre calidad de aire"
    }
    # Text Tweet
    text_tweet = ".:-_-:. "
    # Spanish
    if lang == 'esp':
        text_tweet += """Condiciones de Calidad del Aire: #{hashlabel} #Colombia
{label}: {category}
Indice de Calidad del Aire: {aqi} -> {emoji}
#Calidad #Aire
Hora Local: {local_datetime}
https://sebaxtian.github.io/breezometer-api-demo/
""".format(hashlabel=aqi_data[1].replace(" ", ""), label=aqi_data[1], category=ctg_esp[aqi_data[6]], aqi=aqi_data[5], emoji=ctg_emoji[aqi_data[6]], local_datetime=LOCAL_DATETIME)
    else:
        # English
        text_tweet += """Air Quality Conditions: #{hashlabel} #Colombia
{label}: {category}
Air Quality Index: {aqi} -> {emoji}
#Air #Quality
Local Datetime: {local_datetime}
https://sebaxtian.github.io/breezometer-api-demo/
""".format(hashlabel=aqi_data[1].replace(" ", ""), label=aqi_data[1], category=aqi_data[6], aqi=aqi_data[5], emoji=ctg_emoji[aqi_data[6]], local_datetime=LOCAL_DATETIME)
    # Return Text Tweet
    return text_tweet

# Air Quality Dataframe
def get_air_quality_dataframe():
    #print(locations_data.shape[0])
    data_air_quality = []
    for index in range(locations_data.shape[0]):
        #print(index)
        # GET Attributes
        LAT = locations_data.iloc[[index]]['lat'].values[0]
        LON = locations_data.iloc[[index]]['lon'].values[0]
        # Request
        bm_aq = breezometer_api_request(LAT, LON)
        #print(bm_aq)
        # Location Data
        ID = locations_data.iloc[[index]]['id'].values[0]
        LABEL = locations_data.iloc[[index]]['label'].values[0]
        # BreezoMeter Data
        BM_UTC = bm_aq['data']['datetime']
        AQI = bm_aq['data']['indexes']['baqi']['aqi']
        CATEGORY = bm_aq['data']['indexes']['baqi']['category']
        COLOR = bm_aq['data']['indexes']['baqi']['color']
        # Log
        print(LABEL, '->', CATEGORY)
        # There is?
        there_is = air_quality[(air_quality['id'] == ID) & (air_quality['bm_utc'] == BM_UTC)]
        #print(len(there_is.values))
        if len(there_is.values) == 0:
            # Air Quality Data
            aqi_data = [ID, LABEL, LAT, LON, BM_UTC, AQI, CATEGORY, COLOR]
            # Append Air Quality Data
            data_air_quality.append(aqi_data)
            # Filter Random City
            #print(LABEL, CITY)
            if LABEL == CITY:
                # Get Text Tweet ENG
                text_tweet_eng = get_text_tweet(aqi_data)
                #print(text_tweet_eng)
                # Get Text Tweet ESP
                text_tweet_esp = get_text_tweet(aqi_data, lang='esp')
                #print(text_tweet_esp)
                # Post Text Tweet ENG
                aqi_tweet.post_tweet(text_tweet_eng)
                # Wait
                time.sleep(2)
                # Post Text Tweet ESP
                aqi_tweet.post_tweet(text_tweet_esp)
        # Wait
        time.sleep(2)
    #print(data_air_quality)
    new_air_quality = pd.DataFrame(columns=['id', 'label', 'lat', 'lon', 'bm_utc', 'aqi', 'category', 'color'], data=data_air_quality)
    # Round Lat Lon values and Return
    return air_quality.append(new_air_quality, ignore_index=True).round({'lat': 6, 'lon': 6})

# Get Air Quality Dataframe
air_quality = get_air_quality_dataframe()

# Save dataframe
air_quality.to_csv(os.path.join('../output', 'air_quality.csv'), index=False)

max_stars_repo_name: sebaxtian/breezometer-api-demo

max_issues_repo_name: sebaxtian/breezometer-api-demo

max_forks_repo_name: sebaxtian/breezometer-api-demo


text:
#%%
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
from math import ceil,log2
import pickle

sys.path.append("../src/")
from lifelong_dnn import LifeLongDNN

#%%
def unpickle(file):
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

def get_colors(colors, inds):
    c = [colors[i] for i in inds]
    return c

def generate_2d_rotation(theta=0, acorn=None):
    if acorn is not None:
        np.random.seed(acorn)

    R = np.array([
        [np.cos(theta), np.sin(theta)],
        [-np.sin(theta), np.cos(theta)]
    ])

    return R

def generate_gaussian_parity(n, mean=np.array([-1, -1]), cov_scale=1, angle_params=None, k=1, acorn=None):
    if acorn is not None:
        np.random.seed(acorn)

    d = len(mean)

    if mean[0] == -1 and mean[1] == -1:
        mean = mean + 1 / 2**k

    mnt = np.random.multinomial(n, 1/(4**k) * np.ones(4**k))
    cumsum = np.cumsum(mnt)
    cumsum = np.concatenate(([0], cumsum))

    Y = np.zeros(n)
    X = np.zeros((n, d))

    for i in range(2**k):
        for j in range(2**k):
            temp = np.random.multivariate_normal(mean, cov_scale * np.eye(d),
                                                 size=mnt[i*(2**k) + j])
            temp[:, 0] += i*(1/2**(k-1))
            temp[:, 1] += j*(1/2**(k-1))

            X[cumsum[i*(2**k) + j]:cumsum[i*(2**k) + j + 1]] = temp

            if i % 2 == j % 2:
                Y[cumsum[i*(2**k) + j]:cumsum[i*(2**k) + j + 1]] = 0
            else:
                Y[cumsum[i*(2**k) + j]:cumsum[i*(2**k) + j + 1]] = 1

    if d == 2:
        if angle_params is None:
            angle_params = np.random.uniform(0, 2*np.pi)

        R = generate_2d_rotation(angle_params)
        X = X @ R

    else:
        raise ValueError('d=%i not implemented!'%(d))

    return X, Y.astype(int)

# %% main hyperparameters
delta = 0.05
delta_ = 0.01
#########################

x = np.arange(-1, 1+delta, step=delta)
y = np.arange(-1, 1+delta, step=delta)
X, Y = np.meshgrid(x,y)
X = np.ravel(X).reshape(-1,1)
Y = np.ravel(Y).reshape(-1,1)
noise = np.concatenate((X,Y), axis=1)

x = np.arange(-1, 1+delta_, step=delta_)
y = np.arange(-1, 1+delta_, step=delta_)
X_, Y_ = np.meshgrid(x,y)
X_ = np.ravel(X_).reshape(-1,1)
Y_ = np.ravel(Y_).reshape(-1,1)
noise_ = np.concatenate((X_,Y_), axis=1)

# %%
xor, label_xor = generate_gaussian_parity(750,cov_scale=0.1,angle_params=0)
normalize_xor = np.max(np.abs(xor))
xor = xor/normalize_xor

l2f = LifeLongDNN()
l2f.new_forest(xor, label_xor, n_estimators=10,max_depth=ceil(log2(750)))

label_noise = l2f.predict(noise, representation=0,decider=0).astype(int)
label_noise_ = l2f.predict(noise_, representation=0,decider=0).astype(int)

#%%
sns.set_context("talk")

colors = sns.color_palette('Dark2', n_colors=2)
fig, ax = plt.subplots(ncols=2, nrows=2, sharex=True, sharey=True, figsize=(12,12))
ax[0][0].scatter(X,Y,s=1)
ax[0][0].set_xticks([-1, -.5, 0, .5, 1])
ax[0][0].set_yticks([-1, -.5, 0, .5, 1])

ax[0][1].scatter(X, Y, c=get_colors(colors, label_noise), s=1)
ax[0][1].set_xticks([-1, -.5, 0, .5, 1])
ax[0][1].set_yticks([-1, -.5, 0, .5, 1])

ax[1][0].scatter(X_,Y_,s=1)
ax[1][0].set_xticks([-1, -.5, 0, .5, 1])
ax[1][0].set_yticks([-1, -.5, 0, .5, 1])

ax[1][1].scatter(X_, Y_, c=get_colors(colors, label_noise_), s=1)
ax[1][1].set_xticks([-1, -.5, 0, .5, 1])
ax[1][1].set_yticks([-1, -.5, 0, .5, 1])

plt.savefig('result/figs/partition.pdf')
# %%


text:
#!/usr/bin/env python3

import matplotlib.pyplot as plt
import numpy as np

def main():
    data_hit = []
    data_miss = []
    with open("data/zbuff-cache-miss.dat", "r") as f:
        for line in f:
            if line.isspace():
                continue
            data_miss.append(int(line.split()[0]))

    with open("data/zbuff-cache-hits.dat", "r") as f:
        for line in f:
            if line.isspace():
                continue
            data_hit.append(int(line.split()[0]))

    fig = plt.figure()
    cdf = fig.add_subplot(111)

    sorted_data_miss = np.sort(data_miss)
    yvals=np.arange(len(sorted_data_miss))/float(len(sorted_data_miss)-1)
    cdf.plot(sorted_data_miss,yvals, 'b-', label="Cache Miss")

    sorted_data_hit = np.sort(data_hit)
    yvals=np.arange(len(sorted_data_hit))/float(len(sorted_data_hit)-1)
    cdf.plot(sorted_data_hit,yvals, 'r-', label="Cache Hit")

    cdf.set_ylim(0,1)
    cdf.set_xlim([1,2500])
    cdf.set_xscale('log')

    cdf.set_xlabel('Request Latency (cycles)')
    cdf.set_ylabel('CDF')

    fig.legend(bbox_to_anchor=(.4, .7),
           bbox_transform=plt.gcf().transFigure)

    plt.show()

if __name__=="__main__":
    main()

max_stars_repo_name: ewust/speculake

max_issues_repo_name: ewust/speculake

max_forks_repo_name: ewust/speculake


text:
# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np

from collecte.sauvamer.converters import CoordinatesConverter, SitRepConverter

COLS_NAME = [
    'numero_etablissement', 'numero_rapport',
    'mer', 'zone_intervention', 'date_saisie',
    'compte_rendu_patron', 'commentaires_patron',
    'nom_etablissement', 'zone_littorale',
    'code_postal', 'type_rapport',
    'date_sortie', 'date_alerte',
    'motif_alerte', 'fausse_alerte',
    'nb_personnes_tirees_seules',
    'nb_personnes_secourues',
    'nb_personnes_disparues',
    'nb_personnes_decedees',
    'nom_navire',
    'type_embarcation',
    'resultat_embarcation',
    'moyen_principal',
    'position_latitude',
    'position_longitude',
    'delai_appareillage_minutes',
    'duree_reelle_minutes',
    'details_sitrep'
]

def zone_intervention_converter(src):
    mapping = {
        'Eaux Territoriales (300 m à 12 nq)': 'eaux-territoriales',
        'Plage et 300m': 'plage-300m',
        'Eaux Françaises (sup. à 12 nq)': 'eaux-francaises',
        'Eaux Etrangères': 'eaux-etrangeres',
    }
    return mapping.get(src.encode('utf-8'), np.nan)

def douglas_converter(src):
    mapping = {
        'calme': 0,
        'plate': 0,
        'ridée': 1,
        'belle': 2,
        'peu agitée': 3,
        'agitée': 4,
        'forte': 5,
        'très forte': 6,
        'grosse': 7,
        'très grosse': 8,
        'énorme': 9,
    }
    for start, code in mapping.iteritems():
        if src.encode('utf-8').lower().startswith(start):
            return code
    return np.nan

df = pd.read_excel(
    '/Users/antoineaugusti/Desktop/sauvamer.xlsx',
    names=COLS_NAME,
    converters={
        'Mer (Douglas)': douglas_converter,
        "Zone d'intervention": zone_intervention_converter,
        "Zone littorale": lambda e: e.lower().replace(' ', '_'),
        "Fausse alerte": lambda e: e == 'Oui',
        "CP": lambda e: int(e),
        "Position latitude": CoordinatesConverter.latitude_dms_to_dd,
        "Position longitude": CoordinatesConverter.longitude_dms_to_dd,
    },
)

df['numero_sitrep'] = df.details_sitrep.apply(SitRepConverter.extract_sitrep_name)
df['cross'] = df.details_sitrep.apply(SitRepConverter.extract_cross)
df['type_operation'] = df.details_sitrep.apply(SitRepConverter.extract_operation_type)

if not (df.type_rapport == 'Sauvetage').all():
    raise ValueError('Tous les rapports doivent être des sauvetages')

print df['mer'].value_counts(dropna=False)
print df['zone_intervention'].value_counts(dropna=False)
print df['zone_littorale'].value_counts(dropna=False)
print df['fausse_alerte'].value_counts(dropna=False)
print df['motif_alerte'].value_counts(dropna=False)
print df['position_latitude'].value_counts(dropna=False)
print df['position_longitude'].value_counts(dropna=False)
print df['cross'].value_counts(dropna=False)
print df['type_operation'].value_counts(dropna=False)

# print df.head(10)


text:
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
import math
import random

def distance(a,b):
	return math.pow( (math.pow(a[0]-b[0],2) + math.pow(a[1]-b[1],2) + math.pow(a[2]-b[2],2)) ,0.5)

def assign_labels(img, centroid, index):
	x, y, temp = img.shape
	for i in xrange(x):
		for j in xrange(y):
			if np.array_equal(img[i,j], np.array([0,0,0])):
				index[i,j] = -1
				continue
			dist = 1000000000
			for k, l in enumerate(centroid):
				temp = distance(img[i,j], l)
				if temp < dist:
					index[i,j] = k
					dist = temp
	return index

def new_centroid(img, index, c):
	x, y, temp = img.shape
	centroid = [ [0 for i in xrange(3)] for j in xrange(c)]
	centroid = np.array(centroid)
	centroid_total = [ 0 for i in xrange(c)]
	for i in xrange(x):
		for j in xrange(y):
			if np.array_equal(img[i,j], np.array([0,0,0])):
				continue
			temp = int(index[i,j])
			centroid[temp,0] += img[i,j,0]
			centroid[temp,1] +=	img[i,j,1]
			centroid[temp,2] += img[i,j,2]
			centroid_total[temp] += 1
	for i in xrange(c):
		if centroid_total[i] == 0:
			centroid[i] = 0
		else:
			centroid[i] = centroid[i]*1.0 / centroid_total[i]
	return centroid

def create_index(filename, number_of_centroids):
	#c = int(raw_input('Enter the number of centroids: '))
	c = number_of_centroids
	img = Image.open(filename)
	img = np.array(img)
	x,y, temp = img.shape
	print 'RBG_Data: Creating index matrix | a variation of dimensional reduction'
	np.random.seed()
	#centroid =  np.array([ np.random.randint(0,256,3), np.random.randint(0,256,3)])
	centroid = [[ random.randint(0,256) for i in xrange(3)] for j in xrange(c)]
	centroid = np.array(centroid)

	index = np.zeros((x,y)) - 1

	#number of iterations in K-Means
	for i in xrange(3):
		index = assign_labels(img[:,:,:3],centroid, index)
		centroid = new_centroid(img[:,:,:3], index, centroid.shape[0])
	index = index + 1

	return img, index

def display_index(index):
	x, y = index.shape
	# plt_x = np.empty((x*y,2))
	# plt_y = np.empty((x*y))
	plt_x =[]
	plt_y = []
	index_max = index.max()
	for i in xrange(x):
		for j in xrange(y):
			# if index[i,j]==0:
			# 	continue

			plt_x.append([i,j])
			plt_y.append(index[i,j]*1.5 / index_max)

	plt_x = np.array(plt_x)
	plt_y = np.array(plt_y)

	#change cmap color to change the range of colors that appear in output
	plt.scatter(plt_x[:,0],plt_x[:,1], c=plt_y, cmap='gray')
	plt.show()