text:
from __future__ import division, print_function, absolute_import

import numpy as np
from numpy.testing import (assert_equal, assert_array_equal, assert_allclose,
        run_module_suite, assert_raises)

from scipy.interpolate import griddata, NearestNDInterpolator

class TestGriddata(object):
    def test_fill_value(self):
        x = [(0,0), (0,1), (1,0)]
        y = [1, 2, 3]

        yi = griddata(x, y, [(1,1), (1,2), (0,0)], fill_value=-1)
        assert_array_equal(yi, [-1., -1, 1])

        yi = griddata(x, y, [(1,1), (1,2), (0,0)])
        assert_array_equal(yi, [np.nan, np.nan, 1])

    def test_alternative_call(self):
        x = np.array([(0,0), (-0.5,-0.5), (-0.5,0.5), (0.5, 0.5), (0.25, 0.3)],
                     dtype=np.double)
        y = (np.arange(x.shape[0], dtype=np.double)[:,None]
             + np.array([0,1])[None,:])

        for method in ('nearest', 'linear', 'cubic'):
            for rescale in (True, False):
                msg = repr((method, rescale))
                yi = griddata((x[:,0], x[:,1]), y, (x[:,0], x[:,1]), method=method,
                              rescale=rescale)
                assert_allclose(y, yi, atol=1e-14, err_msg=msg)

    def test_multivalue_2d(self):
        x = np.array([(0,0), (-0.5,-0.5), (-0.5,0.5), (0.5, 0.5), (0.25, 0.3)],
                     dtype=np.double)
        y = (np.arange(x.shape[0], dtype=np.double)[:,None]
             + np.array([0,1])[None,:])

        for method in ('nearest', 'linear', 'cubic'):
            for rescale in (True, False):
                msg = repr((method, rescale))
                yi = griddata(x, y, x, method=method, rescale=rescale)
                assert_allclose(y, yi, atol=1e-14, err_msg=msg)

    def test_multipoint_2d(self):
        x = np.array([(0,0), (-0.5,-0.5), (-0.5,0.5), (0.5, 0.5), (0.25, 0.3)],
                     dtype=np.double)
        y = np.arange(x.shape[0], dtype=np.double)

        xi = x[:,None,:] + np.array([0,0,0])[None,:,None]

        for method in ('nearest', 'linear', 'cubic'):
            for rescale in (True, False):
                msg = repr((method, rescale))
                yi = griddata(x, y, xi, method=method, rescale=rescale)

                assert_equal(yi.shape, (5, 3), err_msg=msg)
                assert_allclose(yi, np.tile(y[:,None], (1, 3)),
                                atol=1e-14, err_msg=msg)

    def test_complex_2d(self):
        x = np.array([(0,0), (-0.5,-0.5), (-0.5,0.5), (0.5, 0.5), (0.25, 0.3)],
                     dtype=np.double)
        y = np.arange(x.shape[0], dtype=np.double)
        y = y - 2j*y[::-1]

        xi = x[:,None,:] + np.array([0,0,0])[None,:,None]

        for method in ('nearest', 'linear', 'cubic'):
            for rescale in (True, False):
                msg = repr((method, rescale))
                yi = griddata(x, y, xi, method=method, rescale=rescale)

                assert_equal(yi.shape, (5, 3), err_msg=msg)
                assert_allclose(yi, np.tile(y[:,None], (1, 3)),
                                atol=1e-14, err_msg=msg)

    def test_1d(self):
        x = np.array([1, 2.5, 3, 4.5, 5, 6])
        y = np.array([1, 2, 0, 3.9, 2, 1])

        for method in ('nearest', 'linear', 'cubic'):
            assert_allclose(griddata(x, y, x, method=method), y,
                            err_msg=method, atol=1e-14)
            assert_allclose(griddata(x.reshape(6, 1), y, x, method=method), y,
                            err_msg=method, atol=1e-14)
            assert_allclose(griddata((x,), y, (x,), method=method), y,
                            err_msg=method, atol=1e-14)

    def test_1d_borders(self):
        # Test for nearest neighbor case with xi outside
        # the range of the values.
        x = np.array([1, 2.5, 3, 4.5, 5, 6])
        y = np.array([1, 2, 0, 3.9, 2, 1])
        xi = np.array([0.9, 6.5])
        yi_should = np.array([1.0, 1.0])

        method = 'nearest'
        assert_allclose(griddata(x, y, xi,
                                 method=method), yi_should,
                        err_msg=method,
                        atol=1e-14)
        assert_allclose(griddata(x.reshape(6, 1), y, xi,
                                 method=method), yi_should,
                        err_msg=method,
                        atol=1e-14)
        assert_allclose(griddata((x, ), y, (xi, ),
                                 method=method), yi_should,
                        err_msg=method,
                        atol=1e-14)

    def test_1d_unsorted(self):
        x = np.array([2.5, 1, 4.5, 5, 6, 3])
        y = np.array([1, 2, 0, 3.9, 2, 1])

        for method in ('nearest', 'linear', 'cubic'):
            assert_allclose(griddata(x, y, x, method=method), y,
                            err_msg=method, atol=1e-10)
            assert_allclose(griddata(x.reshape(6, 1), y, x, method=method), y,
                            err_msg=method, atol=1e-10)
            assert_allclose(griddata((x,), y, (x,), method=method), y,
                            err_msg=method, atol=1e-10)

    def test_square_rescale_manual(self):
        points = np.array([(0,0), (0,100), (10,100), (10,0), (1, 5)], dtype=np.double)
        points_rescaled = np.array([(0,0), (0,1), (1,1), (1,0), (0.1, 0.05)], dtype=np.double)
        values = np.array([1., 2., -3., 5., 9.], dtype=np.double)

        xx, yy = np.broadcast_arrays(np.linspace(0, 10, 14)[:,None],
                                     np.linspace(0, 100, 14)[None,:])
        xx = xx.ravel()
        yy = yy.ravel()
        xi = np.array([xx, yy]).T.copy()

        for method in ('nearest', 'linear', 'cubic'):
            msg = method
            zi = griddata(points_rescaled, values, xi/np.array([10, 100.]),
                          method=method)
            zi_rescaled = griddata(points, values, xi, method=method,
                                   rescale=True)
            assert_allclose(zi, zi_rescaled, err_msg=msg,
                            atol=1e-12)

    def test_xi_1d(self):
        # Check that 1-D xi is interpreted as a coordinate
        x = np.array([(0,0), (-0.5,-0.5), (-0.5,0.5), (0.5, 0.5), (0.25, 0.3)],
                     dtype=np.double)
        y = np.arange(x.shape[0], dtype=np.double)
        y = y - 2j*y[::-1]

        xi = np.array([0.5, 0.5])

        for method in ('nearest', 'linear', 'cubic'):
            p1 = griddata(x, y, xi, method=method)
            p2 = griddata(x, y, xi[None,:], method=method)
            assert_allclose(p1, p2, err_msg=method)

            xi1 = np.array([0.5])
            xi3 = np.array([0.5, 0.5, 0.5])
            assert_raises(ValueError, griddata, x, y, xi1,
                          method=method)
            assert_raises(ValueError, griddata, x, y, xi3,
                          method=method)

def test_nearest_options():
    # smoke test that NearestNDInterpolator accept cKDTree options
    npts, nd = 4, 3
    x = np.arange(npts*nd).reshape((npts, nd))
    y = np.arange(npts)
    nndi = NearestNDInterpolator(x, y)

    opts = {'balanced_tree': False, 'compact_nodes': False}
    nndi_o = NearestNDInterpolator(x, y, tree_options=opts)
    assert_allclose(nndi(x), nndi_o(x), atol=1e-14)

if __name__ == "__main__":
    run_module_suite()

max_stars_repo_name: lefevre-fraser/openmeta-mms

max_issues_repo_name: lefevre-fraser/openmeta-mms

max_forks_repo_name: lefevre-fraser/openmeta-mms

reason: import numpy,from numpy,from scipy


text:
#!/usr/bin/python
# -*- coding: UTF-8 -*-

"""
Please note, this code is only for python 3+. If you are using python 2+, please modify the code accordingly.
"""
from __future__ import print_function
import pandas as pd
import numpy as np

# concatenating
# ignore index
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2, columns=['a','b','c','d'])
res = pd.concat([df1, df2, df3], axis=0, ignore_index=True)
print("(1) test")
print(res)

ext: py

reason: import numpy


text:
"""
Plot full psychometric functions as a function of choice history,
and separately for 20/80 and 80/20 blocks
"""

import pandas as pd
import numpy as np
import time, os
import matplotlib.pyplot as plt
import seaborn as sns
import datajoint as dj
from IPython import embed as shell # for debugging

## INITIALIZE A FEW THINGS
sns.set(style="darkgrid", context="paper", font='Arial')
sns.set(style="darkgrid", context="paper")
sns.set(style="darkgrid", context="paper", font_scale=1.3)

# import wrappers etc
from ibl_pipeline import reference, subject, action, acquisition, data, behavior
from ibl_pipeline.utils import psychofit as psy
from ibl_pipeline.analyses import behavior as behavioral_analyses
from dj_tools import *
new_criteria = dj.create_virtual_module('analyses', 'user_anneurai_analyses')

figpath = os.path.join(os.path.expanduser('~'), 'Data/Figures_IBL')

# ================================= #
# 1. get training status from original DJ table
# ================================= #

use_subjects = subject.Subject * subject.SubjectProject & 'subject_project = "ibl_neuropixel_brainwide_01"'
use_sessions = use_subjects * subject.SubjectLab * subject.Subject.aggr(behavior.TrialSet,
                                                          session_start_time='max(session_start_time)')

# QUICK PIE PLOT
plt.close('all')
sns.set_palette("cubehelix")
fig, ax = plt.subplots(1, 2, figsize=(13, 13))

# ================================= #
# v0
# ================================= #

sess = behavioral_analyses.SessionTrainingStatus() * use_sessions
df1 = pd.DataFrame(sess.fetch(as_dict=True))

df2 = df1.groupby(['training_status'])['subject_uuid'].count().reset_index()
df2.index = df2.training_status
df2 = df2.reindex(['-', 'over40days', 'training in progress', 'trained', '.', 'ready for ephys', '..'])
df2.fillna(0, inplace=True)
original = df2.copy()
print(df2)

ax[0].pie(df2['subject_uuid'], autopct='%1.2f%%', labels=df2.index)
ax[0].set_title('Original criteria (v0), n = %d'%df2['subject_uuid'].sum())

# ================================= #
# v1
# ================================= #

# sns.set_palette("Set2")

sess = new_criteria.SessionTrainingStatus() * use_sessions
df3 = pd.DataFrame(sess.fetch(as_dict=True))

df4 = df3.groupby(['training_status'])['subject_uuid'].count().reset_index()
df4.index = df4.training_status
df4 = df4.reindex(['unbiasable', 'untrainable', 'in_training', 'trained_1a', 'trained_1b',
                   'ready4ephysrig', 'ready4recording'])
df4.fillna(0, inplace=True)
print(df4)
new = df4.copy()

ax[1].pie(df4['subject_uuid'], autopct='%1.2f%%', labels=df4.index)
ax[1].set_title('Alternative criteria (v1), n = %d'%df4['subject_uuid'].sum())

fig.savefig(os.path.join(figpath, "training_success.pdf"))
fig.savefig(os.path.join(figpath, "training_success.png"), dpi=300)
plt.close('all')

# ================================= #
# COMPARE PER LAB
# ================================= #

# WRITE A SUMMARY DOCUMENT
df5 = pd.merge(df1, df3, on='subject_nickname')
df5 = df5[['subject_nickname', 'lab_name_x', 'training_status_x', 'training_status_y']]
df5 = df5.sort_values(by=['lab_name_x', 'subject_nickname'])
df5.to_csv(os.path.join(figpath, "criteria_comparison.csv"))

# ================================= #
# COMPARE PER LAB
# ================================= #

sys.path.insert(0, '/Users/urai/Documents/code/postdoc-analyses/')
import aging_tables

progression_old = pd.DataFrame((behavioral_analyses.SessionTrainingStatus() \
 * (subject.SubjectProject & 'subject_project = "ibl_neuropixel_brainwide_01"') \
 * subject.Subject * subject.SubjectLab * aging_tables.TrainingDay).fetch('subject_nickname', 'lab_name',
                                                                          'training_day', 'training_status',
                                                                          as_dict=True))
progression_old['stage'] = progression_old['training_status'].replace({'training in progress': 1,
                                                                       'trained': 2,
                                                                       'ready for ephys': 3,
                                                                       'over40days': 0,
                                                                       'wrong session type run': np.nan})
# plot these
fig = sns.FacetGrid(progression_old, col="lab_name", col_wrap=3, hue="subject_nickname")
fig.map(sns.lineplot, "training_day", "stage", estimator=None, units=progression_old.subject_nickname)
fig.savefig(os.path.join(figpath, "training_progression.pdf"))

progression_new = pd.DataFrame((new_criteria.SessionTrainingStatus() \
 * (subject.SubjectProject & 'subject_project = "ibl_neuropixel_brainwide_01"') \
 * subject.Subject * subject.SubjectLab * aging_tables.TrainingDay).fetch('subject_nickname', 'lab_name',
                                                                          'training_day', 'training_status',
                                                                          as_dict=True))
progression_new['stage'] = progression_new['training_status'].replace({'in_training': 1,
                                                                       'trained_1a': 2,
                                                                       'trained_1b': 3,
                                                                       'ready4ephysrig': 4,
                                                                       'ready4recording': 5,
                                                                       'untrainable': 0,
                                                                       'unbiasable': 0})
# plot these
fig = sns.FacetGrid(progression_new, col="lab_name", col_wrap=3, hue="subject_nickname")
fig.map(sns.lineplot, "training_day", "stage", estimator=None, units=progression_new.subject_nickname)
fig.savefig(os.path.join(figpath, "training_progression_new.pdf"))

# ================================================================== #
# FOR EACH ANIMAL, COMPUTE TIME TO REACH EACH TRAINING CRITERION
# ================================================================== #

training_time_old = progression_old.groupby(['lab_name', 'subject_nickname', 'training_status']).min().reset_index()
training_time_new = progression_new.groupby(['lab_name', 'subject_nickname', 'training_status']).min().reset_index()

plt.close('all')
sns.set_palette("cubehelix")
fig, ax = plt.subplots(1, 2, figsize=(13,8), sharey=True)

sns.swarmplot(x='stage', y='training_day', hue="lab_name", data=training_time_old, ax=ax[0])
sns.boxplot(x='stage', y='training_day', data=training_time_old, ax=ax[0], boxprops={'facecolor':'None'})
ax[0].set_title("Original criteria")
ax[0].set_xlim([0.5, 5.5])
ax[0].set_xticks([1,2,3])
ax[0].set_xticklabels(['in training', 'trained', 'ready4ephys'],
                  size='small', rotation=45)

sns.swarmplot(x='stage', y='training_day', hue="lab_name", data=training_time_new, ax=ax[1])
sns.boxplot(x='stage', y='training_day', data=training_time_new, ax=ax[1], boxprops={'facecolor':'None'})
ax[1].set_title("New criteria")
ax[1].set_xlim([0.5, 5.5])
ax[1].set_xticks([1,2,3,4,5])
ax[1].set_xticklabels(['in_training', 'trained_1a', 'trained_1b', 'ready4ephysrig', 'ready4recording'],
                  size='small', rotation=45)
ax[1].get_legend().remove()

fig.savefig(os.path.join(figpath, "training_times.pdf"))
fig.savefig(os.path.join(figpath, "training_times.png"), dpi=300)
plt.close('all')

max_stars_repo_name: ineslaranjeira/analysis

max_issues_repo_name: ineslaranjeira/analysis

max_forks_repo_name: ineslaranjeira/analysis


text:
#!/usr/bin/env python

import numpy as np
from kmodes.kprototypes_TZ import KPrototypes
import datetime

# fileNames = ['广东.csv','河北.csv','浙江.csv','陕西.csv','贵州.csv','辽宁.csv','湖北.csv','江苏.csv','北京.csv','上海.csv']
fileNames = ['kprototypes_nor_5000.csv']
resSyms = np.genfromtxt('kprototypes_nor_5000_res.csv',dtype=int,delimiter=',')[:,0]
resClas = np.genfromtxt('kprototypes_nor_5000_res.csv',dtype=str,delimiter=',')[:,1]
resMatrix = {}
for i in range(len(resSyms)):
    if resClas[i] != '':
        if resClas[i] == 'normal':
            resMatrix[resSyms[i]] = 0
        else:
            resMatrix[resSyms[i]] = 1

startTime = datetime.datetime.now()
preMatrix = {}
zeroIds = []
for idx in range(len(fileNames)):
    fileName = fileNames[idx]
    syms = np.genfromtxt(fileName, dtype=str, delimiter=',')[:, 0]
    X = np.genfromtxt(fileName, dtype=object, delimiter=',')[:, 1:]
    X[:, 0] = X[:, 0].astype(float)

    columnNum = len(X[0])
    kproto = KPrototypes(n_clusters=5, init='Huang', verbose=0,n_init=1)
    clusters = kproto.fit_predict(X, categorical=[columnNum-3, columnNum-2, columnNum-1])

    # Print cluster centroids of the trained model.
    print(kproto.cluster_centroids_)
    # Print training statistics
    print(kproto.cost_)
    print(kproto.n_iter_)

    for s, c in zip(syms, clusters):
        s = int(float(s))
        preMatrix[s] = c
        # if c == 0:
        #     zeroIds.append(s)
endTime = datetime.datetime.now()
print("聚类用时："+str(endTime-startTime))

clustNum = [0,0,0,0,0]
for i in preMatrix.keys():
    clustNum[preMatrix[i]] += 1
normalClust = np.argmax(clustNum)

for i in preMatrix.keys():
    if preMatrix[i] == normalClust:
        preMatrix[i] = 0
    else:
        preMatrix[i] = 1

tp = 0
fp = 0
fn = 0
tn = 0
for i in preMatrix.keys():
    if preMatrix[i] == 1 and resMatrix[i] == 1: tp += 1
    elif preMatrix[i] == 1 and resMatrix[i] == 0: fp += 1
    elif preMatrix[i] == 0 and resMatrix[i] == 1: fn += 1
    else: tn += 1

print('TP: '+str(tp))
print('FP: '+str(fp))
print('FN: '+str(fn))
print('TN: '+str(tn))

max_stars_repo_name: dreamiond/TZ-KPrototypes

max_issues_repo_name: dreamiond/TZ-KPrototypes

max_forks_repo_name: dreamiond/TZ-KPrototypes


text:
import numpy as np

G = 6.67e-11

def compute_x_accel(x1,y1,z1,x2,y2,z2,center_mass):
    a_x = (G*center_mass*(x2-x1))/((x2-x1)**2+(y2-y1)**2+(z2-z1)**2)**(3/2)
    return a_x

def compute_y_accel(x1,y1,z1,x2,y2,z2,center_mass):
    a_y = (G*center_mass*(y2-y1))/((x2-x1)**2+(y2-y1)**2+(z2-z1)**2)**(3/2)
    return a_y

def compute_z_accel(x1,y1,z1,x2,y2,z2,center_mass):
    a_z = (G*center_mass*(z2-z1))/((x2-x1)**2+(y2-y1)**2+(z2-z1)**2)**(3/2)
    return a_z

def computeRk4(x1,y1,z1,x2,y2,z2,vx,vy,vz,center_mass,delT):

    k = np.zeros((24, 1))

    k1_x = vx
    k1_y = vy
    k1_z = vz

    k1_vx = compute_x_accel(x1,y1,z1,x2,y2,z2,center_mass)
    k1_vy = compute_y_accel(x1, y1, z1, x2, y2, z2, center_mass)
    k1_vz = compute_z_accel(x1, y1, z1, x2, y2, z2, center_mass)

    k2_x = vx + (delT / 2) * k1_vx
    k2_y = vy + (delT / 2) * k1_vy
    k2_z = vz + (delT / 2) * k1_vz

    k2_vx = compute_x_accel(x1+(delT/2)*k1_x, y1+(delT/2)*k1_y, z1+(delT/2)*k1_z,x2,y2,z2,center_mass)
    k2_vy = compute_y_accel(x1 + (delT / 2) * k1_x, y1 + (delT / 2) * k1_y, z1 + (delT / 2) * k1_z, x2, y2, z2,
                            center_mass)
    k2_vz = compute_z_accel(x1 + (delT / 2) * k1_x, y1 + (delT / 2) * k1_y, z1 + (delT / 2) * k1_z, x2, y2, z2,
                            center_mass)

    k3_x = vx + (delT / 2) * k2_vx
    k3_y = vy + (delT / 2) * k2_vy
    k3_z = vz + (delT / 2) * k2_vz

    k3_vx = compute_x_accel(x1+(delT/2)*k2_x, y1+(delT/2)*k2_y, z1+(delT/2)*k2_z, x2,y2,z2,center_mass)
    k3_vy = compute_y_accel(x1 + (delT / 2) * k2_x, y1 + (delT / 2) * k2_y, z1 + (delT / 2) * k2_z, x2, y2, z2,
                            center_mass)
    k3_vz = compute_z_accel(x1 + (delT / 2) * k2_x, y1 + (delT / 2) * k2_y, z1 + (delT / 2) * k2_z, x2, y2, z2,
                            center_mass)

    k4_x = vx + delT*k3_vx
    k4_y = vy + delT*k3_vy
    k4_z = vz + delT * k3_vz

    k4_vx = compute_x_accel(x1+delT*k3_x, y1+delT*k3_y, z1+delT*k3_z,x2,y2,z2,center_mass)
    k4_vy = compute_y_accel(x1 + delT * k3_x, y1 + delT * k3_y, z1 + delT * k3_z, x2, y2, z2, center_mass)
    k4_vz = compute_z_accel(x1 + delT * k3_x, y1 + delT * k3_y, z1 + delT * k3_z, x2, y2, z2, center_mass)

    k[0] = k1_x
    k[1] = k1_y
    k[2] = k1_z
    k[3] = k1_vx
    k[4] = k1_vy
    k[5] = k1_vz

    k[6] = k2_x
    k[7] = k2_y
    k[8] = k2_z
    k[9] = k2_vx
    k[10] = k2_vy
    k[11] = k2_vz

    k[12] = k3_x
    k[13] = k3_y
    k[14] = k3_z
    k[15] = k3_vx
    k[16] = k3_vy
    k[17] = k3_vz

    k[18] = k4_x
    k[19] = k4_y
    k[20] = k4_z
    k[21] = k4_vx
    k[22] = k4_vy
    k[23] = k4_vz
    return k


text:
#!/usr/bin/env python
# coding: utf-8

# # Tarea #1

# ## Estudiante: Heiner Romero Leiva

# In[2]:

import os
import pandas as pd
import numpy  as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import confusion_matrix
from pandas import DataFrame
from matplotlib import colors as mcolors
from keras.models import Sequential
from keras.layers import Dense
from sklearn.preprocessing import MinMaxScaler

# #### Función para calcular los índices de calidad de la predicción

# In[3]:

def indices_general(MC, nombres = None):
    precision_global = np.sum(MC.diagonal()) / np.sum(MC)
    error_global = 1 - precision_global
    precision_categoria  = pd.DataFrame(MC.diagonal()/np.sum(MC,axis = 1)).T
    if nombres!=None:
        precision_categoria.columns = nombres
    return {"Matriz de Confusión":MC,
            "Precisión Global":precision_global,
            "Error Global":error_global,
            "Precisión por categoría":precision_categoria}

# #### Función para graficar la distribución de la variable a predecir

# In[4]:

def distribucion_variable_predecir(data:DataFrame,variable_predict:str):
    colors = list(dict(**mcolors.CSS4_COLORS))
    df = pd.crosstab(index=data[variable_predict],columns="valor") / data[variable_predict].count()
    fig = plt.figure(figsize=(10,9))
    g = fig.add_subplot(111)
    countv = 0
    titulo = "Distribución de la variable %s" % variable_predict
    for i in range(df.shape[0]):
        g.barh(1,df.iloc[i],left = countv, align='center',color=colors[11+i],label= df.iloc[i].name)
        countv = countv + df.iloc[i]
    vals = g.get_xticks()
    g.set_xlim(0,1)
    g.set_yticklabels("")
    g.set_title(titulo)
    g.set_ylabel(variable_predict)
    g.set_xticklabels(['{:.0%}'.format(x) for x in vals])
    countv = 0
    for v in df.iloc[:,0]:
        g.text(np.mean([countv,countv+v]) - 0.03, 1 , '{:.1%}'.format(v), color='black', fontweight='bold')
        countv = countv + v
    g.legend(loc='upper center', bbox_to_anchor=(1.08, 1), shadow=True, ncol=1)

# #### Función para ver la distribución de una variable categórica respecto a la predecir

# In[5]:

def poder_predictivo_categorica(data:DataFrame, var:str, variable_predict:str):
    df = pd.crosstab(index= data[var],columns=data[variable_predict])
    df = df.div(df.sum(axis=1),axis=0)
    titulo = "Distribución de la variable %s según la variable %s" % (var,variable_predict)
    g = df.plot(kind='barh',stacked=True,legend = True, figsize = (10,9),                 xlim = (0,1),title = titulo, width = 0.8)
    vals = g.get_xticks()
    g.set_xticklabels(['{:.0%}'.format(x) for x in vals])
    g.legend(loc='upper center', bbox_to_anchor=(1.08, 1), shadow=True, ncol=1)
    for bars in g.containers:
        plt.setp(bars, width=.9)
    for i in range(df.shape[0]):
        countv = 0
        for v in df.iloc[i]:
            g.text(np.mean([countv,countv+v]) - 0.03, i , '{:.1%}'.format(v), color='black', fontweight='bold')
            countv = countv + v

# #### Función para ver la distribución de una variable numérica respecto a la predecir

# In[6]:

def poder_predictivo_numerica(data:DataFrame, var:str, variable_predict:str):
    sns.FacetGrid(data, hue=variable_predict, height=6).map(sns.kdeplot, var, shade=True).add_legend()

# ### Ejercicio 1:

# #### En este ejercicio usaremos los datos (voces.csv). Se trata de un problema de reconocimiento de genero mediante el analisis de la voz y el habla. Esta base de datos fue creada para identificar una voz como masculina o femenina, basandose en las propiedades acusticas de la voz y el habla. El conjunto de datos consta de 3.168 muestras de voz grabadas, recogidas de hablantes masculinos y femeninos

# #### 1. Cargue la tabla de datos voces.csv en Python.

# In[6]:

voces = pd.read_csv("voces.csv", delimiter = ',', decimal = '.')

# In[7]:

voces.info()
voces.shape
voces.info

# In[8]:

voces.describe()
# Se sacan estadisticas basicas para ver distribuciones y si es necesario centrar y normalizar las tabla, ya que Redes
# Neuronales es un metodo basado en distancias.

# In[9]:

# Normalizando y centrando la tabla ya que hay valores en diferentes escalas
voices = voces.iloc[:,0:20]
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaled_values = scaler.fit_transform(voices)
voices.loc[:,:] = scaled_values
voices.head()

# #### 2. Genere al azar una tabla de testing con una 20% de los datos y con el resto de los datos genere una tabla de aprendizaje.

# #### Distribución de la variable a predecir

# In[10]:

distribucion_variable_predecir(voces,"genero")
# La variable a predecir esta completamente balanceada, por lo que, en el testing las predicciones
# deben de dar muy parecidas.

# #### Elimina la variable categorica, deja las variables predictoras en X

# In[11]:

X = voices.iloc[:,0:20]
X.head()

# #### Deja la variable a predecir en y

# In[12]:

y = voces.iloc[:,20:21]
y.head()

# #### Se separan los datos con el 80% de los datos para entrenamiento y el 20% para testing

# In[13]:

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8)

# #### 3. Usando MLPClassifier genere un modelo predictivo para la tabla de aprendizaje. Utilice una cantidad suficiente de capas ocultas y nodos para que la prediccion sea buena.

# #### Usando 10000 capas ocultas de 150 nodos cada una

# In[14]:

instancia_red = MLPClassifier(solver='lbfgs', random_state=0,hidden_layer_sizes=[10000, 150])
instancia_red.fit(X_train,y_train.iloc[:,0].values)

# #### 4. Con la tabla de testing calcule la matriz de confusion, la precision, la precision positiva, la precision negativa, los falsos positivos, los falsos negativos, la acertividad positiva y la acertividad negativa. Luego construya un cuadro comparativo.

# #### Indices de Calidad del Modelo

# In[15]:

prediccion = instancia_red.predict(X_test)
MC = confusion_matrix(y_test, prediccion)
indices = indices_general(MC,list(np.unique(y)))
for k in indices:
    print("\n%s:\n%s"%(k,str(indices[k])))

# In[16]:

# Desplegando funcion programada
def indices_personalizados(MC):
    print(MC)
    precision_global = (MC.iloc[0,0] + MC.iloc[1,1]) / (MC.iloc[0,0]
                        + MC.iloc[0,1] + MC.iloc[1,0] + MC.iloc[1,1])
    error_global  = 1 - precision_global
    precision_positiva = (MC.iloc[1,1]) / (MC.iloc[1,0] + MC.iloc[1,1])
    precision_negativa = (MC.iloc[0,0]) / (MC.iloc[0,0] + MC.iloc[0,1])
    falsos_positivos = (MC.iloc[0,1]) / (MC.iloc[0,0] + MC.iloc[0,1])
    falsos_negativos = (MC.iloc[1,0]) / (MC.iloc[1,0] + MC.iloc[1,1])
    asertividad_positiva = (MC.iloc[1,1]) / (MC.iloc[0,1] + MC.iloc[1,1])
    asertividad_negativa = (MC.iloc[0,0]) / (MC.iloc[0,0] + MC.iloc[1,0])
    return {"Precisión Global":precision_global,
            "Error Global":error_global,
            "Precision Positiva (PP)":precision_positiva,
            "Precision Negativa (PN)":precision_negativa,
            "Falsos Positivos (PFP)":falsos_positivos,
            "Falsos Negativos (PFN)":falsos_negativos,
            "Asertividad Positiva (AP)":asertividad_positiva,
            "Asertividad Negativa (AN)":asertividad_negativa}

# #### Desplegando indices personalizados

# In[19]:

datos = (([286, 3],[7, 338]))
df = pd.DataFrame(datos, columns = ["Masculino", "Femenino"])
MC = df
indices_personalizados(MC)

# #### 5. Construya un cuadro comparativo con respecto a las tareas del curso anterior. ¿Cual metodo es mejor?

# In[74]:

cadena = "Cuadro Comparativo entre Modelos Supervisados"
print(cadena.center(35," "))
print(" ========================================")
print(" Modelo K Vecinos Mas Cercanos:\n**************************")
print("Precisión Global: 0.9479495268138801\nError Global: 0.05205047318611988\nPrecision Positiva (PP): 0.9779874213836478\nPrecision Negativa (PN): 0.9177215189873418\nFalsos Positivos (PFP): 0.08227848101265822\nFalsos Negativos (PFN): 0.0220125786163522\nAsertividad Positiva (AP): 0.9228486646884273\nAsertividad Negativa (AN): 0.9764309764309764\n**************************")
print(" Arbol de decision:\n**************************")
print("Precisión Global: 0.9684542586750788\nError Global: 0.03154574132492116\nPrecision Positiva (PP): 0.9688473520249221\nPrecision Negativa (PN): 0.9680511182108626\nFalsos Positivos (PFP): 0.03194888178913738\nFalsos Negativos (PFN): 0.03115264797507788\nAsertividad Positiva (AP): 0.9688473520249221\nAsertividad Negativa (AN): 0.9680511182108626\n**************************")
print(" Arboles Aleatorios:\n**************************")
print("Precisión Global: 0.9889589905362776\nError Global: 0.01104100946372244\nPrecision Positiva (PP): 0.99375\nPrecision Negativa (PN): 0.9840764331210191\nFalsos Positivos (PFP): 0.01592356687898089\nFalsos Negativos (PFN): 0.00625\nAsertividad Positiva (AP): 0.9845201238390093\nAsertividad Negativa (AN): 0.9935691318327974\n**************************")
print(" Modelo ADA Boosting:\n**************************")
print("Precisión Global: 0.9810725552050473,\nError Global: 0.018927444794952675\nPrecision Positiva (PP): 0.990625\nPrecision Negativa (PN): 0.9713375796178344\nFalsos Positivos (PFP): 0.028662420382165606\nFalsos Negativos (PFN): 0.009375\nAsertividad Positiva (AP): 0.9723926380368099\nAsertividad Negativa (AN): 0.9902597402597403\n**************************")
print(" Modelo XG Boosting:\n**************************")
print("Precisión Global: 0.9889589905362776,\nError Global: 0.01104100946372244\nPrecision Positiva (PP): 0.99375\nPrecision Negativa (PN): 0.9840764331210191\nFalsos Positivos (PFP): 0.01592356687898089\nFalsos Negativos (PFN): 0.00625\nAsertividad Positiva (AP): 0.9845201238390093\nAsertividad Negativa (AN): 0.9935691318327974\n**************************")
print(" Modelo Maquinas de Soporte Vectorial:\n**************************")
print("Precisión Global: 0.9826498422712934\nError Global: 0.017350157728706628\nPrecision Positiva (PP): 0.9821958456973294\nPrecision Negativa (PN): 0.9831649831649831\nFalsos Positivos (PFP): 0.016835016835016835\nFalsos Negativos (PFN): 0.017804154302670624\nAsertividad Positiva (AP): 0.9851190476190477\nAsertividad Negativa (AN): 0.9798657718120806\n**************************")
print(" Modelo Redes Neuronales - MLPClassifier\n**************************")
print("Precisión Global: 0.9842271293375394\nError Global: 0.01577287066246058\nPrecision Positiva (PP): 0.9797101449275363\nPrecision Negativa (PN): 0.9896193771626297\nFalsos Positivos (PFP): 0.010380622837370242\nFalsos Negativos (PFN): 0.020289855072463767\nAsertividad Positiva (AP): 0.9912023460410557\nAsertividad Negativa (AN): 0.9761092150170648\n**************************")
print(" ========================================")

# ##### Analisis
# * Con respecto al cuadro comparativo se puede ver que el Modelo que da los mejores resultados es el de Arboles Aleatorios junto con el XG Boosting, ya que ambos tienen la precision global mas alta de casi un 99%, ademas que la Asertividad Positiva es de mas de un 98% mientras que la negativa es de mas de un 99% lo que los hace modelos bastante confiables.

# #### Repita los ejercicios anteriores, pero esta vez utilice el paquete Keras, utilice la misma cantidad de capas ocultas y nodos que la usada arriba. ¿Mejora la prediccion?

# #### Reescalando las variables predictoras y haciendo el split del 80% para el training y el 20% para el testing

# In[89]:

dummy_y = pd.get_dummies(y)
scaler = MinMaxScaler(feature_range = (0, 1))
scaled_X  = pd.DataFrame(scaler.fit_transform(X), columns = list(X))
X_train, X_test, y_train, y_test = train_test_split(scaled_X, dummy_y, train_size = 0.8, random_state = 0)
print(X_train.head())
print(dummy_y)

# #### Creando el Modelo en keras junto con Capas

# In[98]:

model = Sequential()
model.add(Dense(20, input_dim = 20, activation = 'relu'))  # primera capa oculta con 10 neuronas, 20 features
model.add(Dense(100, activation = 'sigmoid'))  # segunda capa oculta con 10 neuronas
model.add(Dense(30, activation = 'relu'))  # tercera capa oculta con 10 neuronas
model.add(Dense(2, activation = 'softmax')) # capa output con 10 neuronas, el 2 son la cantidad de categorias a predecir

# #### Compilacion del Modelo

# In[99]:

model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])

# #### Resumen del Modelo

# In[100]:

print(model.summary())

# #### Se usan 10000 etapas de entrenamiento (Epochs) y se actualizan los pesos cada 150 observaciones procesadas

# In[101]:

model.fit(X_train, y_train, epochs = 10000, batch_size = 150, verbose = 0)
y_pred = model.predict(X_test)
y_test_class = np.argmax(np.asanyarray(y_test), axis = 1)
y_pred_class = np.argmax(y_pred, axis = 1)

# #### Predicciones y Calidad del Modelo

# In[102]:

scores = model.evaluate(X_test, y_test)
MC = confusion_matrix(y_test_class, y_pred_class)
indices = indices_general(MC,list(np.unique(y)))
for k in indices:
    print("\n%s:\n%s"%(k,str(indices[k])))

# In[103]:

datos = (([296, 5],[8, 325]))
df = pd.DataFrame(datos, columns = ["Masculino", "Femenino"])
MC = df
indices_personalizados(MC)

# #### 7. Compare los resultados con los obtenidos en las tareas del curso anterior.

# In[104]:

cadena = "Cuadro Comparativo entre Modelos Supervisados"
print(cadena.center(35," "))
print(" ========================================")
print(" Modelo K Vecinos Mas Cercanos:\n**************************")
print("Precisión Global: 0.9479495268138801\nError Global: 0.05205047318611988\nPrecision Positiva (PP): 0.9779874213836478\nPrecision Negativa (PN): 0.9177215189873418\nFalsos Positivos (PFP): 0.08227848101265822\nFalsos Negativos (PFN): 0.0220125786163522\nAsertividad Positiva (AP): 0.9228486646884273\nAsertividad Negativa (AN): 0.9764309764309764\n**************************")
print(" Arbol de decision:\n**************************")
print("Precisión Global: 0.9684542586750788\nError Global: 0.03154574132492116\nPrecision Positiva (PP): 0.9688473520249221\nPrecision Negativa (PN): 0.9680511182108626\nFalsos Positivos (PFP): 0.03194888178913738\nFalsos Negativos (PFN): 0.03115264797507788\nAsertividad Positiva (AP): 0.9688473520249221\nAsertividad Negativa (AN): 0.9680511182108626\n**************************")
print(" Arboles Aleatorios:\n**************************")
print("Precisión Global: 0.9889589905362776\nError Global: 0.01104100946372244\nPrecision Positiva (PP): 0.99375\nPrecision Negativa (PN): 0.9840764331210191\nFalsos Positivos (PFP): 0.01592356687898089\nFalsos Negativos (PFN): 0.00625\nAsertividad Positiva (AP): 0.9845201238390093\nAsertividad Negativa (AN): 0.9935691318327974\n**************************")
print(" Modelo ADA Boosting:\n**************************")
print("Precisión Global: 0.9810725552050473,\nError Global: 0.018927444794952675\nPrecision Positiva (PP): 0.990625\nPrecision Negativa (PN): 0.9713375796178344\nFalsos Positivos (PFP): 0.028662420382165606\nFalsos Negativos (PFN): 0.009375\nAsertividad Positiva (AP): 0.9723926380368099\nAsertividad Negativa (AN): 0.9902597402597403\n**************************")
print(" Modelo XG Boosting:\n**************************")
print("Precisión Global: 0.9889589905362776,\nError Global: 0.01104100946372244\nPrecision Positiva (PP): 0.99375\nPrecision Negativa (PN): 0.9840764331210191\nFalsos Positivos (PFP): 0.01592356687898089\nFalsos Negativos (PFN): 0.00625\nAsertividad Positiva (AP): 0.9845201238390093\nAsertividad Negativa (AN): 0.9935691318327974\n**************************")
print(" Modelo Maquinas de Soporte Vectorial:\n**************************")
print("Precisión Global: 0.9826498422712934\nError Global: 0.017350157728706628\nPrecision Positiva (PP): 0.9821958456973294\nPrecision Negativa (PN): 0.9831649831649831\nFalsos Positivos (PFP): 0.016835016835016835\nFalsos Negativos (PFN): 0.017804154302670624\nAsertividad Positiva (AP): 0.9851190476190477\nAsertividad Negativa (AN): 0.9798657718120806\n**************************")
print(" Modelo Redes Neuronales - TensorFlow y Keras\n**************************")
print("Precisión Global: 0.9794952681388013\nError Global: 0.02050473186119872\nPrecision Positiva (PP): 0.975975975975976\nPrecision Negativa (PN): 0.9833887043189369\nFalsos Positivos (PFP): 0.016611295681063124\nFalsos Negativos (PFN): 0.024024024024024024\nAsertividad Positiva (AP): 0.9848484848484849\nAsertividad Negativa (AN): 0.9736842105263158\n**************************")
print(" ========================================")

# ##### Analisis
# * Comparando los resultados, se puede ver que la Red Neuronal usando TensorFlow da muy buenos resultados, sin embargo, el que sigue dando los mejores resultados es el Arbol de Decision junto con el Modelo XG Boosting (igual que el ejercicio previo) ya que se alcanza casi un 99% de precision global, y mas de un 98% de Asertividad Positiva y mas de un 99% de Asertividad Negativa.

# #### Ejercicio 2:

# #### Esta pregunta utiliza los datos (tumores.csv). Se trata de un conjunto de datos de caracterısticas del tumor cerebral que incluye cinco variables de primer orden y ocho de textura y cuatro parametros de evaluacion de la calidad con el nivel objetivo. La variables son: Media, Varianza, Desviacion estandar, Asimetrıa, Kurtosis, Contraste, Energıa, ASM (segundo momento angular), Entropıa, Homogeneidad, Disimilitud, Correlacion, Grosor, PSNR (Pico de la relacion senal-ruido), SSIM (Indice de Similitud Estructurada), MSE (Mean Square Error), DC (Coeficiente de Dados) y la variable a predecir tipo (1 = Tumor, 0 = No-Tumor).

# #### 1. Usando el paquete MLPClassifier y el paquete Keras en Python genere modelos predictivos para la tabla Tumores.csv usando 70 % de los datos para tabla aprendizaje y un 30 % para la tabla testing. Utilice una cantidad suficiente de capas ocultas y nodos para que la prediccion sea buena.

# #### Utilizando paquete MLPClassifier

# In[7]:

tumores = pd.read_csv("tumores.csv", delimiter = ',', decimal = '.')
tumores.head()

# In[8]:

tumores.info()

# In[9]:

# Convierte las variables de object a categórica
tumores['imagen'] = tumores['imagen'].astype('category')
print(tumores.info())
print(tumores.head())
# Recodifica las categorías usando números
tumores["imagen"] = tumores["imagen"].cat.codes
print(tumores.info())
print(tumores.head())
# Convierte las variables de entero a categórica
tumores['imagen'] = tumores['imagen'].astype('category')
print(tumores.info())
print(tumores.head())

# In[10]:

tumores.tail() #variable categorica ha sido convertida a numero

# #### Distribución de la variable a predecir

# In[11]:

distribucion_variable_predecir(tumores,"tipo") #Problema altamente desequilibrado.

# In[12]:

# Normalizando y centrando la tabla ya que hay valores en diferentes escalas
tumores_1 = tumores.iloc[:,0:17]
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaled_values = scaler.fit_transform(tumores_1)
tumores_1.loc[:,:] = scaled_values
tumores_1.head()

# Variables con escalas diferentes han sido reescaladas.

# #### Elimina la variable catégorica, deja las variables predictoras en X

# In[13]:

X = tumores_1.iloc[:,0:17]
X.head()

# #### Deja la variable a predecir en y

# In[14]:

y = tumores.iloc[:,17:18]
y.head()

# #### Se separan los datos con el 70% de los datos para entrenamiento y el 30% para testing

# In[15]:

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7)

# #### Usando 10000 capas ocultas de 150 nodos cada una

# In[17]:

instancia_red = MLPClassifier(solver='lbfgs', random_state=0,hidden_layer_sizes=[10000, 150])
print(instancia_red)

# #### Entrenando al modelo mediante el metodo Fit

# In[18]:

instancia_red.fit(X_train,y_train.iloc[:,0].values)
print("Las predicciones en Testing son: {}".format(instancia_red.predict(X_test)))

# #### Índices de Calidad del Modelo

# In[19]:

prediccion = instancia_red.predict(X_test)
MC = confusion_matrix(y_test, prediccion)
indices = indices_general(MC,list(np.unique(y)))
for k in indices:
    print("\n%s:\n%s"%(k,str(indices[k])))

# #### Usando Paquete Keras

# In[20]:

distribucion_variable_predecir(tumores,"tipo") # Es un problema desbalanceado.

# In[21]:

tumores.info() # Hay una categoria dentro de los datos, sin embargo esa variable ya que se habia convertido.

# tumores_1.info()

# #### Elimina la variable catégorica, deja las variables predictoras en X

# In[22]:

X = tumores_1.iloc[:,0:17]
X.head()

# #### Deja la variable a predecir en y

# #### Como la variable a predecir la dan en terminos de 0 y 1, es necesario con vertirla a Si y No.

# In[24]:

import pandas as pd
d = tumores
df = pd.DataFrame(data=d)
df

# In[40]:

df.replace({0: "No", 1: "Si"}, inplace = True)
print(df.iloc[:,17:18]) #Resultado fue reemplazado con exito.

# In[26]:

y = df.iloc[:,17:18]
y.head()

# #### Se separan los datos con el 70% de los datos para entrenamiento y el 30% para testing

# #### Como la variable a predecir ya viene dada por "0" y "1" no es necesario utilizar codigo disyuntivo ni reescalar

# In[27]:

dummy_y = pd.get_dummies(y)
scaler = MinMaxScaler(feature_range = (0, 1))
scaled_X  = pd.DataFrame(scaler.fit_transform(X), columns = list(X))
X_train, X_test, y_train, y_test = train_test_split(scaled_X, dummy_y, train_size = 0.7, random_state = 0)
print(dummy_y)

# #### Creando Modelo en Keras

# In[33]:

model = Sequential()
model.add(Dense(1000, input_dim = 17, activation = 'relu'))  # primera capa oculta con 5000 neuronas
model.add(Dense(500, activation = 'sigmoid'))  # segunda capa oculta con 10000 neuronas
model.add(Dense(300, activation = 'sigmoid'))  # tercera capa oculta con 5000 neuronas
model.add(Dense(50, activation = 'relu'))  # Agregamos tercera capa oculta con 4998 neuronas
model.add(Dense(2, activation = 'softmax')) # Agregamos capa output con 2 neuronas

# #### Complilando el Modelo

# In[34]:

model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])

# #### Resumen del Modelo

# In[35]:

print(model.summary())

# #### Ajustes del Modelo

# #### Usamos 10000 etapas de entrenamiento (epochs) y actualizando los pesos de la red cada 150 observaciones procesadas (batch_size).

# In[36]:

model.fit(X_train, y_train, epochs = 10000, batch_size = 150, verbose = 0)
# La predicción es una matriz con 3 columnas
y_pred = model.predict(X_test)
# Convertimos a columna
y_test_class = np.argmax(np.asanyarray(y_test), axis = 1)  # Convertimos a array
y_pred_class = np.argmax(y_pred, axis = 1)

# #### Predicciones y Calidad del Modelo

# In[37]:

scores = model.evaluate(X_test, y_test)
MC = confusion_matrix(y_test_class, y_pred_class)
indices = indices_general(MC,list(np.unique(y)))
for k in indices:
    print("\n%s:\n%s"%(k,str(indices[k])))

# #### 2. Calcule para los datos de testing la precision global y la matriz de confusion. Interprete la calidad de los resultados. Ademas compare respecto a los resultados obtenidos en las tareas del curso anterior.

# In[43]:

cadena = "Cuadro Comparativo entre Calidades de los Modelos Supervisados"
print(cadena.center(100," "))
print(" ========================================")
print(" Modelo K Vecinos Mas Cercanos:\n**************************")
print("Precisión Global: 0.9479495268138801\nError Global: 0.05205047318611988\n**************************")
print(" Arbol de decision:\n**************************")
print("Precisión Global: 0.9684542586750788\nError Global: 0.03154574132492116\n**************************")
print(" Arboles Aleatorios:\n**************************")
print("Precisión Global: 0.9889589905362776\nError Global: 0.01104100946372244\n**************************")
print(" Modelo ADA Boosting:\n**************************")
print("Precisión Global: 0.9810725552050473,\nError Global: 0.018927444794952675\n**************************")
print(" Modelo XG Boosting:\n**************************")
print("Precisión Global: 0.9889589905362776,\nError Global: 0.01104100946372244\n**************************")
print(" Modelo Maquinas de Soporte Vectorial:\n**************************")
print("Precisión Global: 0.9826498422712934\nError Global: 0.017350157728706628\n**************************")
print(" Modelo utilizando paquete MLPClassifier\n**************************")
print("Precisión Global: 0.9686684073107049\nError Global: 0.031331592689295085\n**************************")
print(" Modelo Redes Neuronales - TensorFlow y Keras\n**************************")
print("Precisión Global: 0.9712793733681462\nError Global: 0.02872062663185382\n**************************")
print(" ========================================")

# #### 3. Compare los resultados con los obtenidos en las tareas del curso anterior.

# #### Analisis
# * Comparando los resultados obtenidos con las redes neuronales con los de las tareas anteriores se puede ver como se mantienen los mejores resultados usando los Arboles Aleatorios y el XG Boosting a nivel de Precision Global (es casi de un 99%) mientras que el error es poco mas de un 1%.
# * Se rescata que utilizando Keras y usando el paquete de TensoFlow de Google, los resultados por la categoria especifica de "no" cuenta con un tumor son considerablemente mejores que en casi todos los modelos, ya que al ser un problema desequilibrado, tiende a solo dar buenos resultados en el "Si" ya quie es donde hay mas datos para hacer la prediccion.

# ### Ejercicio 3:

# #### [no usar MLPClassifier ni Keras] Disene una Red Neuronal de una capa (Perceptron) para la tabla de verdad del nand:

# In[45]:

from IPython.display import Image
Image(filename="/Users/heinerleivagmail.com/cadena.png")

# #### Es decir, encuentre los pesos w1, w2 y el umbral θ para la Red Neuronal que se muestra en el siguiente grafico, usando una funcion de activacion tipo Sigmoidea:

# In[46]:

from IPython.display import Image
Image(filename="/Users/heinerleivagmail.com/screen.png")

# In[2]:

import numpy as np
import pandas as pd
import math

def sigmoidea(pesos, predictoras):
  x = 0
  for i in range(len(predictoras)):
    x += pesos[i] * predictoras[i]

  x -= pesos[len(pesos) - 1]
  return (1 / (1 + math.exp(-(x))))

def NAND(datos, resultado, pesos, ACTIVACION, corte = 0.5):
  calc = []
  for i in range(datos.shape[0]):
    calc.append(ACTIVACION(pesos, datos.iloc[i, :].values))

  return(pd.DataFrame({
    'res': calc,
    'pred': [1 if a >= corte else 0 for a in calc],
    'real': resultado
  }))

x = pd.DataFrame({'X1': [0, 1, 0, 1], 'X2': [0, 0, 1, 1]})
z = [1, 1, 1, 0]
NAND(x, z, [-0.5, -0.5, -0.5], sigmoidea) # El utlimo valor corresponde al umbral

# ### Ejercicio 4:

# [no usar MLPClassifier ni Keras] Para la Tabla de Datos que se muestra seguidamente donde x j para j = 1, 2, 3 son las variables predictoras y la variable a predecir es z disene y programe a pie una Red Neuronal de una capa (Perceptron):

# In[47]:

from IPython.display import Image
Image(filename="/Users/heinerleivagmail.com/mini.png")

# #### Es decir, encuentre todos los posibles pesos w1, w2, w3 y umbrales θ para la Red Neuronal que se muestra en el siguiente grafico:

# In[48]:

from IPython.display import Image
Image(filename="/Users/heinerleivagmail.com/perce.png")

# #### Use una funcion de activacion tipo Tangente hiperbolica, para esto escriba una Clase en Python que incluya los metodos necesarios pra implementar esta Red Neuronal.Se deben hacer variar los pesos wj con j = 1, 2, 3 en los siguientes valores v=(-1,-0.9,-0.8,...,0,...,0.8,0.9,1) y haga variar θ en u=(0,0.1,...,0.8,0.9,1). Escoja los pesos wj con j = 1, 2, 3 y el umbral θ de manera que se minimiza el error cuadratico medio.

# In[3]:

def tan_hiperbolica(pesos, predictoras):
  x = 0
  for i in range(len(predictoras)):
    x += pesos[i] * predictoras[i]

  x -= pesos[len(pesos) - 1]
  return ((2 / (1 + math.exp(-2 * x))) - 1)

def ECM(pred, real):
  ecm = [(pred[i] - real[i])**2 for i in range(len(pred))]
  return(sum(ecm) / len(pred))

x = pd.DataFrame({'X1': [1, 1, 1, 1], 'X2': [0, 0, 1, 1], 'X3': [0, 1, 0, 1]})
z = [1, 1, 1, 0]

resultados = pd.DataFrame()
for w1 in np.arange(-10, 11)/10:
  for w2 in np.arange(-10, 11)/10:
    for w3 in np.arange(-10, 11)/10:
      for umbral in np.arange(0, 11)/10:
        aux = NAND(x, z, [w1, w2, w3, umbral], tan_hiperbolica, corte = 0)
        e = ECM(aux["pred"].values, aux["real"].values)
        nuevo = pd.DataFrame({"w1": [w1], "w2": [w2], "w3": [w3], "umbral": [umbral], "ECM": [e]})
        resultados = resultados.append(nuevo)

resultados.loc[resultados["ECM"] == 0, :]

# #### Como se puede ver, la funcion programada puede predecir los valores dados en las Z.

# In[149]:

############################################################# FIN ###################################################################

max_stars_repo_name: levraines/Portfolio

max_issues_repo_name: levraines/Portfolio

max_forks_repo_name: levraines/Portfolio


text:
import os
from typing import List, Tuple

import logging
from itertools import groupby
from tqdm import tqdm  # type: ignore

import numpy as np  # type: ignore
import tensorflow as tf  # type: ignore
from tensorflow.keras import models, layers, optimizers  # type: ignore

from domain_adaptation.datasets.SwaVDataset import SwaVDataset

logger = logging.getLogger('trainer')
logger.setLevel(logging.INFO)

class SwAV:

    def __init__(self,
                 model: models.Model,
                 p_d1: int = 1024,
                 p_d2: int = 96,
                 p_dim: int = 10,
                 input_shape: Tuple[int] = (2048,),
                 normalize: bool = True,
                 sinkhorn_iter: int = 5,
                 epsilon: float = 0.05,
                 temperature: float = 0.1,
                 crops_for_assign: List[int] = [0, 1]) -> None:
        self.model = model
        self.l2norm = normalize
        self.p_d1, self.p_d2, self.p_dim = p_d1, p_d2, p_dim
        self.input_shape = input_shape
        self.prototype_model = self.prototype(self.p_d1, self.p_d2, self.p_dim,
                                              self.input_shape)
        self.sinkhorn_iter, self.epsilon = sinkhorn_iter, epsilon
        self.temperature = temperature
        self.crops_for_assign = crops_for_assign
        self.epoch_loss: List[float] = []
        self.step_loss: List[float] = []

    def fit(
        self,
        dataloader: SwaVDataset,  # Create a SwaVDataset param
        optimizer: optimizers.Optimizer,
        epochs: int = 50,
    ):
        num_batches = dataloader.dataset_swaved.cardinality().numpy()
        for ep in range(epochs):
            SwAV.norm_layer(self.prototype_model, "prototype")
            if num_batches in [
                    tf.data.INFINITE_CARDINALITY, tf.data.UNKNOWN_CARDINALITY
            ]:
                logger.warn(
                    "couldn't compute number of batches, no progress bar...")
                pbar = tqdm(enumerate(dataloader.dataset_swaved))
            else:
                pbar = tqdm(enumerate(dataloader.dataset_swaved),
                            total=num_batches)
            logger.info(f"Epoch: {ep+1}...")
            self.step_loss = []
            for i, inputs in pbar:
                loss = self.epoch(inputs, optimizer, dataloader.nmb_crops)
                self.step_loss.append(loss)
                pbar.set_description(
                    f"Current loss: {np.mean(self.step_loss):.4f}")
            self.epoch_loss.append(np.mean(self.step_loss))
            logger.info(f"Epoch: {ep+1}/{epochs}\t"
                        f"Loss: {np.mean(self.step_loss):.4f}")

    def epoch(self, inputs: Tuple, optimizer: tf.keras.optimizers,
              nb_crops: List[int]) -> float:
        images = list(inputs)
        b_s = images[0].shape[0]
        # getting a list of consecutive idxs with same crop size
        crop_sizes = [img.shape[1] for img in images]
        idx_crops = tf.math.cumsum(
            [len(list(g)) for _, g in groupby(crop_sizes)], axis=0)
        start = 0

        with tf.GradientTape() as tape:
            for end in idx_crops:
                concat_input = tf.stop_gradient(
                    tf.concat(values=inputs[start:end], axis=0))
                _embedding = self.model(concat_input)
                if start == 0:
                    embeddings = _embedding
                else:
                    embeddings = tf.concat(values=(embeddings, _embedding),
                                           axis=0)
                start = end

            projection, prototype = self.prototype_model(embeddings)
            projection = tf.stop_gradient(projection)

            loss = .0
            for i, crop_id in enumerate(self.crops_for_assign):
                with tape.stop_recording():
                    out = prototype[b_s * crop_id:b_s * (crop_id + 1)]
                    clus = self.sinkhorn(out)[-b_s:]

                subloss = .0
                for v in np.delete(np.arange(np.sum(nb_crops)), crop_id):
                    prob = tf.nn.softmax(prototype[b_s * v:b_s * (v + 1)] /
                                         self.temperature)
                    subloss -= tf.math.reduce_mean(
                        tf.math.reduce_sum(clus * tf.math.log(prob), axis=1))
                loss += subloss / tf.cast(
                    tf.reduce_sum(nb_crops) - 1, tf.float32)
            loss /= len(self.crops_for_assign)
        varrs = (self.model.trainable_variables +
                 self.prototype_model.trainable_variables)
        gradients = tape.gradient(loss, varrs)
        optimizer.apply_gradients(zip(gradients, varrs))

        return loss

    def prototype(self, d1: int, d2: int, dim: int,
                  input_shape: Tuple[int]) -> models.Model:
        inputs = layers.Input(input_shape)
        projection1 = layers.Dense(d1)(inputs)
        projection1 = layers.BatchNormalization()(projection1)
        projection1 = layers.Activation("relu")(projection1)
        projection2 = layers.Dense(d2, name="projection")(projection1)
        if self.l2norm:
            projection2 = tf.math.l2_normalize(projection2,
                                               axis=1,
                                               name="projection_normalized")
        prototype = layers.Dense(dim, use_bias=False,
                                 name="prototype")(projection2)
        return models.Model(inputs=inputs, outputs=[projection2, prototype])

    @staticmethod
    def norm_layer(prototype: models.Model,
                   prototype_layer: str = "prototype") -> None:
        weights = tf.transpose(
            prototype.get_layer(prototype_layer).get_weights())
        weights_norm = tf.math.l2_normalize(weights, axis=1)
        prototype.get_layer(prototype_layer).set_weights(
            tf.transpose(weights_norm))

    def sinkhorn(
        self,
        scores,
    ) -> tf.Tensor:
        Q = tf.transpose(tf.exp(scores / self.epsilon))
        Q /= tf.keras.backend.sum(Q)
        K, B = Q.shape
        u, r, c = tf.zeros(K, dtype=tf.float32), tf.ones(
            K, dtype=tf.float32), tf.ones(B, dtype=tf.float32) / B
        for _ in range(self.sinkhorn_iter):
            u = tf.keras.backend.sum(Q, axis=1)
            Q *= tf.expand_dims((r / u), axis=1)
            Q *= tf.expand_dims(c / tf.keras.backend.sum(Q, axis=0), axis=0)
        return tf.transpose(Q / tf.keras.backend.sum(Q, axis=0, keepdims=True))

    def save(self, path):
        directory = os.mkdir(path)
        self.model.save(os.path.join(directory, "main_model"))
        self.prototype_model.save(os.path.join(directory, "prototype_model"))

    def load(self, path):
        self.model = models.load_model(os.path.join(path, "main_model"))
        self.prototype_model = models.load_model(
            os.path.join(path, "prototype_model"))

max_stars_repo_name: othmanefc/domain-adaptation

max_issues_repo_name: othmanefc/domain-adaptation

max_forks_repo_name: othmanefc/domain-adaptation


text:
import numpy as np
import tensorflow as tf
import argparse
import os
import itk
import sys
from utils import *

def main(args):
	print("Tensorflow version:", tf.__version__)

	saved_model_path = args.model
	out_name = args.out

	model = tf.keras.models.load_model(saved_model_path, custom_objects={'tf': tf})
	model.summary()

	img = itk.imread(args.img)
	img_np = itk.GetArrayViewFromImage(img)

	prediction = model.predict(img_np.reshape((1,) + img_np.shape))

	out_img = GetImage(prediction[0])

	itk.imwrite(out_img, args.out)

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description='Predict a label map from a natural image of an eye', formatter_class=argparse.ArgumentDefaultsHelpFormatter)

	input_group = parser.add_argument_group('Input parameters')
	input_group.add_argument('--img', type=str, help='Input image for prediction')

	model_group = parser.add_argument_group('Model group')
	model_group.add_argument('--model', help='Directory of trained model in saved model format')

	out_group = parser.add_argument_group('Output parameters')
	out_group.add_argument('--out', type=str, help='Output image name', default="out.nrrd")

	args = parser.parse_args()

	main(args)

max_stars_repo_name: hina-shah/trachoma

max_issues_repo_name: hina-shah/trachoma

max_forks_repo_name: hina-shah/trachoma


text:
import cv2
import numpy as np

img = cv2.imread("../1.jpg")
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
kernel = np.ones((5,5),np.float32)/25
gray = cv2.filter2D(gray,-1,kernel)
edges = cv2.Canny(gray,400,600,apertureSize = 5)
cv2.imshow('image',edges)
cv2.waitKey(0)

lines = cv2.HoughLines(edges,1,np.pi/180,15)
for i in range(8):
    for rho,theta in lines[i]:
           a = np.cos(theta)
           b = np.sin(theta)
           x0 = a*rho
           y0 = b*rho
           x1 = int(x0 + 1000*(-b))
           y1 = int(y0 + 1000*(a))
           x2 = int(x0 - 1000*(-b))
           y2 = int(y0 - 1000*(a))
    cv2.line(img,(x1,y1),(x2,y2),(0,0,255),2)

cv2.imshow('image',img)
cv2.waitKey(0)


text:
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly as py
import pandas as pd
import sys
import os
import pandas as pd
from os import walk
import glob
import fnmatch
import numpy as np

from plugins.FabCovid19.FabCovid19 import *
from .facs_postprocess_utils import *

def plot(df, output_dir, locations):
    colors = px.colors.qualitative.G10
    fig = make_subplots(rows=9, cols=1, shared_xaxes=True,
                        vertical_spacing=0.01, horizontal_spacing=0.05)
    row = 1
    for location in enumerate(locations):
        subdf = df[df['type'] == location[1]]
        fig.add_trace(go.Scatter(x=subdf['time'], y=subdf['mean'], name=location[1],  line=dict(
            color=colors[row])), row=row, col=1)
        row += 1

    fig.update_xaxes(showline=True, linewidth=1,
                     linecolor='black', mirror=True)
    fig.update_yaxes(showline=True, linewidth=1,
                     linecolor='black', mirror=True)
    fig.update_traces(mode='lines',  marker=dict(size=1, colorscale='Plotly3'))
    fig.update_layout(legend_orientation="h",  autosize=True, width=800,
                      height=1000, font=dict(family="Courier New, monospace", size=12))
    py.offline.plot(fig, filename='{}.html'.format(output_dir))

@task
@load_plugin_env_vars("FabCovid19")
def facs_locationplot(output_dir, output_file='covid_out_infections.csv'):
    results_dir = locate_results_dir(output_dir)
    borough = extract_location_name(results_dir)

    # Extract cores value
    cores = int(output_dir.split("_")[-1])
    print("CORES = {}".format(cores))

    run_list = []
    for root, dirs, files in os.walk(results_dir, topdown=True):
        # print(root,dirs,files)
        for name in files:
            if "covid_out_infections_" in name and borough in root:
                replica = root.split('\\')[-1].split('_')[1]
                filepath = os.path.join(root, name)
                print("file found at:", filepath)
                df = pd.read_csv(filepath, usecols=[
                                 '#time', 'x', 'y', 'location_type'])
                run_list.append(df)

    print("INFO: {} files read, representing {} runs.".format(len(run_list), len(run_list)/cores))
    # Calculate ensemble size
    runs = int(len(run_list)/cores)

    rows = []
    no = 0
    for df in run_list:
        grouped = df.groupby(['#time', 'location_type'])
        for name, group in grouped:
            d = {'time': name[0], 'run': no,
                 'type': name[1], 'count': len(group)}
            rows.append(d)
        no += 1

    types = ['hospital', 'house', 'office', 'park', 'leisure',
             'school', 'supermarket', 'shopping', 'traffic']
    pdf = pd.DataFrame(rows)
    pdf = pdf[pdf.time != "#t"]
    pdf['time'] = pd.to_numeric(pdf['time'])

    mean_rows = []
    for i in range(pdf['time'].min(), pdf['time'].max()):
        for j in types:
            data = pdf[(pdf['time'] == i) & (pdf['type'] == j)]
            # print(j,data)
            count = data['count'].sum()
            count /= runs
            d = {'time': i, 'type': j, 'mean': count}
            mean_rows.append(d)
    mdf = pd.DataFrame(mean_rows)
    plot(mdf, output_dir, types)

max_stars_repo_name: djgroen/FabCovid

max_issues_repo_name: djgroen/FabCovid

max_forks_repo_name: djgroen/FabCovid


text:
from .context import stadv, call_assert

import tensorflow as tf
import numpy as np

class LBFGSCase(tf.test.TestCase):
    """Test the lbfgs optimization function.
    Note: we are NOT testing the LBFGS implementation from SciPy, instead we
    test our wrapping and its interplay with TensorFlow."""

    def setUp(self):
        self.example_flow = np.array([[0.5, 0.4], [-0.2, 0.7]])
        self.flows = tf.Variable(self.example_flow, name='flows')
        self.loss_l2 = tf.reduce_sum(tf.square(self.flows), name='loss_l2')
        self.loss_dummy = tf.Variable(1.4, name='loss_dummy')

        tf.global_variables_initializer()

    def test_l2_norm_loss(self):
        """Check that simple L2 loss leads to 0 loss and gradient in the end."""
        results = stadv.optimization.lbfgs(
            self.loss_l2, self.flows, flows_x0=self.example_flow
        )
        call_assert(
            self.assertEqual,
            results['flows'].shape, self.example_flow.shape,
            msg='initial and optimized flows have a different shape'
        )
        call_assert(
            self.assertAllClose,
            results['flows'], np.zeros(results['flows'].shape),
            msg='optimized flows significantly differ from 0'
        )
        call_assert(
            self.assertAllClose,
            results['loss'], np.zeros(results['loss'].shape),
            msg='final gradients significantly differ from 0'
        )

    def test_dummy_loss(self):
        """Make sure a dummy loss (no computable gradient) gives an error."""
        with self.assertRaises(ValueError):
            stadv.optimization.lbfgs(
                self.loss_dummy, self.flows, flows_x0=self.example_flow
            )

    def test_overwriting_optimized_function(self):
        """Make sure we cannot overwrite argument defining the function to
        optimize."""
        with self.assertRaises(ValueError):
            stadv.optimization.lbfgs(
                self.loss_dummy, self.flows, flows_x0=self.example_flow,
                fmin_l_bfgs_b_extra_kwargs={'func': np.sqrt}
            )

if __name__ == '__main__':
    tf.test.main()


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Jun  8 00:59:58 2019

@author: atekawade
"""
import os
import numpy as np
import matplotlib as mpl
#mpl.use('Agg')
import matplotlib.pyplot as plt
import h5py
import sys
from tensorflow import keras
from IPython.display import clear_output
import random
import pandas as pd
from multiprocessing import cpu_count
from ct_segnet.data_utils.data_io import Parallelize

def _norm(y):
    if (y.max() != 1.0) | (y.min() != 0.0):
        y = (y - np.min(y)) / (np.max(y) - np.min(y))
    return y.astype(np.uint8)

def data_generator(X, Y, batch_size):
    """Generator that yields randomly sampled data pairs of size batch_size.
    X, Y are DataFile object pairs of train / test / validation data.
    """
    while True:
        idxs = sorted(random.sample(range(X.d_shape[0]), batch_size))
        x = X.read_sequence(idxs)
        y = Y.read_sequence(idxs)
        y = _norm(y)

        yield (x[...,np.newaxis], y[...,np.newaxis])

class Logger(keras.callbacks.Callback):
    """An instance of Logger can be passed to keras model.fit to log stuff at epochs
    # Arguments
        model_paths    : dict, with keys = ["name", "history", "file"]
        Xtest, Ytest   : DataFile instances of test data pairs for calculating model accuracy
        df_prev        : pandas.DataFrame, if retraining, pass previous epochs data
        N              : int, autosave frequency
        n_test         : int, number of test image pairs to be sampled every epoch
    """
    def __init__(self, Xtest, Ytest, model_paths, N, df_prev = None, n_test = None):

        self.test_dg = data_generator(Xtest, Ytest, n_test)
        self.N = N
        self.df_prev = df_prev
        self.model_name = model_paths['name']
        self.model_history = model_paths['history']
        self.model_file = model_paths['file']

    def on_train_begin(self, logs={}):

        if self.df_prev is not None:
            next_epoch = self.df_prev['epoch'].tolist()[-1] + 1
            print("\n\nRestarting training from epoch # %i..."%next_epoch)

        # INITIALIZE VECTORS
        self.i = 1 if self.df_prev is None else next_epoch
        self.x = [] if self.df_prev is None else self.df_prev['epoch'].tolist()
        self.losses = [] if self.df_prev is None else self.df_prev['loss'].tolist()
        self.val_losses = [] if self.df_prev is None else self.df_prev['val_loss'].tolist()
        self.test_losses = [] if self.df_prev is None else self.df_prev['test_loss'].tolist()
        self.metric = [] if self.df_prev is None else self.df_prev['metric'].tolist()
        self.acc_zeros = [] if self.df_prev is None else self.df_prev['acc_zeros'].tolist()
        self.acc_ones = [] if self.df_prev is None else self.df_prev['acc_ones'].tolist()

        self.logs = []

    def on_epoch_end(self, epoch, logs={}):

        self.fig, self.ax = plt.subplots(1,2, figsize = (12,5))
        self.ax[0].set_xlabel('epoch')
        self.ax[0].set_ylabel('loss')
        self.ax[1].set_xlabel('epoch')
        self.ax[1].set_ylabel('metrics')

        # APPEND DATA
        self.logs.append(logs)
        self.x.append(self.i)
        self.losses.append(logs.get('loss'))
        self.val_losses.append(logs.get('val_loss'))
        x_test, y_test = next(self.test_dg)
        evaluation = self.model.evaluate(x = x_test, y = y_test, verbose = 0)
        if type(evaluation) is list:
            if len(evaluation) == 4:
                self.test_losses.append(evaluation[0])
                self.metric.append(evaluation[1])
                self.acc_zeros.append(evaluation[2])
                self.acc_ones.append(evaluation[3])
            else:
                raise ValueError("\n number of metrics provided does not match plotter's configuration.")
        else:
            raise ValueError("\n expecting metric definitions but none found.")

        print("val_loss on test data: %.4f"%self.test_losses[-1])
        print("metric evaluated on test data: %.4f"%self.metric[-1])
        print("acc_zeros: %.2f, acc_ones: %.2f"%(self.acc_zeros[-1], self.acc_ones[-1]))

        clear_output(wait=True)
        self.ax[0].plot(self.x, self.losses, label="loss")
        self.ax[0].plot(self.x, self.val_losses, label="val_loss")
        self.ax[0].plot(self.x, self.test_losses, label = "test_data val_loss")
        self.ax[0].legend(loc = 'upper right')

        self.ax[1].plot(self.x, self.acc_zeros, label = 'acc_zeros')
        self.ax[1].plot(self.x, self.acc_ones, label = 'acc_ones')
        self.ax[1].plot(self.x, self.metric, '--k', label = "IoU")
        self.ax[1].legend(loc = 'upper right')

#         plt.show()
        plt.savefig(os.path.join(self.model_history, 'Loss_plot_' + self.model_name + '.png'))
        plt.close()

        if self.i % self.N == 0:

            # Save loss data in .csv file
            df = pd.DataFrame()
            df['epoch'] = self.x
            df['loss'] = self.losses
            df['val_loss'] = self.val_losses
            df['test_loss'] = self.test_losses
            df['metric'] = self.metric
            df['acc_zeros'] = self.acc_zeros
            df['acc_ones'] = self.acc_ones
            df.set_index('epoch')
            df.to_csv(os.path.join(self.model_history,self.model_name+".csv"))

            if self.test_losses[-1] < 1.5*self.test_losses[-self.N]:
                print("Saving model at epoch # %i..."%self.i)
                self.model.save(self.model_file)
                self.model.save_weights(os.path.join(self.model_history, 'weights_' + self.model_name + '.h5'))
            else:
                print("Terminate Training: Overfitting detected from increasing test loss...")
                # self.model.stop_training = True

        self.i += 1

    def on_train_end(self, logs = {}):

        self.logs.append(logs)

def save_datasnaps(dg, model_history, n_imgs = 20):

    if not os.path.exists(os.path.join(model_history,"data_snaps")):
        os.makedirs(os.path.join(model_history,"data_snaps"))

    # Load and save random train data images
    x, y = next(dg)
    x, y = x[...,0], y[...,0]
    for jj in x.shape[0]:
        fig, ax = plt.subplots(1,2, figsize = (6,3))
        ax[0].axis('off')
        ax[0].imshow(y[jj])
        ax[1].axis('off')
        ax[1].imshow(x[jj])
        fig.suptitle("MIN: %.2f, MAX: %.2f, MEAN: %.2f"%(np.min(x[jj]), np.max(x[jj]), np.mean(x[jj])))
        plt.savefig(os.path.join(model_history,"data_snaps",'randomdata_%5d.png'%jj))
        plt.close()

    return

def ROC(thresh, y_true = None, y_pred = None):
    """Receiver Operating Characteristics (ROC) curve
    """
    y_p = np.zeros_like(y_pred)
    y_p[y_pred > thresh] = 1
    y_true = np.copy(y_true)

    TN = np.sum((1-y_true)*(1-y_p)).astype(np.float32)

    FP = np.sum((1-y_true)*y_p).astype(np.float32)

    TNR = TN / (TN + FP)
    FPR = 1 - TNR

    TP = np.sum(y_true*y_p).astype(np.float32)

    FN = np.sum(y_true*(1-y_p)).astype(np.float32)

    TPR = TP / (TP +  FN)

    return (FPR, TPR)

def calc_jac_acc(y_true, y_pred):
    """Jaccard accuracy or Intersection over Union
    """
    y_pred = np.round(np.copy(y_pred))
    jac_acc = (np.sum(y_pred*y_true) + 1) / (np.sum(y_pred) + np.sum(y_true) - np.sum(y_pred*y_true) + 1)
    return jac_acc

def fidelity(y_true, y_pred, tolerance = 0.95):
    """Fidelity is number of images with IoU > tolerance
    """

    XY = [(y_true[ii], y_pred[ii]) for ii in range(y_true.shape[0])]
    del y_true
    del y_pred

    jac_acc = np.asarray(Parallelize(XY, calc_jac_acc, procs = cpu_count()))

    mean_IoU = np.mean(jac_acc)

    jac_fid = np.zeros_like(jac_acc)
    jac_fid[jac_acc > tolerance] = 1
    jac_fid = np.sum(jac_fid).astype(np.float32) / np.size(jac_acc)

    return jac_fid, mean_IoU, jac_acc

def save_results(dg, model_results, segmenter):
    """Save some results on test images into a folder in the path to model repo
    """
    x_test, y_test = next(dg)
    y_pred = segmenter.predict(x_test)
    y_pred = np.round(y_pred)

    x_test, y_test, y_pred = x_test[...,0], y_test[...,0], y_pred[...,0]

    if not os.path.exists(os.path.join(model_results,"data_snaps")):
        os.makedirs(os.path.join(model_results,"data_snaps"))

    for ii in x_test.shape[0]:

            fig, ax = plt.subplots(1,3, figsize = (9,3))
            ax[0].axis('off')
            ax[0].imshow(x_test[ii])
            ax[1].axis('off')
            ax[1].imshow(y_test[ii])
            ax[2].axis('off')
            ax[2].imshow(y_pred[ii])

            #jac_acc = (np.sum(y_pred[ii]*y_test[ii]) + 1) / (np.sum(y_pred[ii]) + np.sum(y_test[ii]) - np.sum(y_pred[ii]*y_test[ii]) + 1)
            test_loss, jac_acc, acc_zeros, acc_ones = segmenter.evaluate(x = x_test[ii][np.newaxis,...,np.newaxis], y = y_test[ii][np.newaxis,...,np.newaxis], verbose = 0)

            fig.suptitle("IMG_MEAN: %.2f, JAC_ACC: %.2f, LOSS: %.2f"%(np.mean(x_test[ii]),jac_acc, test_loss))
            plt.savefig(os.path.join(model_results,"data_snaps",'snap_%05d.png'%ii))
            plt.close()

    return

max_stars_repo_name: decarlof/CTSegNet

max_issues_repo_name: decarlof/CTSegNet

max_forks_repo_name: decarlof/CTSegNet


text:
import sys
import numpy as np
from typing import *
import scipy.linalg as sla
import scipy.sparse as sparse
import jax.numpy.linalg as nla
import matplotlib.pyplot as mplt
import scipy.sparse.linalg as ssl
from sklearn.utils import shuffle
from sklearn.cluster import KMeans
from scipy.sparse.csr import csr_matrix
from scipy.sparse.dia import dia_matrix
from sklearn.metrics import silhouette_score
from sklearn.metrics.pairwise import rbf_kernel
from scipy.spatial.distance import squareform, pdist

EPSILON = 1e-10 # avoid DivideByZero errors
MEM_PREC = 3    # for memory profiling output
import warnings; warnings.filterwarnings('ignore')

def algo0(S: np.ndarray, k: int, seed: int=None)-> Tuple[np.ndarray, int]:
    '''
        Params:
            - S : {s1, s2, ..., sN}, where si ∊ R^d ==> S ∊ R^(n x d)
            - k : number of clusters
            - action: {'f': KMeans.fit(), 'p': KMeans.predict()}
            - preds: precomputed classifier for predictions, ignored if action != 'p'
            - seed: random seed for sampling - default is None
        Returns:
            - preds: predicted label array
    '''
    return KMeans(n_clusters=k, random_state=seed).fit_predict(np.real(S)), sys.getsizeof(S)

def algo1(S: np.ndarray, k: int, style: str='ng', seed: int=None)-> Tuple[np.ndarray, int]:
    '''
        Params:
            - S : {s1, s2, ..., sN}, where si ∊ R^d ==> S ∊ R^(n x d)
            - k : number of clusters
            - action: {'f': KMeans.fit(), 'p': KMeans.predict()}
            - preds: precomputed classifier for predictions, ignored if action != 'p'
            - style: {'ng': Ng et. al 2001, 'cho': Choromanska et al. 2013}
            - seed: random seed for sampling - default is None
        Returns:
            - preds: predicted label array
    '''
    X: csr_matrix; L: np.ndarray; global EPSILON
    A: np.ndarray = rbf_kernel(squareform(pdist(S))) # A ∊ R^(n x n)
    np.fill_diagonal(A,0)
    dvals: np.ndarray = 1/(np.sqrt(np.sum(A, axis=1)) + EPSILON)
    d: dia_matrix = sparse.dia_matrix(np.diag(dvals)) # d ∊ R^(n x n)

    if style.lower() == "ng":
        L = d@A@d                               # L ∊ R^(n x n), unnormalized
        vals, vecs = nla.eig(L)
        X = vecs[:,np.argsort(vals)[-k:]]       # X ∊ R^(n x k) --> k LARGEST eigenvectors

    elif style.lower() == "cho":
        L = np.eye(S.shape[0]) - d@A@d          # L ∊ R^(n x n), normalized
        vals, vecs = nla.eig(L)
        X = vecs[:,np.argsort(vals)[-k:]]     # X ∊ R^(n x k) --> k SMALLEST eigenvectors

    return KMeans(n_clusters=k,random_state=seed).fit_predict(np.real(X)), sys.getsizeof(A)

def algo3(S: np.ndarray, l: int, r: int,  k: int, seed: int=None)-> Tuple[np.ndarray, int]:
    '''
        Params:
            - S : {s1, s2, ..., sN}, where si ∊ R^d ==> S ∊ R^(n x d)
            - k : number of clusters
            - l : number of columns sampled
            - r : rank approximation
            - action: {'f': KMeans.fit(), 'p': KMeans.predict()}
            - preds: precomputed classifier for predictions, ignored if action != 'p'
            - style: {'ng': Ng et. al 2001, 'cho': Choromanska et al. 2013}
            - seed: random seed for sampling - default is None
        Returns:
            - preds: predicted label array
    '''
    global EPSILON; h = np.sqrt(l/S.shape[0]); p = min(l, S.shape[1])
    if not k <= r <= l <= S.shape[0]: exit("Rank cannot be larger than column sample size")
    col_idxs: np.ndarray = np.random.choice(S.shape[1], p, replace=False)
    A: np.ndarray = rbf_kernel(S)[:,col_idxs] # A ∊ R^(n x l)
    np.fill_diagonal(A,0)

    Dvals: np.ndarray = 1/(np.sqrt(np.sum(A, axis=1)) + EPSILON)
    dvals: np.ndarray = 1/(np.sqrt(np.sum(A, axis=0)) + EPSILON)
    D: dia_matrix = sparse.dia_matrix((Dvals.T, [0]), shape=(Dvals.size, Dvals.size))
    d: dia_matrix = sparse.dia_matrix((dvals.T, [0]), shape=(dvals.size, dvals.size))

    C: np.ndarray = sparse.csr_matrix(np.eye(S.shape[0],p)) - h * D@A@d # C ∊ R^(n x l), not sparse
    W: np.ndarray = C[col_idxs,:] # W ∊ R^(l x l), not sparse
    EW, UW = sla.eig(W) # EW ∊ R^(l x l), UW ∊ R^(l x l)

    r_idxs: np.ndarray = np.argsort(EW)[:r]
    UWr: dia_matrix = sparse.dia_matrix(UW[:, r_idxs]) # UWr ∊ R^(l x r)
    EWr: dia_matrix = sparse.dia_matrix(np.diag(EW[r_idxs])) # EWr ∊ R^(r x r)
    Ut: np.ndarray = h * C@UWr@ssl.inv(EWr) # Ut ∊ R^(n x r)
    Y = sla.svd(Ut)[0][:,-k:] # vh ∊ R^(n x k) --> top k *unitary* eigenvectors
    return KMeans(n_clusters=k, random_state=seed).fit_predict(np.real(Y)), sys.getsizeof(A)

def plot_clustering(X: np.ndarray, true_labels: np.ndarray, pred_labels: np.ndarray)-> None:
    '''
        Utility function: Plot predictions after clustering

        Params
            - X: Dataset to be plotted
            - true_labels: Actual label array
            - pred_labels: Predicted label array
    '''
    utls, upls = np.unique(true_labels), np.unique(pred_labels)
    (_,a), cs = mplt.subplots(2), shuffle(['b','g','r','c','m','y','k','w'])
    if upls.shape != utls.shape: upls = utls
    print(X.shape, pred_labels.shape, true_labels.shape, upls.shape, utls.shape)
    for ki in range(utls.size):
        p, t = X[pred_labels==upls[ki],:], X[true_labels==utls[ki],:]
        a[0].scatter(t[:,0], t[:,1],c=cs[ki],s=5, marker='.')
        a[1].scatter(p[:,0], p[:,1], c=cs[ki],s=5, marker='.')

    a[0].set_title('Original Clustering')
    a[1].set_title("Nystrom Clustering")
    mplt.subplots_adjust(hspace=1, wspace=1); mplt.show()

def cluster0(data: np.ndarray, labels: np.ndarray=None, n_clusters: int=None,
             fit_seed: int=None, score_seed: int=None)-> Tuple[np.ndarray, np.ndarray]:

    if labels is None: labels = np.zeros(data.shape[0])
    if n_clusters is None: n_clusters = np.unique(labels).size

    preds, size = algo0(data, n_clusters, seed=fit_seed)
    score = silhouette_score(data, preds, metric='euclidean', random_state=score_seed)
    return preds, score, size

def cluster1(data: np.ndarray, labels: np.ndarray=None, n_clusters: int=None,
             fit_seed: int=None, score_seed: int=None)-> Tuple[np.ndarray, np.ndarray]:

    if labels is None: labels = np.zeros(data.shape[0])
    if n_clusters is None: n_clusters = np.unique(labels).size

    preds, size = algo1(data, n_clusters, style='ng', seed=fit_seed)
    score = silhouette_score(data, preds, metric='euclidean', random_state=score_seed)
    return preds, score, size

def cluster3(data: np.ndarray, l: int, r: int, labels: np.ndarray=None,
             n_clusters: int=None, fit_seed: int=None, score_seed: int=None)-> Tuple[np.ndarray, np.ndarray]:

    if labels is None: labels = np.zeros(data.shape[0])
    if n_clusters is None: n_clusters = np.unique(labels).size

    preds, size = algo3(data, l, r, n_clusters, seed=fit_seed)
    score = silhouette_score(data, preds, metric='euclidean', random_state=score_seed)
    return preds, score, size

reason: import numpy,import scipy,from scipy,import jax