text:
"""
Copyright, Rinat Maksutov, 2017.
License: GNU General Public License
"""

import numpy as np
import pandas as pd

"""
Exponential moving average
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:moving_averages
Params:
    data: pandas DataFrame
    period: smoothing period
    column: the name of the column with values for calculating EMA in the 'data' DataFrame

Returns:
    copy of 'data' DataFrame with 'ema[period]' column added
"""
def ema(data, period=0, column='close'):
    data['ema' + str(period)] = data[column].ewm(ignore_na=False, min_periods=period, com=period, adjust=True).mean()

    return data

"""
Moving Average Convergence/Divergence Oscillator (MACD)
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:moving_average_convergence_divergence_macd
Params:
    data: pandas DataFrame
    period_long: the longer period EMA (26 days recommended)
    period_short: the shorter period EMA (12 days recommended)
    period_signal: signal line EMA (9 days recommended)
    column: the name of the column with values for calculating MACD in the 'data' DataFrame

Returns:
    copy of 'data' DataFrame with 'macd_val' and 'macd_signal_line' columns added
"""
def macd(data, period_long=26, period_short=12, period_signal=9, column='close'):
    remove_cols = []
    if not 'ema' + str(period_long) in data.columns:
        data = ema(data, period_long)
        remove_cols.append('ema' + str(period_long))

    if not 'ema' + str(period_short) in data.columns:
        data = ema(data, period_short)
        remove_cols.append('ema' + str(period_short))

    data['macd_val'] = data['ema' + str(period_short)] - data['ema' + str(period_long)]
    data['macd_signal_line'] = data['macd_val'].ewm(ignore_na=False, min_periods=0, com=period_signal, adjust=True).mean()

    data = data.drop(remove_cols, axis=1)

    return data

"""
Accumulation Distribution
Source: http://stockcharts.com/school/doku.php?st=accumulation+distribution&id=chart_school:technical_indicators:accumulation_distribution_line
Params:
    data: pandas DataFrame
    trend_periods: the over which to calculate AD
    open_col: the name of the OPEN values column
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'acc_dist' and 'acc_dist_ema[trend_periods]' columns added
"""
def acc_dist(data, trend_periods=21, open_col='<OPEN>', high_col='<HIGH>', low_col='<LOW>', close_col='close', vol_col='<VOL>'):
    for index, row in data.iterrows():
        if row[high_col] != row[low_col]:
            ac = ((row[close_col] - row[low_col]) - (row[high_col] - row[close_col])) / (row[high_col] - row[low_col]) * row[vol_col]
        else:
            ac = 0
        data.set_value(index, 'acc_dist', ac)
    data['acc_dist_ema' + str(trend_periods)] = data['acc_dist'].ewm(ignore_na=False, min_periods=0, com=trend_periods, adjust=True).mean()

    return data

"""
On Balance Volume (OBV)
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:on_balance_volume_obv
Params:
    data: pandas DataFrame
    trend_periods: the over which to calculate OBV
	close_col: the name of the CLOSE values column
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'obv' and 'obv_ema[trend_periods]' columns added
"""
def on_balance_volume(data, trend_periods=21, close_col='close', vol_col='<VOL>'):
    for index, row in data.iterrows():
        if index > 0:
            last_obv = data.at[index - 1, 'obv']
            if row[close_col] > data.at[index - 1, close_col]:
                current_obv = last_obv + row[vol_col]
            elif row[close_col] < data.at[index - 1, close_col]:
                current_obv = last_obv - row[vol_col]
            else:
                current_obv = last_obv
        else:
            last_obv = 0
            current_obv = row[vol_col]

        data.set_value(index, 'obv', current_obv)

    data['obv_ema' + str(trend_periods)] = data['obv'].ewm(ignore_na=False, min_periods=0, com=trend_periods, adjust=True).mean()

    return data

"""
Price-volume trend (PVT) (sometimes volume-price trend)
Source: https://en.wikipedia.org/wiki/Volume%E2%80%93price_trend
Params:
    data: pandas DataFrame
    trend_periods: the over which to calculate PVT
	close_col: the name of the CLOSE values column
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'pvt' and 'pvt_ema[trend_periods]' columns added
"""
def price_volume_trend(data, trend_periods=21, close_col='close', vol_col='<VOL>'):
    for index, row in data.iterrows():
        if index > 0:
            last_val = data.at[index - 1, 'pvt']
            last_close = data.at[index - 1, close_col]
            today_close = row[close_col]
            today_vol = row[vol_col]
            current_val = last_val + (today_vol * (today_close - last_close) / last_close)
        else:
            current_val = row[vol_col]

        data.set_value(index, 'pvt', current_val)

    data['pvt_ema' + str(trend_periods)] = data['pvt'].ewm(ignore_na=False, min_periods=0, com=trend_periods, adjust=True).mean()

    return data

"""
Average true range (ATR)
Source: https://en.wikipedia.org/wiki/Average_true_range
Params:
    data: pandas DataFrame
    trend_periods: the over which to calculate ATR
    open_col: the name of the OPEN values column
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column
	vol_col: the name of the VOL values column
	drop_tr: whether to drop the True Range values column from the resulting DataFrame

Returns:
    copy of 'data' DataFrame with 'atr' (and 'true_range' if 'drop_tr' == True) column(s) added
"""
def average_true_range(data, trend_periods=14, open_col='<OPEN>', high_col='<HIGH>', low_col='<LOW>', close_col='close', drop_tr = True):
    for index, row in data.iterrows():
        prices = [row[high_col], row[low_col], row[close_col], row[open_col]]
        if index > 0:
            val1 = np.amax(prices) - np.amin(prices)
            val2 = abs(np.amax(prices) - data.at[index - 1, close_col])
            val3 = abs(np.amin(prices) - data.at[index - 1, close_col])
            true_range = np.amax([val1, val2, val3])

        else:
            true_range = np.amax(prices) - np.amin(prices)

        data.set_value(index, 'true_range', true_range)
    data['atr'] = data['true_range'].ewm(ignore_na=False, min_periods=0, com=trend_periods, adjust=True).mean()
    if drop_tr:
        data = data.drop(['true_range'], axis=1)

    return data

"""
Bollinger Bands
Source: https://en.wikipedia.org/wiki/Bollinger_Bands
Params:
    data: pandas DataFrame
    trend_periods: the over which to calculate BB
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'bol_bands_middle', 'bol_bands_upper' and 'bol_bands_lower' columns added
"""
def bollinger_bands(data, trend_periods=20, close_col='close'):

    data['bol_bands_middle'] = data[close_col].ewm(ignore_na=False, min_periods=0, com=trend_periods, adjust=True).mean()
    for index, row in data.iterrows():

        s = data[close_col].iloc[index - trend_periods: index]
        sums = 0
        middle_band = data.at[index, 'bol_bands_middle']
        for e in s:
            sums += np.square(e - middle_band)

        std = np.sqrt(sums / trend_periods)
        d = 2
        upper_band = middle_band + (d * std)
        lower_band = middle_band - (d * std)

        data.set_value(index, 'bol_bands_upper', upper_band)
        data.set_value(index, 'bol_bands_lower', lower_band)

    return data

"""
Chaikin Oscillator
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:chaikin_oscillator
Params:
    data: pandas DataFrame
	periods_short: period for the shorter EMA (3 days recommended)
	periods_long: period for the longer EMA (10 days recommended)
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'ch_osc' column added
"""
def chaikin_oscillator(data, periods_short=3, periods_long=10, high_col='<HIGH>',
                       low_col='<LOW>', close_col='close', vol_col='<VOL>'):
    ac = pd.Series([])
    val_last = 0

    for index, row in data.iterrows():
        if row[high_col] != row[low_col]:
            val = val_last + ((row[close_col] - row[low_col]) - (row[high_col] - row[close_col])) / (row[high_col] - row[low_col]) * row[vol_col]
        else:
            val = val_last
        ac.set_value(index, val)
    val_last = val

    ema_long = ac.ewm(ignore_na=False, min_periods=0, com=periods_long, adjust=True).mean()
    ema_short = ac.ewm(ignore_na=False, min_periods=0, com=periods_short, adjust=True).mean()
    data['ch_osc'] = ema_short - ema_long

    return data

"""
Typical Price
Source: https://en.wikipedia.org/wiki/Typical_price
Params:
    data: pandas DataFrame
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'typical_price' column added
"""
def typical_price(data, high_col = '<HIGH>', low_col = '<LOW>', close_col = 'close'):

    data['typical_price'] = (data[high_col] + data[low_col] + data[close_col]) / 3

    return data

"""
Ease of Movement
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:ease_of_movement_emv
Params:
    data: pandas DataFrame
	period: period for calculating EMV
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'emv' and 'emv_ema_[period]' columns added
"""
def ease_of_movement(data, period=14, high_col='<HIGH>', low_col='<LOW>', vol_col='<VOL>'):
    for index, row in data.iterrows():
        if index > 0:
            midpoint_move = (row[high_col] + row[low_col]) / 2 - (data.at[index - 1, high_col] + data.at[index - 1, low_col]) / 2
        else:
            midpoint_move = 0

        diff = row[high_col] - row[low_col]

        if diff == 0:
			#this is to avoid division by zero below
            diff = 0.000000001

        vol = row[vol_col]
        if vol == 0:
            vol = 1
        box_ratio = (vol / 100000000) / (diff)
        emv = midpoint_move / box_ratio

        data.set_value(index, 'emv', emv)

    data['emv_ema_'+str(period)] = data['emv'].ewm(ignore_na=False, min_periods=0, com=period, adjust=True).mean()

    return data

"""
Mass Index
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:mass_index
Params:
    data: pandas DataFrame
	period: period for calculating MI (9 days recommended)
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column

Returns:
    copy of 'data' DataFrame with 'mass_index' column added
"""
def mass_index(data, period=25, ema_period=9, high_col='<HIGH>', low_col='<LOW>'):
    high_low = data[high_col] - data[low_col] + 0.000001	#this is to avoid division by zero below
    ema = high_low.ewm(ignore_na=False, min_periods=0, com=ema_period, adjust=True).mean()
    ema_ema = ema.ewm(ignore_na=False, min_periods=0, com=ema_period, adjust=True).mean()
    div = ema / ema_ema

    for index, row in data.iterrows():
        if index >= period:
            val = div[index-25:index].sum()
        else:
            val = 0
        data.set_value(index, 'mass_index', val)

    return data

"""
Average directional movement index
Source: https://en.wikipedia.org/wiki/Average_directional_movement_index
Params:
    data: pandas DataFrame
	periods: period for calculating ADX (14 days recommended)
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column

Returns:
    copy of 'data' DataFrame with 'adx', 'dxi', 'di_plus', 'di_minus' columns added
"""
def directional_movement_index(data, periods=14, high_col='<HIGH>', low_col='<LOW>'):
    remove_tr_col = False
    if not 'true_range' in data.columns:
        data = average_true_range(data, drop_tr = False)
        remove_tr_col = True

    data['m_plus'] = 0.
    data['m_minus'] = 0.

    for i,row in data.iterrows():
        if i>0:
            data.set_value(i, 'm_plus', row[high_col] - data.at[i-1, high_col])
            data.set_value(i, 'm_minus', row[low_col] - data.at[i-1, low_col])

    data['dm_plus'] = 0.
    data['dm_minus'] = 0.

    for i,row in data.iterrows():
        if row['m_plus'] > row['m_minus'] and row['m_plus'] > 0:
            data.set_value(i, 'dm_plus', row['m_plus'])

        if row['m_minus'] > row['m_plus'] and row['m_minus'] > 0:
            data.set_value(i, 'dm_minus', row['m_minus'])

    data['di_plus'] = (data['dm_plus'] / data['true_range']).ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()
    data['di_minus'] = (data['dm_minus'] / data['true_range']).ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()

    data['dxi'] = np.abs(data['di_plus'] - data['di_minus']) / (data['di_plus'] + data['di_minus'])
    data.set_value(0, 'dxi',1.)
    data['adx'] = data['dxi'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()
    data = data.drop(['m_plus', 'm_minus', 'dm_plus', 'dm_minus'], axis=1)
    if remove_tr_col:
        data = data.drop(['true_range'], axis=1)

    return data

"""
Money Flow Index (MFI)
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:money_flow_index_mfi
Params:
    data: pandas DataFrame
	periods: period for calculating MFI (14 days recommended)
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'money_flow_index' column added
"""
def money_flow_index(data, periods=14, vol_col='<VOL>'):
    remove_tp_col = False
    if not 'typical_price' in data.columns:
        data = typical_price(data)
        remove_tp_col = True

    data['money_flow'] = data['typical_price'] * data[vol_col]
    data['money_ratio'] = 0.
    data['money_flow_index'] = 0.
    data['money_flow_positive'] = 0.
    data['money_flow_negative'] = 0.

    for index,row in data.iterrows():
        if index > 0:
            if row['typical_price'] < data.at[index-1, 'typical_price']:
                data.set_value(index, 'money_flow_positive', row['money_flow'])
            else:
                data.set_value(index, 'money_flow_negative', row['money_flow'])

        if index >= periods:
            period_slice = data['money_flow'][index-periods:index]
            positive_sum = data['money_flow_positive'][index-periods:index].sum()
            negative_sum = data['money_flow_negative'][index-periods:index].sum()

            if negative_sum == 0.:
				#this is to avoid division by zero below
                negative_sum = 0.00001
            m_r = positive_sum / negative_sum

            mfi = 1-(1 / (1 + m_r))

            data.set_value(index, 'money_ratio', m_r)
            data.set_value(index, 'money_flow_index', mfi)

    data = data.drop(['money_flow', 'money_ratio', 'money_flow_positive', 'money_flow_negative'], axis=1)

    if remove_tp_col:
        data = data.drop(['typical_price'], axis=1)

    return data

"""
Negative Volume Index (NVI)
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:negative_volume_inde
Params:
    data: pandas DataFrame
	periods: period for calculating NVI (255 days recommended)
	close_col: the name of the CLOSE values column
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'nvi' and 'nvi_ema' columns added
"""
def negative_volume_index(data, periods=255, close_col='close', vol_col='<VOL>'):
    data['nvi'] = 0.

    for index,row in data.iterrows():
        if index > 0:
            prev_nvi = data.at[index-1, 'nvi']
            prev_close = data.at[index-1, close_col]
            if row[vol_col] < data.at[index-1, vol_col]:
                nvi = prev_nvi + (row[close_col] - prev_close / prev_close * prev_nvi)
            else:
                nvi = prev_nvi
        else:
            nvi = 1000
        data.set_value(index, 'nvi', nvi)
    data['nvi_ema'] = data['nvi'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()

    return data

"""
Positive Volume Index (PVI)
Source: https://www.equities.com/news/the-secret-to-the-positive-volume-index
Params:
    data: pandas DataFrame
	periods: period for calculating PVI (255 days recommended)
	close_col: the name of the CLOSE values column
	vol_col: the name of the VOL values column

Returns:
    copy of 'data' DataFrame with 'pvi' and 'pvi_ema' columns added
"""
def positive_volume_index(data, periods=255, close_col='close', vol_col='<VOL>'):
    data['pvi'] = 0.

    for index,row in data.iterrows():
        if index > 0:
            prev_pvi = data.at[index-1, 'pvi']
            prev_close = data.at[index-1, close_col]
            if row[vol_col] > data.at[index-1, vol_col]:
                pvi = prev_pvi + (row[close_col] - prev_close / prev_close * prev_pvi)
            else:
                pvi = prev_pvi
        else:
            pvi = 1000
        data.set_value(index, 'pvi', pvi)
    data['pvi_ema'] = data['pvi'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()

    return data

"""
Momentum
Source: https://en.wikipedia.org/wiki/Momentum_(technical_analysis)
Params:
    data: pandas DataFrame
	periods: period for calculating momentum
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'momentum' column added
"""
def momentum(data, periods=14, close_col='close'):
    data['momentum'] = 0.

    for index,row in data.iterrows():
        if index >= periods:
            prev_close = data.at[index-periods, close_col]
            val_perc = (row[close_col] - prev_close)/prev_close

            data.set_value(index, 'momentum', val_perc)

    return data

"""
Relative Strenght Index
Source: https://en.wikipedia.org/wiki/Relative_strength_index
Params:
    data: pandas DataFrame
	periods: period for calculating momentum
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'rsi' column added
"""
def rsi(data, periods=14, close_col='close'):
    data['rsi_u'] = 0.
    data['rsi_d'] = 0.
    data['rsi'] = 0.

    for index,row in data.iterrows():
        if index >= periods:

            prev_close = data.at[index-periods, close_col]
            if prev_close < row[close_col]:
                data.set_value(index, 'rsi_u', row[close_col] - prev_close)
            elif prev_close > row[close_col]:
                data.set_value(index, 'rsi_d', prev_close - row[close_col])

    data['rsi'] = data['rsi_u'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean() / (data['rsi_u'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean() + data['rsi_d'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean())

    data = data.drop(['rsi_u', 'rsi_d'], axis=1)

    return data

"""
Chaikin Volatility (CV)
Source: https://www.marketvolume.com/technicalanalysis/chaikinvolatility.asp
Params:
    data: pandas DataFrame
	ema_periods: period for smoothing Highest High and Lowest Low difference
	change_periods: the period for calculating the difference between Highest High and Lowest Low
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'chaikin_volatility' column added
"""
def chaikin_volatility(data, ema_periods=10, change_periods=10, high_col='<HIGH>', low_col='<LOW>', close_col='close'):
    data['ch_vol_hl'] = data[high_col] - data[low_col]
    data['ch_vol_ema'] = data['ch_vol_hl'].ewm(ignore_na=False, min_periods=0, com=ema_periods, adjust=True).mean()
    data['chaikin_volatility'] = 0.

    for index,row in data.iterrows():
        if index >= change_periods:

            prev_value = data.at[index-change_periods, 'ch_vol_ema']
            if prev_value == 0:
                #this is to avoid division by zero below
                prev_value = 0.0001
            data.set_value(index, 'chaikin_volatility', ((row['ch_vol_ema'] - prev_value)/prev_value))

    data = data.drop(['ch_vol_hl', 'ch_vol_ema'], axis=1)

    return data

"""
William's Accumulation/Distribution
Source: https://www.metastock.com/customer/resources/taaz/?p=125
Params:
    data: pandas DataFrame
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'williams_ad' column added
"""
def williams_ad(data, high_col='<HIGH>', low_col='<LOW>', close_col='close'):
    data['williams_ad'] = 0.

    for index,row in data.iterrows():
        if index > 0:
            prev_value = data.at[index-1, 'williams_ad']
            prev_close = data.at[index-1, close_col]
            if row[close_col] > prev_close:
                ad = row[close_col] - min(prev_close, row[low_col])
            elif row[close_col] < prev_close:
                ad = row[close_col] - max(prev_close, row[high_col])
            else:
                ad = 0.

            data.set_value(index, 'williams_ad', (ad+prev_value))

    return data

"""
William's % R
Source: https://www.metastock.com/customer/resources/taaz/?p=126
Params:
    data: pandas DataFrame
	periods: the period over which to calculate the indicator value
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'williams_r' column added
"""
def williams_r(data, periods=14, high_col='<HIGH>', low_col='<LOW>', close_col='close'):
    data['williams_r'] = 0.

    for index,row in data.iterrows():
        if index > periods:
            data.set_value(index, 'williams_r', ((max(data[high_col][index-periods:index]) - row[close_col]) /
                                                 (max(data[high_col][index-periods:index]) - min(data[low_col][index-periods:index]))))

    return data

"""
TRIX
Source: https://www.metastock.com/customer/resources/taaz/?p=114
Params:
    data: pandas DataFrame
	periods: the period over which to calculate the indicator value
	signal_periods: the period for signal moving average
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'trix' and 'trix_signal' columns added
"""
def trix(data, periods=14, signal_periods=9, close_col='close'):
    data['trix'] = data[close_col].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()
    data['trix'] = data['trix'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()
    data['trix'] = data['trix'].ewm(ignore_na=False, min_periods=0, com=periods, adjust=True).mean()
    data['trix_signal'] = data['trix'].ewm(ignore_na=False, min_periods=0, com=signal_periods, adjust=True).mean()

    return data

"""
Ultimate Oscillator
Source: http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:ultimate_oscillator
Params:
    data: pandas DataFrame
	period_1: the period of the first average (7 days recommended)
	period_2: the period of the second average (14 days recommended)
	period_3: the period of the third average (28 days recommended)
	high_col: the name of the HIGH values column
	low_col: the name of the LOW values column
	close_col: the name of the CLOSE values column

Returns:
    copy of 'data' DataFrame with 'ultimate_oscillator' column added
"""
def ultimate_oscillator(data, period_1=7,period_2=14, period_3=28, high_col='<HIGH>', low_col='<LOW>', close_col='close'):
    data['ultimate_oscillator'] = 0.
    data['uo_bp'] = 0.
    data['uo_tr'] = 0.
    data['uo_avg_1'] = 0.
    data['uo_avg_2'] = 0.
    data['uo_avg_3'] = 0.

    for index,row in data.iterrows():
        if index > 0:

            bp = row[close_col] - min(row[low_col], data.at[index-1, close_col])
            tr = max(row[high_col], data.at[index-1, close_col]) - min(row[low_col], data.at[index-1, close_col])

            data.set_value(index, 'uo_bp', bp)
            data.set_value(index, 'uo_tr', tr)
            if index >= period_1:
                uo_avg_1 = sum(data['uo_bp'][index-period_1:index]) / sum(data['uo_tr'][index-period_1:index])
                data.set_value(index, 'uo_avg_1', uo_avg_1)
            if index >= period_2:
                uo_avg_2 = sum(data['uo_bp'][index-period_2:index]) / sum(data['uo_tr'][index-period_2:index])
                data.set_value(index, 'uo_avg_2', uo_avg_2)
            if index >= period_3:
                uo_avg_3 = sum(data['uo_bp'][index-period_3:index]) / sum(data['uo_tr'][index-period_3:index])
                data.set_value(index, 'uo_avg_3', uo_avg_3)
                uo = (4 * uo_avg_1 + 2 * uo_avg_2 + uo_avg_3) / 7
                data.set_value(index, 'ultimate_oscillator', uo)

    data = data.drop(['uo_bp', 'uo_tr', 'uo_avg_1', 'uo_avg_2', 'uo_avg_3'], axis=1)

    return data


text:
import yaml
import json
import os
import numpy as np

class NotinRange(Exception):
    def __init__(self, message = "Values Not in range"):
        self.message = message
        super().__init__(message)

class NotinCols(Exception):
    def __init__(self, message="Values not in Columns"):
        self.message = message
        super().__init__(message)

def get_schema(schema_path="prediction_service/schema_in.json"):
    with open(schema_path) as json_file:
        schema = json.load(json_file)
    return schema

def validate_input(valid_dict=None):
    def _validate_cols(col):
        schema = get_schema()
        actual_cols = schema.keys()
        if col not in actual_cols:
            raise NotinCols
    def _validate_values(col, val):
        schema = get_schema()
        if not (schema[col]["min"] <= float(valid_dict[col]) <= schema[col]["max"]) :
            raise NotinRange

    for col, val in valid_dict.items():
        _validate_cols(col)
        _validate_values(col, val)
    return True

ext: py

max_stars_repo_name: sukilsiva/ML-WorkFlow

max_issues_repo_name: sukilsiva/ML-WorkFlow

max_forks_repo_name: sukilsiva/ML-WorkFlow

reason: import numpy


text:
# -*- coding: utf-8 -*-

"""
Initial AST validation and normalization.
"""

from __future__ import print_function, division, absolute_import

import ast
import copy
import types

from numba import error
from numba import nodes
from numba import visitors
from numba import typesystem

class NormalizeAST(visitors.NumbaTransformer):
    "Normalize AST"

    function_level = 0

    # TODO: Actually use numba.ir.normalized
    ir = types.ModuleType('numba.ir.normalized')
    vars(ir).update(vars(ast))
    vars(ir).update(vars(nodes))

    #------------------------------------------------------------------------
    # Normalization
    #------------------------------------------------------------------------

    def visit_FunctionDef(self, node):
        if self.function_level:
            return self.handle_inner_function(node)

        self.function_level += 1
        self.visitchildren(node)
        self.function_level -= 1
        return node

    def handle_inner_function(self, node):
        "Create assignment code for inner functions and mark the assignment"
        lhs = ast.Name(node.name, ast.Store())
        ast.copy_location(lhs, node)

        rhs = FuncDefExprNode(func_def=node)
        ast.copy_location(rhs, node)

        fields = rhs._fields
        rhs._fields = []
        assmnt = ast.Assign(targets=[lhs], value=rhs)
        result = self.visit(assmnt)
        rhs._fields = fields

        return result

    def visit_FunctionDef(self, node):
        #for arg in node.args:
        #    if arg.default:
        #        self.visitchildren(arg)
        if self.function_level:
            return self.handle_inner_function(node)

        self.visitchildren(node)
        return node

    def visit_ListComp(self, node):
        """
        Rewrite list comprehensions to the equivalent for loops.

        AST syntax:

            ListComp(expr elt, comprehension* generators)
            comprehension = (expr target, expr iter, expr* ifs)

            'ifs' represent a chain of ANDs
        """
        assert len(node.generators) > 0

        # Create innermost body, i.e. list.append(expr)
        # TODO: size hint for PyList_New
        list_create = ast.List(elts=[], ctx=ast.Load())
        list_create.type = typesystem.object_ # typesystem.list_()
        list_create = nodes.CloneableNode(list_create)
        list_value = nodes.CloneNode(list_create)
        list_append = ast.Attribute(list_value, "append", ast.Load())
        append_call = ast.Call(func=list_append, args=[node.elt],
                               keywords=[], starargs=None, kwargs=None)

        # Build up the loops from inwards to outwards
        body = append_call
        for comprehension in reversed(node.generators):
            # Hanlde the 'if' clause
            ifs = comprehension.ifs
            if len(ifs) > 1:
                make_boolop = lambda op1_op2: ast.BoolOp(op=ast.And(),
                                                         values=op1_op2)
                if_test = reduce(make_boolop, ifs)
            elif len(ifs) == 1:
                if_test, = ifs
            else:
                if_test = None

            if if_test is not None:
                body = ast.If(test=if_test, body=[body], orelse=[])

            # Wrap list.append() call or inner loops
            body = ast.For(target=comprehension.target,
                           iter=comprehension.iter, body=[body], orelse=[])

        expr = nodes.ExpressionNode(stmts=[list_create, body], expr=list_value)
        return self.visit(expr)

    def visit_AugAssign(self, node):
        """
        Inplace assignment.

        Resolve a += b to a = a + b. Set 'inplace_op' attribute of the
        Assign node so later stages may recognize inplace assignment.

        Do this now, so that we can correctly mark the RHS reference.
        """
        target = node.target

        rhs_target = copy.deepcopy(target)
        rhs_target.ctx = ast.Load()
        ast.fix_missing_locations(rhs_target)

        bin_op = self.ir.BinOp(rhs_target, node.op, node.value)
        assignment = self.ir.Assign([target], bin_op)
        assignment.inplace_op = node.op
        return self.visit(assignment)

    def DISABLED_visit_Compare(self, node):
        "Reduce cascaded comparisons into single comparisons"

        # Process children
        self.generic_visit(node)

        # TODO: We can't generate temporaries from subexpressions since
        # this may invalidate execution order. For now, set the type so
        # we can clone
        for c in node.comparators:
            c.type = None

        compare_nodes = []
        comparators = [nodes.CloneableNode(c) for c in node.comparators]

        # Build comparison nodes
        left = node.left
        for op, right in zip(node.ops, comparators):
            node = self.ir.Compare(left=left, ops=[op], comparators=[right])
            # We shouldn't need to type this...
            node = nodes.typednode(node, typesystem.bool_)

            left = right.clone
            compare_nodes.append(node)

        # AND the comparisons together
        boolop = lambda left, right: self.ir.BoolOp(ast.And(), [left, right])
        node = reduce(boolop, reversed(compare_nodes))

        return node

#------------------------------------------------------------------------
# Nodes
#------------------------------------------------------------------------

class FuncDefExprNode(nodes.Node):
    """
    Wraps an inner function node until the closure code kicks in.
    """

    _fields = ['func_def']

max_stars_repo_name: liuzhenhai/numba

max_issues_repo_name: liuzhenhai/numba

max_forks_repo_name: liuzhenhai/numba

reason: from numba


text:
"""
=============================================================
Receiver Operating Characteristic (ROC) with cross validation
=============================================================

ROC curves typically feature true positive rate on the Y axis, and false
positive rate on the X axis. This means that the top left corner of the plot is
the "ideal" point - a false positive rate of zero, and a true positive rate of
one. This is not very realistic, but it does mean that a larger area under the
curve (AUC) is usually better.

This example shows the ROC response of different datasets, created from K-fold
cross-validation. Taking all of these curves, it is possible to calculate the
mean area under curve, and see the variance of the curve when the
training set is split into different subsets. This roughly shows how the
classifier output is affected by changes in the training data, and how
different the splits generated by K-fold cross-validation are from one another.

.. note::

    See also :func:`sklearn.metrics.auc_score`,
             :func:`sklearn.cross_validation.cross_val_score`,
             :ref:`example_plot_roc.py`,

"""
# print(__doc__)

import numpy as np
from scipy import interp
import matplotlib.pyplot as plt

from sklearn import svm, datasets
from sklearn.metrics import roc_curve, auc
from sklearn.cross_validation import StratifiedKFold
from sklearn.metrics import roc_curve, auc
from sklearn import metrics
from sys import argv
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression,  RandomizedLogisticRegression
from sklearn.svm import SVC,LinearSVC
from sklearn.grid_search import GridSearchCV
from sklearn.cross_validation import StratifiedKFold,cross_val_score,StratifiedShuffleSplit

from sklearn.feature_selection import RFE, RFECV, SelectFdr,SelectFwe,SelectKBest

from sklearn.pipeline import Pipeline
from sklearn.metrics import f1_score
from PipeTasks import Get_yPred,balance_weights
from collections import Counter
from sklearn.dummy import DummyClassifier
import os, fnmatch

from sklearn.preprocessing import LabelEncoder
import numpy as np

from sklearn import svm, datasets
from sklearn.metrics import roc_curve, auc
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier
import os, fnmatch
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from PipeTasks import Get_yPred,balance_weights
###############################################################################
# Data IO and generation

# os.chdir(r'/a/fr-05/vol/protein/danofer/ProtFeat/feat_extract/test_seq/Thermophile')
##os.chdir(r'/cs/prt3/danofer/ProtFeat/feat_extract/test_seq/NP/SP_Cleaved+NP+Neg_Big')

filePaths = ['/cs/prt3/danofer/ProtFeat/feat_extract/test_seq/NP/SP_Cleaved+NP+Neg_Big','/a/fr-05/vol/protein/danofer/ProtFeat/feat_extract/test_seq/Thermophile']
fileNames = ['Neuropeptide Precursors','Thermophiles']
lineColours = [(':','red'),('-','blue'),('--','green')]

plt.figure(dpi=270)

n=-1
for filepath in filePaths:
    n +=1
    fileName=fileNames[n]

    os.chdir(filepath)
    df = pd.read_csv('trainingSetFeatures.csv')
    ##    df.drop('proteinname',axis=1, inplace=True)
    feature_cols = [col for col in df.columns if col not in ['classname','Id','proteinname']]
    X=df[feature_cols].values
    y=df.classname.values

    Fwe = SelectFwe(alpha=0.01).fit(X,y)
    X=Fwe.transform(X)

    le = LabelEncoder()
    y = le.fit_transform(y)

    #Orig
    # X, y = X[y != 2], y[y != 2]  #Orig
    n_samples, n_features = X.shape #Orig

    n_classes=len(set(y))
    target_names=list(le.classes_)
    print ("n_classes",n_classes,"target_names",target_names)

    ###############################################################################
    # Classification and ROC analysis

    # Run classifier with cross-validation and plot ROC curves
    cv = StratifiedKFold(y, n_folds=13,shuffle=True) #Maybe try shuffle=True ?

    'Best hyper param  classifiers For Thermophiles:'
    # svm=SVC(C=50, cache_size=1500, class_weight='auto', gamma=0.0, kernel='rbf', probability=True)
    # rf = RandomForestClassifier(n_jobs=-1,min_samples_split= 3, max_features= 0.4, min_samples_leaf= 2, n_estimators= 200, max_depth= 8)

    'Best hyper param  classifiers For NP:'
    svm =SVC(C=50, cache_size=1800, class_weight='auto',gamma=0.0, kernel='rbf', max_iter=-1, probability=True )
    rf = RandomForestClassifier(max_depth= None, min_samples_split= 3, min_samples_leaf= 1, n_estimators= 250,  n_jobs= -1, max_features= "auto")

    classifier = svm

    mean_tpr = 0.0
    mean_fpr = np.linspace(0, 1, 100)
    all_tpr = []

    for i, (train, test) in enumerate(cv):
        probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test])
        # Compute ROC curve and area the curve
        fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
        mean_tpr += interp(mean_fpr, fpr, tpr)
        mean_tpr[0] = 0.0
        roc_auc = auc(fpr, tpr)
        # plt.plot(fpr, tpr, lw=1, label='ROC fold %d (area = %0.2f)' % (i, roc_auc))
    mean_tpr /= len(cv)
    mean_tpr[-1] = 1.0
    mean_auc = auc(mean_fpr, mean_tpr)

    # plt.plot(mean_fpr, mean_tpr, 'k--',
    #          label='%s: Mean ROC (area = %0.2f)' % (fileName,mean_auc), lw=2)
    plt.plot(mean_fpr, mean_tpr, lineColours[n][0],
             label='%s \n Mean ROC (area = %0.2f)' % (fileName,mean_auc),lw=2,
             color=lineColours[n][1])
    # sns.plot(mean_fpr, mean_tpr)

plt.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), label='Luck')

# mean_tpr /= len(cv)
# mean_tpr[-1] = 1.0
# mean_auc = auc(mean_fpr, mean_tpr)
# plt.plot(mean_fpr, mean_tpr, 'k--',
#          label='%s: Mean ROC (area = %0.2f)' % (fileName,mean_auc), lw=2)
# plt.plot(mean_fpr, [i/2 for i in mean_tpr], 'k--',
#          label='%s: YYY ROC (area = %0.2f)' % (fileName,mean_auc), lw=2, color='red')

plt.xlim([-0.01, 1.02])
plt.ylim([-0.01, 1.02])
plt.grid('off')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC (Receiver operating characteristic) Curves')
plt.legend(loc="lower right")
plt.tight_layout()
plt.ion()
plt.show()

reason: import numpy,from scipy


text:
# encoding: utf-8

# (c) 2019 Open Risk, all rights reserved
#
# correlationMatrix is licensed under the Apache 2.0 license a copy of which is included
# in the source distribution of correlationMatrix. This is notwithstanding any licenses of
# third-party software included in this distribution. You may not use this file except in
# compliance with the License.
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import pandas as pd
import correlationMatrix as cm

"""
This module provides functionality for the creation of various correlation matrices
or correlated timeseries data (in pandas dataframe format)

Useful for controlled experiments where the generative process in fully known

"""

def generate_random_matrix(n=10):
    """
        Produce a random matrix for testing purposes
        The simple method currently implemented uses an independent factor model with random loadings
        Y = b F + e
    """

    b = np.random.uniform(low=-1.0, high=1.0, size=n)
    C = np.outer(b, b)
    for i in range(n):
        C[i, i] = 1
    myMatrix = cm.CorrelationMatrix(values=C)
    return myMatrix

def multivariate_normal(correlationmatrix, sample):
    """
    Generate samples sampling from a multivariate normal.

    Suitable for testing cohorting algorithms and duration based estimators.

    The data format is a table with columns per entity ID

    :param correlationmatrix: The correlatio matrix use for the simulation
    :param int n: The number of distinct entities to simulate
    :param int sample: The number of samples to simulate
    :rtype: pandas dataframe

    .. note:: This emulates typical downloads of data from market data API's

    """

    mean = np.zeros(correlationmatrix.dimension)
    data = np.random.multivariate_normal(mean, correlationmatrix.matrix, sample)
    columns = ['S' + str(i) for i in range(0, correlationmatrix.dimension)]
    return pd.DataFrame(data, columns=columns)

def capm_model(n, b, sample):
    """
    Generate samples from a stylized CAPM model

    Suitable for testing estimation algorithms

    The data format is a table with columns per entity ID and a final market factor

    :param int n: The number of distinct entities to simulate
    :param list: A list of loading to the market factor
    :param int sample: The number of samples to simulate
    :rtype: pandas dataframe

    .. note:: This emulates typical downloads of data from market data API's

    """
    if len(b) != n:
        print("Inconsistent input values, length of loadings matrix differs from simulated entities")
        exit()

    errors = np.random.normal(loc=0.0, scale=1.0, size=(n, sample))
    market_factor = np.random.normal(loc=0.0, scale=1.0, size=(1, sample))
    returns = np.outer(b, market_factor) + errors
    print(returns.shape)
    print(market_factor.shape)
    print(returns.shape)
    data = np.append(returns, market_factor, 0)
    print(data.shape)
    columns = ['S' + str(i) for i in range(0, n)]
    columns.append('Market Factor')
    print(columns)
    print(data[:, :5])
    df = pd.DataFrame(data.transpose(), columns=columns)
    return df

def apt_model(n, b, m, rho, sample):
    """
    Generate samples from a stylized APT model

    Suitable for testing estimation algorithms

    The data format is a table with columns per entity ID and market factors

    :param int n: The number of distinct entities to simulate
    :param list b: A list of loading to the market factor
    :param int m: The number of market factors
    :param rho: The correlation matrix to use for the simulation of the market factors
    :param int sample: The number of samples to simulate
    :rtype: pandas dataframe

    .. note:: This emulates typical downloads of data from market data API's

    """
    if len(b) != m:
        print("Inconsistent input values, length of loadings matrix differs from simulated market factors")
        exit()

    if rho.dimension != m:
        print("Inconsistent input values, dimension of factor correlation matrix differs from simulated factors")
        exit()

    mean = np.zeros(rho.dimension)
    market_factors = np.random.multivariate_normal(mean, rho.matrix, sample).transpose()

    errors = np.random.normal(loc=0.0, scale=1.0, size=(n, sample))

    print(market_factors.shape)
    print(errors.shape)

    returns = np.dot(b, market_factors) + errors
    print(returns.shape)

    data = np.append(returns, market_factors, 0)
    print(data.shape)
    columns = ['S' + str(i) for i in range(0, n)] + ['F' + str(i) for i in range(0, m)]
    print(columns)
    print(data[:, :5])
    df = pd.DataFrame(data.transpose(), columns=columns)
    return df

def sector_model(n, b, rho, sample):
    """
    Generate samples from a stylized Sector model in the spirit of Credit Metrics

    Suitable for testing estimation algorithms

    The data format is a table with columns per entity ID and market factors

    :param int n: The number of distinct entities to simulate per sector
    :param list b: A list of loadings of the n entities to their market factor
    :param int m: The number of market factors
    :param rho: The correlation matrix to use for the simulation of the market factors
    :param int sample: The number of samples to simulate
    :rtype: pandas dataframe

    """
    # The loadings array
    b = np.array(b)

    # Sector returns on basis of input correlation matrix
    mean = np.zeros(rho.dimension)
    market_factors = np.random.multivariate_normal(mean, rho.matrix, sample).transpose()
    m = rho.dimension

    # Entity return residuals (m * n)
    errors = np.random.normal(loc=0.0, scale=1.0, size=(n * m, sample))

    # Initialize returns array (n*m)
    returns = np.zeros((n * m, sample))
    for sector in range(m):
        for entity in range(n):
            for s in range(sample):
                return_id = entity + n * sector
                returns[return_id][s] = b[entity] * market_factors[sector][s] + errors[return_id][s]

    # put entity and sector returns together
    data = np.append(returns, market_factors, 0)

    columns = ['S' + str(i) for i in range(0, n * m)] + ['F' + str(i) for i in range(0, m)]
    df = pd.DataFrame(data.transpose(), columns=columns)
    print(df.head(5))
    return df

max_stars_repo_name: open-risk/correlationMatrix

max_issues_repo_name: open-risk/correlationMatrix

max_forks_repo_name: open-risk/correlationMatrix


text:
from __future__ import print_function, division

__author__ = 'amrit'

import sys
from demo import cmd
#sys.dont_write_bytecode = True
from collections import OrderedDict
import os
from collections import Counter
from featurization import *
from ML import DT, SVM, RF, FFT1
from sklearn.model_selection import StratifiedKFold
import pickle
import numpy as np
from random import seed, shuffle
import time

ROOT=os.getcwd()
files=["pitsA", "pitsB", "pitsC", "pitsD", "pitsE", "pitsF"]
MLS=[DT,RF, SVM]#,  FFT1]
MLS_para_dic=[OrderedDict([("min_samples_split",2),("min_impurity_decrease",0.0), ("max_depth",None),
                               ("min_samples_leaf", 1)]), OrderedDict([("min_samples_split",2),
                                ("max_leaf_nodes",None), ("min_samples_leaf",1), ("min_impurity_decrease",0.0),("n_estimators",10)]),
                   OrderedDict([("C", 1.0), ("kernel", 'linear'),
                                ("degree", 3)]), OrderedDict()]

metrics=['accuracy','recall','precision','false_alarm']
features=[TFIDF, HASHING]

def readfile1(filename=''):
    dict = []
    labels=[]
    with open(filename, 'r') as f:
        for doc in f.readlines():
            try:
                row = doc.lower().split(">>>")
                dict.append(row[0].strip())
                labels.append(row[1].strip())
            except:
                pass
    count=Counter(labels)
    import operator
    key=max(count.iteritems(), key=operator.itemgetter(1))[0]

    labels=map(lambda x: 1 if x == key else 0, labels)
    return np.array(dict), np.array(labels)

def _test(res=''):
    seed(1)
    np.random.seed(1)
    path=ROOT+"/../data/preprocessed/"+res+".txt"
    raw_data,labels=readfile1(path)
    temp={}

    for i in range(5):
        ranges=range(len(labels))
        shuffle(ranges)
        raw_data=raw_data[ranges]
        labels=labels[ranges]

        for fea in features:
            if fea.__name__ not in temp:
                temp[fea.__name__] = {}
            start_time = time.time()
            corpus,_=fea(raw_data)
            end_time = time.time() - start_time

            skf = StratifiedKFold(n_splits=5)
            for train_index, test_index in skf.split(corpus, labels):
                train_data, train_labels = corpus[train_index], labels[train_index]
                test_data, test_labels = corpus[test_index], labels[test_index]

                for j, le in enumerate(MLS):
                    if le.__name__ not in temp[fea.__name__]:
                        temp[fea.__name__][le.__name__]={}

                    start_time1 = time.time()
                    _,val=MLS[j](MLS_para_dic[j], train_data, train_labels, test_data, test_labels, 'recall')
                    end_time1 = time.time() - start_time1
                    for m in metrics:
                        if m not in temp[fea.__name__][le.__name__]:
                            temp[fea.__name__][le.__name__][m]=[]
                        temp[fea.__name__][le.__name__][m].append(val[0][m])

                    if 'times' not in temp[fea.__name__][le.__name__]:
                        temp[fea.__name__][le.__name__]['times']=[]
                    else:
                        temp[fea.__name__][le.__name__]['times'].append(end_time1 + end_time)
                    if fea.__name__ not in temp[fea.__name__][le.__name__]:
                        temp[fea.__name__][le.__name__]['features']=[]
                    else:
                        temp[fea.__name__][le.__name__]['features'].append(val[1])

    with open('../dump/untuned' +res+ '.pickle', 'wb') as handle:
        pickle.dump(temp, handle)

if __name__ == '__main__':
    eval(cmd ())


text:
from lib.objects.Point import Point
from lib.objects.Organoid import Organoid
import numpy as np
import pandas as pd
from PIL import Image
import cv2 as cv

class ImgObjMpp:

    def __init__(self,path_image, path_region, path_csv):
        df_marks = pd.read_csv(path_csv)
        centers_x = df_marks["Center Col"]
        centers_y = df_marks["Center Row"]
        major_axis = df_marks["Semi Major Axis"]

        self.n_object = len(centers_x)
        self.list_object = []
        self.list_distmap = []
        self.list_reg_watersh = []
        self.list_img_local_object = []

        for i in range(0,self.n_object):
            self.list_object += [Organoid(Point(centers_x[i],centers_y[i]),major_axis[i])]

        self.matrix_img = np.uint8(cv.normalize(np.array(Image.open(path_image)), np.zeros(np.array(
            Image.open(path_image)).shape),0, 255, cv.NORM_MINMAX))
        self.matrix_regions = cv.imread(path_region,0)


text:
from convlab2.nlu.svm.multiwoz import SVMNLU
from convlab2.nlu.jointBERT.multiwoz import BERTNLU
from convlab2.nlu.milu.multiwoz import MILU
from convlab2.dst.rule.multiwoz import RuleDST
from convlab2.policy.rule.multiwoz import RulePolicy
from convlab2.nlg.template.multiwoz import TemplateNLG
from convlab2.dialog_agent import PipelineAgent, BiSession
from convlab2.evaluator.multiwoz_eval import MultiWozEvaluator
from pprint import pprint
import random
import numpy as np
import torch

sys_nlu = BERTNLU(mode='all', config_file='multiwoz_all.json',
                  model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all.zip')
# sys_nlu = SVMNLU(mode='sys')
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = RulePolicy(character='sys')
# template NLG
sys_nlg = TemplateNLG(is_user=False)
# assemble
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')

# user_nlu = sys_nlu
# user_nlu = SVMNLU(mode='all')
user_nlu = MILU(model_file="https://convlab.blob.core.windows.net/models/milu.tar.gz")
# not use dst
user_dst = None
# rule policy
user_policy = RulePolicy(character='usr')
# template NLG
user_nlg = TemplateNLG(is_user=True)
# assemble
user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user')

evaluator = MultiWozEvaluator()
sess = BiSession(sys_agent=sys_agent, user_agent=user_agent, kb_query=None, evaluator=evaluator)

random.seed(20200131)
np.random.seed(20190827)
torch.manual_seed(20200131)
sys_response = ''
sess.init_session()
print('init goal:')
pprint(sess.evaluator.goal)
print('-'*50)
for i in range(40):
    sys_response, user_response, session_over, reward = sess.next_turn(sys_response)
    print('user:', user_response)
    print('sys:', sys_response)
    print()
    if session_over is True:
        print('task complete:', user_policy.policy.goal.task_complete())
        print('task success:', sess.evaluator.task_success())
        print('book rate:', sess.evaluator.book_rate())
        print('inform precision/recall/f1:', sess.evaluator.inform_F1())
        print('-'*50)
        print('final goal:')
        pprint(sess.evaluator.goal)
        print('='*100)
        break

total_dialog = 10
random.seed(20200131)
goal_seeds = [random.randint(1,100000) for _ in range(total_dialog)]
precision = 0
recall = 0
f1 = 0
suc_num = 0
complete_num = 0
for j in range(total_dialog):
    sys_response = ''
    random.seed(goal_seeds[0])
    np.random.seed(goal_seeds[0])
    torch.manual_seed(goal_seeds[0])
    goal_seeds.pop(0)
    sess.init_session()
    # print('init goal:')
    # pprint(sess.evaluator.goal)
    # print('-'*50)
    for i in range(40):
        sys_response, user_response, session_over, reward = sess.next_turn(
            sys_response)
        # print('user:', user_response)
        # print('sys:', sys_response)
        if session_over is True:
            if sess.evaluator.task_success() == 1:
                suc_num = suc_num+1
            if user_policy.policy.goal.task_complete():
                complete_num += 1
            print('task complete:', user_policy.policy.goal.task_complete())
            print('task success:', sess.evaluator.task_success())
            print('book rate:', sess.evaluator.book_rate())
            print('inform precision/recall/f1:', sess.evaluator.inform_F1())
            stats = sess.evaluator.inform_F1()
            if(stats[0] != None):
                precision = precision+stats[0]
            if(stats[1] != None):
                recall = recall+stats[1]
            if(stats[2] != None):
                f1 = f1+stats[2]
            else:
                suc_num = suc_num-1
            # print('-'*50)
            # print('final goal:')
            # pprint(sess.evaluator.goal)
            # print('='*100)
            break
print("complete number of dialogs/tot:", complete_num/total_dialog)
print("success number of dialogs/tot:", suc_num/total_dialog)
print("average precision:", precision/total_dialog)
print("average recall:", recall/total_dialog)
print("average f1:", f1/total_dialog)


text:
import os
import numpy as np
import paddle
from ppcd.datasets import DataLoader
from ppcd.metrics import ComputAccuracy
from ppcd.utils import loss_computation
from tqdm import tqdm

def Eval(model,
         eval_data,
         batch_size=1,
         losses=None,
         threshold=0.5,
         ignore_index=255,
         show_result=True):
    dataloader = DataLoader
    data_lens = len(eval_data)
    model.eval()
    val_losses = []
    val_mious = []
    val_maccs = []
    val_mf1s = []
    val_kappas = []
    val_class_mious = 0.
    val_class_accs = 0.
    val_class_f1s = 0.
    # if isinstance(dataloader, CDataLoader):  # 这个地方待改进
    #     eval_loader = dataloader(eval_data, batch_size=batch_size, is_val=True)
    # else:
    #     eval_loader = dataloader(eval_data, batch_size=batch_size)
    eval_loader = dataloader(eval_data, batch_size=batch_size, is_val=True)
    for val_load_data in tqdm(eval_loader):
        if val_load_data is None:
            break
        val_img, val_lab = val_load_data
        val_pred_list = model(val_img)
        tmp_pred = []
        tmp_lab = []
        # 没有变化区标签的就不评估
        for v_pred, v_lab in zip(val_pred_list, val_lab):
            if np.sum(v_lab.numpy()) != 0:
                tmp_pred.append(v_pred)
                tmp_lab.append(v_lab.astype('int64'))
            else:
                data_lens -= 1
        val_pred_list = val_pred_list
        val_lab = tmp_lab
        if val_lab == []:
            continue
        # val_img = paddle.concat([val_A_img, val_B_img], axis=1)
        # val_pred_list = model(val_img)
        val_loss_list = loss_computation(
            logits_list=val_pred_list,
            labels=val_lab,
            losses=losses)
        val_loss = sum(val_loss_list)
        val_losses.append(val_loss.numpy())
        num_class = val_pred_list[0].shape[1]  # eval_loader.classes_num
        if num_class != 1:
            val_pred = paddle.argmax(val_pred_list[0], axis=1, keepdim=True, \
                                        dtype='int32')
        else:
            val_pred = (val_pred_list[0] > threshold).astype('int32')
        val_miou, val_class_miou, val_macc, val_class_acc, val_mf1, val_class_f1, val_kappa = ComputAccuracy(
            val_pred, val_lab[0], num_classes=num_class, ignore_index=ignore_index)
        val_mious.append(val_miou)
        val_maccs.append(val_macc)
        val_mf1s.append(val_mf1)
        val_kappas.append(val_kappa)
        val_class_mious += val_class_miou
        val_class_accs += val_class_acc
        val_class_f1s += val_class_f1
        # print(val_class_mious, val_class_accs, val_class_f1s)
    vcm = val_class_mious / data_lens
    vca = val_class_accs / data_lens
    vcf = val_class_f1s / data_lens
    if show_result:
        print("[Eval] loss: {:.4f}, miou: {:.4f}, class_miou: {}, acc: {:.4f}, class_acc: {}, f1: {:.4f}, class_f1: {}, kappa: {:.4f}" \
                .format(np.mean(val_losses), np.mean(val_mious), \
                str(vcm), np.mean(val_maccs), \
                str(vca), np.mean(val_mf1s), \
                str(vcf), np.mean(val_kappas)))
        return
    else:
        return np.mean(val_losses), np.mean(val_mious), vcm, np.mean(val_maccs), \
               vca, np.mean(val_mf1s), vcf, np.mean(val_kappas)

max_stars_repo_name: geoyee/PdRSCD

max_issues_repo_name: MinZHANG-WHU/PdRSCD

max_forks_repo_name: MinZHANG-WHU/PdRSCD


text:
from helper.knn import KNN
from collections import namedtuple
import networkx as nx

InitialKnnGraph = namedtuple('InitialKnnGraph', 'directed_knn_graph undirected_knn_graph distances')

def generate_initial_knn_graph(data, parameters: dict) -> InitialKnnGraph:
    directed_knn_graph, distances = __create_directed_knn_graph(database=data, k=int(parameters['k']))
    if parameters['graph-type'] == 'MkNN-Graph':
        undirected_knn_graph = __create_mutual_knn_graph(graph_directed=directed_knn_graph, distances=distances)
    elif parameters['graph-type'] == 'sym kNN-Graph':
        undirected_knn_graph = __create_symmetric_knn_graph(graph_directed=directed_knn_graph, distances=distances)
    else:
        raise AttributeError(parameters['graph-type'])
    return InitialKnnGraph(directed_knn_graph, undirected_knn_graph, distances)

def __create_directed_knn_graph(database, k: int):
    """
    returns an undirected kNN-Graph for the given dataset
    """
    neighbourhood = KNN(k=k, database=database)
    distances_cache = dict()
    graph_directed = dict()
    for index in range(len(database)):
        distances, points = neighbourhood.get_neighbourhood_points(point_index=index)
        graph_directed[index] = set()
        knn_counter = 0
        for p, d in zip(points, distances):
            distances_cache[(index, p)] = d
            distances_cache[(p, index)] = d
            graph_directed[index].add(p)
            knn_counter += 1
    return graph_directed, distances_cache

def __create_symmetric_knn_graph(graph_directed: dict, distances: dict) -> nx.Graph:
    symmetric_knn_graph = nx.Graph()
    for a in graph_directed:
        symmetric_knn_graph.add_node(a)
        for b in graph_directed[a]:
            if a == b:
                continue
            symmetric_knn_graph.add_edge(a, b, weight=distances[(a, b)])
    return symmetric_knn_graph

def __create_mutual_knn_graph(graph_directed: dict, distances: dict) -> nx.Graph:
    mutual_knn_graph = nx.Graph()
    for a in graph_directed:
        mutual_knn_graph.add_node(a)
        for b in graph_directed[a]:
            if a == b:
                continue
            if a in graph_directed[b]:
                mutual_knn_graph.add_edge(a, b, weight=distances[(a, b)])
    return mutual_knn_graph

max_stars_repo_name: sobermeier/cluster-flow

max_issues_repo_name: sobermeier/cluster-flow

max_forks_repo_name: sobermeier/cluster-flow

reason: import networkx


text:
# -*- coding: UTF-8 -*-
# @Author  : Chenyang Wang
# @Email   : THUwangcy@gmail.com

""" FPMC
Reference:
    "Factorizing Personalized Markov Chains for Next-Basket Recommendation"
    Rendle et al., WWW'2010.
CMD example:
    python main.py --model_name FPMC --emb_size 64 --lr 1e-3 --l2 1e-6 --history_max 20 \
    --dataset 'Grocery_and_Gourmet_Food'
"""

import torch
import torch.nn as nn
import numpy as np

from models.BaseModel import SequentialModel

class FPMC(SequentialModel):
    extra_log_args = ['emb_size']

    @staticmethod
    def parse_model_args(parser):
        parser.add_argument('--emb_size', type=int, default=64,
                            help='Size of embedding vectors.')
        return SequentialModel.parse_model_args(parser)

    def __init__(self, args, corpus):
        self.emb_size = args.emb_size
        super().__init__(args, corpus)

    def _define_params(self):
        self.ui_embeddings = nn.Embedding(self.user_num, self.emb_size)
        self.iu_embeddings = nn.Embedding(self.item_num, self.emb_size)
        self.li_embeddings = nn.Embedding(self.item_num, self.emb_size)
        self.il_embeddings = nn.Embedding(self.item_num, self.emb_size)

    def forward(self, feed_dict):
        self.check_list = []
        u_id = feed_dict['user_id']  # [batch_size]
        i_ids = feed_dict['item_id']  # [batch_size, -1]
        li_id = feed_dict['last_item_id']  # [batch_size]

        ui_vectors = self.ui_embeddings(u_id)
        iu_vectors = self.iu_embeddings(i_ids)
        li_vectors = self.li_embeddings(li_id)
        il_vectors = self.il_embeddings(i_ids)

        prediction = (ui_vectors[:, None, :] * iu_vectors).sum(-1) + (li_vectors[:, None, :] * il_vectors).sum(-1)
        return {'prediction': prediction.view(feed_dict['batch_size'], -1)}

    class Dataset(SequentialModel.Dataset):
        def _get_feed_dict(self, index):
            user_id, target_item = self.data['user_id'][index], self.data['item_id'][index]
            if self.phase != 'train' and self.model.test_all:
                neg_items = np.arange(1, self.corpus.n_items)
            else:
                neg_items = self.data['neg_items'][index]
            item_ids = np.concatenate([[target_item], neg_items]).astype(int)
            pos = self.data['position'][index]
            last_item_id = self.corpus.user_his[user_id][pos - 1][0]
            feed_dict = {
                'user_id': user_id,
                'item_id': item_ids,
                'last_item_id': last_item_id
            }
            return feed_dict

max_stars_repo_name: THUwangcy/Chorus

max_issues_repo_name: THUwangcy/Chorus

max_forks_repo_name: THUwangcy/Chorus


text:
from __future__ import division, print_function

from copy import copy

import numpy as np

import matplotlib.colors as colors
import matplotlib
from mpl_toolkits.axes_grid1 import make_axes_locatable

def plot_hexbins(ax,
                 data,
                 extent,
                 sample_weight=None,
                 cmap='viridis',
                 log_c=None,
                 hex_kwargs={},
                 zorder=4):
    if log_c:
        norm = colors.LogNorm()
    else:
        norm = colors.Normalize()

    if sample_weight is None:
        sample_weight = np.ones_like(data[:, 0])
    ax.hexbin(
        data[:, 0],
        data[:, 1],
        C=sample_weight,
        extent=extent,
        cmap=cmap,
        reduce_C_function=np.sum,
        norm=norm,
        zorder=zorder,
        **hex_kwargs)
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="8%", pad=0.05)
    cb = matplotlib.colorbar.ColorbarBase(cax,
                                          cmap=cmap,
                                          norm=norm)
    return cb

class Cut:
    def __init__(self, threshold, feature, comparison):
        self.ignore = False
        self.threshold = threshold
        if feature == 0:
            self.borders = [[threshold, threshold],
                            ['inf', 'inf']]
            self.feature = 0
            self.comparison = comparison
            if comparison == 'R':
                self.side = 'Left'
            elif comparison == 'L':
                self.side = 'Right'
        elif feature == 1:
            self.borders = [['inf', 'inf'],
                            [threshold, threshold]]
            self.feature = 1
            self.comparison = comparison
            if comparison == 'R':
                self.side = 'Bottom'
            elif comparison == 'L':
                self.side = 'Top'
        else:
            print(feature)
            raise ValueError

    def intersect(self, cut2):
        if cut2 is None:
            return None
        else:
            if self.comparison == 'R' and cut2.comparison == 'R':
                cut2.borders[self.feature][0] = self.threshold
                self.borders[cut2.feature][0] = cut2.threshold
            elif self.comparison == 'L' and cut2.comparison == 'L':
                cut2.borders[self.feature][1] = self.threshold
                self.borders[cut2.feature][1] = cut2.threshold
            elif self.comparison == 'R' and cut2.comparison == 'L':
                cut2.borders[self.feature][1] = self.threshold
                self.borders[cut2.feature][0] = cut2.threshold
            elif self.comparison == 'L' and cut2.comparison == 'R':
                cut2.borders[self.feature][0] = self.threshold
                self.borders[cut2.feature][1] = cut2.threshold
            return cut2

class Node:
    def __init__(self):
        self.top = None
        self.left = None
        self.right = None
        self.bottom = None
        self.level = 0
        self.cut_list = []
        self.bin_limits = None
        self.index = -2

    def add_cut(self, threshold, feature, comparison):
        self.level += 1
        new_cut = Cut(threshold, feature, comparison)
        self.cut_list.append(copy(new_cut))
        if new_cut.side == 'Bottom':
            self.bottom = new_cut
            self.left = new_cut.intersect(self.left)
            self.right = new_cut.intersect(self.right)
        elif new_cut.side == 'Top':
            self.top = new_cut
            self.left = new_cut.intersect(self.left)
            self.right = new_cut.intersect(self.right)
        elif new_cut.side == 'Left':
            self.left = new_cut
            self.bottom = new_cut.intersect(self.bottom)
            self.top = new_cut.intersect(self.top)
        elif new_cut.side == 'Right':
            self.right = new_cut
            self.bottom = new_cut.intersect(self.bottom)
            self.top = new_cut.intersect(self.top)

    def __str__(self):
        s = ''
        if self.top is None:
            s += 'Top: None\n'
        else:
            s += 'Top: %d: %f\n' % (self.top.feature,
                                    self.top.threshold)
        if self.bottom is None:
            s += 'Bottom: None\n'
        else:
            s += 'Bottom: %d: %f\n' % (self.bottom.feature,
                                       self.bottom.threshold)
        if self.left is None:
            s += 'Left: None\n'
        else:
            s += 'Left: %d: %f\n' % (self.left.feature,
                                     self.left.threshold)
        if self.right is None:
            s += 'Right: None\n'
        else:
            s += 'Right: %d: %f\n' % (self.right.feature,
                                      self.right.threshold)
        return s

    def __plot_edges__(self, ax, limits, color='k', lw=2., ls='-', zorder=200):
        x1 = self.left
        x2 = self.right
        y1 = self.bottom
        y2 = self.top
        bin_limits = [x1, x2, y1, y2]
        for i, coord in enumerate(bin_limits):
            if coord is None:
                bin_limits[i] = limits[i]
            else:
                bin_limits[i] = coord.threshold
        self.bin_limits = bin_limits
        color = color
        zorder = zorder
        ax.plot([bin_limits[0], bin_limits[0]],
                [bin_limits[2], bin_limits[3]],
                lw=lw, color=color, ls=ls, zorder=zorder)
        ax.plot([bin_limits[1], bin_limits[1]],
                [bin_limits[2], bin_limits[3]],
                lw=lw, color=color, ls=ls, zorder=zorder)
        ax.plot([bin_limits[0], bin_limits[1]],
                [bin_limits[2], bin_limits[2]],
                lw=lw, color=color, ls=ls, zorder=zorder)
        ax.plot([bin_limits[0], bin_limits[1]],
                [bin_limits[3], bin_limits[3]],
                lw=lw, color=color, ls=ls, zorder=zorder)

    def __fill__(self, ax,
                 color,
                 zorder=199):
        bin_limits = self.bin_limits
        ax.fill([bin_limits[0], bin_limits[0], bin_limits[1], bin_limits[1]],
                [bin_limits[2], bin_limits[3], bin_limits[3], bin_limits[2]],
                color=color,
                zorder=zorder)

class TreeCrawlerPlotting:
    def __init__(self, model):
        self.tree = model.tree_
        self.node_indices = list(np.where(self.tree.feature == -2)[0])
        self.threshold = self.tree.threshold
        self.children_left = self.tree.children_left
        self.children_right = self.tree.children_right
        self.feature = self.tree.feature
        self.leaf_list = []
        self.leaf_idents = []

    def start_crawl(self):
        root = Node()
        self.get_cut_call_next(0, root)

    def get_cut_call_next(self, index, node):
        if index in self.node_indices:
            node.ident = index
            self.leaf_list.append(node)
            self.leaf_idents.append(index)
        else:
            threshold = self.tree.threshold[index]
            feature = self.tree.feature[index]
            l_node = copy(node)
            r_node = copy(node)
            l_node.add_cut(threshold, feature, 'L')
            r_node.add_cut(threshold, feature, 'R')
            self.get_cut_call_next(self.children_left[index], l_node)
            self.get_cut_call_next(self.children_right[index], r_node)

    def plot(self,
             ax,
             limits=None,
             data=None,
             cb_label='Number Events',
             sample_weight=None,
             cmap='viridis',
             linecolor='w',
             linewidth=2.,
             log_c=False,
             as_hexbins=False,
             hex_kwargs={},
             zorder=5):
        if limits is None:
            limits = [np.inf, -np.inf, np.inf, -np.inf]
            for i, leaf_i in enumerate(self.node_indices):
                leaf = self.leaf_list[self.leaf_idents.index(leaf_i)]
                if leaf.left is not None:
                    limits[0] = min(leaf.left.threshold, limits[0])
                if leaf.right is not None:
                    limits[1] = max(leaf.right.threshold, limits[1])
                if leaf.bottom is not None:
                    limits[2] = min(leaf.bottom.threshold, limits[2])
                if leaf.top is not None:
                    limits[3] = max(leaf.top.threshold, limits[3])
        for i, leaf_i in enumerate(self.node_indices):
            leaf = self.leaf_list[self.leaf_idents.index(leaf_i)]
            leaf.__plot_edges__(ax, limits,
                                color=linecolor,
                                lw=linewidth,
                                ls='-',
                                zorder=zorder)
        ax.set_xlim(limits[0], limits[1])
        ax.set_ylim(limits[2], limits[3])
        if data is not None:
            if as_hexbins:
                cb = plot_hexbins(ax,
                                  data,
                                  extent=limits,
                                  sample_weight=sample_weight,
                                  cmap=cmap,
                                  log_c=log_c,
                                  hex_kwargs=hex_kwargs,
                                  zorder=zorder - 1)
            else:
                cb = self.fill_bins(ax, data,
                                    cmap=cmap,
                                    log_c=log_c,
                                    zorder=zorder)
        if cb_label is not None:
            cb.set_label(cb_label)
        return limits

    def fill_bins(self, ax, data,
                  cmap='viridis',
                  c_min=None,
                  c_max=None,
                  log_c=False,
                  zorder=199):
        cmap = matplotlib.cm.get_cmap(cmap)
        if c_min is None:
            c_min = np.min(data)
        if c_max is None:
            c_max = np.max(data)
        if log_c:
            norm = colors.LogNorm(vmin=c_min, vmax=c_max)
        else:
            norm = colors.Normalize(vmin=c_min, vmax=c_max)
        colz = cmap(norm(data))
        for i, leaf_i in enumerate(self.node_indices):
            leaf = self.leaf_list[self.leaf_idents.index(leaf_i)]
            color = colz[i]
            leaf.__fill__(ax, color, zorder=zorder - 1)

        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="8%", pad=0.05)
        cb = matplotlib.colorbar.ColorbarBase(cax,
                                              cmap=cmap,
                                              norm=norm)
        return cb

def plot_binning(ax,
                 binning,
                 X=None,
                 limits=None,
                 sample_weight=None,
                 cb_label='Number of Events',
                 cmap='viridis',
                 linecolor='0.5',
                 linewidth=1.,
                 log_c=False,
                 as_hexbins=True,
                 hex_kwargs={'rasterized': True},
                 zorder=5):
    if binning.n_dims != 2:
        raise binning.InvalidDimension
    if X is not None:
        if as_hexbins:
            data = X
            sample_weight = sample_weight
        else:
            data = binning.histogram(X, sample_weight=sample_weight)
            sample_weight = None
    else:
        data = None
    tree_crawler = TreeCrawlerPlotting(binning.tree)
    tree_crawler.start_crawl()
    limits = tree_crawler.plot(ax=ax,
                               limits=limits,
                               data=data,
                               cmap=cmap,
                               cb_label=cb_label,
                               linecolor=linecolor,
                               linewidth=linewidth,
                               log_c=log_c,
                               as_hexbins=as_hexbins,
                               hex_kwargs=hex_kwargs,
                               zorder=zorder)
    return limits

max_stars_repo_name: mbrner/funfolding

max_issues_repo_name: tudo-astroparticlephysics/funfolding

max_forks_repo_name: tudo-astroparticlephysics/funfolding


text:
from pathlib import Path
import numpy as np
import nibabel as nb
from ..freesurfer import FSInjectBrainExtracted

def test_inject_skullstrip(tmp_path):
    t1_mgz = tmp_path / "sub-01" / "mri" / "T1.mgz"
    t1_mgz.parent.mkdir(parents=True)
    # T1.mgz images are uint8
    nb.MGHImage(np.ones((5, 5, 5), dtype=np.uint8), np.eye(4)).to_filename(str(t1_mgz))

    mask_nii = tmp_path / "mask.nii.gz"
    # Masks may be in a different space (and need resampling), but should be boolean,
    # or uint8 in NIfTI
    nb.Nifti1Image(np.ones((6, 6, 6), dtype=np.uint8), np.eye(4)).to_filename(
        str(mask_nii)
    )

    FSInjectBrainExtracted(
        subjects_dir=str(tmp_path), subject_id="sub-01", in_brain=str(mask_nii)
    ).run()

    assert Path.exists(tmp_path / "sub-01" / "mri" / "brainmask.auto.mgz")
    assert Path.exists(tmp_path / "sub-01" / "mri" / "brainmask.mgz")

    # Run a second time to hit "already exists" condition
    FSInjectBrainExtracted(
        subjects_dir=str(tmp_path), subject_id="sub-01", in_brain=str(mask_nii)
    ).run()

max_stars_repo_name: dangom/niworkflows

max_issues_repo_name: dangom/niworkflows

max_forks_repo_name: dangom/niworkflows


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 22 14:19:06 2021

@author: vandervegt
"""

import numpy as np
import matplotlib.pylab as plt
import networkx as nx

# basic parameter
# beta: probability of infection
b = 0.15
# mu: probability of recovery
m = 0.07
# lambda: probability of losing immunity
l = 0.01
# p: probability of moving
p = 0.05

# try for n nodes, fully connected
# initial conditions of the form S1, I1, R1, S2, I2, R2, etc.
n = 10 # number of nodes
c = 3 # number of compartments
approx_total_number_individuals = 5000
avg_pop_size = approx_total_number_individuals/n
x = np.repeat(0,n*c) # initialized compartments for all nodes
x[0:-1:3] = np.rint(np.random.normal(1,0.5,10)*avg_pop_size) # initial all susceptible populations
x[0:3] = [x[0]*0.95, x[0]*0.05, 0] # initial infections in first node
x[x < 0] = 0
x_store = [x]
t_store = [0]
t = 0
t_final = 365*3

# draw random numbers now
rand_to_generate = 10**6
random_numbers = np.random.uniform(0,1,rand_to_generate)
random_number_count = 0

# matrix of egdes between nodes
random_mat = np.random.random((n,n))
edges_mat = np.zeros((n,n))
for i in range(n):
    for j in range(i-1):
        if (random_mat[i][j] <= 0.4 and i != j):
            edges_mat[i][j] = 1
            edges_mat[j][i] = 1 # assuming the connections go both directions

# list of reactions
rxn = np.zeros((n*(3+(n-1)*c), n*c))
for i in range(n):
    # compartment reactions
    StoI = np.repeat(0,n*c)
    StoI[i*c] = -1
    StoI[i*c+1] = 1
    rxn[i*(3+((n-1)*c))] = StoI
    ItoR = np.repeat(0,n*c)
    ItoR[i*c+1] = -1
    ItoR[i*c+2] = 1
    rxn[i*(3+((n-1)*c))+1] = ItoR
    RtoS = np.repeat(0,n*c)
    RtoS[i*c+2] = -1
    RtoS[i*c] = 1
    rxn[i*(3+((n-1)*c))+2] = RtoS
    # movement reactions
    count = 0
    for j in range(n):
        if edges_mat[i][j] == 1:
            Sitoj = np.repeat(0,n*c)
            Sitoj[i*c] = -1
            Sitoj[j*c] = 1
            rxn[i*(3+((n-1)*c)) + c + count*c] = Sitoj
            Iitoj = np.repeat(0,n*c)
            Iitoj[i*c + 1] = -1
            Iitoj[j*c + 1] = 1
            rxn[i*(3+((n-1)*c)) + c + count*c + 1] = Iitoj
            Ritoj = np.repeat(0,n*c)
            Ritoj[i*c+2] = -1
            Ritoj[j*c+2] = 1
            rxn[i*(3+((n-1)*c)) + c + count*c + 2] = Ritoj
            count += 1

# start loop over time
while t <= t_final:
    # compute propensities
    prop = np.array([])
    for i in range(n):
        # patch 1: S to I, I to R, R to S, move 1 to 2 (for each comp), move 1 to 3 (for each comp)
        N = np.sum(x[i*c:(i+1)*c])
        if N == 0:
            pinfect = np.zeros(c)
            pmove = np.zeros((n-1)*c)
        else:
            pinfect = np.array([(1-(1-b/N)**x[i*c+1])*x[i*c], m*x[i*c+1], l*x[i*c+2]])
            pmove = (n-1)*[(p/2)*(x[i*c]/N), (p/2)*(x[i*c+1]/N), (p/2)*(x[i*c+2]/N)]
            pmove = np.array(pmove)
        p_temp = np.concatenate((pinfect, pmove))
        # total propensities: concatenate lists
        prop = np.concatenate((prop,p_temp))

    prop_cumsummed = np.cumsum(prop)

    if random_number_count + 1 >= rand_to_generate:
        random_numbers = np.random.uniform(0,1,rand_to_generate)
        random_number_count = 0

    # compute time step
    dt = - np.log(random_numbers[random_number_count])*(1/prop_cumsummed[-1])
    random_number_count += 1

    t = t + dt
    t_store.append(t)

    # choose reaction
    rn = random_numbers[random_number_count]
    random_number_count += 1

    prop_cumsummed = np.divide(prop_cumsummed,prop_cumsummed[-1])

    for count in range(len(prop)):
        if rn <= prop_cumsummed[count]:
            break

    # find reaction in list from above
    current_rxn = rxn[count]
    # add reaction to population vector
    x = [sum(temp) for temp in zip(x, current_rxn)]
    x_store.append(x)

# plot results
x_store = np.array(x_store)
# plot results overall
store_total = np.zeros([c,len(t_store)])
for i in [0,1,2]:
    temp = x_store[:,i:-1:3]
    store_total[i] = np.sum(temp,axis = 1)

plt.plot(t_store,store_total[0],color='darkred',lw=2)
plt.plot(t_store,store_total[1],color='darkgreen',lw=2)
plt.plot(t_store,store_total[2],color='darkblue',lw=2)
plt.legend(["Total S","Total I","Total R"],fontsize=12)
plt.tick_params(direction='in',size=6)
plt.title("Compartments summed over all patches")

# plot results per node
plt.subplots(figsize = (14,15))
for i in range(1,n+1):
    plt.subplot(5,2,i)
    plt.plot(t_store,[item[(i-1)*c] for item in x_store],color='darkred',lw=2)
    plt.plot(t_store,[item[(i-1)*c+1] for item in x_store],color='darkgreen',lw=2)
    plt.plot(t_store,[item[(i-1)*c+2] for item in x_store],color='darkblue',lw=2)
    strS = "Patch %d: S" % i
    strI = "Patch %d: I" % i
    strR = "Patch %d: R" % i
    plt.legend([strS,strI,strR],fontsize=12)
    plt.tick_params(direction='in',size=6)

# plot network graph
G = nx.from_numpy_array(edges_mat, create_using=nx.DiGraph)
labels = {}
for i in range(n):
    labels[i] = i+1
# G.degree
plt.figure(figsize=(9, 6))
nx.draw(G, labels = labels)

max_stars_repo_name: solveigvandervegt/CoMoMetapopulations

max_issues_repo_name: solveigvandervegt/CoMoMetapopulations

max_forks_repo_name: solveigvandervegt/CoMoMetapopulations

reason: import numpy,import networkx


text:
"""
This project is created by NAMAN MADHARIA

Dataset Source: kaggle.com
"""
#-----------------------------Importing libraries-----------------------
import tkinter

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import rcParams
from matplotlib.cm import rainbow

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

#-----------------Takinf inputs Fn-----------------
def takeInput():
    inputValues = []

    bmiValues = []

    weight1=int(weight.get())
    height1=int(height.get())
#---------conversions---------------------
    age1 = ((int(age.get()) - 29)  / (77-29 ))
    print(age1)
    trestbps1 = ((int(rbp.get()) - 94)/(200-94))
    chol1 = ((int (serumChol.get()) - 126)/(564-126))
    thalach1 = ((int(thalach.get()) - 71)/(202-71))
    oldpeak1 = (int(oldpeak.get())/ (6.2))

    bmiValues.append(name.get())
    bmiValues.append(weight1)
    bmiValues.append(height1)

    inputValues.append(age1)
    inputValues.append(sex.get())
    inputValues.append(chestPain.get())
    inputValues.append(trestbps1)
    inputValues.append(chol1)
    inputValues.append(FBS.get())
    inputValues.append(ECG.get())
    inputValues.append(thalach1)
    inputValues.append(trestbps1)
    inputValues.append(oldpeak1)
    inputValues.append(slope.get())
    inputValues.append(ca.get())
    inputValues.append(thal.get())

    print(inputValues)
    print(" name weight height", bmiValues)

    print("\n")
    final_Result = knn_classifier.predict([inputValues])
    print(final_Result)

    #-----------------------BMI calc-------------------------

    bmi = bmiValues[1]/((bmiValues[2]/100)**2)
    out=""
    if (bmi < 18.5):
        out="underweight"

    elif ( bmi >= 18.5 and bmi < 24.9):
        out="healthy"

    elif ( bmi >= 24.9 and bmi < 30):
        out="overweight"

    elif ( bmi >=30):
        out="suffering from Obesity"

    print("bmi and out :",bmi,out)

#------------------Second Window config tkinter
    substituteWindow = tkinter.Tk()
    substituteWindow.geometry('640x480-15-200')
    substituteWindow.title("PREDICTION")
    substituteWindow.configure(bg='#006064')

    substituteWindow.columnconfigure(0, weight=2)
    substituteWindow.columnconfigure(1, weight=1)
    substituteWindow.columnconfigure(2, weight=2)
    substituteWindow.columnconfigure(3, weight=2)

    substituteWindow.rowconfigure(0, weight=1)
    substituteWindow.rowconfigure(1, weight=10)
    substituteWindow.rowconfigure(2, weight=10)
    substituteWindow.rowconfigure(3, weight=1)
    substituteWindow.rowconfigure(4, weight=1)
    substituteWindow.rowconfigure(5, weight=1)
    substituteWindow.rowconfigure(6, weight=1)

    if final_Result[0] == 1:
        label1 = tkinter.Label(substituteWindow, text="HEART DISEASE DETECTED !!!!!", font=('Times', -35), fg='red')
        label1.grid(row=1, column=1, columnspan=6)
        label2 = tkinter.Label(substituteWindow, text="Dear "+str(bmiValues[0])+" PLEASE VISIT NEAREST CARDIOLOGIST AT THE EARLIEST", font=('Times', -20), fg='red')
        label2.grid(row=2, column=1, columnspan=6)
        label3 = tkinter.Label(substituteWindow, text = "Your BMI is "+str(bmi)+"% and you are  "+out, font=('Times', -20))
        label3.grid(row=3, column = 2)

    else:
        label1 = tkinter.Label(substituteWindow, text="NO DETECTIOIN OF HEART DISEASES", font=('Times', -35) )
        label1.grid(row=1, column=1, columnspan=6)
        label2 = tkinter.Label(substituteWindow, text="Dear "+str(bmiValues[0]), font=('Times', -30))
        label2.grid(row=2, column=1, columnspan=6)
        label3 = tkinter.Label(substituteWindow, text = "Your BMI is "+str(bmi)+"% and you are  "+out, font=('Times', -20))
        label3.grid(row=3, column = 1)

    substituteWindow.mainloop()

#------------------------------------Readig dataset------------------
heart = pd.read_csv("heart_disease.csv")

#using min max scaler
min_max = MinMaxScaler()
columns_to_scale = ['age', 'trestbps', 'chol', 'thalach', 'oldpeak']
heart[columns_to_scale ] = min_max.fit_transform(heart[columns_to_scale])
y = heart['target']
X = heart.drop(['target'], axis = 1)

#training and testing of datasets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.33, random_state = 0)
# heart = pd.get_dummies(heart, columns = ['sex', 'cp', 'fbs', 'restecg', 'exang', 'slope', 'ca', 'thal'])

# print(len(X_train))
# len(X_test)

#---------------------knn classifier-----------------------
knn_classifier = KNeighborsClassifier(n_neighbors = 4)
knn_classifier.fit(X_train, y_train)

#-----------------------starting main window of tkinter---------------------
mainWindow = tkinter.Tk()
mainWindow.geometry('640x480-15-200')
mainWindow['padx']=20
mainWindow.title("HEART DISEASE PREDICTION")
mainWindow.configure(bg='#006064') # hash49B

mainWindow.columnconfigure(0, weight=2)
mainWindow.columnconfigure(1, weight=1)
mainWindow.columnconfigure(2, weight=2)
mainWindow.columnconfigure(3, weight=2)
mainWindow.rowconfigure(0, weight=0)
mainWindow.rowconfigure(1, weight=0)
mainWindow.rowconfigure(2, weight=1)
mainWindow.rowconfigure(3, weight=1)
mainWindow.rowconfigure(4, weight=1)
mainWindow.rowconfigure(5, weight=1)
mainWindow.rowconfigure(6, weight=1)
mainWindow.rowconfigure(7, weight=1)
mainWindow.rowconfigure(8, weight=1)
mainWindow.rowconfigure(9, weight=1)
mainWindow.rowconfigure(10, weight=10)

label1 = tkinter.Label(mainWindow, text="Enter all your details", font=('Impact', -25),bg='#006064' )
label1.grid(row=1, column=0, columnspan=6)

#-------------------------Input boxes----------------------

ageFrame = tkinter.LabelFrame(mainWindow, text="Age(in years)")
ageFrame.grid(row=3, column=0)
ageFrame.config(font=("Courier", -15))
age= tkinter.Entry(ageFrame)
age.grid(row=2, column=2, sticky='nw')

sexFrame = tkinter.LabelFrame(mainWindow, text="Sex(0-female,1-male)")
sexFrame.grid(row=3, column=1)
sexFrame.config(font=("Courier", -15))
sex= tkinter.Entry(sexFrame)
sex.grid(row=2, column=2, sticky='nw')

chestPainFrame = tkinter.LabelFrame(mainWindow, text="CP (0/1/2/3/4)")
chestPainFrame.grid(row=3, column=2)
chestPainFrame.config(font=("Courier", -15))
chestPain= tkinter.Entry(chestPainFrame)
chestPain.grid(row=2, column=2, sticky='nw')

rbpFrame = tkinter.LabelFrame(mainWindow, text="RBP (94-200)")
rbpFrame.grid(row=4, column=0)
rbpFrame.config(font=("Courier", -15))
rbp= tkinter.Entry(rbpFrame)
rbp.grid(row=2, column=2, sticky='nw')

serumCholFrame = tkinter.LabelFrame(mainWindow, text="Serum Chol")
serumCholFrame.grid(row=4, column=1)
serumCholFrame.config(font=("Courier", -15))
serumChol = tkinter.Entry(serumCholFrame)
serumChol.grid(row=2, column=2, sticky='n')

FBSFrame = tkinter.LabelFrame(mainWindow, text="Fasting BP(0-4)")
FBSFrame.grid(row=4, column=2)
FBSFrame.config(font=("Courier", -15))
FBS= tkinter.Entry(FBSFrame)
FBS.grid(row=2, column=2, sticky='nw')

ECGFrame = tkinter.LabelFrame(mainWindow, text="ECG (0/1/2)")
ECGFrame.grid(row=5, column=0)
ECGFrame.config(font=("Courier", -15))
ECG = tkinter.Entry(ECGFrame)
ECG.grid(row=2, column=2, sticky='nw')

thalachFrame = tkinter.LabelFrame(mainWindow, text="thalach(71-202)")
thalachFrame.grid(row=5, column=1)
thalachFrame.config(font=("Courier", -15))
thalach = tkinter.Entry(thalachFrame)
thalach.grid(row=2, column=2, sticky='nw')

exangFrame = tkinter.LabelFrame(mainWindow, text="exAngina(0/1)")
exangFrame.grid(row=5, column=2)
exangFrame.config(font=("Courier", -15))
exang = tkinter.Entry(exangFrame)
exang.grid(row=2, column=2, sticky='nw')

oldpeakFrame = tkinter.LabelFrame(mainWindow, text="Old Peak(int(0-6))")
oldpeakFrame.grid(row=6, column=0)
oldpeakFrame.config(font=("Courier", -15))
oldpeak = tkinter.Entry(oldpeakFrame)
oldpeak.grid(row=2, column=2, sticky='nw')

slopeFrame = tkinter.LabelFrame(mainWindow, text="Slope(0/1/2)")
slopeFrame.grid(row=6, column=1)
slopeFrame.config(font=("Courier", -15))
slope = tkinter.Entry(slopeFrame)
slope.grid(row=2, column=2, sticky='nw')

caFrame = tkinter.LabelFrame(mainWindow, text=" C. A (0/1/2/3)")
caFrame.grid(row=6, column=2)
caFrame.config(font=("Courier", -15))
ca = tkinter.Entry(caFrame)
ca.grid(row=2, column=2, sticky='nw')

thalFrame = tkinter.LabelFrame(mainWindow, text=" THAL(0/1/2/3)")
thalFrame.grid(row=7, column=1)
thalFrame.config(font=("Courier", -15))
thal = tkinter.Entry(thalFrame)
thal.grid(row=2, column=2, sticky='nw')

#---------------------------------BMI-----------------------------
nameFrame = tkinter.LabelFrame(mainWindow, text=" Full Name ")
nameFrame.grid(row=8, column=1)
nameFrame.config(font=("Courier", -15))
name = tkinter.Entry(nameFrame)
name.grid(row=2, column=2, sticky='nw')

wFrame = tkinter.LabelFrame(mainWindow, text=" Weight in kgs ")
wFrame.grid(row=7, column=0)
wFrame.config(font=("Courier", -15))
weight = tkinter.Entry(wFrame)
weight.grid(row=2, column=2, sticky='nw')

hFrame = tkinter.LabelFrame(mainWindow, text=" Height in cms ")
hFrame.grid(row=7, column=2)
hFrame.config(font=("Courier", -15))
height = tkinter.Entry(hFrame)
height.grid(row=2, column=2, sticky='nw')

#------------------------Button setup--------------------------------------
analyseButton = tkinter.Button(mainWindow, text="  PROCEED   ", font=('Impact', -30),command=takeInput,bg='red')
analyseButton.grid(row=10, column=0, columnspan=5)

mainWindow.mainloop()


text:
import numpy as np
from scipy import signal as sg
import tensorflow as tf

h = [2, 1, 0]
x = [3, 4, 5]

y = np.convolve(x, h)
print(y)

x = [6,2]
h = [1,2,5,4]

y = np.convolve(x, h, "full")  #now, because of the zero padding, the final dimension of the array is bigger
print(y)

x = [6, 2]
h = [1, 2, 5, 4]

y = np.convolve(x, h, "same")  # it is same as zero padding, but with returns an ouput with the same length as max of x or h
print(y)

I= [[255,   7,  3],
    [212, 240,  4],
    [218, 216, 230],]

g= [[-1, 1]]

print('Without zero padding \n')
print('{0} \n'.format(sg.convolve( I, g, 'valid')))
# The 'valid' argument states that the output consists only of those elements
# that do not rely on the zero-padding.

print('With zero padding \n')
print(sg.convolve( I, g))

# Using tensorflow
#Building graph

input = tf.Variable(tf.random_normal([1, 10, 10, 1]))
filter = tf.Variable(tf.random_normal([3, 3, 1, 1]))
op = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], padding='VALID')
op2 = tf.nn.conv2d(input, filter, strides=[1, 1, 1, 1], padding='SAME')

#Initialization and session
init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)

    print("Input \n")
    print('{0} \n'.format(input.eval()))
    print("Filter/Kernel \n")
    print('{0} \n'.format(filter.eval()))
    print("Result/Feature Map with valid positions \n")
    result = sess.run(op)
    print(result)
    print('\n')
    print("Result/Feature Map with padding \n")
    result2 = sess.run(op2)
    print(result2)

# Importing image
#Importing
import numpy as np
from scipy import signal
from scipy import misc
import matplotlib.pyplot as plt
from PIL import Image

im = Image.open('images/bird.jpg')  # type here your image's name

image_gr = im.convert("L")    # convert("L") translate color images into black and white
                              # uses the ITU-R 601-2 Luma transform (there are several
                              # ways to convert an image to grey scale)
print("\n Original type: %r \n\n" % image_gr)

# convert image to a matrix with values from 0 to 255 (uint8)
arr = np.asarray(image_gr)
print("After conversion to numerical representation: \n\n %r" % arr)
### Activating matplotlib for Ipython

### Plot image

imgplot = plt.imshow(arr)
imgplot.set_cmap('gray')  #you can experiment different colormaps (Greys,winter,autumn)
print("\n Input image converted to gray scale: \n")
plt.show(imgplot)

kernel = np.array([[ 0, 1, 0],
                   [ 1,-4, 1],
                   [ 0, 1, 0],])

grad = signal.convolve2d(arr, kernel, mode='same', boundary='symm')

print('GRADIENT MAGNITUDE - Feature map')

fig, aux = plt.subplots(figsize=(10, 10))
aux.imshow(np.absolute(grad), cmap='gray')

# normalization
type(grad)

grad_biases = np.absolute(grad) + 100

grad_biases[grad_biases > 255] = 255

print('GRADIENT MAGNITUDE - Feature map')

fig, aux = plt.subplots(figsize=(10, 10))
aux.imshow(np.absolute(grad_biases), cmap='gray')

# importing digit
#Importing
import numpy as np
from scipy import signal
from scipy import misc
import matplotlib.pyplot as plt
from PIL import Image

im = Image.open('images/num3.jpg')  # type here your image's name

image_gr = im.convert("L")    # convert("L") translate color images into black and white
                              # uses the ITU-R 601-2 Luma transform (there are several
                              # ways to convert an image to grey scale)
print("\n Original type: %r \n\n" % image_gr)

# convert image to a matrix with values from 0 to 255 (uint8)
arr = np.asarray(image_gr)
print("After conversion to numerical representation: \n\n %r" % arr)
### Activating matplotlib for Ipython

### Plot image
fig, aux = plt.subplots(figsize=(10, 10))
imgplot = plt.imshow(arr)
imgplot.set_cmap('gray')  #you can experiment different colormaps (Greys,winter,autumn)
print("\n Input image converted to gray scale: \n")
plt.show(imgplot)

# edge detection for digit
kernel = np.array([
                        [ 0, 1, 0],
                        [ 1,-4, 1],
                        [ 0, 1, 0],
                                     ])

grad = signal.convolve2d(arr, kernel, mode='same', boundary='symm')

print('GRADIENT MAGNITUDE - Feature map')

fig, aux = plt.subplots(figsize=(10, 10))
aux.imshow(np.absolute(grad), cmap='gray')

max_stars_repo_name: indervirbanipal/tensorflow

max_issues_repo_name: indervirbanipal/tensorflow

max_forks_repo_name: indervirbanipal/tensorflow