text:
"""Demo of how to use pandas to multiply one table by another."""
import logging
import multiprocessing
import os
import re
import sys

from osgeo import gdal
import pandas
import pygeoprocessing
import numpy
import taskgraph

gdal.SetCacheMax(2**27)

# treat this one column name as special for the y intercept
INTERCEPT_COLUMN_ID = 'intercept'
OPERATOR_FN = {
    '+': numpy.add,
    '*': numpy.multiply,
    '^': numpy.power,
}
N_CPUS = multiprocessing.cpu_count()

logging.basicConfig(
    level=logging.DEBUG,
    format=(
        '%(asctime)s (%(relativeCreated)d) %(levelname)s %(name)s'
        ' [%(funcName)s:%(lineno)d] %(message)s'),
    stream=sys.stdout)

LOGGER = logging.getLogger(__name__)
logging.getLogger('taskgraph').setLevel(logging.INFO)

def raster_rpn_calculator_op(*args_list):
    """Calculate RPN expression.

    Args:
        args_list (list): a length list of N+4 long where:
            - the first N elements are array followed by nodata
            - the N+1th element is the target nodata
            - the N+2nd  element is an RPN stack containing either
              symbols, numeric values, or an operator in OPERATOR_SET.
            - N+3rd value is a dict mapping the symbol to a dict with
              "index" in it showing where index*2 location it is in the
              args_list.
            - N+4th value is a set of symbols that if present should set their
              nodata to 0.
            - N+5th value is "conversion factor" to multiply the final result
              by if it is not None.

    Returns:
        evaluation of the RPN calculation
    """
    n = len(args_list)-5
    result = numpy.empty(args_list[0].shape, dtype=numpy.float32)
    result[:] = args_list[n]  # target nodata
    rpn_stack = list(args_list[n+1])
    info_dict = args_list[n+2]
    zero_nodata_indexes = args_list[n+3]
    conversion_factor = args_list[n+4]

    if conversion_factor is None:
        conversion_factor = 1

    valid_mask = numpy.ones(args_list[0].shape, dtype=numpy.bool)
    # build up valid mask where all pixel stacks are defined
    for index in range(0, n, 2):
        nodata_value = args_list[index+1]
        if nodata_value is not None and index//2 not in zero_nodata_indexes:
            valid_mask &= \
                ~numpy.isclose(args_list[index], args_list[index+1])

    # process the rpn stack
    accumulator_stack = []
    while rpn_stack:
        val = rpn_stack.pop(0)
        if val in OPERATOR_FN:
            operator = val
            operand_b = accumulator_stack.pop()
            operand_a = accumulator_stack.pop()
            val = OPERATOR_FN[operator](operand_a, operand_b)
            accumulator_stack.append(val)
        else:
            if isinstance(val, str):
                arg_index = info_dict[val]['index']
                if arg_index in zero_nodata_indexes:
                    nodata_mask = numpy.isclose(
                        args_list[2*arg_index],  args_list[2*arg_index+1])
                    args_list[2*arg_index][nodata_mask] = 0.0
                accumulator_stack.append(args_list[2*arg_index][valid_mask])
            else:
                accumulator_stack.append(val)

    result[valid_mask] = accumulator_stack.pop(0) * conversion_factor
    if accumulator_stack:
        raise RuntimeError(
            f'accumulator_stack not empty: {accumulator_stack}')
    return result

def mult_by_columns(
        lasso_table_path, data_dir, workspace_dir,
        base_convolution_raster_id, target_raster_id, bounding_box,
        pixel_size, target_result_path, task_graph,
        zero_nodata_symbols=None, target_nodata=numpy.finfo('float32').min,
        conversion_factor=None):
    """Calculate large regression.

    Args:
        lasso_table_path (str): path to lasso table
        data_dir (str): path to directory containing rasters in lasso
            table path
        workspace_dir (str): path to output directory, will contain
            "result.tif" after completion
        base_convolution_raster_id (str): The convolution columns in
            the lasso table have the form  [base]_[mask_type]_gs[kernel_size],
            this parameter matches [base] so it can be replaced with a
            filename of the form [target_raster_id]_[mask_type]_[kernel_size].
        target_raster_id (str): this is the base of the target raster that
            to use in the table.
        bounding_box (list): If not `None`, manual bounding box in the form
            of four  consecutive floats: "min_lng, min_lat, max_lng,
            max_lat, ex: " "-180.0, -58.3, 180.0, 81.5".
        pixel_size (tuple): desired target pixel size in raster units
        target_result_path (str): path to desired output raster
        task_graph (TaskGraph): TaskGraph object that can be used for
            scheduling.
        zero_nodata_symbols (set): set of symbols whose nodata values should be
            treated as 0.
        target_nodata (float): desired target nodata value
        conversion_factor (float): if not None, this factor is multiplied by
            the final result befor going into target

    Returns:
        None

    """
    lasso_df = pandas.read_csv(lasso_table_path, header=None)
    LOGGER.debug(f"parsing through {lasso_table_path}")
    # built a reverse polish notation stack for the operations and their order
    # that they need to be executed in
    rpn_stack = []
    first_term = True
    for row_index, row in lasso_df.iterrows():
        header = row[0]
        if header == INTERCEPT_COLUMN_ID:
            # special case of the intercept, just push it
            rpn_stack.append(float(row[1]))
        else:
            # it's an expression/coefficient row
            LOGGER.debug(f'{row_index}: {row}')
            coefficient = float(row[1])
            # put on the coefficient first since it's there, we'll multiply
            # it later
            rpn_stack.append(coefficient)

            # split out all the multiplcation terms
            product_list = header.split('*')
            for product in product_list:
                if product.startswith(base_convolution_raster_id):
                    LOGGER.debug(f'parsing out base and gs in {product}')
                    match = re.match(
                        fr'{base_convolution_raster_id}(.*)',
                        product)
                    suffix = match.group(1)
                    product = \
                        f'''{target_raster_id}{suffix}'''
                # for each multiplication term split out an exponent if exists
                if '^' in product:
                    rpn_stack.extend(product.split('^'))
                    # cast the exponent to an integer so can operate directly
                    rpn_stack[-1] = int(rpn_stack[-1])
                    # push the ^ to exponentiate the last two operations
                    rpn_stack.append('^')
                else:
                    # otherwise it's a single value
                    rpn_stack.append(product)
                # multiply this term and the last
                rpn_stack.append('*')

        # if it's not the first term we want to add the rest
        if first_term:
            first_term = False
        else:
            rpn_stack.append('+')

    LOGGER.debug(rpn_stack)

    # find the unique symbols in the expression
    raster_id_list = [
        x for x in set(rpn_stack)-set(OPERATOR_FN)
        if not isinstance(x, (int, float))]

    LOGGER.debug(raster_id_list)

    # translate symbols into raster paths and get relevant raster info
    raster_id_to_info_map = {}
    missing_raster_path_list = []
    min_size = sys.float_info.max
    bounding_box_list = []
    for index, raster_id in enumerate(raster_id_list):
        raster_path = os.path.join(data_dir, f'{raster_id}.tif')
        if not os.path.exists(raster_path):
            missing_raster_path_list.append(raster_path)
            continue
        else:
            raster_info = pygeoprocessing.get_raster_info(raster_path)
            raster_id_to_info_map[raster_id] = {
                'path': raster_path,
                'nodata': raster_info['nodata'][0],
                'index': index,
            }
            min_size = min(
                min_size, abs(raster_info['pixel_size'][0]))
            bounding_box_list.append(raster_info['bounding_box'])

    if missing_raster_path_list:
        LOGGER.error(
            f'expected the following '
            f'{"rasters" if len(missing_raster_path_list) > 1 else "raster"} given '
            f'the entries in the table, but could not find them locally:\n'
            + "\n".join(missing_raster_path_list))
        sys.exit(-1)

    LOGGER.info(
        f'raster paths:\n{str(raster_id_to_info_map)}')

    if bounding_box:
        target_bounding_box = bounding_box
    else:
        target_bounding_box = pygeoprocessing.merge_bounding_box_list(
            bounding_box_list, 'intersection')

    if not pixel_size:
        pixel_size = (min_size, -min_size)

    LOGGER.info(f'target pixel size: {pixel_size}')
    LOGGER.info(f'target bounding box: {target_bounding_box}')

    LOGGER.debug('align rasters, this might take a while')
    align_dir = os.path.join(workspace_dir, 'aligned_rasters')
    try:
        os.makedirs(align_dir)
    except OSError:
        pass

    # align rasters and cast to list because we'll rewrite
    # raster_id_to_path_map object
    for raster_id in raster_id_to_info_map:
        raster_path = raster_id_to_info_map[raster_id]['path']
        raster_basename = os.path.splitext(os.path.basename(raster_path))[0]
        aligned_raster_path = os.path.join(
            align_dir,
            f'{raster_basename}_{target_bounding_box}_{pixel_size}.tif')
        raster_id_to_info_map[raster_id]['aligned_path'] = \
            aligned_raster_path
        task_graph.add_task(
            func=pygeoprocessing.warp_raster,
            args=(
                raster_path, pixel_size, aligned_raster_path,
                'near'),
            kwargs={
                'target_bb': target_bounding_box,
                'working_dir': workspace_dir
            })

    LOGGER.info('construct raster calculator raster path band list')
    raster_path_band_list = []
    LOGGER.debug(raster_id_list)
    LOGGER.debug(raster_id_to_info_map)
    for index, raster_id in enumerate(raster_id_list):
        raster_path_band_list.append(
            (raster_id_to_info_map[raster_id]['aligned_path'], 1))
        raster_path_band_list.append(
            (raster_id_to_info_map[raster_id]['nodata'], 'raw'))
        if index != raster_id_to_info_map[raster_id]['index']:
            raise RuntimeError(
                f"indexes dont match: {index} {raster_id} "
                f"{raster_id_to_info_map}")

    zero_nodata_indexes = {
        raster_id_to_info_map[raster_id]['index']
        for raster_id in zero_nodata_symbols
        if raster_id in raster_id_to_info_map}

    raster_path_band_list.append((target_nodata, 'raw'))
    raster_path_band_list.append((rpn_stack, 'raw'))
    raster_path_band_list.append((raster_id_to_info_map, 'raw'))
    raster_path_band_list.append((zero_nodata_indexes, 'raw'))
    raster_path_band_list.append((conversion_factor, 'raw'))
    LOGGER.debug(rpn_stack)

    # wait for rasters to align
    task_graph.join()

    LOGGER.debug(raster_path_band_list)
    pygeoprocessing.raster_calculator(
        raster_path_band_list, raster_rpn_calculator_op, target_result_path,
        gdal.GDT_Float32, float(target_nodata))
    LOGGER.debug('all done with mult by raster')

ext: py

reason: import numpy


text:
# Data format and transformations
import numpy as np
import pandas as pd

# Importing data
from yahoo_finance_api2 import share
from yahoo_finance_api2.exceptions import YahooFinanceError

# Technical analysis
import ta

# Machine Learning
from sklearn.linear_model import LinearRegression
from sklearn import preprocessing, model_selection, svm

HISTORY_LENGTH=100
PREDICTIONS_DAYS=[1,5,10]
PREDICTION_PREFIX='prediction'

def initHistory(share_name):
   """
   This function generates a history from a share name

   Parameters
   ----------
   share_name: str
      Share name in yahoo format

   Returns
   -------
   A pandas with last share values and TA
   """
   df = __getLastPrices(share_name, HISTORY_LENGTH)
   if df is not None:
      df = ta.add_all_ta_features(
         df, "open", "high", "low", "close", "volume", fillna=True)
      for day in PREDICTIONS_DAYS:
         column_name = PREDICTION_PREFIX+str(day)
         df[column_name]=np.nan
      for i in range(0,len(df)):
         makePredictions(df, i)
   return df

def readHistory(file_name):
   return pd.read_csv(file_name).head(HISTORY_LENGTH)

def writeHistory(file_name, history):
   # TODO: saved data is not the same as the read one.
   history.to_csv(file_name, index=False)

def updateHistory(history, share_name):
   entry = __getLastPrices(share_name)
   if entry['timestamp'][0] not in history['timestamp']:
      df = history.append(entry, ignore_index=True)
      df = ta.add_all_ta_features(
          df, "open", "high", "low", "close", "volume", fillna=True)
      return makePredictions(df, len(history)-1)
   else:
      return None

def getPredictionError(history, days):
   column_name = PREDICTION_PREFIX + str(days)
   df = pd.DataFrame()
   df['prediction'] = history[column_name]
   df['real'] = history['close'].shift(-days)
   df.dropna(inplace=True)

   return ((df['prediction']-df['real'])/df['real']).abs().median()

def __getLastPrices(share_name, days = 1):
   """
   This function returns last values of a share

   Parameters
   ----------
   share_name: str
      Share name in yahoo format

   Returns
   -------
   A pandas with last share values and TA
   """
   my_share = share.Share(share_name)
   symbol_data = None
   try:
      symbol_data = my_share.get_historical(
         share.PERIOD_TYPE_DAY,
         days,
         share.FREQUENCY_TYPE_DAY,
         1)
   except YahooFinanceError as e:
      print(e.message)
   finally:
      if symbol_data is None:
         return None
      else:
         df = pd.DataFrame(symbol_data)
         return df

def __removePredictions(history):
   df = history
   for day in PREDICTIONS_DAYS:
      column_name = PREDICTION_PREFIX+str(day)
      if column_name in df.columns:
         df = df.drop(column_name, 1)
   return df

def __makePrediction(history, element, days=1):
   df = history.drop(['timestamp'],1)
   df = __removePredictions(df)
   df['future']= df['close'].shift(-days)
   df_dropped = df.dropna(inplace=False)

   X = np.array(df_dropped.drop('future', 1))
   Y = np.array(df_dropped['future'])
   X = preprocessing.scale(X)

   # Split data into train and test
   X_train, X_test, Y_train, Y_test = model_selection.train_test_split(
      X, Y, test_size=0.2)

   # Perform the regresion on the training data
   clf = LinearRegression()
   clf.fit(X_train, Y_train)
   to_predict = preprocessing.scale(
       np.array(df.drop('future', 1)))[element:element+1]
   prediction = clf.predict(to_predict)
   return prediction

def makePredictions(history, element):
   for day in PREDICTIONS_DAYS:
      column_name = PREDICTION_PREFIX+str(day)
      history.at[element, column_name] = __makePrediction(
          history, element=element, days=day)
   return history

if __name__ == "__main__":

   import argparse
   parser = argparse.ArgumentParser(description='Generates a prediction for a share symbol.')
   parser.add_argument('historic_file', help='historic filename')
   parser.add_argument('symbol', help='Share symbol name')
   parser.add_argument('-i', action='store_true', default=False,
                     dest='init',
                     help='Initialise the historic file')

   try:
      args = parser.parse_args()
      if args.init:
         history = initHistory(args.symbol)
      else:
         history = readHistory(args.historic_file)
         history = updateHistory(history, args.symbol)

      writeHistory(args.historic_file, history)
   except:
      pass

   import matplotlib.pyplot as plt
   from matplotlib import style
   style.use('ggplot')
   #df = pd.DataFrame(predictions)
   #df = history['close'].append(df, ignore_index=True)
   #plt.plot(df[:-N])
   #plt.plot(df[-N:])

   #plt.show()

max_stars_repo_name: jbenleo/invester

max_issues_repo_name: jbenleo/invester

max_forks_repo_name: jbenleo/invester


text:
from __future__ import print_function

import numpy as np
import rpy2.robjects as robjects
import collections
import warnings
import pickle
import time
import sys

import aelsem

def analyse_quadruple(classify_data, S_full, N, other_results_line, mB,
                      results, verbose=1):
    more_results_line = np.zeros(6+6, dtype=int)
    more_results_line[0:6] = other_results_line

    vars = other_results_line[0:4]
    S_sub = S_full[vars,:][:,vars]

    # Determine the LSEM equivalence classes with the best BIC scores
    best_class_by_type = aelsem.classify(classify_data, S=S_sub, N=N,
                                         epsilon=1e-6,
                                         get_scores_by_type=True,
                                         smooth=False)
    ##(best_DAG, best_MAG, best_BAP, best_ADMG, best_PD)
    best_class = best_class_by_type[-1]
    more_results_line[7:12] = best_class_by_type

    p = mB.shape[0]
    mO = np.eye(p, dtype=bool)
    vars_mask = np.zeros(p, dtype=bool)
    vars_mask[vars] = True
    (mB_marg, mO_marg) = aelsem.marginalized_model((mB, mO), vars_mask)
    #aelsem.print_model((mB_marg, mO_marg))
    mB_marg = mB_marg[vars,:][:,vars]
    mO_marg = mO_marg[vars,:][:,vars]
    #aelsem.print_model((mB_marg, mO_marg))
    true_class = classify_data['model_index'][aelsem.hash_model((mB_marg,
                                                                 mO_marg))]
    if verbose >= 3:
        print("True class:", true_class)
    more_results_line[6] = true_class

    return more_results_line

def analyse_file_sanity_check(more_results, results):
    num_lines = more_results.shape[0]
    agree = 0
    for i in range(num_lines):
        m_true = more_results[i,4]
        c_true2 = more_results[i,6]
        m_true2 = 0
        if c_true2 == 90:
            m_true2 = 4
        elif c_true2 == 246:
            m_true2 = 1
        elif c_true2 == 238:
            m_true2 = -1
        if m_true2 != m_true:
            print("Disagreement: {0} -> {1} vs. {2}"
                  .format(c_true2, m_true2, m_true))
        else:
            agree += 1
    print("Agreements:", agree)

def analyse_file_aggregate(more_results, results, use_column):
    num_lines = more_results.shape[0]
    for i in range(num_lines):
        best_class = more_results[i,use_column]
        results[more_results[i,4]][more_results[i,5]][best_class] += 1

def analyse_file_compute(base_filename, classify_data, results, verbose=1):
    R_filename = base_filename + ".Rdata"
    csv_filename = base_filename + "_struc.csv"
    csv2_filename = base_filename + "_sel.csv"

    robjects.r['load'](R_filename)
    #data_p = int(robjects.r['data'][robjects.r['data'].names.index('p')][0])
    data_nObs = int(robjects.r['data'][robjects.r['data'].names.index('nObs')][0])
    #data_nInt = int(robjects.r['data'][robjects.r['data'].names.index('nInt')][0])
    data_obs = np.array(robjects.r['data'][robjects.r['data'].names.index('obs')])
    #data_int = np.array(robjects.r['data'][robjects.r['data'].names.index('int')])
    #data_intpos = np.array(robjects.r['data'][robjects.r['data'].names.index('intpos')],dtype='int') - 1
    data_Lambda = np.array(robjects.r['data'][robjects.r['data'].names.index('B')])

    #print(data_p, "variables,", data_nObs, "data points")
    S_full = aelsem.sample_covariance_matrix(data_obs)
    #np.set_printoptions(threshold=np.inf)
    #print(S_full)

    mB = (np.fabs(data_Lambda) > 1e-12).T

    with warnings.catch_warnings():
        # ignore warnings about empty files
        warnings.filterwarnings("ignore")#, message="empty input file")
        other_results = np.loadtxt(csv_filename, delimiter=',', dtype=int,
                                   ndmin=2)
    #print(other_results.shape) # look at prev_results.shape[0]

    num_lines = other_results.shape[0]

    # To just report the file sizes without doing anything:
    #print(base_filename, "-", num_lines, "lines")
    #return

    more_results = np.zeros((num_lines, 6+6), dtype=int)
    for line_no in range(num_lines):
        if verbose >= 2 or (verbose and line_no % 100 == 0):
            print("File", base_filename, "- line", line_no+1,
                  "of", other_results.shape[0])
        more_results[line_no,:] = analyse_quadruple(classify_data,
                                                    S_full, data_nObs,
                                                    other_results[line_no,:],
                                                    mB, results,
                                                    verbose=verbose)
    # store more_results in file csv2_filename
    np.savetxt(csv2_filename, more_results, fmt='%1i', delimiter=',')
    print("File", base_filename, "completed")

    return more_results

def analyse_file(base_filename, classify_data, results, use_column, verbose=1):
    csv2_filename = base_filename + "_sel.csv"

    try:
        with warnings.catch_warnings():
            # ignore warnings about empty files
            warnings.filterwarnings("ignore")#, message="empty input file")
            more_results = np.loadtxt(csv2_filename, delimiter=',', dtype=int,
                                      ndmin=2)
    except IOError:
        more_results = analyse_file_compute(base_filename, classify_data,
                                            results, verbose)

    analyse_file_aggregate(more_results, results, use_column)

def main(argv):
    # Usage: run with just one command-line argument (e.g. p10_final) to
    # analyse all data in that experiment (consisting of 100 runs). To
    # parallelize, add an additional argument 0-99 to specify which of these
    # runs to analyse. The results for each run will be stored in a file, so
    # that running the command without the numeric argument afterwards will
    # just load those results and summarize them.

    #exp_name = "p10_final"
    #exp_name = "p30_final"
    #exp_name = "p50_final"
    exp_name = argv[1]

    if len(argv) >= 3:
        file_numbers = [int(argv[2])]
        report_aggregates = False
    else:
        file_numbers = range(100)
        report_aggregates = True

    use_column = 11 # Change this to 8 to get results for MAGs instead
    columns = {6: "truth", 7: "DAGs", 8: "MAGs", 9: "BAPs", 10: "ADMGs",
               11: "all path diagrams"}
    report_details = True #False

    store_aggregates = False

    verbose = 1
    # Meaning of verbose:
    # 0: only output notification when file is done
    # 1: current line number (every hundred lines) only
    # 2: above for every line, plus current model class being tested
    # 3: also some intermediate results

    base_filename = "ystruct/experiments_uai2015/exp_{0}/simulData".format(exp_name)

    classify_data = aelsem.classify_prepare(4, allow_cycles=False)

    results = {}
    for i in [0,-1,1,4]:
        results[i] = {}
        for j in range(-1,5):
            results[i][j] = collections.Counter()

    starting_time = time.clock()
    for file_no in file_numbers:
        analyse_file(base_filename + str(file_no), classify_data, results,
                     use_column, verbose=verbose)
    time_used = time.clock() - starting_time
    num_selections = 0
    for i in [0,-1,1,4]:
        for j in range(-1,5):
            num_selections += sum(results[i][j].values())
    print("Total time:", time_used)
    print("Number of model selections:", num_selections)
    if num_selections > 0:
        print("Average time per selection:", time_used / num_selections)

    if store_aggregates:
        pickle.dump(results, open("ystr_results_{0}.p".format(exp_name), "wb"))
        # To load: results = pickle.load(open("ystr-results.p", "rb"))

    if not report_aggregates:
        return

    print("*** RESULTS FOR: {0}, {1} ***".format(argv[1], columns[use_column]))

    # numbering used in csv files
    m_name = {0: "nothing", 1: "extended Y", -1: "extended Y (refl)",
              2: "Y [0]", 3: "Y [1]", 4: "Y [2]"}
    # numbering used by aelsem.py
    model_name = {90: "Y", 105: "upside-down Y",
                  246: "extended Y", 238: "extended Y (refl)",
                  243: "Verma Y", 233: "Verma Y (refl)",
                  247: "tetrad + indep"}
    test_TP = np.zeros(5, dtype=int)
    test_FP = np.zeros(5, dtype=int)
    filter_TP = np.zeros(5, dtype=int)
    filter_FP = np.zeros(5, dtype=int)
    for m_true in [0,1,-1,4]:
        for m_indep in [0,1,-1,2,3,4]:
            c = results[m_true][m_indep]
            if len(c) == 0:
                continue
            total = sum(c.values())
            if report_details:
                print("True structure:", m_name[m_true],
                      "\tStructure found by tests:", m_name[m_indep])

            # If 4 nodes test positively as a Y-structure, then so would the 4
            # nodes with a and b swapped, but the second 4-tuple isn't reported
            # in the input file
            mult = 2 if m_indep >= 2 else 1
            if m_indep != 0:
                if m_true == m_indep or m_true == 4:
                    test_TP[abs(m_indep)] += mult * total
                else:
                    test_FP[abs(m_indep)] += mult * total

            for model, number in c.most_common():
                if report_details:
                    print("    {0}/{1} times model class {2}"
                          .format(number, total, model), end="")
                    if model in model_name:
                        print(" (", model_name[model], ")", sep="")
                    else:
                        print("")
                    aelsem.print_model_class(classify_data['model_classes'][model], indent=2)
                m_sel = 0
                if model in model_name:
                    if model_name[model] == "Y":
                        m_sel = 4
                    elif model_name[model] == "extended Y":
                        m_sel = 1
                    elif model_name[model] == "extended Y (refl)":
                        m_sel = -1

                m_filter = 0
                if m_indep >= 2 and m_sel == 4:
                    m_filter = m_indep
                elif m_indep != 0 and m_sel != 0:
                    if m_indep * m_sel == -1:
                        print("indep and sel disagreed on orientation of extY ({0} times here)".format(number))
                        # filter these out as well
                    elif m_indep == 1 or m_sel == 1:
                        m_filter = 1
                    else:
                        m_filter = -1
                mult = 2 if m_filter >= 2 else 1
                if m_filter != 0:
                    if m_true == m_filter or m_true == 4:
                        filter_TP[abs(m_indep)] += mult * number
                    else:
                        filter_FP[abs(m_indep)] += mult * number

    for m_indep in [3,2,1]:
        test_TP[m_indep] += test_TP[m_indep + 1]
        test_FP[m_indep] += test_FP[m_indep + 1]
        filter_TP[m_indep] += filter_TP[m_indep + 1]
        filter_FP[m_indep] += filter_FP[m_indep + 1]

    print("Results for independence test method:")
    for m_indep in [1,2,3,4]:
        print("{0}:\nTP = {1}\tFP = {2}\tprec = {3}"
              .format(m_name[m_indep], test_TP[m_indep], test_FP[m_indep],
                      test_TP[m_indep] * 1.0 / (test_TP[m_indep] + test_FP[m_indep])))
    print("Results for combined (filtering) method:")
    for m_indep in [1,2,3,4]:
        print("{0}:\nTP = {1}\tFP = {2}\tprec = {3}"
              .format(m_name[m_indep], filter_TP[m_indep], filter_FP[m_indep],
                      filter_TP[m_indep] * 1.0 / (filter_TP[m_indep] + filter_FP[m_indep])))
    print("Relative recall of filtering compared to tests:")
    for m_indep in [1,2,3,4]:
        print("{0}:\nTPfilter = {1}\tTPtests = {2}\trelative recall = {3}"
              .format(m_name[m_indep], filter_TP[m_indep], test_TP[m_indep],
                      "n/a" if test_TP[m_indep] == 0 else
                      1.0 * filter_TP[m_indep] / test_TP[m_indep]))

    print("*** END OF RESULTS FOR: {0}, {1} ***".format(argv[1], columns[use_column]))

    # Print all model classes with a known arrow X -> Y:
    #for m, model in enumerate(model_classes):
    #    marks = aelsem.model_class_marks(model)
    #    if marks[0,1] == "-->":
    #        print(m)
    #        aelsem.print_model_class(model)

if __name__ == "__main__":
    main(sys.argv)

max_stars_repo_name: caus-am/aelsem

max_issues_repo_name: caus-am/aelsem

max_forks_repo_name: caus-am/aelsem


text:
from numpy.testing import assert_allclose
from plum import dispatch

from algebra import Element

__all__ = ["approx", "a", "b", "c"]

approx = assert_allclose

# Some extra atomic elements to test with:

class A(Element):
    @dispatch
    def __eq__(self, other: "A"):
        return True

    def render(self, formatter):
        return "a"

a = A()

class B(Element):
    @dispatch
    def __eq__(self, other: "B"):
        return True

    def render(self, formatter):
        return "b"

b = B()

class C(Element):
    @dispatch
    def __eq__(self, other: "C"):
        return True

    def render(self, formatter):
        return "c"

c = C()

max_stars_repo_name: wesselb/ring

max_issues_repo_name: wesselb/algebra

max_forks_repo_name: wesselb/algebra

reason: from numpy


text:
from os import truncate
from websocket import create_connection
import json
import random
import string
import re
import csv
import numpy as np
from datetime import datetime
import pandas as pd
import matplotlib.pyplot as plt
from pandas_datareader import data, wb
from ichimoku import getValue
from ichimoku import getClose
from ichimoku import getSenkouA
from ichimoku import getSenkouB

import threading
import schedule
import time
import datetime as dt
import smtplib, ssl

class ThreadWithResult(threading.Thread):
    def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, *, daemon=None):
        def function():
            self.result = target(*args, **kwargs)
        super().__init__(group=group, target=function, name=name, daemon=daemon)

values = []
threads = []

def filter_raw_message(text):
    try:
        found = re.search('"m":"(.+?)",', text).group(1)
        found2 = re.search('"p":(.+?"}"])}', text).group(1)
    #    print(found)
    #    print(found2)
        return found1, found2
    except AttributeError:
        print("error")

def isNowInTimePeriod(startTime, endTime, nowTime):
    if startTime < endTime:
        return nowTime >= startTime and nowTime <= endTime
    else:
        #Over midnight:
        return nowTime >= startTime or nowTime <= endTime

def generateSession():
    stringLength=12
    letters = string.ascii_lowercase
    random_string= ''.join(random.choice(letters) for i in range(stringLength))
    return "qs_" +random_string

def generateChartSession():
    stringLength=12
    letters = string.ascii_lowercase
    random_string= ''.join(random.choice(letters) for i in range(stringLength))
    return "cs_" +random_string

def prependHeader(st):
    return "~m~" + str(len(st)) + "~m~" + st

def constructMessage(func, paramList):
    #json_mylist = json.dumps(mylist, separators=(',', ':'))
    return json.dumps({
        "m":func,
        "p":paramList
        }, separators=(',', ':'))

def createMessage(func, paramList):
    return prependHeader(constructMessage(func, paramList))

def sendRawMessage(ws, message):
    ws.send(prependHeader(message))

def sendMessage(ws, func, args):
    ws.send(createMessage(func, args))

def generate_csv(a,id):
    out= re.search('"s":\[(.+?)\}\]', a).group(1)
    x=out.split(',{\"')

    with open(id+'.csv', mode='w', newline='') as data_file:
        employee_writer = csv.writer(data_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

        employee_writer.writerow(['index', 'date', 'open', 'high', 'low', 'close', 'volume'])

        for xi in x:
            xi= re.split('\[|:|,|\]', xi)
           # print(xi)
            ind= int(xi[1])
            ts= datetime.fromtimestamp(float(xi[4])).strftime("%Y/%m/%d, %H:%M:%S")
            employee_writer.writerow([ind, ts, float(xi[5]), float(xi[6]), float(xi[7]), float(xi[8]), float(xi[9])])

def schedule_actions():

  # Every Monday task() is called at 20:
  # TODO CHANGE TO 9
  global bool_flag
  schedule.every(60).minutes.do(start_thread)
  signal_list = start_thread()
  # Checks whether a scheduled task is pending to run or no p.panthalattiyil@gmail.com
  while True:
    schedule.run_pending()
    time.sleep(1)

def send_mail(text):
    port = 587  # For starttls
    smtp_server = "smtp.gmail.com"
    sender_email = " "
    receiver_email = " "
    password = " "
    message = """\
    Subject: Ichimoku Sensei

    This message is sent from Python

    """ + text

    context = ssl.create_default_context()
    with smtplib.SMTP(smtp_server, port) as server:
        server.ehlo()  # Can be omitted
        server.starttls(context=context)
        server.ehlo()  # Can be omitted
        server.login(sender_email, password)
        server.sendmail(sender_email, receiver_email, message)

def job_function(id):
    try:
        if dt.datetime.today().weekday() != 5 or dt.datetime.today().weekday() != 6:
            if True:        # Initialize the headers needed for the websocket connection
                headers = json.dumps({
                    # 'Connection': 'upgrade',
                    # 'Host': 'data.tradingview.com',
                    'Origin': 'https://data.tradingview.com'
                    # 'Cache-Control': 'no-cache',
                    # 'Upgrade': 'websocket',
                    # 'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits',
                    # 'Sec-WebSocket-Key': '2C08Ri6FwFQw2p4198F/TA==',
                    # 'Sec-WebSocket-Version': '13',
                    # 'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36 Edg/83.0.478.56',
                    # 'Pragma': 'no-cache',
                    # 'Upgrade': 'websocket'
                })

                print("schedule job started")

                interval = 1
                values = []

                # TODO CHANGE TO 16

                while( 16 >= interval):
                    val = interval * 15

                    print("Value at " + str(val))
                    # Then create a connection to the tunnel
                    ws = create_connection(
                        'wss://data.tradingview.com/socket.io/websocket',headers=headers)

                    session= generateSession()
            #     print("session generated {}".format(session))

                    chart_session= generateChartSession()
            #     print("chart_session generated {}".format(chart_session))

                    # Then send a message through the tunnel
                    sendMessage(ws, "set_auth_token", ["unauthorized_user_token"])
                    sendMessage(ws, "chart_create_session", [chart_session, ""])
                    sendMessage(ws, "quote_create_session", [session])
                    sendMessage(ws,"quote_set_fields", [session,"ch","chp","current_session","description","local_description","language","exchange","fractional","is_tradable","lp","lp_time","minmov","minmove2","original_name","pricescale","pro_name","short_name","type","update_mode","volume","currency_code","rchp","rtc"])
                    sendMessage(ws, "quote_add_symbols",[session, id, {"flags":['force_permission']}])
                    sendMessage(ws, "quote_fast_symbols", [session,id])

                    #st='~m~140~m~{"m":"resolve_symbol","p":}'
                    #p1, p2 = filter_raw_message(st)
                    sendMessage(ws, "resolve_symbol", [chart_session,"symbol_1","={\"symbol\":\""+id+"\",\"adjustment\":\"splits\",\"session\":\"extended\"}"])
                    sendMessage(ws, "create_series", [chart_session, "s1", "s1", "symbol_1", str(val), 2000])
                    #sendMessage(ws, "create_study", [chart_session,"st4","st1","s1","ESD@tv-scripting-101!",{"text":"BNEhyMp2zcJFvntl+CdKjA==_DkJH8pNTUOoUT2BnMT6NHSuLIuKni9D9SDMm1UOm/vLtzAhPVypsvWlzDDenSfeyoFHLhX7G61HDlNHwqt/czTEwncKBDNi1b3fj26V54CkMKtrI21tXW7OQD/OSYxxd6SzPtFwiCVAoPbF2Y1lBIg/YE9nGDkr6jeDdPwF0d2bC+yN8lhBm03WYMOyrr6wFST+P/38BoSeZvMXI1Xfw84rnntV9+MDVxV8L19OE/0K/NBRvYpxgWMGCqH79/sHMrCsF6uOpIIgF8bEVQFGBKDSxbNa0nc+npqK5vPdHwvQuy5XuMnGIqsjR4sIMml2lJGi/XqzfU/L9Wj9xfuNNB2ty5PhxgzWiJU1Z1JTzsDsth2PyP29q8a91MQrmpZ9GwHnJdLjbzUv3vbOm9R4/u9K2lwhcBrqrLsj/VfVWMSBP","pineId":"TV_SPLITS","pineVersion":"8.0"}])

                    # Printing all the result
                    a=""
                    i = 0
                    while i  < 10:
                        try:
                            result = ws.recv()
                #            print(result)
                            a=a+result+"\n"
                            i=i+1
                        except Exception as e:
                            print(e)
                            break

                    generate_csv(a,id)

                    values = np.append(values,getValue(id))

                    interval = interval + 1

                    if interval == 5:
                        interval = 16

                    if interval == 3:
                        interval = 4

                print(values)

                check = all(x==values[0] for x in values)

                check_false = True

                print("schedule job finished "+id)

                if check:
                    check_false = True
                    if values[0] == 1:
                        result_signal = ["Buy", id, "Close Price: "+ str(getClose()), "Senkou A "+ str(getSenkouA()), "Senkou B "+ str(getSenkouB())]
                        return result_signal

                    else:
                        result_signal = ["Sell", id,"Close Price: "+ str(getClose()), "Senkou A "+ str(getSenkouA()), "Senkou B "+ str(getSenkouB())]
                        return result_signal
                else:
                    result_signal = ["Nothing", id,"Close Price: "+ str(getClose()), "Senkou A "+ str(getSenkouA()), "Senkou B "+ str(getSenkouB())]
                    print("Elements are not equal")
                    return result_signal

            else:
                print("BOT offline")
                time.sleep(600)
                print("BOT Waking up")
                result_signal = ["Ignore State", id,"Close Price: "+ str(getClose()), "Senkou A "+ str(getSenkouA()), "Senkou B "+ str(getSenkouB())]
        else:
            time.sleep(172800)
            return job_function(id)
    except:
        result_signal = ["Server error", id,"Close Price: NA ", "Senkou A NA", "Senkou B NA"]
        print("Connection could not be established, retry soon")
        return result_signal

def checkSame(list1,list2):
    for f, b in zip(list1, list2):

        if f is None and b is None:
            continue

        print(f[0] + " check both " + b[0])

        if f[0] != b[0] and f[0] != "Server error":
            print("Returning new value ")
            return False

    print("Returning ever True ")

    return True

currency_list = ["FOREXCOM:EURUSD","FOREXCOM:USDCHF","FOREXCOM:EURCHF","FOREXCOM:CADCHF","FOREXCOM:EURGBP","FOREXCOM:EURCAD"]
thread_list = []
currentThreadListResult = []

boot_flag = [True]

def start_thread():
    print("Thread started")
    global currentThreadListResult
    currentThreadListResult = [None]

    if boot_flag[0]:
        global previousThreadListResult
        previousThreadListResult = [None]
        print("Boot flag set False")
        boot_flag[0] = False

    for x in range (len(currency_list)):
        thread = ThreadWithResult(target=job_function, args=(currency_list[x],))
        thread_list.append(thread)
        thread.start()

    message = ""
    for x in range(len(thread_list)):
        thread_list[x].join()

    for x in range(len(thread_list)):
        currentThreadListResult.append(thread_list[x].result)
        message+=(str(thread_list[x].result) + " \n\n")

#     thread1.start()
#     thread2.start()
#     thread3.start()
#     thread4.start()
#     thread5.start()
#    # thread6.start()
#    # thread7.start()
#    # thread8.start()

#     thread1.join()
#     thread2.join()
#     thread3.join()
#     thread4.join()
#     thread5.join()
#    # thread6.join()
#    # thread7.join()
#    # thread8.join()
    print("current list: "+ str(currentThreadListResult))
    print("prev list: "+ str(previousThreadListResult))

#   message = str(thread1.result) + " \n\n" + str(thread2.result) + " \n\n" + str(thread3.result)  + " \n\n" +   str(thread4.result)+ " \n\n" + str(thread5.result)
    if isNowInTimePeriod(dt.time(8,00), dt.time(18,00), dt.datetime.now().time()) and not checkSame(previousThreadListResult, currentThreadListResult):

        print("Different Signal than previous")
        send_mail(message)

    print(message)

    message = "BOT offline"
    thread_list.clear()
    previousThreadListResult = currentThreadListResult.copy()
    currentThreadListResult.clear()

schedule_actions()

max_stars_repo_name: merwinmu/mabataki

max_issues_repo_name: merwinmu/mabataki

max_forks_repo_name: merwinmu/mabataki


text:
""" Contains implementation for epochs plot
"""
import logging

import matplotlib.pyplot as plt
import numpy as np

from meggie.utilities.messaging import exc_messagebox

from meggie.mainwindow.dynamic import Action
from meggie.mainwindow.dynamic import subject_action

class PlotEpochsImage(Action):
    """ Opens MNE's image plot.
    """

    def run(self):

        try:
            selected_name = self.data['outputs']['epochs'][0]
        except IndexError as exc:
            return

        subject = self.experiment.active_subject
        try:
            self.handler(subject, {'name': selected_name})
        except Exception as exc:
            exc_messagebox(self.window, exc)

    @subject_action
    def handler(self, subject, params):
        epochs = subject.epochs.get(params['name'])
        figs = epochs.content.plot_image()
        for fig in figs:
            ch_type = fig.canvas.get_window_title()
            title = '{0}_{1}'.format(params['name'], ch_type)
            fig.canvas.set_window_title(title)

max_stars_repo_name: Teekuningas/meggie

max_issues_repo_name: Teekuningas/meggie

max_forks_repo_name: cibr-jyu/meggie


text:
"""
LIST OF FUNCTIONS
-----------------

Columns:
- `prettify_column_names`: Replace whitespace in column labels with
   an underscore and, by default, change to all lowercase.
- `delete_columns`: Delete selected columns permanently from the
   input dataframe.

Outliers:
- `get_outlier_values_with_iqr_method`: Return a list of outlier
  values as well as the lower and upper cut-off values.
- `count_outliers_IQR_method`: Detect outliers in specified columns
   depending on specified distance from 1th / 3rd quartile. NaN ignored.
- `remove_outliers_IQR_method`: Remove outliers in specified columns
   depending on specified distance from 1th / 3rd quartile. NaN ignored.
- `winsorize_values`: Return a winsorized version of the selected
   columns.

Transformations:
- `transfrom_data`: Apply the desired transformation on the selected
   columns. (Methods are log, log10, box-cox or yeo-johnson.)
"""

from typing import Dict, Iterable, List, Optional, Tuple, Union

import numpy as np
import pandas as pd
from scipy import stats

# COLUMNS - Names, Datatypes and Removal

def prettify_column_names(df: pd.DataFrame, lowercase: bool = True) -> pd.DataFrame:
    """Replace whitespace in column labels with underscore and,
    by default, change to all lowercase.
    """
    df = df.copy()
    if lowercase:
        df.columns = (col.lower().replace(" ", "_") for col in df.columns)
    else:
        df.columns = (col.replace(" ", "_") for col in df.columns)
    return df

def delete_columns(df: pd.DataFrame, cols_to_delete: Iterable[str]) -> pd.DataFrame:
    """Delete columns permanently from the input dataframe. Note:
    This function is structured such that more and more columns can be
    added to the list and deleted iteratively during EDA. In the end
    you hold the full list of deleted columns.
    """
    df = df.copy()
    for col in cols_to_delete:
        try:
            df.drop(col, axis=1, inplace=True)
            print(f"Column {col} successfully deleted.")
        except KeyError:
            pass
    return df

def downcast_dtypes(
    df: pd.DataFrame,
    category_threshold: Optional[int] = None,
    verbose: bool = True,
) -> pd.DataFrame:
    """Return a copy of the input dataframe with reduced memory usage.
    Numeric dtypes will be downcast to the smallest possible format
    depending on the actual data, object dtypes with less distinct
    values than an optional threshold (default is the rowcount) will
    be transformed to dtype 'category'.

    Limitations: Only 'object' cols are considered for conversion
    to dtype 'category'.
    """
    if verbose:
        print(
            f" Original df size before downcasting: "
            f"{df.memory_usage(deep=True).sum() / (1024**2):,.2f} MB"
        )

    df = df.copy()

    for col in df.columns:
        col_type = str(df[col].dtype)
        col_cat_threshold = category_threshold or df[col].count()
        col_unique_items = df[col].nunique()

        if col_type == "object" and col_unique_items < col_cat_threshold:
            df[col] = df[col].astype("category")
        if col_type.startswith("int"):
            df[col] = pd.to_numeric(df[col], downcast="integer")
        if col_type.startswith("float"):
            df[col] = pd.to_numeric(df[col], downcast="float")

    if verbose:
        print(
            f" New df size after downcasting:"
            f"{df.memory_usage(deep=True).sum() / (1024**2):,.2f} MB"
        )

    return df

# OUTLIERS - Return (for an iterable), Count and Remove

def get_outlier_values_with_iqr_method(
    data: Iterable[Union[int, float]], iqr_dist: float
) -> Tuple[List[Union[int, float]], float, float]:
    """Return a list of outlier values as well as the lower and upper
    cut-off values for the numerical data input. The outliers are
    defined by the 'IQR-Method' for which an IQR-distance has to be
    passed as the second parameter.
    """
    q25, q75 = np.nanpercentile(data, 25), np.nanpercentile(data, 75)
    iqr = q75 - q25
    cut_off = iqr * iqr_dist
    lower, upper = q25 - cut_off, q75 + cut_off
    outliers = [x for x in data if x < lower or x > upper]
    return outliers, lower, upper

def count_outliers_IQR_method(df: pd.DataFrame, iqr_dist: Union[int, float] = 1.5):
    """Display outlier count for numeric columns in the passed
    dataframe based on the IQR distance from the 1th / 3rd quartile
    (defaults to 1.5). NaN values are ignored for the calculations.
    """
    outlier_cols = df.select_dtypes(include=[np.number]).columns.tolist()

    for col in outlier_cols:
        outliers, lower, upper = get_outlier_values_with_iqr_method(
            df[col], iqr_dist=iqr_dist
        )

        if len(outliers) > 0:
            outlier_pct = len(outliers) / len(df[col])
            print(
                f"\n{col}:\n"
                f" - effective upper cut-off value: "
                f"{min(df[col].max(), upper):,.2f}\n"
                f" - effective lower cut-off value: "
                f"{max(df[col].min(), lower):,.2f}\n"
                f" - Identified outliers: {len(outliers):,.0f}\n"
                f" - of total values: {outlier_pct:.1%}"
            )

def remove_outliers_IQR_method(
    df: pd.DataFrame,
    outlier_cols: Optional[List[str]] = None,
    iqr_dist: Union[int, float] = 1.5,
    return_idx_deleted: bool = False,
) -> (pd.DataFrame, Optional[List[str]]):
    """Return a dataframe with outliers removed for selected columns.
    If no specific `outlier_cols` are specified (default), the cleaning
    is applied to all numeric columns. Outliers are removed depending
    on the desired distance from 1th and 3rd quartile (defaults to 1.5).
    NaN values are ignored.

    If `return_idx_deleted` is set to True (it defaults to Flase),
    then not only the cleaned dataframe is returned, but also the list
    of the deleted index values as the second return object.
    """
    df = df.copy()
    if outlier_cols is None:
        outlier_cols = df.select_dtypes(include=[np.number]).columns.tolist()

    len_df_in = len(df)
    rows_to_delete = []

    for col in outlier_cols:
        _, lower, upper = get_outlier_values_with_iqr_method(df[col], iqr_dist=iqr_dist)

        idx_low = df[df[col] < lower].index.tolist()
        idx_high = df[df[col] > upper].index.tolist()

        print(f"\n{col}: \nRows to remove: {len(idx_low + idx_high)}\n")
        rows_to_delete = rows_to_delete + idx_low + idx_high

    rows_to_delete = set(rows_to_delete)
    df.drop(rows_to_delete, inplace=True, axis=0)
    len_df_out = len(df)
    assert len(rows_to_delete) == (len_df_in - len_df_out)

    print(
        f"\nRows removed in total: {len_df_in - len_df_out}"
        f"\n(Percentage of original DataFrame: "
        f"{len(rows_to_delete) / len_df_in:0.1%})"
    )

    if not return_idx_deleted:
        return df
    else:
        return df, list(rows_to_delete)

def winsorize_outliers(
    df: pd.DataFrame, w_dict: Dict[str, Tuple[float, float]], **kwargs
) -> pd.DataFrame:
    """Return a winsorized version of the selected columns. Besides
    the input dataframe, a `w_dict` has to be passed, consisting
    of the column names as keys and tuples of the quantiles on each
    end to be winsorized. (Note you can substitute the floats with
    "None" when you don't want to transfrom on one side.) Additional
    kwargs can be passed to the underlying `winsorize` function
    from scipy.stats.mstats.

    Example w_dict:
        w_dict = {
            "col1": (None, 0.05),
            "col2": (0.1, 0.02),
        }

    Scipy reference here:
        https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.mstats.winsorize.html
    """
    df = df.copy()
    for col, limits in w_dict.items():
        df[col] = stats.mstats.winsorize(df[col], limits=limits, **kwargs)
    return df

# TRANSFORMATIONS

def transform_data(
    df: pd.DataFrame,
    method: Optional[str] = "log",
    cols_to_transform: Optional[List[str]] = None,
    treat_nan: Optional[bool] = False,
    add_suffix: Optional[bool] = True,
) -> pd.DataFrame:
    """Apply the desired transformation on the selected columns of
    the input dataframe. Missing values are ignored (see below). YJ
    is the only option that can handle 0 or negative values, the
    other methods will shift the data accoringly if they encounter
    0 or negative values.

    Possible transformations are:
    - log (compress values more the larger they are)
    - log10 (same, but somewhat easier to interpret)
    - box-cox (optimized generalization of the log transformation)
    - yeo-johnson (same but able to handle zero and negative values)

    Note: The `treat_nan` option simply sets all nan values to -1.
    Use with caution.
    """
    df = df.copy()
    cols_to_transform = (
        cols_to_transform or df.select_dtypes(include=np.number).columns.tolist()
    )
    suffix_dict = {
        "log": "_log",
        "log10": "_log10",
        "box_cox": "_bc",
        "yeo_johnson": "_yj",
    }
    function_dict = {
        "log": lambda x: np.log(x),
        "log10": lambda x: np.log10(x),
    }

    for col in df[cols_to_transform]:
        shift_value = 0
        if (np.min(df[col]) <= 0) & (method != "yeo_johnson"):
            shift_value = np.min(df[col]) * -1 + 0.1
            print(
                f"Zero or negative value(s) in col {col}, "
                f"all data is shifted by {shift_value}. "
                "Alternatively use method 'yeo-johnson'."
            )
        try:
            if method in ["log", "log10"]:
                df[col] = df[col] + shift_value
                df[col] = df[col].apply(function_dict.get(method))
            elif method == "box_cox":
                df[col] = df[col] + shift_value
                df[col] = stats.boxcox(df[col])[0]
            elif method == "yeo_johnson":
                df[col] = stats.yeojohnson(df[col])[0]
            if treat_nan:
                df[col].replace(np.nan, -1, inplace=True)
            if add_suffix:
                df.rename(columns={col: col + suffix_dict.get(method)}, inplace=True)
        except KeyError:
            print(col + " not found!")
    return df

### MISSING VALUES - Handling

# def handle_nan(
#     df,
#     cols_to_impute_num=None,
#     cols_to_impute_cat=None,
#     cols_to_drop=None,
#     drop_all_NaN=False,
# ):
#     """Handle NaN with different strategies for selected columns. Return a
#     transformed copy of the DataFrame. Note: Use with caution, as there are
#     more sophisticated solutions around.

#     Arguments:
#     ----------
#     - cols_to_impute_num: list of num columns to impute median,
#         (default=None)
#     - cols_to_impute_cat: list of categorical columns to impute mode,
#         (default=None)
#     - cols_to_drop: list of columns to drop entirely,
#         (default=None)
#     - drop_all_NaN: bool, if True ALL remaining rows with NaN will be removed,
#         (default=False)

#     Returns:
#     --------
#     - df_NaN: DataFrame, transformed copy of original DataFrame
#     """
#     df_NaN = df.copy()
#     if cols_to_impute_num is not None:
#         for col in cols_to_impute_num:
#             if col in df_NaN.columns:
#                 print(
#                     "{} - median value to impute: {}".format(
#                         col, df_NaN[col].median()
#                     )
#                 )
#                 df_NaN[col] = df_NaN[col].fillna(df_NaN[col].median())
#             else:
#                 print(col + " not found")
#     if cols_to_impute_cat is not None:
#         for col in cols_to_impute_cat:
#             if col in df_NaN.columns:
#                 print(
#                     "{} - most frequent value to impute: {}".format(
#                         col, df_NaN[col].value_counts().index[0]
#                     )
#                 )
#                 df_NaN[col] = df_NaN[col].fillna(
#                     df_NaN[col].value_counts().index[0]
#                 )
#             else:
#                 print(col + " not found")
#     if cols_to_drop is not None:
#         for col in cols_to_drop:
#             if col in df_NaN.columns:
#                 df_NaN.drop(col, axis=1, inplace=True)
#             else:
#                 print(col + " not found")
#     if drop_all_NaN:
#         df_NaN = df_NaN.dropna(how="any")

#     return df_NaN

max_stars_repo_name: rbuerki/codebook

max_issues_repo_name: rbuerki/codebook

max_forks_repo_name: rbuerki/codebook

reason: import numpy,from scipy


text:
import numpy as np
import torch

eps = 2.2204e-16

##Tensor operation

def np2tensor_complex(complex_num, dtype):
    # numpy complex to torch complex
    real = torch.tensor(complex_num.real, dtype=dtype)
    imag = torch.tensor(complex_num.imag, dtype=dtype)
    tensor_complex = torch.complex(real, imag)
    return tensor_complex

def replace_amp(field, amplitude):
    # replace the amplitude of a complex number
    input_phase = field.angle()
    out_field = torch.polar(amplitude, input_phase)
    return out_field

def cart2polar(real, imag):
    # Converts the polar form to polar form
    mag = torch.pow(real ** 2 + imag ** 2, 0.5)
    ang = torch.atan2(imag, real)
    return torch.polar(mag, ang)

def polar2cart(mag, ang):
    # Converts the polar form to cartesian form
    real = mag * torch.cos(ang)
    imag = mag * torch.sin(ang)
    return torch.complex(real, imag)

def torch_fftshift(arg_in):
    # torch fftshift
    real = arg_in.real
    imag = arg_in.imag
    for dim in range(0, len(real.size())):
        real = torch.roll(real, dims=dim, shifts=real.size(dim) // 2)
        imag = torch.roll(imag, dims=dim, shifts=imag.size(dim) // 2)
    arg_out = torch.complex(real, imag)
    return arg_out

def torch_ifftshift(arg_in):
    # torch ifftshift
    real = arg_in.real
    imag = arg_in.imag
    for dim in range(0, len(real.size())):
        real = torch.roll(real, dims=dim, shifts=-real.size(dim) // 2)
        imag = torch.roll(imag, dims=dim, shifts=-imag.size(dim) // 2)
    arg_out = torch.complex(real, imag)

    return arg_out

def angle2map(phasemap):
    # continuous phase angle to 8 bit
    out_slm = phasemap.cpu().detach().squeeze().numpy() + 2 * np.pi
    out_slm = np.remainder(out_slm, 2 * np.pi) / (2 * np.pi)
    phase_out_8bit = (out_slm * 255).round().astype(np.uint8)  # quantized to 8 bits

    return phase_out_8bit

def torch2np(tensor_in):
    return tensor_in.cpu().detach().numpy().squeeze()


text:
import numpy as np

class PolicyPlayer(object):
    """AI player based on MCTS"""

    def __init__(self, policy_network, is_selfplay):
        self._is_selfplay = is_selfplay
        self._policy_network = policy_network

    def set_player_ind(self, p):
        self.player = p

    def reset_player(self):
        pass

    def get_action(self, board, temp=1e-3, return_prob=0):
        sensible_moves = board.availables
        # the pi vector returned by MCTS as in the alphaGo Zero paper
        move_probs = np.zeros(board.width*board.height)
        if len(sensible_moves) > 0:
            probs_legal = self._policy_network.policy_fn(board)
            acts = [p[0] for p in probs_legal]
            probs = [p[1] for p in probs_legal]

            # with the default temp=1e-3, it is almost equivalent
            # to choosing the move with the highest prob
            probs_norm = probs / np.sum(probs)
            move = np.random.choice(acts, p=probs_norm)
            # reset the root node

            if return_prob:
                return move, probs_norm
            else:
                return move
        else:
            print("WARNING: the board is full")

    def __str__(self):
        return "Policy {}".format(self.player)


text:
from matplotlib.collections import LineCollection
from matplotlib.pyplot import show, subplots
from numpy import absolute, amax, angle, array, concatenate, pi

def plot_far_field(Theta, ff):
    am = absolute(ff)
    ph = angle((ff[:-1] + ff[1:]) / 2)

    points = array([Theta, am]).T.reshape(-1, 1, 2)
    segments = concatenate([points[:-2], points[1:-1], points[2:]], axis=1)

    fig, ax = subplots(subplot_kw={"polar": True})

    lc = LineCollection(segments, cmap="twilight_shifted_r", linewidth=2)
    lc.set_array(ph)
    im = ax.add_collection(lc)

    ax.set_ylim(0, 1.05 * amax(am))

    cbar = fig.colorbar(im, ax=ax, ticks=[-pi, -pi / 2, 0, pi / 2, pi])
    cbar.ax.set_ylim(-pi, pi)
    cbar.ax.set_yticklabels(["-π", "-π / 2", "0", "π / 2", "π"])
    cbar.set_label("Argument")
    ax.set_title("Modulus of the Far field")

    show()

max_stars_repo_name: zmoitier/accoster

max_issues_repo_name: zmoitier/accoster

max_forks_repo_name: zmoitier/accoster


text:
__author__ = 'tilmannbruckhaus'

import numpy
import sys

class HighlyDivisible:

    # Highly divisible triangular number
    # Problem 12
    # The sequence of triangle numbers is generated by adding the natural numbers.
    # So the 7th triangle number would be 1 + 2 + 3 + 4 + 5 + 6 + 7 = 28.
    # The first ten terms would be:
    #
    # 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, ...
    #
    # Let us list the factors of the first seven triangle numbers:
    #
    #  1: 1
    #  3: 1,3
    #  6: 1,2,3,6
    # 10: 1,2,5,10
    # 15: 1,3,5,15
    # 21: 1,3,7,21
    # 28: 1,2,4,7,14,28
    # We can see that 28 is the first triangle number to have over five divisors.
    #
    # What is the value of the first triangle number to have over five hundred divisors?
    #
    # See: http://www.wikihow.com/Determine-the-Number-of-Divisors-of-an-Integer

    def __init__(self):
        pass

    @staticmethod
    def find(limit):
        index = 1
        candidate = 1
        while True:
            factors = HighlyDivisible.factor(candidate)
            if HighlyDivisible.num_divisors(factors) > limit:
                return candidate
            index += 1
            candidate += index

    @staticmethod
    def factor(candidate):
        factors = []
        for i in range(2, int(numpy.ma.sqrt(candidate)) + 1):
            exponent = 0
            while candidate % i == 0:
                # i is a factor
                exponent += 1
                candidate /= i
            if exponent > 0:
                factors.append([i, exponent])
        if candidate > 1:
            # we are left with a prime:
            factors.append([candidate, 1])
        return factors

    @staticmethod
    def num_divisors(factors):
        num_divisors = 1
        for (divisor, exponent) in factors:
            # see wikiHow link above
            num_divisors *= exponent + 1
        return num_divisors

if __name__ == '__main__':
    count = 500
    result = HighlyDivisible.find(count)
    print "\nThe value of the first triangle number to have over", count, "divisors is", result

max_stars_repo_name: bruckhaus/challenges

max_issues_repo_name: bruckhaus/challenges

max_forks_repo_name: bruckhaus/challenges


text:
import numpy as np
import matplotlib.pyplot as plt
import math

#在[0,2pi]取100个随机方向
dirc = np.random.random(100)*2*np.pi

#竖向排列两个图，第一个为极坐标，第二个为直角坐标
fig, ax = plt.subplots(2,1)

#极坐标下展现了每步的方向和大小（定为单位步长），用每个点与原点连线组成的向量表示
ax[0] = plt.subplot(211, projection='polar')
for i in range(100):
    ax[0].plot([dirc[i],dirc[i]],[0,1],'r')

#初始化x
x = np.random.rand(100)
x[0] = 0
a = 0
#初始化y
y = np.random.rand(100)
y[0] = 0
b = 0
#每一步在x的投影为cos(dirx[i]), 在y的投影为sin(dirx[i])，然后将每一步连起来即为走过的路径
for i in range(1,100):
    a += math.cos(dirc[i-1])
    x[i] = a
    b += math.sin(dirc[i-1])
    y[i] = b

ax[1].plot(x,y,marker='+',markeredgecolor="red")
#处理坐标轴，使起点位于原点（坐标轴交点）
ax[1].xaxis.set_ticks_position("bottom")
ax[1].yaxis.set_ticks_position("left")
ax[1].spines["bottom"].set_position(('data',0))
ax[1].spines["left"].set_position(('data',0))
ax[1].spines["right"].set_color("none")
ax[1].spines["top"].set_color("none")

fig.savefig('exercise_2-3.pdf')

max_stars_repo_name: haohlin/Pattern-Recognition-Homework

max_issues_repo_name: haohlin/Pattern-Recognition-Homework

max_forks_repo_name: haohlin/Pattern-Recognition-Homework