text:
from flask import Flask, jsonify, render_template, request, send_file, url_for
import json
import sys, traceback
import os
import csv
import re
import numpy as np
import pprint
from datetime import datetime, timedelta
import jinja2
from jinja2 import Template
import getFiles
pp = pprint.PrettyPrinter()

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/bullet.html')
def bullet():
    return render_template('bullet.html')

@app.route('/bulletChart.html')
def bulletChart():
    return render_template('bulletChart.html')

@app.route('/crossfilter.html')
def crossfilter():
    return render_template('crossfilter.html')

@app.route('/crossfilterWithDimentions.html')
def crossfilterWithDimentions():
    return render_template('crossfilterWithDimentions.html')

@app.route('/crossfilterWithTables.html')
def crossfilterWithTables():
    return render_template('crossfilterWithTables.html')

@app.route('/cumulativeLineChart.html')
def cumulativeLineChart():
    return render_template('cumulativeLineChart.html')

@app.route('/discreteBarChart.html')
def discreteBarChart():
    return render_template('discreteBarChart.html')

@app.route('/historicalBar.html')
def historicalBar():
    return render_template('historicalBar.html')

@app.route('/historicalBarChart.html')
def historicalBarChart():
    return render_template('historicalBarChart.html')

@app.route('/horizon.html')
def horizon():
    return render_template('horizon.html')

@app.route('/indentedtree.html')
def indentedtree():
    return render_template('indentedtree.html')

@app.route('/legend.html')
def legend():
    return render_template('legend.html')

@app.route('/test_line.html')
def line():

    filename = './data_files/US_population.txt'
    pattern_regrex = ('(?i)(\S+\s\S+\s\S+)\W+(\S+)\W+(\S+)\W+(\S+)')
    diclist, json_dic = get_CSV(filename, pattern_regrex)
    pp.pprint(diclist[1])
    dic = {'diclist' : diclist, 'json_dic': json_dic, 'Graph Name' : 'Test Graph'}
    return render_template('test_line.html', dic=dic)

@app.route('/lineChart.html')
def lineChart():
    return render_template('lineChart.html')

@app.route('/lineChartSVGResize.html')
def lineChartSVGResize():
    return render_template('lineChartSVGResize.html')

@app.route('/linePlusBarChart.html')
def linePlusBarChart():
    return render_template('linePlusBarChart.html')

@app.route('/linePlusBarWithFocusChart.html')
def linePlusBarWithFocusChart():
    return render_template('linePlusBarWithFocusChart.html')

@app.route('/lineWithFisheyeChart.html')
def lineWithFisheyeChart():
    return render_template('lineWithFisheyeChart.html')

@app.route('/lineWithFocusChart.html')
def lineWithFocusChart():

    mortalityFile = './data_files/Mortality_Rate_US.csv'
    popFile = './data_files/totalPop.txt'
    filename = './data_files/US_population.txt'
    imigrationFile = './data_files/US_Imigration.txt'

    pattern_regex = ('(\d{1,2})/(\d{1,2})/(\d{4})\s+(\S+)')
    diclist, json_dic, popArray, jsonPopArray = getFiles.get_US2000(filename, pattern_regex, 0)
    diclist2000, json_dic2000, pop2kArray, jsonPop2kArray = getFiles.get_US2000(
    './data_files/US_population_2000-Current.txt', pattern_regex, 0)
    pp.pprint(diclist2000)
    diclistDebt, jsonDebt, debtArray, jsonDebtArray = getFiles.get_US2000('./data_files/US_Debt.txt',
    pattern_regex,0)

    diclistStamps, jsonStamps, stampArray, jsonStampArray = getFiles.get_US2000(
    './data_files/US_FoodStampsCost.csv', pattern_regex,1)

    mortalityArray, jsonMortality = getFiles.birthRateAdjust(popFile, mortalityFile)

    dicListImigration, jsonDicList, imigrationArray, jsonImigration = getFiles.get_US2000(imigrationFile,pattern_regex,0)
    totalPop = popArray
    totalPop += pop2kArray
    print("Length of total pop = ", len(popArray))
    jsonDebtPerPerson = getFiles.getDebtPerPerson(totalPop ,debtArray)
    print("JSON debt per person = " , jsonDebtPerPerson)
    # Append all the data to pass off into a json readable object
    data = {'jsonPop2k' : jsonPop2kArray, 'jsonPop' : jsonPopArray, 'jsonDebt' : jsonDebtArray
    , 'jsonDebtPerPerson' : jsonDebtPerPerson, 'jsonMortality' : jsonMortality,
    'jsonImigration' : jsonImigration}
    array = debtArray
    return render_template('lineWithFocusChart.html', data=data)

@app.route('/multiBar.html')
def multiBar():
    return render_template('multiBar.html')

@app.route('/multiBarChart.html')
def multiBarChart():
    return render_template('multiBarChart.html')

@app.route('/multiBarHorizontalChart.html')
def multiBarHorizontalChart():
    return render_template('multiBarHorizontalChart.html')

@app.route('/multiChart.html')
def multiChart():
    return render_template('multiChart.html')

@app.route('/parallelCoordinates.html')
def parallelCoordinates():
    return render_template('parallelCoordinates.html')

@app.route('/pie.html')
def pie():
    return render_template('pie.html')

@app.route('/pieChart.html')
def pieChart():
    return render_template('pieChart.html')

@app.route('/scatter.html')
def scatter():
    return render_template('scatter.html')

@app.route('/scatterChart.html')
def scatterChart():
    return render_template('scatterChart.html')

@app.route('/scatterPlusLineChart.html')
def scatterPlusLineChart():
    return render_template('scatterPlusLineChart.html')

@app.route('/sparkline.html')
def sparkline():
    return render_template('sparkline.html')

@app.route('/sparklinePlus.html')
def sparklinePlus():
    return render_template('sparklinePlus.html')

@app.route('/stackedArea.html')
def stackedArea():
    return render_template('stackedArea.html')

@app.route('/stackedAreaChart.html')
def stackedAreaChart():

    mortalityFile = './data_files/Mortality_Rate_US.csv'
    popFile = './data_files/totalPop.txt'
    filename = './data_files/US_population.txt'
    imigrationFile = './data_files/US_Imigration.txt'

    pattern_regex = ('(\d{1,2})/(\d{1,2})/(\d{4})\s+(\S+)')
    diclist, json_dic, popArray, jsonPopArray = getFiles.get_US2000(filename, pattern_regex, 0)
    diclist2000, json_dic2000, pop2kArray, jsonPop2kArray = getFiles.get_US2000(
    './data_files/US_population_2000-Current.txt', pattern_regex, 0)
    diclistDebt, jsonDebt, debtArray, jsonDebtArray = getFiles.get_US2000('./data_files/US_Debt.txt',
    pattern_regex,0)
    print("debt array = ", debtArray)
    print("pop array = " , popArray)
    print("length of pop = " , len(popArray))
    print("length of debt = ", len(debtArray))
    print("length of pop2k =" , len(pop2kArray))
    diclistStamps, jsonStamps, stampArray, jsonStampArray = getFiles.get_US2000(
    './data_files/US_FoodStampsCost.csv', pattern_regex,1)
    totalPop = popArray
    totalPop += pop2kArray
    print("Length of total pop = ", len(popArray))

    mortalityArray, jsonMortality = getFiles.birthRateAdjust(popFile, mortalityFile)

    dicListImigration, jsonDicList, imigrationArray, jsonImigration = getFiles.get_US2000(
    imigrationFile,pattern_regex,0)

    totalPop = popArray
    totalPop += pop2kArray
    print("Length of total pop = ", len(popArray))
    jsonDebtPerPerson = getFiles.getDebtPerPerson(totalPop ,debtArray)
    print("JSON debt per person = " , jsonDebtPerPerson)
    # Append all the data to pass off into a json readable object
    data = {'jsonPop2k' : jsonPop2kArray, 'jsonPop' : jsonPopArray, 'jsonDebt' : jsonDebtArray
    , 'jsonDebtPerPerson' : jsonDebtPerPerson, 'jsonMortality' : jsonMortality,
    'jsonImigration' : jsonImigration}

    array = debtArray

    return render_template('stackedAreaChart.html',data=data, array=array)

@app.route('/test/lineChartTest.html')
def test_lineChartTest():
    return render_template('test/lineChartTest.html')

@app.route('/test/pieChartTest.html')
def test_pieChartTest():
    return render_template('test/pieChartTest.html')

@app.route('/test/ScatterChartTest.html')
def test_ScatterChartTest():
    return render_template('test/ScatterChartTest.html')

@app.route('/test/stackedAreaChartTest.html')
def test_stackedAreaChartTest():
    return render_template('test/stackedAreaChartTest.html')

@app.route('/test/multiBarChartTest.html')
def test_multiBarChartTest():
    return render_template('test/multiBarChartTest.html')

@app.route('/test/polylinearTest.html')
def test_polylinearTest():
    return render_template('test/polylinearTest.html')

@app.route('/test/realTimeChartTest.html')
def test_realTimeChartTest():

    imigrationFile = './data_files/US_Imigration.txt'
    filename = './data_files/US_population.txt'
    pattern_regex = ('(\d{1,2})/(\d{1,2})/(\d{4})\s+(\S+)')

    diclist, json_dic, popArray, jsonPopArray = getFiles.get_US2000(filename, pattern_regex, 0)

    diclist2000, json_dic2000, pop2kArray, jsonPop2kArray = getFiles.get_US2000(
    './data_files/US_population_2000-Current.txt', pattern_regex, 0)

    diclist, json_dic, popArray, jsonPopArray = getFiles.get_US2000(filename, pattern_regex, 0)

    diclistDebt, jsonDebt, debtArray, jsonDebtArray = getFiles.get_US2000('./data_files/US_Debt.txt',
    pattern_regex,0)

    dicListImigration, jsonDicList, imigrationArray, jsonImigration = getFiles.get_US2000(imigrationFile,pattern_regex,0)
    data = {'json_dic2000' : json_dic2000, 'json_dic': json_dic, 'jsonImigration' : jsonImigration,
    'jsonDebt' : jsonDebtArray}

    return render_template('test/realTimeChartTest.html', data=data)

"""
@app.route('/images/grey-plus.png')
def test_realTimeChartTest():
    return send_file('static/images/grey-plus.png', mimetype='image/png')

@app.route('/images/grey-minus.png')
def test_realTimeChartTest():
    return send_file('static/images/grey-minus.png', mimetype='image/png')

@app.route('/images/nvd3_sampleLineChart1.png')
def test_realTimeChartTest():
    return send_file('static/images/nvd3_sampleLineChart1.png', mimetype='image/png')
"""
if __name__ == "__main__":
    app.run(debug=True, port=8080)


text:
'''
SOI Proprietorship Tax Data (pull_soi_proprietorship.py):
-------------------------------------------------------------------------------
Last updated: 6/29/2015.

This module creates functions for pulling the proprietorship soi tax data into
NAICS trees.
'''
# Packages:
import os.path
import numpy as np
import pandas as pd
import xlrd
# Directories:
_CUR_DIR = os.path.dirname(__file__)
_OUT_DIR = os.path.join(os.path.dirname(_CUR_DIR), "output")
_DATA_DIR = os.path.join(os.path.dirname(_CUR_DIR), "data")
_PROP_DIR = os.path.join(_DATA_DIR, "soi_proprietorship")
# Importing custom packages:
import naics_processing as naics
import file_processing as fp
import constants as cst
# Dataframe names:
_FARM_DF_NM = cst.FARM_PROP_DF_NM
_NFARM_DF_NM = cst.NON_FARM_PROP_DF_NM
_CODE_DF_NM = cst.CODE_DF_NM
_TOT_CORP_DF_NM = cst.TOT_CORP_DF_NM
_AST_PRT_DF_NM = cst.AST_PRT_DF_NM
# (Optional) Hardcode the year that the partner data is from:
_YR = ""
_YR = str(_YR)
# Filenames:
_DDCT_IN_FILE = fp.get_file(dirct=_PROP_DIR, contains=[_YR+"sp01br.xls"])
_FARM_IN_FILE = fp.get_file(dirct=_PROP_DIR, contains=["farm_data.csv"])
_DDCT_IN_CROSS_FILE = fp.get_file(dirct=_PROP_DIR,
                                  contains=[_YR+"sp01br_Crosswalk.csv"])
# Full path for files:
_DDCT_IN_PATH = os.path.join(_PROP_DIR, _DDCT_IN_FILE)
_FARM_IN_PATH = os.path.join(_PROP_DIR, _FARM_IN_FILE)
_DDCT_IN_CROSS_PATH = os.path.join(_PROP_DIR, _DDCT_IN_CROSS_FILE)
_NFARM_PROP_OUT_PATH = os.path.join(_OUT_DIR, _NFARM_DF_NM+".csv")
_FARM_PROP_OUT_PATH = os.path.join(_OUT_DIR, _FARM_DF_NM+".csv")
# Constant factors:
_DDCT_FILE_FCTR = 10**3
# Dataframe columns:
_NFARM_DF_COL_NMS = cst.DFLT_PROP_NFARM_DF_COL_NMS_DICT
_NFARM_DF_COL_NM = _NFARM_DF_COL_NMS["DEPR_DDCT"]
_AST_DF_COL_NMS_DICT = cst.DFLT_PRT_AST_DF_COL_NMS_DICT
_LAND_COL_NM = _AST_DF_COL_NMS_DICT["LAND_NET"]
_DEPR_COL_NM = _AST_DF_COL_NMS_DICT["DEPR_AST_NET"]
# Input--relevant row/column names in the nonfarm prop types excel worksheet:
_IN_COL_DF_DICT = dict([
                    ("Depreciation\ndeduction", "DEPR_DDCT")
                    ])
_SECTOR_COL = "Industrial sector"
_DDCT_COL1 = "Depreciation\ndeduction"
_DDCT_COL2 = "Depreciation\ndeduction"

def load_soi_nonfarm_prop(data_tree=naics.generate_tree(),
                          blue_tree=None, blueprint=None,
                          from_out=False, out_path=_NFARM_PROP_OUT_PATH):
    """ This function loads the soi nonfarm proprietorship data:

    :param data_tree: The NAICS tree to read the data into.
    :param cols_dict: A dictionary mapping dataframe columns to the name of
           the column names in the input file
    :param blueprint: The key corresponding to a dataframe in a tree to be
           used as a "blueprint" for populating the df_list dataframes forward.
    :param blue_tree: A NAICS tree with the "blueprint" dataframe. The default
           is the original NAICS tree.
    :param from_out: Whether to read in the data from output.
    :param output_path: The path of the output file.
    """
    # If from_out, load the data tree from output:
    if from_out:
        data_tree = naics.load_tree_dfs(input_path=out_path, tree=data_tree)
        return data_tree
    # Opening nonfarm proprietor data:
    wb = xlrd.open_workbook(_DDCT_IN_PATH)
    ws = wb.sheet_by_index(0)
    cross = pd.read_csv(_DDCT_IN_CROSS_PATH)
    # Finding the relevant positions in worksheet:
    pos1 = naics.search_ws(ws, _SECTOR_COL, 20, True, [0,0], True)
    pos2 = naics.search_ws(ws, _DDCT_COL1, 20)
    pos3 = naics.search_ws(ws,_DDCT_COL2, 20,
                           True, np.array(pos2) + np.array([0,1]))
    #
    data_tree.append_all(df_nm=_NFARM_DF_NM, df_cols=[_NFARM_DF_COL_NM])
    #
    cross_index = cross.shape[0]-1
    enum_index = len(data_tree.enum_inds)-1
    for i in xrange(pos1[0],ws.nrows):
        cur_cell = str(ws.cell_value(i,pos1[1])).lower().strip()
        #
        tot_proportions = 0
        for j in xrange(0, cross.shape[0]):
            cross_index = (cross_index+1) % cross.shape[0]
            cur_ind_name = str(cross.iloc[cross_index,0]).lower().strip()
            if(cur_cell == cur_ind_name):
                if pd.isnull(cross.iloc[cross_index,1]):
                    continue
                ind_codes = str(cross.iloc[cross_index,1]).split(".")
                for k in xrange(0, len(data_tree.enum_inds)):
                    enum_index = (enum_index+1) % len(data_tree.enum_inds)
                    cur_data = data_tree.enum_inds[enum_index].data
                    cur_codes = cur_data.dfs[_CODE_DF_NM]
                    cur_proportions = naics.compare_codes(ind_codes, cur_codes.iloc[:,0])
                    if cur_proportions == 0:
                        continue
                    tot_proportions += cur_proportions
                    cur_dfs = cur_data.dfs[_NFARM_DF_NM][_NFARM_DF_COL_NM]
                    cur_dfs[0] += (_DDCT_FILE_FCTR * cur_proportions
                                        * (ws.cell_value(i,pos2[1])
                                        + ws.cell_value(i,pos3[1])))
            if(tot_proportions == 1):
                break
    # Default:
    if blueprint == None and _TOT_CORP_DF_NM in data_tree.enum_inds[0].data.dfs.keys():
        blueprint = _TOT_CORP_DF_NM
    naics.pop_back(tree=data_tree, df_list=[_NFARM_DF_NM])
    naics.pop_forward(tree=data_tree, df_list=[_NFARM_DF_NM],
                      blueprint=blueprint, blue_tree=blue_tree)
    #
    return data_tree

def load_soi_farm_prop(data_tree=naics.generate_tree(),
                       blue_tree=None, blueprint=None,
                       from_out=False, out_path=_FARM_PROP_OUT_PATH):
    """ This function loads the soi nonfarm proprietorship data:

    :param data_tree: The NAICS tree to read the data into.
    :param cols_dict: A dictionary mapping dataframe columns to the name of
           the column names in the input file
    :param blueprint: The key corresponding to a dataframe in a tree to be
           used as a "blueprint" for populating the df_list dataframes forward.
    :param blue_tree: A NAICS tree with the "blueprint" dataframe. The default
           is the original NAICS tree.
    :param from_out: Whether to read in the data from output.
    :param output_path: The path of the output file.
    """
    # If from_out, load the data tree from output:
    if from_out:
        data_tree = naics.load_tree_dfs(input_path=out_path, tree=data_tree)
        return data_tree
    # Load Farm Proprietorship data:
    farm_data = pd.read_csv(_FARM_IN_PATH)
    new_farm_cols = ["Land", "FA"]
    #
    data_tree.append_all(df_nm=_FARM_DF_NM, df_cols=new_farm_cols)
    #
    land_mult = ((farm_data["R_sp"][0] + farm_data["Q_sp"][0]) *
                        (float(farm_data["A_sp"][0])/farm_data["A_p"][0]))
    total = farm_data["R_p"][0] + farm_data["Q_p"][0]
    total_pa = 0
    cur_codes = [111,112]
    proportions = np.zeros(len(cur_codes))
    proportions = naics.get_proportions(cur_codes, data_tree, _AST_PRT_DF_NM,
                                 [_LAND_COL_NM, _DEPR_COL_NM])
    #
    for ind_code in cur_codes:
        cur_ind = naics.find_naics(data_tree, ind_code)
        cur_df = cur_ind.data.dfs[_AST_PRT_DF_NM]
        total_pa += (cur_df[_LAND_COL_NM][0] + cur_df[_DEPR_COL_NM][0])
    #
    for i in xrange(0,len(cur_codes)):
        cur_ind = naics.find_naics(data_tree, cur_codes[i])
        cur_ind.data.dfs[_FARM_DF_NM]["Land"][0] = (land_mult *
                            cur_ind.data.dfs[_AST_PRT_DF_NM][_LAND_COL_NM][0]/
                            total_pa)
        cur_ind.data.dfs[_FARM_DF_NM]["FA"][0] = ((proportions.iloc[1,i]*total)
                                    - cur_ind.data.dfs[_FARM_DF_NM]["Land"][0])
    # Default:
    if blueprint == None and _TOT_CORP_DF_NM in data_tree.enum_inds[0].data.dfs.keys():
        blueprint = _TOT_CORP_DF_NM
    naics.pop_back(tree=data_tree, df_list=[_FARM_DF_NM])
    naics.pop_forward(tree=data_tree, df_list=[_FARM_DF_NM],
                      blueprint=blueprint, blue_tree=blue_tree)
    #
    return data_tree

ext: py

max_stars_repo_name: lnsongxf/OG-USA

max_issues_repo_name: lnsongxf/OG-USA

max_forks_repo_name: lnsongxf/OG-USA

reason: import numpy


text:
#!/usr/bin/python
# BSD 3-Clause License

# Copyright (c) 2019, Noam C. Golombek
# All rights reserved.

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.

# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import timeit
import numpy as np
import cv2

import rospy
from cv_bridge import CvBridge, CvBridgeError
from sensor_msgs.msg import Image, CompressedImage

class ImageSubscriber(object):
    '''A class to subscribe to a ROS camera'''

    def __init__(self, topic="usb_cam/image_raw"):
        '''Initialize ros publisher, ros subscriber'''

        self.currentImage = None
        self.currentHeader = None
        self.compressed = True if "compressed" in topic else False
        self.frame_count = 0
        self.start_time = timeit.default_timer()

        if not self.compressed:
            self.bridge = CvBridge()
            self.subscriber = rospy.Subscriber(topic,
                                               Image, self.callback,  queue_size=1)
        else:
            self.subscriber = rospy.Subscriber(
                topic, CompressedImage, self.callback, queue_size=1)

    def callback(self, ros_data):
        '''Callback function of subscribed topic.
        Here images get converted to cv2 format'''

        if not self.compressed:
            self.currentImage = cv2.resize(
                self.bridge.imgmsg_to_cv2(ros_data, "rgb8"),
                (640, 480))
        else:
            np_arr = np.fromstring(ros_data.data, np.uint8)
            image_np = cv2.imdecode(np_arr, 1)

            self.currentImage = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)

        self.currentHeader = ros_data.header

        self.frame_count += 1


text:
# imports and basic notebook setup
from cStringIO import StringIO
import numpy as np
import scipy.ndimage as nd
import PIL.Image
from IPython.display import clear_output, Image, display
from google.protobuf import text_format

import sys
import os
sys.path.append("/home/ahranov/REMember/dep/caffe/python")
os.system("export LD_LIBRARY_PATH=/deepdream/caffe/distribute/lib")

import caffe

# If your GPU supports CUDA and Caffe was built with CUDA support,
# uncomment the following to run Caffe operations on the GPU.
# caffe.set_mode_gpu()
# caffe.set_device(0) # select GPU device if multiple devices exist

def showarray(a, fmt='jpeg'):
    a = np.uint8(np.clip(a, 0, 255))
    f = StringIO()
    PIL.Image.fromarray(a).save(f, fmt)
    display(Image(data=f.getvalue()))

#Loading DNN model
#In this notebook we are going to use a GoogLeNet model trained on ImageNet dataset. Feel free to experiment with other models from Caffe Model Zoo. One particularly interesting model was trained in MIT Places dataset. It produced many visuals from the original blog post.

model_path = '/deepdream/caffe/models/bvlc_googlenet/' # substitute your path here
net_fn   = model_path + 'deploy.prototxt'
param_fn = model_path + 'bvlc_googlenet.caffemodel'

# Patching model to be able to compute gradients.
# Note that you can also manually add "force_backward: true" line to "deploy.prototxt".
model = caffe.io.caffe_pb2.NetParameter()
text_format.Merge(open(net_fn).read(), model)
model.force_backward = True
open('tmp.prototxt', 'w').write(str(model))

net = caffe.Classifier('tmp.prototxt', param_fn,
                       mean = np.float32([104.0, 116.0, 122.0]), # ImageNet mean, training set dependent
                       channel_swap = (2,1,0)) # the reference model has channels in BGR order instead of RGB

# a couple of utility functions for converting to and from Caffe's input image layout
def preprocess(net, img):
    return np.float32(np.rollaxis(img, 2)[::-1]) - net.transformer.mean['data']
def deprocess(net, img):
    return np.dstack((img + net.transformer.mean['data'])[::-1])

#Producing dreams
#Making the "dream" images is very simple. Essentially it is just a gradient ascent process that tries to maximize the L2 norm of activations of a particular DNN layer. Here are a few simple tricks that we found useful for getting good images:

#*offset image by a random jitter
#*normalize the magnitude of gradient ascent steps
#*apply ascent across multiple scales (octaves)
#First we implement a basic gradient ascent step function, applying the first two tricks:

def objective_L2(dst):
    dst.diff[:] = dst.data

def make_step(net, step_size=1.5, end='inception_4c/output',
              jitter=32, clip=True, objective=objective_L2):
    '''Basic gradient ascent step.'''

    src = net.blobs['data'] # input image is stored in Net's 'data' blob
    dst = net.blobs[end]

    ox, oy = np.random.randint(-jitter, jitter+1, 2)
    src.data[0] = np.roll(np.roll(src.data[0], ox, -1), oy, -2) # apply jitter shift

    net.forward(end=end)
    objective(dst)  # specify the optimization objective
    net.backward(start=end)
    g = src.diff[0]
    # apply normalized ascent step to the input image
    src.data[:] += step_size/np.abs(g).mean() * g

    src.data[0] = np.roll(np.roll(src.data[0], -ox, -1), -oy, -2) # unshift image

    if clip:
        bias = net.transformer.mean['data']
        src.data[:] = np.clip(src.data, -bias, 255-bias)

#Next we implement an ascent through different scales. We call these scales "octaves".

def deepdream(net, base_img, iter_n=10, octave_n=4, octave_scale=1.4,
              end='inception_4c/output', clip=True, **step_params):
    # prepare base images for all octaves
    octaves = [preprocess(net, base_img)]
    for i in xrange(octave_n-1):
        octaves.append(nd.zoom(octaves[-1], (1, 1.0/octave_scale,1.0/octave_scale), order=1))

    src = net.blobs['data']
    detail = np.zeros_like(octaves[-1]) # allocate image for network-produced details
    for octave, octave_base in enumerate(octaves[::-1]):
        h, w = octave_base.shape[-2:]
        if octave > 0:
            # upscale details from the previous octave
            h1, w1 = detail.shape[-2:]
            detail = nd.zoom(detail, (1, 1.0*h/h1,1.0*w/w1), order=1)

        src.reshape(1,3,h,w) # resize the network's input image size
        src.data[0] = octave_base+detail
        for i in xrange(iter_n):
            make_step(net, end=end, clip=clip, **step_params)

            # visualization
            vis = deprocess(net, src.data[0])
            if not clip: # adjust image contrast if clipping is disabled
                vis = vis*(255.0/np.percentile(vis, 99.98))
            showarray(vis)
            print octave, i, end, vis.shape
            clear_output(wait=True)

        # extract details produced on the current octave
        detail = src.data[0]-octave_base
    # returning the resulting image
    return deprocess(net, src.data[0])

#Now we are ready to let the neural network reveal its dreams! Let's take a cloud image as a starting point:

img = np.float32(PIL.Image.open('Mockus.jpg'))
showarray(img)

#Running the next code cell starts the detail generation process. You may see how new patterns start to form, iteration by iteration, octave by octave.

_=deepdream(net, img)

#The complexity of the details generated depends on which layer's activations we try to maximize. Higher layers produce complex features, while lower ones enhance edges and textures, giving the image an impressionist feeling:

_=deepdream(net, img, end='inception_3b/5x5_reduce')

#We encourage readers to experiment with layer selection to see how it affects the results. Execute the next code cell to see the list of different layers. You can modify the make_step function to make it follow some different objective, say to select a subset of activations to maximize, or to maximize multiple layers at once. There is a huge design space to explore!

net.blobs.keys()

#What if we feed the deepdream function its own output, after applying a little zoom to it? It turns out that this leads to an endless stream of impressions of the things that the network saw during training. Some patterns fire more often than others, suggestive of basins of attraction.

#We will start the process from the same sky image as above, but after some iteration the original image becomes irrelevant; even random noise can be used as the starting point.

os.system("mkdir frames")
frame = img
frame_i = 0

h, w = frame.shape[:2]
s = 0.05 # scale coefficient
for i in xrange(100):
    frame = deepdream(net, frame)
    PIL.Image.fromarray(np.uint8(frame)).save("frames/%04d.jpg"%frame_i)
    frame = nd.affine_transform(frame, [1-s,1-s,1], [h*s/2,w*s/2,0], order=1)
    frame_i += 1

#Be careful running the code above, it can bring you into very strange realms!
Image(filename='frames/0029.jpg')

#Controlling dreams
#The image detail generation method described above tends to produce some patterns more often the others. One easy way to improve the generated image diversity is to tweak the optimization objective. Here we show just one of many ways to do that. Let's use one more input image. We'd call it a "guide".

guide = np.float32(PIL.Image.open('sourceImages/flowers.jpg'))
showarray(guide)

#Note that the neural network we use was trained on images downscaled to 224x224 size. So high resolution images might have to be downscaled, so that the network could pick up their features. The image we use here is already small enough.

#Now we pick some target layer and extract guide image features.

end = 'inception_3b/output'
h, w = guide.shape[:2]
src, dst = net.blobs['data'], net.blobs[end]
src.reshape(1,3,h,w)
src.data[0] = preprocess(net, guide)
net.forward(end=end)
guide_features = dst.data[0].copy()

#Instead of maximizing the L2-norm of current image activations, we try to maximize the dot-products between activations of current image, and their best matching correspondences from the guide image.

def objective_guide(dst):
    x = dst.data[0].copy()
    y = guide_features
    ch = x.shape[0]
    x = x.reshape(ch,-1)
    y = y.reshape(ch,-1)
    A = x.T.dot(y) # compute the matrix of dot-products with guide features
    dst.diff[0].reshape(ch,-1)[:] = y[:,A.argmax(1)] # select ones that match best

_=deepdream(net, img, end=end, objective=objective_guide)

#This way we can affect the style of generated images without using a different training set.

max_stars_repo_name: ShanTheNaan/REMember

max_issues_repo_name: ShanTheNaan/REMember

max_forks_repo_name: ShanTheNaan/REMember

reason: import numpy,import scipy


text:
import matplotlib.pyplot as plt
import numpy as np
import csv
from utils import *

class Plotter():
	def __init__(self, value_dict, team_csv):
		# self.values, self.teams = values
		self.team_map = read_team_csv(team_csv)
		self.value_dict = value_dict

	def compare_rankings(self, test=False):
		q_vals = []
		team_names = []
		power_rankings = []
		standings = []

		i = 0
		for team_id in self.team_map:
			name, power_ranking, standing = self.team_map[team_id]
			team_names.append(name)
			power_rankings.append(power_ranking)
			standings.append(standing)
			if test:
				q_vals.append((np.random.randint(1000), i))
			else:
				info_dict = self.value_dict[int(team_id)]
				q_val=info_dict['total_value']
				games=info_dict['games']
				possessions=info_dict["possesions"]
				movements=info_dict["movements"]
				q_vals.append((q_val/games, i))
			i += 1

		q_val_rankings = [t[1] for t in sorted(q_vals,reverse=True)]

		plt.xlabel('Teams')
		plt.ylabel('Ranking')
		plt.title('2015-16 NBA Team Rankings')

		# print(power_rankings)
		plt.plot(range(len(team_names)), power_rankings, label='Standing', marker='o')
		plt.plot(range(len(team_names)), q_val_rankings, label='Q Val per Game Ranking', marker='o')
		plt.plot(range(len(team_names)), standings, label='Power Ranking', marker='o')

		plt.xticks(range(len(team_names)), team_names, size='small', rotation='vertical')
		plt.legend()
		plt.show()
		plt.savefig('rankings_game_plot.png')

def read_team_csv(path):
	teams = {}
	with open(path) as csvfile:
		reader = csv.DictReader(csvfile)
		for row in reader:
			team_id = row['TEAM_ID']
			teams[team_id] = row['TEAM_NAME'], row['POWER_RANKING'], row['STANDING']
	return teams

team_csv = 'team_mapping.csv'
test_value_dict = load_obj("team_eval_dict")
plotter = Plotter(test_value_dict, team_csv)
# print(plotter.team_map)
plotter.compare_rankings(test=False)

max_stars_repo_name: frmccann/DRL-ice-hockey

max_issues_repo_name: frmccann/DRL-ice-hockey

max_forks_repo_name: frmccann/DRL-ice-hockey


text:
"""
|----------------------------------------------|
| CS488 -  Senior Capstone Project             |
| Porter Libby                                 |
| Spring 2020                                  |
|----------------------------------------------|
"""

import argparse, glob, cv2, time, os, sys, shutil

from PIL import Image
from fastai.vision import *
from fastai.imports import *
from fastai import *
from torchvision import transforms
from matplotlib import pyplot as plt
import numpy as np

def predict(image, model, topk=5):
    learn = load_learner(model)

    img = open_image(image)

    pred_class, pred_idx, outputs = learn.predict(img)
    print(pred_class)
    print(pred_idx)
    print(outputs)

model = "out"
image = "../training/plant-id/docks/3113_12449_30098.jpg"

predict(image, model)

max_stars_repo_name: probably-not-porter/CEML

max_issues_repo_name: probably-not-porter/CEML

max_forks_repo_name: probably-not-porter/Composite-Edge-ML


text:
import numpy as np

M = np.matrix('5 1 3; 1 2 0; 3 0 7')
print('M: ')
print(M)
print('DET: ')
print(np.linalg.det(M))
print('Cholesky: ')
print(np.linalg.cholesky(M))

U = np.linalg.cholesky(M)
W = np.transpose(U) * U

print('W: ')
print(W)

max_stars_repo_name: andres-root/numerical-analysis

max_issues_repo_name: andres-root/numerical-analysis

max_forks_repo_name: andres-root/numerical-analysis


text:
# Copyright (c) 2020 Anita Hu and Kevin Su
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import glob
import os
import cv2
import torch
import numpy as np
import librosa
from torch.utils.data import Dataset

"""
Video label in filename:
Modality (01 = full-AV, 02 = video-only, 03 = audio-only).
Vocal channel (01 = speech, 02 = song).
Emotion (01 = neutral, 02 = calm, 03 = happy, 04 = sad, 05 = angry, 06 = fearful, 07 = disgust, 08 = surprised).
Emotional intensity (01 = normal, 02 = strong). NOTE: There is no strong intensity for the 'neutral' emotion.
Statement (01 = "Kids are talking by the door", 02 = "Dogs are sitting by the door").
Repetition (01 = 1st repetition, 02 = 2nd repetition).
Actor (01 to 24. Odd numbered actors are male, even numbered actors are female).
"""
video_dict = {
    "modality": [1, 2, 3],
    "vocal_channel": [1, 2],
    "emotion": [1, 2, 3, 4, 5, 6, 7, 8],
    "emotional_intensity": [1, 2],
    "statement": [1, 2],
    "repetition": [1, 2],
    "actor": list(range(1, 25))
}

def parse_gt(video_name):
    video_keys = video_dict.keys()
    gt = {}
    y = video_name.strip(".mp4")
    y = y.split("-")
    for k, v in zip(video_keys, y):
        gt[k] = int(v)
    return gt

class RAVDESSDataset(Dataset):
    """RAVDESS dataset."""

    def __init__(self, actor_folds, modality=('video', 'audio'), transform=None):
        """
        :param actor_folds: a list of actor folder paths
        :param modality: list of modalities, default ('video', 'audio')
        :param transform: data transform or a dict with
        {
            "image_transform": transform function or None
            "audio_transform": transform function or None
        }
        """
        self.video_list = []
        self.transform = transform
        self.modality = modality

        actor_counter = 0
        for each_actor in actor_folds:
            for each_video in glob.glob(os.path.join(each_actor, "*.mp4")):
                self.video_list.append(each_video)
            actor_counter += 1

        print("{} videos for {} actors found.".format(str(len(self.video_list)), str(actor_counter)))

    def __len__(self):
        return len(self.video_list)

    def _read_images(self, idx, transform):
        X = []
        video_path = self.video_list[idx]
        frames_path = os.path.join(video_path, "frames")
        frames_lst = sorted(glob.glob(os.path.join(frames_path, "*.jpg")), key=lambda x: x[6:])

        for frame in frames_lst:
            img = cv2.imread(frame)
            if transform:
                img = transform(img)
                img = img.squeeze_(0)  # remove fake batch dimension
            else:
                img = torch.tensor(img)
            X.append(img)
        X = torch.stack(X, dim=0)

        return {'images': X.float()}

    def _read_audio(self, idx, transform):
        video_path = self.video_list[idx]
        audio_path = os.path.join(video_path, "audios/featuresMFCC.npy")

        if transform:
            wav_file = os.path.join(video_path, "audios/audio.wav")
            X, sample_rate = librosa.load(wav_file, duration=2.45, sr=22050 * 2, offset=0.5)
            X = transform(X)
            sample_rate = np.array(sample_rate)
            features = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
        else:
            features = np.load(audio_path)

        return {'mfcc': torch.tensor(features)}

    def __getitem__(self, idx):
        gt = parse_gt(os.path.basename(self.video_list[idx]))
        y = torch.LongTensor([gt["emotion"] - 1])  # id starts in 1

        sample = {'emotion': y}

        if "video" in self.modality:
            if isinstance(self.transform, dict) and "image_transform" in self.transform:
                image_transform = self.transform["image_transform"]
            else:
                image_transform = self.transform
            image_feature = self._read_images(idx, image_transform)
            sample.update(image_feature)
        if "audio" in self.modality:
            if isinstance(self.transform, dict) and "audio_transform" in self.transform:
                audio_transform = self.transform["audio_transform"]
            else:
                audio_transform = self.transform
            audio_features = self._read_audio(idx, audio_transform)
            sample.update(audio_features)

        return sample

max_stars_repo_name: sharanramjee/st-ctr

max_issues_repo_name: sharanramjee/st-ctr

max_forks_repo_name: sharanramjee/st-ctr


text:
#!/usr/bin/env python
# coding: utf-8

import requests
import pandas as pd
import numpy as np
from bs4 import BeautifulSoup
from alphacast import Alphacast
from dotenv import dotenv_values
API_KEY = dotenv_values(".env").get("ALPHACAST_API_KEY")
alphacast = Alphacast(API_KEY)

#Url base del sitio
url = 'https://www.indec.gob.ar/Nivel4/Tema/3/3/42'
html_page = requests.get(url).text
soup = BeautifulSoup(html_page, 'html.parser')
links = soup.find_all('a', href=True)

#Obtengo la url del archivo del ISAC
xls_links = []
keyword = 'Indicador sintético de la actividad de la construcción, insumos para la construcción'
for link in links:
    if keyword in link.get_text():
        xls_links.append(link.get('href'))

#Mantengo solo el primer elemento de la lista
xls_links = 'https://www.indec.gob.ar' + xls_links[0]

# Levanto la data y armo el dataframe
df1 = pd.read_excel(xls_links, sheet_name = 'Cuadro 1', skiprows=2, header=[0,1,2])

#Elimino las columnas con NaNs
df1.dropna(axis=1, how='all', inplace=True)
df1.dropna(axis=0, how='all', inplace=True)

#Se eliminan las filas que no tengan datos en la tercer columna
df1.dropna(subset=[df1.columns[2]], how='all', inplace=True)

#Completo de la fila 1 en adelante
df1.iloc[1:, 0].fillna(method='ffill', inplace=True)

# Realizo un replacement de los meses
dict_meses = {"Enero": '01', "Febrero": '02', "Marzo": '03', "Abril": '04', "Mayo": '05', "Junio": '06',
              "Julio": '07', "Agosto": '08', "Septiembre": '09', "Octubre": '10', "Noviembre": '11', "Diciembre": '12'}
df1.iloc[:, 1].replace(dict_meses, inplace=True)

#Unifico los nombres de las columnas
df1.columns = df1.columns.map(' - '.join)
df1 = df1.dropna(how='any', axis=0)

#Se cambian los nombres de las variables
df1 = df1.rename(columns={'Serie original -     ISAC nivel general - Unnamed: 2_level_2': 'ISAC nivel general',
                          'Serie original - Variación porcentual - respecto al mismo mes del año anterior': 'Serie original - Variación porcentual - respecto al mismo mes del año anterior - %',
                          'Serie original - Variación porcentual - acumulada del año respecto a igual acumulado del año anterior':'Serie original - Variación porcentual - acumulada del año respecto a igual acumulado del año anterior - %',
                          'Serie desestacionalizada (1) -     ISAC     nivel general - Unnamed: 6_level_2':'ISAC - sa_orig',
                          'Serie desestacionalizada (1) - Variación porcentual respecto al mes anterior - Unnamed: 7_level_2':'Serie desestacionalizada - Variación porcentual respecto al mes anterior - %',
                          'Serie tendencia-ciclo -     ISAC     nivel general - Unnamed: 9_level_2':'ISAC tendencia-ciclo',
                          'Serie tendencia-ciclo - Variación porcentual respecto al mes anterior - Unnamed: 10_level_2':'Serie tendencia-ciclo - Variación porcentual respecto al mes anterior - %'
                         })

#Creamos la columna fecha y le cambiamos el formato
df1['Date'] = df1[df1.columns[0]].astype(str) + '-' + df1[df1.columns[1]].astype(int).astype(str)
df1['Date'] = pd.to_datetime(df1['Date'], format='%Y-%m')
df1.set_index('Date', inplace=True)
df1 = df1.iloc[:, 2:]

#Levanto la hoja con el ISAC desagregado y creo un segundo dataframe
df2 = pd.read_excel(xls_links, sheet_name = 'Cuadro 2.1', skiprows=2)

#Elimino las filas con NaNs
df2.dropna(subset=[df2.columns[1]], inplace=True)

#Completo las filas de 'Período' con el dato del año
df2.iloc[:, 0].fillna(method='ffill', inplace=True)

#Reemplazo los meses con el diccionario
df2.iloc[:, 1].replace(dict_meses, inplace=True)

#Creo la columna de fecha, le cambio el formato y la seteo como índice
df2['Date'] = df2[df2.columns[0]].astype(str) + '-' + df2[df2.columns[1]].astype(int).astype(str)
df2['Date'] = pd.to_datetime(df2['Date'], format='%Y-%m')
df2.set_index('Date', inplace=True)
df2 = df2.iloc[:, 2:]

#Uno ambos dataframes en uno solo
df = df1.merge(df2, left_index=True, right_index=True, how='outer')

#Realizo limpieza de ciertos strings y agrego la columna country para hacer el upload
df.replace('///', np.nan, inplace=True)
df.columns = df.columns.str.replace('\s+\-\s+\-\s+', ' - ', regex=True)
df.columns = df.columns.str.replace('\s+\(1\)', '', regex=True)

#Cargo la data a Alphacast
df['country'] = 'Argentina'
alphacast.datasets.dataset(15).upload_data_from_df(df,
                 deleteMissingFromDB = False, onConflictUpdateDB = True, uploadIndex=True)

max_stars_repo_name: alphacastio/connectors-argentina-INDEC

max_issues_repo_name: alphacastio/connectors-argentina-INDEC

max_forks_repo_name: alphacastio/connectors-argentina-INDEC


text:
import numba
import cmath
import numpy as np
from numba import cuda
from .util import port_mask_nan

__all__ = ["fractional_diff", "get_weights_floored", "port_fractional_diff"]

def get_weights_floored(d, num_k, floor=1e-3):
    r"""Calculate weights ($w$) for each lag ($k$) through
    $w_k = -w_{k-1} \frac{d - k + 1}{k}$ provided weight above a minimum value
    (floor) for the weights to prevent computation of weights for the entire
    time series.

    Args:
        d (int): differencing value.
        num_k (int): number of lags (typically length of timeseries) to
        calculate w.
        floor (float): minimum value for the weights for computational
        efficiency.
    """
    w_k = np.array([1])
    k = 1

    while k < num_k:
        w_k_latest = -w_k[-1] * ((d - k + 1)) / k
        if abs(w_k_latest) <= floor:
            break

        w_k = np.append(w_k, w_k_latest)

        k += 1

    w_k = w_k.reshape(-1, 1)

    return w_k

@cuda.jit(device=True)
def conv_window(shared, history_len, out_arr, window_size,
                arr_len, offset, offset2, min_size):
    """
    This function is to do convolution for one thread

    Arguments:
    ------
     shared: numba.cuda.DeviceNDArray
        3 chunks of data are stored in the shared memory
        the first [0, window_size) elements is the chunk of data that is
        necessary to compute the first convolution element.
        then [window_size, window_size + thread_tile * blockDim) elements
        are the inputs allocated for this block of threads
        the last [window_size + thread_tile,
        window_size + thread_tile + window_size) is to store the kernel values
     history_len: int
        total number of historical elements available for this chunk of data
     out_arr: numba.cuda.DeviceNDArray
        output gpu_array of size of `thread_tile`
     window_size: int
        the number of elements in the kernel
     arr_len: int
        the chunk array length, same as `thread_tile`
     offset: int
        indicate the starting index of the chunk array in the shared for
        this thread.
     offset: int
        indicate the starting position of the weights/kernel array
     min_size: int
         the minimum number of non-na elements
    """
    for i in range(arr_len):
        if i + history_len < window_size-1:
            out_arr[i] = np.nan
        else:
            s = 0.0
            average_size = 0
            for j in range(0, window_size):
                if not (cmath.isnan(
                        shared[offset + i - j])):
                    s += (shared[offset + i - j] *
                          shared[offset2 + window_size - 1 - j])
                    average_size += 1
            if average_size >= min_size:
                out_arr[i] = s
            else:
                out_arr[i] = np.nan

@cuda.jit
def kernel(in_arr, weight_arr, out_arr, window,
           arr_len, thread_tile, min_size):
    """
    This kernel is to do 1D convlution on `in_arr` array with `weight_arr`
    as kernel. The results is saved on `out_arr`.

    Arguments:
    ------
     in_arr: numba.cuda.DeviceNDArray
        input gpu array
     weight_arr: numba.cuda.DeviceNDArray
        convolution kernel gpu array
     out_arr: numba.cuda.DeviceNDArray
        output gpu_array
     window: int
        the number of elements in the weight_arr
     arr_len: int
        the input/output array length
     thread_tile: int
        each thread is responsible for `thread_tile` number of elements
     min_size: int
         the minimum number of non-na elements
    """
    shared = cuda.shared.array(shape=0,
                               dtype=numba.float64)
    block_size = cuda.blockDim.x  # total number of threads
    tx = cuda.threadIdx.x
    # Block id in a 1D grid
    bid = cuda.blockIdx.x
    starting_id = bid * block_size * thread_tile

    # copy the thread_tile * number_of_thread_per_block into the shared
    for j in range(thread_tile):
        offset = tx + j * block_size
        if (starting_id + offset) < arr_len:
            shared[offset + window - 1] = in_arr[
                starting_id + offset]
        cuda.syncthreads()

    # copy the window - 1 into the shared
    for j in range(0, window - 1, block_size):
        if (((tx + j) <
             window - 1) and (
                 starting_id - window + 1 + tx + j >= 0)):
            shared[tx + j] = \
                in_arr[starting_id - window + 1 + tx + j]
        cuda.syncthreads()
    # copy the weights into the shared
    for j in range(0, window, block_size):
        element_id = tx + j
        if (((tx + j) < window) and (element_id < window)):
            shared[thread_tile * block_size + window - 1 + tx +
                   j] = weight_arr[tx + j]
        cuda.syncthreads()
    # slice the shared memory for each threads
    start_shared = tx * thread_tile
    his_len = min(window - 1,
                  starting_id + tx * thread_tile)
    # slice the global memory for each threads
    start = starting_id + tx * thread_tile
    end = min(starting_id + (tx + 1) * thread_tile, arr_len)
    sub_outarr = out_arr[start:end]
    sub_len = end - start
    conv_window(shared, his_len, sub_outarr,
                window, sub_len,
                window - 1 + start_shared,
                thread_tile * block_size + window - 1,
                min_size)

def fractional_diff(input_arr, d=0.5, floor=1e-3, min_periods=None,
                    thread_tile=2, number_of_threads=512):
    """
    The fractional difference computation method.

    Arguments:
    -------
      input_arr: numba.cuda.DeviceNDArray or cudf.Series
        the input array to compute the fractional difference
      d: float
        the differencing value. range from 0 to 1
      floor: float
        minimum value for the weights for computational efficiency.
      min_periods: int
        default the lengths of the weights. Need at least min_periods of
        non-na elements to get fractional difference value
      thread_tile: int
        each thread will be responsible for `thread_tile` number of
        elements in window computation
      number_of_threads: int
        number of threads in a block for CUDA computation

    Returns
    -------
    (numba.cuda.DeviceNDArray, np.array)
        the computed fractional difference array and the weight array tuple

    """
    if isinstance(input_arr, numba.cuda.cudadrv.devicearray.DeviceNDArray):
        gpu_in = input_arr
    else:
        gpu_in = input_arr.to_gpu_array()

    # compute the weights for the fractional difference
    weights = get_weights_floored(d=d,
                                  num_k=len(input_arr),
                                  floor=floor)[::-1, 0]
    weights_out = np.ascontiguousarray(weights)
    weights = numba.cuda.to_device(weights_out)

    window = len(weights)

    if min_periods is None:
        min_periods = window
    else:
        min_periods = min_periods

    number_of_threads = number_of_threads
    array_len = len(gpu_in)

    # allocate the output array
    gpu_out = numba.cuda.device_array_like(gpu_in)

    number_of_blocks = \
        (array_len + (number_of_threads * thread_tile - 1)) // \
        (number_of_threads * thread_tile)

    shared_buffer_size = (number_of_threads * thread_tile +
                          window - 1 + window)

    # call the conv kernel
    kernel[(number_of_blocks,),
           (number_of_threads,),
           0,
           shared_buffer_size * 8](gpu_in,
                                   weights,
                                   gpu_out,
                                   window,
                                   array_len,
                                   thread_tile,
                                   min_periods)
    return gpu_out, weights_out

def port_fractional_diff(asset_indicator, input_arr, d=0.5, floor=1e-3,
                         min_periods=None, thread_tile=2,
                         number_of_threads=512):
    """
    Calculate the fractional differencing signal for all the financial
    assets indicated by asset_indicator.

    Arguments:
    -------
      asset_indicator: cudf.Series
        the integer indicator array to indicate the start of the different
        asset
      input_arr: numba.cuda.DeviceNDArray or cudf.Series
        the input array to compute the fractional difference
      d: float
        the differencing value. range from 0 to 1
      floor: float
        minimum value for the weights for computational efficiency.
      min_periods: int
        default the lengths of the weights. Need at least min_periods of
        non-na elements to get fractional difference value
      thread_tile: int
        each thread will be responsible for `thread_tile` number of
        elements in window computation
      number_of_threads: int
        number of threads in a block for CUDA computation

    Returns
    -------
    (numba.cuda.DeviceNDArray, np.array)
        the computed fractional difference array and the weight array tuple
    """
    out, weights = fractional_diff(input_arr, d=d, floor=floor,
                                   min_periods=min_periods,
                                   thread_tile=thread_tile,
                                   number_of_threads=number_of_threads)
    port_mask_nan(asset_indicator.to_gpu_array(), out, 0,
                  len(weights) - 1)
    return out, weights

reason: import numpy,import numba,from numba


text:
""" Helper functions to run GCP AutoML Tables.
    Note: GCP only works for datasets with > 1000 rows.

    Link to GCP AutoML Tables Client API reference: https://googleapis.dev/python/automl/latest/gapic/v1beta1/tables.html
"""
import time, warnings, os, sys
import re, math
import numpy as np
import pandas as pd
from autogluon.utils.tabular.utils.loaders import load_pd
from autogluon.utils.tabular.ml.constants import BINARY, MULTICLASS, REGRESSION

from google.cloud import storage
from google.cloud import automl_v1beta1 as automl
import google.cloud.automl_v1beta1.proto.data_types_pb2 as data_types
from google.oauth2 import service_account
from google.api_core import operations_v1

GCS_PREFIX = "gs://"
GCS_TRAIN_FILENAME = 'train.csv' # name of all training data files on GCS
GCS_TEST_FILENAME = 'test.csv' # name of all test data files on GCS
LOCAL_PREDS_FILENAME = 'tables_pred.csv' # name of file containing GCP Tables predictions
GCP_DISPLAY_NAME_MAXCHARS = 32
GCP_MODEL_PREFIX = "tm_" # model-display name appends this prefix to dataset-name
id_column = 'unique_id_col_gcptables' # name of ID column appended to data because GCP shuffles predictions

def gcptables_fit_predict(train_data, test_data, dataset_name, label_column, problem_type, output_directory, gcp_info,
                          eval_metric = None, runtime_sec = 3600, fit_model = True, model_name = None, make_predictions = True):
    """ Use GCP AutoML tables for both fitting and prediction.
        Returns all outputs of AbstractBaseline.fit(), AbstractBaseline.predict() as one big tuple, with one final element: class_order
        Also takes in the same arguments as these methods, except for num_cores.
        Other Args:
            dataset_name: str Name
                GCP data and outputs will be stored in GCS Storage Bucket under this name, should be unique for every GCP run on a new dataset.
            gcp_info: dict of critical informtion regarding GCP configuration, project, and access keys.
            fit_model: bool indicating whether or not to actually fit models using GCP AutoML Tables.
                If a previous run of this function crashed after the model had been trained, then you just produce predictions via:
                fit_model = False. Similarly, you can set this False in order to get predictions in a separate process from the fit() call.
                When False, you must specify: model_name as the string corresponding to the model.name entry from previous fit(),
                but without the project/path prefix (this thus matches the display name of the model in the GCP console).
            make_predictions: bool indicating whether or not we should return after fit() without making predictions.

        Note: For classification, your class labels cannot end with suffix: '_score'
    """

    train_data = train_data.copy()
    test_data = test_data.copy()

    # Reformat column names to only contain alphanumeric characters:
    label_column_index = train_data.columns.get_loc(label_column)
    train_data.columns = [re.sub(r'\W+', '_', col) for col in train_data.columns.tolist()]  # ensure alphanumeric-only column-names
    test_data.columns = [re.sub(r'\W+', '_', col) for col in test_data.columns.tolist()]  # ensure alphanumeric-only column-names
    label_column = train_data.columns[label_column_index]  # re-assign as it may have changed
    train_data[id_column] = list(train_data.index)
    test_data[id_column] = list(test_data.index)
    data_colnames = list(set(train_data.columns))

    # Drop test labels if they exist:
    if label_column in test_data.columns:
        test_data = test_data.drop([label_column], axis=1)

    og_dataset_name = dataset_name
    dataset_name = re.sub(r'\W+', '_', dataset_name) # Ensure GCP will not complain about names
    dataset_name = dataset_name[:(GCP_DISPLAY_NAME_MAXCHARS-len(GCP_MODEL_PREFIX))]
    if model_name is None:
        model_display_name = GCP_MODEL_PREFIX + dataset_name
    else:
        model_display_name = model_name
    if og_dataset_name != dataset_name:
        print("GCP will complain about provided dataset_name, renamed to: %s" % dataset_name)

    PROJECT_ID = gcp_info['PROJECT_ID']
    BUCKET_NAME = gcp_info['BUCKET_NAME']
    COMPUTE_REGION = gcp_info['COMPUTE_REGION']
    GOOGLE_APPLICATION_CREDENTIALS = gcp_info['GOOGLE_APPLICATION_CREDENTIALS']
    num_models_trained = None
    num_models_ensemble = None
    fit_time = None
    y_pred = None
    y_prob = None
    predict_time = None
    class_order = None
    if len(train_data) < 1000:
        raise ValueError("GCP AutoML tables can only be trained on datasets with >= 1000 rows")

    # Create GCP clients:
    storage_client = storage.Client.from_service_account_json(GOOGLE_APPLICATION_CREDENTIALS)
    bucket = storage_client.get_bucket(BUCKET_NAME)
    credentials = service_account.Credentials.from_service_account_file(GOOGLE_APPLICATION_CREDENTIALS)
    automl_client = automl.AutoMlClient(credentials = credentials)
    tables_client = automl.TablesClient(project=PROJECT_ID, region=COMPUTE_REGION, credentials=credentials)

    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    #  Upload training data to GCS:
    gcs_train_path = dataset_name + "/" + GCS_TRAIN_FILENAME # target file-name
    train_file_exists = storage.Blob(bucket=bucket, name=gcs_train_path).exists(storage_client)
    if not train_file_exists:
        print('Uploading training data')
        train_file_path = output_directory + GCS_TRAIN_FILENAME
        train_data.to_csv(train_file_path, index=False) # write reformatted train-data to CSV file.
        # Upload to GCS:
        blob = bucket.blob(gcs_train_path)
        blob.upload_from_filename(train_file_path)
    else: # need to rename columns anyway to process predictions.
        print('Training data already uploaded')

    # Upload test data:
    gcs_test_path = dataset_name + "/" + GCS_TEST_FILENAME # target file-name
    test_file_exists = storage.Blob(bucket=bucket, name=gcs_test_path).exists(storage_client)
    if not test_file_exists:
        print('Uploading test data')
        test_file_path = output_directory + GCS_TEST_FILENAME
        test_data.to_csv(test_file_path, index=False) # write reformatted test-data to CSV file.
        # Upload to GCS:
        blob = bucket.blob(gcs_test_path)
        blob.upload_from_filename(test_file_path)
    else:
        print('Test data already uploaded')

    if not train_file_exists:
        os.remove(train_file_path)

    if not test_file_exists:
        os.remove(test_file_path)

    # print("train_data.columns", train_data.columns)
    # print("test_data.columns", test_data.columns) # TODO remove

    # Use AutoML-Tables to fit models with training data:
    dataset = tables_client.create_dataset(dataset_display_name = dataset_name)
    tables_dataset_name = dataset.name
    import_data_response = tables_client.import_data(
        dataset=dataset,
        gcs_input_uris=GCS_PREFIX+BUCKET_NAME+"/"+gcs_train_path
    )
    print('Dataset import operation: {}'.format(import_data_response.operation))
    print('Dataset import response: {}'.format(import_data_response.result())) # print ensures block until dataset has been uploaded.
    list_table_specs_response = tables_client.list_table_specs(dataset=dataset)
    table_specs = [s for s in list_table_specs_response]
    print(table_specs)
    # list_column_specs_response = tables_client.list_column_specs(dataset=dataset)
    # column_specs = [s for s in list_column_specs_response]
    # label_spec = [column_specs[i] for i in range(len(column_specs)) if column_specs[i].display_name == label_column]
    # print(label_spec[0])

    # Set label column:
    if problem_type in [BINARY, MULTICLASS]:
        type_code='CATEGORY'
        update_column_response = tables_client.update_column_spec(
            dataset=dataset,
            column_spec_display_name=label_column,
            type_code=type_code,
            nullable=False,
        ) # ensure label_column is categorical
        print(update_column_response)

    update_dataset_response = tables_client.set_target_column(
        dataset=dataset,
        column_spec_display_name=label_column,
    )
    print(update_dataset_response)

    # Fit AutoML Tables:
    gcp_metric = None # Metric passed to GCP as optimization_objective argument
    if fit_model:
        if eval_metric is not None:
            metrics_map = {  # Mapping of benchmark metrics to GCP AutoML Tables metrics: https://googleapis.dev/python/automl/latest/gapic/v1beta1/types.html
                'accuracy': 'MINIMIZE_LOG_LOSS',
                'f1': 'MAXIMIZE_AU_PRC',
                'log_loss': 'MINIMIZE_LOG_LOSS',
                'roc_auc': 'MAXIMIZE_AU_ROC',
                'balanced_accuracy': 'MAXIMIZE_AU_ROC',
                'precision': 'MAXIMIZE_PRECISION_AT_RECALL',
                'recall': 'MAXIMIZE_RECALL_AT_PRECISION',
                'mean_squared_error': 'MINIMIZE_RMSE',
                'median_absolute_error': 'MINIMIZE_MAE',
                'mean_absolute_error': 'MINIMIZE_MAE',
                'r2': 'MINIMIZE_RMSE',
            }
            if eval_metric in metrics_map:
                gcp_metric = metrics_map[eval_metric]
        else:
            warnings.warn("Unknown metric will not be used by GCP AutoML Tables: %s" % eval_metric)
        t0 = time.time()
        model_train_hours = math.ceil(runtime_sec / 3600.)
        print('Training model for %s hours' % model_train_hours)
        print('Training model with name: %s' % model_display_name)
        # TODO FIXME TODO FIXME:
        #  exclude_column_spec_names (Optional[str]) – The list of the names of the columns you want to exclude and not train your model on.
        # FIXME: ADD AN ID COLUMN
        create_model_response = tables_client.create_model(
            model_display_name=model_display_name,
            dataset=dataset,
            train_budget_milli_node_hours=model_train_hours*1000,
            optimization_objective=gcp_metric,
            exclude_column_spec_names=[id_column, label_column],
        )
        operation_id = create_model_response.operation.name
        print('Create GCP model operation: {}'.format(create_model_response.operation))
        check_interval = 60 # check for model status updates every check_interval seconds
        keep_checking = True
        check_time = time.time()
        while keep_checking: # and time.time() - t0 <= runtime_sec: # check on current model status
            if time.time() - check_time > check_interval:
                api = operations_v1.OperationsClient(channel = automl_client.transport.channel)
                status_update = api.get_operation(operation_id)
                print("Status update on GCP model: \n {}".format(status_update))
                print('Time Elapsed: %s of %s' % ((time.time() - t0), runtime_sec))
                check_time = time.time()
                if hasattr(status_update, 'done') and status_update.done:
                    keep_checking = False

        # Waits until model training is done:
        model = create_model_response.result()
        model_name = model.name
        print("GCP training completed, produced model object with name: %s" % model_name)
        print("You can use this trained model for batch prediction by specifying model_name=%s" % model_display_name)
        print(model)
        t1 = time.time()
        fit_time = t1 - t0
        print("GCP Tables Model fit complete, runtime: %s" % fit_time)
        print("GCP model name = %s" % model_name)
    else: #skip model fitting:
        fit_time = None
        print("Skipping GCP Tables Model fit, just using trained model for prediction")
        if model_name is None:
            raise ValueError("When fit_model=False, model_name must be specified.")
        model = tables_client.get_model(model_display_name = model_name)

    # Automatically-generated held-out validation performance estimates:
    num_models_trained = -1
    num_models_ensemble = -1
    summary_list = tables_client.list_model_evaluations(model=model)
    model_eval_summaries = [s for s in summary_list]
    if problem_type in [BINARY, MULTICLASS]:
        log_losses = [model_eval_summaries[i+1].classification_evaluation_metrics.log_loss for i in range(len(model_eval_summaries)-1)]
        log_loss = np.mean(np.array(log_losses))
        print("Validation log_loss = %s" % log_loss)

    if problem_type == BINARY:
        auc_rocs = [model_eval_summaries[i+1].classification_evaluation_metrics.au_roc for i in range(len(model_eval_summaries)-1)]
        auc_roc = np.mean(np.array(auc_rocs))
        print("Validation AUC_ROC = %s" % auc_roc)

    if not make_predictions:
        print("Skipping predictions, set model_name = %s to use this trained model for prediction later on" % model_name)
        return num_models_trained, num_models_ensemble, fit_time, y_pred, y_prob, predict_time, class_order

    # Predict (using batch inference, so no need to deploy model):
    t2 = time.time()
    preds_file_prefix = GCS_PREFIX+BUCKET_NAME+"/"+dataset_name+"/pred"
    batch_predict_response = tables_client.batch_predict(
        model=model,
        gcs_input_uris=GCS_PREFIX+BUCKET_NAME+"/"+gcs_test_path,
        gcs_output_uri_prefix=preds_file_prefix
    )
    print('Batch prediction operation: {}'.format(
        batch_predict_response.operation))

    # Wait until batch prediction is done.
    batch_predict_result = batch_predict_response.result()
    print(batch_predict_response.metadata)
    t3 = time.time()
    predict_time = t3 - t2

    # Fetch predictions from GCS bucket to local file:
    preds_gcs_folder = batch_predict_response.metadata.batch_predict_details.output_info.gcs_output_directory # full path to GCS file containing predictions
    preds_gcs_filename = 'tables_1.csv' # default file name created by GCP Tables.
    preds_gcs_file = preds_gcs_folder + "/" + preds_gcs_filename
    local_preds_file = output_directory + LOCAL_PREDS_FILENAME

    with open(local_preds_file, 'wb') as file_obj:
        storage_client.download_blob_to_file(preds_gcs_file, file_obj)

    # Load predictions into python and format:
    test_pred_df = load_pd.load(local_preds_file)
    same_cols = [col for col in test_pred_df.columns if col in data_colnames]
    keep_cols = [col for col in test_pred_df.columns if col not in data_colnames]
    original_gcp_length = len(test_pred_df)
    original_test_length = len(test_data)
    print('test orig:', )
    print(test_data)
    print('before dedupe...')
    print(test_pred_df)
    test_pred_df = test_pred_df.drop_duplicates(subset=[id_column]) # drop any duplicate rows in predictions before join
    print('before merge...')
    print(test_pred_df)
    test_pred_df = test_data.merge(test_pred_df, on=[id_column], how='left') # un-shuffle the predictions so order matches test data.
    print('after merge...')
    print(test_pred_df)
    test_pred_df = test_pred_df[keep_cols]
    if len(test_pred_df) != len(test_data):
        warnings.warn("GCP failed to produce predictions for some test data rows")
        print('diff: %s | %s' % (len(test_pred_df), len(test_data)))
        print('DIFF ORIGINAL:')
        print(original_test_length)
        print(original_gcp_length)

    if problem_type != REGRESSION:
        gcp_classes = list(test_pred_df.columns)
        og_classes = list(train_data[label_column].unique())

        print('Num Classes orig:', len(og_classes))
        print('Num Classes GCP: ', len(gcp_classes))
        print('GCP Class Names                 : ', gcp_classes)
        print('Original Class Names            : ', og_classes)
        orig_colnames = [column[(len(label_column) + 1):-len('_score')] for column in gcp_classes]
        print('Original Class Names (Reordered): ', orig_colnames)

        if len(gcp_classes) != len(og_classes):
            warnings.warn("GCP AutoML Tables predictions are missing classes")
            raise AssertionError('GCP AutoML did not predict with all classes! GCP returned %s of %s classes!' % (len(gcp_classes), len(og_classes)))

        test_pred_df.columns = orig_colnames
    else:
        test_pred_df.columns = [label_column]

    if test_pred_df.isnull().values.any():  # Some missing predictions exist that need to be imputed.
        test_pred_df = impute_dummy_predictor(test_pred_df=test_pred_df, train_data=train_data, label_column=label_column, problem_type=problem_type)

    if problem_type == REGRESSION:
        if len(keep_cols) != 1:
            warnings.warn("GCP AutoML Tables regression predictions are incorrectly formatted")
            print('keep_cols:', keep_cols)
            raise AssertionError('GCP AutoML did not return a valid regression prediction! GCP returned %s of %s classes!' % (len(keep_cols), 1))
        y_pred = test_pred_df[label_column]
        y_prob = None
        return num_models_trained, num_models_ensemble, fit_time, y_pred, y_prob, predict_time, class_order
    else:
        y_pred = test_pred_df.idxmax(axis=1)
        class_order = list(test_pred_df.columns)
        y_prob = np.array(test_pred_df)
        return num_models_trained, num_models_ensemble, fit_time, y_pred, y_prob, predict_time, class_order

def impute_dummy_predictor(test_pred_df, train_data, label_column, problem_type):
    print("Some test data predictions are missing (due to GCP error), imputing these with dummy predictor.")
    if problem_type == REGRESSION:
        y_pred_dummy = np.mean(train_data[label_column])
        test_pred_df = test_pred_df.fillna(y_pred_dummy)
    else:
        num_train = float(len(train_data))
        clss_counts = train_data[label_column].value_counts()
        clss_names = list(clss_counts.keys())
        gcp_df_names = set(list(test_pred_df.columns))
        to_str_clss = False # whether we should convert this to string
        to_int_clss = False # whether we should convert this to int
        if set(clss_names) != gcp_df_names:
            str_clss_names = set([str(clss) for clss in clss_names])
            if str_clss_names == gcp_df_names:
                to_str_clss = True
            else:
                int_clss_names = set([int(name) for name in clss_names])
                if int_clss_names == gcp_df_names:
                    to_int_clss = True
                else:
                    print("test_pred_df.columns = ", list(test_pred_df.columns))
                    print("unique values in train_data[label_column] = ", clss_names)
                    raise ValueError("impute_dummy: mismatched test_pred_df.columns and clss_names:")
        # clss_names = [str(x) for x in list(clss_counts.keys())] # convert to string if ints.
        # if id_column in clss_names:
        #     clss_names.remove(id_column)
        y_marginal_probs = {}
        for clss in clss_names:
            y_marginal_probs[clss] = clss_counts[clss] / num_train
        for clss in clss_names:
            if to_str_clss:
                gcp_df_name = str(clss)
            elif to_int_clss:
                gcp_df_name = int(clss)
            else:
                gcp_df_name = clss
            test_pred_df[gcp_df_name] = test_pred_df[gcp_df_name].fillna(y_marginal_probs[clss])
    if test_pred_df.isnull().values.any():
        raise ValueError("impute_dummy failed to fill in all null values in prediction dataframe")
    return test_pred_df.copy()

max_stars_repo_name: jwmueller/autogluon-benchmarking

max_issues_repo_name: jwmueller/autogluon-benchmarking

max_forks_repo_name: jwmueller/autogluon-benchmarking


text:
import unittest
import numpy as np
import pandas as pd
import bilby.hyper as hyp

class TestHyperLikelihood(unittest.TestCase):
    def setUp(self):
        self.keys = ["a", "b", "c"]
        self.lengths = [300, 400, 500]
        self.posteriors = list()
        for ii, length in enumerate(self.lengths):
            self.posteriors.append(
                pd.DataFrame({key: np.random.normal(0, 1, length) for key in self.keys})
            )
        self.log_evidences = [2, 2, 2]
        self.model = hyp.model.Model(list())
        self.sampling_model = hyp.model.Model(list())

    def tearDown(self):
        del self.keys
        del self.lengths
        del self.posteriors
        del self.log_evidences

    def test_evidence_factor_with_evidences(self):
        like = hyp.likelihood.HyperparameterLikelihood(
            self.posteriors,
            self.model,
            self.sampling_model,
            log_evidences=self.log_evidences,
        )
        self.assertEqual(like.evidence_factor, 6)

    def test_evidence_factor_without_evidences(self):
        like = hyp.likelihood.HyperparameterLikelihood(
            self.posteriors, self.model, self.sampling_model
        )
        self.assertTrue(np.isnan(like.evidence_factor))

    def test_len_samples_with_max_samples(self):
        like = hyp.likelihood.HyperparameterLikelihood(
            self.posteriors,
            self.model,
            self.sampling_model,
            log_evidences=self.log_evidences,
            max_samples=10,
        )
        self.assertEqual(like.samples_per_posterior, 10)

    def test_len_samples_without_max_samples(self):
        like = hyp.likelihood.HyperparameterLikelihood(
            self.posteriors,
            self.model,
            self.sampling_model,
            log_evidences=self.log_evidences,
        )
        self.assertEqual(like.samples_per_posterior, min(self.lengths))

    def test_resample_with_max_samples(self):
        like = hyp.likelihood.HyperparameterLikelihood(
            self.posteriors,
            self.model,
            self.sampling_model,
            log_evidences=self.log_evidences,
        )
        resampled = like.resample_posteriors()
        self.assertEqual(resampled["a"].shape, (len(self.lengths), min(self.lengths)))

    def test_resample_without_max_samples(self):
        like = hyp.likelihood.HyperparameterLikelihood(
            self.posteriors,
            self.model,
            self.sampling_model,
            log_evidences=self.log_evidences,
        )
        resampled = like.resample_posteriors(10)
        self.assertEqual(resampled["a"].shape, (len(self.lengths), 10))

if __name__ == "__main__":
    unittest.main()


text:
from __future__ import absolute_import

from .tesisfunctions import histogram, brightness,Plotim, graphmath, graphHistogram, \
    overlay, findmaxima, findminima, smooth, graph_filter, getOtsuThresh, find_near
from RRtoolbox.lib.arrayops import Bandpass,convexityRatio
import cv2
import numpy as np
import pylab as plt
from scipy.signal import savgol_filter
# I should  try by finding the minimum value or maximum value, find their indexes and then apply region growing from there
# to detect the main body, perhaps i should find the maximum value in the histogram and then find that color, and apply
# region growing from there. Using these three i should be able to distinct the background, main body and brightest areas

fn1 = r'im5_1.jpg'
#fn1 = r'im1_1.jpg'
#fn1 = r'im3_1.jpg'
name = fn1.split('\\')[-1].split(".")[0]

# read image
fore = cv2.imread(fn1)
fore = cv2.resize(fore,(300,300)) # resize image
#Plotim(name,fore).show() # show image

# get intensity
P = brightness(fore)
#P = (255-P)

# create color pallet: unknown, background, body, brightest, boundaries
pallet = np.array([[255,0,0],[0,0,0],[0,0,255],[255,255,255],[255,0,255]],np.uint8)

lines,comments = [],[]
fig1 = plt.figure("Histogram")
# calculate histogram
#hist_P = histogram(P)[0]
hist_P, bins = np.histogram(P.flatten(),256,[0,256])
lines.extend(graphmath(hist_P,show=False)[1]) # plots original histogram

# filter histogram to smooth it.
window = "hamming" # 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
window_len = 51
hist_PS = smooth(hist_P,window_len,window=window, correct=True) # it smooths and expands the histogram
#hist_PS = savgol_filter(hist_P, window_len, polyorder = 3)
lines.extend(graphmath(hist_PS, show=False)[1]) # plots smooth histogram

x = bins[:-1] #np.indices(hist_P.shape) #np.arange(len(hist_PS)) # get indexes

otsu = getOtsuThresh(hist_P) # Otsu value
minima = findminima(hist=hist_PS) # minima values
maxima = findmaxima(hist=hist_PS) # maxima values
fig2 = plt.figure("Min - Max")
plt.plot(x,hist_PS)
plt.plot(x[minima], hist_PS[minima], "o", label="min")
plt.plot(x[maxima], hist_PS[maxima], "o", label="max")
plt.plot(x[otsu], hist_PS[otsu], "o", label="Otsu")
plt.legend()

# INITIAL DATA
data_min_left = P.min() # initial minimum value
data_min = find_near(maxima, thresh=data_min_left)
data_min_right = find_near(minima, thresh=data_min, side="right")
data_max = P.max()
data_max_left = find_near(minima, thresh=data_max, side="left")
"""
# filter to reduce histogram ends
# create Filter to reduce histogram ends
filterr = Bandpass(10, data_min, data_max) # FIXMED this should use P.min(), P.max() and local maximas, or a relation of porcentage
                              # like min_val % body_val % max_val
graph_filter(filterr,show=False)

hist_PF = filterr(x)*hist_P # this is applyed to find the maximum value at the main body.
lines.extend(graphmath(hist_PF,show=False)[1]) # plots damped histogram at the ends"""

plt.figure(fig1.number) # continue Histogram

# SECONDARY DATA
data_body = find_near(maxima, thresh=otsu,side="right")
data_body_left = find_near(minima,thresh=data_body,side="left")
data_body_right = find_near(minima,thresh=data_body,side="right")
annotate = [(x[data_min_left], hist_P[data_min_left], "min left"),
            (x[data_min], hist_P[data_min], "min"),
            (x[data_min_right], hist_P[data_min_right], "min right"),
            (x[data_body_left], hist_P[data_body_left],"body left"),
            (x[data_body], hist_P[data_body],"body"),
            (x[data_body_right], hist_P[data_body_right],"body right"),
            (x[data_max_left], hist_P[data_max_left],"max left"),
            (x[data_max], hist_P[data_max],"max")]
arrowprops = dict(facecolor='black', shrink=0.05)

ax = plt.gca() # get current axes
for xp,yp,ann in annotate:
    ax.annotate(ann.title(), xy=(xp,yp), textcoords='data', xytext=(xp,yp), arrowprops=arrowprops)
    plt.plot(xp,yp, "o", label=ann)
#plt.legend(handles=lines,loc = 'upper left')
#graphHistogram(P)
plt.show()
# create markers
markers = np.zeros_like(P).astype("int32")
mk_back,mk_body,mk_flare = 1,2,3
markers[P <= data_min]=mk_back # background FIXMED use P.min() and aproaching to local maxima
markers[np.bitwise_and(P>data_body_left,P<data_body)]=mk_body # main body
markers[P >= data_max_left]=mk_flare # Flares. this can be used approaching
                                    # to local maxima, but brightest areas are almost
                                    # always saturated so no need to use it
plotc = Plotim(name + " markers", overlay(fore.copy(), pallet[markers], alpha=0.5))
plotc.sample = P
plotc.show() # show markers using pallet

# apply watershed to markers
cv2.watershed(fore,markers) # FIXME perhaps the function should be cv2.floodFill?
# convert processed markers to colors
water = pallet[markers]
plotc = Plotim(name + " watershed", overlay(fore.copy(), water, alpha=0.3)).show() # show colored watershed in image

#data = water==pallet[mk_flare]
#brightest = np.uint8(data[:,:,0]&data[:,:,1]&data[:,:,2])
brightest = np.uint8(markers==mk_flare)

#plotc = Plotim(name +" flares", brightest).show()

contours,hierarchy = cv2.findContours(brightest.copy(),cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
Crs = [(convexityRatio(cnt),cnt) for cnt in contours] # convexity ratios
Crs.sort(reverse=True)
optic_disc = Crs.pop()
ellipse = cv2.fitEllipse(optic_disc[1])
optic_disc_ellipse = np.zeros_like(brightest)
cv2.ellipse(optic_disc_ellipse, ellipse, 1, -1) # get elliptical ROI

#plotc = Plotim(name +" optic disc mask", optic_disc_ellipse).show()
plotc = Plotim(name + " optic_dist", overlay(fore.copy(), optic_disc_ellipse * 255, alpha=0.3)).show()

max_stars_repo_name: davtoh/RRTools

max_issues_repo_name: davtoh/RRTools

max_forks_repo_name: davtoh/RRTools

reason: import numpy,from scipy


text:
import copy
import os
import pickle

import torch
import torch.nn as nn
import numpy as np

from matplotlib import pyplot as plt

from library import inputs
from Utils.checkpoints import save_context, Logger
from Utils import flags
from Utils import config

import Torture
from library import evaluation, eval_inception_score

FLAGS = flags.FLAGS
KEY_ARGUMENTS = config.load_config(FLAGS.config_file)
FLAGS.subfolder = "Test"
old_c = FLAGS.old_model_c
old_gan = FLAGS.old_model_gan
FLAGS.old_model_c = "loaded"
FLAGS.old_model_gan = "loaded"
text_logger, MODELS_FOLDER, SUMMARIES_FOLDER = save_context(__file__, KEY_ARGUMENTS)

torch.manual_seed(1234)
torch.cuda.manual_seed(1235)
np.random.seed(1236)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
FLAGS.device = device

# model = FLAGS.old_model_c
# dirname = os.path.dirname(model)
# basename = os.path.basename(model)
# config_path = os.path.join(dirname, "..", "source", "configs_dict.pkl")
# summary_path = os.path.join(dirname, "..", "summary")
# with open(config_path, "rb") as f:
#     new_dict = pickle.load(f)
# FLAGS.set_dict(new_dict)

itr = inputs.get_data_iter(batch_size=FLAGS.bs_c, subset=1000)
itr_u = inputs.get_data_iter(batch_size=FLAGS.bs_c)
itr_test = inputs.get_data_iter(batch_size=FLAGS.bs_c, train=False)
# itr_t = inputs.get_data_iter_twice(subset=1000)
# itr_ut = inputs.get_data_iter_twice()
netG, optim_G = inputs.get_generator_optimizer()
netD, optim_D = inputs.get_discriminator_optimizer()
netC, optim_c = inputs.get_classifier_optimizer()
netC_T, _ = inputs.get_classifier_optimizer()

netG, netD, netC = netG.to(device), netD.to(device), netC.to(device)
netG = nn.DataParallel(netG)
netD = nn.DataParallel(netD)
netC = nn.DataParallel(netC)
netC_T = nn.DataParallel(netC_T)

checkpoint_io_gan = Torture.utils.checkpoint.CheckpointIO(checkpoint_dir=MODELS_FOLDER)
checkpoint_io_gan.register_modules(netG=netG, netD=netD)
checkpoint_io_c = Torture.utils.checkpoint.CheckpointIO(checkpoint_dir=MODELS_FOLDER)
checkpoint_io_c.register_modules(netC=netC, netC_T=netC_T)

checkpoint_io_gan.load_file(FLAGS.old_model_gan)
# checkpoint_io_c.load_file(old_c)
logger = Logger(log_dir=SUMMARIES_FOLDER)

# total, correct, _ = evaluation.test_classifier(netC_T)
# print(total, correct, (100 * (correct / total)))

# with torch.no_grad():
#     netG.eval()
#     data, label = itr.__next__()
#     sample_z = torch.randn(FLAGS.bs_g, FLAGS.g_z_dim).to(device)
#     tlabel = label[: FLAGS.bs_g // 10]
#     tlabel = torch.cat([tlabel for _ in range(10)], 0)
#     x_fake = netG(sample_z, tlabel)
#     logger.add_imgs(x_fake, "imgtest", nrow=FLAGS.bs_g // 10)

# # # # Inception score
with torch.no_grad():
    netG.eval()
    img_list = []
    for _ in range(100):
        sample_z = torch.randn(FLAGS.bs_g, FLAGS.g_z_dim).to(device)
        data, label = itr.__next__()
        x_fake = netG(sample_z.to(device), label.to(device))
        img_list.append(x_fake.data.cpu().numpy() * 0.5 + 0.5)
    img_list = np.concatenate(img_list, axis=0)
    img_list = np.transpose(img_list, [0, 2, 3, 1])
    print(img_list.shape)
    print(eval_inception_score.get_inception_score(img_list))

with torch.no_grad():
    netG.eval()
    confidence_sum = 0
    total, correct = 0, 0
    softmax = nn.Softmax(dim=1)
    conf_cor_list, conf_wor_list = [], []
    for _ in range(100):
        sample_z = torch.randn(FLAGS.bs_g, FLAGS.g_z_dim).to(device)
        data, label = itr_test.__next__()
        label = label.to(device)
        # x_fake = netG(sample_z.to(device), label.to(device))
        x_fake = data.to(device)

        outputs = netC_T(x_fake)
        confidence, predicted = torch.max(softmax(outputs).data, 1)
        confidence_sum += confidence.sum()
        total += label.size(0)
        correct += (predicted == label).sum().item()

        confidence_cor = confidence[predicted == label]
        confidence_wor = confidence[predicted != label]
        conf_cor_list.append(confidence_cor.data.cpu().numpy())
        conf_wor_list.append(confidence_wor.data.cpu().numpy())
    print(total, correct, (100 * (correct / total)), confidence_sum / total)
    conf_cor_list = np.concatenate(conf_cor_list)
    conf_wor_list = np.concatenate(conf_wor_list)
    print(conf_cor_list.shape)
    bins = []
    for i in range(10):
        bins.append(0.9 + 0.01 * i)
    figure = plt.figure()
    plt.hist([conf_cor_list, conf_wor_list], bins, label=["c", "w"])
    plt.legend()
    figure.savefig("confidence.pdf")

max_stars_repo_name: AmirHosseinAmeli/Triple-GAN

max_issues_repo_name: AmirHosseinAmeli/Triple-GAN

max_forks_repo_name: AmirHosseinAmeli/Triple-GAN