text:
import sys
sys.path.append('../')

from pathlib import Path
import shutil
import os
import numpy as np

from py_diff_pd.common.renderer import PbrtRenderer
from py_diff_pd.common.hex_mesh import voxelize, generate_hex_mesh
from py_diff_pd.core.py_diff_pd_core import HexMesh3d
from py_diff_pd.common.common import print_info, create_folder
from py_diff_pd.common.project_path import root_path

if __name__ == '__main__':
    folder = Path('pbrt_renderer_demo')
    create_folder(folder)

    # Create a mesh.
    voxels = np.ones((4, 4, 4))
    dx = 0.25
    origin = [0, 0, 0]
    bin_file_name = '.tmp.bin'
    generate_hex_mesh(voxels, dx, origin, bin_file_name)
    mesh = HexMesh3d()
    mesh.Initialize(bin_file_name)
    os.remove(bin_file_name)

    # Render.
    options = {
        'file_name': str(folder / 'demo.png'),
        'light_map': 'uffizi-large.exr',
        'sample': 64,
        'max_depth': 4,
        'camera_pos': (0, -2, 0.8),
        'camera_lookat': (0, 0, 0),
    }
    renderer = PbrtRenderer(options)
    renderer.add_hex_mesh(mesh, render_voxel_edge=True, color=(.3, .7, .5), transforms=[
        ('t', (-1, 0, 0)),
        ('s', 0.25),
        ('r', (1, 0, 1, 0)),
        ('r', (1, 0, 0, 1)),
        ('t', (0, 0, 0.05))
    ])
    renderer.add_tri_mesh(Path(root_path) / 'asset/mesh/curved_ground.obj',
        texture_img='chkbd_24_0.7')
    renderer.render()

    # Display.
    os.system('eog {}'.format(folder / 'demo.png'))


text:
#          Copyright Rein Halbersma 2020-2021.
# Distributed under the Boost Software License, Version 1.0.
#    (See accompanying file LICENSE_1_0.txt or copy at
#          http://www.boost.org/LICENSE_1_0.txt)

import numpy as np

from ..enums import Action, Card, Hand, nC, nH

class StandOn20Agent:
    """
    A blackjack agent that stands on 20 or higher (Sutton and Barto, 2018).
    """
    def __init__(self, env):
        self.policy = np.full((nH, nC), Action.HIT)
        self.policy[Hand.H20:(Hand.H21 + 1), :] = Action.STAND
        self.policy[Hand.S20:(Hand.BJ  + 1), :] = Action.STAND

        self.observe = {
            'Blackjack-v0': (lambda obs: (Hand[('S' if obs[2] else 'H') + str(obs[0])], Card((obs[1] - 2) % 10))),
            'Blackjack-v1': (lambda obs: obs)
        }[env.unwrapped.spec.id]

    def act(self, obs, reward, done):
        return self.policy[self.observe(obs)]

ext: py

max_stars_repo_name: rhalbersma/blackjack

reason: import numpy


text:
# utility to process downloaded netcdf files into processed output data
# HWITW (C) 2021, 2022
#
import math
import copy
import struct
import datetime
import os.path
import gc
import json
import numpy
import netCDF4
import argparse
import pathlib
import concurrent.futures

from generate_HWITW_stats import *
from data_settings import data_settings
from data_groups import data_groups, all_variables

# ignoring leap years (i.e. week 53)!
# Calculations will be off by one day after Feb 28.!
HOURS_PER_WEEK = 24 * 7
HOURS_PER_YEAR = 364 * 24
WEEKS_PER_YEAR = 52

# this is how CDS global netcdf files are indexed - by qtr degree
NUM_LONGIDX_GLOBAL = 1440
NUM_LATIDX_GLOBAL  = 721

APP_VERSION = "0.72"
current_time = datetime.datetime.now()

# what data processing to do for the whole globe (as opposed to specific locations)
global_data_groups = ['temperature_and_humidity','wind','precipitation','cloud_cover']

# make full pathname string for Global var netcdf input file
def var_filename( inp_path:str, dir_name:str, year:int, var_name:str ) -> (str, str):
    # file naming scheme
    pathname = f'{inp_path}/{dir_name}/{year}/'
    # filename = f'gn{grid_num}-{year}-{var_name}.nc'
    filename = f'global-{year}-{var_name}.nc'
    fullname = pathname + filename
    return fullname, filename

# make full pathname string for a Global output netcdf file
def out_filename( out_path:str, dgroup_name:str, year:int ) -> (dict):
    pathname = f'{out_path}/tt_output/{year}/'
    # filename = f'gn{grid_num}-{year}-{var_name}.nc'
    filestr = f'hwglobal-{dgroup_name}-{year}'
    filename = filestr + '.nc'
    tempfilename = filestr +'.tempnc'
    fullname = pathname + filename
    tempfullname = pathname + tempfilename
    o_names = {
        'pathname':pathname,
        'fullname':fullname,
        'filename':filename,
        'tempfullname':tempfullname
    }
    return o_names

# does output file already exist
def output_exists( out_path:str, dgroup_name:str, year:int ) -> bool:
    o_names = out_filename( out_path, dgroup_name, year )
    return os.path.isfile( o_names['fullname'] )

# create output netcdf
def create_output( out_path:str, dgroup_name:str, year:int, num_weeks:int ) -> (netCDF4.Dataset, str):
    # create & initialize the output dataset
    o_names = out_filename( out_path, dgroup_name, year )
    os.makedirs( o_names['pathname'], exist_ok=True )    # create the path if necessary

    try:
        ods = netCDF4.Dataset( o_names['tempfullname'], mode="w", clobber=True, format="NETCDF4" )
    except OSError:
        print( f'output {o_names["tempfullname"]} could not be created!' )
        exit(-1)

    ods.createDimension( "week",      num_weeks )
    ods.createDimension( "latitude",  NUM_LATIDX_GLOBAL )
    ods.createDimension( "longitude", NUM_LONGIDX_GLOBAL )

    week_var = ods.createVariable( 'Week', 'f4', 'week' )
    week_var[:] = [ range(1, num_weeks+1) ]
    week_var.units = 'weeks'

    latitude = ods.createVariable( 'Latitude', 'f4', 'latitude' )
    latitude[:] = [(90 - (i * 180.25 / NUM_LATIDX_GLOBAL)) for i in range( NUM_LATIDX_GLOBAL )]
    latitude.units = 'degrees_north'

    longitude = ods.createVariable( 'Longitude', 'f4', 'longitude' )
    longitude[:] = [(i * 360.0 / NUM_LONGIDX_GLOBAL) for i in range( NUM_LONGIDX_GLOBAL )]
    longitude.units = 'degrees_east'

    return (ods, o_names)

# rename temporary output
def rename_finished_output( o_names:dict ):
    os.rename( o_names['tempfullname'], o_names['fullname'] )

# write to netcdf output Dataset
def save_output( ods:netCDF4.Dataset, week_idx:int, odat:dict ):
    # iterate thru the odat dictionary & save stuff to ods
    for ovar_name, ovar in odat.items():
        for stat_name, lat_dat in ovar.items():
            comb_name = ovar_name + '.' + stat_name
            if not comb_name in ods.variables:
                nc_var = ods.createVariable( comb_name, 'u1', ("week","latitude", "longitude") )
                #todo: annotate with descriptino compression type etc
                nc_var.description = data_settings['variables'][ovar_name][stat_name]['long_name']
                comp_type = data_settings['variables'][ovar_name][stat_name]['compression']
                nc_var.units = comp_type + json.dumps( data_settings['compression'][comp_type] )

            nc_var = ods.variables[ comb_name ]
            for lat_idx, stat in lat_dat.items():
                nc_var[week_idx, lat_idx] = numpy.asarray( stat, dtype=numpy.uint8 )

# store the hig stats output data in a dictionary.
def store_out_lat( odat:dict, var_name:str, stat_name:str, lat_i:int, long_i:int, wk_value ):
    if not var_name in odat.keys():
        odat[ var_name ] = { stat_name: { lat_i: [None for i in range(NUM_LONGIDX_GLOBAL)] } }

    if not stat_name in odat[ var_name ].keys():
        odat[ var_name ][ stat_name ] = { lat_i: [None for i in range(NUM_LONGIDX_GLOBAL)] }

    odat[ var_name ][ stat_name ][ lat_i ][long_i] = wk_value

def process_lat( lat_i:int, lat_data, data_group:dict ):
    out_lat = {}
    # for each longitude
    for long_i in range( NUM_LONGIDX_GLOBAL ):
        # process 1 location 1 week
        #if ( data_group['analysis_kwargs'].contains( 'lon' ) ):
        latitude_deg_n = 90.0 - (lat_i * 180.25 / NUM_LATIDX_GLOBAL)
        longitude_deg_e = long_i * 360.0 / NUM_LONGIDX_GLOBAL - 180.0
        lat0 = math.ceil( latitude_deg_n * 4 ) / 4
        lat1 = (math.floor( latitude_deg_n * 4 ) / 4) #+ 0.01 # edge is not inclusive
        long0 = math.floor( longitude_deg_e * 4 ) / 4
        long1 = (math.ceil( longitude_deg_e * 4 ) / 4) #- 0.01
        area_lat_long = [ lat0, long0, lat1, long1 ]

        # get hourly datas' as ndarrays for this location
        loc_data = []
        for var in lat_data:
            var_loc = var[ long_i ]
            loc_data.append( var_loc )

        analyze_func = data_group['analyze']
        results = analyze_func( loc_data, area_lat_long )

        # store the results
        for variable, variable_info in results.items():
            for stat, wk_value in variable_info.items():
                store_out_lat( out_lat, variable, stat, lat_i, long_i, wk_value )

    return out_lat

# process one full year for one data_group
def process_data_group( flag_args:dict, inp_path:str, out_path:str, dir_name:str, year:int, dg_name:str, data_group:dict ):

    fcalc = flag_args[ 'force_recalc' ]
    show_progress = flag_args[ 'show_progress' ]

    # if output file already exists and not force_recalc, skip this one. it is done.
    if output_exists( out_path, dg_name, year ) and (fcalc == False):
        print( f'\rSkipping completed output {dg_name}-{year}' )
        return

    if show_progress: print( f"Analyzing {dg_name}..." )
    # open all the netcdf input files needed to process this data_group
    ncds_group = []
    for var in data_group['files']:
        var_name = var[0]
        short_var_name = var[1]

        # make sure input file exists
        fullname, filename = var_filename( inp_path, dir_name, year, var_name )
        already_exists = os.path.isfile( fullname )
        if not already_exists:
            print( f'{filename} is missing!', flush=True )
            return

        # load file
        try:
            ds = netCDF4.Dataset( fullname, 'r' )
        except OSError:
            print( f'{filename} could not be opened!' )
            exit(-1)

        # these should all be true for a global var nc
        total_num_hours = min( ds.dimensions['time'].size, HOURS_PER_YEAR )
        num_lat         = ds.dimensions['latitude'].size
        num_long        = ds.dimensions['longitude'].size
        num_dimensions  = len(ds.dimensions)

        assert total_num_hours >= 0
        assert num_long == NUM_LONGIDX_GLOBAL
        assert num_lat == NUM_LATIDX_GLOBAL
        assert (num_dimensions == 3 or num_dimensions == 4)

        # store the Dataset
        ncds_group.append( {
            'dataset':          ds,
            'var_name':         var_name,
            'short_var_name':   short_var_name
        } )

    # create netcdf Output file
    num_weeks = min( total_num_hours // HOURS_PER_WEEK, WEEKS_PER_YEAR )
    ods, o_names = create_output( out_path, dg_name, year, num_weeks )
    if show_progress:
        print( f'\rOutput {o_names["filename"]}', end='', flush=True )
    else:
        print( f'\rOutput {o_names["filename"]}', flush=True )

    # for each week present in the data
    for week_i in range( num_weeks ):

        # for each Dataset needed by the data_group
        week_data = []
        for ncds in ncds_group:
            ds =                ncds['dataset']
            short_var_name =    ncds['short_var_name']

            # load the weeks worth of hours (for all locations in file). transpose to lat,long,hour
            hr_0 = week_i * HOURS_PER_WEEK
            hr_1 = hr_0 + HOURS_PER_WEEK
            if num_dimensions == 3:
                var_week = numpy.transpose( ds[ short_var_name ][hr_0:hr_1][:][:], axes=(1, 2, 0) )

            elif num_dimensions == 4:
                # handle current year. it has a weird expver column / extra dimension
                var_week = numpy.transpose( ds[ short_var_name ][hr_0:hr_1][:][:][:], axes=(2, 3, 0, 1) )
                # convert invalids to 0
                var_week = numpy.ma.filled( var_week, 0 )
                # sum columns on dimension 3 and then drop column 1
                var_week[:, :, :, 0] = var_week[:, :, :, 0] + var_week[:, :, :, 1]
                var_week = numpy.delete( var_week, 1, axis=3 )

            else:
                raise RuntimeError(f"Unknown dataset - it should have either 3 or 4 dimensions, but instead has {num_dimensions}.")

            assert( var_week.dtype == numpy.float64 )
            week_data.append( var_week )

        # for each latitude in the file, spawn processes
        with concurrent.futures.ProcessPoolExecutor() as execute:   #max_workers=6
            lfuts = []
            for lat_i in range( num_lat ):
                # get hourly datas' as ndarrays for this entire latitude
                lat_data = []
                for var_week in week_data:
                    var_lat = numpy.ma.getdata( var_week[:][lat_i] )
                    lat_data.append( var_lat )

                lfuts.append( execute.submit( process_lat, lat_i, lat_data, data_group ) )

            # wait for futures and save to ods
            cnt = 1
            for lf in lfuts:
                if show_progress: print( f'\rOutput {o_names["filename"]} {year} week {week_i+1}/{num_weeks} latitude {cnt}/{num_lat} ', end='', flush=True )
                olat = lf.result()
                save_output( ods, week_i, olat )
                cnt += 1

    # clear the progress output line from screen
    if show_progress: print( f'\rOutput {o_names["filename"]} done.                                 ', flush=True )

    # close ods file and rename to mark as done
    ods.close()
    rename_finished_output( o_names )

    # free the input ds objects
    for ncds in ncds_group:
        ncds['dataset'].close()

# for all years process data_groups'
def load_netcdfs( flag_args:dict, inp_path:str, out_path:str, dir_name:str, start_year:int, end_year:int ):
    # calculate a unique number for each quarter degree on the planet.
    # makes having a unique filename for the coordinates simpler.
    #grid_num = CalcQtrDegGridNum( area_lat_long )
    years = list( range( start_year, end_year + 1 ) )
    for year in years:
        yidx = year - start_year

        for dg_name in global_data_groups:
            dg = data_groups[ dg_name ]
            process_data_group( flag_args, inp_path, out_path, dir_name, year, dg_name, dg );

def CalcQtrDegGridNum( area_lat_long ):
    grid_num = int( ((area_lat_long[0] + 90) * 4) * 360 * 4 + ((area_lat_long[1] + 180) * 4) )
    return grid_num

##########################################################
# main
def main():
    print( f'** HWITW data processing tool v{APP_VERSION} **\n')

    # default options
    input_path = '.'
    output_path = '.'

    # Initialize parser
    parser = argparse.ArgumentParser()
    parser.add_argument( "-i", "--input", help = "Set input path" )
    parser.add_argument( "-o", "--output", help = "Set output path" )
    parser.add_argument( "-f", "--force", action='store_true', help = "Force recalculation of all output" )
    parser.add_argument( "-p", "--progress", action='store_true', help = "Show fancy progress" )
    args = parser.parse_args()

    flag_args = {
        'force_recalc':args.force,
        'show_progress':args.progress }

    show_progress = args.progress

    if args.input:
        input_path = args.input

    if args.output:
        output_path = args.output

    if not pathlib.Path( input_path ).exists():
        print( f'input path does not exist: {input_path}' )
        exit( -1 )

    if not pathlib.Path( output_path ).exists():
        print( f'output path does not exist: {output_path}' )
        exit( -1 )

    # era5 goes from 1979 to present
    load_netcdfs( flag_args, input_path, output_path, 'cds_era5', 1979, current_time.year )

    # era5 back extension goes from 1950 to 1978
    load_netcdfs( flag_args, input_path, output_path, 'cds_era5_backext', 1950, 1978 )

if __name__ == '__main__':
    main()

max_stars_repo_name: NCEAS/cdstool

max_issues_repo_name: NCEAS/cdstool

max_forks_repo_name: NCEAS/cdstool


text:
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

# dataset
x_train = np.array([[1.1], [1.3], [1.5], [2.0], [2.2], [2.9], [3.0], [3.2], [3.2], [3.7], [3.9], [4.0], [4.0], [4.1], [4.5], [4.9], [5.1], [5.3], [5.9], [6.0], [6.8], [7.1], [7.9], [8.2], [8.7], [9.0], [9.5], [9.6], [10.3], [10.5]])
y_train = np.array([39343, 46205, 37731, 43525, 39891, 56642, 60150, 54445, 64445, 57189, 63218, 55794, 56957, 57081, 61111, 67938, 66029, 83088, 81363, 93940, 91738, 98273, 101302, 113812, 109431, 105582, 116969, 112635, 122391, 121872])
X_train, X_test, Y_train, Y_test = train_test_split( x_train, y_train, test_size=0.333, random_state=0)

class RidgeRegression():
    # Function for model training
    def fit( self, X, Y, n_epochs=1000 ):

        # no_of_training_examples, no_of_features
        self.training_size, self.input_dim = X.shape

        # weight initialization
        self.W = np.zeros(self.input_dim)
        self.X, self.Y = X, Y
        self.b = 0

        # gradient descent learning
        for epoch in range(n_epochs):
            self.update_weights()

    # Helper function to update weights in gradient descent
    def update_weights(self, learning_rate=0.01, delta=1):
        Y_pred = self.predict( self.X )

        error = self.Y - Y_pred

        # calculate gradients
        dW = ( -(2 * self.X.T.dot(error)) + (2 * delta * self.W)) / self.training_size
        db = -2 * np.sum(error) / self.training_size

        # update weights
        self.W = self.W - learning_rate * dW
        self.b = self.b - learning_rate * db
        return self

    def predict( self, X ) :
        return X.dot(self.W) + self.b

# Driver code
if __name__ == "__main__":
    # Model training
    model = RidgeRegression()
    model.fit(X_train, Y_train)

    # Prediction on test set
    Y_pred = model.predict( X_test )
    print( "Predicted values ", np.round( Y_pred[:3], 2 ) )
    print( "Real values      ", Y_test[:3] )
    print( "Trained W        ", round( model.W[0], 2 ) )
    print( "Trained b        ", round( model.b, 2 ) )

    # Visualization on test set
    plt.scatter( X_test, Y_test, color = 'blue' )
    plt.plot( X_test, Y_pred, color = 'orange' )
    plt.title( 'Salary vs Experience' )
    plt.xlabel( 'Years of Experience' )
    plt.ylabel( 'Salary' )
    plt.show()


text:
from __future__ import absolute_import, print_function, division

from os.path import join, exists
from tqdm import tqdm
import time
import json
import numpy as np
import sys
from datetime import datetime

import theano

import keras.backend as K
from keras.engine import Model, merge
from keras.layers import Embedding, LSTM, Input, Lambda, \
                         Distribute, RepeatVector, \
                         Reshape, Flatten, Dense, Dropout, \
                         InputLayer, DataLayer, Convolution2D, MaxPooling2D
from keras.optimizers import Adam
from keras.callbacks import Callback, ProgbarLogger, ModelCheckpoint, ProgbarV2, \
                            LearningRateScheduler, CSVLogger
from keras.regularizers import l2
from keras.utils.visualize_util import plot as kplot

from ikelos.layers import ProbabilityTensor, SoftAttention, Summarize, Fix, LambdaMask, set_name, compose, \
                          BranchLSTM, LastDimDistribute, DynamicEmbedding
from ikelos.data import Vocabulary, VocabManager

from .igor import Igor
from ..common import make_convolutional_embedding, make_logger, make_token_embedding, \
                     make_shallow_convolutional_embedding, make_minimal_token_embedding

try:
    import cPickle as pickle
except:
    import pickle

sys.setrecursionlimit(40000)

concat = lambda layers: merge(layers, mode='concat')
merge_batch = lambda layers: merge(layers, mode='concat', concat_axis=0)

class FergusRModel(object):
    def __init__(self, igor):

        now = datetime.now()
        self.run_name = "fergusr_{}mo_{}day_{}hr_{}min".format(now.month, now.day,
                                                                now.hour, now.minute)
        log_location = join(igor.log_dir, self.run_name+".log")
        self.logger = igor.logger = make_logger(igor, log_location)
        igor.verify_directories()
        self.igor = igor

    @classmethod
    def from_yaml(cls, yamlfile, kwargs=None):
        igor = Igor.from_file(yamlfile)
        model = cls(igor)

        igor.prep()
        model.make(kwargs)
        return model

    @classmethod
    def from_config(cls, config, kwargs=None):
        igor = Igor(config)
        model = cls(igor)

        igor.prep()
        model.make(kwargs)
        return model

    def load_checkpoint_weights(self):
        weight_file = join(self.igor.model_location,
                           self.igor.saving_prefix,
                           self.igor.checkpoint_weights)
        if exists(weight_file):
            self.logger.info("+ Loading checkpoint weights")
            self.model.load_weights(weight_file, by_name=True)
        else:
            self.logger.warning("- Checkpoint weights do not exist; {}".format(weight_file))

    def plot(self):
        filename = join(self.igor.model_location,
                        self.igor.saving_prefix,
                        'model_visualization.png')
        kplot(self.model, to_file=filename)
        self.logger.debug("+ Model visualized at {}".format(filename))

    def make(self, theano_kwargs=None):
        """Construct the Fergus-Recurrent model

        Model:
            Input at time t:
                - Soft attention over embedded lexemes of children of node_t
                - Embedded lexeme of node_t
            Compute:
                - Inputs are fed into a recurrent tree s.t. hidden states travel down branches
                - node_t's supertag embeddings are retrieved
                - output of recurrent tree at time t is aligned with each supertag vector
                - a vectorized probability function computes a distribution
            Output:
                - Distribution over supertags for node_t
        """
        if self.igor.embedding_type == "convolutional":
            make_convolutional_embedding(self.igor)
        elif self.igor.embedding_type == "token":
            make_token_embedding(self.igor)
        elif self.igor.embedding_type == "shallowconv":
            make_shallow_convolutional_embedding(self.igor)
        elif self.igor.embedding_type == "minimaltoken":
            make_minimal_token_embedding(self.igor)
        else:
            raise Exception("Incorrect embedding type")

        spine_input_shape = (self.igor.batch_size,
                             self.igor.max_sequence,
                             self.igor.max_num_supertags)

        node_input_shape = (self.igor.batch_size, self.igor.max_sequence)

        dctx_input_shape = (self.igor.batch_size, self.igor.max_sequence, self.igor.max_daughter_size)

        E, V = self.igor.word_embedding_size, self.igor.word_vocab_size # for word embeddings
        repeat_N = self.igor.max_num_supertags # for lex
        repeat_D = self.igor.max_daughter_size
        mlp_size = self.igor.mlp_size

        ## dropout parameters
        p_emb = self.igor.p_emb_dropout
        p_W = self.igor.p_W_dropout
        p_U = self.igor.p_U_dropout
        w_decay = self.igor.weight_decay
        p_mlp = self.igor.p_mlp_dropout

        #### make layer inputs
        spineset_in = Input(batch_shape=spine_input_shape, name='parent_spineset_in', dtype='int32')
        phead_in = Input(batch_shape=node_input_shape, name='parent_head_input', dtype='int32')
        dctx_in = Input(batch_shape=dctx_input_shape, name='daughter_context_input', dtype='int32')
        topology_in = Input(batch_shape=node_input_shape, name='node_topology', dtype='int32')

        ##### params
        def predict_params():
            return {'output_dim': 1,
                    'W_regularizer': l2(w_decay),
                    'activation': 'relu',
                    'b_regularizer':l2(w_decay)}

        ### Layer functions
        ############# Convert the word indices to vectors
        F_embedword = Embedding(input_dim=V, output_dim=E, mask_zero=True,
                                W_regularizer=l2(w_decay), dropout=p_emb, name='embedword')
        if self.igor.saved_embeddings is not None:
            print("Loading saved embeddings....")
            F_embedword.initial_weights = [self.igor.saved_embeddings]

        F_probability = ProbabilityTensor(name='predictions',
                                          dense_function=Dense(**predict_params()))
        ### composition functions

        F_softdaughters = compose(LambdaMask(lambda x, mask: None, name='remove_attention_mask'),
                                  Distribute(SoftAttention(name='softdaughter'), name='distribute_softdaughter'),
                                  F_embedword)

        F_align = compose(Distribute(Dropout(p_mlp)),
                           Distribute(Dense(mlp_size, activation='relu')),
                           concat)

        F_rtn = compose(RepeatVector(repeat_N, axis=2, name='repeattree'),
                        BranchLSTM(self.igor.rtn_size, name='recurrent_tree1', return_sequences=True))

        F_predict = compose(Distribute(F_probability, name='distribute_probability'),
                            Distribute(Dropout(p_mlp)),  ### need a separate one because the 'concat' is different for the two situations
                            LastDimDistribute(Dense(mlp_size, activation='relu')),
                            concat)

        ############################ new ###########################

        dctx = F_softdaughters(dctx_in)
        parent = F_embedword(phead_in)
        #node_context = F_align([parent, dctx])
        #import pdb
        #pdb.set_trace()

        ### put into tree
        aligned_node = F_align([parent, dctx])
        node_context = F_rtn([aligned_node, topology_in])

        parent_spines = self.igor.F_embedspine(spineset_in)
        ### get probability
        predictions = F_predict([node_context, parent_spines])

        ##################
        ### make model
        ##################
        self.model = Model(input=[dctx_in, phead_in, topology_in, spineset_in],
                           output=predictions,
                           preloaded_data=self.igor.preloaded_data)

        ##################
        ### compile model
        ##################
        optimizer = Adam(self.igor.LR, clipnorm=self.igor.max_grad_norm,
                                       clipvalue=self.igor.grad_clip_threshold)
        theano_kwargs = theano_kwargs or {}
        self.model.compile(loss='categorical_crossentropy', optimizer=optimizer,
                           metrics=['accuracy'], **theano_kwargs)

        if self.igor.from_checkpoint:
            self.load_checkpoint_weights()
        elif not self.igor.in_training:
            raise Exception("No point in running this without trained weights")

    def train(self):
        train_data = self.igor.train_gen(forever=True)
        dev_data = self.igor.dev_gen(forever=True)
        N = self.igor.num_train_samples
        E = self.igor.num_epochs
        # generator, samplers per epoch, number epochs
        callbacks = [ProgbarV2(3, 10)]

        checkpoint_fp = join(self.igor.model_location,
                             self.igor.saving_prefix,
                             self.igor.checkpoint_weights)
        self.logger.info("+ Model Checkpoint: {}".format(checkpoint_fp))

        callbacks += [ModelCheckpoint(filepath=checkpoint_fp, verbose=1, save_best_only=True)]
        callbacks += [LearningRateScheduler(lambda epoch: self.igor.LR * 0.9**(epoch))]

        csv_location = join(self.igor.log_dir, self.run_name+".csv")
        callbacks += [CSVLogger(csv_location)]
        self.model.fit_generator(generator=train_data, samples_per_epoch=N, nb_epoch=E,
                                 callbacks=callbacks, verbose=1,
                                 validation_data=dev_data,
                                 nb_val_samples=self.igor.num_dev_samples)

    def debug(self):
        dev_data = self.igor.dev_gen(forever=False)
        X,Y = next(dev_data)
        self.model.predict_on_batch(X)
        #self.model.evaluate_generator(dev_data, self.igor.num_dev_samples)

    def profile(self, num_iterations=1):
        train_data = self.igor.train_gen(forever=True)
        dev_data = self.igor.dev_gen(forever=True)
        # generator, samplers per epoch, number epochs
        callbacks = [ProgbarV2(1, 10)]
        self.logger.debug("+ Beginning the generator")
        self.model.fit_generator(generator=train_data,
                                 samples_per_epoch=self.igor.batch_size*10,
                                 nb_epoch=num_iterations,
                                 callbacks=callbacks,
                                 verbose=1,
                                 validation_data=dev_data,
                                 nb_val_samples=self.igor.batch_size)
        self.logger.debug("+ Calling theano's pydot print.. this might take a while")
        theano.printing.pydotprint(self.model.train_function.function,
                                   outfile='theano_graph.png',
                                   var_with_name_simple=True,
                                   with_ids=True)
        self.logger.debug("+ Calling keras' print.. this might take a while")
        self.plot("keras_graph.png")
        #self.model.profile.print_summary()

    def __call__(self, data):
        if self.model is None:
            raise Exception("model not instantiated yet; please call make()")
        assert isinstance(data, list)
        B = data[0].shape[0]
        return self.model.predict(data, batch_size=B)

reason: import numpy,import theano


text:
# coding: utf-8

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import torch
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def get_features(df):
    x = df.x.values
    y = df.y.values
    z = df.z.values

    r = np.sqrt(x**2 + y**2 + z**2)

    x2 = x/r
    y2 = y/r

    r2 = np.sqrt(x**2 + y**2)
    z2 = z/r2

    df['x2'] = x2
    df['y2'] = y2
    df['z2'] = z2
    df['r2'] = r2
    df['r'] = r

    # eta, and phi
    theta = np.arccos(z/r)
    phi = np.arctan2(y, x)
    eta = - np.log(np.tan(theta/2.))
    df['eta'] = eta
    df['phi'] = phi
    df['absZ'] = np.abs(z)

    df['z1'] = z/r2

    return df

import matplotlib.ticker as plticker
loc = plticker.MultipleLocator(base=200)
loc0p4 = plticker.MultipleLocator(base=0.4)

def view(df, pID, numb=10, pp=None, pID_name='particle_id'):
    fig = plt.figure(figsize=(15, 12))
    ax1 = fig.add_subplot(221)
    ax2 = fig.add_subplot(222)
    ax3 = fig.add_subplot(223)
    ax4 = fig.add_subplot(224, projection='polar')
    if pp is None:
        if numb < 0:
            pp = pID
        else:
            pp = pID.sample(numb)

    for p in pp:
        if p == 0:
            continue
        data = df[df[pID_name] == p][['r2', 'eta', 'phi', 'z', 'absZ']].sort_values(by=['absZ']).values
        ax1.plot(data[:,3], data[:,0], '-', alpha=0.5, lw=4, label='{}'.format(p))
        ax1.scatter(data[:,3], data[:,0], marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)

        ax2.plot(data[:,3], data[:,1], '-', alpha=0.5, lw=4, label='{}'.format(p))
        ax2.scatter(data[:,3], data[:,1], marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)

        ax3.plot(data[:,3], data[:,2], '-', alpha=0.5, lw=4, label='{}'.format(p))
        ax3.scatter(data[:,3], data[:,2], marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)

        ax4.plot(data[:,2], np.abs(data[:,3]), '-', alpha=0.5, lw=4, label='{}'.format(p))
        ax4.scatter(data[:,2], np.abs(data[:,3]), marker='o', edgecolor='black', s=np.ones(len(data))*30, alpha=0.5)

    ax1.set_xlabel("Z [mm]")
    ax1.set_ylabel("r [mm]")
    ax1.xaxis.set_minor_locator(loc)
    ax1.set_xlim(-3200, 3200)
    ax1.set_ylim(0, 1100)

    ax2.set_xlabel("Z [mm]")
    ax2.set_ylabel("eta")
    ax2.yaxis.set_minor_locator(loc0p4)
    ax2.xaxis.set_minor_locator(loc)
    ax2.set_xlim(-3200, 3200)
    ax2.set_ylim(-5, 5)
    ax2.grid(True)

    ax3.set_xlabel("Z [mm]")
    ax3.set_ylabel("phi")
    ax3.xaxis.set_minor_locator(loc)
    ax3.set_xlim(-3200, 3200)
    ax3.set_ylim(-np.pi, np.pi)

    ax4.grid(True)
    ax4.set_ylim(0, 3200)
    if numb < 12:
        ax4.legend(loc='center left', bbox_to_anchor=(1, 0.5))

    fig.tight_layout()
    plt.show()
    return pp, ax1, ax2, ax3, ax4

def make_uID(det):
    data = []
    icount = 0
    volumes = np.unique(det['volume_id'])
    for vol in volumes:
        layers = np.unique(det[det['volume_id'] == vol][['layer_id']])
        for layer in layers:
            modules = np.unique(det[(det['volume_id'] == vol) & (det['layer_id'] == layer)][['module_id']])
            for module in modules:
                data.append([vol, layer, module, icount])
                icount += 1
    return pd.DataFrame(data=data, columns=['volume_id', 'layer_id', 'module_id', 'uID'])

import time
import math

def timeSince(since):
    now = time.time()
    s = now - since
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

def tunable_parameters(model):
    return sum([p.numel() for p in model.parameters() if p.requires_grad])

import random
def random_choice(l):
    return l[random.randint(0, len(l) - 1)]

def plot_detector(hits):
    fig = plt.figure(figsize=(8, 6))
    ax = fig.add_subplot(111)
    hits.plot.scatter(x='z', y='r2', ax=ax)
    eta = [0.1, 1, 2, 3, 3.2, 4, 4.5]
    # eta = - np.log(np.tan(theta/2.))
    theta = [2 * math.atan(math.exp(-x)) for x in eta]
    xp = np.arange(0, 3200, 200)
    xm = np.arange(-3200, 200, 200)
    for th in theta:
        yp = xp * np.tan(th)
        ym = xm * np.tan(-1 * th)
        ax.plot(xp, yp, '-', alpha=0.5, lw=3)
        ax.plot(xm, ym, '-', alpha=0.5, lw=3)

    ax.set_xlim(-3200, 3200)
    ax.set_ylim(0, 1100)
    ax.xaxis.set_minor_locator(loc)
    plt.tight_layout()
    plt.show()


text:
from setuptools import setup
from setuptools.extension import Extension
from Cython.Build import cythonize
import numpy
#import Cython.Compiler.Options
#Cython.Compiler.Options.get_directive_defaults()['cdivision'] = True
#Cython.Compiler.Options.get_directive_defaults()['boundscheck'] = False
#Cython.Compiler.Options.get_directive_defaults()['wraparound'] = False
#Cython.Compiler.Options.get_directive_defaults()['profile'] = True

ext_modules=[
    Extension('slowquant.molecularintegrals.runMIcython',['slowquant/molecularintegrals/runMIcython.pyx']),
    Extension('slowquant.coupledcluster.CythonCC',['slowquant/coupledcluster/CythonCC.pyx'])]

setup(ext_modules=cythonize(ext_modules), include_dirs=[numpy.get_include()])

max_stars_repo_name: Melisius/Hartree-Fock

max_issues_repo_name: erikkjellgren/SlowQuant

max_forks_repo_name: Melisius/Hartree-Fock


text:
"""
This class creates a personalized K-model instance
"""

#Common imports
from .k_model import KroneckerModel
import numpy as np
import pandas as pd

#Import kronecker model for annotation purposes

class PersonalKModel():
    """This class holds a reference to a kronecker model and
        a personalization map"""

    def __init__(self, kronecker_model: KroneckerModel, personalization_map: np.ndarray, average_fit: np.ndarray, output_name: str,
                 gait_fingerprint: np.ndarray = None,
                 subject_name: np.ndarray = None):
        """
        Initialize the PersonalKModel Object

        Keyword arguments:
        kronecker_model -- KroneckerModel object
        personalization_map -- Numpy array with shape
                              (num_gait_fingerprints, model_output_size)
        average_fit -- Numpy array with
                       shape (1, model_output_size)
        gait_fingerprint -- an individual's optimized gait fingerprint with
                            shape(1,model_output_size)
        subject_name -- string of the subject name
        """
        #Get the model and the amount of functions
        self.model = kronecker_model
        self.num_models = kronecker_model.get_num_basis()

        #Get the personalization_map
        self.pmap = personalization_map
        self.num_gait_fingerpints = personalization_map.shape[0]

        #If we have the personalized fit, add it as well
        self.average_fit = average_fit

        #Personalized fit for one person
        # Store the name so you can keep track of who's fit it is
        self.subject_gait_fingerprint = gait_fingerprint
        self.subject_name = subject_name

        #Keep the output name
        self.output_name = output_name

    def get_subject_name(self):
        """
        Returns the subjects name, if any

        This is useful is the instance to this object is stored
        """
        return self.subject_name

    def set_pmap(self, new_pmap):
        """
        Set the personalization map of the model
        """
        self.pmap = new_pmap

    def evaluate(self,input_data,use_personalized_fit=False, use_average_fit=False):
        """
        Evaluate a Kronecker Model multiplied by
            the (average fit plus personalization map times gait fingerprint)
            which gives a scalar output

        In the default behavior, it will use the gait fingerprint in the input_dataset,
        which is a numpy array that has a column corresponding to each model input and
        then the gait fingerprints

        The subject-average fit can also be used by setting use_average_fit to true

        If the model was initialized with a subject gait fingerprint, use_personalized_fit
        will override the gait fingerprint in the dataset for the least squares optimal gait fingerprint
        for that subject

        Keyword arguments:
        dataset -- numpy array that contains the model inputs and the gait fingeprints
                   shape (num_datapoints, num_models + num_gait_fingerprints)

        use_personalized_fit -- Boolean if true just use the fit for
                                the subject that was initialized in
        use_average_fit -- Boolean, if true just use the average fit

        Returns
        output -- evaluated personalized kronecker model output
                  shape (num_datapoints,1)

        Notes:
        user_personalized_fit takes precedence over use_average_fit

        e.g. if user_personalized_fit is true, it does not matter what
        use_average_fit is
        """

        #Calculate the output of the kronecker model
        # shape(num_datapoints, model_output_size)
        kronecker_output = self.model.evaluate(input_data)

        #Default behaviour, use the gait fingerprints from the dataset
        #The gait fingeprints will be after the model variables
        gait_fingerprints = input_data[:, self.num_models:]

        #Get the gait_fingerprints
        if(use_personalized_fit == True):

            #Return error if the object was not initialized
            if(self.subject_gait_fingerprint is None):
                return TypeError("Object was not initialized \
                            with a personalized_fit parameter")

            gait_fingerprints = self.subject_gait_fingerprint

        #Don't do any personalization and just do average fit
        elif(use_average_fit == True):
            #Get the number of datapoints
            num_datapoints = input_data.shape[0]

            #To not include personalization, just set the gait fingerprints
            # to zero
            gait_fingerprints = np.zeros((num_datapoints,self.num_gait_fingerpints))

        #Get the personalized vector
        # shape (num_datapoints, output_model_size)
        personalization_vector = self.average_fit + gait_fingerprints @ self.pmap

        ## Reshape in order to get the output of the shape (num_datapoints, 1)

        #Shape (num_datapoints, 1, model_size)
        kronecker_output = kronecker_output[:,np.newaxis,:]
        #Shape (num_datapoints, model_size, 1)
        personalization_vector = personalization_vector[:,:,np.newaxis]

        #Output is kronecker_output multiplied with personalization vector
        # shape(num_datapoints, 1)
        output = (kronecker_output @ personalization_vector).reshape(-1,1)

        return output


text:
# -*- coding: utf-8 -*-
#   ramaplot.DiffDataset.py
#
#   Copyright (C) 2015 Karl T Debiec
#   All rights reserved.
#
#   This software may be modified and distributed under the terms of the
#   BSD license. See the LICENSE file for details.
"""
Manages difference datasets.
"""
################################### MODULES ###################################
from __future__ import absolute_import,division,print_function,unicode_literals
from .myplotspec.Dataset import Dataset
################################### CLASSES ###################################
class DiffDataset(Dataset):
    """
    Manages difference datasets.
    """

    @classmethod
    def get_cache_key(cls, mask_cutoff=None, *args, **kwargs):
        """
        Generates tuple of arguments to be used as key for dataset
        cache.

        Arguments documented under :func:`__init__`.
        """
        from .myplotspec import multi_get_copy
        import six

        minuend_kw = multi_get_copy(["minuend", "minuend_kw"], kwargs, {})
        minuend_cls = minuend_kw.pop("cls")
        if minuend_cls is None:
            minuend_cls = Dataset
        elif isinstance(minuend_cls, six.string_types):
            mod_name = ".".join(minuend_cls.split(".")[:-1])
            minuend_cls_name   = minuend_cls.split(".")[-1]
            mod = __import__(mod_name, fromlist=[minuend_cls_name])
            minuend_cls = getattr(mod, minuend_cls_name)
        key = [cls, mask_cutoff, minuend_cls.get_cache_key(**minuend_kw)]

        subtrahend_kw = multi_get_copy(["subtrahend", "subtrahend_kw"],
          kwargs, {})
        if isinstance(subtrahend_kw, dict):
            subtrahend_kw = [subtrahend_kw]
        for sh_kw in subtrahend_kw:
            sh_cls = sh_kw.pop("cls")
            if sh_cls is None:
                sh_cls = Dataset
            elif isinstance(sh_cls, six.string_types):
                mod_name = ".".join(sh_cls.split(".")[:-1])
                sh_cls_name   = sh_cls.split(".")[-1]
                mod = __import__(mod_name, fromlist=[sh_cls_name])
                sh_cls = getattr(mod, sh_cls_name)
            key.append(sh_cls.get_cache_key(**sh_kw))

        return tuple(key)

    def __init__(self, dataset_classes=None, mask_cutoff=None, verbose=1,
        debug=0, **kwargs):
        """
        Initializes dataset.

        Arguments:
          minuend_kw (dict): Keyword arguments used to generate first
            dataset
          subtrahend_kw (dict, list): Keyword arguments used to generate
            datasets to subtract; if dict, arguments for single dataset;
            if list, list of dicts of keyword arguments used to generate
            multiple datasets
          dataset_cache (dict, optional): Cache of previously-loaded
            datasets
          mask_cutoff (float): Maximum free energy; points in the
            nascent free energy difference surface at which the free
            energy of both dataset_1 and dataset_2 are greater than
            mask_cutoff will be set to NaN; used to hide distracting
            large differences in depopulated regions; Currently only
            supports free energy maximum and not probability minimum
        """
        from copy import copy
        import numpy as np
        from .myplotspec import multi_get_copy

        # Load minuend
        minuend_kw = multi_get_copy(["minuend", "minuend_kw"], kwargs, {})
        if not "dataset_cache" in minuend_kw:
            minuend_kw["dataset_cache"] = kwargs.get("dataset_cache", {})
        minuend = self.load_dataset(verbose=verbose,
                    debug=debug, **minuend_kw)
        self.x_centers = copy(minuend.x_centers)
        self.y_centers = copy(minuend.y_centers)
        self.x_width = copy(minuend.x_width)
        self.y_width = copy(minuend.y_width)
        self.x_bins = copy(minuend.x_bins)
        self.y_bins = copy(minuend.y_bins)
        self.dist = copy(minuend.dist)

        # Prepare mask
        cutmask = np.zeros_like(self.dist)
        nanmask = np.zeros_like(self.dist)
        if mask_cutoff is not None:
            cutmask[self.dist >= mask_cutoff ] = 1
        nanmask[np.isnan(self.dist)] = 1

        # Subtract subtrahends
        subtrahend_kw = multi_get_copy(["subtrahend", "subtrahend_kw"],
          kwargs, {})
        if isinstance(subtrahend_kw, dict):
            subtrahend_kw = [subtrahend_kw]
        for sh_kw in subtrahend_kw:
            if not "dataset_cache" in sh_kw:
                sh_kw["dataset_cache"] = kwargs.get("dataset_cache", {})
            sh = self.load_dataset(verbose=verbose, debug=debug, **sh_kw)

            # Validate comparability of datasets
            if not ((minuend.x_centers == sh.x_centers).all()
            and     (minuend.y_centers == sh.x_centers).all()
            and      minuend.x_width   == sh.x_width
            and      minuend.y_width   == sh.y_width
            and     (minuend.x_bins    == sh.x_bins).all()
            and     (minuend.y_bins    == sh.y_bins).all()):
                raise TypeError()

            # Prepare data
            self.dist -= sh.dist

            # Prepare mask
            if mask_cutoff is not None:
                cutmask[np.logical_and(cutmask==1, sh.dist<=mask_cutoff)] = 0
            nanmask[np.isnan(sh.dist)] = 1

        self.mask = np.ma.masked_array(np.ones_like(self.dist),
          np.logical_not(np.logical_or(cutmask, nanmask)))

max_stars_repo_name: KarlTDebiec/Ramaplot

max_issues_repo_name: KarlTDebiec/Ramaplot

max_forks_repo_name: KarlTDebiec/Ramaplot


text:
import torch
from fastprogress.fastprogress import master_bar, progress_bar
import numpy as np
from time import time
from time import strftime, gmtime
import pdb

__all__ = ["LearnerCallback", "Learner"]

class LearnerCallback():

    def get_metric_names(self):
        return []

    def on_train_begin(self):
        return

    def on_train_end(self):
        return

    def on_epoch_begin(self):
        return

    def on_epoch_end(self):
        return

    def on_batch_begin(self, features, target, train):
        return

    def on_batch_end(self, output, target, train):
        return

class Learner:

    def __init__(self, model, loss_func, train_dl, valid_dl, optimizer, learner_callback=None, gpu_id=0, predict_smaple_func=None, masks_and_hulls=True, save_func=None, stacked_net=False, early_stopping_metric=None):
        self.device = torch.device("cuda:"+str(gpu_id))
        self.predict_smaple_func = predict_smaple_func if predict_smaple_func is not None else lambda epoch : None
        self.save_func = save_func if save_func is not None else lambda epoch : None
        self.early_stopping_metric = early_stopping_metric
        self.model = model
        self.train_dl = train_dl
        self.valid_dl = valid_dl
        self.optimizer = optimizer
        self.loss_func = loss_func
        self.learner_callback = learner_callback if learner_callback is not None else LearnerCallback()
        self.metric_names = self.learner_callback.get_metric_names()
        self.metrics = None
        self.train_losses = None
        self.valid_losses = None
        self.masks_and_hulls = masks_and_hulls
        self.stacked_net = stacked_net

    def set_loss_func(self, loss_func):
        self.loss_func = loss_func

    def loss_batch(self, x_data, y_data, masks, hulls, scheduler=None, is_train=True):
        "Calculate loss and metrics for a batch."

        self.learner_callback.on_batch_begin(x_data, y_data, train=is_train)
        self.optimizer.zero_grad()

        out = self.model(x_data)

        if self.masks_and_hulls:
            if self.stacked_net:
                loss = sum(self.loss_func(o, y_data, masks, hulls) for o in out)
            else:
                loss = self.loss_func(out, y_data, masks, hulls)
        else:
            if self.stacked_net:
                loss = sum(self.loss_func(o, out[1].shape) for o in out)
            else:
                loss = self.loss_func(out, y_data)

        if is_train:
            loss.backward()
            self.optimizer.step()
            if scheduler is not None:
                scheduler.step()

        if self.stacked_net:
            self.learner_callback.on_batch_end(out[-1].detach().cpu(), y_data.detach().cpu(), train=is_train)
        else:
            self.learner_callback.on_batch_end(out.detach().cpu(), y_data.detach().cpu(), train=is_train)

        return float(loss.detach().cpu())

    def validate(self,parrent_bar):
        "Calculate `loss_func` in evaluation mode."
        self.model.eval()
        with torch.no_grad():
            val_losses = []

            for data_zip in progress_bar(self.valid_dl, parent=parrent_bar):

                if self.masks_and_hulls:
                    names,x_data,y_data, masks, hulls = (data_zip[0],data_zip[1],data_zip[2],data_zip[3],data_zip[4])
                    masks = masks.to(self.device)
                    hulls = hulls[:,None].repeat(1,masks.shape[1],1,1).to(self.device)
                else:
                    names,x_data,y_data,masks,hulls = (data_zip[0],data_zip[1],data_zip[2],None,None)

                x_data = x_data.to(self.device)
                y_data = y_data.to(self.device)
                val_loss = self.loss_batch(x_data, y_data, masks, hulls, is_train=False)
                parrent_bar.child.comment = str(round(val_loss,4))
                val_losses.append(val_loss)

        return np.array(val_losses).mean()

    def get_losses(self, train=True, valid=True):
        assert (train or valid), "train or valid must be True"
        losses = []
        if train:
            losses.append(np.array(self.train_losses))

        if valid:
            losses.append(np.array(self.valid_losses))

        return losses[0] if len(losses) == 1 else losses

    def predict(self, inp, eval_mode=True):
        if eval_mode:
            self.model.eval()
            with torch.no_grad():
                return self.model(inp)
        else:
            return self.model(inp)

    def get_metrics(self):
        return np.array(self.metrics)

    def fit(self, epochs, one_cylce=True):
        assert self.loss_func is not None, "loss function not definied!"
        self.train_losses = []
        self.valid_losses = []
        self.metrics = []
        scheduler = torch.optim.lr_scheduler.OneCycleLR(self.optimizer, max_lr=self.optimizer.param_groups[0]["lr"]*10, steps_per_epoch=len(self.train_dl), epochs=epochs) if one_cylce else None
        pbar = master_bar(range(epochs))
        pbar.write(["Epoch","train_loss","valid_loss"] + self.metric_names + ["time"], table=True)
        self.learner_callback.on_train_begin()
        Stop = False
        for epoch in pbar:
            if Stop:
                break
            t1 = time()
            self.model.train()
            self.learner_callback.on_epoch_begin()

            for data_zip in progress_bar(self.train_dl, parent=pbar):

                if self.masks_and_hulls:
                    names,x_data,y_data, masks, hulls = (data_zip[0],data_zip[1],data_zip[2],data_zip[3],data_zip[4])
                    masks = masks.to(self.device)
                    hulls = hulls[:,None].repeat(1,masks.shape[1],1,1).to(self.device)
                else:
                    names,x_data,y_data,masks,hulls = (data_zip[0],data_zip[1],data_zip[2],None,None)

                x_data = x_data.to(self.device)
                y_data = y_data.to(self.device)

                train_loss = self.loss_batch(x_data, y_data, masks, hulls, scheduler=scheduler, is_train=True)

                pbar.child.comment = str(round(train_loss,4))

            valid_loss = self.validate(parrent_bar=pbar)

            self.train_losses.append(train_loss)
            self.valid_losses.append(valid_loss)
            self.predict_smaple_func(epoch)
            self.save_func(epoch)
            met = self.learner_callback.on_epoch_end()

            t2 = time()
            metrics_table = []
            if met is not None:
                self.metrics.append(met)
                for name,value in zip(self.metric_names, met):
                    metrics_table.append(f'{round(value,4):.4f}')

            metrics_table.append(strftime('%M:%S', gmtime(round(t2-t1))))
            graphs = [[np.arange(len(self.train_losses)),np.array(self.train_losses)], [np.arange(len(self.valid_losses)),np.array(self.valid_losses)]]
            pbar.update_graph(graphs, [0,epochs], [0,np.array([self.train_losses,self.valid_losses]).max()])
            pbar.write([f'{epoch:04d}',f'{round(train_loss,4):.4f}',f'{round(valid_loss,4):.4f}'] + metrics_table, table=True)

            if self.early_stopping_metric is not None:
                idx = self.metric_names.index(self.early_stopping_metric[0])
                if float(metrics_table[idx])>self.early_stopping_metric[1]:
                    Stop=True

        self.learner_callback.on_train_end()

max_stars_repo_name: johansi/nnlib

max_issues_repo_name: johansi/nnlib

max_forks_repo_name: johansi/nnlib


text:
# column names Survived,Pclass,Name,Sex,Age,
# Siblings/Spouses Aboard, Parents/Children Aboard,Fare
# inspect what kind of datatypes would be needed
# integer columns - survived(0/1 boolean)
# Pclass, Age, Siblings, Parents, Fare

import pandas as pd
import os, psycopg2
import numpy as np
from dotenv import load_dotenv
from psycopg2.extras import execute_values

CSV_FILEPATH = os.path.join(os.path.dirname(__file__), "titanic.csv")

titanic = pd.read_csv(CSV_FILEPATH)
print(titanic.head())

psycopg2.extensions.register_adapter(np.int64, psycopg2._psycopg.AsIs)
# connect to pg db
load_dotenv()

DB_NAME = os.getenv("DB_NAME", default="OOPS")
DB_USER = os.getenv("DB_USER", default="OOPS")
DB_PW = os.getenv("DB_PW", default="OOPS")
DB_HOST = os.getenv("DB_HOST", default="OOPS")

connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PW, host=DB_HOST)
print("------------------------------------------------")
print(type(connection)) #> <class 'psycopg2.extensions.connection'>

cursor = connection.cursor()
print("------------------------------------------------")
print(type(cursor)) #> <class 'psycopg2.extensions.cursor'>

# create table
create_table =  """
                DROP TABLE IF EXISTS titanic;
                CREATE TABLE IF NOT EXISTS titanic (
                    id SERIAL PRIMARY KEY,
                    "Survived" int4,
                    "Pclass" int4,
                    "Name" text,
                    "Sex" text,
                    "Age" int4,
                    "Siblings/Spouses Aboard" int4,
                    "Parents/Children Aboard" int4,
                    "Fare" float8
);
                """

cursor.execute(create_table)

# insert titanic data

insert_data = """
            INSERT INTO titanic(
                                Survived, Pclass, Name,
                                Sex, Age, Siblings/Spouse Aboard,
                                Parents/Children Aboard, Fare
                                )
                """

list_to_tuples = list(titanic.to_records(index=False))

execute_values(cursor, insert_data, list_to_tuples)

connection.commit()

cursor.close()
connection.close()


text:
# Copyright 2018 D-Wave Systems Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
# ================================================================================================

from __future__ import division, absolute_import

import dimod
import numpy as np

from six import iteritems

from dwave.embedding.chain_breaks import broken_chains

__all__ = ['target_to_source', 'chain_to_quadratic', 'chain_break_frequency', 'adjacency_to_edges']

def target_to_source(target_adjacency, embedding):
    """Derive the source adjacency from an embedding and target adjacency.

    Args:
        target_adjacency (dict/:class:`networkx.Graph`):
            A dict of the form {v: Nv, ...} where v is a node in the target graph and Nv is the
            neighbors of v as an iterable. This can also be a networkx graph.

        embedding (dict):
            A mapping from a source graph to a target graph.

    Returns:
        dict: The adjacency of the source graph.

    Raises:
        ValueError: If any node in the target_adjacency is assigned more
            than  one node in the source graph by embedding.

    Examples:

        >>> target_adjacency = {0: {1, 3}, 1: {0, 2}, 2: {1, 3}, 3: {0, 2}}  # a square graph
        >>> embedding = {'a': {0}, 'b': {1}, 'c': {2, 3}}
        >>> source_adjacency = dwave.embedding.target_to_source(target_adjacency, embedding)
        >>> # triangle graph:
        >>> source_adjacency   # doctest: +SKIP
        {'a': {'b', 'c'}, 'b': {'a', 'c'}, 'c': {'a', 'b'}}

        This function also works with networkx graphs.

        >>> import networkx as nx
        >>> target_graph = nx.complete_graph(5)
        >>> embedding = {'a': {0, 1, 2}, 'b': {3, 4}}
        >>> dwave.embedding.target_to_source(target_graph, embedding)   # doctest: +SKIP
        {'a': {'b'}, 'b': {'a'}}

    """
    # the nodes in the source adjacency are just the keys of the embedding
    source_adjacency = {v: set() for v in embedding}

    # we need the mapping from each node in the target to its source node
    reverse_embedding = {}
    for v, chain in iteritems(embedding):
        for u in chain:
            if u in reverse_embedding:
                raise ValueError("target node {} assigned to more than one source node".format(u))
            reverse_embedding[u] = v

    # v is node in target, n node in source
    for v, n in iteritems(reverse_embedding):
        neighbors = target_adjacency[v]

        # u is node in target
        for u in neighbors:

            # some nodes might not be assigned to chains
            if u not in reverse_embedding:
                continue

            # m is node in source
            m = reverse_embedding[u]

            if m == n:
                continue

            source_adjacency[n].add(m)
            source_adjacency[m].add(n)

    return source_adjacency

def chain_to_quadratic(chain, target_adjacency, chain_strength):
    """Determine the quadratic biases that induce the given chain.

    Args:
        chain (iterable):
            The variables that make up a chain.

        target_adjacency (dict/:class:`networkx.Graph`):
            Should be a dict of the form {s: Ns, ...} where s is a variable
            in the target graph and Ns is the set of neighbours of s.

        chain_strength (float):
            The magnitude of the quadratic bias that should be used to create chains.

    Returns:
        dict[edge, float]: The quadratic biases that induce the given chain.

    Raises:
        ValueError: If the variables in chain do not form a connected subgraph of target.

    Examples:

        >>> chain = {1, 2}
        >>> target_adjacency = {0: {1, 2}, 1: {0, 2}, 2: {0, 1}}
        >>> dimod.embedding.chain_to_quadratic(chain, target_adjacency, 1)
        {(1, 2): -1}

    """
    quadratic = {}  # we will be adding the edges that make the chain here

    # do a breadth first search
    seen = set()
    try:
        next_level = {next(iter(chain))}
    except StopIteration:
        raise ValueError("chain must have at least one variable")
    while next_level:
        this_level = next_level
        next_level = set()
        for v in this_level:
            if v not in seen:
                seen.add(v)

                for u in target_adjacency[v]:
                    if u not in chain:
                        continue
                    next_level.add(u)
                    if u != v and (u, v) not in quadratic:
                        quadratic[(v, u)] = -chain_strength

    if len(chain) != len(seen):
        raise ValueError('{} is not a connected chain'.format(chain))

    return quadratic

def chain_break_frequency(samples_like, embedding):
    """Determine the frequency of chain breaks in the given samples.

    Args:
        samples_like (samples_like/:obj:`dimod.SampleSet`):
            A collection of raw samples. 'samples_like' is an extension of NumPy's array_like.
            See :func:`dimod.as_samples`.

        embedding (dict):
            Mapping from source graph to target graph as a dict of form {s: {t, ...}, ...},
            where s is a source-model variable and t is a target-model variable.

    Returns:
        dict: Frequency of chain breaks as a dict in the form {s: f, ...},  where s
        is a variable in the source graph and float f the fraction
        of broken chains.

    Examples:
        This example embeds a single source node, 'a', as a chain of two target nodes (0, 1)
        and uses :func:`.chain_break_frequency` to show that out of two synthetic samples,
        one ([-1, +1]) represents a broken chain.

        >>> import numpy as np
        ...
        >>> samples = np.array([[-1, +1], [+1, +1]])
        >>> embedding = {'a': {0, 1}}
        >>> print(dwave.embedding.chain_break_frequency(samples, embedding)['a'])
        0.5

    """
    if isinstance(samples_like, dimod.SampleSet):
        labels = samples_like.variables
        samples = samples_like.record.sample
        num_occurrences = samples_like.record.num_occurrences
    else:
        samples, labels = dimod.as_samples(samples_like)
        num_occurrences = np.ones(samples.shape[0])

    if not all(v == idx for idx, v in enumerate(labels)):
        labels_to_idx = {v: idx for idx, v in enumerate(labels)}
        embedding = {v: {labels_to_idx[u] for u in chain} for v, chain in embedding.items()}

    if not embedding:
        return {}

    variables, chains = zip(*embedding.items())

    broken = broken_chains(samples, chains)

    return {v: float(np.average(broken[:, cidx], weights=num_occurrences))
            for cidx, v in enumerate(variables)}

def edgelist_to_adjacency(edgelist):
    """Converts an iterator of edges to an adjacency dict.

    Args:
        edgelist (iterable):
            An iterator over 2-tuples where each 2-tuple is an edge.

    Returns:
        dict: The adjacency dict. A dict of the form `{v: Nv, ...}` where `v` is
        a node in a graph and `Nv` is the neighbors of `v` as an set.

    """
    adjacency = dict()
    for u, v in edgelist:
        if u in adjacency:
            adjacency[u].add(v)
        else:
            adjacency[u] = {v}
        if v in adjacency:
            adjacency[v].add(u)
        else:
            adjacency[v] = {u}
    return adjacency

def adjacency_to_edges(adjacency):
    """Converts an adjacency dict, networkx graph, or bqm to an edge iterator.

    Args:
        adjacency (dict/:class:`networkx.Graph`/:class:`dimod.BQM`):
            Should be a dict of the form {s: Ns, ...} where s is a variable
            in the graph and Ns is the set of neighbours of s.

    Yields:
        tuple: A 2-tuple, corresponding to an edge in the provided graph

    """
    if hasattr(adjacency, 'edges'):
        yield from adjacency.edges()

    elif hasattr(adjacency, 'quadratic'):
        yield from adjacency.quadratic

    elif hasattr(adjacency, 'items'):
        seen = set()
        for v, Nv in adjacency.items():
            seen.add(v)
            for u in Nv:
                if u not in seen:
                    yield (u, v)
    else:
        raise TypeError("unrecognized type for adjacency -- provide a dict, "
                        "Mapping, networkx.Graph or dimod.BQM")

class intlabel_disjointsets:
    """A disjoint sets implementation with size and path-halving, for graphs
    labeled [0, ..., n-1]

    Args:
        n (int):
            The number of items in the disjoint sets

    """
    def __init__(self, n):
        self._parent = list(range(n))
        self._size = [1] * n

    def find(self, q):
        """Find the current root for q.

        Args:
            q (int):
                A number in range(n)

        Returns:
            int: the root of the set containing q

        """
        parent = self._parent
        p = parent[q]
        while q != p:
            r = parent[q] = parent[p]
            q, p = p, r
        return p

    def union(self, p, q):
        """Merges the sets containing p and q.

        Args:
            p (int):
                A number in range(n)
            q (int):
                A number in range(n)

        """
        p = self.find(p)
        q = self.find(q)
        a = self._size[p]
        b = self._size[q]
        if p == q:
            return
        if a > b:
            p, q = q, p
        self._parent[p] = q
        self._size[q] = a + b

    def size(self, q):
        """Returns the size of the set containing q.

        Args:
            p (int):
                A number in range(n)

        Returns:
            int: the size of the set containing q
        """
        return self._size[self.find(q)]

max_stars_repo_name: somethingLC/dwave-system

max_issues_repo_name: somethingLC/dwave-system

max_forks_repo_name: somethingLC/dwave-system

reason: import numpy,import networkx


text:
# -*- coding: utf-8 -*-
# @Author: YangZhou
# @Date:   2017-06-14 22:16:16
# @Last Modified by:   YangZhou
# @Last Modified time: 2017-06-23 20:15:34
from aces.tools import *
from aces.graph import fig, setLegend, pl, fit
import numpy as np

def ff(p, x):
    return p[0] * (1.0 - np.exp(-x / p[1]))
    # return 1.0/(p[1]/x+1/p[0])-p[2]
    # return p[0]*p[1]**x

with fig('vl_length.eps', figsize=(7, 8)):
    markers = ['s', '8', '^', 'D', '>']
    colors = "k,r,b,purple,g".split(',')
    vs = '3l1,3l2,4l1,5l1'.split(',')
    text_style = dict(horizontalalignment='left', verticalalignment='center',
                      fontsize=14, fontdict={'family': 'serif'})
    fi, axes = pl.subplots(2, 1, sharex=True, figsize=(7, 8))
    ax1 = axes[0]
    for s in ['z', 'a']:
        for i, v in enumerate(vs):

            file = "multi_si/tc/%s.txt" % (v + s)
            m = markers[i]
            c = colors[i]
            if s == 'z':
                opts = dict(
                    markeredgecolor='w',
                    marker=m,
                    color=c,
                    markersize=8 + 2,
                    ls='.')
                o1 = "-"
            else:
                opts = dict(
                    markeredgecolor=c,
                    marker=m,
                    color='w',
                    markersize=8,
                    ls='.')
                o1 = "--"
            y = np.loadtxt(file, skiprows=2)[:, [1, 5]]
            print y
            label = v + s
            ax1.plot(y[:, 0], y[:, 1], label=label, **opts)
            p = fit(y[:, 0], y[:, 1], [1, 1, 1], ff)
            xx = np.linspace(10, 100, 1000)
            ax1.plot(xx, ff(p, xx), color=c, ls=o1)
            ax1.set_ylabel("Thermal Conductivity(W/mK)")
            # ax1.set_xticks([])
            ax1.set_ylim([0, 14])
            ax1.text(-.15, 1, 'a)', transform=ax1.transAxes, **text_style)
    setLegend(ax1, ncol=2, loc=0, fontsize=8)
    ax1 = axes[1]
    vs = '8l1,10l1,6l1,Silicene'.split(',')
    for s in ['z', 'a']:
        for i, v in enumerate(vs):

            file = "multi_si/tc/%s.txt" % (v + s)
            m = markers[i]
            c = colors[i]
            if s == 'z':
                opts = dict(
                    markeredgecolor='w',
                    marker=m,
                    color=c,
                    markersize=8 + 2,
                    ls='.')
                o1 = "-"
            else:
                opts = dict(
                    markeredgecolor=c,
                    marker=m,
                    color='w',
                    markersize=8,
                    ls='.')
                o1 = "--"
            y = np.loadtxt(file, skiprows=2)[:, [1, 5]]
            print y
            label = v + s
            ax1.plot(y[:, 0], y[:, 1], label=label, **opts)
            p = fit(y[:, 0], y[:, 1], [1, 1, 1], ff)
            xx = np.linspace(10, 100, 1000)
            ax1.plot(xx, ff(p, xx), color=c, ls=o1)
            ax1.set_xlabel("Length(nm)")
            ax1.set_ylim([0, 18])
            ax1.set_ylabel("Thermal Conductivity(W/mK)")
            ax1.text(-.15, 1, 'b)', transform=ax1.transAxes, **text_style)
    setLegend(ax1, ncol=2, loc=0, fontsize=8)

    fi.subplots_adjust(
        left=None,
        bottom=None,
        right=None,
        top=None,
        wspace=0,
        hspace=0)


text:
import numpy as np
import matplotlib as plt
import matplotlib.cm
import matplotlib.colors
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import *
from tensorflow.keras.preprocessing import sequence
from tf2bert.layers import MaskedGlobalMaxPooling1D
from tf2bert.text.tokenizers import CharTokenizer
from tf2bert.text.rendering import print_color_text
import dataset

# Grad-CAM在文本可视化理解中的应用，并对比纯梯度和积分梯度的效果

def gelu(x):
    return 0.5 * x * (1.0 + tf.math.erf(x / tf.sqrt(2.0)))

def batch_pad(X, maxlen=None, dtype="int32"):
    """批填充或截断"""
    if maxlen is None:
        maxlen = max([len(i) for i in X])
    X = sequence.pad_sequences(
        X,
        maxlen=maxlen,
        dtype=dtype,
        padding="post",
        truncating="post",
        value=0
    )
    return X

def compute_grad_cam_weights(model_pred, model_features, sample, normalize=True):
    """Grad-CAM的基本实现"""
    with tf.GradientTape() as tape:
        features = model_features(sample)
        tape.watch(features)
        y_preds = model_pred(features)
        label_id = tf.argmax(y_preds[0])
        label = y_preds[:, label_id]

    grads = tape.gradient(label, features)
    pooled_grads = tf.reduce_mean(grads, axis=(0, 1))

    features = features.numpy()[0]
    pooled_grads = pooled_grads.numpy()
    for i in range(pooled_grads.shape[-1]):
        features[:, i] *= pooled_grads[i]

    weights = np.mean(features, axis=-1)
    if normalize:
        weights = np.maximum(weights, 0) / np.max(weights)
    return weights

def convert_to_heatmap(weights, cm="Reds"):
    """把权重转化为热力图，可是根据可视化需要选择colormap
    https://matplotlib.org/3.1.0/tutorials/colors/colormaps.html"""
    cmap = matplotlib.cm.get_cmap(cm)
    colors = []
    for i in range(cmap.N):
        rgb = cmap(i)[:3]
        colors.append(matplotlib.colors.rgb2hex(rgb))
    colors = np.array(colors)

    weights = np.uint8(255 * weights)
    heatmap = colors[weights]
    return heatmap

class DataGenerator(tf.keras.utils.Sequence):

    def __init__(self, X, y, num_classes, batch_size):
        self.X = X
        self.y = y
        self.num_classes = num_classes
        self.batch_size = batch_size

    def __len__(self):
        return len(self.X) // self.batch_size

    def __getitem__(self, index):
        i = index * self.batch_size
        j = i + self.batch_size
        X = self.X[i:j]
        y = self.y[i:j]
        y = tf.keras.utils.to_categorical(y, num_classes)
        return batch_pad(X, maxlen=None), np.array(y)

    def on_epoch_end(self):
        np.random.RandomState(773).shuffle(self.X)
        np.random.RandomState(773).shuffle(self.y)

def split_kfolds(X, y, n_splits=8):
    X_train = [j for i, j in enumerate(X) if i % n_splits != 1]
    y_train = [j for i, j in enumerate(y) if i % n_splits != 1]
    X_test = [j for i, j in enumerate(X) if i % n_splits == 1]
    y_test = [j for i, j in enumerate(y) if i % n_splits == 1]
    return (X_train, y_train), (X_test, y_test)

def show_visualization(model, gradient_model, text, labelid, id2label, tokenizer):
    """对比梯度和基本梯度的效果"""
    embeddings = model.layers[1].embeddings # embedding矩阵
    values = tf.Variable(embeddings) # 保存embedding矩阵以便恢复
    X = [text]
    X = tokenizer.transform(X)
    X = batch_pad(X)
    labelid_pred = np.argmax(model.predict(X)[0])
    labelid_pred_in = np.array([labelid_pred])

    # 纯梯度
    based_grads = gradient_model.predict([X, labelid_pred_in])[0]
    based_grads = np.sqrt(np.square(based_grads).sum(axis=1))
    based_grads = (based_grads - based_grads.min()) / (based_grads.max() - based_grads.min())

    grads = []
    n = 30
    # 计算积分梯度
    for alpha in np.linspace(0, 1, n):
        # 让embedding矩阵渐变
        embeddings.assign(alpha * values)
        pred_grads = gradient_model.predict([X, labelid_pred_in])[0]
        grads.append(pred_grads)

    # 还原embedding矩阵值以便做其他样本的预测
    embeddings.assign(values)

    # 不同embedding值下梯度的均值
    grads = np.mean(grads, axis=0)
    weights = np.sqrt(np.square(grads).sum(axis=1))
    weights = (weights - weights.min()) / (weights.max() - weights.min())

    print("gradients:")
    print_color_text(text, based_grads)
    print("integrated gradients:")
    print_color_text(text, weights)
    print("=>y_true:", id2label[labelid])
    print("=>y_pred:", id2label[labelid_pred])

X, y, classes = dataset.load_hotel_comment()
num_classes = len(classes)

tokenizer = CharTokenizer(mintf=10)
tokenizer.fit(X)
num_words = len(tokenizer)
maxlen = None
embedding_dim = 128
hdim = 128

embedding = Embedding(
    input_dim=num_words,
    output_dim=embedding_dim,
    mask_zero=False,
    name="embedding"
)

conv1 = Conv1D(filters=hdim, kernel_size=2, padding="same", activation=gelu)
conv2 = Conv1D(filters=hdim, kernel_size=2, padding="same", activation=gelu)
conv3 = Conv1D(filters=hdim, kernel_size=3, padding="same", activation=gelu)
pool = MaskedGlobalMaxPooling1D(return_scores=False)

inputs = Input(shape=(maxlen,))
mask = Lambda(lambda x: tf.not_equal(x, 0))(inputs)
x = embedding(inputs)
x = LayerNormalization()(x)
x = Dropout(0.2)(x)
x = conv1(x)
x = conv2(x)
x = conv3(x)

# 提取特征模型
model_features = Model(inputs, x)

x = pool(x, mask=mask)
x = Dense(128)(x)
x = Dropout(0.2)(x)
outputs = Dense(num_classes, activation="softmax")(x)

model = Model(inputs, outputs)
model.compile(
    loss="categorical_crossentropy",
    optimizer="adam",
    metrics=["accuracy"]
)

model.summary()

# 特征到类别模型
pred_inputs = tf.keras.Input(shape=(maxlen, filters))
x = pred_inputs
layer_names = ["masked_global_max_pooling1d", "dense",
               "dropout_1", "activation", "dense_1"]
for name in layer_names:
    x = model.get_layer(name)(x)
outputs = x
model_pred = Model(pred_inputs, outputs)

if __name__ == "__main__":
    print(__file__)
    batch_size = 64
    epochs = 10
    Xv = X[:100].copy()
    yv = y[:100].copy()
    X = tokenizer.transform(X)
    (X_train, y_train), (X_test, y_test) = split_kfolds(X, y, 5)
    dataset_train = DataGenerator(X_train, y_train, num_classes, batch_size)
    dataset_val = DataGenerator(X_test, y_test, num_classes, batch_size)
    model.fit(
        dataset_train,
        batch_size=batch_size,
        epochs=epochs,
        validation_data=dataset_val,
        validation_batch_size=batch_size
    )

    id2label = {j:i for i,j in classes.items()}
    for text, labelid in zip(Xv, yv):
        show_visualization(model, gradient_model, text, labelid, id2label, tokenizer)
        input()


text:
import csv
import os
import json
import numpy as np
from tqdm import tqdm
csvFile_train = open("ava-predicted-box-csv/ava_train_predicted_boxes.csv", "r")
reader_train = csv.reader(csvFile_train)

csvFile_val = open("ava-predicted-box-csv/ava_val_predicted_boxes.csv", "r")
reader_val = csv.reader(csvFile_val)
vt_box_dict = {}

for set_idx, reader in enumerate([reader_train, reader_val]):
  if set_idx == 0:
    set_name = 'train'
  elif set_idx == 1:
    set_name = 'val'
  for idx, item in enumerate(tqdm(reader)):
    video_id = item[0]
    time = int(item[1])
    h_box = [float(item[2]), float(item[3]), float(item[4]), float(item[5])]
    score = float(item[-1])
    vt = video_id+'.'+str(time)
    if vt not in vt_box_dict:
       vt_box_dict[vt] = []
    hit = False
    for exist_box in vt_box_dict[vt]:
        if np.sum(np.abs(np.array(h_box)-np.array(exist_box[0]))) <= 1e-5:
            hit = True
    if not hit and score >= 0.8:
        vt_box_dict[vt].append([h_box, score])

json.dump(vt_box_dict, open('vt_box_dict.json','w'))


text:
# %%

import os
from os import listdir
from typing import Optional

import chart_studio.plotly as py
import cufflinks
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import PIL
import plotly.express as px
import plotly.graph_objs as go
import pydicom
from PIL import Image, ImageDraw
from plotly.offline import iplot
from scipy.stats import boxcox
from skimage.io import imread
from sklearn.cluster import KMeans
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import MinMaxScaler, StandardScaler

cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme="pearl")

import seaborn as sns

sns.set(style="whitegrid")

import warnings

warnings.filterwarnings("ignore")

plt.style.use("fivethirtyeight")
plt.show()

# %%

path = "../input/siim-isic-melanoma-classification/"
train = pd.read_csv(path + "train.csv")
print("Shape of train :", train.shape)

# %% [markdown]

"""
+ image_name: 이미지 파일 이름.
+ patient_id: 환자 개인의 고유값.
+ sex: 성별(Nan값은 성별을 알 수 없음)
+ age_approx: 환자 나이.
+ anatom_site_general_challenge: 흑색종 위치
+ diagnosis: maligant의 진단명
+ benign_malignant: target값의 실제 명
"""

# 결측값 확인
total = train.isnull().sum().sort_values(ascending=False)
percent = (train.isnull().sum() / train.isnull().count() * 100).sort_values(
    ascending=False
)
missing_train_data = pd.concat([total, percent], axis=1, keys=["Total", "Percent"])
missing_train_data.head()
# %%

def plot_count(df: pd.DataFrame, feature: str, title: str = "", size: float = 2.5):
    f, ax = plt.subplots(1, 1, figsize=(4 * size, 3 * size))
    total = float(len(df))
    sns.countplot(df[feature], order=df[feature].value_counts().index, palette="Set2")
    plt.title(title)
    for p in ax.patches:
        height = p.get_height()
        ax.text(
            p.get_x() + p.get_width() / 2.0,
            height + 3,
            "{:1.2f}%".format(100 * height / total),
            ha="center",
        )
    plt.show()

# %%
plot_count(train, "benign_malignant")

# %%
plot_count(train, "sex")

# %%
plot_count(train, "anatom_site_general_challenge")

# %%
train["diagnosis"].value_counts(normalize=True).sort_values().iplot(
    kind="barh",
    xTitle="Percentage",
    linecolor="black",
    opacity=0.7,
    color="blue",
    theme="pearl",
    bargap=0.2,
    gridcolor="white",
    title="Distribution in the training set",
)

# %%

def plot_relative_distribution(
    df: pd.DataFrame, feature: str, hue: str, title: str = "", size: float = 2
):
    f, ax = plt.subplots(1, 1, figsize=(4 * size, 3 * size))
    total = float(len(df))
    sns.countplot(x=feature, hue=hue, data=df, palette="Set2")
    plt.title(title)
    for p in ax.patches:
        height = p.get_height()
        ax.text(
            p.get_x() + p.get_width() / 2.0,
            height + 3,
            "{:1.2f}%".format(100 * height / total),
            ha="center",
        )
    plt.show()

# %%
plot_relative_distribution(
    df=train,
    feature="sex",
    hue="benign_malignant",
    title="relative count plot of sex with benign_malignant",
    size=2.8,
)

# %%
plot_relative_distribution(
    df=train,
    feature="anatom_site_general_challenge",
    hue="benign_malignant",
    title="relative count plot of anatom_site_general_challenge with benign_malignant",
    size=3,
)

# %%
train["age_approx"].iplot(
    kind="hist",
    bins=30,
    color="blue",
    xTitle="Age",
    yTitle="Count",
    title="Age Distribution",
)

def display_images(images: np.ndarray, title: Optional[str] = None):
    f, ax = plt.subplots(5, 3, figsize=(18, 22))
    if title:
        f.suptitle(title, fontsize=30)

    for i, image_id in enumerate(images):
        image_path = os.path.join(path, f"jpeg/train/{image_id}.jpg")
        image = Image.open(image_path)

        ax[i // 3, i % 3].imshow(image)
        image.close()
        ax[i // 3, i % 3].axis("off")

        benign_malignant = train[train["image_name"] == image_id][
            "benign_malignant"
        ].values[0]
        ax[i // 3, i % 3].set_title(
            f"image_name: {image_id}\nSource: {benign_malignant}", fontsize="15"
        )

    plt.show()

# %%
df = train.groupby(["age_approx", "target"])["target"].count().unstack(level="target")
df.plot(kind="bar", figsize=(12, 10), title="Age Target")
plt.show()
# %%
df["age_approx"]
# %%
df = train.groupby(["age_approx", "target"])["target"].agg(["count"])
df["percent"] = df.groupby(level=0).transform(lambda x: (x / x.sum()))
df = df.unstack(level="target")
df["percent"].plot(kind="bar", figsize=(12, 10), title="Age Target")
plt.show()
# %%
df = train.groupby(["age_approx", "target"])["target"].agg(["count"])
df["percent"] = df.groupby(level=0).transform(lambda x: (x / x.sum()))
df
# %%

max_stars_repo_name: ds-wook/ml-project

max_issues_repo_name: ds-wook/ml-project

max_forks_repo_name: ds-wook/ml-project

reason: import numpy,from scipy


text:
# Chapter matplotlib pyplot
'''
Pyplot

Most of the Matplotlib utilities lies under the pyplot submodule, and are usually imported under the plt alias:
import matplotlib.pyplot as plt

Now the Pyplot package can be referred to as plt.
Example

Draw a line in a diagram from position (0,0) to position (6,250):
'''

import sys
import matplotlib

# matplotlib.use('agg') Don't use this line because this makes code un run

import matplotlib.pyplot as plt
import numpy as r

xpoint=r.array([0,6])
ypint=r.array([0,340])

plt.plot(xpoint,ypint)
plt.show()

#Two  lines to make our compiler able to draw:

plt.savefig(sys.stdout.buffer)
sys.stdout.flush()

max_stars_repo_name: JayramMardi/codeMania

max_issues_repo_name: JayramMardi/codeMania

max_forks_repo_name: JayramMardi/codeMania


text:
#!/usr/bin/env python

"""Load the MASYM dataset."""

import numpy
from hasy_tools import generate_index, load_images
from six.moves import xrange
from tensorflow.contrib.learn.python.learn.datasets import base
from tensorflow.python.framework import dtypes

SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'  # TODO

class DataSet:
    """DataSet from tensorflow.contrib.learn.python.learn.datasets.mnist."""

    def __init__(self,
                 images,
                 labels,
                 fake_data=False,
                 one_hot=False,
                 dtype=dtypes.float32,
                 reshape=True):
        """
        Construct a DataSet.

        one_hot arg is used only if fake_data is true.  `dtype` can be either
        `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
        `[0, 1]`.
        """
        dtype = dtypes.as_dtype(dtype).base_dtype
        if dtype not in (dtypes.uint8, dtypes.float32):
            raise TypeError(('Invalid image dtype %r, expected uint8 or '
                             'float32') % dtype)
        if fake_data:
            self._num_examples = 10000
            self.one_hot = one_hot
        else:
            assert images.shape[0] == labels.shape[0], (
                'images.shape: {} labels.shape: {}'.format(images.shape,
                                                       labels.shape))
            self._num_examples = images.shape[0]

            # Convert shape from [num examples, rows, columns, depth]
            # to [num examples, rows*columns] (assuming depth == 1)
            if reshape:
                assert images.shape[3] == 1
                images = images.reshape(images.shape[0],
                                        images.shape[1] * images.shape[2])
            if dtype == dtypes.float32:
                # Convert from [0, 255] -> [0.0, 1.0].
                images = images.astype(numpy.float32)
                images = numpy.multiply(images, 1.0 / 255.0)
        self._images = images
        self._labels = labels
        self._epochs_completed = 0
        self._index_in_epoch = 0

    @property
    def images(self):
        return self._images

    @property
    def labels(self):
        return self._labels

    @property
    def num_examples(self):
        return self._num_examples

    @property
    def epochs_completed(self):
        return self._epochs_completed

    def next_batch(self, batch_size, fake_data=False):
        """Return the next `batch_size` examples from this data set."""
        if fake_data:
            fake_image = [1] * 784
            if self.one_hot:
                fake_label = [1] + [0] * 9
            else:
                fake_label = 0
            return ([fake_image for _ in xrange(batch_size)],
                    [fake_label for _ in xrange(batch_size)])
        start = self._index_in_epoch
        self._index_in_epoch += batch_size
        if self._index_in_epoch > self._num_examples:
            # Finished epoch
            self._epochs_completed += 1
            # Shuffle the data
            perm = numpy.arange(self._num_examples)
            numpy.random.shuffle(perm)
            self._images = self._images[perm]
            self._labels = self._labels[perm]
            # Start next epoch
            start = 0
            self._index_in_epoch = batch_size
            assert batch_size <= self._num_examples
        end = self._index_in_epoch
        return self._images[start:end], self._labels[start:end]

def read_data_sets(train_dir,
                   fake_data=False,
                   one_hot=False,
                   dtype=dtypes.float32,
                   reshape=True,
                   validation_size=5000):
    """Read MASYM data."""
    if fake_data:

        def fake():
            return DataSet([], [],
                           fake_data=True, one_hot=one_hot, dtype=dtype)

        train = fake()
        validation = fake()
        test = fake()
        return base.Datasets(train=train, validation=validation, test=test)

    symbol_id2index = generate_index('HASYv2/')
    test_images, test_labels = load_images(train_dir,
                                           'hasy-test-labels.csv',
                                           symbol_id2index)
    train_images, train_labels = load_images(train_dir,
                                             'hasy-train-labels.csv',
                                             symbol_id2index)

    if not 0 <= validation_size <= len(train_images):
        raise ValueError(
            'Validation size should be between 0 and {}. Received: {}.'
            .format(len(train_images), validation_size))
    # Shuffle data
    perm = numpy.arange(len(train_labels))
    numpy.random.shuffle(perm)
    train_images = train_images[perm]
    train_labels = train_labels[perm]
    # Split training set in training and validation set
    validation_images = train_images[:validation_size]
    validation_labels = train_labels[:validation_size]
    train_images = train_images[validation_size:]
    train_labels = train_labels[validation_size:]

    train = DataSet(train_images, train_labels, dtype=dtype, reshape=reshape)
    validation = DataSet(validation_images,
                         validation_labels,
                         dtype=dtype,
                         reshape=reshape)
    test = DataSet(test_images, test_labels, dtype=dtype, reshape=reshape)

    return base.Datasets(train=train, validation=validation, test=test)

if __name__ == '__main__':
    # loaded = load_images('HASYv1/', 'hasy-test-labels.csv', symbol_id2index)
    # print(loaded.shape)
    pass

max_issues_repo_name: Kerwin-Xie/algorithms

max_forks_repo_name: Kerwin-Xie/algorithms


text:
from scipy.io import wavfile
from pesq import pesq
from pystoi.stoi import stoi
from glob import glob
import os, librosa, sys, argparse
import numpy as np
from tqdm import tqdm

parser = argparse.ArgumentParser()
parser.add_argument('-r', "--results_root", help="Path to test results folder", required=True)
args = parser.parse_args()

sr = 16000

all_files = glob("{}/wavs/*.wav".format(args.results_root))
gt_folder = args.results_root + '/gts/{}'

print('Calculating for {} files'.format(len(all_files)))

total_pesq = 0
total_stoi = 0
total_estoi = 0

for filename in tqdm(all_files):
	gt_filename = gt_folder.format(os.path.basename(filename))
	rate, deg = wavfile.read(filename)
	rate, ref = wavfile.read(gt_filename)
	if len(ref.shape) > 1: ref = np.mean(ref, axis=1) #raise ValueError('Audio should be a mono band')

	if rate != sr:
		ref = librosa.resample(ref.astype(np.float32), rate, sr).astype(np.int16)
		rate = sr

	if len(ref) > len(deg): x = ref[0 : deg.shape[0]]
	elif len(deg) > len(ref):
		deg = deg[: ref.shape[0]]
		x = ref
	else: x = ref

	total_pesq += pesq(rate, x, deg, 'nb')
	total_stoi += stoi(x, deg, rate, extended=False)
	total_estoi += stoi(x, deg, rate, extended=True)

print('Mean PESQ: {}'.format(total_pesq / len(all_files)))
print('Mean STOI: {}'.format(total_stoi / len(all_files)))
print('Mean ESTOI: {}'.format(total_estoi / len(all_files)))


text:
import numpy as np
import random
import os
from PIL import Image
from scipy.ndimage import imread
from skimage import img_as_float
import scipy.misc
import time
from model import *
from weights import *
from utils import *
import torch
from torch.autograd import Variable
from torchvision.utils import save_image
import skimage.io as io
import sys
class DepthPrediction:
	def __init__(self, weight_file, batch_size):
		self.weight_file = weight_file
		self.model = Model(batch_size)
		self.dtype = torch.cuda.FloatTensor
		self.model.load_state_dict(load_weights(self.model, self.weight_file, self.dtype))
		print("Model on cuda? {0}".format(next(self.model.parameters()).is_cuda))
	def print_model(self):
		print(self.model)
	def predict(self, img):
		#cropped_img = center_crop(img, 304, 228)
		#cropped_img=img
		resize_img = down_size(img,320,240)
		resize_img_save = torch.from_numpy(resize_img).permute(2, 0, 1).unsqueeze(0).float()
		save_image(resize_img_save, "resize_image.jpg")
		cropped_img = center_crop(resize_img, 304, 228)
		scipy.misc.toimage(cropped_img, cmin = 0.0, cmax = 1.0).save('cropped_img.jpg')
		pytorch_img = torch.from_numpy(cropped_img).permute(2,0,1).unsqueeze(0).float()
		save_image(pytorch_img, "input_image.jpg")
		pytorch_input = Variable(pytorch_img)
		print(list(pytorch_input.size()))
		t = time.time()
		out_img = self.model(pytorch_input)
		save_image(out_img.data, "output_image.jpg")# normalize=True)
		print("Finished image in {0} s".format(time.time() - t))
	def export_model(self):
		x = Variable(torch.randn(1, 3, 228, 304), requires_grad=True)
		# Export the model
		torch_out = torch.onnx._export(self.model,             # model being run
                               x,                       # model input (or a tuple for multiple inputs)
                               "depth_pred.onnx", # where to save the model (can be a file or file-like object)
                               export_params=True)      # store the trained parameter weights inside the model file
if __name__ == '__main__':
	prediction = DepthPrediction('NYU_ResNet-UpProj.npy', 1)
	#img = img_as_float(imread(sys.argv[1]))
	# img=imread(sys.argv[1])
	img=Image.open(sys.argv[1])
	prediction.predict(img)
	prediction.export_model()

reason: import numpy,import scipy,from scipy


text:
from joblib import Parallel, delayed
import numpy as np

import cv2
import matplotlib.pyplot as plt

def high_contrast(image):
    # Converting image to LAB Color model
    lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)

    # Splitting the LAB image to different channels
    l, a, b = cv2.split(lab)

    # Applying CLAHE to L-channel
    clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8))
    cl = clahe.apply(l)

    # Merge the CLAHE enhanced L-channel with the a and b channel
    limg = cv2.merge((cl, a, b))

    # Converting image from LAB Color model to RGB model
    final = cv2.cvtColor(limg, cv2.COLOR_LAB2BGR)
    return final

def prepare_image(path, width=None, height=None, invert=True, high_contrast=False):
    """
    Prepare image

    Args:
        path: str
            Path to the image

        width : int
            Width of the final image in pixels

        height: int
            Height of the final image in pixels
    Returns:

    """

    image = cv2.imread(path)

    # increase contrast
    if high_contrast:
        image = high_contrast(image)

    # convert to grayscale
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    if width is None and height is None:
        height, width = image.shape[:2]
    elif width is None:
        width = height / image.size[1] * image.size[0]
    elif height is None:
        height = width / image.size[0] * image.size[1]

    image = cv2.resize(image, (width, height))

    if invert:
        image = 255 - image
    return image

def get_hooks(n_hooks, height, width):
    """
    Generate hooks coordinates for the square image

    Args:
        n_hooks: int
            Number of hooks to generate

        width: int
            Width of the image in pixels

        height: int
            Height of the image in pixels

    Returns:
        list(tuple(int, int))
    """

    ratio = width / height

    n_hooks_list = [
        int(n_hooks * ratio / (2 + 2 * ratio)),
        None,
        int(n_hooks * ratio / (2 + 2 * ratio)),
        None
    ]
    n_hooks_list[1] = (n_hooks - 2 * n_hooks_list[0]) // 2
    n_hooks_list[-1] = n_hooks - sum(n_hooks_list[:-1])

    hooks = [
        np.stack((
            np.linspace(0, width - 1, n_hooks_list[0], endpoint=False),
            np.zeros(n_hooks_list[0])),
            axis=1
        ),
        np.stack((
            np.ones(n_hooks_list[1]) * (width - 1),
            np.linspace(0, height - 1, n_hooks_list[1], endpoint=False)),
            axis=1
        ),
        np.stack((
            np.linspace(width - 1, 0, n_hooks_list[2], endpoint=False),
            np.ones(n_hooks_list[2]) * (height - 1)),
            axis=1
        ),
        np.stack((
            np.zeros(n_hooks_list[3]),
            np.linspace(height - 1, 0, n_hooks_list[3], endpoint=False)),
            axis=1
        )
    ]

    return np.vstack(hooks).astype(int)

def get_pixel_path(p0, p1):
    """
    Return pixel path from p0 to p1

    Args:
        p0: tuple(float, float)
            Start pixel

        p1: tuple(float, float)
            End pixel

    Returns:
        list(tuple(float, float))
            Ordered list of pixels
    """
    p0 = np.array(p0)
    p1 = np.array(p1)

    max_x = max(p0[0], p1[0])
    max_y = max(p0[1], p1[1])

    path_length = max(np.linalg.norm(p1 - p0), 1)

    path = p0 + (p1 - p0).reshape(1, -1) / path_length * \
           np.arange(path_length + 1).reshape(-1, 1)
    path = np.clip(path, [0, 0], [max_x, max_y])

    path = np.unique(np.round(path), axis=0).astype(int)
    return path

def get_all_pixel_paths(hooks):
    def get_path(i, j):
        return i, j, get_pixel_path(hooks[i], hooks[j])

    paths = Parallel(-1)(delayed(get_path)(i, j)
                         for i in range(len(hooks))
                         for j in range(i + 1, len(hooks)))
    paths = {(i, j): p for i, j, p in paths}

    for i in range(len(hooks)):
        for j in range(i + 1, len(hooks)):
            paths[(j, i)] = paths[(i, j)]

    # for i in range(len(hooks)):
    #     for j in range(i + 1, len(hooks)):
    #         pixel_paths[(i, j)] = get_pixel_path(
    #             hooks[i], hooks[j])
    #         pixel_paths[(j, i)] = pixel_paths[(i, j)]

    return paths

def loss(image, pixel_path, line_norm='length'):
    old_pixel_values = image[pixel_path[:, 1], pixel_path[:, 0]]
    error = old_pixel_values.sum()
    if line_norm == 'none':
        pass
    elif line_norm == 'length':
        error /= len(pixel_path)
    return error

def optimize(image, n_lines, hooks, pixel_paths, line_weight=15, line_width=3,
             min_offset=30, show_progress=False, min_loss=-500, **loss_kwargs):
    lines = []

    line_mask = np.zeros_like(image)
    image = image.copy().astype(int)
    n_hooks = len(hooks)

    if show_progress:
        fig, ax = plt.subplots(1, 1, figsize=(6, 6))
        ax.scatter(hooks[:, 0], hooks[:, 1], s=5)
        ax.set_xlim([-5, image.shape[1] + 5])
        ax.set_ylim([image.shape[0] + 5, -5])
        ax.set_aspect('equal')

    start_hook = 0
    prev_hooks = [start_hook]
    for i in range(n_lines):

        best_loss = -np.inf
        best_line = None

        for start_hook in np.random.permutation(n_hooks)[:10]:
            for offset in range(min_offset, n_hooks - min_offset):

                cur_hook = (start_hook + offset) % n_hooks

                if cur_hook in prev_hooks:
                    continue

                path = pixel_paths[(start_hook, cur_hook)]
                loss_val = loss(image, path, **loss_kwargs)

                if loss_val > best_loss:
                    best_loss = loss_val
                    best_line = (start_hook, cur_hook)

        if best_loss < min_loss:
            continue

        lines.append(best_line)
        line_mask = line_mask * 0
        cv2.line(line_mask, tuple(hooks[best_line[0]]),
                 tuple(hooks[best_line[1]]), line_weight, line_width)
        image = image - line_mask.astype(int)

        prev_hooks.append(best_line[1])
        if len(prev_hooks) > 20:
            prev_hooks = prev_hooks[1:]

        if show_progress:
            p0, p1 = hooks[best_line[0]], hooks[best_line[1]]
            ax.plot([p0[0], p1[0]], [p0[1], p1[1]],
                         c='k', lw=0.2, alpha=1)
            fig.suptitle(f'N lines: {i + 1}\nLoss: {best_loss}')
            fig.canvas.draw()

    return image, lines

def thread_length(lines, hooks):
    total_length = 0
    for p0, p1 in lines:
        total_length += np.linalg.norm(hooks[p0] - hooks[p1])

    return total_length

def pixel_to_meters(pixel_length, image_size, canvas_size):
    pixel_width, pixel_height = np.array(canvas_size) / np.array(image_size)
    pixel_diag = np.sqrt(pixel_width**2 + pixel_height**2)
    return pixel_length * pixel_diag

def find_lines(image, n_hooks, n_lines, optimizer_kwargs, loss_kwargs):
    hooks = get_hooks(n_hooks, *image.shape)

    pixel_paths = get_all_pixel_paths(hooks)

    image_opt, lines = optimize(image, n_lines, hooks, pixel_paths,
                                **optimizer_kwargs, **loss_kwargs)
    return lines

def main():
    pass

if __name__ == "__main__":
    main()