text:
# --------------
#Importing header files

import pandas as pd
from sklearn.model_selection import train_test_split

# Code starts here
data = pd.read_csv(path)
X = data.drop(['customer.id','paid.back.loan'], axis= 1)
y = data['paid.back.loan']
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.3 , random_state = 0)

# Code ends here

# --------------
#Importing header files
import matplotlib.pyplot as plt

# Code starts here
fully_paid = y_train.value_counts()

# Code ends here

# --------------
#Importing header files
import numpy as np
from sklearn.preprocessing import LabelEncoder

# Code starts here
X_train['int.rate'] = (X_train['int.rate'].str.replace('%', '').astype(float))/100
X_test['int.rate'] = (X_test['int.rate'].str.replace('%', '').astype(float))/100
cat_df = X_train.select_dtypes(include = ['object'])
num_df = X_train.select_dtypes(exclude = ['object'])
# Code ends here

# --------------
#Importing header files
import seaborn as sns

# Code starts here
cols = num_df.columns
# Code ends here

# --------------
# Code starts here
cols = cat_df.columns

# Code ends here

# --------------
#Importing header files
from sklearn.tree import DecisionTreeClassifier

# Code starts here
for col in cat_df.columns:
    X_train.fillna('NA')
    le = LabelEncoder()
    X_train[col] = le.fit_transform(X_train[col])
for col in cat_df.columns:
    X_test.fillna('NA')
    le = LabelEncoder()
    X_test[col] = le.fit_transform(X_test[col])
le = LabelEncoder()
y_test = le.fit_transform(y_test)
y_train = le.fit_transform(y_train)
model = DecisionTreeClassifier(random_state = 0)
model.fit(X_train,y_train)
from sklearn.metrics import accuracy_score
test_pred = model.predict(X_test)
acc = accuracy_score(y_test,test_pred)

# Code ends here

# --------------
#Importing header files
from sklearn.model_selection import GridSearchCV

#Parameter grid
parameter_grid = {'max_depth': np.arange(3,10), 'min_samples_leaf': range(10,50,10)}

# Code starts here
model_2 = DecisionTreeClassifier(random_state = 0)
p_tree = GridSearchCV(estimator = model_2, param_grid = parameter_grid, cv = 5)
p_tree.fit(X_train,y_train)
test_pred  = p_tree.predict(X_test)
from sklearn.metrics import accuracy_score
acc_2 = accuracy_score(y_test,test_pred)

# Code ends here

# --------------
#Importing header files

from io import StringIO
from sklearn.tree import export_graphviz
from sklearn import tree
from sklearn import metrics
from IPython.display import Image
import pydotplus

# Code starts here
dot_data = tree.export_graphviz(decision_tree=p_tree.best_estimator_, out_file=None, feature_names=X.columns, filled = True ,class_names=['loan_paid_back_yes','loan_paid_back_no'])
graph_big = pydotplus.graph_from_dot_data(dot_data)
Image(graph_big.create_png())

# show graph - do not delete/modify the code below this line
img_path = user_data_dir+'/file.png'
graph_big.write_png(img_path)

plt.figure(figsize=(20,15))
plt.imshow(plt.imread(img_path))
plt.axis('off')
plt.show()

# Code ends here

max_stars_repo_name: iamacityzen/ga-learner-dsmp-repo

max_issues_repo_name: iamacityzen/ga-learner-dsmp-repo

max_forks_repo_name: iamacityzen/ga-learner-dsmp-repo


text:
import random
import time
import math
import os.path

import numpy as np
import pandas as pd

from pysc2.agents import base_agent
from pysc2.env import sc2_env, run_loop
from pysc2.lib import actions, features, units
from absl import app

import torch
from torch.utils.tensorboard import SummaryWriter

from skdrl.pytorch.model.mlp import NaiveMultiLayerPerceptron
from skdrl.pytorch.model.naivedqn import NaiveDQN
from skdrl.pytorch.util.train_util import EMAMeter

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
writer = SummaryWriter()

class ProtossAgentWithRawActsAndRawObs(base_agent.BaseAgent):
    actions = ("do_nothing",
               "harvest_minerals",
               "build_pylons",
               "build_gateways",
               "train_zealot",
               "attack")

    def get_my_units_by_type(self, obs, unit_type):
        return [unit for unit in obs.observation.raw_units
                if unit.unit_type == unit_type
                and unit.alliance == features.PlayerRelative.SELF]

    def get_enemy_units_by_type(self, obs, unit_type):
        return [unit for unit in obs.observation.raw_units
                if unit.unit_type == unit_type
                and unit.alliance == features.PlayerRelative.ENEMY]

    def get_my_completed_units_by_type(self, obs, unit_type):
        return [unit for unit in obs.observation.raw_units
                if unit.unit_type == unit_type
                and unit.build_progress == 100
                and unit.alliance == features.PlayerRelative.SELF]

    def get_enemy_completed_units_by_type(self, obs, unit_type):
        return [unit for unit in obs.observation.raw_units
                if unit.unit_type == unit_type
                and unit.build_progress == 100
                and unit.alliance == features.PlayerRelative.ENEMY]

    def get_distances(self, obs, units, xy):
        units_xy = [(unit.x, unit.y) for unit in units]
        return np.linalg.norm(np.array(units_xy) - np.array(xy), axis=1)

    def getMeanLocation(self, unitList):
        sum_x = 0
        sum_y = 0
        for unit in unitList:
            sum_x += unit.x
            sum_y += unit.y
        mean_x = sum_x / len(unitList)
        mean_y = sum_y / len(unitList)

        return [mean_x, mean_y]

    def transformDistance(self, x, x_distance, y, y_distance):
        if not self.base_top_left:
            return [x - x_distance, y - y_distance]

        return [x + x_distance, y + y_distance]

    def step(self, obs):
        super(ProtossAgentWithRawActsAndRawObs, self).step(obs)
        if obs.first():
            nexus = self.get_my_units_by_type(
                obs, units.Protoss.Nexus)[0]
            self.base_top_left = (nexus.x < 32)

    def do_nothing(self, obs):
        return actions.RAW_FUNCTIONS.no_op()

    def harvest_minerals(self, obs):
        probes = self.get_my_units_by_type(obs, units.Protoss.Probe)
        idle_probes = [probe for probe in probes if probe.order_length == 0]
        if len(idle_probes) > 0:
            mineral_patches = [unit for unit in obs.observation.raw_units
                               if unit.unit_type in [
                                   units.Neutral.BattleStationMineralField,
                                   units.Neutral.BattleStationMineralField750,
                                   units.Neutral.LabMineralField,
                                   units.Neutral.LabMineralField750,
                                   units.Neutral.MineralField,
                                   units.Neutral.MineralField750,
                                   units.Neutral.PurifierMineralField,
                                   units.Neutral.PurifierMineralField750,
                                   units.Neutral.PurifierRichMineralField,
                                   units.Neutral.PurifierRichMineralField750,
                                   units.Neutral.RichMineralField,
                                   units.Neutral.RichMineralField750
                               ]]
            probe = random.choice(idle_probes)
            distances = self.get_distances(obs, mineral_patches, (probe.x, probe.y))
            mineral_patch = mineral_patches[np.argmin(distances)]
            return actions.RAW_FUNCTIONS.Harvest_Gather_unit(
                "now", probe.tag, mineral_patch.tag)
        return actions.RAW_FUNCTIONS.no_op()

    def build_pylons(self, obs):
        nexus = self.get_my_units_by_type(obs, units.Protoss.Nexus)
        pylons = self.get_my_units_by_type(obs, units.Protoss.Pylon)
        probes = self.get_my_units_by_type(obs, units.Protoss.Probe)

        if len(nexus) == 1 and len(pylons) == 0 and obs.observation.player.minerals >= 100:
            mean_x, mean_y = self.getMeanLocation(nexus)
            pylon_xy = self.transformDistance(int(mean_x), -20, int(mean_y), 30)
            distances = self.get_distances(obs, probes, pylon_xy)
            probe = probes[np.argmin(distances)]

            return actions.RAW_FUNCTIONS.Build_Pylon_pt(
                "now", probe.tag, pylon_xy)
        elif len(pylons) > 0:
            pylon_coordinate = []
            for pylon in pylons:
                pylon_coordinate.append((pylon.x, pylon.y))
            x_coordinate, y_coordinate = max(pylon_coordinate, key=lambda t: t[1])
            pylon_xy = self.transformDistance(int(x_coordinate), 10, int(y_coordinate), 0)
            distances = self.get_distances(obs, probes, pylon_xy)
            probe = probes[np.argmin(distances)]
            return actions.RAW_FUNCTIONS.FUNCTIONS.Build_Pylon_pt("now", probe.tag, pylon_xy)
        return actions.RAW_FUNCTIONS.no_op()

    def build_gateways(self, obs):
        completed_pylons = self.get_my_completed_units_by_type(obs, units.Protoss.Pylon)
        gateways = self.get_my_units_by_type(obs, units.Protoss.Gateway)
        probes = self.get_my_units_by_type(obs, units.Protoss.Probe)

        if (len(completed_pylons) > 0 and len(gateways) == 0 and
                obs.observation.player.minerals >= 150 and len(probes) > 0):
            target_pylon = random.choice(completed_pylons)
            gateway_xy = self.transformDistance(int(target_pylon.x), 0, int(target_pylon.y), -10)
            distances = self.get_distances(obs, probes, gateway_xy)
            probe = probes[np.argmin(distances)]
            return actions.RAW_FUNCTIONS.Build_Gateway_pt("now", probe.tag, gateway_xy)
        return actions.RAW_FUNCTIONS.no_op()

    def train_marine(self, obs):
        completed_gateways = self.get_my_completed_units_by_type(
            obs, units.Protoss.Gateway)
        free_supply = (obs.observation.player.food_cap -
                       obs.observation.player.food_used)
        if (len(completed_gateways) > 0 and obs.observation.player.minerals >= 100
                and free_supply > 1):
            gateways = self.get_my_units_by_type(obs, units.Protoss.Gateway)
            gateway = random.choice(gateways)
            if gateway.order_length < 5:
                return actions.RAW_FUNCTIONS.Train_Zealot_quick("now", gateway.tag)
        return actions.RAW_FUNCTIONS.no_op()

    def attack(self, obs):
        zealots = self.get_my_units_by_type(obs, units.Protoss.Zealot)
        if len(zealots) > 0:
            attack_xy = (38, 44) if self.base_top_left else (19, 23)
            x_offset = random.randint(-4, 4)
            y_offset = random.randint(-4, 4)
            zealots_tags = [zealot.tag for zealot in zealots]

            return actions.RAW_FUNCTIONS.Attack_pt(
                "now", zealots_tags, (attack_xy[0] + x_offset, attack_xy[1] + y_offset))
        return actions.RAW_FUNCTIONS.no_op()

class ProtossRandomAgent(ProtossAgentWithRawActsAndRawObs):
    def step(self, obs):
        super(ProtossRandomAgent, self).step(obs)
        action = random.choice(self.actions)
        return getattr(self, action)(obs)

class ProtossRLAgentWithRawActsAndRawObs(ProtossAgentWithRawActsAndRawObs):
    def __init__(self):
        super(ProtossRLAgentWithRawActsAndRawObs, self).__init__()

        self.s_dim = 21
        self.a_dim = 6
        self.lr = 1e-4
        self.gamma = 1.0
        self.epsilon = 1.0

        self.qnetwork = NaiveMultiLayerPerceptron(input_dim=self.s_dim,
                           output_dim=self.a_dim,
                           num_neurons=[128],
                           hidden_act_func='ReLU',
                           out_act_func='Identity').to(device)

        self.data_file = 'rlagent_with_naive_dqn'
        if os.path.isfile(self.data_file + '.pt'):
            self.qnetwork.load_state_dict(torch.load(self.data_file + '.pt'))

        self.dqn = NaiveDQN(state_dim=self.s_dim,
                             action_dim=self.a_dim,
                             qnet=self.qnetwork,
                             lr=self.lr,
                             gamma=self.gamma,
                             epsilon=self.epsilon).to(device)

        self.print_every = 50
        self.ema_factor = 0.5
        self.ema = EMAMeter(self.ema_factor)
        self.cum_reward = 0
        self.cum_loss = 0
        self.episode_count = 0

        self.new_game()

    def reset(self):
        super(TerranRLAgentWithRawActsAndRawObs, self).reset()
        self.new_game()

    def new_game(self):
        self.base_top_left = None
        self.previous_state = None
        self.previous_action = None
        self.cum_reward = 0
        self.cum_loss = 0

    def get_state(self, obs):
        scvs = self.get_my_units_by_type(obs, units.Terran.SCV)
        idle_scvs = [scv for scv in scvs if scv.order_length == 0]
        command_centers = self.get_my_units_by_type(obs, units.Terran.CommandCenter)
        supply_depots = self.get_my_units_by_type(obs, units.Terran.SupplyDepot)
        completed_supply_depots = self.get_my_completed_units_by_type(
            obs, units.Terran.SupplyDepot)
        barrackses = self.get_my_units_by_type(obs, units.Terran.Barracks)
        completed_barrackses = self.get_my_completed_units_by_type(
            obs, units.Terran.Barracks)
        marines = self.get_my_units_by_type(obs, units.Terran.Marine)

        queued_marines = (completed_barrackses[0].order_length
        if len(completed_barrackses) > 0 else 0)

        free_supply = (obs.observation.player.food_cap -
                       obs.observation.player.food_used)
        can_afford_supply_depot = obs.observation.player.minerals >= 100
        can_afford_barracks = obs.observation.player.minerals >= 150
        can_afford_marine = obs.observation.player.minerals >= 100

        enemy_scvs = self.get_enemy_units_by_type(obs, units.Terran.SCV)
        enemy_idle_scvs = [scv for scv in enemy_scvs if scv.order_length == 0]
        enemy_command_centers = self.get_enemy_units_by_type(
            obs, units.Terran.CommandCenter)
        enemy_supply_depots = self.get_enemy_units_by_type(
            obs, units.Terran.SupplyDepot)
        enemy_completed_supply_depots = self.get_enemy_completed_units_by_type(
            obs, units.Terran.SupplyDepot)
        enemy_barrackses = self.get_enemy_units_by_type(obs, units.Terran.Barracks)
        enemy_completed_barrackses = self.get_enemy_completed_units_by_type(
            obs, units.Terran.Barracks)
        enemy_marines = self.get_enemy_units_by_type(obs, units.Terran.Marine)

        return (len(command_centers),
                len(scvs),
                len(idle_scvs),
                len(supply_depots),
                len(completed_supply_depots),
                len(barrackses),
                len(completed_barrackses),
                len(marines),
                queued_marines,
                free_supply,
                can_afford_supply_depot,
                can_afford_barracks,
                can_afford_marine,
                len(enemy_command_centers),
                len(enemy_scvs),
                len(enemy_idle_scvs),
                len(enemy_supply_depots),
                len(enemy_completed_supply_depots),
                len(enemy_barrackses),
                len(enemy_completed_barrackses),
                len(enemy_marines))

    def step(self, obs):
        super(TerranRLAgentWithRawActsAndRawObs, self).step(obs)

        #time.sleep(0.5)

        state = self.get_state(obs)
        state = torch.tensor(state).float().view(1, self.s_dim).to(device)
        action_idx = self.dqn.choose_action(state)
        action = self.actions[action_idx]
        done = True if obs.last() else False
        if self.previous_action is not None:
            loss = self.dqn.learn(self.previous_state.to(device),
                              torch.tensor(self.previous_action).view(1, 1).to(device),
                              torch.tensor(obs.reward).view(1, 1).to(device),
                              state.to(device),
                              torch.tensor(done).float().view(1, 1).to(device)
                              )
            self.cum_loss += loss.detach().numpy()
        self.cum_reward += obs.reward
        self.previous_state = state
        self.previous_action = action_idx

        if obs.last():
            self.episode_count = self.episode_count + 1
            torch.save(self.dqn.qnet.state_dict(), self.data_file + '.pt')

            self.ema.update(self.cum_reward)
            writer.add_scalar("Loss/online", self.cum_loss/obs.observation.game_loop, self.episode_count)
            writer.add_scalar("Score", self.ema.s, self.episode_count)

            if self.episode_count % self.print_every == 0:
                print("Episode {} || EMA: {} || EPS : {}".format(self.episode_count, self.ema.s, self.dqn.epsilon))

            if self.episode_count >= 150:
                self.dqn.epsilon *= 0.999

        return getattr(self, action)(obs)

# def main(unused_argv):
#    agent1 = TerranRLAgentWithRawActsAndRawObs()
#    agent2 = TerranRandomAgent()
#    try:
#        with sc2_env.SC2Env(
#                map_name="Simple64",
#                players=[sc2_env.Agent(sc2_env.Race.terran),
#                         sc2_env.Agent(sc2_env.Race.terran)],
#                agent_interface_format=features.AgentInterfaceFormat(
#                    action_space=actions.ActionSpace.RAW,
#                    use_raw_units=True,
#                    raw_resolution=64,
#                ),
#                step_mul=8,
#                disable_fog=True,
#        ) as env:
#            run_loop.run_loop([agent1, agent2], env, max_episodes=1000)
#    except KeyboardInterrupt:
#        pass

# def main(unused_argv):
#     agent = TerranRLAgentWithRawActsAndRawObs()
#     try:
#         with sc2_env.SC2Env(
#                 map_name="Simple64",
#                 players=[sc2_env.Agent(sc2_env.Race.terran),
#                          sc2_env.Bot(sc2_env.Race.terran,
#                                      sc2_env.Difficulty.very_easy)],
#                 agent_interface_format=features.AgentInterfaceFormat(
#                     action_space=actions.ActionSpace.RAW,
#                     use_raw_units=True,
#                     raw_resolution=64,
#                 ),
#                 step_mul=8,
#                 disable_fog=True,
#         ) as env:
#             agent.setup(env.observation_spec(), env.action_spec())
#
#             timesteps = env.reset()
#             agent.reset()
#
#             while True:
#                 step_actions = [agent.step(timesteps[0])]
#                 if timesteps[0].last():
#                     break
#                 timesteps = env.step(step_actions)
#     except KeyboardInterrupt:
#         pass

# def main(unused_argv):
#     agent = TerranRLAgentWithRawActsAndRawObs()
#     try:
#         while True:
#             with sc2_env.SC2Env(
#                     map_name="Simple64",
#                     players=[sc2_env.Agent(sc2_env.Race.terran),
#                              sc2_env.Bot(sc2_env.Race.terran,
#                                          sc2_env.Difficulty.very_easy)],
#                     agent_interface_format=features.AgentInterfaceFormat(
#                         action_space=actions.ActionSpace.RAW,
#                         use_raw_units=True,
#                         raw_resolution=64,
#                     ),
#                     step_mul=8,
#                     disable_fog=True,
#                     game_steps_per_episode=0,
#                     visualize=False) as env:
#
#               agent.setup(env.observation_spec(), env.action_spec())
#
#               timesteps = env.reset()
#               agent.reset()
#
#               while True:
#                   step_actions = [agent.step(timesteps[0])]
#                   if timesteps[0].last():
#                       break
#                   timesteps = env.step(step_actions)
#
#     except KeyboardInterrupt:
#         pass

def main(unused_argv):
   agent1 = TerranRLAgentWithRawActsAndRawObs()
   try:
       with sc2_env.SC2Env(
               map_name="Simple64",
               players=[sc2_env.Agent(sc2_env.Race.terran),
                        sc2_env.Bot(sc2_env.Race.terran,
                                    sc2_env.Difficulty.very_easy)],
               agent_interface_format=features.AgentInterfaceFormat(
                   action_space=actions.ActionSpace.RAW,
                   use_raw_units=True,
                   raw_resolution=64,
               ),
               step_mul=8,
               disable_fog=True,
               visualize=False
       ) as env:
           run_loop.run_loop([agent1], env, max_episodes=1000)
   except KeyboardInterrupt:
       pass

if __name__ == "__main__":
    app.run(main)

ext: py

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 19 08:26:08 2015

@author: rlabbe
"""

import numpy as np
import matplotlib.pyplot as plt

from math import sin, cos, factorial
def df(x, p):
    if p == 0:
        return sin(x)

        return x
    if p % 4 == 1:
        return cos(x)

    if p % 4 == 2:
        return -sin(x)

    if p % 4 == 3:
        return -cos(x)

    return sin(x)

def taylor(df, x, a, n):

    f = 0.0

    for i in range(n+1):
        term = df(a, i) * (x - a)**i / factorial(i)
        f += term

    return f

x = 0.1
a = 0.8
n = 1

plt.cla()

xs = np.linspace(-2, 2, 100)
ts = [taylor(df, i, a, n) for i in xs]

plt.plot(xs, np.sin(xs))
plt.plot(xs, ts)


text:
'''
Molecule class for parsing molecule information from various chemical filetypes.
'''
import numpy as np
import cclib
import os
import qcelemental as qcel

class Molecule:
    """
    Class to store molecule information

    Attributes
    ----------
    n_atom : int
        number of atoms
    xyz : array
        xyz coordinates. Size: (n_atom,3)
    sym :  list
        list of atomic symbol. Size: (n_atom,1)
    at_num : list
        list of atomic numbers. Size: (n_atom,1)
    n_connect : int
        number of bonds (not for xyz)
    connect : list
        list of bond connectivity from file (Note: index starts at 1 from file
        so need to subtract 1 from connectivity when converting to atomic
        symbol). Size: (n_atom,2) (not for xyz)
    """
    __accepted_file_formats = ['xyz', 'sdf', 'mol', 'cml']

    def __init__(self, fname=None):
        if fname is not None:
            self.import_file(fname)
        return None

    def sym2num(self, sym):
        """
        Given a chemical symbol, returns the atomic number defined within the class

        Parameters
        -----------
        sym : string
            chemical symbol

        Returns
        --------
        at_num : int
            atomic number for symbol argument
        """
        try:
            atomic_number = qcel.periodictable.to_Z(sym)
            return atomic_number
        except:
            raise KeyError('{} is not defined.'.format(sym))

    def import_file(self, fname):
        filetype = os.path.splitext(fname)[1].split('.')[1]
        if filetype not in Molecule.__accepted_file_formats:
            parsed_properly = self.import_cclib(fname)
            if not parsed_properly:
                formatted_aff = str(
                    Molecule.__accepted_file_formats).strip('[]')
                raise NotImplementedError(
                    'file type \'{}\'  is unsupported. Accepted formats: {} or any cclib supported format.'.format(filetype, formatted_aff))
        if filetype == 'xyz':
            self.import_xyz(fname)
        elif filetype == 'sdf' or filetype == 'mol':
            self.import_sdf(fname)
        elif filetype == 'cml':
            self.import_cml(fname)

    def import_xyz(self, fname):
        """
        Imports xyz file as a Molecule class instance

        Parameters
        ----------
        fname : string
            xyz filename
        """
        self.ftype = 'xyz'
        with open(fname) as f:
            lines = f.readlines()
        self.n_atom = int(lines[0].split()[0])

        # reading lines to build up class data
        self.sym = []
        self.at_num = []
        self.xyz = np.zeros((self.n_atom, 3))
        for i, line in enumerate(lines[2:]):
            tmp = line.split()
            self.sym.append(tmp[0])
            self.at_num.append(self.sym2num(tmp[0]))
            self.xyz[i, 0] = float(tmp[1])
            self.xyz[i, 1] = float(tmp[2])
            self.xyz[i, 2] = float(tmp[3])
        self.connectivity_matrix()

    def import_sdf(self, fname):
        """
        Imports sdf and mol file as a Molecule class instance

        Parameters
        ----------
        fname : string
            sdf or mol file name
        """
        self.ftype = 'sdf'
        with open(fname) as f:
            lines = f.readlines()
        self.n_atom = int(lines[3].split()[0])
        self.n_connect = int(lines[3].split()[1])
        self.sym = []
        self.at_num = []
        self.xyz = np.zeros((self.n_atom, 3))
        for i, line in enumerate(lines[4:4+self.n_atom]):
            tmp = line.split()
            self.sym.append(tmp[3])
            self.at_num.append(self.sym2num(tmp[3]))
            self.xyz[i, 0] = float(tmp[0])
            self.xyz[i, 1] = float(tmp[1])
            self.xyz[i, 2] = float(tmp[2])
        self.connect = np.zeros((self.n_connect, 2))
        for i, line in enumerate(lines[4+self.n_atom:4+self.n_atom+self.n_connect]):
            tmp = line.split()
            self.connect[i, 0] = tmp[0]
            self.connect[i, 1] = tmp[1]

    def import_cml(self, fname):
        """
        Imports cml file as a Molecule class instance

        Parameters
        ----------
        fname : string
            cml file name
        """
        self.ftype = 'cml'
        with open(fname) as f:
            lines = f.readlines()
        self.n_atom = 0
        self.n_connect = 0
        self.sym = []
        self.at_num = []
        self.xyz = []
        self.connect = []
        for i in range(len(lines)):
            if lines[i].split()[0] == '<atom':
                self.n_atom += 1
                tmp = lines[i].split()
                self.sym.append(tmp[2].split('"')[1])
                self.at_num.append(self.sym2num(tmp[2].split('"')[1]))
                x = float(tmp[3].split('"')[1])
                y = float(tmp[4].split('"')[1])
                z = float(tmp[5].split('"')[1])
                self.xyz.append([x, y, z])
            elif lines[i].split()[0] == '<bond':
                self.n_connect += 1
                tmp = lines[i].split()
                a = int(tmp[1].split('"')[1].split('a')[1])
                b = int(tmp[2].split('"')[0].split('a')[1])
                self.connect.append([a, b])
        self.xyz = np.array(self.xyz)

    def import_cclib(self, fname):
        """
        Imports any cclib parsable file as a Molecule class instance

        Parameters
        -----------
        fname : string
            cclib parsable output file name
        """
        try:
            self.ftype = 'cclib'
            data = cclib.io.ccread(fname)
            self.n_atom = data.natom
            self.at_num = data.atomnos
            # This gets the atomic symbols by looking up the keys of the
            # atomic numbers. It looks somewhat crazy but it is looking
            # through a list of the values stored in the dictionary,
            # matching the value to the atomic number and returning
            # the key that corresponds to that atomic number. It works
            # with this dictionary because the keys to values are 1 to 1.
            self.sym = []
            for i in data.atomnos:
                self.sym.append(qcel.periodictable.to_E(i))
            # cclib stores the atomic coordinates in a array of shape
            # [molecule, num atoms, 3 for xyz] because I think they might
            # have many "molecules" from each step of an optimization or
            # something. Here we are taking just the last one.
            self.xyz = data.atomcoords[-1]
            return True
        except:
            return False

    def bond(self, i, j):
        """
        Determines whether atoms are bound by comparing distance between radii
        with covalent radii of two atoms.

        Parameters
        ----------
        i: int
            index of and first atom
        j: int
            index of second atom

        Returns
        -------
        boolean
            1 if bond exists 0 else
        """
        i_covr = qcel.covalentradii.get(self.sym[i], units='angstrom')
        j_covr = qcel.covalentradii.get(self.sym[j], units='angstrom')
        r = np.linalg.norm(self.xyz[i] - self.xyz[j])
        if r < 1.1*(i_covr + j_covr):
            return int(1)
        return int(0)

    def connectivity_matrix(self):
        """
        Constructs list of bonded atoms which corresponds to non-zero elements of the upper triangle of the connectivity matrix. Bonds are determined with covalent radii. To access elements of self.connect[i,j], ensure that i < j.

        Parameters
        ----------
        None

        Returns
        -------
        None
        """
        # TODO: make this more memory efficient by ordering i,j in code when needed.
        temp = []
        for i in range(self.n_atom):
            for j in range(i+1, self.n_atom):
                if self.bond(i, j):
                    temp.append([i+1, j+1])
        self.connect = np.asarray(temp)

max_stars_repo_name: amandadumi/chemreps

max_issues_repo_name: amandadumi/chemreps

max_forks_repo_name: shivupa/chemreps


text:
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import sqlite3 as sqlite
import pandas as pd
from scipy import stats
import pylab
from sklearn.neighbors import KernelDensity
from scipy.stats import mode
import json
from json2html import *
from scipy.stats import norm
from sklearn.preprocessing import StandardScaler
import warnings
warnings.filterwarnings('ignore')

def ImportData():
  conn = sqlite.connect("/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/data/yelpCleanDB.sqlite")
  data = pd.read_sql_query("SELECT * FROM DataCompetitionCityAdded", conn)
  # for index, row in data.iterrows():
  #   if row['city'] == 'San Francisco - Downtown' or row['city'] == 'San Francisco - Outer':
  #     # print(row)
  #     del row
  print(len(data))
  data = data[data['category'] != 'food']
  data = data[data['category'] != 'nightlife']
  data = data[data['city'] != 'San Francisco - Downtown']
  data = data[data['city'] != 'San Francisco - Outer']
  print(len(data))
  return data

def Stats(data):
  pd.options.display.float_format = '{:,.2f}'.format #display a pandas dataframe with a given format

  data = data[['rating', 'review_count', 'category', 'query_price',
               'closest2Distance','closest2Price', 'closest2Rating',
               'closest5Distance', 'closest5Price','closest5Rating',
               'closest10Distance', 'closest10Price','closest10Rating',
               'closest15Distance', 'closest15Price','closest15Rating']]

  # data.columns =
  dataByCategory = data.groupby('category')

  dataPivotTable = pd.DataFrame(columns=['category', 'variable', 'Stats', 'value'])
  categories = ['bars', 'beautysvc', 'coffee', 'giftshops', 'restaurants'] #data index
  variables= ['closest10Distance', 'closest10Price', 'closest10Rating', 'closest15Distance',
    'closest15Price', 'closest15Rating', 'closest2Distance', 'closest2Price', 'closest2Rating',
    'closest5Distance', 'closest5Price', 'closest5Rating', 'query_price', 'rating', 'review_count'] #labels
  columns = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max']
  stats = pd.DataFrame(dataByCategory.describe())
  for category in categories:
    for variable in variables:
      for i in range(len(columns)):
        value = stats.loc[category][variable][i]
        dataPivotTable.loc[len(dataPivotTable)] = [category, variable, columns[i], value]

  statsTable = pd.pivot_table(dataPivotTable, values='value', index=['category', 'variable'], columns=['Stats'])

  statsTable.index.names = ['Category', 'Variables']
  # category:
  levels0 = ['Bars', 'Beauty and Spas', 'Coffee and Tea', 'Giftshops', 'Restaurants']
  # print(statsTable.index.levels[0])
  # variables:
  levels1 = ['Distance of Closest 10 Competitors', 'Price of Closest 10 Competitors', 'Rating of Closest 10 Competitors',
       'Distance of Closest 15 Competitors', 'Price of Closest 15 Competitors', 'Rating of Closest 15 Competitors',
       'Distance of Closest 2 Competitors', 'Price of Closest 2 Competitors', 'Rating of Closest 2 Competitors',
       'Distance of Closest 5 Competitors', 'Price of Closest 5 Competitors', 'Rating of Closest 5 Competitors', 'Price',
       'Rating', 'Review Count']
  # print(statsTable.index.levels[1])

  statsTable.index.set_levels(levels0, level=0, inplace=True)
  statsTable.index.set_levels(levels1, level=1, inplace=True)

  statsTable.to_csv('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/statsTable.csv')
  return statsTable

def MissingDataCheck(data):
  total = data.isnull().sum().sort_values(ascending=False) #number of missing data
  percent = (data.isnull().sum()/data.isnull().count()).sort_values(ascending=False) #of missing data
  missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
  print(missing_data.head(20))

  # dealing with missing data:
  #drop the ones with no competition variable
  data = data.drop((missing_data[missing_data['Total'] == 254]).index,1)
  # we can drop the ones with missing lattitude/longitude
  # data = data.drop(data.loc[data['longitude'].isnull()].index)
  # data.isnull().sum().max() #just checking
  return data

def OutlierCheck(data):
  #standardizing data to check outliers:
  # closest2DistanceScaled = StandardScaler().fit_transform(data['closest2Distance'].dropna()[:,np.newaxis]);
  # lowRange = closest2DistanceScaled[closest2DistanceScaled[:,0].argsort()][:10]
  # highRange= closest2DistanceScaled[closest2DistanceScaled[:,0].argsort()][-50:]

  # closest5DistanceScaled = StandardScaler().fit_transform(data['closest5Distance'].dropna()[:,np.newaxis]);
  # lowRange = closest5DistanceScaled[closest5DistanceScaled[:,0].argsort()][:10]
  # highRange= closest5DistanceScaled[closest5DistanceScaled[:,0].argsort()][-50:]
  # print('outer range (low) of the distribution:')
  # print(lowRange)
  # print('\nouter range (high) of the distribution:')
  # print(highRange)

  #bivariate analysis rating/distance
  # var = 'closest2Distance'
  # data = pd.concat([data['rating'], data[var]], axis=1)
  # data.plot.scatter(x=var, y='rating', xlim=(0,50), ylim=(0.5,5.5)) #detected 3 outliers that does not follow the crowd

  # #deleting 3 points: rating 1,15,2 & 10< distance <20
  toBeDeleted = data.sort_values(by = 'closest2Distance', ascending = False)[:35]
  print(toBeDeleted[['closest2Distance', 'rating']])
  data = data.drop(data[data['id'] == 7893].index)
  data = data.drop(data[data['id'] == 3480].index)
  data = data.drop(data[data['id'] == 10569].index)

  # can do this bivariate analysis with other distance variables as well but checked and saw values are consistent
  plt.show()

def NormalityCheck(data):
  res = stats.probplot(data['rating'], plot=plt)
  res = stats.probplot(data['query_price'], plot=plt)
  plt.show()

    # THIS WORKS JUST FINE:
  # sns.distplot(data['query_price'], bins=4, kde_kws={'bw': .4})
  # plt.show()

  # #skewness and kurtosis
  # print("Skewness: %f" % data['query_price'].skew())
  # print("Kurtosis: %f" % data['query_price'].kurt())

def ScatterPlots(data):
  #scatter plot closest2Distance/saleprice
  var = 'closest2Distance'
  data = pd.concat([data['rating'], data[var]], axis=1)
  data.plot.scatter(x=var, y='rating', xlim=(0,50), ylim=(0.5,5.5))

  #scatter plot closest5Distance/rating
  var = 'closest5Distance'
  data = pd.concat([data['rating'], data[var]], axis=1)
  data.plot.scatter(x=var, y='rating', xlim=(0,50), ylim=(0.5,5.5))

  # Relationship with categorical features

  #box plot rating/price
  var = 'query_price'
  data = pd.concat([data['rating'], data[var]], axis=1)
  f, ax = plt.subplots(figsize=(8, 6))
  fig = sns.boxplot(x=var, y="rating", data=data)
  fig.axis(ymin=0, ymax=5.5)

  # Scatter plots between 'rating' and correlated variables

  #scatterplot
  sns.set()
  cols = ['rating', 'query_price', 'closest2Distance', 'closest5Distance', 'closest10Distance', 'closest15Distance']
  sns.pairplot(data[cols], size = 2.5)
  plt.show()

def CategoryHeatmaps(data):
  dataAxes1 = data.pivot_table(values='query_price', index='rating', columns='category') #VERY GOOD
  dataAxes2 = data.pivot_table(values='rating', index='query_price', columns='category') #VERY GOOD

  # fig1 = plt.figure(figsize=(10,10))
  fig1, axes = plt.subplots(nrows=2, ncols=1, figsize=(10,10), dpi = 100)
  fig1.suptitle('Price-Rating Relationship Across Categories', fontsize=14)
  plt.subplot(2, 1, 1)
  axes = sns.heatmap(dataAxes1.dropna(), cmap='coolwarm', linecolor='white', linewidths=1, annot=True, #vmax=4, vmin=0,
    xticklabels=['Bars', 'Beauty & Spas', 'Coffee & Tea', 'Giftshops', 'Restaurants'],
    cbar_kws={'label': ''},
    annot_kws={'fontsize': 10})
  # axes = fig.add_axes([0.1, 0.1, 0.8, 0.8])
  axes.set_ylabel('Ratings', rotation=0)
  axes.yaxis.set_label_coords(-0.05 ,1.02)
  axes.set_xlabel('')
  # axes.collections[0].colorbar.set_label("Hello", rotation=0)
  axes.set_title('Colormap: Price Levels', fontsize=12)

  # Only y-axis labels need their rotation set, x-axis labels already have a rotation of 0
  _, labels = plt.yticks()
  plt.setp(labels, rotation=0)
  # _, labels = plt.xticks()
  # plt.setp(labels, rotation=0)

  # fig2 = plt[0].figure(figsize=(10,10))
  plt.subplot(2, 1, 2)
  axes = sns.heatmap(dataAxes2.dropna(), cmap='coolwarm', linecolor='white', linewidths=1, annot=True, #vmax=5, vmin=0,
    xticklabels=['Bars', 'Beauty & Spas', 'Coffee & Tea', 'Giftshops', 'Restaurants'],
    cbar_kws={'label': ''},
    annot_kws={'fontsize': 10})
  # axes = fig.add_axes([0.1, 0.1, 0.8, 0.8])
  axes.set_ylabel('Price Levels', rotation=0)
  axes.yaxis.set_label_coords(-0.05 ,1.02)
  axes.set_xlabel('')
  axes.set_title('Colormap: Ratings', fontsize=12)

  # Only y-axis labels need their rotation set, x-axis labels already have a rotation of 0
  _, labels = plt.yticks()
  plt.setp(labels, rotation=0)
  # _, labels = plt.xticks()
  # plt.setp(labels, rotation=0)
  plt.savefig('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/categoryheatmaps.png', dpi=200)
  plt.show()

  #correlation matrix
  corrmat = data.corr()
  f, ax = plt.subplots(figsize=(12, 9))
  sns.heatmap(corrmat, vmax=.8, square=True);

  # SalePrice' correlation matrix (zoomed heatmap style)
  #saleprice correlation matrix
  k = 10 #number of variables for heatmap
  cols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index
  cm = np.corrcoef(data[cols].values.T)
  sns.set(font_scale=1.25)
  hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
  plt.show()

def Histograms(data):
  # fig3 = plt.figure(figsize=(10,10))
  def kde_sklearn(x, x_grid, bandwidth=0.2, **kwargs):
    """Kernel Density Estimation with Scikit-learn"""
    kde_skl = KernelDensity(bandwidth=bandwidth, **kwargs, kernel='gaussian')
    kde_skl.fit(x[:, np.newaxis])
    # score_samples() returns the log-likelihood of the samples
    log_pdf = kde_skl.score_samples(x_grid[:, np.newaxis])
    return np.exp(log_pdf)

  plt.style.use('seaborn-deep') #bmh

  categories = data['category'].unique()

  dataRestaurants = data[data['category'] == 'restaurants']
  dataBars = data[data['category'] == 'bars']
  dataCoffee = data[data['category'] == 'coffee']
  dataBeauty = data[data['category'] == 'beautysvc']
  dataGiftshop = data[data['category'] == 'giftshops']

  dataPriceByCategory = [dataRestaurants['query_price'], dataBars['query_price'],
    dataCoffee['query_price'], dataBeauty['query_price'], dataGiftshop['query_price']]
  dataRatingByCategory = [dataRestaurants['rating'], dataBars['rating'],
    dataCoffee['rating'], dataBeauty['rating'], dataGiftshop['rating']]

  dataPriceByCategoryDF = pd.DataFrame(columns = ['Restaurants','Bars','Coffee','Giftshops','Beauty'])
  dataPriceByCategoryDF['Restaurants'] = dataRestaurants['query_price']
  dataPriceByCategoryDF['Bars'] = dataBars['query_price']
  dataPriceByCategoryDF['Coffee'] = dataCoffee['query_price']
  dataPriceByCategoryDF['Giftshops'] = dataGiftshop['query_price']
  dataPriceByCategoryDF['Beauty'] = dataBeauty['query_price']
  # print(dataRestaurants['query_price'])
  # print(dataPriceByCategoryDF.head())

  fig4 = plt.figure(figsize=(10,10))
  binsPrice = np.linspace(1, 4, num= 5)
  binsRating = np.linspace(1, 5, num = 10)
  x_gridPrice = np.linspace(0, 5, 100)
  x_gridRating = np.linspace(0, 6, 100)

  plt.subplot(2, 2, 1)
  plt.hist(dataPriceByCategory, alpha=0.7, bins=binsPrice,
    label=['Restaurants', 'Bars', 'Coffee & Tea', 'Beauty & Spas', 'Giftshops'], histtype = "bar")
  plt.title('Price by Category')
  plt.legend(loc='upper right')

  plt.subplot(2, 2, 2)
  plt.plot(x_gridPrice, kde_sklearn(data['query_price'], x_gridPrice, bandwidth=0.4),
               label='KDE', linewidth=3, alpha=0.5)
  plt.hist(data['query_price'], bins=4, fc='gray', histtype='stepfilled', alpha=0.3, normed=True, label=None)
  plt.title('Price All Categories')
  # labels = [1,2,3,4]
  # plt.set_xticklabels(['$%.lf$' % x for x in labels])
  mean = data['query_price'].mean()
  median = data['query_price'].median()
  (_, idx, counts) = np.unique(data['query_price'], return_index=True, return_counts=True)
  index = idx[np.argmax(counts)]
  mode = data['query_price'].iloc[index]
  plt.axvline(mean, label='Mean:{:2.1f}'.format(mean), color='b', linestyle='-', linewidth=1) #label='Mean:{:2.1f}'.format(mean)
  plt.axvline(median, label='Median:{:2.1f}'.format(median), color='r', linestyle='--', linewidth=1) #label='Median:{:2.1f}'.format(median)
  # print(modeSkewness, medianSkewness)
  # ax[row, col].text(.5,1.3, title, weight='heavy', horizontalalignment='center', fontsize=13, transform=ax[row, col].transAxes)
  plt.axvline(mode, label='Mode:{:2.1f}'.format(mode), color='g', linestyle='-.', linewidth=1) #label='Mode:{:2.1f}'.format(mode)
  plt.legend(loc='upper right')

  plt.subplot(2, 2, 3)
  plt.hist(dataRatingByCategory, alpha=0.7, bins=binsRating,
    label=['Restaurants', 'Bars',  'Coffee & Tea', 'Beauty & Spas', 'Giftshops'], histtype = "bar")
  plt.title('Rating by Category')
  plt.legend(loc='upper left')

  plt.subplot(2, 2, 4)
  plt.plot(x_gridRating, kde_sklearn(data['rating'], x_gridRating, bandwidth=0.4),
               label='KDE', linewidth=3, alpha=0.5)
  plt.hist(data['rating'], bins=4, fc='gray', histtype='stepfilled', alpha=0.3, normed=True, label=None)
  plt.title('Rating All Categories')
  # labels = [1,2,3,4,5]
  # plt.set_xticklabels(['$%.lf$' % x for x in labels])
  mean = data['rating'].mean()
  median = data['rating'].median()
  (_, idx, counts) = np.unique(data['rating'], return_index=True, return_counts=True)
  index = idx[np.argmax(counts)]
  mode = data['rating'].iloc[index]
  plt.axvline(mean, label='Mean:{:2.1f}'.format(mean), color='b', linestyle='-', linewidth=1) #label='Mean:{:2.1f}'.format(mean)
  plt.axvline(median, label='Median:{:2.1f}'.format(median), color='r', linestyle='--', linewidth=1) #label='Median:{:2.1f}'.format(median)
  # print(modeSkewness, medianSkewness)
  # ax[row, col].text(.5,1.3, title, weight='heavy', horizontalalignment='center', fontsize=13, transform=ax[row, col].transAxes)
  plt.axvline(mode, label='Mode:{:2.1f}'.format(mode), color='g', linestyle='-.', linewidth=1) #label='Mode:{:2.1f}'.format(mode)
  plt.legend(loc='upper left')

  plt.savefig('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/histogramsByCategory.png', dpi=200)
  # plt.hist(data['rating'], alpha=0.7, bins=binsRating, histtype = "bar")
  # plt.title('Rating All Categories')

  # kde = stats.gaussian_kde(x)
  # xx = np.linspace(0, 9, 1000)
  # plt, ax = plt.subplots(figsize=(8,6))
  # ax.hist(x, normed=True, bins=bins, alpha=0.3)
  # ax.plot(xx, kde(xx))

  cols = [0,0,1,1,2,2]
  rows = [0,1,0,1,0,1]
  title = ['Restaurants', 'Bars', 'Coffee & Tea', 'Beauty & Spas', 'Giftshops', 'All Categories']
  priceLabels = ['$', '$$', '$$$', '$$$$']
  categories = [dataRestaurants, dataBars, dataCoffee, dataBeauty, dataGiftshop, data]
  # fig = plt.figure(figsize=(8,4))
  fig4, axes = plt.subplots(nrows=2, ncols=3,figsize=(10,10), dpi = 100)
  fig4.suptitle('Price Histograms', fontsize=14)
  # for category, row, col, ax in zip(categories, rows, cols, axes):
  for i, row, col in zip(range(len(categories)), rows, cols):
    print(i, row, col)
    axes[row, col].hist(categories[i]['query_price'], histtype='stepfilled', alpha=0.5, bins=[0.5,1,2,3,4,5])
    # axes[row, col].set_xlabel(priceLabels)
    # axes[row, col].set_xticks([1, 2, 3, 4])
    x = range(1, 5)
    axes[row, col].set_xticks(x)
    labels = ['$', '$$', '$$$', '$$$$']
    labels = [1,2,3,4]
    axes[row, col].set_xticklabels(['$%.lf$' % y for y in labels])
    # axes[row, col].set_xticklabels('$, $$, $$$, $$$$', fontsize=12)
    axes[row, col].set_title(title[i])
  fig4.savefig('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/priceHistograms.png', dpi=200)

  fig5, axes = plt.subplots(nrows=2, ncols=3,figsize=(10,10), dpi = 100)
  fig5.suptitle('Rating Histograms', fontsize=14)
  # for category, row, col, ax in zip(categories, rows, cols, axes):
  for i, row, col in zip(range(len(categories)), rows, cols):
    print(i, row, col)
    axes[row, col].hist(categories[i]['rating'], histtype='stepfilled', alpha=0.5, bins=[0.5,1,1.5,2,2.5,3,3.5,4,4.5,5,5.5])
    # axes[row, col].set_xlabel(priceLabels)
    # axes[row, col].set_xticks([1, 2, 3, 4])
    x = range(1, 6)
    axes[row, col].set_xticks(x)
    labels = [1,2,3,4,5]
    axes[row, col].set_xticklabels(['$%.lf$' % y for y in labels])
    # axes[row, col].set_xticklabels('$, $$, $$$, $$$$', fontsize=12)
    axes[row, col].set_title(title[i])

  fig5.savefig('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/ratingHistograms.png', dpi=200)

  plt.show()

def JointPlots(data):
  categories = ['restaurants', 'bars', 'coffee', 'beautysvc', 'giftshops']

  dataRestaurants = data[data['category'] == 'restaurants']
  dataBars = data[data['category'] == 'bars']
  dataCoffee = data[data['category'] == 'coffee']
  dataBeauty = data[data['category'] == 'beautysvc']
  dataGiftshop = data[data['category'] == 'giftshops']

  rows = [0,1,0,1,0,1]
  cols = [0,0,1,1,2,2]
  title = ['Restaurants', 'Bars', 'Coffee & Tea', 'Beauty & Spas', 'Giftshops', 'All Categories']

  fig6, axes = plt.subplots(nrows=2, ncols=3, figsize=(10,10), dpi = 100)
  fig6.suptitle('Price-Rating Relationship Across Categories', fontsize=14)
  for i, row, col, t in zip(categories, rows, cols, title):
    print(i, row, col)
    g = sns.jointplot(x='query_price',y='rating', data=data[data['category'] == i], kind='kde',
      marginal_kws=dict(bw=1),  #, ax=axes[row, col]
      annot_kws=dict(stat="pearsonr"),
      edgecolor="w", linewidth=1, ax=axes[row, col])
    g.plot_joint(sns.kdeplot, zorder=3, n_levels=5) #ax=axes[row, col]
    axes[row, col].set_title(t, fontsize=10)

    dataPearson = data[data['category'] == i]
    x = dataPearson['query_price']
    y = dataPearson['rating']
    pearsonr = stats.pearsonr(x, y)
    axes[row, col].set_xlabel('Pearsonr:{:2.3f}  p-value:{:2.2f}'.format(pearsonr[0], pearsonr[1]))
    axes[row, col].xaxis.set_label_coords(0.5,0.10)
    # axes[row, col].set_ylabel("Rating")
    # axes[row, col].xaxis.labelpad = 5
    # axes[row, col].yaxis.labelpad = 5

  k = sns.jointplot(x='query_price',y='rating', data=data, kind='kde',
      marginal_kws=dict(bw=1),
      annot_kws=dict(stat="pearsonr"),
      edgecolor="w", linewidth=1, ax=axes[1, 2])
  k.plot_joint(sns.kdeplot, zorder=3, n_levels=5)
  axes[1, 2].set_title(title[5], fontsize=10)
  pearsonr = stats.pearsonr(data['rating'], data['query_price'])
  axes[1, 2].set_xlabel('Pearsonr:{:2.3f}  p-value:{:2.2f}'.format(pearsonr[0], pearsonr[1]))
  axes[1, 2].xaxis.set_label_coords(0.5,0.10)

  fig6.text(0.5, 0.04, 'Price Levels (1:Low; 4:High)', ha='center', fontsize=12)
  fig6.text(0.04, 0.5, 'Ratings', va='center', rotation='vertical', fontsize=12)

  fig6.savefig('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/jointplots.png', dpi=200)
  plt.show()

def Correlations(data): #only for pooled data
  fig7, axes = plt.subplots(nrows=3, ncols=2, figsize=(10,10), dpi = 100)
  dataCorrCity = data[['numberOfCompetitorsCity', 'avgPriceCity', 'avgRatingCity', 'population', 'area']].drop_duplicates()
  dataCorrCompClosest2 = data[['closest2Distance', 'closest2Rating', 'closest2Price']]
  dataCorr = data[['rating', 'query_price', 'review_count']]
  dataCorrZipcode = data[['numberOfCompetitorsZipcode', 'avgPriceZipcode', 'avgRatingZipcode', 'population', 'area']].drop_duplicates()
  dataCorrCompDistance = data[['closest2Distance', 'closest5Distance', 'closest10Distance', 'closest15Distance']]
  dataCorrFinal = data[['rating', 'query_price', 'review_count', 'closest2Distance', 'closest2Rating', 'closest2Price']]

  datasets = [dataCorrCity, dataCorrCompClosest2, dataCorr,
    dataCorrZipcode, dataCorrCompDistance, dataCorrFinal]
  rows = [0,1,2,0,1,2] #from left to right= 1-4-5-2-3-6
  cols = [0,1,0,1,0,1]
  titles = ['Competition & City Variables',
    'Closest 2 Competitors',
    'Business Variables',
    'Competition & Zipcode Variables',
    'Distance Variables',
    'Business & Competition Variables']

  xLabels = [['Competitors', 'Comp.Price', 'Comp.Rating', 'Population', ''],
    ['Dist.2', 'Comp.Rating', ''],
    ['Rating', 'Price', ''],
    ['Competitors', 'Comp.Price', 'Comp.Rating', 'Population', ''],
    ['Distance-2', 'Distance-5', 'Distance-10', ''],
    ['Rating', 'Price', 'Review', 'Comp.Dist.', 'Comp.Rating', '']]

  yLabels = [['', ' Price', 'Rating', 'Population', 'Area'],
    ['', 'Comp.Rating', 'Comp.Price'],
    ['', 'Price', 'Review'],
    ['', 'Price', 'Rating', 'Population', 'Area'],
    ['', 'Distance-5', 'Distance-10', 'Distance-15'],
    ['', 'Price', 'Review', 'Dist.2', 'Comp.Rating', 'Comp.Price']]

  fig7.suptitle('Correlations Among Variables', fontsize=14)
  # for category, row, col, ax in zip(categories, rows, cols, axes):
  for dataset, row, col, title, xLabel, yLabel in zip(datasets, rows, cols, titles, xLabels, yLabels):
    corr = dataset.corr() # np.corrcoef
    mask = np.zeros_like(corr)
    mask[np.triu_indices_from(mask)] = True
    with sns.axes_style("white"):
      c = sns.heatmap(corr, mask=mask, vmax=1, vmin=-0.3, square=True, cbar_kws={"pad": 0.01, 'shrink': 0.7, 'fraction': 0.10}, ax=axes[row, col]) #annot=True, annot_kws : dict of key, value mappings, optional Keyword arguments for ax.text when annot is True.
      axes[row, col].text(.5,.85, title, horizontalalignment='center', transform=axes[row, col].transAxes)
      axes[row, col].set_xticklabels(xLabel, rotation=25, fontsize=8)
      axes[row, col].xaxis.set_ticks_position('none')
      axes[row, col].set_yticklabels(yLabel, rotation='horizontal', fontsize=8)
      axes[row, col].yaxis.set_ticks_position('none')
      # axes[row, col].set_xlabel("Price")
    # axes[row, col].set_ylabel("Rating")
    # axes[row, col].xaxis.labelpad = 5
    # axes[row, col].yaxis.labelpad = 5

    # axes[row, col].hist(categories[i]['query_price'], bins=[0.5,1,2,3,4,5])
    # # axes[row, col].set_xlabel(priceLabels)
    # # axes[row, col].set_xticks([1, 2, 3, 4])
    # x = range(1, 5)
    # axes[row, col].set_xticks(x)
    # labels = ['$', '$$', '$$$', '$$$$']
    # labels = [1,2,3,4]
    # axes[row, col].set_xticklabels(['$%.lf$' % y for y in labels])

  fig7.savefig('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/correlations.png', dpi=200)
  plt.show()

def KDEs(data):
  def kde_sklearn(x, x_grid, bandwidth=0.2, **kwargs):
    """Kernel Density Estimation with Scikit-learn"""
    kde_skl = KernelDensity(bandwidth=bandwidth, **kwargs, kernel='gaussian')
    kde_skl.fit(x[:, np.newaxis])
    # score_samples() returns the log-likelihood of the samples
    log_pdf = kde_skl.score_samples(x_grid[:, np.newaxis])
    return np.exp(log_pdf)
  #TRY KDE WITH DIFFERENT BANDWITHS

  dataRestaurants = data[data['category'] == 'restaurants']
  dataBars = data[data['category'] == 'bars']
  dataCoffee = data[data['category'] == 'coffee']
  dataBeauty = data[data['category'] == 'beautysvc']
  dataGiftshop = data[data['category'] == 'giftshops']

  dataRestaurants['Price'] = dataRestaurants['query_price']
  dataBars['Price'] = dataBars['query_price']
  dataCoffee['Price'] = dataCoffee['query_price']
  dataBeauty['Price'] = dataBeauty['query_price']
  dataGiftshop['Price'] = dataGiftshop['query_price']
  data['Price'] = data['query_price']

  dataRestaurants['Rating'] = dataRestaurants['rating']
  dataBars['Rating'] = dataBars['rating']
  dataCoffee['Rating'] = dataCoffee['rating']
  dataBeauty['Rating'] = dataBeauty['rating']
  dataGiftshop['Rating'] = dataGiftshop['rating']
  data['Rating'] = data['rating']

  x_grid = np.linspace(0, 6, 70)
  datasets = [dataRestaurants, dataBars, dataCoffee, dataBeauty, dataGiftshop, data,
    dataRestaurants, dataBars, dataCoffee, dataBeauty, dataGiftshop, data]
  plots = ['Rating','Rating','Rating','Rating','Rating','Rating',
    'Price','Price','Price','Price','Price','Price']
  rows = [0,1,2,3,4,5,0,1,2,3,4,5] #from left to right= 1-4-5-2-3-6
  cols = [0,0,0,0,0,0,1,1,1,1,1,1]

  titles = ['RATING','','','','','','PRICE','','','','','']
  ts = ['Restaurants', 'Bars', 'Coffee & Tea', 'Beauty & Spas', 'Giftshops', 'All Categories',
    'Restaurants', 'Bars', 'Coffee & Tea', 'Beauty & Spas', 'Giftshops', 'All Categories']

  labels = ['Mode', 'Median']
  fig8, ax = plt.subplots(nrows=6, ncols=2, figsize=(10,10), dpi = 100)
  fig8.suptitle('Kernel Density Estimations of Rating and Price Variables Across Categories',
    fontsize=14)

  for row, col, title, t, dataset, plot in zip(rows, cols, titles, ts, datasets, plots):
    mean = dataset[plot].mean()
    median = dataset[plot].median()
    (_, idx, counts) = np.unique(dataset[plot], return_index=True, return_counts=True)
    index = idx[np.argmax(counts)]
    mode = dataset[plot].iloc[index]
    standardDev = np.std(dataset[plot])
    medianSkewness =  (3 * (mean - median)) / standardDev #P = (3 * (X - Med))/s Pearson’s Median Skewness Coefficient
    modeSkewness =  (mean - mode) / standardDev #P = (3 * (X - Med))/s Pearson’s Median Skewness Coefficient

    ax[row, col].plot(x_grid, kde_sklearn(dataset[plot], x_grid, bandwidth=0.4),
               label='Mode skewness={:2.1f}'.format(modeSkewness), linewidth=3, alpha=0.5) #label='bw={0}'.format(bandwidth),
    ax[row, col].hist(dataset[plot], bins=4, fc='gray', histtype='stepfilled', alpha=0.3, normed=True, label=None)
    if plot == 'Rating':
      ax[row, col].set_xlim(0, 5.5)
    else:
      ax[row, col].set_xlim(0, 4.5)
    # ll1 = ('Mode skewness={:2.1f}'.format(modeSkewness))
    l1 = ax[row, col].axvline(mean, color='b', linestyle='-', linewidth=1) #label='Mean:{:2.1f}'.format(mean)
    l2 = ax[row, col].axvline(median, color='r', linestyle='--', linewidth=1) #label='Median:{:2.1f}'.format(median)
    # print(modeSkewness, medianSkewness)
    ax[row, col].text(.5,1.3, title, weight='heavy', horizontalalignment='center', fontsize=13, transform=ax[row, col].transAxes)
    l3 = ax[row, col].axvline(mode, color='g', linestyle='-.', linewidth=1) #label='Mode:{:2.1f}'.format(mode)
    ax[row, col].legend(loc=0, fontsize=8)
    ax[row, col].set_yticklabels('')
    ax[row, col].yaxis.set_ticks_position('none')
    ax[row, col].set_yticklabels('')
    ax[row, col].set_ylabel(t, rotation=90)
    # ax[row, col].set_title(t, fontsize=10)
  fig8.legend((l1, l2, l3), ('Mean', 'Median', 'Mode'), loc = 'lower center')
  # fig8.legend((l1,), ('Mean',), loc =(4))
  # fig8.legend((l2,), ('Median',), loc = (5))
  # fig8.legend((l3,), ('Mode',), loc = (3))

  fig8.savefig('/Users/selinerguncu/Desktop/PythonProjects/Fun Projects/Yelp/yelp/static/plots/KDEs.png', dpi=200)
  plt.show()

data = ImportData()
# Stats(data)
# CategoryHeatmaps(data)
# Histograms(data)
# # JointPlots(data)
# # Correlations(data)
# # KDEs(data)

# MissingDataCheck(data)
# OutlierCheck(data)
NormalityCheck(data)
# ScatterPlots(data)

# df2.plot.box()
# data1 = pd.DataFrame()
# dataByCategory = data.groupby('category')
# data1['x'] = [1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9,1,1,1,2,3,4,5,6,7,8,9]

# df2['a'].plot.kde()
# sns.distplot(data['rating'], fit=norm);
# fig = plt.figure()

#8 plots -- number 5 might be unnecessary

    # ax.text(3, 8, 'boxed italics text in data coords', style='italic',
    #     bbox={'facecolor':'red', 'alpha':0.5, 'pad':10})
  # x_grid = np.linspace(0, 3, 100)
  # x = dataBars['closest2Distance'].dropna()

  # fig, ax = plt.subplots()
  # for bandwidth in [0.1, 0.3, 1.0]:
  #     ax.plot(x_grid, kde_sklearn(x, x_grid, bandwidth=bandwidth),
  #             label='bw={0}'.format(bandwidth), linewidth=3, alpha=0.5)
  # ax.hist(x, 300, fc='gray', histtype='stepfilled', alpha=0.3, normed=True)
  # ax.set_xlim(0, 3)
  # ax.legend(loc='upper right')

  # FIND THE OPTIMUM BANDWIDTH: https://jakevdp.github.io/blog/2013/12/01/kernel-density-estimation/
  # from sklearn.grid_search import GridSearchCV
  # grid = GridSearchCV(KernelDensity(kernel='gaussian'),
  #                     {'bandwidth': np.linspace(0.01, 1.0, 30)},
  #                     cv=20) # 20-fold cross-validation
  # grid.fit(x[:, None])
  # print(grid.best_params_) #{'bandwidth': 0.10000000000000001}

#BUNUNLA IC ICE PLOT YAPMAYI DENE
# grid_kws = {"height_ratios": (.9, .05), "hspace": .3}
# f, (ax, cbar_ax) = plt.subplots(2, gridspec_kw=grid_kws)
# ax = sns.heatmap(flights, ax=ax,
#   cbar_ax=cbar_ax,
#   cbar_kws={"orientation": "horizontal"})
#
  #

  #axes[2].set_xlim([0,60])
# xlim=xlim, ylim=ylim

#  data = data[data['category'] != 'food']
  # data = data[data['category'] != 'nightlife']
  # dataAxes1 = data.pivot_table(values='query_price', index='rating', columns='category')
  # dataAxes2 = data.pivot_table(values='rating', index='query_price', columns='category')
  # # print(deneme)

  # fig1 = plt.figure()

  # axes1 = sns.heatmap(dataAxes1.dropna(), cmap='coolwarm', linecolor='white', linewidths=1, annot=True,
  #   xticklabels=['Bars', 'Beauty & Spas', 'Coffee & Tea', 'Giftshops', 'Restaurants'],
  #   cbar_kws={'label': 'Price'})
  # axes1 = fig1.add_axes([0.1, 0.1, 0.8, 0.8])
  # axes1.set_ylabel('Rating')
  # axes1.set_xlabel('')
  # axes1.set_title('Rating-Price Relationship Across Categories')

  # # Only y-axis labels need their rotation set, x-axis labels already have a rotation of 0
  # _, labels = plt.yticks()
  # plt.setp(labels, rotation=0)

  # fig2 = plt.figure(figsize=(10,10))
  # axes2 = sns.heatmap(dataAxes2.dropna(), cmap='coolwarm', linecolor='white', linewidths=1, annot=True,
  #   xticklabels=['Bars', 'Beauty & Spas', 'Coffee & Tea', 'Giftshops', 'Restaurants'],
  #   cbar_kws={'label': 'Rating'})
  # axes2 = fig2.add_axes([0.1, 0.1, 0.8, 0.8])
  # axes2.set_ylabel('Price')
  # axes2.set_xlabel('')
  # axes2.set_title('Price-Rating Relationship Across Categories')

  # # Only y-axis labels need their rotation set, x-axis labels already have a rotation of 0
  # _, labels = plt.yticks()
  # plt.setp(labels, rotation=0)

# "boundaries": np.linspace(-1, 1, 4)

#SBPLOTS:

# data = range(1,10);
# fig = figure()
# for i in range(6):
#     ax = fig.add_subplot(2,3,i)

#     ax.text(.5,.9,'centered title',
#         horizontalalignment='center',
#         transform=ax.transAxes)

#     ax.plot(data)
# show()

# TIC ON AXES:
# plt.tick_params(
#     axis='x',          # changes apply to the x-axis
#     which='both',      # both major and minor ticks are affected
#     bottom='off',      # ticks along the bottom edge are off
#     top='off',         # ticks along the top edge are off
#     labelbottom='off') # labels along the bottom edge are off

max_stars_repo_name: selinerguncu/Yelp-Spatial-Analysis

max_issues_repo_name: selinerguncu/Yelp-Spatial-Analysis

max_forks_repo_name: selinerguncu/Yelp-Spatial-Analysis

reason: import numpy,from scipy


text:
'''
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
'''

import os
import random
import librosa
import librosa.display
import soundfile as sf
import numpy as np
import cv2
import scipy.interpolate as interp
import matplotlib.pyplot as plt

from tqdm import tqdm
from copy import deepcopy

from PIL import Image

from scipy.signal import lfilter

import speechproc
import spectrogating

# %%
############
# Load data
############
x, sr = librosa.load('./5AE9F749.wav', sr=48000)

# %%
plt.figure(figsize=(14, 5))
librosa.display.waveplot(x ,sr=sr, x_axis='time')

# %%
################
# Mel Spectrogram
###############
x_mel = librosa.feature.melspectrogram(y=x, sr=sr, n_mels=64)
x_mel_db = librosa.power_to_db(x_mel, ref=np.max)
plt.figure(figsize=(14, 5))
librosa.display.specshow(x_mel_db, x_axis='time', y_axis='mel')
plt.axis('off')

# %%
############
# PCEN
############
x_pcen = librosa.pcen(x_mel, sr=sr, gain=1.3, hop_length=512,
                      bias=2, power=0.3, time_constant=0.4, eps=1e-06, max_size=1)

plt.figure(figsize=(14, 5))
librosa.display.specshow(x_pcen, x_axis='time', y_axis='mel')
plt.axis('off')

# %%
kernel = np.ones((3, 3), np.float32)/25
dst = cv2.filter2D(x_pcen, -1, kernel)
plt.figure(figsize=(14, 5))
librosa.display.specshow(dst, x_axis='time', y_axis='mel')
plt.axis('off')

# %%
##############
# Denoise with spectral gating
##############
noise2 = x[0:1*sr]
x_dn = spectrogating.removeNoise(audio_clip=x, noise_clip=noise2,
    n_grad_freq=2,
    n_grad_time=4,
    n_fft=2048,
    win_length=2048,
    hop_length=512,
    n_std_thresh=2.5,
    prop_decrease=1.0,
    verbose=False,
    visual=False)

# %%
plt.figure(figsize=(14, 5))
librosa.display.waveplot(x, sr=sr, x_axis='time');
librosa.display.waveplot(x_dn,sr=sr, x_axis='time');
plt.axis('off')

# %%
x_mel_dn = librosa.feature.melspectrogram(y=x_dn, sr=sr, n_mels=64)
x_mel_dn_db = librosa.power_to_db(x_mel_dn, ref=np.max)
plt.figure(figsize=(14, 5))
librosa.display.specshow(x_mel_dn_db, x_axis='time', y_axis='mel')
plt.axis('off')

# %%
###########
# rVAD segmentation
###########
winlen, ovrlen, pre_coef, nfilter, nftt = 0.025, 0.01, 0.97, 20, 2048
ftThres = 0.4
vadThres = 0.2
opts = 1

# %%
ft, flen, fsh10, nfr10 = speechproc.sflux(x_dn, sr, winlen, ovrlen, nftt)

# %%
# --spectral flatness --
pv01 = np.zeros(nfr10)
pv01[np.less_equal(ft, ftThres)] = 1
pitch = deepcopy(ft)
pvblk = speechproc.pitchblockdetect(pv01, pitch, nfr10, opts)

# %%
# --filtering--
ENERGYFLOOR = np.exp(-50)
b = np.array([0.9770,   -0.9770])
a = np.array([0.3,   -0.3])
fdata = lfilter(b, a, x_dn, axis=0)

# %%
#--pass 1--
noise_samp, noise_seg, n_noise_samp=speechproc.snre_highenergy(fdata, nfr10, flen, fsh10,
                                                               ENERGYFLOOR, pv01, pvblk)

# %%
#sets noisy segments to zero
for j in range(n_noise_samp):
    fdata[range(int(noise_samp[j, 0]),  int(noise_samp[j, 1]) + 1)] = 0

# %%
vad_seg = speechproc.snre_vad(fdata, nfr10, flen, fsh10, ENERGYFLOOR, pv01, pvblk, vadThres)

# %%
plt.figure(figsize=(14, 5))
librosa.display.waveplot(vad_seg.astype('float'), sr=sr, x_axis='time')

# %%
import scipy.interpolate as interp
interp = interp.interp1d(np.arange(vad_seg.size), vad_seg)
vad_seg_st = interp(np.linspace(0, vad_seg.size-1, fdata.size))

# %%
plt.figure(figsize=(14, 5))
librosa.display.waveplot(vad_seg_st.astype('float') * fdata.max(), sr=sr, x_axis='time')
librosa.display.waveplot(fdata, sr=sr, x_axis='time')

no_events_starttime = [i / len(vad_seg) * audio.duration() for i in range(len(vad_seg)) if vad_seg[i] == 0 and vad_seg[i-1] == 1]

max_stars_repo_name: microsoft/Akoustos

max_issues_repo_name: microsoft/Akoustos

max_forks_repo_name: microsoft/Akoustos

reason: import numpy,import scipy,from scipy


text:
"""
Framework described in
Data augmentation using learned transformationsfor one-shot medical image segmentation
http://www.mit.edu/~adalca/files/papers/cvpr2019_brainstorm.pdf

"""
from .net_utils import gen_identity_map
import mermaid.finite_differences as fdt
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from .net_utils import Bilinear
import pynd.segutils as pynd_segutils

class convBlock(nn.Module):
    """
    A convolutional block including conv, BN, nonliear activiation, residual connection
    """
    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1,
                 bias=True, batchnorm=False, residual=False, max_pool=False, nonlinear=nn.LeakyReLU(0.2)):
        """

        :param in_channels:
        :param out_channels:
        :param kernel_size:
        :param stride:
        :param padding:
        :param bias:
        :param batchnorm:
        :param residual:
        :param nonlinear:
        """

        super(convBlock, self).__init__()
        self.conv = nn.Conv3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=bias)
        self.bn = nn.BatchNorm3d(out_channels) if batchnorm else None
        self.nonlinear = nonlinear
        self.residual = residual
        self.max_pool = nn.MaxPool3d(kernel_size=(2,2,2),stride=2) if max_pool else None

    def forward(self, x):
        x= self.conv(x)
        if self.bn:
            x = self.bn(x)
        if self.nonlinear:
            x = self.nonlinear(x)
        if self.residual:
            x += x

        if not self.max_pool:
            return x
        else:
            y= self.max_pool(x)
            return x, y

class TransformCVPR2019(nn.Module):
    """
    unet architecture for voxelmorph models presented in the CVPR 2018 paper.
    You may need to modify this code (e.g., number of layers) to suit your project needs.

    :param vol_size: volume size. e.g. (256, 256, 256)
    :param enc_nf: list of encoder filters. right now it needs to be 1x4.
           e.g. [16,32,32,32]
    :param dec_nf: list of decoder filters. right now it must be 1x6 (like voxelmorph-1) or 1x7 (voxelmorph-2)
    :return: the keras reg_model
    """
    def __init__(self, img_sz, opt=None):
        super(TransformCVPR2019, self).__init__()
        self.is_train = opt['tsk_set'][('train',False,'if is in train mode')]
        opt_voxelmorph = opt['tsk_set']['reg']['aug_trans_net']
        self.initial_reg_factor = opt_voxelmorph[('initial_reg_factor', 1., 'initial regularization factor')]
        enc_filters = [16, 32, 32, 32 ]
        dec_filters = [32, 32, 32, 32, 32, 16, 16]
        self.enc_filter = enc_filters
        self.dec_filter = dec_filters
        input_channel =2
        output_channel= 3
        self.input_channel = input_channel
        self.output_channel = output_channel
        self.img_sz = img_sz
        self.spacing = 1. / ( np.array(img_sz) - 1)
        self.loss_fn = None #NCCLoss()
        self.epoch = -1
        self.print_count = 0
        self.id_transform = gen_identity_map(self.img_sz, 1.0).cuda()
        self.encoders = nn.ModuleList()
        self.decoders = nn.ModuleList()
        self.bilinear = Bilinear(zero_boundary=True)
        for i in range(len(enc_filters)):
            if i==0:
                self.encoders.append(convBlock(input_channel, enc_filters[i], stride=1, max_pool=True, bias=True))
            if i>0 and i<len(enc_filters)-1:
                self.encoders.append(convBlock(enc_filters[i-1], enc_filters[i], stride=1,max_pool=True, bias=True))
            if i ==len(enc_filters)-1:
                self.encoders.append(convBlock(enc_filters[i-1], enc_filters[i], stride=1,max_pool=False, bias=True))

        self.decoders.append(convBlock(enc_filters[3] + enc_filters[2],dec_filters[0], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[0] + enc_filters[1],dec_filters[1], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[1] + enc_filters[0],dec_filters[2], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[2],dec_filters[3], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[3],dec_filters[4], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[4],dec_filters[5], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[5], dec_filters[6],stride=1, bias=True))
        self.final_conv = nn.Conv3d(dec_filters[6], output_channel, kernel_size=3, stride=1, padding=1, bias=True)

        self.flow = nn.Conv3d(output_channel, output_channel, kernel_size=3, stride=1, padding=1, bias=True)

    def set_loss_fn(self, loss_fn):
        """ set loss function"""
        self.loss_fn = loss_fn

    def set_cur_epoch(self, cur_epoch=-1):
        """ set current epoch"""
        self.epoch = cur_epoch

    def forward(self, source, target,source_mask=None, target_mask=None):
        id_map = self.id_transform.clone()

        x_enc_0, x = self.encoders[0](torch.cat((source, target), dim=1))
        x_enc_1, x = self.encoders[1](x)
        x_enc_2, x = self.encoders[2](x)
        x_enc_3 = self.encoders[3](x)

        x = F.interpolate(x_enc_3,scale_factor=2)
        x = torch.cat((x, x_enc_2),dim=1)
        x = self.decoders[0](x)
        x = F.interpolate(x, scale_factor=2)
        x = torch.cat((x, x_enc_1), dim=1)
        x = self.decoders[1](x)
        x = F.interpolate(x, scale_factor=2)
        x = torch.cat((x, x_enc_0), dim=1)
        x = self.decoders[2](x)
        x = self.decoders[3](x)
        x = self.decoders[4](x)
        x = self.decoders[5](x)
        x = self.decoders[6](x)
        x = self.final_conv(x)
        disp_field = self.flow(x)

        deform_field = disp_field + id_map
        warped_source = self.bilinear(source, deform_field)
        self.warped = warped_source
        self.target = target
        self.disp_field = disp_field
        if self.train:
            self.print_count += 1
        return warped_source, deform_field, disp_field

    def get_extra_to_plot(self):
        return None, None

    def check_if_update_lr(self):
        return False, None

    def scale_reg_loss(self,sched='l2'):
        disp = self.disp_field
        fd = fdt.FD_torch(self.spacing*2)
        dfx = fd.dXc(disp[:, 0, ...])
        dfy = fd.dYc(disp[:, 1, ...])
        dfz = fd.dZc(disp[:, 2, ...])
        l2 = dfx**2+dfy**2+dfz**2
        reg = l2.mean()
        return reg

    def get_sim_loss(self):
        sim_loss = self.loss_fn.get_loss(self.warped,self.target)
        sim_loss =  sim_loss / self.warped.shape[0]
        return sim_loss

    def weights_init(self):
        for m in self.modules():
            classname = m.__class__.__name__
            if classname.find('Conv') != -1:
                if not m.weight is None:
                    nn.init.xavier_normal_(m.weight.data)
                if not m.bias is None:
                    m.bias.data.zero_()

    def get_loss(self):
        reg_factor =self.initial_reg_factor
        sim_loss = self.get_sim_loss()
        reg_loss = self.scale_reg_loss()
        if self.print_count % 10 == 0:
            print('current sim loss is{}, current_reg_loss is {}, and reg_factor is {} '.format(sim_loss.item(),
                                                                                reg_loss.item(),
                                                                                reg_factor))
        return sim_loss+ reg_factor*reg_loss

class AppearanceCVPR2019(nn.Module):
    """
    unet architecture for voxelmorph models presented in the CVPR 2018 paper.
    You may need to modify this code (e.g., number of layers) to suit your project needs.

    :param vol_size: volume size. e.g. (256, 256, 256)
    :param enc_nf: list of encoder filters. right now it needs to be 1x4.
           e.g. [16,32,32,32]
    :param dec_nf: list of decoder filters. right now it must be 1x6 (like voxelmorph-1) or 1x7 (voxelmorph-2)
    :return: the keras reg_model
    """
    def __init__(self, img_sz, opt=None):
        super(AppearanceCVPR2019, self).__init__()
        self.is_train = opt['tsk_set'][('train',False,'if is in train mode')]
        opt_voxelmorph = opt['tsk_set']['reg']['aug_appear_net']
        self.initial_reg_factor = opt_voxelmorph[('initial_reg_factor', 1., 'initial regularization factor')]
        self.sim_factor = opt_voxelmorph[('sim_factor', 1., 'initial regularization factor')]
        enc_filters = [16, 32, 32, 32, 32, 32]
        dec_filters = [64, 64, 32, 32, 32, 16, 16]
        self.enc_filter = enc_filters
        self.dec_filter = dec_filters
        input_channel =2
        output_channel= 3
        self.input_channel = 2
        self.output_channel = 3
        self.img_sz = img_sz
        self.spacing = 1. / ( np.array(img_sz) - 1)
        self.loss_fn = None #NCCLoss()
        self.epoch = -1
        self.print_count = 0
        self.encoders = nn.ModuleList()
        self.decoders = nn.ModuleList()
        self.bilinear = Bilinear(zero_boundary=True)
        for i in range(len(enc_filters)):
            if i == 0:
                self.encoders.append(convBlock(input_channel, enc_filters[i], stride=1, max_pool=True, bias=True))
            if i > 0 and i < len(enc_filters) - 1:
                self.encoders.append(convBlock(enc_filters[i - 1], enc_filters[i], stride=1, max_pool=True, bias=True))
            if i == len(enc_filters) - 1:
                self.encoders.append(convBlock(enc_filters[i - 1], enc_filters[i], stride=1, max_pool=False, bias=True))

        self.decoders.append(convBlock(enc_filters[5] + enc_filters[4], dec_filters[0], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[0] + enc_filters[3], dec_filters[1], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[1] + enc_filters[2], dec_filters[2], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[2] + enc_filters[1], dec_filters[3], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[3] + enc_filters[0], dec_filters[4], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[4], dec_filters[5], stride=1, bias=True))
        self.decoders.append(convBlock(dec_filters[5], dec_filters[6], stride=1, bias=True))
        self.final_conv = nn.Conv3d(dec_filters[6], output_channel, kernel_size=3, stride=1, padding=1, bias=True)
        self.color = nn.Conv3d(output_channel, 1, kernel_size=3, stride=1, padding=1, bias=True)

        self.mask = None
        self.target =None
        self.reconst = None
        self.delta = None

        # identity transform for computing displacement

    def set_loss_fn(self, loss_fn):
        """ set loss function"""
        self.loss_fn = loss_fn

    def set_cur_epoch(self, cur_epoch=-1):
        """ set current epoch"""
        self.epoch = cur_epoch

    def forward(self, source, target,source_mask=None, target_mask=None):

        x_enc_0,x = self.encoders[0](torch.cat((source, target), dim=1))
        x_enc_1,x = self.encoders[1](x)
        x_enc_2,x = self.encoders[2](x)
        x_enc_3,x = self.encoders[3](x)
        x_enc_4,x = self.encoders[4](x)
        x_enc_5 = self.encoders[5](x)

        x = F.interpolate(x_enc_5,scale_factor=2)
        x = torch.cat((x, x_enc_4),dim=1)
        x = self.decoders[0](x)
        x = F.interpolate(x,size=x_enc_3.shape[2:])
        x = torch.cat((x, x_enc_3), dim=1)
        x = self.decoders[1](x)
        x = F.interpolate(x, scale_factor=2)
        x = torch.cat((x, x_enc_2), dim=1)
        x = self.decoders[2](x)
        x = F.interpolate(x, scale_factor=2)
        x = torch.cat((x, x_enc_1), dim=1)
        x = self.decoders[3](x)
        x = F.interpolate(x, scale_factor=2)
        x = torch.cat((x, x_enc_0), dim=1)
        x = self.decoders[4](x)
        x = self.decoders[5](x)
        x = self.decoders[6](x)
        x = self.final_conv(x)

        delta = self.color(x)
        reconst = source + delta
        self.delta = delta
        self.reconst = reconst
        self.target = target
        if self.train:
            self.print_count += 1
        return reconst,delta,delta

    def get_extra_to_plot(self):
        return None, None

    def check_if_update_lr(self):
        return False, None

    def get_sim_loss(self):

        sim_loss = self.loss_fn.get_loss(self.reconst,self.target)
        sim_loss =  sim_loss / self.reconst.shape[0]
        sim_loss = sim_loss *self.sim_factor
        return sim_loss

    def weights_init(self):
        for m in self.modules():
            classname = m.__class__.__name__
            if classname.find('Conv') != -1:
                if not m.weight is None:
                    nn.init.xavier_normal_(m.weight.data)
                if not m.bias is None:
                    m.bias.data.zero_()

    def scale_reg_loss(self):
        def __compute_contour(seg_data):
            contours = pynd_segutils.seg2contour(seg_data,exclude_zero=True, contour_type='both')[None]
            contours[contours > 0] = 1
            return torch.Tensor(contours).cuda()
        if self.mask is None:
            import SimpleITK as sitk
            atlas_path = '/playpen-raid/zyshen/data/oai_seg/atlas_label.nii.gz'
            seg  = sitk.GetArrayFromImage(sitk.ReadImage(atlas_path))
            contour = __compute_contour(seg)
            self.mask = 1.0 - contour

        delta = self.delta
        fd = fdt.FD_torch(self.spacing * 2)
        dfx = fd.dXc(delta[:, 0, ...])
        dfy = fd.dYc(delta[:, 0, ...])
        dfz = fd.dZc(delta[:, 0, ...])
        dabs = dfx.abs() + dfy.abs() + dfz.abs()
        l2 = self.mask*dabs
        reg = l2.mean()
        return reg

    def get_loss(self):
        reg_factor = self.initial_reg_factor
        sim_loss = self.get_sim_loss()
        reg_loss = self.scale_reg_loss()
        if self.print_count % 10 == 0:
            print('current sim loss is{}, current_reg_loss is {}, and reg_factor is {} '.format(sim_loss.item(),
                                                                                reg_loss.item(),
                                                                                reg_factor))
        return sim_loss+ reg_factor*reg_loss

max_stars_repo_name: uncbiag/easyreg

max_issues_repo_name: uncbiag/easyreg

max_forks_repo_name: uncbiag/easyreg


text:
# coding:utf-8
import tensorflow as tf
from tensorflow.contrib import slim
import numpy as np

num_keep_radio = 0.7

def prelu(inputs):
    alphas = tf.get_variable("alphas", shape=inputs.get_shape()[-1], dtype=tf.float32,
                             initializer=tf.constant_initializer(0.25))
    pos = tf.nn.relu(inputs)
    neg = alphas * (inputs - abs(inputs)) * 0.5
    return pos + neg

def dense_to_one_hot(labels_dense, num_classes):
    num_labels = labels_dense.shape[0]
    index_offset = np.arange(num_labels) * num_classes
    labels_one_hot = np.zeros((num_labels, num_classes))
    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
    return labels_one_hot

def cls_ohem(cls_prob, label):
    zeros = tf.zeros_like(label)
    label_filter_invalid = tf.where(tf.less(label, 0), zeros, label)
    num_cls_prob = tf.size(cls_prob)
    cls_prob_reshape = tf.reshape(cls_prob, [num_cls_prob, -1])
    label_int = tf.cast(label_filter_invalid, tf.int32)
    # get the number of rows of class_prob
    num_row = tf.to_int32(cls_prob.get_shape()[0])
    row = tf.range(num_row) * 2
    indices_ = row + label_int
    label_prob = tf.squeeze(tf.gather(cls_prob_reshape, indices_))
    loss = -tf.log(label_prob + 1e-10)
    zeros = tf.zeros_like(label_prob, dtype=tf.float32)
    ones = tf.ones_like(label_prob, dtype=tf.float32)
    # set pos and neg to be 1, rest to be 0
    valid_inds = tf.where(label < zeros, zeros, ones)
    num_valid = tf.reduce_sum(valid_inds)

    keep_num = tf.cast(num_valid * num_keep_radio, dtype=tf.int32)
    loss = loss * valid_inds
    loss, _ = tf.nn.top_k(loss, k=keep_num)
    return tf.reduce_mean(loss)

def bbox_ohem_smooth_L1_loss(bbox_pred, bbox_target, label):
    sigma = tf.constant(1.0)
    threshold = 1.0 / (sigma ** 2)
    zeros_index = tf.zeros_like(label, dtype=tf.float32)
    valid_inds = tf.where(label != zeros_index, tf.ones_like(label, dtype=tf.float32), zeros_index)
    abs_error = tf.abs(bbox_pred - bbox_target)
    loss_smaller = 0.5 * ((abs_error * sigma) ** 2)
    loss_larger = abs_error - 0.5 / (sigma ** 2)
    smooth_loss = tf.reduce_sum(tf.where(abs_error < threshold, loss_smaller, loss_larger), axis=1)
    keep_num = tf.cast(tf.reduce_sum(valid_inds) * num_keep_radio, dtype=tf.int32)
    smooth_loss = smooth_loss * valid_inds
    _, k_index = tf.nn.top_k(smooth_loss, k=keep_num)
    smooth_loss_picked = tf.gather(smooth_loss, k_index)
    return tf.reduce_mean(smooth_loss_picked)

def bbox_ohem_orginal(bbox_pred, bbox_target, label):
    zeros_index = tf.zeros_like(label, dtype=tf.float32)
    valid_inds = tf.where(label != zeros_index, tf.ones_like(label, dtype=tf.float32), zeros_index)
    square_error = tf.reduce_sum(tf.square(bbox_pred - bbox_target), axis=1)
    keep_num = tf.cast(tf.reduce_sum(valid_inds) * num_keep_radio, dtype=tf.int32)
    square_error = square_error * valid_inds
    _, k_index = tf.nn.top_k(square_error, k=keep_num)
    square_error = tf.gather(square_error, k_index)
    return tf.reduce_mean(square_error)

def bbox_ohem(bbox_pred, bbox_target, label):
    zeros_index = tf.zeros_like(label, dtype=tf.float32)
    ones_index = tf.ones_like(label, dtype=tf.float32)
    valid_inds = tf.where(tf.equal(tf.abs(label), 1), ones_index, zeros_index)
    square_error = tf.square(bbox_pred - bbox_target)
    square_error = tf.reduce_sum(square_error, axis=1)
    num_valid = tf.reduce_sum(valid_inds)
    keep_num = tf.cast(num_valid, dtype=tf.int32)
    square_error = square_error * valid_inds
    _, k_index = tf.nn.top_k(square_error, k=keep_num)
    square_error = tf.gather(square_error, k_index)

    return tf.reduce_mean(square_error)

def landmark_ohem(landmark_pred, landmark_target, label):
    ones = tf.ones_like(label, dtype=tf.float32)
    zeros = tf.zeros_like(label, dtype=tf.float32)
    valid_inds = tf.where(tf.equal(label, -2), ones, zeros)
    square_error = tf.square(landmark_pred - landmark_target)
    square_error = tf.reduce_sum(square_error, axis=1)
    num_valid = tf.reduce_sum(valid_inds)
    keep_num = tf.cast(num_valid, dtype=tf.int32)
    square_error = square_error * valid_inds
    _, k_index = tf.nn.top_k(square_error, k=keep_num)
    square_error = tf.gather(square_error, k_index)
    return tf.reduce_mean(square_error)

def cal_accuracy(cls_prob, label):
    pred = tf.argmax(cls_prob, axis=1)
    label_int = tf.cast(label, tf.int64)
    cond = tf.where(tf.greater_equal(label_int, 0))
    picked = tf.squeeze(cond)
    label_picked = tf.gather(label_int, picked)
    pred_picked = tf.gather(pred, picked)
    accuracy_op = tf.reduce_mean(tf.cast(tf.equal(label_picked, pred_picked), tf.float32))
    return accuracy_op

def _activation_summary(x):
    tensor_name = x.op.name
    print('load summary for : ', tensor_name)
    tf.summary.histogram(tensor_name + '/activations', x)

def P_Net(inputs, label=None, bbox_target=None, landmark_target=None, training=True):
    with slim.arg_scope([slim.conv2d],
                        activation_fn=prelu,
                        weights_initializer=slim.xavier_initializer(),
                        biases_initializer=tf.zeros_initializer(),
                        weights_regularizer=slim.l2_regularizer(0.0005),
                        padding='valid'):
        print(inputs.get_shape())

        net = slim.conv2d(inputs, 10, 3, stride=1, scope='conv1')
        _activation_summary(net)
        print(net.get_shape())
        net = slim.max_pool2d(net, kernel_size=[2, 2], stride=2, scope='pool1', padding='SAME')
        _activation_summary(net)
        print(net.get_shape())
        net = slim.conv2d(net, num_outputs=16, kernel_size=[3, 3], stride=1, scope='conv2')
        _activation_summary(net)
        print(net.get_shape())
        net = slim.conv2d(net, num_outputs=32, kernel_size=[3, 3], stride=1, scope='conv3')
        _activation_summary(net)
        print(net.get_shape())
        conv4_1 = slim.conv2d(net, num_outputs=2, kernel_size=[1, 1], stride=1, scope='conv4_1',
                              activation_fn=tf.nn.softmax)
        _activation_summary(conv4_1)
        # conv4_1 = slim.conv2d(net,num_outputs=1,kernel_size=[1,1],stride=1,scope='conv4_1',activation_fn=tf.nn.sigmoid)

        print(conv4_1.get_shape())
        bbox_pred = slim.conv2d(net, num_outputs=4, kernel_size=[1, 1], stride=1, scope='conv4_2', activation_fn=None)
        _activation_summary(bbox_pred)
        print(bbox_pred.get_shape())
        landmark_pred = slim.conv2d(net, num_outputs=10, kernel_size=[1, 1], stride=1, scope='conv4_3',
                                    activation_fn=None)
        _activation_summary(landmark_pred)
        print(landmark_pred.get_shape())

        if training:
            cls_prob = tf.squeeze(conv4_1, [1, 2], name='cls_prob')
            cls_loss = cls_ohem(cls_prob, label)
            bbox_pred = tf.squeeze(bbox_pred, [1, 2], name='bbox_pred')
            bbox_loss = bbox_ohem(bbox_pred, bbox_target, label)

            accuracy = cal_accuracy(cls_prob, label)
            L2_loss = tf.add_n(slim.losses.get_regularization_losses())
            return cls_loss, bbox_loss, L2_loss, accuracy
        # test
        else:
            # when test,batch_size = 1
            cls_pro_test = tf.squeeze(conv4_1, axis=0)
            bbox_pred_test = tf.squeeze(bbox_pred, axis=0)
            landmark_pred_test = tf.squeeze(landmark_pred, axis=0)
            return cls_pro_test, bbox_pred_test, landmark_pred_test

def R_Net(inputs, label=None, bbox_target=None, landmark_target=None, training=True):
    with slim.arg_scope([slim.conv2d],
                        activation_fn=prelu,
                        weights_initializer=slim.xavier_initializer(),
                        biases_initializer=tf.zeros_initializer(),
                        weights_regularizer=slim.l2_regularizer(0.0005),
                        padding='valid'):
        print(inputs.get_shape())
        net = slim.conv2d(inputs, num_outputs=28, kernel_size=[3, 3], stride=1, scope="conv1")
        print(net.get_shape())
        net = slim.max_pool2d(net, kernel_size=[3, 3], stride=2, scope="pool1", padding='SAME')
        print(net.get_shape())
        net = slim.conv2d(net, num_outputs=48, kernel_size=[3, 3], stride=1, scope="conv2")
        print(net.get_shape())
        net = slim.max_pool2d(net, kernel_size=[3, 3], stride=2, scope="pool2")
        print(net.get_shape())
        net = slim.conv2d(net, num_outputs=64, kernel_size=[2, 2], stride=1, scope="conv3")
        print(net.get_shape())
        fc_flatten = slim.flatten(net)
        print(fc_flatten.get_shape())
        fc1 = slim.fully_connected(fc_flatten, num_outputs=128, scope="fc1")
        print(fc1.get_shape())
        # batch*2
        cls_prob = slim.fully_connected(fc1, num_outputs=2, scope="cls_fc", activation_fn=tf.nn.softmax)
        print(cls_prob.get_shape())
        # batch*4
        bbox_pred = slim.fully_connected(fc1, num_outputs=4, scope="bbox_fc", activation_fn=None)
        print(bbox_pred.get_shape())
        # batch*10
        landmark_pred = slim.fully_connected(fc1, num_outputs=10, scope="landmark_fc", activation_fn=None)
        print(landmark_pred.get_shape())
        # train
        if training:
            cls_loss = cls_ohem(cls_prob, label)
            bbox_loss = bbox_ohem(bbox_pred, bbox_target, label)
            accuracy = cal_accuracy(cls_prob, label)
            landmark_loss = landmark_ohem(landmark_pred, landmark_target, label)
            L2_loss = tf.add_n(slim.losses.get_regularization_losses())
            return cls_loss, bbox_loss, landmark_loss, L2_loss, accuracy
        else:
            return cls_prob, bbox_pred, landmark_pred

def O_Net(inputs, label=None, bbox_target=None, landmark_target=None, training=True):
    with slim.arg_scope([slim.conv2d],
                        activation_fn=prelu,
                        weights_initializer=slim.xavier_initializer(),
                        biases_initializer=tf.zeros_initializer(),
                        weights_regularizer=slim.l2_regularizer(0.0005),
                        padding='valid'):
        print(inputs.get_shape())
        net = slim.conv2d(inputs, num_outputs=32, kernel_size=[3, 3], stride=1, scope="conv1")
        print(net.get_shape())
        net = slim.max_pool2d(net, kernel_size=[3, 3], stride=2, scope="pool1", padding='SAME')
        print(net.get_shape())
        net = slim.conv2d(net, num_outputs=64, kernel_size=[3, 3], stride=1, scope="conv2")
        print(net.get_shape())
        net = slim.max_pool2d(net, kernel_size=[3, 3], stride=2, scope="pool2")
        print(net.get_shape())
        net = slim.conv2d(net, num_outputs=64, kernel_size=[3, 3], stride=1, scope="conv3")
        print(net.get_shape())
        net = slim.max_pool2d(net, kernel_size=[2, 2], stride=2, scope="pool3", padding='SAME')
        print(net.get_shape())
        net = slim.conv2d(net, num_outputs=128, kernel_size=[2, 2], stride=1, scope="conv4")
        print(net.get_shape())
        fc_flatten = slim.flatten(net)
        print(fc_flatten.get_shape())
        fc1 = slim.fully_connected(fc_flatten, num_outputs=256, scope="fc1")
        print(fc1.get_shape())
        # batch*2
        cls_prob = slim.fully_connected(fc1, num_outputs=2, scope="cls_fc", activation_fn=tf.nn.softmax)
        print(cls_prob.get_shape())
        # batch*4
        bbox_pred = slim.fully_connected(fc1, num_outputs=4, scope="bbox_fc", activation_fn=None)
        print(bbox_pred.get_shape())
        # batch*10
        landmark_pred = slim.fully_connected(fc1, num_outputs=10, scope="landmark_fc", activation_fn=None)
        print(landmark_pred.get_shape())
        # train
        if training:
            cls_loss = cls_ohem(cls_prob, label)
            bbox_loss = bbox_ohem(bbox_pred, bbox_target, label)
            accuracy = cal_accuracy(cls_prob, label)
            landmark_loss = landmark_ohem(landmark_pred, landmark_target, label)
            L2_loss = tf.add_n(slim.losses.get_regularization_losses())
            return cls_loss, bbox_loss, landmark_loss, L2_loss, accuracy
        else:
            return cls_prob, bbox_pred, landmark_pred


text:
# --------------
# Importing header files
import numpy as np
import pandas as pd
from scipy.stats import mode

import warnings
warnings.filterwarnings('ignore')

#Reading file
bank_data = pd.read_csv(path)
#Checking all categorical values
categorical_var = bank_data.select_dtypes(include = 'object')
print(categorical_var)
#Checking all numerical values
numerical_var = bank_data.select_dtypes(include = 'number')
print(numerical_var)
#drop the column Loan_ID to create a new dataframe banks
banks = bank_data.drop('Loan_ID',axis=1)
#see the null values
print(banks.isnull().sum())

bank_mode = banks.mode().iloc[0]
banks.fillna(bank_mode, inplace = True)
banks.isnull()
print(banks.isnull().sum())

#Now let's check the loan amount of an average person based on 'Gender', 'Married', 'Self_Employed'.
avg_loan_amount = banks.pivot_table(index=['Gender', 'Married', 'Self_Employed'],values='LoanAmount',aggfunc='mean')

loan_approved_se = banks[(banks['Self_Employed'] == 'Yes') & (banks['Loan_Status'] == 'Y')]
loan_approved_nse = banks[(banks['Self_Employed'] == 'No') & (banks['Loan_Status'] == 'Y')]
percentage_se = round((len(loan_approved_se)/614)*100,2)
print(percentage_se)
percentage_nse = round((len(loan_approved_nse)/614)*100,2)
print(percentage_nse)

loan_term = banks['Loan_Amount_Term'].apply(lambda x:x/12)
big_loan_term = 0
for i in loan_term:
    if i>=25:
        big_loan_term+=1
print(big_loan_term)

loan_groupby = banks.groupby('Loan_Status')[['ApplicantIncome', 'Credit_History']]
mean_values = loan_groupby.mean()
print(mean_values)