text:
'''
Created on Nov, 2016

@author: hugo

'''
from __future__ import absolute_import
from os import path
import numpy as np
from keras.layers import Input, Dense, Lambda, Dropout
from keras.models import Model
from keras.optimizers import Adadelta, Adam, Adagrad
from keras.models import load_model
from keras import regularizers
import keras.backend as K
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.layers.core import Activation
# from keras.layers.normalization import BatchNormalization

from ..utils.keras_utils import Dense_tied, weighted_binary_crossentropy, KCompetitive
from ..utils.io_utils import dump_json, load_json

class DeepAutoEncoder(object):
    """Deep AutoEncoder for topic modeling.

        Parameters
        ----------
        dim : dimensionality of encoding space.

        nb_epoch :

        batch_size :

        """

    def __init__(self, input_size, dim, comp_topk=None, \
        weights_file=None, model_save_path='./'):
        self.input_size = input_size
        self.dim = dim
        self.comp_topk = comp_topk
        self.model_save_path = model_save_path

        self.build(weights_file)

    def build(self, weights_file=None):
        h1_dim = 256

        # this is our input placeholder
        input_layer = Input(shape=(self.input_size,))

        # "encoded" is the encoded representation of the input
        h1_layer = Dense(h1_dim, init='glorot_normal', activation='tanh')
        encoded_layer = Dense(self.dim, init='glorot_normal', activation='tanh')

        encoded = h1_layer(input_layer)

        if self.comp_topk:
            encoded = KCompetitive(self.comp_topk)(encoded)
            print('add k-competitive layer')

        encoded = encoded_layer(encoded)
        if self.comp_topk:
            encoded = KCompetitive(self.comp_topk)(encoded)
            print('add k-competitive layer')

        # "decoded" is the lossy reconstruction of the input
        decoder_layer = Dense_tied(h1_dim, init='glorot_normal', activation='tanh', tied_to=encoded_layer)
        rev_h1_layer = Dense_tied(self.input_size, init='glorot_normal', activation='sigmoid', tied_to=h1_layer)
        decoded = decoder_layer(encoded)

        if self.comp_topk:
            decoded = KCompetitive(self.comp_topk)(decoded)
            print('add k-competitive layer')

        decoded = rev_h1_layer(decoded)

        # this model maps an input to its reconstruction
        self.autoencoder = Model(input=input_layer, output=decoded)

        # this model maps an input to its encoded representation
        self.encoder = Model(input=input_layer, output=encoded)

        # create a placeholder for an encoded (32-dimensional) input
        encoded_input = Input(shape=(self.dim,))

        # create the decoder model
        self.decoder = Model(input=encoded_input, output=rev_h1_layer(decoder_layer(encoded_input)))

        if not weights_file is None:
            self.autoencoder.load_weights(weights_file, by_name=True)
            print('Loaded pretrained weights')

    def fit(self, train_X, val_X, nb_epoch=50, batch_size=100, feature_weights=None):
        print('Training autoencoder')
        optimizer = Adadelta(lr=1.5)
        # optimizer = Adam()
        # optimizer = Adagrad()
        if feature_weights is None:
            self.autoencoder.compile(optimizer=optimizer, loss='binary_crossentropy') # kld, binary_crossentropy, mse
        else:
            print('Using weighted loss')
            self.autoencoder.compile(optimizer=optimizer, loss=weighted_binary_crossentropy(feature_weights)) # kld, binary_crossentropy, mse

        self.autoencoder.fit(train_X[0], train_X[1],
                        nb_epoch=nb_epoch,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(val_X[0], val_X[1]),
                        callbacks=[
                                    ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=0.01),
                                    EarlyStopping(monitor='val_loss', min_delta=1e-5, patience=5, verbose=1, mode='auto'),
                                    # ModelCheckpoint(self.model_save_path, monitor='val_loss', save_best_only=True, verbose=0),
                        ]
                        )

        return self

max_stars_repo_name: qmeeus/KATE

max_issues_repo_name: qmeeus/KATE

max_forks_repo_name: qmeeus/KATE


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Desc: create a excel, read data from it, and save them to the datatbase, remain to refactoring.
Author: Kris Peng
Copyright (c) 2018 - Kris Peng <kris.dacpc@gmail.com>
'''
from openpyxl import load_workbook
from pandas import DataFrame
from sqlalchemy import Column, String, create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
import numpy as np

# 创建对象的基类:
Base = declarative_base()

class Account(Base):
    __tablename__ = 'account'

    id = Column(String(20), primary_key=True)
    name = Column(String(50))
    email = Column(String(50))

dest_filename = 'workbook.xlsx'
wb = load_workbook(dest_filename)
ws = wb.get_sheet_by_name('MySheet')
list = []

for columns in ws.iter_rows(row_offset = 0):
    for column in columns:
        list.append(str(column.value))
list = np.array(list)
list = list.reshape([4,3])
list = list[1:]
# DB part
engine = create_engine('mysql+pymysql://root:pengcheng00@localhost:3306/test?charset=utf8mb4')
DBSession = sessionmaker(bind=engine)
# 创建session对象:
session = DBSession()
# 创建新User对象:
for row in list:
    new_Account = Account(id = row[0], name = row[1], email = row[2])
    # 添加到session:
    session.add(new_Account)
# 提交即保存到数据库:
session.commit()
# 关闭session:
session.close()

ext: py

max_stars_repo_name: KrisCheng/HackerApplication

max_issues_repo_name: KrisCheng/HackerProblem

max_forks_repo_name: KrisCheng/HackerProblem

reason: import numpy


text:
from __future__ import print_function
from wisc_tools.structures import Mode, Position, Quaternion, Pose, ModeTrajectory, PoseTrajectory, AnnotationTrajectory
from wisc_tools.control import EventController
import rospy
import math
import numpy as np

def serialize(input):
    if isinstance(input, dict):
        return {key:serialize(value) for key,value in input.iteritems()}
    elif isinstance(input, list):
        return [serialize(element) for element in input]
    elif isinstance(input, Pose):
        return input.dict
    else:
        return input

class StateController(object):
    '''
    Generic StateController Object.
    Handles updates to goals, modes, annotations, and actions
    '''

    next_group_id = 0

    def __init__(self, rosnode, arms=[], joints=[], modes={}, actions={}, poses={}, annotations={}):
        self.rosnode = rosnode
        self.new(arms, joints, modes, actions, poses, annotations)

    @property
    def now(self):
        return rospy.get_time()

    @property
    def current_serializable(self):
        return serialize(self.current)

    @property
    def future(self):
        return {
            'armData':[self.pose_future(arm,trajectory) for (arm, trajectory) in self.event_controller.arm_trajectories.iteritems()],
            'modeData':[self.mode_future(mode,trajectory) for (mode, trajectory) in self.event_controller.mode_trajectories.iteritems()],
        }

    def new(self, arms, joints, modes, actions, poses, annotations):

        self.current = {'actions':[],'modes':{},'arms':{},'annotations':{},'poses':{}}
        self.arms = arms
        self.joints = joints
        self.modes = modes
        self.actions = actions
        self.annotations = annotations
        self.poses = {}
        for arm, pose in poses.iteritems():
            self.poses[arm] = {pose_name:{'pose':Pose.from_eulerpose_dict(pose_info),'default':pose_info['default']} for (pose_name,pose_info) in pose.iteritems()}
        default_poses = []
        for arm in self.arms:
            for pose,poseinfo in self.poses[arm].iteritems():
                if poseinfo['default']:
                    default_poses.append(poseinfo['pose'].ros_pose)
        print(default_poses)

        self.event_controller = EventController({arm:[info['pose'] for pose,info in poses.iteritems() if info['default']][0] for arm,poses in self.poses.iteritems()},
                                                self.annotations,
                                                self.modes)
        self.initialize()

    def pose_future(self,arm,trajectory):
        now = self.now
        times = list(np.linspace(now, now+5, 10))
        plan = [trajectory[time] for time in times]
        return {
                'name':arm,
                'points':[[point.position.y,point.position.z,point.position.x] for point in plan]
               }
        # return {
        #   'name': arm,
        #   'x': [t.position.x for t in plan],
        #   'y': [t.position.y for t in plan],
        #   'z': [t.position.z for t in plan],
        #   'type': 'scatter3d',
        #   'mode': 'lines+markers',
        #   'line': {'color': [t-now for t in times],
        #            'colorscale': "Viridis"},
        #   'marker': {'color': [t-now for t in times],
        #              'colorscale': "Viridis"}
        # }

    def mode_future(self,mode,trajectory):
        now = self.now
        times = list(np.linspace(now, now+5, 20))
        plan = [trajectory[time] for time in times]
        return {
                'name':mode,
                'values':plan
               }
        # return {
        #   'name': mode,
        #   'x': [t-now for t in times],
        #   'y': plan,
        #   'type': 'scatter',
        #   'opacity': 0.5,
        #   'mode': 'lines+markers'
        # }

    def set_action(self,action):
        # Actions piggyback off poses and modes.
        print('Setting action: {0}'.format(action))
        # Get the time to do the first action, and then specify the offsets based on that
        now = self.now
        times = []
        for arm in self.actions[action].keys():
            current_pose = self.event_controller.arm_trajectories[arm][now]
            goal_pose = self.poses[arm][self.actions[action][arm][0]['pose']]['pose']
            times.append(self.time_to_pose(current_pose,goal_pose))

        # Offset is the max time estimate across arms
        ttp = max(times)

        for arm in self.actions[action].keys():
            current_pose = self.event_controller.arm_trajectories[arm][now]
            goal_pose = self.poses[arm][self.actions[action][arm][0]['pose']]['pose']
            last = None
            for event in self.actions[action][arm]:
                if last is None:
                    self.event_controller.add_pose_at_time(self.now, now + ttp, arm, self.poses[arm][event['pose']]['pose'], self.next_group_id)
                    for e in event['modes']:
                        self.event_controller.add_mode_at_time(self.now, now + ttp, e, self.modes[e]['values'][event['modes'][e]], False, self.next_group_id)
                    for e in event['annotations']:
                        self.event_controller.add_annotation_at_time(self.now, now + ttp, e, event['annotations'][e], self.next_group_id)
                else:
                    self.event_controller.add_pose_at_time(self.now, now + ttp + last['time'], arm, self.poses[arm][event['pose']]['pose'], self.next_group_id)
                    for e in event['modes']:
                        self.event_controller.add_mode_at_time(self.now, now + ttp + last['time'], e, self.modes[e]['values'][event['modes'][e]], False, self.next_group_id)
                    for e in event['annotations']:
                        self.event_controller.add_annotation_at_time(self.now, now + ttp, e, event['annotations'][e], self.next_group_id)
                last = event
        self.next_group_id += 1
        self.timestep()
        [print({'time': event.time, 'poses': event.poses}) for event in self.event_controller.events]

    def set_pose(self,arm,pose,offset=None):
        # Estimate the amount of time needed to get to that pose
        print('Setting pose for {0} to {1}'.format(arm, pose))
        # If offset is none, calculate the time to do the event
        goal_pose = self.poses[arm][pose]['pose']
        current_time = self.now
        current_pose =  self.event_controller.arm_trajectories[arm][current_time]
        if offset == None:
            offset = self.time_to_pose(current_pose,goal_pose)

        # spatial_dist,rotation_dist = self.current['arms'][arm].distance_to(self.poses[arm][pose]['pose'])
        # print('Estimated distance {0}:{1}'.format(spatial_dist,rotation_dist))
        self.event_controller.add_pose_at_time(self.now, self.now+offset, arm, goal_pose, self.next_group_id)
        self.next_group_id += 1
        # [print({'time': event.time, 'poses': event.poses}) for event in self.event_controller.events]
        self.timestep()
        # self.event_controller.add_pose_at_time()

    def set_mode(self,mode,value,offset=None,override=True):
        # Estimate time needed to smoothly apply that mode
        print('Mode',mode)
        print('Value',value)
        print('Override',override)
        current_time = self.now
        current_value =  self.event_controller.mode_trajectories[mode][current_time]
        if value != None:
            goal_value = self.modes[mode]['values'][value]
        else:
            goal_value = current_value

        time_to_mode = self.time_to_mode(current_value,goal_value)
        mode_time = current_time + time_to_mode
        print('Setting mode for {0} to {1} in {2}s'.format(mode, value, time_to_mode))
        if override:
            self.event_controller.add_mode_at_time(current_time,mode_time,mode,goal_value,True, self.next_group_id)
            self.event_controller.set_mode_override(current_time,mode,True)
            #print(self.event_controller.events)
        else:
            self.event_controller.set_mode_override(current_time,mode,False)
        self.timestep()

    def set_annotation(self,annotation,data):
        print('setting annotation {} with {}'.format(annotation, data))
        self.event_controller.add_annotation_at_time(self.now, self.now, annotation, data, self.next_group_id)
        self.next_group_id += 1

    def initialize(self):
        initial = {'actions':[],'modes':{},'arms':{},'annotations':{},'poses':{}}
        for arm in self.arms:
            now = self.now
            defaults = [pose for pose in self.poses[arm].keys() if self.poses[arm][pose]['default']]
            if len(defaults) >= 1:
                initial['arms'][arm] = defaults[0]
            else:
                initial['arms'][arm] = self.poses[arm].keys()[0]
            pose = self.poses[arm][initial['arms'][arm]]['pose']
            self.event_controller.add_pose_at_time(now,now,arm,pose,0)
        for mode in self.modes.keys():
            override = self.modes[mode]['override']
            value = self.modes[mode]['value']
            current_value = self.modes[mode]['values'][value]
            initial['modes'][mode] = {'override':override,'name':value,'value':current_value}
            self.event_controller.add_mode_at_time(now,now,mode,current_value,override,0)
        self.current = initial
        return initial

    def timestep(self):
        time = self.now
        annotations = self.event_controller.timestep_to(time)
        self.current["annotations"] = annotations
        for arm in self.arms:
            try:
                self.current['arms'][arm] = self.event_controller.arm_trajectories[arm][time]
            except:
                print("could not find arm "+arm)
        for mode in self.modes.keys():
            try:
                current = self.event_controller.mode_trajectories[mode][time]
                name = None
                for value_name,value in self.modes[mode]['values'].iteritems():
                    if value == current:
                        name = value_name
                self.current['modes'][mode] = {'override':self.event_controller.mode_overrides[mode],
                                               'name':name,
                                               'value':self.event_controller.mode_trajectories[mode][time]}
            except:
                print("could not find mode "+mode)

        return serialize(self.current)

    @staticmethod
    def time_to_pose(current_pose,goal_pose):
        # return 3;
        spatial_dist, rotational_dist = current_pose.distance_to(goal_pose)
        #print(spatial_dist,rotational_dist)
        t = max([spatial_dist*5,rotational_dist*2])
        #print(t)
        return t + 0.5

    @staticmethod
    def time_to_mode(current_mode,mode_goal):
        # Hard coded for the time being.
        return math.fabs(current_mode-mode_goal)*10 + 0.5

max_stars_repo_name: Wisc-HCI/wiscutils

max_issues_repo_name: Wisc-HCI/interpolation

max_forks_repo_name: Wisc-HCI/interpolation


text:
# from napari.layers.base.base import Layer
# from napari.utils.events import Event
# from napari.utils.colormaps import AVAILABLE_COLORMAPS

from typing import Dict, List, Union
from warnings import warn

import numpy as np

from ...utils.colormaps import AVAILABLE_COLORMAPS, Colormap
from ...utils.events import Event
from ...utils.status_messages import format_float
from ..base import Layer
from ._track_utils import TrackManager

class Tracks(Layer):
    """Tracks layer.

    Parameters
    ----------
    data : array (N, D+1)
        Coordinates for N points in D+1 dimensions. ID,T,(Z),Y,X. The first
        axis is the integer ID of the track. D is either 3 or 4 for planar
        or volumetric timeseries respectively.
    properties : dict {str: array (N,)}, DataFrame
        Properties for each point. Each property should be an array of length N,
        where N is the number of points.
    graph : dict {int: list}
        Graph representing associations between tracks. Dictionary defines the
        mapping between a track ID and the parents of the track. This can be
        one (the track has one parent, and the parent has >=1 child) in the
        case of track splitting, or more than one (the track has multiple
        parents, but only one child) in the case of track merging.
        See examples/tracks_3d_with_graph.py
    color_by: str
        Track property (from property keys) by which to color vertices.
    tail_width : float
        Width of the track tails in pixels.
    tail_length : float
        Length of the track tails in units of time.
    colormap : str
        Default colormap to use to set vertex colors. Specialized colormaps,
        relating to specified properties can be passed to the layer via
        colormaps_dict.
    colormaps_dict : dict {str: napari.utils.Colormap}
        Optional dictionary mapping each property to a colormap for that
        property. This allows each property to be assigned a specific colormap,
        rather than having a global colormap for everything.
    name : str
        Name of the layer.
    metadata : dict
        Layer metadata.
    scale : tuple of float
        Scale factors for the layer.
    translate : tuple of float
        Translation values for the layer.
    rotate : float, 3-tuple of float, or n-D array.
        If a float convert into a 2D rotation matrix using that value as an
        angle. If 3-tuple convert into a 3D rotation matrix, using a yaw,
        pitch, roll convention. Otherwise assume an nD rotation. Angles are
        assumed to be in degrees. They can be converted from radians with
        np.degrees if needed.
    shear : 1-D array or n-D array
        Either a vector of upper triangular values, or an nD shear matrix with
        ones along the main diagonal.
    affine: n-D array or napari.utils.transforms.Affine
        (N+1, N+1) affine transformation matrix in homogeneous coordinates.
        The first (N, N) entries correspond to a linear transform and
        the final column is a lenght N translation vector and a 1 or a napari
        AffineTransform object. If provided then translate, scale, rotate, and
        shear values are ignored.
    opacity : float
        Opacity of the layer visual, between 0.0 and 1.0.
    blending : str
        One of a list of preset blending modes that determines how RGB and
        alpha values of the layer visual get mixed. Allowed values are
        {'opaque', 'translucent', and 'additive'}.
    visible : bool
        Whether the layer visual is currently being displayed.

    """

    # The max number of tracks that will ever be used to render the thumbnail
    # If more tracks are present then they are randomly subsampled
    _max_tracks_thumbnail = 1024

    def __init__(
        self,
        data,
        *,
        properties=None,
        graph=None,
        tail_width=2,
        tail_length=30,
        name=None,
        metadata=None,
        scale=None,
        translate=None,
        rotate=None,
        shear=None,
        affine=None,
        opacity=1,
        blending='additive',
        visible=True,
        colormap='turbo',
        color_by='track_id',
        colormaps_dict=None,
    ):

        # if not provided with any data, set up an empty layer in 2D+t
        if data is None:
            data = np.empty((0, 4))
        else:
            # convert data to a numpy array if it is not already one
            data = np.asarray(data)

        # in absence of properties make the default an empty dict
        if properties is None:
            properties = {}

        # set the track data dimensions (remove ID from data)
        ndim = data.shape[1] - 1

        super().__init__(
            data,
            ndim,
            name=name,
            metadata=metadata,
            scale=scale,
            translate=translate,
            rotate=rotate,
            shear=shear,
            affine=affine,
            opacity=opacity,
            blending=blending,
            visible=visible,
        )

        self.events.add(
            tail_width=Event,
            tail_length=Event,
            display_id=Event,
            display_tail=Event,
            display_graph=Event,
            color_by=Event,
            colormap=Event,
            properties=Event,
            rebuild_tracks=Event,
            rebuild_graph=Event,
        )

        # track manager deals with data slicing, graph building and properties
        self._manager = TrackManager()
        self._track_colors = None
        self._colormaps_dict = colormaps_dict or {}  # additional colormaps
        self._color_by = color_by  # default color by ID
        self._colormap = colormap

        # use this to update shaders when the displayed dims change
        self._current_displayed_dims = None

        # track display properties
        self.tail_width = tail_width
        self.tail_length = tail_length
        self.display_id = False
        self.display_tail = True
        self.display_graph = True

        # set the data, properties and graph
        self.data = data
        self.properties = properties
        self.graph = graph or {}

        self.color_by = color_by
        self.colormap = colormap

        self._update_dims()

        # reset the display before returning
        self._current_displayed_dims = None

    @property
    def _extent_data(self) -> np.ndarray:
        """Extent of layer in data coordinates.

        Returns
        -------
        extent_data : array, shape (2, D)
        """
        if len(self.data) == 0:
            extrema = np.full((2, self.ndim), np.nan)
        else:
            maxs = np.max(self.data, axis=0)
            mins = np.min(self.data, axis=0)
            extrema = np.vstack([mins, maxs])
        return extrema[:, 1:]

    def _get_ndim(self) -> int:
        """Determine number of dimensions of the layer."""
        return self._manager.ndim

    def _get_state(self):
        """Get dictionary of layer state.

        Returns
        -------
        state : dict
            Dictionary of layer state.
        """
        state = self._get_base_state()
        state.update(
            {
                'data': self.data,
                'properties': self.properties,
                'graph': self.graph,
                'color_by': self.color_by,
                'colormap': self.colormap,
                'colormaps_dict': self.colormaps_dict,
                'tail_width': self.tail_width,
                'tail_length': self.tail_length,
            }
        )
        return state

    def _set_view_slice(self):
        """Sets the view given the indices to slice with."""

        # if the displayed dims have changed, update the shader data
        if self._dims.displayed != self._current_displayed_dims:
            # store the new dims
            self._current_displayed_dims = self._dims.displayed
            # fire the events to update the shaders
            self.events.rebuild_tracks()
            self.events.rebuild_graph()

        return

    def _get_value(self) -> int:
        """ use a kd-tree to lookup the ID of the nearest tree """
        coords = np.array(self.coordinates)
        return self._manager.get_value(coords)

    def _update_thumbnail(self):
        """Update thumbnail with current points and colors."""
        colormapped = np.zeros(self._thumbnail_shape)
        colormapped[..., 3] = 1

        if self._view_data is not None and self.track_colors is not None:
            de = self._extent_data
            min_vals = [de[0, i] for i in self._dims.displayed]
            shape = np.ceil(
                [de[1, i] - de[0, i] + 1 for i in self._dims.displayed]
            ).astype(int)
            zoom_factor = np.divide(
                self._thumbnail_shape[:2], shape[-2:]
            ).min()
            if len(self._view_data) > self._max_tracks_thumbnail:
                thumbnail_indices = np.random.randint(
                    0, len(self._view_data), self._max_tracks_thumbnail
                )
                points = self._view_data[thumbnail_indices]
            else:
                points = self._view_data
                thumbnail_indices = range(len(self._view_data))

            # get the track coords here
            coords = np.floor(
                (points[:, :2] - min_vals[1:] + 0.5) * zoom_factor
            ).astype(int)
            coords = np.clip(
                coords, 0, np.subtract(self._thumbnail_shape[:2], 1)
            )

            # modulate track colors as per colormap/current_time
            colors = self.track_colors[thumbnail_indices]
            times = self.track_times[thumbnail_indices]
            alpha = (self.current_time - times) / self.tail_length
            alpha[times > self.current_time] = 1.0
            colors[:, -1] = np.clip(1.0 - alpha, 0.0, 1.0)
            colormapped[coords[:, 1], coords[:, 0]] = colors

        colormapped[..., 3] *= self.opacity
        self.thumbnail = colormapped

    @property
    def _view_data(self):
        """ return a view of the data """
        return self._pad_display_data(self._manager.track_vertices)

    @property
    def _view_graph(self):
        """ return a view of the graph """
        return self._pad_display_data(self._manager.graph_vertices)

    def _pad_display_data(self, vertices):
        """ pad display data when moving between 2d and 3d """
        if vertices is None:
            return

        data = vertices[:, self._dims.displayed]
        # if we're only displaying two dimensions, then pad the display dim
        # with zeros
        if self._dims.ndisplay == 2:
            data = np.pad(data, ((0, 0), (0, 1)), 'constant')
            return data[:, (1, 0, 2)]  # y, x, z -> x, y, z
        else:
            return data[:, (2, 1, 0)]  # z, y, x -> x, y, z

    @property
    def current_time(self):
        """ current time according to the first dimension """
        # TODO(arl): get the correct index here
        time_step = self._slice_indices[0]

        if isinstance(time_step, slice):
            # if we are visualizing all time, then just set to the maximum
            # timestamp of the dataset
            return self._manager.max_time

        return time_step

    @property
    def use_fade(self) -> bool:
        """toggle whether we fade the tail of the track, depending on whether
        the time dimension is displayed"""
        return 0 in self._dims.not_displayed

    @property
    def data(self) -> np.ndarray:
        """ array (N, D+1): Coordinates for N points in D+1 dimensions. """
        return self._manager.data

    @data.setter
    def data(self, data: np.ndarray):
        """ set the data and build the vispy arrays for display """
        # set the data and build the tracks
        self._manager.data = data
        self._manager.build_tracks()

        # reset the properties and recolor the tracks
        self.properties = {}
        self._recolor_tracks()

        # reset the graph
        self._manager.graph = {}
        self._manager.build_graph()

        # fire events to update shaders
        self.events.rebuild_tracks()
        self.events.rebuild_graph()
        self.events.data()
        self._update_dims()

    @property
    def properties(self) -> Dict[str, np.ndarray]:
        """dict {str: np.ndarray (N,)}, DataFrame: Properties for each track."""
        return self._manager.properties

    @property
    def properties_to_color_by(self) -> List[str]:
        """ track properties that can be used for coloring etc... """
        return list(self.properties.keys())

    @properties.setter
    def properties(self, properties: Dict[str, np.ndarray]):
        """ set track properties """
        if self._color_by not in [*properties.keys(), 'track_id']:
            warn(
                (
                    f"Previous color_by key {self._color_by} not present in"
                    " new properties. Falling back to track_id"
                ),
                UserWarning,
            )
            self._color_by = 'track_id'
        self._manager.properties = properties
        self.events.properties()
        self.events.color_by()

    @property
    def graph(self) -> Dict[int, Union[int, List[int]]]:
        """dict {int: list}: Graph representing associations between tracks."""
        return self._manager.graph

    @graph.setter
    def graph(self, graph: Dict[int, Union[int, List[int]]]):
        """ Set the track graph. """
        self._manager.graph = graph
        self._manager.build_graph()
        self.events.rebuild_graph()

    @property
    def tail_width(self) -> Union[int, float]:
        """float: Width for all vectors in pixels."""
        return self._tail_width

    @tail_width.setter
    def tail_width(self, tail_width: Union[int, float]):
        self._tail_width = tail_width
        self.events.tail_width()
        self.status = format_float(self.tail_width)

    @property
    def tail_length(self) -> Union[int, float]:
        """float: Width for all vectors in pixels."""
        return self._tail_length

    @tail_length.setter
    def tail_length(self, tail_length: Union[int, float]):
        self._tail_length = tail_length
        self.events.tail_length()
        self.status = format_float(self.tail_length)

    @property
    def display_id(self) -> bool:
        """ display the track id """
        return self._display_id

    @display_id.setter
    def display_id(self, value: bool):
        self._display_id = value
        self.events.display_id()
        self.refresh()

    @property
    def display_tail(self) -> bool:
        """ display the track tail """
        return self._display_tail

    @display_tail.setter
    def display_tail(self, value: bool):
        self._display_tail = value
        self.events.display_tail()

    @property
    def display_graph(self) -> bool:
        """ display the graph edges """
        return self._display_graph

    @display_graph.setter
    def display_graph(self, value: bool):
        self._display_graph = value
        self.events.display_graph()

    @property
    def color_by(self) -> str:
        return self._color_by

    @color_by.setter
    def color_by(self, color_by: str):
        """ set the property to color vertices by """
        if color_by not in self.properties_to_color_by:
            raise ValueError(f'{color_by} is not a valid property key')
        self._color_by = color_by
        self._recolor_tracks()
        self.events.color_by()

    @property
    def colormap(self) -> str:
        return self._colormap

    @colormap.setter
    def colormap(self, colormap: str):
        """ set the default colormap """
        if colormap not in AVAILABLE_COLORMAPS:
            raise ValueError(f'Colormap {colormap} not available')
        self._colormap = colormap
        self._recolor_tracks()
        self.events.colormap()

    @property
    def colormaps_dict(self) -> Dict[str, Colormap]:
        return self._colormaps_dict

    @colormaps_dict.setter
    def colomaps_dict(self, colormaps_dict: Dict[str, Colormap]):
        # validate the dictionary entries?
        self._colormaps_dict = colormaps_dict

    def _recolor_tracks(self):
        """ recolor the tracks """

        # this catch prevents a problem coloring the tracks if the data is
        # updated before the properties are. properties should always contain
        # a track_id key
        if self.color_by not in self.properties_to_color_by:
            self.color_by = 'track_id'

        # if we change the coloring, rebuild the vertex colors array
        vertex_properties = self._manager.vertex_properties(self.color_by)

        def _norm(p):
            return (p - np.min(p)) / np.max([1e-10, np.ptp(p)])

        if self.color_by in self.colormaps_dict:
            colormap = self.colormaps_dict[self.color_by]
        else:
            # if we don't have a colormap, get one and scale the properties
            colormap = AVAILABLE_COLORMAPS[self.colormap]
            vertex_properties = _norm(vertex_properties)

        # actually set the vertex colors
        self._track_colors = colormap.map(vertex_properties)

    @property
    def track_connex(self) -> np.ndarray:
        """ vertex connections for drawing track lines """
        return self._manager.track_connex

    @property
    def track_colors(self) -> np.ndarray:
        """return the vertex colors according to the currently selected
        property"""
        return self._track_colors

    @property
    def graph_connex(self) -> np.ndarray:
        """ vertex connections for drawing the graph """
        return self._manager.graph_connex

    @property
    def track_times(self) -> np.ndarray:
        """ time points associated with each track vertex """
        return self._manager.track_times

    @property
    def graph_times(self) -> np.ndarray:
        """ time points assocaite with each graph vertex """
        return self._manager.graph_times

    @property
    def track_labels(self) -> tuple:
        """ return track labels at the current time """

        # check that current time is still within the frame map
        if self.current_time < 0 or self.current_time > self._manager.max_time:
            # need to return a tuple for pos to clear the vispy text visual
            return None, (None, None)

        labels, positions = self._manager.track_labels(self.current_time)
        padded_positions = self._pad_display_data(positions)
        return labels, padded_positions

max_stars_repo_name: ddawsari/napari

max_issues_repo_name: ddawsari/napari

max_forks_repo_name: ddawsari/napari


text:
# ======================================================================================================================

# Functions to compute fitted predictions

# ======================================================================================================================

import numpy as np
import sys
from boostIV.primitives import sigmoid_wl, cubic_spline, blockwise_view

###################################################

### Get fit ###

###################################################

# get estimated basis functions
def get_bf_hat(Xmat, param_mat, wl_model, M, n_split):
    if wl_model == sigmoid_wl:
        n_param = 1 + Xmat.shape[1]
    elif wl_model == cubic_spline:
        n_param = ((3 + 2) * (Xmat.shape[1] - 1) + 1) + 1
    else:
        sys.exit('No weak learner type matched: pick sigmoid_wl or cubic_spline.')
    n_obs = Xmat.shape[0]
    bfs_hat_by_fold = np.empty((n_obs, M, n_split))
    # split the parameter matrix by fold
    # split the parameter matrix by fold: to maintain C-contiguity for blockwise_view function
    param_to_split = np.zeros((n_param * n_split, M))
    for m in range(M):
        param_to_split[:, m] = param_mat[:, m + 1]
    param_mat_by_fold = blockwise_view(param_to_split, (n_param, M))
    for l in range(n_split):
        for k in range(M):
            bfs_hat_by_fold[:, k, l] = wl_model(Xmat, param_mat_by_fold[l][0][1:n_param, k])
    bfs_hat = np.mean(bfs_hat_by_fold, axis=2)
    return bfs_hat

def get_boostIV_fit(Xmat, param_mat, wl_model, M, n_split):
    # number of parameters
    if wl_model == sigmoid_wl:
        n_param = 1 + Xmat.shape[1]
    elif wl_model == cubic_spline:
        n_param = ((3 + 2) * (Xmat.shape[1] - 1) + 1) + 1
    else:
        sys.exit('No weak learner type matched: pick sigmoid_wl or cubic_spline.')
    n_obs = Xmat.shape[0]
    avg_model = np.empty((n_obs, n_split))
    # split the parameter matrix by fold: to maintain C-contiguity for blockwise_view function
    param_to_split = np.zeros((n_param * n_split, M))
    for m in range(M):
        param_to_split[:, m] = param_mat[:, m + 1]
    param_mat_by_fold = blockwise_view(param_to_split, (n_param, M))  # first column is the intercept
    for l in range(n_split):
        bfs_fit = np.zeros((n_obs, M))
        for k in range(M):
            bfs_fit[:, k] = wl_model(Xmat, param_mat_by_fold[l][0][1:n_param, k])
        avg_model[:, l] = bfs_fit @ param_mat_by_fold[l][0][0, :]
    boost_fit = np.mean(avg_model, axis=1) + param_mat[:, 0].mean()
    return boost_fit

def get_post_boostIV_fit(param, wl_model, M, X, n_split, cf=True):
    """
    :param param: output of post_boostIV_crossfit
    :param wl_model: type of weak learner as a function: sigmoid_wl or cubic_spline
    :param M: number of boosting iterations
    :param X: (out of sample) data to evaluate the structural function
    :param cf: if True, add a cross-fitting step for the boosting step (I and II stages)
    :return: h_fun estimate
    """
    # dimension check
    if len(X.shape) == 1:
        X = X.reshape(1, -1)
    # extract parameters
    bf_param = param['bf_param']
    model_fit = param['bf_weights']
    K = len(bf_param)
    h_post_boost = []
    for k in range(K):
        if cf:
            bfs_hat_k = get_bf_hat(X, bf_param[k], wl_model, M, n_split)
            h_post_boost_k = model_fit[k].predict(bfs_hat_k)
        else:
            h_post_boost_k = model_fit[k].predict(np.vstack([wl_model(X, bf_param[k][1:, j]) for j in range(M)]).T)
        h_post_boost.append(h_post_boost_k)
    h_post_boost = np.vstack(h_post_boost).mean(axis=0)
    return h_post_boost

max_stars_repo_name: edbakhitov/boostIV

max_issues_repo_name: edbakhitov/boostIV

max_forks_repo_name: edbakhitov/boostIV


text:
import pandas as pd
import numpy as np
import argparse
import pdb
from tensorflow.python import keras
from tensorflow.python.keras import optimizers

from stock.data import Fields as fld, get_ticker_df, index_to_map
from common.lgb import LGBModel
from common.nn import NNModel

def get_delta_columns(prefix):
    return [f'{prefix}_d_{2**i * 5}' for i in range(7)]

def add_ticker_features(df, prefix):
    df_copy = df.copy()

    # add 5, 10, 20, 40, 80, 160, 320 day delta
    for i in range(7):
        days_interval = 2**i * 5
        df_copy.index += days_interval
        df[f'{prefix}_d_{days_interval}'] = df[fld.COL_ADJ_CLOSE] / df_copy[fld.COL_ADJ_CLOSE] - 1
        df_copy.index -= days_interval

    df.dropna(inplace=True)

def add_cash_features(df):
    # add 5, 10, 20, 40, 80, 160, 320 day delta
    for i in range(7):
        days_interval = 2**i * 5
        df[f'cash_d_{days_interval}'] = 0.0

def add_rank_features(df, index):
    rank_list = []
    index_map = index_to_map(index)

    def get_rank(row):
        result = {}
        for i in range(7):
            result[f'{2**i * 5}'] = []
        ordered_index = row.sort_values().index
        for i in ordered_index:
            split_i = i.split('_')
            result[split_i[2]].append(index_map[split_i[0]])

        flat = []
        for i in range(7):
            flat += result[f'{2**i * 5}']
        rank_list.append(flat)

    df.drop(fld.COL_TIME, axis=1).apply(get_rank, axis=1)

    ticker_count = int(len(rank_list[0]) / 7)
    ar = np.array(rank_list)
    assert(len(df) == len(ar))
    new_col_list = [f'r_{2**i * 5}_{j}' for i in range(7) for j in range(ticker_count)]

    df[new_col_list] = pd.DataFrame(ar, columns=new_col_list)
    return

def add_put_call_features(df, path):
    pc_df = pd.read_csv(f'{path}/equitypc.csv')
    pc_df.timestamp = pd.to_datetime(pc_df.timestamp)
    df.timestamp = pd.to_datetime(df.timestamp)
    df = df.merge(pc_df, how='left', on='timestamp')

    # sometimes put-call feature is delayed. We just use the last 80 day's mean if that's the case.
    if not (df.iloc[-1].pc_ratio > 0):
        df.iloc[-1, df.columns.get_loc('pc_ratio')] = df.iloc[-81:-1].pc_ratio.mean()
    for i in range(5):
        days = 2**i * 5
        df[f'pc_{days}'] = df.pc_ratio.rolling(days).mean()
    return df

def add_target(df, days, index):
    df_future = df.copy()
    # by attaching the result onto a past row, we look into the future
    df_future.index -= days
    df['target'] = df_future[f'r_{days}_{len(index)}']

def get_all_delta_data(path, index):
    ticker_dfs = {}
    for ticker in index.ticker:
        df = get_ticker_df(path, ticker)
        add_ticker_features(df, ticker)
        ticker_dfs[ticker] = df

    # merge just the delta columns
    merged = None
    for ticker, v in ticker_dfs.items():
        if merged is None:
            merged = v[[fld.COL_TIME] + get_delta_columns(ticker)]
        else:
            merged = pd.merge(merged, v[[fld.COL_TIME] + get_delta_columns(ticker)], on=fld.COL_TIME)

    add_cash_features(merged)

    return merged

def get_lgb_features(df):
    exclude_cols = ['timestamp', 'target', 'cash_d_5', 'cash_d_10', 'cash_d_20', 'cash_d_40', 'cash_d_80',
                    'cash_d_160', 'cash_d_320']
    for col in df.columns:
        if not (col.startswith('r_') or col.startswith('pc_')) or col.startswith('r_5') or col.startswith('r_10'):
            exclude_cols.append(col)
    return [f for f in df.columns if f not in exclude_cols]

def get_nn_features(df):
    exclude_cols = {'timestamp', 'target', 'cash_d_5', 'cash_d_10', 'cash_d_20', 'cash_d_40', 'cash_d_80',
                    'cash_d_160', 'cash_d_320'}
    for col in df.columns:
        if col.startswith('r_') or col.endswith('_d_5') or col.endswith('_d_10'):
            exclude_cols.add(col)
    return [f for f in df.columns if f not in exclude_cols]

def train_lgb(path, index, df, name):
    params = {
        'boosting': 'gbdt',
        'objective': 'multiclass',
        'num_class': len(index) + 1,  # including cash as a target
        'metric': 'multi_logloss',
        'learning_rate': 0.005,
        'verbose': 1,

        'num_rounds': 30000,
        #'is_unbalance': True,
        #'scale_pos_weight': 8.951238929246692,
        'early_stopping': 1000,

        # 'bagging_freq': 5,
        # 'bagging_fraction': 0.33,
        'boost_from_average': 'false',
        'feature_fraction': 1.0,
        'max_depth': -1,
        'min_data_in_leaf': 20,
        # 'min_sum_hessian_in_leaf': 5.0,
        'num_leaves': 3,
        'num_threads': 8,
        'tree_learner': 'serial',
    }

    lgb_model = LGBModel(f'lgb_{name}', path, 'timestamp', 'target', num_folds=5,
                         feat_cols=get_lgb_features(df))
    lgb_model.train(df, params, stratified=False, random_shuffle=True)

def train_nn(path, index, df, name):
    feat_cols = get_nn_features(df)
    model = keras.models.Sequential([
        keras.layers.Dense(1024, activation='relu', input_shape=(len(feat_cols), ),
                           kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.Dropout(0.5),
        keras.layers.Dense(64, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.Dropout(0.5),
        # keras.layers.Dense(64, activation='relu'),
        # keras.layers.Dropout(0.2),
        keras.layers.Dense(len(index) + 1, activation='softmax')
    ])
    model.compile(optimizer=optimizers.Adam(lr=0.0005),
                  loss='categorical_crossentropy',
                  metrics=[keras.metrics.categorical_accuracy])
    nn_model = NNModel(f'nn_{name}', path, 'timestamp', 'target', model, num_folds=1,
                       feat_cols=feat_cols, classification=True, monitor='categorical_accuracy')
    nn_model.train(df, None)

def main():
    parser = argparse.ArgumentParser(description='testing performance')
    parser.add_argument("-a", "--algo", help="The algorithm we want to test ")
    parser.add_argument("-b", "--by", help="The market segment breakdown method, sector or size")

    args = parser.parse_args()
    if args.algo is None:
        print('Must have algo name')
        return
    if args.by is None:
        print('Must specify -b sector or size')
        return

    path = 'data'

    index = pd.read_csv(f'{path}/index_by_{args.by}.csv')
    df = get_all_delta_data(path, index)
    add_rank_features(df, index)
    add_target(df, 160, index)
    df = add_put_call_features(df, path)

    train_end = int(len(df) * 3 / 5)
    df = df.iloc[0:train_end]
    if args.algo == 'lgb':
        train_lgb(path, index, df, args.by)
    elif args.algo == 'nn':
        train_nn(path, index, df, args.by)
    else:
        print('unknown algorithm')
    print('done')

if __name__ == '__main__':
    main()

max_stars_repo_name: looselyconnected/fastai

max_issues_repo_name: looselyconnected/fastai

max_forks_repo_name: looselyconnected/fastai


text:
import numpy as np
from matplotlib import pyplot as pl
from FacetFlowNetwork import demffn

pw = 0.01
xr = np.arange(-1.5, 1.5, pw)
yr = np.arange(-1.4, 1.4, pw)

x, y = np.meshgrid(xr, yr)
r = np.sqrt(x*x + y*y)

dem = np.exp(-r*r)
tsca = r / 2.0

f = demffn(dem, pw)
sca = f.fatp(f.sca())
sca.shape = dem.shape

pl.figure(1, (8, 6))
im = pl.imshow(sca, origin = 'lower',
        interpolation = 'none', extent = [0,1.5,0,1.4],
        cmap = pl.cm.viridis_r,
        vmin = 0, vmax = tsca.max())
cb = pl.colorbar()
cb.set_label('SCA [m]')
pl.gca().set_aspect('equal')
pl.savefig('Gauss_demffn_sca.png')

pl.close('all')

pl.figure(1, (8, 6))
im = pl.imshow(100*(sca-tsca)/tsca, origin = 'lower',
        interpolation = 'none', extent = [0,1.5,0,1.4],
        cmap = pl.cm.bwr,
        vmin = -25, vmax = 25)
cb = pl.colorbar()
cb.set_label(r'Relative SCA error [%]')
pl.gca().set_aspect('equal')
pl.savefig('Gauss_demffn_sca_error.png')

max_stars_repo_name: Rheinwalt/FacetFlowNetwork

max_issues_repo_name: Rheinwalt/FacetFlowNetwork

max_forks_repo_name: Rheinwalt/FacetFlowNetwork


text:
#----------------------------------------------------------------------
# Deep learning for classification for contrast CT;
# Transfer learning using Google Inception V3;
#-------------------------------------------------------------------------------------------

import os
import numpy as np
import pandas as pd
import seaborn as sn
import matplotlib.pyplot as plt
import glob
import tensorflow
from tensorflow import keras
from tensorflow.keras.models import Model
from tensorflow.keras.preprocessing.image import img_to_array, load_img, ImageDataGenerator
from tensorflow.keras.layers import GlobalAveragePooling2D
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.applications import ResNet152
from tensorflow.keras.applications import ResNet101
from tensorflow.keras.applications import ResNet50V2
from tensorflow.keras.applications import ResNet101V2
from tensorflow.keras.applications import ResNet152V2

# ----------------------------------------------------------------------------------
# transfer learning CNN model
# ----------------------------------------------------------------------------------
def GoogleNet(resnet, transfer_learning, input_shape, batch_momentum, activation,
              activation_out, loss_function, optimizer, dropout_rate):

    ### determine if use transfer learnong or not
    if transfer_learning == True:
        weights = 'imagenet'
    elif transfer_learning == False:
	    weights = None

	### dermine input shape
    default_shape = (224, 224, 3)
    if input_shape == default_shape:
	    include_top = True
    else:
	    include_top = False

    ### determine ResNet base model
    if resnet == 'ResNet50V2':
        base_model = ResNet50V2(
            weights=weights,
            include_top=include_top,
            input_shape=input_shape,
            pooling=None
            )
    elif resnet == 'ResNet101V2':
        base_model = ResNet101V2(
            weights=weights,
            include_top=include_top,
            input_shape=input_shape,
            pooling=None
            )
    elif resnet == 'ResNet152V2':
        base_model = ResNet152V2(
            weights=weights,
            include_top=include_top,
            input_shape=input_shape,
            pooling=None
            )
#    base_model.trainable = True
#
#    inputs = keras.Input(shape=input_shape)
#    x = inputs
#    x = base_model(x, training=True)
#    x = BatchNormalization(momentum=batch_momentum)(x)
#    x = GlobalAveragePooling2D()(x)
#    x = Dropout(0.2)(x)
#    outputs = Dense(1)(x)
#    model = Model(inputs, outputs)
#    model.summary()
    ### create top model
    out = base_model.output
    out = BatchNormalization(momentum=batch_momentum)(out)
    out = GlobalAveragePooling2D()(out)
    out = Dropout(dropout_rate)(out)
    ### layer 3
#    out = BatchNormalization(momentum=batch_momentum)(out)
#    out = Dense(512, activation=activation)(out)
#    out = Dropout(dropout_rate)(out)
#    ### lyaer 2
#    out = BatchNormalization(momentum=batch_momentum)(out)
#    out = Dense(128, activation=activation)(out)
#    out = Dropout(dropout_rate)(out)
#    ### layer 1
#    out = BatchNormalization(momentum=batch_momentum)(out)
    predictions = Dense(1, activation=activation_out)(out)
    model = Model(inputs=base_model.input, outputs=predictions)

	### only if we want to freeze layers
#    for layer in base_model.layers:
#        layer.trainable = True

    print('complie model')
    model.compile(
                  loss=loss_function,
                  optimizer=optimizer,
                  metrics=['accuracy']
                  )
    model.summary()

    return model

max_stars_repo_name: xmuyzz/IVContrast

max_issues_repo_name: xmuyzz/IVContrast

max_forks_repo_name: xmuyzz/IVContrast


text:
import os
import re
from collections import defaultdict
from typing import TYPE_CHECKING, List

import numpy
from typing_extensions import Literal

from nonbonded.library.models.forcefield import Parameter
from nonbonded.library.models.projects import Optimization
from nonbonded.library.models.results import OptimizationResult, TargetResultType
from nonbonded.library.models.targets import OptimizationTarget
from nonbonded.library.plotting.seaborn.utilities import plot_categories
from nonbonded.library.plotting.utilities import (
    combine_target_rmse,
    sort_categories_key,
)
from nonbonded.library.utilities.string import camel_to_kebab_case

if TYPE_CHECKING:
    import simtk.unit
    from openff.toolkit.typing.engines.smirnoff.forcefield import (
        ForceField as OFFForceField,
    )

def _default_parameter_unit(parameter: Parameter) -> "simtk.unit":
    """Returns the default unit for a parameter.

    Parameters
    ----------
    parameter
        The parameter to retrieve the unit of.

    Returns
    -------
        The default parameter unit.
    """

    import simtk.unit

    default_units = {
        "epsilon": simtk.unit.kilocalories_per_mole,
        "rmin_half": simtk.unit.angstrom,
        "sigma": simtk.unit.angstrom,
        "charge_increment1": simtk.unit.elementary_charge,
    }

    return default_units[parameter.attribute_name]

def _extract_parameter_value(
    force_field: "OFFForceField", parameter: Parameter
) -> float:
    """Extracts the value of a parameter from a force field object
    in its default unit.

    Parameters
    ----------
    force_field
        The force field which contains the parameter.
    parameter
        The parameter to extract the value of

    Returns
    -------
        The value of the parameter.
    """

    handler = force_field.get_parameter_handler(parameter.handler_type)

    off_parameter = (
        handler if parameter.smirks is None else handler.parameters[parameter.smirks]
    )

    value = getattr(off_parameter, parameter.attribute_name)

    return value.value_in_unit(_default_parameter_unit(parameter))

def _parameter_attribute_to_title(parameter: Parameter) -> str:
    """Maps a parameter attribute into a human and ``matplotlib`` friendly plot title
    where possible.

    Parameters
    ----------
    parameter
        The parameter which contains the attribute to map.

    Returns
    -------
        A fitting title.
    """

    attribute_symbols = {
        "epsilon": r"\varepsilon",
        "sigma": r"\sigma",
        "rmin_half": r"\dfrac{r_{min}}{2}",
        "charge_increment": "q",
    }

    indexless_attribute = re.sub(r"\d", "", parameter.attribute_name)

    if indexless_attribute not in attribute_symbols:
        return parameter.attribute_name

    if parameter.attribute_name in attribute_symbols:
        return f"${attribute_symbols[parameter.attribute_name]}$"

    symbol = attribute_symbols[indexless_attribute]

    title = re.sub(r"(\d+)", r"_{\1}", parameter.attribute_name)
    title = title.replace(indexless_attribute, symbol)

    return f"${title}$"

def plot_parameter_changes(
    optimizations: List[Optimization],
    optimization_results: List[OptimizationResult],
    mode: Literal["percent", "absolute"],
    show_legend: bool,
    output_directory: str,
    file_type: Literal["png", "pdf"] = "png",
):
    """Plots the change in the force field parameters which have been trained over
    the course of an optimization.

    Parameters
    ----------
    optimizations
        The optimizations which have been performed.
    optimization_results
        The analyzed outputs of the optimizations.
    mode
        The type of change to plot. This may either be absolute changes
        or percent changes.
    show_legend
        Whether to show the legend.
    output_directory
        The directory in which to save the plots.
    file_type
        The file type to use for the plots.
    """
    import pandas
    import seaborn
    from matplotlib import pyplot

    # Determine which parameters were refit
    parameters = defaultdict(dict)

    for index, (optimization, optimization_result) in enumerate(
        zip(optimizations, optimization_results)
    ):

        original_force_field = optimization.force_field.to_openff()
        refit_force_field = optimization_result.refit_force_field.to_openff()

        for parameter in optimization.parameters_to_train:

            parameters[parameter][index] = (
                _extract_parameter_value(original_force_field, parameter),
                _extract_parameter_value(refit_force_field, parameter),
            )

    # Re-shape the parameter values into a plottable pandas frame.
    data_rows = []

    for parameter in parameters:
        for optimization_index in parameters[parameter]:

            original_value, refit_value = parameters[parameter][optimization_index]

            percentage_change = (
                0.0
                if numpy.isclose(original_value, 0.0)
                else (refit_value - original_value) / original_value * 100.0
            )
            absolute_change = refit_value - original_value

            unit = _default_parameter_unit(parameter).get_symbol()

            if unit == "A":
                unit = "Å"

            data_row = {
                "Name": optimizations[optimization_index].name,
                "Handler": parameter.handler_type,
                "SMIRKS": parameter.smirks,
                "Attribute": f"{_parameter_attribute_to_title(parameter)} ({unit})",
                r"$\Delta$": absolute_change,
                r"% $\Delta$": percentage_change,
            }

            data_rows.append(data_row)

    # Plot the data.
    parameter_data = pandas.DataFrame(data_rows)
    palette = seaborn.color_palette(n_colors=len(optimizations))

    header = r"$\Delta$" if mode == "absolute" else r"% $\Delta$"
    handlers = {*parameter_data["Handler"].unique()}

    for handler in handlers:

        handler_data = parameter_data[parameter_data["Handler"] == handler]

        n_attributes = len(handler_data["Attribute"].unique())
        n_smirks = len(handler_data["SMIRKS"].unique())

        plot = seaborn.FacetGrid(
            handler_data,
            row="Attribute",
            height=1.5 * n_attributes,
            aspect=1.0 + 0.1 * (n_smirks - 1),
            sharey=False,
        )
        plot.map_dataframe(plot_categories, "SMIRKS", header, "Name", color=palette)
        plot.set_titles("{row_name}")

        for index in range(n_attributes):

            # Move the x-axis to y = 0
            plot.axes[index, 0].spines["bottom"].set_position(("data", 0.0))
            # Remove the x-axis tick marks.
            plot.axes[index, 0].tick_params(axis="x", which="both", length=0)

        # Move the x-axis labels back under the plot, rather than on the x axis.
        pyplot.setp(
            plot.axes[n_attributes - 1, 0].get_xticklabels(),
            transform=plot.axes[n_attributes - 1, 0].get_xaxis_transform(),
            rotation=90,
        )

        if show_legend:
            plot.add_legend()

        handler_name = camel_to_kebab_case(handler.replace("vdW", "vdw"))

        plot.savefig(
            os.path.join(output_directory, f"{handler_name}-{mode}-change.{file_type}")
        )
        pyplot.close(plot.fig)

def plot_objective_per_iteration(
    optimizations: List[Optimization],
    optimization_results: List[OptimizationResult],
    show_legend: bool,
    output_directory: str,
    file_type: Literal["png", "pdf"] = "png",
):
    """Plots the objective function for each optimizations target as a function of the
    optimization iteration.

    Different optimization are coloured separately while different targets are styled
    with different markers.

    Parameters
    ----------
    optimizations
        The optimizations which have been performed.
    optimization_results
        The analyzed outputs of the optimizations.
    show_legend
        Whether to show the legend.
    output_directory
        The directory in which to save the plots.
    file_type
        The file type to use for the plots.
    """

    import pandas
    import seaborn
    from matplotlib import pyplot

    data_rows = []

    for optimization, optimization_result in zip(optimizations, optimization_results):
        for iteration, iteration_result in optimization_result.target_results.items():
            for target_id, target_result in iteration_result.items():

                data_rows.append(
                    {
                        "Optimization": optimization.name,
                        "Target": target_id,
                        "Iteration": iteration,
                        "Objective Function": target_result.objective_function,
                    }
                )

    plot_data = pandas.DataFrame(data_rows)

    hue = "Optimization"
    style = "Target" if len(plot_data["Target"].unique()) > 1 else None

    colors = seaborn.color_palette(
        "colorblind", n_colors=len(plot_data["Optimization"].unique())
    )

    plot = seaborn.relplot(
        data=plot_data,
        x="Iteration",
        y="Objective Function",
        hue=hue,
        style=style,
        height=4.0,
        legend="full" if show_legend else False,
        palette=colors,
    )
    for target in plot_data["Target"].unique():

        target_data = plot_data[plot_data["Target"] == target]

        seaborn.lineplot(
            data=target_data,
            x="Iteration",
            y="Objective Function",
            hue=hue,
            legend=False,
            estimator=None,
            ci=None,
            palette=colors,
        )

    plot.savefig(os.path.join(output_directory, f"objective-function.{file_type}"))
    pyplot.close(plot.fig)

def plot_target_rmse(
    targets: List[OptimizationTarget],
    target_results: List[TargetResultType],
    target_labels: List[str],
    file_prefix: str,
    output_directory: str,
    file_type: Literal["png", "pdf"] = "png",
):
    """Plots the RMSE for each of the specified targets and for each data type
    contained within the target results.

    Parameters
    ----------
    targets
        The targets which the results were collected for.
    target_results
        The target results to plot.
    target_labels
        The label associated with each target result.
    file_prefix
        The prefix to append to the file name of each produced plot.
    output_directory
        The directory in which to save the plots.
    file_type
        The file type to use for the plots.
    """
    import seaborn
    from matplotlib import pyplot

    plot_data = combine_target_rmse(targets, target_results, target_labels)

    if len(plot_data) == 0:
        return

    # Extract the unique data types (e.g. property types) which will be plotted
    # in separate figures.
    data_types = plot_data["Data Type"].unique()

    # Select a set of colors for each label
    color_palette = (
        ["skyblue", "limegreen"] + []
        if len(target_labels) < 3
        else seaborn.color_palette(n_colors=len(target_labels) - 2)
    )
    colors = {label: color_palette[index] for index, label in enumerate(target_labels)}

    for data_type in data_types:

        # Extract a data frame containing only the data type which should
        # be included in this figure.
        type_plot_data = plot_data[
            (
                plot_data["Data Type"] == data_type
                if data_type is not None
                else plot_data["Data Type"].isnull()
            )
        ]

        categories = sorted(
            type_plot_data["Category"].unique(), key=sort_categories_key, reverse=True
        )
        category_indices = [x * 2 for x in range(1, len(categories) + 1)]

        labels = type_plot_data["Label"].unique()

        with seaborn.axes_style("white"):

            figure, axis = pyplot.subplots(figsize=(5.0, 1.5 + (0.5 * len(categories))))

            # Plot data for each label.
            for label in labels:

                label_plot_data = type_plot_data[type_plot_data["Label"] == label]

                axis.errorbar(
                    label_plot_data["Value"],
                    label_plot_data["Category"]
                    .replace(categories, category_indices)
                    .values,
                    xerr=[
                        label_plot_data["Lower CI"],
                        label_plot_data["Upper CI"],
                    ],
                    color=colors[label],
                    capsize=5.0,
                    capthick=1.5,
                    linestyle="none",
                    marker="o",
                    label=label,
                )

            # Add a simple legend.
            axis.legend()

            # Add title and axis names
            axis.set_yticks(category_indices)
            axis.set_yticklabels(categories)

            axis.set_xlim(left=0.0)
            axis.set_ylim(0.5, (len(categories) + 1) * 2.0 - 0.5)

            axis.set_xlabel("RMSE")

        data_name = "-" if data_type is None else f"-{camel_to_kebab_case(data_type)}-"

        # Save the figure.
        figure.savefig(
            os.path.join(
                output_directory,
                f"{file_prefix}{data_name}rmse.{file_type}",
            ),
            bbox_inches="tight",
        )
        pyplot.close(figure)

def plot_rmse_change(
    optimization: Optimization,
    optimization_result: OptimizationResult,
    output_directory: str,
    file_type: Literal["png", "pdf"] = "png",
):
    """Plots the change in the RMSE between the first and the last iteration of an
    optimization for each optimization target.

    Parameters
    ----------
    optimization
        The optimization which the results were collected for.
    optimization_result
        The analyzed outputs of the optimization.
    output_directory
        The directory in which to save the plots.
    file_type
        The file type to use for the plots.
    """
    if 0 not in optimization_result.target_results:

        raise KeyError(
            "The optimization results must contain the statistics for iteration "
            "0 to plot the change in RMSE values."
        )

    if len(optimization_result.target_results) <= 1:

        raise KeyError(
            "The optimization results must contain the statistics for at least "
            "two iterations to plot the change in RMSE values."
        )

    targets_by_id = {target.id: target for target in optimization.targets}

    final_iteration = sorted(optimization_result.target_results)[-1]

    for target_id, initial_result in optimization_result.target_results[0].items():

        final_result = optimization_result.target_results[final_iteration][target_id]

        plot_target_rmse(
            [targets_by_id[target_id], targets_by_id[target_id]],
            [initial_result, final_result],
            ["Initial", "Final"],
            target_id,
            output_directory,
            file_type,
        )

max_stars_repo_name: SimonBoothroyd/nonbonded

max_issues_repo_name: SimonBoothroyd/nonbonded

max_forks_repo_name: SimonBoothroyd/nonbonded


text:
# -*- coding: utf-8 -*-
"""MRI-tumor-segmentation-Brats.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1BTXwPQ3kXM3cODnJeay1LuqCoiSiHZnH
"""
from tqdm import tqdm

# !pip install medpy

# Dataset BraTS 2019 training em https://drive.google.com/drive/folders/1fdHMkpGIuMNJTZioDGskxmueIkFF_Ssn?usp=sharing
# Para funcionar sem modificar o path abaixo, copie esta pasta no seu Google Drive em <raiz>/datasets/
# Ou modifique o path abaixo para o seu diretório, onde /content/drive/My Drive/ é a raiz do seu Google Drive

# from google.colab import drive
# drive.mount('/content/drive')

# Commented out IPython magic to ensure Python compatibility.
# %tensorflow_version 1.x
# import argparse
import tensorflow as tf

# from tensorflow.contrib.keras.python.keras.layers import *
from keras.layers import *
from tensorboardX import SummaryWriter

def Conv3DWithBN(
    x,
    filters,
    ksize,
    strides,
    name,
    padding="same",
    dilation_rate=1,
    center=True,
    scale=True,
    decay=0.99,
):
    x = Conv3D(
        filters=filters,
        kernel_size=ksize,
        strides=strides,
        padding=padding,
        dilation_rate=dilation_rate,
        use_bias=False,
        kernel_initializer="he_normal",
        name=name + "_conv",
    )(x)
    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn"
    )(x)
    x = Activation("relu", name=name + "_relu")(x)
    return x

def Conv2DWithBN(
    x,
    filters,
    ksize,
    strides,
    name,
    padding="same",
    dilation_rate=1,
    center=True,
    scale=True,
    decay=0.99,
):
    x = Conv2D(
        filters=filters,
        kernel_size=ksize,
        strides=strides,
        padding=padding,
        dilation_rate=dilation_rate,
        use_bias=False,
        kernel_initializer="he_normal",
        name=name + "_conv",
    )(x)
    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn"
    )(x)
    x = Activation("relu", name=name + "_relu")(x)
    return x

def Conv1DWithBN(
    x,
    filters,
    ksize,
    strides,
    name,
    padding="same",
    dilation_rate=1,
    center=True,
    scale=True,
    decay=0.99,
):
    x = Conv1D(
        filters=filters,
        kernel_size=ksize,
        strides=strides,
        padding=padding,
        dilation_rate=dilation_rate,
        use_bias=False,
        kernel_initializer="he_normal",
        name=name + "_conv",
    )(x)
    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn"
    )(x)
    x = Activation("relu", name=name + "_relu")(x)
    return x

def DenseWithBN(
    x, units, name, kernel_regularizer=None, center=True, scale=True, decay=0.99
):
    x = Dense(
        units=units,
        use_bias=False,
        kernel_regularizer=kernel_regularizer,
        name=name + "_weight",
    )(x)
    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bias"
    )(x)
    x = Activation("relu", name=name + "_relu")(x)
    return x

def ResNetUnit2D(
    x, filters, ksize, name, end=False, center=True, scale=True, decay=0.99
):
    identity = x

    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn_1"
    )(x)
    x = Activation("relu", name=name + "_relu_1")(x)
    x = Conv2D(
        filters,
        kernel_size=ksize,
        strides=1,
        padding="same",
        kernel_initializer="he_normal",
        name=name + "_conv_1",
    )(x)

    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn_2"
    )(x)
    x = Activation("relu", name=name + "_relu_2")(x)
    x = Conv2D(
        filters,
        kernel_size=ksize,
        strides=1,
        padding="same",
        kernel_initializer="he_normal",
        name=name + "_conv2",
    )(x)

    x = add([x, identity])
    if end:
        x = BatchNormalization(center=center, scale=scale, momentum=decay)(x)
        x = Activation("relu")(x)
    return x

def ResNetUnitIncreasingDims2D(
    x, filters, ksize, strides, name, begin=False, center=True, scale=True, decay=0.99
):
    identity = x

    if not begin:
        x = BatchNormalization(
            center=center, scale=scale, momentum=decay, name=name + "_bn_1"
        )(x)
        x = Activation("relu", name=name + "_relu_1")(x)
    x = Conv2D(
        filters,
        kernel_size=ksize,
        strides=strides[0],
        padding="same",
        kernel_initializer="he_normal",
        use_bias=False,
        name=name + "_conv_1",
    )(x)

    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn_2"
    )(x)
    x = Activation("relu", name=name + "_relu_2")(x)
    x = Conv2D(
        filters,
        kernel_size=ksize,
        strides=strides[1],
        padding="same",
        kernel_initializer="he_normal",
        use_bias=False,
        name=name + "_conv_2",
    )(x)

    identity = Conv2D(
        filters,
        kernel_size=1,
        strides=strides[0],
        padding="same",
        kernel_initializer="he_normal",
        name=name + "_conv_identity",
    )(identity)
    x = add([x, identity])
    return x

def ResNetUnit1D(
    x, filters, ksize, name, end=False, center=True, scale=True, decay=0.99
):
    identity = x

    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn_1"
    )(x)
    x = Activation("relu", name=name + "_relu_1")(x)
    x = Conv1D(
        filters,
        kernel_size=ksize,
        strides=1,
        padding="same",
        kernel_initializer="he_normal",
        use_bias=False,
        name=name + "_conv_1",
    )(x)

    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn_2"
    )(x)
    x = Activation("relu", name=name + "_relu_2")(x)
    x = Conv1D(
        filters,
        kernel_size=ksize,
        strides=1,
        padding="same",
        kernel_initializer="he_normal",
        use_bias=False,
        name=name + "_conv_2",
    )(x)

    x = add([x, identity])
    if end:
        x = BatchNormalization(
            center=center, scale=scale, momentum=decay, name=name + "_bn_3"
        )(x)
        x = Activation("relu", name=name + "_relu_3")(x)
    return x

def ResNetUnitIncreasingDims1D(
    x, filters, ksize, strides, name, begin=False, center=True, scale=True, decay=0.99
):
    """
    ResNet unit without BottleNeck. 2 layers
    :param x:
    :param filters:
    :param ksize:
    :param strides: list with 2 elements, stride for each layer
    :param begin:
    :return:
    """

    identity = x

    if not begin:
        x = BatchNormalization(
            center=center, scale=scale, momentum=decay, name=name + "_bn_1"
        )(x)
        x = Activation("relu", name=name + "_relu_1")(x)
    x = Conv1D(
        filters,
        kernel_size=ksize,
        strides=strides[0],
        padding="same",
        kernel_initializer="he_normal",
        use_bias=False,
        name=name + "_conv_1",
    )(x)

    x = BatchNormalization(
        center=center, scale=scale, momentum=decay, name=name + "_bn_2"
    )(x)
    x = Activation("relu", name=name + "_relu_2")(x)
    x = Conv1D(
        filters,
        kernel_size=ksize,
        strides=strides[1],
        padding="same",
        kernel_initializer="he_normal",
        use_bias=False,
        name=name + "_conv_2",
    )(x)

    identity = Conv1D(
        filters,
        kernel_size=1,
        strides=strides[0],
        padding="same",
        kernel_initializer="he_normal",
        name=name + "_conv_identity",
    )(identity)
    x = add([x, identity])
    return x

def ContextualAvgPooling(x, ksizes, strides):
    """
    Concatenate input with pooling result
    :param x:
    :param ksizes:
    :param strides:
    :return:
    """
    out = None
    for ks in ksizes:
        x_pooled = AvgPool1D(pool_size=ks, strides=strides, padding="same")
        if out is None:
            out = x_pooled
        else:
            out = concatenate([out, x_pooled], axis=-1)
    x = concatenate([x, out], axis=-1)
    return x

def ContextualAtrousConv1D(x, filters, ksize, strides, dilation_rates, name):
    """
    Retrieve contextual information with Atrous Convolution
    :param x:
    :param filters:
    :param ksize:
    :param strides:
    :param dilation_rates:
    :return:
    """
    concat = x
    for dr in dilation_rates:
        x_atrous = Conv1DWithBN(
            x,
            filters=filters,
            ksize=ksize,
            strides=strides,
            dilation_rate=dr,
            name=name + "a_conv_s" + str(dr),
        )
        concat = concatenate([concat, x_atrous], axis=-1)
    concat = Conv1DWithBN(
        concat,
        filters=256,
        ksize=ksize,
        strides=strides,
        name=name + "atrou_post_conv1",
    )
    return concat

def ContextualAtrousConv3D(x, filters, ksize, strides, dilation_rates, name):
    """
    Retrieve contextual information with Atrous Convolution
    :param x:
    :param filters:
    :param ksize:
    :param strides:
    :param dilation_rates:
    :return:
    """
    concat = None
    for dr in dilation_rates:
        x_atrous = Conv3DWithBN(
            x,
            filters=filters,
            ksize=ksize,
            strides=strides,
            dilation_rate=dr,
            name=name + "a_conv_s" + str(dr),
        )
        if concat is None:
            concat = x_atrous
        else:
            concat = concatenate([concat, x_atrous], axis=-1)
    concat = Conv3DWithBN(
        concat,
        filters=filters,
        ksize=ksize,
        strides=strides,
        name=name + "atrou_post_conv1",
    )
    return concat

def SharedAtrousConv1D(x, SharedConvs, PostConv):
    concat = x
    for SharedConv in SharedConvs:
        x_atrous = SharedConv(x)
        x_atrous = BatchNormalization()(x_atrous)
        x_atrous = Activation("relu")(x_atrous)
        concat = concatenate([concat, x_atrous], axis=-1)
    concat = PostConv(concat)
    concat = BatchNormalization()(concat)
    concat = Activation("relu")(concat)
    return concat

def densenet_block3d(x, k, rep):
    dense_input = x
    for i in range(rep):
        x_dense = Conv3D(
            filters=k, kernel_size=3, strides=1, padding="same", activation="relu"
        )(dense_input)
        dense_input = concatenate([dense_input, x_dense])
    return dense_input

def DenseNetTransit(x, rate=1, name=None):
    if rate != 1:
        out_features = x.get_shape().as_list()[-1] * rate
        x = BatchNormalization(center=True, scale=True, name=name + "_bn")(x)
        x = Activation("relu", name=name + "_relu")(x)
        x = Conv3D(
            filters=out_features,
            kernel_size=1,
            strides=1,
            padding="same",
            kernel_initializer="he_normal",
            use_bias=False,
            name=name + "_conv",
        )(x)
    x = AveragePooling3D(pool_size=2, strides=2, padding="same")(x)
    return x

def DenseNetUnit3D(x, growth_rate, ksize, rep, bn_decay=0.99, name=None):
    for i in range(rep):
        concat = x
        x = BatchNormalization(
            center=True, scale=True, momentum=bn_decay, name=name + "_bn_rep_" + str(i)
        )(x)
        x = Activation("relu")(x)
        x = Conv3D(
            filters=growth_rate,
            kernel_size=ksize,
            padding="same",
            kernel_initializer="glorot_normal",
            use_bias=False,
            name=name + "_conv_rep_" + str(i),
        )(x)
        x = concatenate([concat, x])
    return x

from tf_layers import (
    BilinearUpsampling1D,
    BilinearUpsampling3D,

)

from tf_models import (
    BraTS2ScaleDenseNetConcat,
    BraTS2ScaleDenseNetConcat_large,
    BraTS2ScaleDenseNet,
    BraTS3ScaleDenseNet,
    BraTS1ScaleDenseNet,
    PlainCounterpart,
)

from sklearn.model_selection import train_test_split

from pathlib import Path
from typing import List

def read_file(file_path: Path) -> str:
    with open(file_path) as text_file:
        return text_file.read()

def read_multiline(file_path: Path) -> List[str]:
    """Read, split by LF, and pop final empty line if present."""
    text = read_file(file_path)
    lines = text.split("\n")
    if lines[-1] == "":
        lines.pop()
    return lines

files = read_multiline("train.txt")
val_files = read_multiline("val.txt")

files = [file_name[:-1] for file_name in files]

val_files = [file_name[:-1] for file_name in val_files]

# print(files[2])
# print(len(files))

# create training and testing vars
files, val_files = train_test_split(files, test_size=0.8)
print("%d training   samples" % len(files))
print("%d validation samples" % len(val_files))

import glob

import numpy as np
from keras.backend import learning_phase
from keras.layers import concatenate, Conv3D

# import tf_models
from sklearn.preprocessing import scale
import tensorflow.compat.v1 as tf

tf.disable_v2_behavior()
# from tensorflow.contrib.keras.python.keras.backend import learning_phase
# from tensorflow.contrib.keras.python.keras.layers import concatenate, Conv3D
from nibabel import load as load_nii
import os
import argparse
import keras

def parse_inputs():

    parser = argparse.ArgumentParser(description="train the model")
    parser.add_argument(
        "-r",
        "--root-path",
        dest="root_path",
        default="./MICCAI_BraTS_2019_Data_Training/HGG/"
    )
    parser.add_argument(
        "-sp", "--save-path", dest="save_path", default="dense24_correction"
    )
    parser.add_argument(
        "-lp", "--load-path", dest="load_path", default="dense24_correction"
    )
    parser.add_argument("-ow", "--offset-width", dest="offset_w", type=int, default=12)
    parser.add_argument("-oh", "--offset-height", dest="offset_h", type=int, default=12)
    parser.add_argument(
        "-oc", "--offset-channel", dest="offset_c", nargs="+", type=int, default=12
    )
    parser.add_argument("-ws", "--width-size", dest="wsize", type=int, default=38)
    parser.add_argument("-hs", "--height-size", dest="hsize", type=int, default=38)
    parser.add_argument("-cs", "--channel-size", dest="csize", type=int, default=38)
    parser.add_argument("-ps", "--pred-size", dest="psize", type=int, default=12)
    parser.add_argument("-bs", "--batch-size", dest="batch_size", type=int, default=4)
    parser.add_argument("-e", "--num-epochs", dest="num_epochs", type=int, default=5)
    parser.add_argument(
        "-c", "--continue-training", dest="continue_training", type=bool, default=False
    )
    parser.add_argument(
        "-mn", "--model_name", dest="model_name", type=str, default="dense24"
    )  # dense48 no_dense
    parser.add_argument(
        "-nc", "--n4correction", dest="correction", type=bool, default=False
    )
    parser.add_argument("-gpu", "--gpu_id", dest="gpu_id", type=str, default="0")
    return vars(parser.parse_args())

options = parse_inputs()

os.environ["CUDA_VISIBLE_DEVICES"] = options['gpu_id']

def acc_tf(y_pred, y_true):
    correct_prediction = tf.equal(
        tf.cast(tf.argmax(y_pred, -1), tf.int32),
        tf.cast(tf.argmax(y_true, -1), tf.int32),
    )
    return 100 * tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

def get_patches_3d(data, labels, centers, hsize, wsize, csize, psize, preprocess=True):
    """
    :param data: 4D nparray (h, w, c, ?)
    :param centers:
    :param hsize:
    :param wsize:
    :param csize:
    :return:
    """
    patches_x, patches_y = [], []
    offset_p = (hsize - psize) / 2
    for i in range(len(centers[0])):
        h, w, c = centers[0, i], centers[1, i], centers[2, i]
        h_beg = int(min(max(0, h - hsize / 2), 240 - hsize))
        w_beg = int(min(max(0, w - wsize / 2), 240 - wsize))
        c_beg = int(min(max(0, c - csize / 2), 155 - csize))
        ph_beg = int(h_beg + offset_p)
        pw_beg = int(w_beg + offset_p)
        pc_beg = int(c_beg + offset_p)
        vox = data[
            h_beg : h_beg + hsize, w_beg : w_beg + wsize, c_beg : c_beg + csize, :
        ]
        vox_labels = labels[
            ph_beg : ph_beg + psize, pw_beg : pw_beg + psize, pc_beg : pc_beg + psize
        ]
        patches_x.append(vox)
        patches_y.append(vox_labels)
    return np.array(patches_x), np.array(patches_y)

def positive_ratio(x):
    return float(np.sum(np.greater(x, 0))) / np.prod(x.shape)

def norm(image):
    image = np.squeeze(image)
    image_nonzero = image[np.nonzero(image)]
    return (image - image_nonzero.mean()) / image_nonzero.std()

def segmentation_loss(y_true, y_pred, n_classes):
    y_true = tf.reshape(y_true, (-1, n_classes))
    y_pred = tf.reshape(y_pred, (-1, n_classes))
    return tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)
    )

def vox_preprocess(vox):
    vox_shape = vox.shape
    vox = np.reshape(vox, (-1, vox_shape[-1]))
    vox = scale(vox, axis=0)
    return np.reshape(vox, vox_shape)

def one_hot(y, num_classes):
    y_ = np.zeros([len(y), num_classes])
    y_[np.arange(len(y)), y] = 1
    return y_

def dice_coef_np(y_true, y_pred, num_classes):
    """
    :param y_true: sparse labels
    :param y_pred: sparse labels
    :param num_classes: number of classes
    :return:
    """
    y_true = y_true.astype(int)
    y_pred = y_pred.astype(int)
    y_true = y_true.flatten()
    y_true = one_hot(y_true, num_classes)
    y_pred = y_pred.flatten()
    y_pred = one_hot(y_pred, num_classes)
    intersection = np.sum(y_true * y_pred, axis=0)
    return (2.0 * intersection) / (np.sum(y_true, axis=0) + np.sum(y_pred, axis=0))

def vox_generator(all_files, n_pos, n_neg, correction=False):
    path = options["root_path"]

    def get_filename(prefix):
        return glob.glob(os.path.join(path, file, prefix))[0].split("/")[-1]

    while 1:
        for file in all_files:
            if correction:
                flair = load_nii(
                    os.path.join(path, file, file + "_flair_corrected.nii.gz")
                ).get_data()
                t2 = load_nii(
                    os.path.join(path, file, file + "_t2_corrected.nii.gz")
                ).get_data()
                t1 = load_nii(
                    os.path.join(path, file, file + "_t1_corrected.nii.gz")
                ).get_data()
                t1ce = load_nii(
                    os.path.join(path, file, file + "_t1ce_corrected.nii.gz")
                ).get_data()
            else:
                flair = load_nii(
                    os.path.join(path, file, file + "_flair.nii.gz")
                ).get_data()
                t2 = load_nii(os.path.join(path, file, file + "_t2.nii.gz")).get_data()
                t1 = load_nii(os.path.join(path, file, file + "_t1.nii.gz")).get_data()
                t1ce = load_nii(
                    os.path.join(path, file, file + "_t1ce.nii.gz")
                ).get_data()

            data_norm = np.array([norm(flair), norm(t2), norm(t1), norm(t1ce)])
            data_norm = np.transpose(data_norm, axes=[1, 2, 3, 0])
            labels = load_nii(os.path.join(path, file, file + "_seg.nii.gz")).get_data()

            foreground = np.array(np.where(labels > 0))
            background = np.array(np.where((labels == 0) & (flair > 0)))

            # n_pos = int(foreground.shape[1] * discount)
            foreground = foreground[
                :, np.random.permutation(foreground.shape[1])[:n_pos]
            ]
            background = background[
                :, np.random.permutation(background.shape[1])[:n_neg]
            ]

            centers = np.concatenate((foreground, background), axis=1)
            centers = centers[:, np.random.permutation(n_neg + n_pos)]

            yield data_norm, labels, centers

def label_transform(y, nlabels):
    return [
        keras.utils.to_categorical(
            np.copy(y).astype(dtype=np.bool), num_classes=2
        ).reshape([y.shape[0], y.shape[1], y.shape[2], y.shape[3], 2]),
        keras.utils.to_categorical(y, num_classes=nlabels).reshape(
            [y.shape[0], y.shape[1], y.shape[2], y.shape[3], nlabels]
        ),
    ]

def train():
    tb_writer = SummaryWriter(comment='initial')

    NUM_EPOCHS = options["num_epochs"]
    LOAD_PATH = options["load_path"]
    SAVE_PATH = options["save_path"]
    PSIZE = options["psize"]
    HSIZE = options["hsize"]
    WSIZE = options["wsize"]
    CSIZE = options["csize"]
    model_name = options["model_name"]
    BATCH_SIZE = options["batch_size"]
    continue_training = options["continue_training"]

    num_labels = 5
    global files
    """
    files = []
    with open('train.txt') as f:
        for line in f:
            files.append(line[:-1])
    """

    flair_t2_node = tf.placeholder(
        dtype=tf.float32, shape=(None, HSIZE, WSIZE, CSIZE, 2)
    )
    t1_t1ce_node = tf.placeholder(
        dtype=tf.float32, shape=(None, HSIZE, WSIZE, CSIZE, 2)
    )
    flair_t2_gt_node = tf.placeholder(
        dtype=tf.int32, shape=(None, PSIZE, PSIZE, PSIZE, 2)
    )
    t1_t1ce_gt_node = tf.placeholder(
        dtype=tf.int32, shape=(None, PSIZE, PSIZE, PSIZE, 5)
    )

    if model_name == "dense48":
        flair_t2_15, flair_t2_27 = BraTS2ScaleDenseNetConcat_large(
            input=flair_t2_node, name="flair"
        )
        t1_t1ce_15, t1_t1ce_27 = BraTS2ScaleDenseNetConcat_large(
            input=t1_t1ce_node, name="t1"
        )
    elif model_name == "no_dense":

        flair_t2_15, flair_t2_27 = PlainCounterpart(input=flair_t2_node, name="flair")
        t1_t1ce_15, t1_t1ce_27 = PlainCounterpart(input=t1_t1ce_node, name="t1")

    elif model_name == "dense24":

        flair_t2_15, flair_t2_27 = BraTS2ScaleDenseNetConcat(
            input=flair_t2_node, name="flair"
        )
        t1_t1ce_15, t1_t1ce_27 = BraTS2ScaleDenseNetConcat(
            input=t1_t1ce_node, name="t1"
        )
    else:
        print(" No such model name ")

    t1_t1ce_15 = concatenate([t1_t1ce_15, flair_t2_15])
    t1_t1ce_27 = concatenate([t1_t1ce_27, flair_t2_27])

    flair_t2_15 = Conv3D(
        2, kernel_size=1, strides=1, padding="same", name="flair_t2_15_cls"
    )(flair_t2_15)
    flair_t2_27 = Conv3D(
        2, kernel_size=1, strides=1, padding="same", name="flair_t2_27_cls"
    )(flair_t2_27)
    t1_t1ce_15 = Conv3D(
        num_labels, kernel_size=1, strides=1, padding="same", name="t1_t1ce_15_cls"
    )(t1_t1ce_15)
    t1_t1ce_27 = Conv3D(
        num_labels, kernel_size=1, strides=1, padding="same", name="t1_t1ce_27_cls"
    )(t1_t1ce_27)

    flair_t2_score = (
        flair_t2_15[:, 13:25, 13:25, 13:25, :] + flair_t2_27[:, 13:25, 13:25, 13:25, :]
    )

    t1_t1ce_score = (
        t1_t1ce_15[:, 13:25, 13:25, 13:25, :] + t1_t1ce_27[:, 13:25, 13:25, 13:25, :]
    )

    loss = segmentation_loss(flair_t2_gt_node, flair_t2_score, 2) + segmentation_loss(
        t1_t1ce_gt_node, t1_t1ce_score, 5
    )

    acc_flair_t2 = acc_tf(y_pred=flair_t2_score, y_true=flair_t2_gt_node)
    acc_t1_t1ce = acc_tf(y_pred=t1_t1ce_score, y_true=t1_t1ce_gt_node)

    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    with tf.control_dependencies(update_ops):
        optimizer = tf.train.AdamOptimizer(learning_rate=5e-4).minimize(loss)

    saver = tf.train.Saver(max_to_keep=15)
    data_gen_train = vox_generator(
        all_files=files, n_pos=200, n_neg=200, correction=options["correction"]
    )
    config = tf.ConfigProto(allow_soft_placement=True)
    config.gpu_options.allow_growth = True
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    with tf.Session(config=config) as sess:
        if continue_training:
            saver.restore(sess, LOAD_PATH)
        else:
            sess.run(tf.global_variables_initializer())
        curr_iteration = 0
        for ei in tqdm(range(NUM_EPOCHS)):
            for pi in tqdm(range(len(files))):
                acc_pi, loss_pi = [], []
                data, labels, centers = data_gen_train.__next__()
                n_batches = int(np.ceil(float(centers.shape[1]) / BATCH_SIZE))
                for nb in tqdm(range(n_batches)):
                    offset_batch = min(nb * BATCH_SIZE, centers.shape[1] - BATCH_SIZE)
                    data_batch, label_batch = get_patches_3d(
                        data,
                        labels,
                        centers[:, offset_batch : offset_batch + BATCH_SIZE],
                        HSIZE,
                        WSIZE,
                        CSIZE,
                        PSIZE,
                        False,
                    )
                    label_batch = label_transform(label_batch, 5)
                    _, l, acc_ft, acc_t1c = sess.run(
                        fetches=[optimizer, loss, acc_flair_t2, acc_t1_t1ce],
                        feed_dict={
                            flair_t2_node: data_batch[:, :, :, :, :2],
                            t1_t1ce_node: data_batch[:, :, :, :, 2:],
                            flair_t2_gt_node: label_batch[0],
                            t1_t1ce_gt_node: label_batch[1],
                            learning_phase(): 1,
                        },
                    )
                    acc_pi.append([acc_ft, acc_t1c])
                    loss_pi.append(l)
                    n_pos_sum = np.sum(np.reshape(label_batch[0], (-1, 2)), axis=0)
                    tb_writer.add_scalar("train-loss", l,  curr_iteration)
                    tb_writer.add_scalar("acc/flair_t2", acc_ft, curr_iteration)
                    tb_writer.add_scalar("acc/t1_t1ce", acc_t1c, curr_iteration)
                    curr_iteration += 1
                    tqdm.write(
                        f"epoch-patient: {ei + 1}, {pi + 1}, "
                        f"iter: {nb + 1}-{n_batches}, "
                        f"p%%: {n_pos_sum[1]/float(np.sum(n_pos_sum)):.4f}, "
                        f"loss: {l:.4f}, acc_flair_t2: {acc_ft:.2f}, "
                        f"acc_t1_t1ce: {acc_t1c:.2f}"
                    )

                current_patient_i = (ei + 1) * (pi + 1)
                tb_writer.add_scalar(
                    f"patient{pi}/loss",
                    np.mean(loss_pi),
                    current_patient_i,
                )
                tb_writer.add_scalar(
                    f"patient{pi}/acc",
                    np.mean(acc_pi),
                    current_patient_i,
                )
                tqdm.write(
                    "patient loss: %.4f, patient acc: %.4f"
                    % (np.mean(loss_pi), np.mean(acc_pi))
                )

            saver.save(sess, SAVE_PATH, global_step=ei)
            tqdm.write("Model saved.")

if __name__ == "__main__":

    train()

max_stars_repo_name: maeda/MRI-tumor-segmentation-Brats

max_issues_repo_name: maeda/MRI-tumor-segmentation-Brats

max_forks_repo_name: maeda/MRI-tumor-segmentation-Brats