import os
import sys

import wx
import wx.aui
from wx.lib.pubsub import pub
import pandac.PandaModules as pm
from direct.actor.Actor import Actor

import p3d
from .. import commands as cmds
from wxExtra import utils as wxUtils, ActionItem, LogPanel, CustomMenu
from wxExtra import AuiManagerConfig, CustomAuiToolBar, CustomMenu
from viewport import Viewport
from pandaEditor import game
from document import Document
from baseDialog import BaseDialog
from resourcesPanel import ResourcesPanel
from propertiesPanel import PropertiesPanel
from sceneGraphPanel import SceneGraphPanel
from preferencesFrame import PreferencesFrame
from lightLinkerPanel import LightLinkerPanel
from projectSettingsPanel import ProjectSettingsPanel

FRAME_TITLE = 'Panda Editor 0.1'
TBAR_ICON_SIZE = (24, 24)
WILDCARD_MODEL = 'Model (*.egg; *.bam; *.egg.pz)|*.egg;*.bam;*.egg.pz'
WILDCARD_SCENE = '.xml|*.xml'
WILDCARD_P3D = '.p3d|*.p3d'

ID_FILE_NEW = wx.NewId()
ID_FILE_OPEN = wx.NewId()
ID_FILE_SAVE = wx.NewId()
ID_FILE_SAVE_AS = wx.NewId()
ID_FILE_IMPORT = wx.NewId()
ID_FILE_PROJ = wx.NewId()

ID_PROJ_NEW = wx.NewId()
ID_PROJ_SET = wx.NewId()
ID_PROJ_BUILD = wx.NewId()

ID_EDIT_UNDO = wx.NewId()
ID_EDIT_REDO = wx.NewId()
ID_EDIT_GROUP = wx.NewId()
ID_EDIT_UNGROUP = wx.NewId()
ID_EDIT_PARENT = wx.NewId()
ID_EDIT_UNPARENT = wx.NewId()

ID_MODIFY_PHYSICS = wx.NewId()

ID_XFORM_SEL = wx.NewId()
ID_XFORM_POS = wx.NewId()
ID_XFORM_ROT = wx.NewId()
ID_XFORM_SCL = wx.NewId()
ID_XFORM_WORLD = wx.NewId()

ID_VIEW_GRID = wx.NewId()
ID_VIEW_TOP = wx.NewId()
ID_VIEW_BOTTOM = wx.NewId()
ID_VIEW_FRONT = wx.NewId()
ID_VIEW_BACK = wx.NewId()
ID_VIEW_RIGHT = wx.NewId()
ID_VIEW_LEFT = wx.NewId()

ID_LAYOUT_GAME = wx.NewId()
ID_LAYOUT_EDITOR = wx.NewId()
ID_LAYOUT_BOTH = wx.NewId()

ID_WIND_PANEL = wx.NewId()
ID_WIND_FILE_TOOLBAR = wx.NewId()
ID_WIND_EDIT_TOOLBAR = wx.NewId()
ID_WIND_MODIFY_TOOLBAR = wx.NewId()
ID_WIND_XFORM_TOOLBAR = wx.NewId()
ID_WIND_LAYOUT_TOOLBAR = wx.NewId()
ID_WIND_VIEWPORT = wx.NewId()
ID_WIND_SCENE_GRAPH = wx.NewId()
ID_WIND_LIGHT_LINKER = wx.NewId()
ID_WIND_PROPERTIES = wx.NewId()
ID_WIND_RESOURCES = wx.NewId()
ID_WIND_LOG = wx.NewId()
ID_WIND_PREFERENCES = wx.NewId()

ID_PLAY = wx.NewId()
ID_PAUSE = wx.NewId()

class MainFrame( wx.Frame ):

    """Panda Editor user interface."""

    def __init__( self, *args, **kwargs ):
        wx.Frame.__init__( self, *args, **kwargs )

        self.app = wx.GetApp()
        self.preMaxPos = None
        self.preMaxSize = None
        self.actns = {
            ID_EDIT_UNDO:self.app.Undo,
            ID_EDIT_REDO:self.app.Redo,
            ID_EDIT_GROUP:self.app.Group,
            ID_EDIT_UNGROUP:self.app.Ungroup,
            ID_EDIT_PARENT:self.app.Parent,
            ID_EDIT_UNPARENT:self.app.Unparent
        }

        # Bind frame events
        self.Bind( wx.EVT_CLOSE, self.OnClose )
        self.Bind( wx.EVT_KEY_UP, p3d.wx.OnKeyUp )
        self.Bind( wx.EVT_KEY_DOWN, p3d.wx.OnKeyDown )
        self.Bind( wx.EVT_SIZE, self.OnSize )
        self.Bind( wx.EVT_MOVE, self.OnMove )

        # Bind publisher events
        pub.subscribe( self.OnUpdate, 'Update' )

        # Build application preferences
        self.cfg = wx.Config( 'pandaEditor' )

        # Build toolbars
        self.BuildFileActions()
        self.BuildEditActions()
        self.BuildModifyActions()
        self.BuildXformActions()
        self.BuildLayoutActions()

        # Build viewport. Don't initialise just yet as ShowBase has not yet
        # been created.
        self.pnlViewport = Viewport( self )

        # Build editor panels
        self.pnlSceneGraph = SceneGraphPanel( self, style=wx.SUNKEN_BORDER )
        self.pnlLightLinker = LightLinkerPanel( self, style=wx.SUNKEN_BORDER )
        self.pnlProps = PropertiesPanel( self, style=wx.SUNKEN_BORDER )
        self.pnlRsrcs = ResourcesPanel( self, style=wx.SUNKEN_BORDER )
        self.pnlLog = LogPanel( self, style=wx.SUNKEN_BORDER )

        # Build aui manager to hold all the widgets
        self.BuildAuiManager()

        # Build menus and menu bar
        self.mb = wx.MenuBar()
        self.BuildViewMenu()
        self.BuildCreateMenu()
        self.BuildWindowMenu()
        self.BuildMenuBar()

        # Populate the panels menu bar with items representing each floating
        # panel.
        self.RebuildPanelMenu()

        # Update the view menu based on the perspective saved in preferences
        self.OnUpdateWindowMenu( None )

    def _GetSavePath( self ):

        # Get default paths from current project directory, or the scene's
        # current location on disk
        defaultDir = ''
        defaultFile = ''
        if self.app.doc.filePath is not None:
            defaultDir, defaultFile = os.path.split( self.app.doc.filePath )
        elif self.app.project.path is not None:
            defaultDir = self.app.project.GetScenesDirectory()

        # Open file browser
        filePath = wxUtils.FileSaveDialog( 'Save Scene As', WILDCARD_SCENE, defaultDir=defaultDir, defaultFile=defaultFile )
        if filePath and os.path.exists( filePath ):

            # Warn user if the chosen file path already exists
            msg = ''.join( ['The file "', filePath, '" already exists.\nDo you want to replace it?'] )
            if wxUtils.YesNoDialog( msg, 'Replace File?', wx.ICON_WARNING ) == wx.ID_NO:
                return False

        return filePath

    def _CheckForSave( self ):
        """
        If there is already a file loaded and it is dirty, query the user to
        save the file. Return False for cancel, True otherwise.
        """
        if self.app.doc.dirty:

            # Show dialog, record result
            msg = ''.join( ['The document "', self.app.doc.title, '" was modified after last save.\nSave changes before continuing?'] )
            result = wxUtils.YesNoCancelDialog( msg, 'Save Changes?', wx.ICON_WARNING )
            if result == wx.ID_YES:
                self.OnFileSave( None )
            elif result == wx.ID_CANCEL:
                return False

        # Document not dirty, return True
        return True

    def OnClose( self, evt ):
        """Save frame and aui preferences, hide the window and quit."""
        # Check if ok to continue, stop the closing process if the user
        # cancelled
        if not self._CheckForSave():
            evt.Veto()
            return

        # Save prefs, hide window and quit
        self.auiCfg.Save()
        if self.preMaxPos is not None:
            self.auiCfg.SavePosition( *self.preMaxPos )
        if self.preMaxSize is not None:
            self.auiCfg.SaveSize( *self.preMaxSize )
        if self.app.project.path is not None:
            self.cfg.Write( 'projDirPath', self.app.project.path )
        self.Show( False )
        self.app.Quit()

    def OnFileNew( self, evt ):
        """Show project settings panel and create new scene."""
        # Check if ok to continue, return if the user cancelled
        if not self._CheckForSave():
            return

        # Create new document
        self.app.CreateScene()
        self.app.doc.OnRefresh()

    def OnFileOpen( self, evt, filePath=None ):
        """Create a new document and load the scene."""
        # Check if ok to continue, return if the user cancelled
        if not self._CheckForSave():
            return

        # Create new document from file path and load it
        if filePath is None:

            # Get the start directory. This will be the current working
            # directory if the project is not set.
            scnsDirPath = self.app.project.GetScenesDirectory()
            if scnsDirPath is None:
                scnsDirPath = os.getcwd()

            filePath = wxUtils.FileOpenDialog( 'Open Scene', WILDCARD_SCENE,
                                               defaultDir=scnsDirPath )

        # Create new document
        if filePath:
            self.app.CreateScene( filePath )
            self.app.doc.Load()

    def OnFileSave( self, evt, saveAs=False ):
        """Save the document."""
        # Set a file path for the document if one does not exist, or for save
        # as
        if self.app.doc.filePath is None or saveAs:

            # Query a new save path
            filePath = self._GetSavePath()
            if filePath:
                self.app.doc.filePath = filePath
            else:
                return

        # Save the file
        self.app.doc.Save()

    def OnFileSaveAs( self, evt ):
        """
        Call save using the saveAs flag in order to bring up a new dialog box
        so the user may set an alternate save path.
        """
        self.OnFileSave( evt, True )

    def OnFileImport( self, evt ):
        """Import assets to project."""
        filePaths = wxUtils.FileOpenDialog( 'Import Models', WILDCARD_MODEL,
                                            wx.MULTIPLE )
        if filePaths:
            for filePath in filePaths:
                self.app.project.ImportAsset( filePath )

    def OnFileNewProject( self, evt ):
        """Build project directory and set project."""
        dirPath = wxUtils.DirDialog( 'Set New Project Directory' )
        if dirPath:
            self.app.project.New( dirPath )
            self.SetProjectPath( dirPath )
            self.app.doc.OnRefresh()

    def OnFileSetProject( self, evt ):
        """
        Set the active project directory path and rebuild the resources panel.
        """
        dirPath = wxUtils.DirDialog( 'Set Project Directory' )
        if dirPath:
            self.SetProjectPath( dirPath )
            self.app.doc.OnRefresh()

    def OnFileBuildProject( self, evt ):
        """Build the current project to a p3d file."""
        filePath = wxUtils.FileSaveDialog( 'Build Project', WILDCARD_P3D )
        if not filePath:
            return

        if filePath and os.path.exists( filePath ):

            # Warn user if the chosen file path already exists
            msg = ''.join( ['The file "', filePath, '" already exists.\nDo you want to replace it?'] )
            if wxUtils.YesNoDialog( msg, 'Replace File?', wx.ICON_WARNING ) == wx.ID_NO:
                return

        self.app.project.Build( filePath )

    def OnSingleCommand( self, evt ):
        id = evt.GetId()
        fn = self.actns[id]
        fn()

    def OnEngagePhysics( self, evt ):
        wrpr = base.game.nodeMgr.Wrap( base.scene.physicsWorld )
        if base.scene.physicsTask not in taskMgr.getAllTasks():
            wrpr.EnablePhysics()
        else:
            wrpr.DisablePhysics()

    def OnViewGrid( self, evt ):
        """
        Show or hide the grid based on the checked value of the menu item.
        """
        checked = evt.Checked()
        if checked:
            self.app.grid.show()
        else:
            self.app.grid.hide()

    def OnViewCamera( self, evt, yaw_pitch ):
        """
        Orbit camera top or bottom by manipulating delta values
        See p3d.camera.Orbit for more
        """
        delta = pm.Vec2( -base.edCamera.getH() + yaw_pitch[0], -base.edCamera.getP() + yaw_pitch[1] )
        base.edCamera.Orbit( delta )

    def OnCreate( self, evt, typeStr ):
        self.app.AddComponent( typeStr )

    def OnCreateActor( self, evt ):
        """
        Turn the selection into actors. This is still a massive hack - we need
        a more concise way of storing this information.
        """
        comps = []
        for wrpr in self.app.selection.wrprs:
            attr = wrpr.FindProperty( 'modelPath' )
            if attr is None:
                continue

            wrprCls = base.game.nodeMgr.nodeWrappers['Actor']
            aWrpr = wrprCls.Create( modelPath=attr.Get() )
            aWrpr.data.setTransform( wrpr.data.getTransform() )
            aWrpr.SetDefaultValues()
            aWrpr.SetParent( wrpr.GetDefaultParent() )
            cmds.Replace( wrpr.data, aWrpr.data )

    def OnCreatePrefab( self, evt ):
        """
        Create a new prefab for the selected object in the prefab directory.
        """
        np = self.app.selection.GetNodePaths()[0]
        dirPath = self.app.project.GetPrefabsDirectory()
        assetName = self.app.project.GetUniqueAssetName( 'prefab.xml', dirPath )
        assetPath = os.path.join( dirPath, assetName )
        base.game.scnParser.Save( np, assetPath )

    def OnCreateCgShader( self, evt ):
        """

        """
        self.app.project.CreateCgShader()

    def OnCreateGlslShader( self, evt ):
        """

        """
        self.app.project.CreateGlslShader()

    def OnShowHidePane( self, evt ):
        """
        Show or hide the pane based on the menu item that was (un)checked.
        """
        pane = self.paneDefs[evt.GetId()][0]
        self._mgr.GetPane( pane ).Show( evt.Checked() )

        # Make sure to call or else we won't see any changes.
        self._mgr.Update()
        self.app.doc.OnRefresh()

    def OnXformSetActiveGizmo( self, evt ):
        if evt.GetId() == ID_XFORM_WORLD:
            self.app.SetGizmoLocal( not evt.Checked() )
            return

        arg = None
        if evt.GetId() == ID_XFORM_POS:
            arg = 'pos'
        elif evt.GetId() == ID_XFORM_ROT:
            arg = 'rot'
        elif evt.GetId() == ID_XFORM_SCL:
            arg = 'scl'
        self.app.SetActiveGizmo( arg )

    def OnLayout( self, evt ):
        if evt.GetId() == ID_LAYOUT_GAME:
            base.LayoutGameView()
        elif evt.GetId() == ID_LAYOUT_EDITOR:
            base.LayoutEditorView()
        elif evt.GetId() == ID_LAYOUT_BOTH:
            base.LayoutBothView()

    def OnUpdateWindowMenu( self, evt ):
        """
        Set the checks in the window menu to match the visibility of the
        panes.
        """
        def UpdateWindowMenu():

            # Check those menus representing panels which are still shown
            # after the event
            for id in self.paneDefs:
                pane = self.paneDefs[id][0]
                if self.mPnl.FindItemById( id ) and self._mgr.GetPane( pane ).IsShown():
                    self.mPnl.Check( id, True )

        # Uncheck all menus
        for id in self.paneDefs:
            if self.mPnl.FindItemById( id ):
                self.mPnl.Check( id, False )

        # Call after or IsShown() won't return a useful value
        wx.CallAfter( UpdateWindowMenu )

    def OnUpdate( self, comps=None ):
        """
        Change the appearance and states of buttons on the form based on the
        state of the loaded document.

        NOTE: Don't use freeze / thaw as this will cause the 3D viewport to
        flicker.
        """
        self.OnUpdateFile( comps )
        self.OnUpdateEdit( comps )
        self.OnUpdateModify( comps )
        self.OnUpdateView( comps )
        self.OnUpdateProject( comps )
        self.OnUpdateXform( comps )

        # Set the frame's title to include the document's file path, include
        # dirty 'star'
        title = ''.join( [FRAME_TITLE, ' - ', self.app.doc.title] )
        if self.app.doc.dirty:
            title += ' *'
        self.SetTitle( title )

        self.app.game.pluginMgr.OnUpdate( comps )

    def OnUpdateFile( self, msg ):
        """
        Update the file menu. Disable all menu and toolbar items then turn
        those back on depending on the document's state.
        """
        self.mFile.EnableAllTools( False )
        self.tbFile.EnableAllTools( False )

        self.mFile.Enable( ID_FILE_NEW, True )
        self.mFile.Enable( ID_FILE_OPEN, True )
        self.mFile.Enable( ID_FILE_SAVE_AS, True )
        self.mFile.Enable( ID_FILE_PROJ, True )
        self.tbFile.EnableTool( ID_FILE_NEW, True )
        self.tbFile.EnableTool( ID_FILE_OPEN, True )
        self.tbFile.EnableTool( ID_FILE_SAVE_AS, True )

        if self.app.doc.dirty:
            self.mFile.Enable( ID_FILE_SAVE, True )
            self.tbFile.EnableTool( ID_FILE_SAVE, True )
        if self.app.project.path is not None:
            self.mFile.Enable( ID_FILE_IMPORT, True )

        self.tbFile.Refresh()

    def OnUpdateEdit( self, msg ):
        """
        Update the edit menu. Disable undo or redo queus if they are empty
        and make sure to refresh the toolbar.
        """
        val = len( self.app.actnMgr.undoList ) > 0
        self.mEdit.Enable( ID_EDIT_UNDO, val )
        self.tbEdit.EnableTool( ID_EDIT_UNDO, val )

        val = len( self.app.actnMgr.redoList ) > 0
        self.mEdit.Enable( ID_EDIT_REDO, val )
        self.tbEdit.EnableTool( ID_EDIT_REDO, val )

        self.tbEdit.Refresh()

    def OnUpdateModify( self, msg ):
        self.tbModify.EnableTool( ID_MODIFY_PHYSICS, False )
        if base.scene.physicsWorld is not None:
            self.tbModify.EnableTool( ID_MODIFY_PHYSICS, True )

            if base.scene.physicsTask not in taskMgr.getAllTasks():
                self.tbModify.ToggleTool( ID_MODIFY_PHYSICS, False )
            else:
                self.tbModify.ToggleTool( ID_MODIFY_PHYSICS, True )

        self.tbModify.Refresh()

    def OnUpdateView( self, msg ):
        """
        Update the view menu. Ensure the grid menu item's checked state
        matches the visibility of the grid.
        """
        self.mView.Check( ID_VIEW_GRID, False )
        if not self.app.grid.isHidden():
            self.mView.Check( ID_VIEW_GRID, True )

    def OnUpdateProject( self, msg ):
        self.mProj.EnableAllTools( False )

        self.mProj.Enable( ID_PROJ_NEW, True )
        self.mProj.Enable( ID_PROJ_SET, True )

        if self.app.project.path is not None:
            self.mProj.EnableAllTools( True )

    def OnUpdateXform( self, msg ):
        gizmo = self.app.gizmoMgr.GetActiveGizmo()
        if gizmo is None:
            self.tbXform.ToggleTool( ID_XFORM_SEL, True )
        elif gizmo.getName() == 'pos':
            self.tbXform.ToggleTool( ID_XFORM_POS, True )
        elif gizmo.getName() == 'rot':
            self.tbXform.ToggleTool( ID_XFORM_ROT, True )
        elif gizmo.getName() == 'scl':
            self.tbXform.ToggleTool( ID_XFORM_SCL, True )

        val = not self.app.gizmoMgr.GetGizmoLocal( 'pos' )
        self.tbXform.ToggleTool( ID_XFORM_WORLD, val )

        self.tbXform.Refresh()

    def OnShowPreferences( self, evt ):
        try:
            self.frmPrefs.Close()
        except:
            pass
        self.frmPrefs = PreferencesFrame( self )
        self.frmPrefs.Center()
        self.frmPrefs.Show()

    def OnMove( self, evt ):
        """
        Keep the window's position on hand before it gets maximized as this is
        the number we need to save to preferences.
        """
        if not self.IsMaximized():
            self.preMaxPos = self.GetPosition()

    def OnSize( self, evt ):
        """
        Keep the window's size on hand before it gets maximized as this is the
        number we need to save to preferences.
        """
        if not self.IsMaximized():
            self.preMaxSize = self.GetSize()

    def SetProjectPath( self, dirPath ):
        """
        Set the project path and rebuild the resources panel.
        """
        self.app.project.Set( dirPath )
        self.pnlRsrcs.Build( self.app.project.path )

    def BuildFileActions( self ):
        """Add tools, set long help strings and bind toolbar events."""
        commonActns = [
            ActionItem( 'New', os.path.join( 'data', 'images', 'document.png' ), self.OnFileNew, ID_FILE_NEW ),
            ActionItem( 'Open', os.path.join( 'data', 'images', 'folder-horizontal-open.png' ), self.OnFileOpen, ID_FILE_OPEN ),
            ActionItem( 'Save', os.path.join( 'data', 'images', 'disk-black.png' ), self.OnFileSave, ID_FILE_SAVE ),
            ActionItem( 'Save As', os.path.join( 'data', 'images', 'disk-black-pencil.png' ), self.OnFileSaveAs, ID_FILE_SAVE_AS ),
        ]

        # Create file menu
        self.mFile = CustomMenu()
        self.mFile.AppendActionItems( commonActns, self )
        self.mFile.AppendSeparator()
        self.mFile.AppendActionItem( ActionItem( 'Import...', '', self.OnFileImport, ID_FILE_IMPORT ), self )

        # Create project actions as a submenu
        self.mProj = CustomMenu()
        actns = [
            ActionItem( 'New...', '', self.OnFileNewProject, ID_PROJ_NEW ),
            ActionItem( 'Set...', '', self.OnFileSetProject, ID_PROJ_SET ),
            ActionItem( 'Build...', '', self.OnFileBuildProject, ID_PROJ_BUILD )
        ]
        self.mProj.AppendActionItems( actns, self )
        self.mFile.AppendMenu( ID_FILE_PROJ, '&Project', self.mProj )

        # Create file toolbar
        self.tbFile = CustomAuiToolBar( self, -1 )
        self.tbFile.SetToolBitmapSize( TBAR_ICON_SIZE )
        self.tbFile.AppendActionItems( commonActns )
        self.tbFile.Realize()

    def BuildEditActions( self ):
        """Add tools, set long help strings and bind toolbar events."""
        commonActns = [
            ActionItem( 'Undo', os.path.join( 'data', 'images', 'arrow-curve-flip.png' ), self.OnSingleCommand, ID_EDIT_UNDO ),
            ActionItem( 'Redo', os.path.join( 'data', 'images', 'arrow-curve.png' ), self.OnSingleCommand, ID_EDIT_REDO )
        ]

        grpActns = [
            ActionItem( 'Group', '', self.OnSingleCommand, ID_EDIT_GROUP ),
            ActionItem( 'Ungroup', '', self.OnSingleCommand, ID_EDIT_UNGROUP )
        ]

        pntActns = [
            ActionItem( 'Parent', '', self.OnSingleCommand, ID_EDIT_PARENT ),
            ActionItem( 'Unparent', '', self.OnSingleCommand, ID_EDIT_UNPARENT )
        ]

        # Create edit menu
        self.mEdit = CustomMenu()
        self.mEdit.AppendActionItems( commonActns, self )
        self.mEdit.AppendSeparator()
        self.mEdit.AppendActionItems( grpActns, self )
        #self.mEdit.AppendSeparator()
        #self.mEdit.AppendActionItems( pntActns, self )

        # Create edit toolbar
        self.tbEdit = CustomAuiToolBar( self, -1 )
        self.tbEdit.SetToolBitmapSize( TBAR_ICON_SIZE )
        self.tbEdit.AppendActionItems( commonActns )
        self.tbEdit.Realize()

    def BuildModifyActions( self ):
        """Add tools, set long help strings and bind toolbar events."""
        actns = [
            ActionItem( 'Engage Physics', os.path.join( 'data', 'images', 'point.png' ), self.OnEngagePhysics, ID_MODIFY_PHYSICS, kind=wx.ITEM_CHECK )
        ]

        # Create edit menu
        self.mModify = CustomMenu()
        self.mModify.AppendActionItems( actns, self )

        # Create edit toolbar
        self.tbModify = CustomAuiToolBar( self, -1 )
        self.tbModify.SetToolBitmapSize( TBAR_ICON_SIZE )
        self.tbModify.AppendActionItems( actns )
        self.tbModify.Realize()

    def BuildXformActions( self ):
        """Add tools, set long help strings and bind toolbar events."""
        fn = self.OnXformSetActiveGizmo
        actns = [
            ActionItem( 'Select', os.path.join( 'data', 'images', 'select.png' ), fn, ID_XFORM_SEL, kind=wx.ITEM_RADIO ),
            ActionItem( 'Move', os.path.join( 'data', 'images', 'move.png' ), fn, ID_XFORM_POS, kind=wx.ITEM_RADIO ),
            ActionItem( 'Rotate', os.path.join( 'data', 'images', 'rotate.png' ), fn, ID_XFORM_ROT, kind=wx.ITEM_RADIO ),
            ActionItem( 'Scale', os.path.join( 'data', 'images', 'scale.png' ), fn, ID_XFORM_SCL, kind=wx.ITEM_RADIO ),
            ActionItem( 'World Transform', os.path.join( 'data', 'images', 'globe.png' ), fn, ID_XFORM_WORLD, kind=wx.ITEM_CHECK )
        ]

        # Create xform toolbar
        self.tbXform = CustomAuiToolBar( self, -1 )
        self.tbXform.SetToolBitmapSize( TBAR_ICON_SIZE )
        self.tbXform.AddSpacer( 0 )   # Need to insert a null object here or the radio buttons don't seem to work (win7 at least).
        self.tbXform.AppendActionItems( actns )
        self.tbXform.Realize()

    def BuildLayoutActions( self ):
        """Add tools, set long help strings and bind toolbar events."""
        actns = [
            ActionItem( 'Editor', os.path.join( 'data', 'images', 'application-sidebar-list.png' ), self.OnLayout, ID_LAYOUT_EDITOR, kind=wx.ITEM_RADIO ),
            ActionItem( 'Game', os.path.join( 'data', 'images', 'layout-game.png' ), self.OnLayout, ID_LAYOUT_GAME, kind=wx.ITEM_RADIO ),
            ActionItem( 'Both', os.path.join( 'data', 'images', 'layout-both.png' ), self.OnLayout, ID_LAYOUT_BOTH, kind=wx.ITEM_RADIO )
        ]

        # Create layout toolbar
        self.tbLayout = CustomAuiToolBar( self, -1 )
        self.tbLayout.SetToolBitmapSize( TBAR_ICON_SIZE )
        self.tbLayout.AddSpacer( 0 )   # Need to insert a null object here or the radio buttons don't seem to work (win7 at least).
        self.tbLayout.AppendActionItems( actns )
        self.tbLayout.ToggleTool( ID_LAYOUT_EDITOR, True )
        self.tbLayout.Realize()

    def BuildViewMenu( self ):
        """Build the view menu."""
        viewActns = [
            ActionItem( 'Grid', '', self.OnViewGrid, ID_VIEW_GRID, kind=wx.ITEM_CHECK )
        ]

        camActns = [
            ActionItem( 'Top', '', self.OnViewCamera, ID_VIEW_TOP, args=(0, -90) ),
            ActionItem( 'Bottom', '', self.OnViewCamera, ID_VIEW_BOTTOM, args=(0, 90) ),
            ActionItem( 'Left', '', self.OnViewCamera, ID_VIEW_LEFT, args=(-90, 0) ),
            ActionItem( 'Right', '', self.OnViewCamera, ID_VIEW_RIGHT, args=(90, 0) ),
            ActionItem( 'Front', '', self.OnViewCamera, ID_VIEW_FRONT, args=(0, 0) ),
            ActionItem( 'Back', '', self.OnViewCamera, ID_VIEW_BACK, args=(-180, 0) )
        ]
        self.mCameras = CustomMenu()
        self.mCameras.AppendActionItems( camActns, self )

        # Append to view menu
        self.mView = CustomMenu()
        self.mView.AppendActionItems( viewActns, self )
        self.mView.AppendSeparator()
        self.mView.AppendSubMenu( self.mCameras, '&Camera' )

    def BuildCreateMenu( self ):
        """Build the create menu."""
        lightActns = [
            ActionItem( 'Ambient', '', self.OnCreate, args='AmbientLight' ),
            ActionItem( 'Point', '', self.OnCreate, args='PointLight' ),
            ActionItem( 'Directional', '', self.OnCreate, args='DirectionalLight' ),
            ActionItem( 'Spot', '', self.OnCreate, args='Spotlight' )
        ]
        mLights = CustomMenu()
        mLights.AppendActionItems( lightActns, self )

        collActns = [
            ActionItem( 'Node', '', self.OnCreate, args='CollisionNode' ),
            ActionItem( 'Box', '', self.OnCreate, args='CollisionBox' ),
            ActionItem( 'Ray', '', self.OnCreate, args='CollisionRay' ),
            ActionItem( 'Sphere', '', self.OnCreate, args='CollisionSphere' ),
            ActionItem( 'Inverse Sphere', '', self.OnCreate, args='CollisionInvSphere' ),
            ActionItem( 'Tube', '', self.OnCreate, args='CollisionTube' )
        ]
        mColl = CustomMenu()
        mColl.AppendActionItems( collActns, self )

        texActns = [
            ActionItem( 'Texture', '', self.OnCreate, args='Texture' )#,
            #ActionItem( 'Texture Stage', '', self.OnCreate, args='TextureStage' )
        ]
        mTex = CustomMenu()
        mTex.AppendActionItems( texActns, self )

        shaActns = [
            ActionItem( 'Shader', '', self.OnCreate, args='Shader' )
        ]
        mSha = CustomMenu()
        mSha.AppendActionItems( shaActns, self )

        bltActions = [
            ActionItem( 'World', '', self.OnCreate, args='BulletWorld' ),
            ActionItem( 'Debug Node', '', self.OnCreate, args='BulletDebugNode' ),
            ActionItem( 'Rigid Body Node', '', self.OnCreate, args='BulletRigidBodyNode' ),
            ActionItem( 'Character Controller Node', '', self.OnCreate, args='BulletCharacterControllerNode' ),
            ActionItem( 'Box Shape', '', self.OnCreate, args='BulletBoxShape' ),
            ActionItem( 'Plane Shape', '', self.OnCreate, args='BulletPlaneShape' ),
            ActionItem( 'Capsule Shape', '', self.OnCreate, args='BulletCapsuleShape' )
        ]
        mBlt = CustomMenu()
        mBlt.AppendActionItems( bltActions, self )

        self.mCreate = CustomMenu()
        self.mCreate.AppendActionItem( ActionItem( 'Panda Node', '', self.OnCreate, args='PandaNode' ), self )
        self.mCreate.AppendActionItem( ActionItem( 'Actor', '', self.OnCreateActor ), self )
        self.mCreate.AppendActionItem( ActionItem( 'Fog', '', self.OnCreate, args='Fog' ), self )
        self.mCreate.AppendSubMenu( mColl, '&Collision' )
        self.mCreate.AppendSubMenu( mLights, '&Lights' )
        self.mCreate.AppendSubMenu( mTex, '&Texture' )
        self.mCreate.AppendSubMenu( mSha, '&Shader' )
        self.mCreate.AppendSubMenu( mBlt, '&Bullet' )
        self.mCreate.AppendSeparator()
        self.mCreate.AppendActionItem( ActionItem( 'Prefab', '', self.OnCreatePrefab ), self )
        #self.mCreate.AppendSeparator()
        #self.mCreate.AppendActionItem( ActionItem( 'Cg Shader', '', self.OnCreateCgShader ), self )
        #self.mCreate.AppendActionItem( ActionItem( 'Glsl Shader', '', self.OnCreateGlslShader ), self )

    def BuildWindowMenu( self ):
        """Build show / hide controls for panes."""
        self.mPnl = CustomMenu()

        self.mWind = CustomMenu()
        self.mWind.AppendMenu( ID_WIND_PANEL, '&Panel', self.mPnl )
        self.mWind.AppendActionItem( ActionItem( 'Preferences', '', self.OnShowPreferences, ID_WIND_PREFERENCES ), self )

    def RebuildPanelMenu( self ):
        self.Freeze()

        self.mPnl.Clear()
        for id, paneDef in self.paneDefs.items():
            if paneDef[1]:
                self.mPnl.AppendCheckItem( id, paneDef[2].caption )
                self.Bind( wx.EVT_MENU, self.OnShowHidePane, id=id )
        self.OnUpdateWindowMenu( None )

        self.Thaw()

    def BuildMenuBar( self ):
        """Build the menu bar and attach all menus to it."""
        self.mb.Append( self.mFile, '&File' )
        self.mb.Append( self.mEdit, '&Edit' )
        self.mb.Append( self.mView, '&View' )
        self.mb.Append( self.mCreate, '&Create' )
        self.mb.Append( self.mWind, '&Window' )

        self.SetMenuBar( self.mb )

    def BuildAuiManager( self ):
        """
        Define the behaviour for each aui manager panel, then add them to the
        manager.
        """
        # Define aui manager panes
        # Each tuple is defined as: widget, show in window menu, aui panel
        # info
        self.paneDefs = {
            ID_WIND_FILE_TOOLBAR:(self.tbFile, True,
                wx.aui.AuiPaneInfo()
                .Name( 'tbFile' )
                .Caption( 'File Toolbar' )
                .ToolbarPane()
                .Top()),

            ID_WIND_EDIT_TOOLBAR:(self.tbEdit, True,
                wx.aui.AuiPaneInfo()
                .Name( 'tbEdit' )
                .Caption( 'Edit Toolbar' )
                .ToolbarPane()
                .Top()),

            ID_WIND_MODIFY_TOOLBAR:(self.tbModify, True,
                wx.aui.AuiPaneInfo()
                .Name( 'tbModify' )
                .Caption( 'Modify Toolbar' )
                .ToolbarPane()
                .Top()),

            ID_WIND_XFORM_TOOLBAR:(self.tbXform, True,
                wx.aui.AuiPaneInfo()
                .Name( 'tbXform' )
                .Caption( 'Transform Toolbar' )
                .ToolbarPane()
                .Top()),

            ID_WIND_LAYOUT_TOOLBAR:(self.tbLayout, True,
                wx.aui.AuiPaneInfo()
                .Name( 'tbLayout' )
                .Caption( 'Layout Toolbar' )
                .ToolbarPane()
                .Top()),

            ID_WIND_VIEWPORT:(self.pnlViewport, False,
                wx.aui.AuiPaneInfo()
                .Name( 'pnlViewport' )
                .Caption( 'Viewport' )
                .CloseButton( False )
                .MaximizeButton( True )
                .Center()),

            ID_WIND_SCENE_GRAPH:(self.pnlSceneGraph, True,
                wx.aui.AuiPaneInfo()
                .Name( 'pnlSceneGraph' )
                .Caption( 'Scene Graph' )
                .CloseButton( True )
                .MaximizeButton( True )
                .MinSize( (100, 100) )
                .Left()
                .Position( 2 )),

            ID_WIND_LIGHT_LINKER:(self.pnlLightLinker, True,
                wx.aui.AuiPaneInfo()
                .Name( 'pnlLightLinker' )
                .Caption( 'Light Linker' )
                .CloseButton( True )
                .MaximizeButton( True )
                .MinSize( (100, 100) )
                .Right()
                .Position( 2 )),

            ID_WIND_RESOURCES:(self.pnlRsrcs, True,
                wx.aui.AuiPaneInfo()
                .Name( 'pnlRsrcs' )
                .Caption( 'Resources' )
                .CloseButton( True )
                .MaximizeButton( True )
                .MinSize( (100, 100) )
                .Right()
                .Position( 2 )),

            ID_WIND_PROPERTIES:(self.pnlProps, True,
                wx.aui.AuiPaneInfo()
                .Name( 'pnlProps' )
                .Caption( 'Properties' )
                .CloseButton( True )
                .MaximizeButton( True )
                .MinSize( (100, 100) )
                .Right()),

            ID_WIND_LOG:(self.pnlLog, True,
                wx.aui.AuiPaneInfo()
                .Name( 'pnlLog' )
                .Caption( 'Log' )
                .CloseButton( True )
                .MaximizeButton( True )
                .MinSize( (100, 100) )
                .Bottom()
                .Position( 1 ))
        }

        # Build aui manager and add each pane
        self._mgr = wx.aui.AuiManager( self )
        for paneDef in self.paneDefs.values():
            self._mgr.AddPane( paneDef[0], paneDef[2] )

        # Bind aui manager events
        self._mgr.Bind( wx.aui.EVT_AUI_PANE_CLOSE, self.OnUpdateWindowMenu )

        # Create config and load preferences for all panels
        self.auiCfg = AuiManagerConfig( self._mgr, 'pandaEditorWindow' )
        self.auiCfg.Load()
        self._mgr.Update()


import pyexcel
import sqlite3

from datetime import datetime
from graph import Graph

def parse(file):
    excel = {'content': file.read(),
             'name': file.filename,
             'type': file.filename.split('.')[-1]}

    data = pyexcel.get_sheet(file_content=excel['content'], file_type=excel['type'], name_columns_by_row=0)
    nodes = list(filter(lambda x: isinstance(x, int),
                        set(data.column_at(2))))
    edges = list(filter(lambda x: x[0] in nodes and x[1] in nodes,
                        list(zip(data.column_at(0), data.column_at(1)))))

    return {'nodes': nodes, 'edges': edges}

class History:
    def __init__(self, dbname):
        self._dbname = dbname
        conn = sqlite3.connect(self._dbname)
        cursor = conn.cursor()
        cursor.execute("""CREATE TABLE IF NOT EXISTS Graphs (
            ID integer primary key autoincrement,
            NET text not null,
            FILENAME text not null,
            DATETIME text not null
        )""")

    def save(self, file):
        data = parse(file)
        net = Graph(data['edges'], data['nodes']).to_visjs()
        time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        conn = sqlite3.connect(self._dbname)
        cursor = conn.cursor()
        cursor.execute("INSERT INTO Graphs (NET, FILENAME, DATETIME) VALUES ( ?, ?, ? )",
                       (net, file.filename, time))
        conn.commit()
        conn.close()

    def tolist(self):
        conn = sqlite3.connect(self._dbname)
        cursor = conn.cursor()
        req = "SELECT ID, DATETIME, FILENAME FROM Graphs ORDER BY ID DESC"
        data = [row for row in cursor.execute(req)]
        conn.close()

        return data

    def _get_values_by_id(self, values, id):
        conn = sqlite3.connect(self._dbname)
        cursor = conn.cursor()
        req = 'SELECT {} FROM Graphs WHERE id == {}'.format(
            ','.join(values), '(SELECT MAX(ID) FROM Graphs)' if id < 0 else str(id))
        data = next(cursor.execute(req))
        conn.close()

        return data

    def get_metadata_by_id(self, id):
        return self._get_values_by_id(['ID', 'DATETIME', 'FILENAME'], id)

    def get_graph_by_id(self, id):
        return self._get_values_by_id(['NET'], id)

    def get_all_by_id(self, id):
        return self._get_values_by_id(['ID', 'DATETIME', 'FILENAME', 'NET'], id)


from utils import *
from config import cfg
from framework import Sentence

def _valid(word_list):
    """

    :param list[str] word_list:
    :rtype: bool
    """
    if len(word_list) == 1:
        return word_list
    word_list = [item.lower() for item in word_list]
    if not all([word.isalpha() for word in word_list]):
        return False
    if collide(word_list, small_words):
        return False
    return True

def build_vocabulary(sentences):
    """
    Build vocabulary from corpus.
    :param list[Sentence] sentences: Tokenized words.
    :rtype: Vocabulary
    """
    uni_gram = Vocabulary(cfg.use_lemma)
    for sentence in sentences:
        sentence = sentence.token
        for token in sentence:
            uni_gram.add([token])

    voc = uni_gram.copy()
    # Maximum length of word spans
    k = cfg.k
    for sentence in sentences:
        sentence = sentence.token
        s = len(sentence)
        for start_idx in range(s):
            for end_idx in range(start_idx+1, min(start_idx+k, s)):
                word_span = sentence[start_idx:end_idx+1]
                if _valid(word_span):
                    lexicon = ' '.join(word_span)
                    voc.add(lexicon)
    voc.trim(word_freq=cfg.word_freq)
    return uni_gram, voc

def build_vocabulary_(sentences):
    """
    Build vocabulary from corpus.
    :param list[Sentence] sentences: Tokenized words.
    :rtype: Vocabulary
    """
    voc = Vocabulary(cfg.use_lemma)
    # Maximum length of word spans
    k = cfg.k
    for sentence in sentences:
        sentence = sentence.token
        s = len(sentence)
        for start_idx in range(s):
            for end_idx in range(start_idx, min(start_idx+1, s)):
                word_span = sentence[start_idx:end_idx+1]
                if _valid(word_span):
                    lexicon = ' '.join(word_span)
                    voc.add(lexicon)
    voc.trim(word_freq=cfg.word_freq)

    for sentence in sentences:
        sentence = sentence.token
        s = len(sentence)
        for start_idx in range(s):
            for end_idx in range(start_idx+1, min(start_idx+k, s)):
                word_span = sentence[start_idx:end_idx+1]
                if _valid(word_span):
                    lexicon = ' '.join(word_span)
                    voc.add(lexicon)
    voc.trim(word_freq=cfg.word_freq)

    return voc

def to_matrix(sentence, voc):
    """
    Convert sentences to matrices.
    :param list[str] sentence: A series of words.
    :param Vocabulary voc: Auxiliary vocabulary.
    :rtype: np.ndarray
    """
    offset = k = cfg.k
    s = len(sentence)
    ret = np.zeros(shape=(s, k), dtype=np.int32)
    sentence = ['<PAD>'] * offset + sentence
    for s_idx in range(s):
        for k_idx in range(k):
            word = ' '.join(sentence[offset+s_idx-k_idx: offset+s_idx+1])
            ret[s_idx, k_idx] = voc[word]
    return ret


import copy

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

import torchvision
import torchvision.transforms as transforms

from tqdm import tqdm as pbar
# from torch.utils.tensorboard import SummaryWriter
from resnet import *

def make_dataloaders(params):
    """
    Make a Pytorch dataloader object that can be used for traing and valiation
    Input:
        - params dict with key 'path' (string): path of the dataset folder
        - params dict with key 'batch_size' (int): mini-batch size
        - params dict with key 'num_workers' (int): number of workers for dataloader
    Output:
        - trainloader and testloader (pytorch dataloader object)
    """
    transform_train = transforms.Compose([transforms.RandomCrop(32, padding=4),
                                          transforms.RandomHorizontalFlip(),
                                          transforms.ToTensor(),
                                          transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])])

    transform_validation = transforms.Compose([transforms.ToTensor(),
                                               transforms.Normalize([0.4914, 0.4822, 0.4465],
                                                                    [0.2023, 0.1994, 0.2010])])

    transform_validation = transforms.Compose([transforms.ToTensor()])

    trainset = torchvision.datasets.CIFAR10(root=params['path'], train=True, transform=transform_train,download=True)
    testset = torchvision.datasets.CIFAR10(root=params['path'], train=False, transform=transform_validation,download=True)

    trainloader = torch.utils.data.DataLoader(trainset, batch_size=params['batch_size'], shuffle=True, num_workers=4)
    testloader = torch.utils.data.DataLoader(testset, batch_size=params['batch_size'], shuffle=False, num_workers=4)
    return trainloader, testloader

def train_model(model, params):
    model = model.to(params['device'])
    optimizer = optim.Adam(model.parameters())
    total_updates = params['num_epochs'] * len(params['train_loader'])

    criterion = nn.CrossEntropyLoss()
    # best_accuracy = test_model(model, params)
    best_model = copy.deepcopy(model.state_dict())

    for epoch in pbar(range(params['num_epochs'])):
        # Each epoch has a training and validation phase
        for phase in ['train', 'validation']:

            # Loss accumulator for each epoch
            logs = {'Loss': 0.0,
                    'Accuracy': 0.0}

            # Set the model to the correct phase
            model.train() if phase == 'train' else model.eval()

            # Iterate over data
            for image, label in params[phase + '_loader']:
                image = image.to(params['device'])
                label = label.to(params['device'])

                # Zero gradient
                optimizer.zero_grad()

                with torch.set_grad_enabled(phase == 'train'):

                    # Forward pass
                    prediction = model(image)
                    loss = criterion(prediction, label)
                    accuracy = torch.sum(torch.max(prediction, 1)[1] == label.data).item()

                    # Update log
                    logs['Loss'] += image.shape[0] * loss.detach().item()
                    logs['Accuracy'] += accuracy

                    # Backward pass
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

            # Normalize and write the data to TensorBoard
            logs['Loss'] /= len(params[phase + '_loader'].dataset)
            logs['Accuracy'] /= len(params[phase + '_loader'].dataset)

if __name__ == '__main__':

    model = resnet18()

    # Train on cuda if available
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    device = torch.device('cpu')
    print("Using", device)

    data_params = {'path': './pytorch_dataset/cifar10', 'batch_size': 256}

    train_loader, validation_loader = make_dataloaders(data_params)

    train_params = {'description': 'Test',
                'num_epochs': 300,
                'check_point': 50, 'device': device,
                'train_loader': train_loader, 'validation_loader': validation_loader}

    train_model(model, train_params)


#!/usr/bin/python
# coding: utf8

import click
from geonames import Geonames
import os

@click.argument('input')
@click.command()
def cli(input):
    basename = os.path.basename(input)
    file_name, ext = os.path.splitext(basename)
    g = Geonames(input)
    g.export_shp(file_name)
    g.export_csv(file_name)
    click.echo(g.label)

if __name__ == '__main__':
    cli()


# coding: utf-8
"""
Models for SBML mapping.
"""

import logging

from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from .core import Attachment, EDDObject
from .fields import VarCharField
from .measurement_type import MeasurementType

logger = logging.getLogger(__name__)

@python_2_unicode_compatible
class SBMLTemplate(EDDObject):
    """ Container for information used in SBML export. """
    class Meta:
        db_table = "sbml_template"
    object_ref = models.OneToOneField(EDDObject, parent_link=True)
    biomass_calculation = models.DecimalField(
        decimal_places=5,
        default=-1,
        help_text=_('The calculated multiplier converting OD to weight of biomass.'),
        max_digits=16,
        verbose_name=_('Biomass Factor'),
    )
    biomass_calculation_info = models.TextField(
        default='',
        help_text=_('Additional information on biomass calculation.'),
        verbose_name=_('Biomass Calculation'),
    )
    biomass_exchange_name = models.TextField(
        help_text=_('The reaction name in the model for Biomass.'),
        verbose_name=_('Biomass Reaction'),
    )
    # FIXME would like to limit this to attachments only on parent EDDObject, and remove null=True
    sbml_file = models.ForeignKey(
        Attachment,
        blank=True,
        help_text=_('The Attachment containing the SBML model file.'),
        null=True,
        on_delete=models.PROTECT,
        verbose_name=_('SBML Model'),
    )

    def __str__(self):
        return self.name

    @property
    def xml_file(self):
        return self.sbml_file

    def load_reactions(self):
        read_sbml = self.parseSBML()
        if read_sbml.getNumErrors() > 0:
            log = read_sbml.getErrorLog()
            for i in range(read_sbml.getNumErrors()):
                logger.error("--- SBML ERROR --- %s" % log.getError(i).getMessage())
            raise Exception("Could not load SBML")
        model = read_sbml.getModel()
        rlist = model.getListOfReactions()
        return rlist

    def parseSBML(self):
        if not hasattr(self, '_sbml_document'):
            # self.sbml_file = ForeignKey
            # self.sbml_file.file = FileField on Attachment
            # self.sbml_file.file.file = File object on FileField
            # self.sbml_file.file.file.name = path to file
            import libsbml
            self._sbml_document = libsbml.readSBML(self.sbml_file.file.file.name)
        return self._sbml_document

    def save(self, *args, **kwargs):
        # may need to do a post-save signal; get sbml attachment and save in sbml_file
        super(SBMLTemplate, self).save(*args, **kwargs)

    def to_json(self, depth=0):
        return {
            "id": self.pk,
            "name": self.name,
            "biomassCalculation": self.biomass_calculation,
        }

@python_2_unicode_compatible
class MetaboliteExchange(models.Model):
    """ Mapping for a metabolite to an exchange defined by a SBML template. """
    class Meta:
        db_table = "measurement_type_to_exchange"
        index_together = (
            ("sbml_template", "reactant_name"),  # reactants not unique, but should be searchable
            ("sbml_template", "exchange_name"),  # index implied by unique, making explicit
        )
        unique_together = (
            ("sbml_template", "exchange_name"),
            ("sbml_template", "measurement_type"),
        )
    sbml_template = models.ForeignKey(
        SBMLTemplate,
        help_text=_('The SBML Model containing this exchange reaction.'),
        on_delete=models.CASCADE,
        verbose_name=_('SBML Model'),
    )
    measurement_type = models.ForeignKey(
        MeasurementType,
        blank=True,
        help_text=_('Measurement type linked to this exchange reaction in the model.'),
        null=True,
        on_delete=models.CASCADE,
        verbose_name=_('Measurement Type'),
    )
    reactant_name = VarCharField(
        help_text=_('The reactant name used in for this exchange reaction.'),
        verbose_name=_('Reactant Name'),
    )
    exchange_name = VarCharField(
        help_text=_('The exchange name used in the model.'),
        verbose_name=_('Exchange Name'),
    )

    def __str__(self):
        return self.exchange_name

@python_2_unicode_compatible
class MetaboliteSpecies(models.Model):
    """ Mapping for a metabolite to an species defined by a SBML template. """
    class Meta:
        db_table = "measurement_type_to_species"
        index_together = (
            ("sbml_template", "species"),  # index implied by unique, making explicit
        )
        unique_together = (
            ("sbml_template", "species"),
            ("sbml_template", "measurement_type"),
        )
    sbml_template = models.ForeignKey(
        SBMLTemplate,
        help_text=_('The SBML Model defining this species link to a Measurement Type.'),
        on_delete=models.PROTECT,
        verbose_name=_('SBML Model'),
    )
    measurement_type = models.ForeignKey(
        MeasurementType,
        blank=True,
        help_text=_('Mesurement type linked to this species in the model.'),
        null=True,
        on_delete=models.CASCADE,
        verbose_name=_('Measurement Type'),
    )
    species = VarCharField(
        help_text=_('Species name used in the model for this metabolite.'),
        verbose_name=_('Species'),
    )

    def __str__(self):
        return self.species


from askapdev.rbuild.setup import setup
from askapdev.rbuild.dependencies import Dependency
from setuptools import find_packages
from setuptools.extension import Extension
import os
import glob

dep = Dependency()
dep.add_package()

ROOTPKG = "askap"
PKGNAME = "accessors"

setup(name = "%s.%s" % (ROOTPKG, PKGNAME),
      version = 'current',
      description = 'Python wrapper for accessors package',
      author = 'Keith Bannister',
      author_email = 'keith.bannister@csiro.au',
      url = 'http://svn.atnf.csiro.au/askap',
      keywords = ['ASKAP', 'Base', 'accessor'],
      long_description = '''Python accessor wrapper''',
      packages = find_packages(),
      namespace_packages = [ROOTPKG],
      license = 'GPL',
      zip_safe = 0,
      dependency = dep,
      ext_modules=[Extension("askap/accessors/_accessors", glob.glob('src/*.cpp'),
                             include_dirs=[os.path.join(dep.get_install_path('accessors'), 'include')])],
      test_suite = "nose.collector",
      )


from typing import List

class Solution:
    def findOcurrences(self, text: str, first: str, second: str) -> List[str]:
        # d = {}
        # d[first] = second
        # text = text.split()
        # ans = []
        # for k in range(len(text)-2):
        #     if text[k] in d:
        #         if text[k+1] == d[text[k]]:
        #             ans.append(text[k+2])
        #         else:
        #             pass
        #     else:
        #         pass
        # return ans

        text = text.split()
        tmp = zip(text, text[1:], text[2:])
        ans = []
        for e in tmp:
            if e[0] == first and e[1] == second:
                ans.append(e[2])
        return ans

text = 'alice is a good girl she is a good student'
first = 'a'
second = 'good'
text = 'we will we will rock you'
first = 'we'
second = 'will'
solu = Solution()
print(solu.findOcurrences(text, first, second))


import torch
import torch.nn as nn
import numpy as np
import random
import matplotlib.pyplot as plt
import tokenization as t
import model as m
import sample as s
import pickle
import argparse

parser = argparse.ArgumentParser(description="train the model")
parser.add_argument("--model_checkpoint", type=str, help="")

levels, chars, indx_to_chars, chars_to_indx, n_chars = t.get_everything("levels/*.txt")

with open("character_data.pickle", "wb") as f:
    pickle.dump((chars, indx_to_chars, chars_to_indx, n_chars), f)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = m.Model(n_chars, 50, 0.25, device, 2).to(device)

args = parser.parse_args()
if args.model_checkpoint != None:
    model.load_state_dict(torch.load(args.model_checkpoint, map_location=device))
    model.eval()

def random_data(batch_size):
    levels_to_process = []
    x = []
    y = []
    max_length = 0
    for _ in range(batch_size):
        level = random.choice(levels)
        if len(level) > max_length:
            max_length = len(level)
        levels_to_process.append(level)

    for i in range(len(levels_to_process)):
        levels_to_process[i] = levels_to_process[i].ljust(max_length, "@")

    for level in levels_to_process:
        x.append(model.input_process(level, chars_to_indx))
        y.append(model.output_process(level, chars_to_indx))

    x = torch.stack(x)
    y = torch.stack(y)
    return x, y

lr = 0.01
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr)

def train(input_train, output_train):
    hidden = model.init_hidden(input_train.shape[0])
    model.zero_grad()
    loss = 0
    for i in range(input_train.shape[1]):
        y = output_train[:, i]
        x = input_train[:, i, :]
        x = x.view(x.shape[0], 1, x.shape[1])
        output, hidden = model(hidden, x)
        output = output.squeeze(1)
        l = criterion(output, y)
        loss += l

    loss.backward()
    optimizer.step()

    return output, loss.item() / len(input_train)

n_iters = 5000
print_every = 20
plot_every = 5
all_loses = []
total_loss = 0

for i in range(0, n_iters):
    output, loss = train(*random_data(25))
    total_loss += loss

    if i % print_every == 0:
        print(f"====================EPOCH: {i}=================")
        print(loss)
        print(
            s.sample(model, n_chars, indx_to_chars, chars_to_indx, 255).replace(
                "\n", "°"
            )
        )
        torch.save(model.state_dict(), "model.mod")

    if i % plot_every == 0:
        all_loses.append(loss)

plt.figure()
plt.plot(all_loses)
plt.show()


import math
import os

import matplotlib.pyplot as plt
import numpy as np

from ChartMaker import load_json, get_list_of_attributs
from ChartUtils import get_all_dirs_in, should_skip_file

def roundup(x, roundup=100):
    return int(math.ceil(x / roundup)) * roundup

plt.rcParams.update({'figure.max_open_warning': 0})
plt.rcParams["figure.autolayout"] = True

def get_max_of_files(data):
    max_x = data["logs"][-1]["trained_pop"]

    if "velocity" in data["metadata"]["hof"][0]:
        max_y = data["metadata"]["hof"][0]["velocity"]
    elif "vertpos" in data["metadata"]["hof"][0]:
        max_y = data["metadata"]["hof"][0]["vertpos"]

    return max_x, max_y

def single_population_chart(fig,
                            ax,
                            all_data,
                            show_max_hof=False,
                            x_attribute='trained_pop'):
    data = all_data["logs"]
    type = all_data["metadata"]["type"]
    p_cx = all_data["metadata"]["p_cx"]
    p_mut = all_data["metadata"]["p_mut"]
    popsize = all_data["metadata"]["population_size"]

    x = get_list_of_attributs(x_attribute, data, 0)

    if show_max_hof:
        color = "tab:red"
        if type == "Haploid":
            color = "tab:blue"

        y_hof = get_list_of_attributs('hof_fitness', data, 0)
        ax.plot(x, y_hof, color=color)

    title = f"Haploid vs diploid. p_cx={p_cx} pmut={p_mut}. Popsize={popsize}"
    ax.set_title(title)
    ax.set_xlabel(x_attribute)
    ax.set_ylabel("Fitness (velocity)")
    ax.set_ylim(ymin=0)
    ax.legend()
    ax.set_xlim(xmin=0)

# rootdir = 'data/history/slurm_exp1_fail'
rootdir = 'data'
for dir in get_all_dirs_in(rootdir):
    fig, ax = plt.subplots()

    ax.plot([], [], color='tab:red', label="Diploid")
    ax.plot([], [], color='tab:blue', label="Haploid")

    max_x = 0
    max_y = 0

    for log_file_name in os.listdir(dir):
        if should_skip_file(log_file_name):
            continue

        print(f"Creating {dir}/{log_file_name}")
        f = open(f"{dir}/{log_file_name}")

        try:
            data = load_json(file=f)
        except Exception:
            continue

        single_population_chart(fig, ax, data, show_max_hof=True)

        x, y = get_max_of_files(data)
        if x > max_x:
            max_x = x
        if y > max_y:
            max_y = y

    plt.xlabel(f"{dir}")

    plt.savefig(f'{dir}/all-logs-chart.png')
    plt.show()


import sqlite3

conn = sqlite3.connect("./database/database.db")
c = conn.cursor()
c.execute(
    """
        CREATE TABLE IF NOT EXISTS employes
        (
            prenom text,
            nom text,
            salaire int
        )
    """
)

# employe = {"prenom": "Paul", "nom": "Dupond", "salaire": 10000}
# c.execute("INSERT INTO employes VALUES (:prenom, :nom, :salaire)", employe)

search: dict[str, str] = {"prenom": "Paul"}
c.execute("SELECT * FROM employes WHERE prenom=:prenom", search)
results: list[tuple] = c.fetchall()
print(f"results = {results}")

employe = {"prenom": "Paul", "nom": "Dupond", "salaire": 12000}
c.execute("""
    UPDATE employes SET salaire=:salaire WHERE prenom=:prenom AND nom=:nom
""", employe)

conn.commit()
conn.close()


import matplotlib.pyplot as plt
import numpy as np
plt.style.use('Solarize_Light2')

#TP 1.1 Simulación de Una Ruleta
#Datos

def frecuenciaR(nro, valores):
    frecuenciaA = 0
    for n in valores:
        if n == nro:
            frecuenciaA+= 1
    freqRel= frecuenciaA/len(valores)
    return freqRel

frecuencia= 1/37
esperanza= np.arange(0, 37).mean()  #Media aritmética
desvio= np.arange(0, 37).std() #Desviación Estándar
varianza= np.arange(0, 37).var() #Varianza

t = 30 #número de tiradas
c = 1 #número de corridas
nroEvaluar = 5 #np.random.randint(0,37)

frecuencias= [[0 for x in range(t)] for y in range(c)]
medias= [[0 for x in range(t)] for y in range(c)]
desvios= [[0 for x in range(t)] for y in range(c)]
varianzas= [[0 for x in range(t)] for y in range(c)]

for i in range(0, c):
    numeros= np.random.randint(0, 37, t)
    print(numeros)
    for n in range(0, t):
        lista= numeros[:n+1]
        frecuencias[i][n]= frecuenciaR(nroEvaluar, lista)
        medias[i][n]= lista.mean()
        desvios[i][n]= lista.std()
        varianzas[i][n]= lista.var()

# Graficos
fig, axs = plt.subplots(ncols=2, nrows=2, constrained_layout=True, figsize=[9, 6])

axs[0, 0].set_title('Frecuencia Relativa (fr)')
axs[0, 0].set(xlabel='Tiradas', ylabel='Frecuencia Relativa (fr)')
axs[0, 0].set_ylim(bottom=0, top=max(np.amax(frecuencias), frecuencia)+0.05)
for i in range(0, c):
    axs[0, 0].plot(range(1, t+1), frecuencias[i])#, label='corrida '+str(i+1)+'°')
axs[0, 0].axhline(y=frecuencia, color='b', linestyle='-', label='fre: '+str(round(frecuencia,3)))

axs[0, 1].set_title('Valor Promedio (vp)')
axs[0, 1].set(xlabel='Tiradas', ylabel='Valor Promedio (vp)')
axs[0, 1].set_ylim(bottom=0, top=max(np.amax(medias), esperanza)+1)
for i in range(0, c):
    axs[0, 1].plot(range(1, t+1), medias[i])#, label='corrida '+str(i+1)+'°')
axs[0, 1].axhline(y=esperanza, color='b', linestyle='-', label='vpe: '+str(round(esperanza,3)))

axs[1, 0].set_title('Valor del Desvío (vd)')
axs[1, 0].set(xlabel='Tiradas', ylabel='Valor del Desvío (vd)')
axs[1, 0].set_ylim(bottom=0, top=max(np.amax(desvios), desvio)+1)
for i in range(0, c):
    axs[1, 0].plot(range(1, t+1), desvios[i])#, label='corrida '+str(i+1)+'°')
axs[1, 0].axhline(y=desvio, color='b', linestyle='-', label='vde: '+str(round(desvio,3)))

axs[1, 1].set_title('Valor de la Varianza (vve)')
axs[1, 1].set(xlabel='Tiradas', ylabel='Valor de la Varianza (vv)')
axs[1, 1].set_ylim(bottom=0, top=max(np.amax(varianzas), varianza)+10)
for i in range(0, c):
    axs[1, 1].plot(range(1, t+1), varianzas[i])#, label='corrida '+str(i+1)+'°')
axs[1, 1].axhline(y=varianza, color='b', linestyle='-', label='vve: '+str(round(varianza,3)))

for ax in axs.flat:
    ax.legend()
    ax.set_xlim(left=0, right=t)

plt.show()

"""def graficoProm(resultados, capAcotado):
    plt.title("Distribucion de medias muestrales")
    plt.axhline(capAcotado, color='k', ls="solid")
    plt.plot(resultados, linewidth=1.2)
    plt.xlabel("")
    plt.ylabel("")
    # Me ajustan los x e y
    ax = plt.gca()
    ax.relim()
    ax.autoscale_view()
    # Linea invisible para agregar legend
    plt.axhline(50, color='w', ls="solid", visible=False)
    #dineroIni = 'Capital inicial: ' + str(capAcotado)
    #dineroFin = (f'Capital Final{resultados[-1]}')
    #plt.legend((dineroIni, dineroFin), loc="best")
    plt.ioff()
    plt.show()

prom_med=[]
listas_medias=[]
for i in medias:
    listas_medias.append(i[-1])

graficoProm(resultados=prom_med, capAcotado=0)"""


import torch

from lavis.models import load_model_and_preprocess
from PIL import Image
from typing import List
from werkzeug.datastructures import FileStorage

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
caption_model, caption_vis_processors, _ = load_model_and_preprocess(
    name="blip_caption", model_type="base_coco", is_eval=True, device=device
)

def caption_image(file: FileStorage):
    rgb_image = Image.open(file.stream).convert("RGB")
    preprocessed_image = (
        caption_vis_processors["eval"](rgb_image).unsqueeze(0).to(device)
    )
    generated_captions: List[str] = caption_model.generate(
        {"image": preprocessed_image}
    )
    print(f"generated captions {generated_captions}")
    caption = None
    if len(generated_captions) != 0:
        caption = generated_captions[0]

    return caption

def caption_images(files: List[FileStorage]):
    names = []
    captions = []

    for image in files:
        names.append(image.filename)
        captions.append(caption_image(image))

    return list(zip(names, captions))


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Union
import numpy as np
import pytest

import hidet as hi
from hidet import ops
from hidet.testing import check_unary
from hidet.utils import prod

def check_ternary(
    a_shape, b_shape, c_shape, numpy_op, hidet_op, dtype: Union[str, np.dtype] = np.float32, atol=0.0, rtol=0.0
):
    np.random.seed(1)
    a = np.array(np.random.randn(*a_shape)).astype(dtype)
    b = np.array(np.random.randn(*b_shape)).astype(dtype)
    c = np.array(np.random.randn(*c_shape)).astype(dtype)

    c = np.abs(c)

    numpy_result = numpy_op(a, b, c)
    hidet_args = [hi.asarray(v).cuda() for v in [a, b, c]]
    hidet_result = hidet_op(*hidet_args).cpu().numpy()
    np.testing.assert_allclose(actual=hidet_result, desired=numpy_result, atol=atol, rtol=rtol)

def numpy_batch_norm_2d(
    data: np.ndarray, running_mean: np.ndarray, running_var: np.ndarray, epsilon: float = 1e-5, axis: int = 1
):
    n, c, h, w = data.shape
    assert axis == 1
    assert len(running_mean.shape) == 1 and running_mean.shape[0] == c
    assert len(running_var.shape) == 1 and running_var.shape[0] == c
    running_mean = np.expand_dims(running_mean, axis=(0, 2, 3))
    running_var = np.expand_dims(running_var, axis=(0, 2, 3))
    return (data - running_mean) * np.reciprocal(np.sqrt(running_var + epsilon))

def numpy_instance_norm(data: np.ndarray, epsilon: float = 1e-5) -> np.ndarray:
    dims = tuple(range(2, len(data.shape)))
    mean = data.mean(axis=dims, keepdims=True)
    var = data.var(axis=dims, keepdims=True)
    return (data - mean) / np.sqrt(var + epsilon)

@pytest.mark.parametrize("shape", [[1, 1, 1, 1], [1, 200, 20, 20], [1, 10, 1, 1], [1, 128, 32, 32], [1, 32, 24, 24]])
def test_batch_norm_2d(shape):
    epsilon = 1e-5
    check_ternary(
        a_shape=shape,
        b_shape=[shape[1]],
        c_shape=[shape[1]],
        numpy_op=lambda a, b, c: numpy_batch_norm_2d(a, b, c, epsilon, 1),
        hidet_op=lambda a, b, c: ops.batch_norm_infer(a, b, c, epsilon, 1),
        dtype='float32',
        atol=1e-5,
        rtol=1e-5,
    )

@pytest.mark.parametrize(
    "shape", [[1, 1, 1, 1], [1, 2, 1, 1], [1, 32, 48], [1, 20, 20, 20], [1, 20, 20, 5, 5], [1, 32, 26214]]
)
def test_instance_norm(shape):
    check_unary(shape, numpy_op=numpy_instance_norm, hidet_op=ops.instance_norm, atol=1e-4, rtol=1e-4)


from models.notifications.notification import Notification

class AllianceSelectionNotification(Notification):

    def __init__(self, event, team=None):
        self.event = event
        self.team = team

        # Find alliance for Team
        if self.team and self.event.alliance_selections:
            self.alliance = next((a for a in self.event.alliance_selections if self.team.key_name in a['picks']), None)

    @classmethod
    def _type(cls):
        from consts.notification_type import NotificationType
        return NotificationType.ALLIANCE_SELECTION

    @property
    def fcm_notification(self):
        body = ['{} alliances have been updated.'.format(self.event.normalized_name)]
        # Add alliance information for team
        if self.team and self.alliance:
            sub_body = ['Team {}'.format(self.team.team_number)]
            if self.alliance['picks'][0] == self.team.key_name:
                sub_body.append('is Captain of')
            else:
                sub_body.append('is on')
            # Get alliance number
            alliance_number = self.event.alliance_selections.index(self.alliance) + 1
            sub_body.append('Alliance {} with'.format(alliance_number))
            # [3:] to remove 'frc' prefix
            team_names = ['Team {}'.format(team_key[3:]) for team_key in self.alliance['picks'] if team_key != self.team.key_name]
            # A gorgeous, gramatically correct list comprehension
            # Format is like 'Team 1, Team 2, and Team 3' or just 'Team 2 and Team 3'
            sub_body.append(' and '.join([', '.join(team_names[:-1]), team_names[-1]]) + '.')
            body.append(' '.join(sub_body))

        from firebase_admin import messaging
        return messaging.Notification(
            title='{} Alliances Updated'.format(self.event.event_short.upper()),
            body=' '.join(body)
        )

    @property
    def data_payload(self):
        payload = {
            'event_key': self.event.key_name
        }

        if self.team:
            payload['team_key'] = self.team.key_name

        return payload

    @property
    def webhook_message_data(self):
        from helpers.model_to_dict import ModelToDict
        payload = self.data_payload
        payload['event_name'] = self.event.name
        payload['event'] = ModelToDict.eventConverter(self.event)
        return payload


import torch
import torch.optim as optim
import torch.nn.functional as F
from torch.distributions import Categorical
import gymnasium as gym

from utils import episode_reward_plot
from vpg import compute_returns, ActorNetwork
from a2c import CriticNetwork, compute_advantages, compute_generalized_advantages

class TransitionMemory:
    """Datastructure to store episode transitions and perform return/advantage/generalized advantage calculations (GAE) at the end of an episode."""

    def __init__(self, gamma, lamb, use_gae):

        self.obs_lst = []
        self.action_lst = []
        self.reward_lst = []
        self.value_lst = []
        self.logprob_lst = []
        self.logprob_old_lst = []
        self.return_lst = []
        self.adv_lst = []

        self.gamma = gamma
        self.lamb = lamb
        self.use_gae = use_gae

        self.traj_start = 0

    def put(self, obs, action, reward, logprob, logprob_old, value):
        """Put a transition into the memory."""

        self.obs_lst.append(obs)
        self.action_lst.append(action)
        self.reward_lst.append(reward)
        self.logprob_lst.append(logprob)
        self.logprob_old_lst.append(logprob_old)
        self.value_lst.append(value)

    def get(self):
        """Get all stored transition attributes in the form of lists."""

        return self.obs_lst, self.reward_lst, self.logprob_lst, self.logprob_old_lst, self.value_lst, self.return_lst, self.adv_lst

    def clear(self):
        """Reset the transition memory."""

        self.obs_lst = []
        self.action_lst = []
        self.reward_lst = []
        self.value_lst = []
        self.logprob_lst = []
        self.logprob_old_lst = []
        self.return_lst = []
        self.adv_lst = []

        self.traj_start = 0

    def finish_trajectory(self, next_value):
        """Call on end of an episode. Will perform episode return and advantage or generalized advantage estimation.

        Parameters
        ----------
        next_value:
            The value of the state the episode ended in. Should be 0.0 for terminal state, critic output otherwise.
        """

        reward_traj = self.reward_lst[self.traj_start:]
        value_traj = self.value_lst[self.traj_start:]
        traj_returns = compute_returns(reward_traj, next_value, self.gamma)
        self.return_lst.extend(traj_returns)
        self.traj_start = len(self.obs_lst)

        if self.use_gae:
            traj_adv = compute_generalized_advantages(reward_traj, value_traj, next_value, self.gamma, self.lamb)
        else:
            traj_adv = compute_advantages(traj_returns, value_traj)
        self.adv_lst.extend(traj_adv)

class ProximalPolicyOptimization:
    """The Actor-Critic approach."""

    def __init__(self, env, batch_size=500, gamma=0.99, lamb=0.99, lr=0.01, eps=0.5, use_gae=True):
        """ Constructor.

        Parameters
        ----------
        env : gym.Environment
            The object of the gym environment the agent should be trained in.
        batch_size : int, optional
            Number of transitions to use for one opimization step.
        gamma : float, optional
            Discount factor.
        lamb : float, optional
            Lambda parameters of GAE.
        lr : float, optional
            Learning rate used for actor and critic Adam optimizer.
        """

        if isinstance(env.action_space, gym.spaces.Box):
            raise NotImplementedError('Continous actions not implemented!')

        self.obs_dim, self.act_dim = env.observation_space.shape[0], env.action_space.n
        self.batch_size = batch_size
        self.env = env
        self.eps = eps
        self.memory = TransitionMemory(gamma, lamb, use_gae)

        self.actor_net = ActorNetwork(self.obs_dim, self.act_dim)
        self.critic_net = CriticNetwork(self.obs_dim)
        self.optim_actor = optim.Adam(self.actor_net.parameters(), lr=lr)
        self.optim_critic = optim.Adam(self.critic_net.parameters(), lr=lr)

    def learn(self, total_timesteps):
        """Train the actor-critic.

        Parameters
        ----------
        total_timesteps : int
            Number of timesteps to train the agent for.
        """
        obs, _ = self.env.reset()

        # For plotting
        overall_rewards = []
        episode_rewards = []

        logprob_old = None

        episode_counter = 0
        for timestep in range(1, total_timesteps + 1):

            action, logprob, value = self.predict(obs, train_returns=True)
            obs_, reward, terminated, truncated, info = self.env.step(action)
            episode_rewards.append(reward)

            # Put into transition buffer (only if `logprob_old` exists, i.e. we are not in the first step)
            if logprob_old:
                self.memory.put(obs, action, reward, logprob, logprob_old.detach(), value)
            logprob_old = logprob

            # Update current obs
            obs = obs_

            if terminated or truncated:

                obs, _ = self.env.reset()
                overall_rewards.append(sum(episode_rewards))
                episode_rewards = []
                self.memory.finish_trajectory(0.0)
                logprob_old = None

            if (timestep - episode_counter) == self.batch_size:

                # Finish partial trajectory
                self.memory.finish_trajectory(self.critic_net(torch.Tensor(obs)).item())
                # Get transitions inside memory
                obs_lst, reward_lst, logprob_lst, logprob_old_lst, value_lst, return_lst, adv_lst = self.memory.get()

                # Calculate losses
                actor_loss = self.calc_actor_loss(logprob_lst, logprob_old_lst, adv_lst, eps=self.eps)
                critic_loss = self.calc_critic_loss(value_lst, return_lst)
                loss = actor_loss + critic_loss

                # Back-propagate and optimize
                self.optim_actor.zero_grad()
                self.optim_critic.zero_grad()
                loss.backward()
                self.optim_actor.step()
                self.optim_critic.step()

                # Clear memory
                self.memory.clear()
                episode_counter = timestep

            # Episode reward plot
            if timestep % 500 == 0:
                episode_reward_plot(overall_rewards, timestep, window_size=5, step_size=1)

    @staticmethod
    def calc_critic_loss(value_lst, return_lst):
        """Calculate critic loss for one batch of transitions."""

        return F.mse_loss(torch.Tensor(value_lst), torch.Tensor(return_lst))

    @staticmethod
    def calc_actor_loss(logprob_lst, logprob_old_lst, adv_lst, eps):
        """Calculate actor "loss" for one batch of transitions."""

        adv_list = torch.Tensor(adv_lst)
        ratio = (torch.stack(logprob_lst) - torch.stack(logprob_old_lst)).exp()
        clipped = torch.clamp(ratio, 1-eps, 1+eps) * adv_list
        return -(torch.min(ratio*adv_list, clipped)).mean()

    def predict(self, obs, train_returns=False):
        """Sample the agents action based on a given observation.

        Parameters
        ----------
        obs : numpy.array
            Observation returned by gym environment
        train_returns : bool, optional
            Set to True to get log probability of decided action and predicted value of obs.
        """

        obs = torch.Tensor(obs)
        probs = self.actor_net(obs)
        policy = Categorical(probs=probs)
        action = policy.sample()
        logprob = policy.log_prob(action)

        value = self.critic_net(obs)

        if train_returns:
            return action.item(), logprob, value.item()
        else:
            return action.item()

if __name__ == '__main__':
    env_id = "CartPole-v1"
    use_gae = True

    _env = gym.make(env_id)
    PPO = ProximalPolicyOptimization(_env, use_gae=use_gae)
    PPO.learn(100000)


"""
(c) 2020 Spencer Rose, MIT Licence
Python Landscape Classification Tool (PyLC)
 Reference: An evaluation of deep learning semantic segmentation
 for land cover classification of oblique ground-based photography,
 MSc. Thesis 2020.
 <http://hdl.handle.net/1828/12156>
Spencer Rose <spencerrose@uvic.ca>, June 2020
University of Victoria

Module: HDF5 Database Wrapper
File: database.py
"""
import json
import os
import math
import h5py
import numpy as np
from config import defaults
from utils.tools import confirm_write_file

class DB(object):
    """
     Wrapper class for Hierarchical Data Format
     version 5 (HDF5) database
     - General database operations for image/mask datasets
     - Multiprocessing enabled
     Data model schema:
     - img: images dataset
     - mask: masks dataset
     - meta: metadata (see Profiler for metadata schema)
    """

    def __init__(self, path=None, data=None, partition=None, clip=None):
        """
            Load database from path or from input data array.

            Parameters
            ------
            path: str
                h5 database path.
            data: np.array
                Input data as {'img':img_data, 'mask':mask_data, 'meta':metadata}
            partition: tuple
                Training/validation dataset ratio [optional].
        """

        assert (path is not None or data is not None) and not (path is not None and data is not None), \
            "Database requires either a path or input data to load."

        self.data = None
        self.path = None

        # db dims
        self.partition = partition if partition is not None else (0., 1.)
        self.partition_size = None
        self.img_shape = None
        self.mask_shape = None
        self.clip = clip if clip is not None else defaults.clip

        # db indicies
        self.start = None
        self.end = None
        self.current = None
        self.next = None

        try:
            # load db with input data
            if data is not None:
                self.size = int(self.clip * len(data['img']))
                self.img_shape = data['img'].shape
                self.mask_shape = data['mask'].shape
                self.data = data
            # load db from file
            else:
                assert os.path.exists(path), "Database path {} does not exist."
                self.path = path
                f = self.open()
                self.size = int(self.clip * len(f['img']))
                self.img_shape = f['img'].shape
                self.mask_shape = f['mask'].shape
                f.close()
        except Exception as e:
            print('Error loading database:\n\t{}'.format(data.shape if data is not None else path))
            print(e)

        # partition database for dataset
        self.start = int(math.ceil(self.partition[0] * self.size))
        self.end = int(math.ceil(self.partition[1] * self.size))
        self.partition_size = self.end - self.start

        # initialize buffer size and iterator
        self.buffer_size = min(defaults.buffer_size, self.partition_size)
        self.current = self.start
        self.next = self.current + self.buffer_size

    def __iter__(self):
        """
        Iterator to load indicies for next dataset chunk
        """
        return self

    def __next__(self):
        """
        Iterate next to load indicies for next dataset chunk
        """
        if self.current == self.end:
            # reset index pointer
            raise StopIteration

        # iterate; if last chunk, truncate
        db_sl = (np.s_[self.current:self.next], self.next - self.current)
        self.current = self.next
        self.next = self.next + self.buffer_size if self.next + self.buffer_size < self.end else self.end

        return db_sl

    def reset(self):
        """
        Reset database index pointers
        """
        self.current = self.start
        self.next = self.current + self.buffer_size

    def __len__(self):
        return self.size

    def init_worker(self, worker_id, n_workers):
        """
         Partition buffer per worker thread.

         Parameters
         -------
         worker_id: int
             Worker pool id.
         n_workers: int
            Number of workers.
         """
        per_worker = int(math.ceil(self.partition_size / float(n_workers)))
        self.start += worker_id * per_worker
        self.end = min(self.start + per_worker, self.end)
        self.start = self.end if self.end < self.start else self.start
        self.partition_size = self.end - self.start

        # update buffer size and iterator
        self.buffer_size = min(defaults.buffer_size, self.partition_size)
        self.current = self.start
        self.next = self.current + self.buffer_size

    def get_meta(self):
        """
        Get metadata attribute from dataset.

        Returns
        -------
        attr: Parameters
            Database metadata.
        """
        if self.path:
            f = self.open()
            attr = f.attrs.get('meta')
            f.close()
            return defaults.update(json.loads(attr))
        else:
            return self.data['meta']

    def get_data(self, dset_key):
        """
        Get entire dataset from database by key. (Use for testing only)

        Parameters
        ------
        dset_key: str
            Dataset key.

        Returns
        -------
        data: np.array
            Dataset array.
        """
        if self.path:
            f = self.open()
            data = f[dset_key][()]
            f.close()
            return data
        else:
            return self.data[dset_key]

    def open(self):
        """
        Open dataset file pointer. Uses H5PY Single-Writer/Multiple-Reader (SWMR).
        """
        return h5py.File(self.path, mode='r', libver='latest', swmr=True)

    def save(self, file_path):
        """
        Saves data buffer to HDF5 database file.

        Parameters
        ------
        file_path: str
            Database file path.
        """

        assert file_path is not None, "File path must be specified to save data to database."

        if len(self.data['img']) == 0 or len(self.data['mask']) == 0:
            print('\n --- Note: Image or mask data is empty.\n')

        n_samples = len(self.data['img'])
        print('\nSaving buffer to database ... ')

        if confirm_write_file(file_path):
            print('\nCopying {} samples to:\n\t{}  '.format(n_samples, file_path))
            with h5py.File(file_path, 'w') as f:
                # create image dataset partition
                f.create_dataset(
                    "img",
                    self.data['img'].shape,
                    compression='gzip',
                    chunks=True,
                    data=self.data['img']
                )
                # create masks dataset partition
                f.create_dataset(
                    "mask",
                    self.data['mask'].shape,
                    compression='gzip',
                    chunks=True,
                    data=self.data['mask']
                )
                # include dataset metadata as new attribute to database
                # - store metadata as JSON string
                f.attrs['meta'] = json.dumps(vars(self.data['meta']))
                f.close()
                print('File saved.')
        else:
            print('Database was not saved.')

    def print_meta(self):
        """
        Print database metadata to console.
        """

        # get database metadata
        meta = self.get_meta()

        hline = '-' * 40
        print('\nDatabase Configuration')
        print(hline)
        print('{:30s} {}'.format('Database ID', meta.id))
        print('{:30s} {} ({})'.format('Channels', meta.ch, 'Grayscale' if meta.ch == 1 else 'Colour'))
        print('{:30s} {}px x {}px'.format('Tile size (WxH)', meta.tile_size, meta.tile_size))
        print('{:30s} {}'.format('Database Size', self.size))
        print('{:30s} {}'.format('Partition', self.partition))
        print(' - {:27s} {}'.format('Size', self.partition_size))
        print(' - {:27s} {}'.format('Start Index', self.start))
        print(' - {:27s} {}'.format('End Index', self.end))
        print('{:30s} {} ({})'.format('Buffer Size (Default)', self.buffer_size, defaults.buffer_size))
        print('{:30s} {}'.format('Clipping', meta.clip))
        print(hline)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import re
import argparse

def printError(message):
    if not _quiet:
        print('\033[91m[ERROR]\033[0m ' + str(message))
        exit(1)

def parseArgs():
    parser = argparse.ArgumentParser()
    parser.add_argument('template', help='template file')
    parser.add_argument('-q', '--quiet', action='store_true', help='be quiet, no error message')
    parser.add_argument('-s', '--seed', type=int, help='set seed value')
    return parser.parse_args()

def fetchContent(templateFile):
    try:
        with open(templateFile) as f:
            return f.read().splitlines()
    except FileNotFoundError as err:
        printError(err)

def generateData(templateContent):
    from faker import Faker
    fake = Faker()
    if _seed:
        fake.seed_instance(_seed)

    try:
        from customprovider import CustomProvider
        fake.add_provider(CustomProvider)
    except (ModuleNotFoundError, ImportError):
        pass

    pattern = r'\{\{\w+\([\w=,\s?#\'\":-]*\)\}\}'

    for line in templateContent:
        num = len(re.findall(pattern, line))
        if num != 0:
            newline = line
            for i in range(1, num+1):
                item = re.findall(pattern, newline)
                try:
                    newline = newline.replace(item[0], str(eval('fake.' + re.sub(r'[\{\}]', '', item[0]))), 1)
                except AttributeError as err:
                    printError(err)
            print(newline)
        else:
            print(line)

def main():
    args = parseArgs()

    global _quiet
    global _seed
    _seed = args.seed or 0
    _quiet = args.quiet or False

    template = str(args.template)
    content = fetchContent(template)
    generateData(content)

if __name__ == '__main__':
    main()


import re, collections, csv, requests, tqdm, pickle

class JudgmentsProcessor():

    def __init__(self, raw_judgments, raw_signatures, most_common_words_tagged_file_path, most_common_words_nontagged_file_path):
        self.__raw_judgments__ = raw_judgments
        self.__raw_signatures__ = raw_signatures
        self.__most_common_words_tagged_file_path__ = most_common_words_tagged_file_path
        self.__most_common_words_nontagged_file_path__ = most_common_words_nontagged_file_path
        self.__signatures_regexes__ = {
            re.compile(r'A?C.*'): 'sprawy cywilne',
            re.compile(r'A?U.*'): 'sprawy z zakresu ubezpieczenia społecznego',
            re.compile(r'A?K.*'): 'sprawy karne',
            re.compile(r'G.*'): 'sprawy gospodarcze',
            re.compile(r'A?P.*'): 'sprawy w zakresie prawa pracy',
            re.compile(r'R.*'): 'sprawy w zakresie prawa rodzinnego',
            re.compile(r'W.*'): 'sprawy o wykroczenia',
            re.compile(r'Am.*'): 'sprawy w zakresie prawa konkurencji'
        }

    def process_judgments(self):
        '''
        Transforms judgments and signatures, removing obsolete words,
        matching signatures codes to signatures regex map,
        removing most common words from judgments texts and removing underrepresented signatures
        :return:
        X, Y: X is a list of judgments texts, Y is a list of corresponding signatures
        '''
        signatures_categories = self.__extract_signatures_categories__()
        judgments_signatures = list(zip(self.__raw_judgments__, signatures_categories))
        judgments_signatures_after_matching_signatures, not_matching_signatures = self.__match_signatures__(judgments_signatures)
        judgments_signatures = self.__filter_non_matching_codes__(judgments_signatures_after_matching_signatures)
        judgments_categories_counter = self.__count_judgments_categories__(judgments_signatures)
        judgments_signatures = self.__filter_categories_with_few_occurences__(judgments_signatures)
        judgments_signatures = self.__filter__all_decisions_containing_reason__(judgments_signatures)

        tagged_judgments_signatures = []
        non_tagged_judgments_signatures = []
        print ("Before tagging:" + str(len(judgments_signatures)))
        ###############################
        for pair in tqdm.tqdm(judgments_signatures):
            result = self.__make_query_to_tager__(pair)
            if result[0]:
                non_tagged_judgments_signatures.append(pair)
                tagged_judgments_signatures.append(result)
        ###############################
        print ("After tagging:"+ str(len(non_tagged_judgments_signatures)))
        print ("After tagging:" + str(len(tagged_judgments_signatures)))

        print ("Saving judgments to files")

        with open("non_tagged_judgments.pickle", "wb") as file:
            pickle.dump(non_tagged_judgments_signatures, file)

        with open("tagged_judgments_signatures.pickle", "wb") as file:
            pickle.dump(tagged_judgments_signatures, file)

        judgments_categories_counter = self.__count_judgments_categories__(tagged_judgments_signatures)

        most_common_words_tagged = self.__extract_most_common_words__(self.__most_common_words_tagged_file_path__)
        most_common_words_non_tagged = self.__extract_most_common_words__(self.__most_common_words_nontagged_file_path__)

        tagged_judgments_signatures =  self.__remove_most_common_words_from_judgments__(
            tagged_judgments_signatures, most_common_words_tagged)
        non_tagged_judgments_signatures = self.__remove_most_common_words_from_judgments__(
            non_tagged_judgments_signatures, most_common_words_non_tagged)

        X_tagged_tmp, Y_tagged, X_non_tagged_tmp, Y_non_tagged = [], [], [], []
        X_tagged, X_non_tagged = [], []

        for pair in tagged_judgments_signatures:
            X_tagged_tmp.append(pair[0])
            Y_tagged.append(pair[1])

        for sublist in X_tagged_tmp:
            words = " ".join(sublist)
            X_tagged.append(words)

        for pair in non_tagged_judgments_signatures:
            X_non_tagged_tmp.append(pair[0])
            Y_non_tagged.append(pair[1])

        for sublist in X_non_tagged_tmp:
            words = " ".join(sublist)
            X_non_tagged.append(words)

        return X_tagged, Y_tagged, X_non_tagged, Y_non_tagged, judgments_categories_counter

    def __make_query_to_tager__(self, judgment_signature_content):
        data_to_query = (",").join(judgment_signature_content[0])
        r = requests.post(data=data_to_query.encode("utf-8"), url="http://localhost:9200")
        response_text = r.text
        splited_response = response_text.splitlines()
        splited_response = [" ".join(x.replace("\t", " ").replace("none", "")[1:].split(":")[:2][:1]).replace(" ", ":")
                            for x in splited_response if ":" in x]
        splited_response = [x.split(":")[0] for x in splited_response]
        return (splited_response, judgment_signature_content[1])

    def __extract_signatures_categories__(self):
        signatures_flatten = [item for sublist in self.__raw_signatures__ for item in sublist]
        signatures_categories = []
        for signature in signatures_flatten:
            splited = signature.split(" ")
            signatures_categories.append(splited[1])
        return signatures_categories

    def __match_signatures_to_code__(self, code):
        for judg_regex in self.__signatures_regexes__.keys():
            if (re.match(judg_regex, code)):
                return self.__signatures_regexes__[judg_regex]
        return None

    def __match_signatures__(self, judgments_and_signatures_list):
        judgments_and_signatures_list_after_transformation = []
        not_matching_signatures = []
        for pair in judgments_and_signatures_list:
            code = pair[1]
            res = self.__match_signatures_to_code__(code)
            judgments_and_signatures_list_after_transformation.append((pair[0], res))
            if res is None:
                not_matching_signatures.append(code)
        return judgments_and_signatures_list_after_transformation, not_matching_signatures

    def __filter_non_matching_codes__(self, judgments_and_signatures_list):
        result = list(filter(lambda x: x[1] is not None,
                                           judgments_and_signatures_list))
        return result

    def __count_judgments_categories__(self, judgments_signatures):
        cases_types = [case[1] for case in judgments_signatures]
        cases_type_counter = collections.Counter(cases_types)
        return cases_type_counter

    def __filter_categories_with_few_occurences__(self, judgments_signatures):
        filtered_judgments_signatures = []
        for pair in judgments_signatures:
            if pair[1] != 'sprawy w zakresie prawa konkurencji':
                filtered_judgments_signatures.append(pair)
        return filtered_judgments_signatures

    def __filter_decision_containing_reason__(self, decision):
        try:
            idx = decision.index('uzasadnienie')
            new_decision = decision[idx + 1:]
            new_decision = [word for word in new_decision
                            if (not word.isdigit() and len(word) > 1)]
            return new_decision
        except ValueError:
            return None

    def __filter__all_decisions_containing_reason__(self, judgs_signs):
        filtered_judgments_signatures = []
        for pair in judgs_signs:
            decision = pair[0]
            new_decision = self.__filter_decision_containing_reason__(decision)
            if new_decision:
                filtered_judgments_signatures.append((new_decision, pair[1]))
        return filtered_judgments_signatures

    def __extract_most_common_words__(self, path):
        most_common_words = []
        with open(path) as file:
            idx, number_of_rows = 0, 20
            csv_reader = csv.reader(file)
            for row in csv_reader:
                word = ",".join(row).split(",")[1]
                most_common_words.append(word)
                idx += 1
                if idx >= number_of_rows:
                    break
        print (most_common_words)
        return most_common_words

    def __remove_most_common_words_from_judgments__(self, judgments_signatures, most_common_words_list):
        filtered_judgments = []
        for pair in judgments_signatures:
            judgment = [word for word in pair[0] if word not in most_common_words_list]
            filtered_judgments.append((judgment, pair[1]))
        return filtered_judgments