import sys
import requests
import os

class ImageDownload:
    def __init__(self, img, id):
        self.images = img
        self.id = id

    def GetLinks(self):
        return [x['src'] for x in self.images]

    def SaveToDisk(self):
        try:
            # creating the folder based on their ids
            os.makedirs(f"C://Users/cojoc/PycharmProjects/scraping/photos/{self.id}")
            for i, image in enumerate(self.GetLinks()):  # using enumerate to index through the links of photos
                r = requests.get(image, stream=True)  # reuqesting to get the image
                # This will allow us to save the images in the folder based by their ids
                with open(f'C://Users/cojoc/PycharmProjects/scraping/photos/{self.id}/image_' + str(i) + '.jpg',
                          'wb') as f:
                    for chunk in r:
                        f.write(chunk)
        except:
            print("The Folder already exists")
            sys.exit()  # exiting the program if the folder with the car's id already exist


from pydantic import BaseModel
from datetime import date

class Metrics(BaseModel):
    date: date
    channel: str
    country: str
    os: str
    impressions: int
    clicks: int
    installs: int
    spend: float
    revenue: float
    cpi: float

    class Config:
        orm_mode = True


import zope.interface
import zope.component
from zope import schema

from Products.CMFPlone import PloneMessageFactory as _

OVERRIDE_RESOURCE_DIRECTORY_NAME = 'resource_overrides'

class IResourceRegistry(zope.interface.Interface):

    url = schema.ASCIILine(
        title=_(u"Resources base URL"),
        required=False)

    js = schema.ASCIILine(
        title=_(u"Main js file"),
        required=False)

    css = schema.List(
        title=_(u"CSS/LESS files"),
        value_type=schema.ASCIILine(title=_(u"URL")),
        default=[],
        required=False)

    init = schema.ASCIILine(
        title=_(u"Init instruction for shim"),
        required=False)

    deps = schema.ASCIILine(
        title=_(u"Dependencies for shim"),
        description=_(u"Comma separated values of resource for shim"),
        required=False)

    export = schema.ASCIILine(
        title=_(u"Export vars for shim"),
        required=False)

    conf = schema.Text(
        title=_(u"Configuration in JSON for the widget"),
        description=_(u"Should be accessible on @@getWCconfig?id=name"),
        required=False)

class IBundleRegistry(zope.interface.Interface):

    jscompilation = schema.ASCIILine(
        title=_(u"URL of the last js compilation"),
        required=False)

    csscompilation = schema.ASCIILine(
        title=_(u"URL of the last css compilation"),
        required=False)

    last_compilation = schema.Datetime(
        title=_(u"Last compiled date"),
        description=_(u"Date time of the last compilation of this bundle"),
        required=False)

    expression = schema.ASCIILine(
        title=_(u"Expression to render"),
        description=_(u"In case its a bundle we can have a condition to render it"),
        required=False)

    conditionalcomment = schema.ASCIILine(
        title=_(u"Conditional comment"),
        description=_(u"In case you want to render this resource on conditional comment"),
        required=False)

    resources = schema.List(
        title=_(u"Loaded resources"),
        description=_(u"The resources that is going to be loaded on this bundle in order"),
        value_type=schema.ASCIILine(title=_(u"Resource name")),
        required=False)

    enabled = schema.Bool(
        title=_(u"It's enabled?"),
        default=True,
        required=False)

    compile = schema.Bool(
        title=_(u"Does your bundle contains any RequireJS or LESS file?"),
        description=_(u"If its true and you modify this bundle you need to build it before production"),
        default=True,
        required=False)

    depends = schema.ASCIILine(
        title=_(u"Depends on another bundle"),
        description=_(u"In case you want to be the last: *, in case its the first should be empty"),
        required=False)

    develop_javascript = schema.Bool(
        title=_(u'Develop JavaScript'),
        default=False)

    develop_css = schema.Bool(
        title=_(u'Develop CSS'),
        default=False)


from sonzo.telnet import TelnetServer, TelnetProtocol
import logging
import time

LMAGENTA = chr(27) + "[1;35m"
WHITE    = chr(27) + "[37m"
LGREEN   = chr(27) + "[1;32m"
LOGIN    = "\n\r\n\r\n\r                             {}Welcome to Sonzo Chat!\n\r\n\r{}"

class ChatClient(TelnetProtocol):
    """
    Custom server side client object inherited from sonzo.SonzoClient.

    This client IS NOT an actual client side telnet client.
    """
    def _init_(self, sock, addr):
        # Overridden medthod
#        SonzoClient.__init__(self, sock, addr)
        self.name = addr

    def onConnect(self):
        # Overridden medthod
        logging.info(" {} has connected.".format(self.addrport()))

        for user in USERLIST:
            systemMessage(user, "{} has joined the chat!\n\r".format(self.addrport()))
        USERLIST.append(self)
        systemMessage(self, LOGIN.format(LMAGENTA, WHITE))

    def onDisconnect(self):
        # Over-ridden medthod
        logging.info(" {} disconnecting.".format(self.addrport()))
        USERLIST.remove(self)
        for user in USERLIST:
            if user is not self:
                systemMessage(user, "{} logged off.\n\r".format(self.addrport()))

    def dataRecieved(self, data):
        """
        Overriden function.  Called when data is recieved from client.
        """
        chat(self, data)

    def disconnect(self):
        """
        Disconnect user.
        """
        systemMessage(self, "Goodbye!\n\r")
        self._new_messages = False
        self._connected = False

def color(c, color):
    if c._ansi:
        return color
    else:
        return ""

def chat(client, msg):
    #Check to see if someone issues a command.
    if msg.startswith("=a".lower()):
        client.setANSIMode()
        systemMessage(client, "ANSI: {}\n\r".format(client._ansi))
        logging.info(" {} changing ANSI to: {}.".format(client.addrport(), client._ansi))
        return
    if msg.startswith("/quit".lower()):
        client.disconnect()
        return
    if msg.startswith("~".lower()):
        logging.info(" {} changing character mode to: {}.".format(client.addrport(), client._character_mode))
        client.setCharacterMode()
        client.systemMessage("Character Mode is now: {}\n\r".format(client._character_mode))
        return
    if msg.startswith("/runlater".lower()):
        chatsrvr.callLater("Ran 2 seconds later.", func=print, runtime=2)
    if msg.startswith("/install".lower()):
        chatsrvr.install("Fart!", func=print)
    # If no command, say it in the chat room.
    for c in USERLIST:
        sendMessage(client, c, msg)

def sendMessage(sender, client, message):
    message = "{}{} says, {}{}".format(color(sender, LGREEN), sender.addrport(), color(client, WHITE), message)
    client.send(message)

def systemMessage(client, message):
    client.send(message)

if __name__ == '__main__':
    logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
    USERLIST = []
    chatclient = ChatClient
    chatsrvr = TelnetServer(clientclass=chatclient, address='', port=23)
    logging.info(" Sonzo Chat Server starting up...")
    # Example of adding a looping call.
    tensecondloop = chatsrvr.loopingCall("Looping at 10 seconds", func=print)
    tensecondloop.start(10)

    chatsrvr.run()


# -*- coding: utf-8 -*-

import wx.animate

###########################################################################
## Class main_frame
###########################################################################

class main_frame ( wx.Frame ):

	def __init__( self, parent ):
		wx.Frame.__init__ ( self, parent, id = wx.ID_ANY, title = u"Partition Text", pos = wx.DefaultPosition, size = wx.Size( 700,500 ), style = wx.CAPTION|wx.CLOSE_BOX|wx.MINIMIZE_BOX|wx.SYSTEM_MENU|wx.TAB_TRAVERSAL )

		self.SetSizeHintsSz( wx.DefaultSize, wx.DefaultSize )

		sizer_main = wx.BoxSizer( wx.VERTICAL )

		self.panel = wx.Panel( self, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize, wx.TAB_TRAVERSAL )
		bSizer3 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer41 = wx.BoxSizer( wx.VERTICAL )

		bSizer27 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer271 = wx.BoxSizer( wx.VERTICAL )

		bSizer23 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer24 = wx.BoxSizer( wx.VERTICAL )

		self.filename_label = wx.StaticText( self.panel, wx.ID_ANY, u"File", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.filename_label.Wrap( -1 )
		bSizer24.Add( self.filename_label, 0, wx.ALL|wx.EXPAND, 5 )

		bSizer23.Add( bSizer24, 1, wx.ALL|wx.EXPAND, 5 )

		bSizer25 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer241 = wx.BoxSizer( wx.VERTICAL )

		self.file_box = wx.TextCtrl( self.panel, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, wx.TE_NO_VSCROLL )
		bSizer241.Add( self.file_box, 0, wx.ALL|wx.EXPAND, 5 )

		bSizer25.Add( bSizer241, 1, wx.EXPAND, 5 )

		bSizer251 = wx.BoxSizer( wx.VERTICAL )

		self.file_open_button = wx.Button( self.panel, wx.ID_ANY, u"...", wx.DefaultPosition, wx.Size( 20,-1 ), 0 )
		bSizer251.Add( self.file_open_button, 0, wx.ALL, 5 )

		bSizer25.Add( bSizer251, 0, wx.EXPAND, 5 )

		bSizer23.Add( bSizer25, 3, wx.EXPAND, 5 )

		bSizer271.Add( bSizer23, 0, wx.EXPAND, 5 )

		bSizer19 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer20 = wx.BoxSizer( wx.VERTICAL )

		self.output_folder_label = wx.StaticText( self.panel, wx.ID_ANY, u"Output Folder", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.output_folder_label.Wrap( -1 )
		bSizer20.Add( self.output_folder_label, 0, wx.ALL|wx.EXPAND, 5 )

		bSizer19.Add( bSizer20, 1, wx.ALL|wx.EXPAND, 5 )

		bSizer22 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer28 = wx.BoxSizer( wx.VERTICAL )

		self.output_folder_box = wx.TextCtrl( self.panel, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, wx.TE_NO_VSCROLL )
		bSizer28.Add( self.output_folder_box, 0, wx.ALL|wx.EXPAND, 5 )

		bSizer22.Add( bSizer28, 1, wx.EXPAND, 5 )

		bSizer29 = wx.BoxSizer( wx.VERTICAL )

		self.output_folder_browse = wx.Button( self.panel, wx.ID_ANY, u"...", wx.DefaultPosition, wx.Size( 20,-1 ), 0 )
		bSizer29.Add( self.output_folder_browse, 0, wx.ALL, 5 )

		bSizer22.Add( bSizer29, 0, wx.EXPAND, 5 )

		bSizer19.Add( bSizer22, 3, wx.EXPAND, 5 )

		bSizer271.Add( bSizer19, 0, wx.EXPAND, 5 )

		bSizer7 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer8 = wx.BoxSizer( wx.VERTICAL )

		self.staticText_linesperfile = wx.StaticText( self.panel, wx.ID_ANY, u"Lines Per File", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.staticText_linesperfile.Wrap( -1 )
		self.staticText_linesperfile.Hide()

		bSizer8.Add( self.staticText_linesperfile, 0, wx.ALL|wx.EXPAND, 5 )

		self.staticText_numberofparts = wx.StaticText( self.panel, wx.ID_ANY, u"Number Of Parts", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.staticText_numberofparts.Wrap( -1 )
		bSizer8.Add( self.staticText_numberofparts, 0, wx.ALL|wx.EXPAND, 5 )

		bSizer7.Add( bSizer8, 1, wx.ALL|wx.EXPAND, 5 )

		bSizer10 = wx.BoxSizer( wx.VERTICAL )

		self.spinCtrl_divisionParameter = wx.SpinCtrl( self.panel, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, wx.SP_ARROW_KEYS|wx.TE_PROCESS_ENTER, 1, 9999, 1 )
		bSizer10.Add( self.spinCtrl_divisionParameter, 0, wx.ALL, 5 )

		bSizer7.Add( bSizer10, 3, wx.EXPAND, 5 )

		bSizer271.Add( bSizer7, 0, wx.EXPAND, 5 )

		bSizer27.Add( bSizer271, 2, wx.EXPAND, 5 )

		bSizer281 = wx.BoxSizer( wx.HORIZONTAL )

		self.m_staticline1 = wx.StaticLine( self.panel, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize, wx.LI_VERTICAL )
		bSizer281.Add( self.m_staticline1, 0, wx.EXPAND|wx.LEFT, 5 )

		bSizer282 = wx.BoxSizer( wx.VERTICAL )

		bSizer291 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer33 = wx.BoxSizer( wx.HORIZONTAL )

		self.m_staticText6 = wx.StaticText( self.panel, wx.ID_ANY, u"Size", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.m_staticText6.Wrap( -1 )
		bSizer33.Add( self.m_staticText6, 0, wx.ALL, 5 )

		self.file_size_diplay = wx.StaticText( self.panel, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0 )
		self.file_size_diplay.Wrap( -1 )
		bSizer33.Add( self.file_size_diplay, 1, wx.ALL, 5 )

		bSizer291.Add( bSizer33, 1, wx.EXPAND, 5 )

		bSizer34 = wx.BoxSizer( wx.HORIZONTAL )

		self.m_staticText8 = wx.StaticText( self.panel, wx.ID_ANY, u"Lines", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.m_staticText8.Wrap( -1 )
		bSizer34.Add( self.m_staticText8, 0, wx.ALL, 5 )

		self.lines_label = wx.StaticText( self.panel, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0 )
		self.lines_label.Wrap( -1 )
		bSizer34.Add( self.lines_label, 1, wx.ALL, 5 )

		self.animCtrl_scanner = wx.animate.AnimationCtrl( self.panel, wx.ID_ANY, wx.animate.NullAnimation, wx.DefaultPosition, wx.DefaultSize, wx.animate.AC_DEFAULT_STYLE )
		self.animCtrl_scanner.LoadFile( u"C:\\Users\\Dibyendu\\Workspace\\Text_Splitter\\icon\\scanning.GIF" )

		self.animCtrl_scanner.Play()
		self.animCtrl_scanner.Hide()

		bSizer34.Add( self.animCtrl_scanner, 0, wx.ALL|wx.EXPAND, 5 )

		bSizer291.Add( bSizer34, 1, wx.EXPAND, 5 )

		bSizer282.Add( bSizer291, 0, wx.ALL|wx.EXPAND, 5 )

		bSizer30 = wx.BoxSizer( wx.VERTICAL )

		radioBox_divideoptionChoices = [ u"Number Of Parts", u"Number Of Lines" ]
		self.radioBox_divideoption = wx.RadioBox( self.panel, wx.ID_ANY, u"Divide According To", wx.DefaultPosition, wx.DefaultSize, radioBox_divideoptionChoices, 1, wx.RA_SPECIFY_COLS )
		self.radioBox_divideoption.SetSelection( 0 )
		bSizer30.Add( self.radioBox_divideoption, 0, wx.ALL, 5 )

		bSizer282.Add( bSizer30, 1, wx.EXPAND, 5 )

		bSizer32 = wx.BoxSizer( wx.VERTICAL )

		bSizer282.Add( bSizer32, 1, wx.EXPAND, 5 )

		bSizer281.Add( bSizer282, 1, wx.EXPAND, 5 )

		bSizer27.Add( bSizer281, 1, wx.EXPAND, 5 )

		bSizer41.Add( bSizer27, 0, wx.EXPAND, 5 )

		bSizer11 = wx.BoxSizer( wx.VERTICAL )

		self.split_button = wx.Button( self.panel, wx.ID_ANY, u"&Split", wx.DefaultPosition, wx.DefaultSize, 0 )
		bSizer11.Add( self.split_button, 0, wx.ALIGN_CENTER|wx.ALL, 5 )

		bSizer41.Add( bSizer11, 0, wx.EXPAND, 5 )

		sbSizer1 = wx.StaticBoxSizer( wx.StaticBox( self.panel, wx.ID_ANY, u"File Preview" ), wx.VERTICAL )

		self.text_display = wx.TextCtrl( self.panel, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, wx.TE_MULTILINE|wx.TE_READONLY )
		sbSizer1.Add( self.text_display, 1, wx.EXPAND, 5 )

		bSizer41.Add( sbSizer1, 1, wx.EXPAND, 0 )

		self.progress_bar = wx.Gauge( self.panel, wx.ID_ANY, 100, wx.DefaultPosition, wx.Size( -1,20 ), wx.GA_HORIZONTAL|wx.GA_SMOOTH )
		self.progress_bar.SetValue( 0 )
		bSizer41.Add( self.progress_bar, 0, wx.ALL|wx.EXPAND, 0 )

		bSizer3.Add( bSizer41, 1, wx.ALL|wx.EXPAND, 5 )

		self.panel.SetSizer( bSizer3 )
		self.panel.Layout()
		bSizer3.Fit( self.panel )
		sizer_main.Add( self.panel, 1, wx.ALL|wx.EXPAND, 0 )

		self.SetSizer( sizer_main )
		self.Layout()
		self.statusbar = self.CreateStatusBar( 1, wx.ST_SIZEGRIP, wx.ID_ANY )
		self.menubar = wx.MenuBar( 0 )
		self.file_menu = wx.Menu()
		self.file_menu_open = wx.MenuItem( self.file_menu, wx.ID_ANY, u"&Open"+ u"\t" + u"Ctrl+O", u"Open file to split or extract from..", wx.ITEM_NORMAL )
		self.file_menu_open.SetBitmap( wx.Bitmap( u"icon/open_icon.png", wx.BITMAP_TYPE_ANY ) )
		self.file_menu.AppendItem( self.file_menu_open )

		self.file_menu_exit = wx.MenuItem( self.file_menu, wx.ID_ANY, u"&Exit"+ u"\t" + u"ALT+F4", u"Exit Partition Text", wx.ITEM_NORMAL )
		self.file_menu_exit.SetBitmap( wx.Bitmap( u"icon/close_icon.png", wx.BITMAP_TYPE_ANY ) )
		self.file_menu.AppendItem( self.file_menu_exit )

		self.menubar.Append( self.file_menu, u"&File" )

		self.help_menu = wx.Menu()
		self.help_menu_about = wx.MenuItem( self.help_menu, wx.ID_ANY, u"&About"+ u"\t" + u"F1", u"About Partition Text", wx.ITEM_NORMAL )
		self.help_menu_about.SetBitmap( wx.Bitmap( u"icon/about_icon.png", wx.BITMAP_TYPE_ANY ) )
		self.help_menu.AppendItem( self.help_menu_about )

		self.menubar.Append( self.help_menu, u"&Help" )

		self.SetMenuBar( self.menubar )

		self.Centre( wx.BOTH )

		# Connect Events
		self.file_open_button.Bind( wx.EVT_BUTTON, self.OnOpenFile )
		self.output_folder_browse.Bind( wx.EVT_BUTTON, self.OnOutputFolderSelect )
		self.spinCtrl_divisionParameter.Bind( wx.EVT_TEXT_ENTER, self.on_spinCtrl_edit )
		self.radioBox_divideoption.Bind( wx.EVT_RADIOBOX, self.on_divide_mode_select )
		self.split_button.Bind( wx.EVT_BUTTON, self.OnSplit )
		self.Bind( wx.EVT_MENU, self.OnOpenFile, id = self.file_menu_open.GetId() )
		self.Bind( wx.EVT_MENU, self.OnExit, id = self.file_menu_exit.GetId() )
		self.Bind( wx.EVT_MENU, self.OnAbout, id = self.help_menu_about.GetId() )

	def __del__( self ):
		pass

	# Virtual event handlers, overide them in your derived class
	def OnOpenFile( self, event ):
		event.Skip()

	def OnOutputFolderSelect( self, event ):
		event.Skip()

	def on_spinCtrl_edit( self, event ):
		event.Skip()

	def on_divide_mode_select( self, event ):
		event.Skip()

	def OnSplit( self, event ):
		event.Skip()

	def OnExit( self, event ):
		event.Skip()

	def OnAbout( self, event ):
		event.Skip()

###########################################################################
## Class about_frame
###########################################################################

class about_frame ( wx.Dialog ):

	def __init__( self, parent ):
		wx.Dialog.__init__ ( self, parent, id = wx.ID_ANY, title = u"About Partition Text", pos = wx.DefaultPosition, size = wx.Size( 232,178 ), style = wx.DEFAULT_DIALOG_STYLE|wx.STAY_ON_TOP )

		self.SetSizeHintsSz( wx.DefaultSize, wx.DefaultSize )

		bSizer12 = wx.BoxSizer( wx.VERTICAL )

		self.m_panel4 = wx.Panel( self, wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize, wx.TAB_TRAVERSAL )
		bSizer13 = wx.BoxSizer( wx.VERTICAL )

		bSizer121 = wx.BoxSizer( wx.HORIZONTAL )

		bSizer15 = wx.BoxSizer( wx.VERTICAL )

		self.m_bitmap1 = wx.StaticBitmap( self.m_panel4, wx.ID_ANY, wx.Bitmap( u"icon/about_pic.png", wx.BITMAP_TYPE_ANY ), wx.DefaultPosition, wx.DefaultSize, 0 )
		bSizer15.Add( self.m_bitmap1, 0, wx.ALL, 5 )

		bSizer121.Add( bSizer15, 1, wx.EXPAND, 5 )

		bSizer17 = wx.BoxSizer( wx.VERTICAL )

		self.m_staticText7 = wx.StaticText( self.m_panel4, wx.ID_ANY, u"PARTITION TEXT\n    Version 1.0", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.m_staticText7.Wrap( -1 )
		self.m_staticText7.SetFont( wx.Font( wx.NORMAL_FONT.GetPointSize(), 70, 90, 92, False, "Arial" ) )

		bSizer17.Add( self.m_staticText7, 0, wx.ALL, 10 )

		bSizer121.Add( bSizer17, 3, wx.EXPAND, 0 )

		bSizer13.Add( bSizer121, 1, wx.EXPAND, 0 )

		bSizer14 = wx.BoxSizer( wx.VERTICAL )

		self.m_staticText8 = wx.StaticText( self.m_panel4, wx.ID_ANY, u"Created By : Dibyendu Das", wx.DefaultPosition, wx.DefaultSize, 0 )
		self.m_staticText8.Wrap( -1 )
		bSizer14.Add( self.m_staticText8, 0, wx.ALIGN_CENTER_HORIZONTAL|wx.ALL, 5 )

		bSizer13.Add( bSizer14, 1, wx.EXPAND, 5 )

		self.m_panel4.SetSizer( bSizer13 )
		self.m_panel4.Layout()
		bSizer13.Fit( self.m_panel4 )
		bSizer12.Add( self.m_panel4, 1, wx.EXPAND |wx.ALL, 10 )

		self.SetSizer( bSizer12 )
		self.Layout()

		self.Centre( wx.BOTH )

	def __del__( self ):
		pass


import argparse

from train import main

parser = argparse.ArgumentParser(description='Train prototypical networks')

# data args
default_dataset = 'omniglot'
parser.add_argument('--data.dataset', type=str, default=default_dataset, metavar='DS',
                    help="data set name (default: {:s})".format(default_dataset))
default_split = 'vinyals'
parser.add_argument('--data.split', type=str, default=default_split, metavar='SP',
                    help="split name (default: {:s})".format(default_split))
parser.add_argument('--data.way', type=int, default=60, metavar='WAY',
                    help="number of classes per episode (default: 60)")
parser.add_argument('--data.shot', type=int, default=5, metavar='SHOT',
                    help="number of support examples per class (default: 5)")
parser.add_argument('--data.query', type=int, default=5, metavar='QUERY',
                    help="number of query examples per class (default: 5)")
parser.add_argument('--data.test_way', type=int, default=5, metavar='TESTWAY',
                    help="number of classes per episode in test. 0 means same as data.way (default: 5)")
parser.add_argument('--data.test_shot', type=int, default=0, metavar='TESTSHOT',
                    help="number of support examples per class in test. 0 means same as data.shot (default: 0)")
parser.add_argument('--data.test_query', type=int, default=15, metavar='TESTQUERY',
                    help="number of query examples per class in test. 0 means same as data.query (default: 15)")
parser.add_argument('--data.train_episodes', type=int, default=100, metavar='NTRAIN',
                    help="number of train episodes per epoch (default: 100)")
parser.add_argument('--data.test_episodes', type=int, default=100, metavar='NTEST',
                    help="number of test episodes per epoch (default: 100)")
parser.add_argument('--data.trainval', action='store_true', help="run in train+validation mode (default: False)")
parser.add_argument('--data.sequential', action='store_true', help="use sequential sampler instead of episodic (default: False)")
parser.add_argument('--data.cuda', action='store_true', help="run in CUDA mode (default: False)")

# model args
default_model_name = 'protonet_conv'
default_encoding = 'C64'
parser.add_argument('--model.model_name', type=str, default=default_model_name, metavar='MODELNAME',
                    help="model name (default: {:s})".format(default_model_name))
parser.add_argument('--model.x_dim', type=str, default='1,28,28', metavar='XDIM',
                    help="dimensionality of input images (default: '1,28,28')")
parser.add_argument('--model.hid_dim', type=int, default=64, metavar='HIDDIM',
                    help="dimensionality of hidden layers (default: 64)")
parser.add_argument('--model.z_dim', type=int, default=64, metavar='ZDIM',
                    help="dimensionality of input images (default: 64)")
parser.add_argument('--model.encoding', type=str, default=default_encoding, metavar='MODELENC',
                    help="model encoding (default: {:s})".format(default_encoding))
# train args
parser.add_argument('--train.epochs', type=int, default=10000, metavar='NEPOCHS',
                    help='number of epochs to train (default: 10000)')
parser.add_argument('--train.optim_method', type=str, default='Adam', metavar='OPTIM',
                    help='optimization method (default: Adam)')
parser.add_argument('--train.learning_rate', type=float, default=0.001, metavar='LR',
                    help='learning rate (default: 0.0001)')
parser.add_argument('--train.decay_every', type=int, default=20, metavar='LRDECAY',
                    help='number of epochs after which to decay the learning rate')
default_weight_decay = 0.0
parser.add_argument('--train.weight_decay', type=float, default=default_weight_decay, metavar='WD',
                    help="weight decay (default: {:f})".format(default_weight_decay))
parser.add_argument('--train.patience', type=int, default=200, metavar='PATIENCE',
                    help='number of epochs to wait before validation improvement (default: 1000)')

# log args
default_fields = 'loss,acc'
parser.add_argument('--log.fields', type=str, default=default_fields, metavar='FIELDS',
                    help="fields to monitor during training (default: {:s})".format(default_fields))
default_exp_dir = 'results'
parser.add_argument('--log.exp_dir', type=str, default=default_exp_dir, metavar='EXP_DIR',
                    help="directory where experiments should be saved (default: {:s})".format(default_exp_dir))

# speech data args
parser.add_argument('--speech.include_background', action='store_true', help="mix background noise with samples (default: False)")
parser.add_argument('--speech.include_silence', action='store_true', help="one of the classes out of n should be silence (default: False)")
parser.add_argument('--speech.include_unknown', action='store_true', help="one of the classes out of n should be unknown (default: False)")
parser.add_argument('--speech.sample_rate', type=int, default=16000, help='desired sampling rate of the input')
parser.add_argument('--speech.clip_duration', type=int, default=1000, help='clip duration in milliseconds')
parser.add_argument('--speech.window_size', type=int, default=40)
parser.add_argument('--speech.window_stride', type=int,default=20)
parser.add_argument('--speech.num_features', type=int, default=40, help='Number of mfcc features to extract')
parser.add_argument('--speech.time_shift', type=int, default=100, help='time shift the audio in milliseconds')
parser.add_argument('--speech.bg_volume', type=float, default=0.1, help='background volumen to mix in between 0 and 1')
parser.add_argument('--speech.bg_frequency', type=float, default=1.0, help='Amount of samples that should be mixed with background noise (between 0 and 1)')
parser.add_argument('--speech.num_silence', type=int, default=1000, help='Number of silence samples to generate')
parser.add_argument('--speech.foreground_volume', type=float, default=1)
args = vars(parser.parse_args())

main(args)


# -*- coding: utf-8 -*-
"""
Created on Sat Oct 30 12:57:54 2021

@author: Gary

Used to create a full set of data, code and documentation that can be
the source of any use of the Open-FF products.  It is meant to
create an anchor that anyone working on the data can refer to.

Provided in the repo are:
    - data table pickles (for recreating analysis sets)
    - zips of filtered and full sets
    - copies of the translation tables used to created the database
    - a readme file that explains things like when the data were downloaded,
      when the data were compiled, what code version was used to create it,
      etc.
    - maybe a simple script to help user extract their own dataset from the
      pickles.
    - a csv file of hashes for most of the files in the repo.  These can
      be used to verify that the files have not been changed since creation.
"""

import os, shutil
from pathlib import Path
from hashlib import sha256
import pandas as pd
import core.Analysis_set as ana_set
import datetime
#import zipfile
import build_common
outdir = build_common.get_pickle_dir()
sources = build_common.get_data_dir()
trans_dir = build_common.get_transformed_dir()
tempfolder = './tmp/'

data_source = 'bulk'  # 'bulk' for typical processing
repo_name = 'v16_2023_03_18'
inc_zipped = False # should be true for major repositories

# data_source = 'NM_scrape_2022_05'
# repo_name = 'NM_scrape_2022_10_09'

if inc_zipped == False:
    repo_name = repo_name + '_min'

repo_dir = build_common.get_repo_dir() + repo_name
pklsource = 'currentData_pickles'

descriptive_notes = f""" This is an OpenFF data repository for the
bulk download of FracFocus from March 18, 2023

Created {datetime.date.today()}
CodeOcean version: 16
Since the version 16 release, this beta version incorporates:
 - added bgFederalWell and bgNativeAmericanWell that is derived from
    the PADUS-3 data set.

"""

# =============================================================================
# descriptive_notes = f""" This is an OpenFF data repository for the
# **New Mexico state-held** data set as scraped by OpenFF in May 2022.
#
# NOTE THAT THESE DATA ARE OLD AND MAY HAVE BEEN UPDATED BY THE INDUSTRY
# SINCE THE TIME THEY WERE SCRAPED.  USE THESE DATA WITH CARE AND DISCRETION.
#
# Created {datetime.date.today()}
# CodeOcean version: 15
# Since the version 15 release, this beta version incorporates:
#  - The field "rq_lbs" was added from the the ver 15 code.
#  - Cleaned up the bgCAS that were not striped of trailing spaces.
#  - Using new comptox batch cleanup routines. Old EPA names are no longer
#      available.
#  - bgLatitude and bgLongitude are now replaced with state-derived values
#      when errors are detected in the FracFocus versions of those varibales.
#      The field bgLocationSource indictates where the bg values come from.
#  - added bgFederalWell and bgNativeAmericanWell that is derived from
#     the PADUS-3 data set
#
# """
# =============================================================================

boilerplate = """This directory contains a data set generated by the Open-FF
project.
"""
print(descriptive_notes)
print(f'Including big zipped files? {inc_zipped}')
q = input(f'Data source is set to < {data_source} > and repo to {repo_name}. \n\n -- Enter "y" to start building process.  > ')
assert q=='y'

print(f'Starting creation of new Data Repo set: {repo_name}')
# create new directory
try:
    os.mkdir(repo_dir)
except:
    print(f'\nCreation of Directory <{repo_dir}> not allowed;  already created?')

# create and store README
with open(repo_dir+'/README.txt','w') as f:
    f.write(descriptive_notes+'\n')
    f.write(boilerplate)  # see below for the text

# # generate output csv's
# first try to delete any existing big files
try:
    os.remove(repo_dir+'/standard_filtered.zip')
    print('removed old std_filtered')
except:
    pass
try:
    os.remove(repo_dir+'/full_no_filter.zip')
    print('removed old full')
except:
    pass
try:
    os.remove(repo_dir+'/catalog_set.zip')
    print('removed old catalog_set')
except:
    pass

if inc_zipped:
    # now make the zipped sets
    ana_set.Standard_data_set().save_compressed()
    shutil.move(outdir+'standard_filtered.zip',repo_dir+'/standard_filtered.zip')

    ana_set.Full_set().save_compressed()
    shutil.move(outdir+'full_no_filter.zip',repo_dir+'/full_no_filter.zip')

    ana_set.Catalog_set().save_compressed()
    shutil.move(outdir+'catalog_set.zip',repo_dir+'/catalog_set.zip')

# copy pickles
pickledir = repo_dir+'/pickles'
try:
    os.mkdir(pickledir)
except:
    print(f'\nDirectory <{pickledir}> not created;  already created?')
flst = os.listdir(outdir+pklsource)
for fn in flst:
    if fn[-4:]=='.pkl':
        if not (fn[-7:]=='_df.pkl'):  # ignore pickled analysis sets
            shutil.copyfile(outdir+pklsource+'/'+fn, pickledir+'/'+fn)
            print(f'copied {fn}')

# copy curation files

cfiles = ['carrier_list_auto.csv',
          'carrier_list_curated.csv',
          'carrier_list_prob.csv']
files = ['CAS_curated.csv',
         'casing_curated.csv','company_xlate.csv','ST_api_without_pdf.csv',
         'ING_curated.csv','CAS_synonyms.csv',
         'CAS_synonyms_CompTox.csv','CAS_ref_and_names.csv',
         'tripwire_summary.csv','upload_dates.csv',
         'missing_values.csv']

cdir = 'curation_files'
os.mkdir(cdir) # made in the cwd.

for fn in files:
    print(f'  - zipping {fn}')
    shutil.copy(trans_dir+fn,cdir)
for fn in cfiles:
    print(f'  - zipping {fn}')
    shutil.copy(trans_dir+f'{data_source}/{fn}',cdir)
shutil.make_archive(os.path.join(repo_dir,cdir),'zip',cdir)
shutil.rmtree(cdir)

# copy CAS and CompTox reference files

cdir = 'CAS_ref_files'
sdir = r"C:\MyDocs\OpenFF\data\external_refs\CAS_ref_files"

cdir = 'CompTox_ref_files'
sdir = r"C:\MyDocs\OpenFF\data\external_refs\CompTox_ref_files"
shutil.make_archive(os.path.join(repo_dir,cdir),'zip',sdir)

# now create hashfile
#  this is a pandas df with all files (except the hashfile) in the "filename"
#  field and the sha256 hash of the file in the "sha256" field.
#  These hashes can be used to verify that the data is in the original state and
#  has not been modified.

print('\nMaking file hashes for validation')

to_hash = ['pickles/bgCAS.pkl',
           'pickles/cas_ing.pkl',
           'pickles/chemrecs.pkl',
           'pickles/companies.pkl',
           'pickles/disclosures.pkl',
           'curation_files.zip',
           'CAS_ref_files.zip',
           'CompTox_ref_files.zip',
           'README.txt',
           'standard_filtered.zip',
           'full_no_filter.zip',
           'catalog_set.zip']

fnout = []; fnhash = []
for fn in to_hash:
    path = Path(os.path.join(repo_dir,fn))
    if path.is_file():
        print('  -- '+fn)
        fnout.append(fn)
        with open(path,'rb') as f:
            fnhash.append(sha256(f.read()).hexdigest())
    else:
        print(f'  >> file not in repo: {fn} <<')
pd.DataFrame({'filename':fnout,'filehash':fnhash}).to_csv(repo_dir+'/filehash.csv',
                                                          index=False)

print(f'Repo creation completed: {repo_dir}')


#!/usr/bin/env python3
import os
from datetime import datetime
import tempfile
from typing import List
import pynvim

class BlameLine:
    def __init__(self, commit=None, email=None, timestamp=None, lineno=None, line=None):
        self.commit = commit
        self.email = email
        self.timestamp = timestamp
        self.lineno = lineno
        self.line = line
    def __repr__(self):
        return f'|{self.commit}|{self.email}|{self.timestamp}|{self.lineno}|{self.line}|'
    @staticmethod
    def from_line(line):
        s = line.split(' ', maxsplit=1)
        commit = s[0].strip()
        s = s[1].split('(', maxsplit=1)
        s = s[1].split('>', maxsplit=1)
        email = s[0][1:].strip()
        date_format = "%Y-%m-%d %H:%M:%S %z"
        ts = datetime.strptime(s[1].lstrip()[:10+1+2+2+2+2+1+5], date_format)
        datestr = ts.strftime(date_format)
        s = s[1].split(datestr, maxsplit=1)[1].lstrip()
        s = s.split(')', maxsplit=1)
        lineno, line = s
        lineno, line = int(lineno), line[1:]
        return BlameLine(
            commit=commit,
            email=email,
            timestamp=ts,
            lineno=lineno,
            line=line,
        )

def get_buffer_blame(nvim, bnr, memo={}) -> List[BlameLine]:
    text = '\n'.join(bnr[:])
    if bnr.name not in memo:
        memo[bnr.name] = {}
    if text not in memo.get(bnr.name):
        fullpath = nvim.funcs.expand('%:p')
        d = os.path.dirname(fullpath)
        filename = os.path.basename(fullpath)

        fd, path = tempfile.mkstemp()

        os.write(fd, bytes(text, 'utf-8'))
        os.close(fd)

        with os.popen(f"git -C '{d}' blame '{filename}' --content '{path}' -e") as p:
            blame = p.read()
            lines = [BlameLine.from_line(line) for line in blame.split('\n') if line != ""]

        memo[text] = lines
    return memo[text]

@pynvim.plugin
class TestPlugin(object):

    def __init__(self, nvim):
        self.nvim = nvim
        self.ns_id = nvim.api.create_namespace('demo')
        self.blame_lines = {}
        self.marks = {}

    @pynvim.command("Glame", nargs="*", range="", sync=False)
    def testfunction(self, *args, **kwargs):
        self.update()

    @pynvim.autocmd('BufReadPost')
    def on_bufenter(self, *args, **kwargs):
        self.update()

    @pynvim.autocmd('TextChanged')
    def on_textchanged(self, *args, **kwargs):
        self.update()

    def update(self):
        bnr = self.nvim.current.buffer
        new_blame_lines = get_buffer_blame(self.nvim, bnr)
        if new_blame_lines == self.blame_lines.get(bnr.name):
            return
        self.blame_lines[bnr.name] = new_blame_lines

        for blame in self.blame_lines[bnr.name]:
            hl_id = f"DiffGutter_{blame.commit}"

            background = blame.commit[:6]
            red, green, blue = [int(background[i:i+2], base=16) for i in range(0, len(background), 2)]
            is_light = ( red + green + blue ) / 3 > 128
            text_color = 'black' if is_light else 'white'
            self.nvim.command(f"highlight {hl_id} guibg=#{background} guifg={text_color}")

            sign_id = f"DiffGutterSign_{blame.commit}"
            sign_text = blame.email.split('@')[0][:2]
            self.nvim.command(f"sign define {sign_id} texthl={hl_id} numhl={hl_id} text={sign_text}")
            signref = f'614{blame.lineno}'
            if signref in self.marks:
                self.nvim.command(f'sign unplace {signref}')
            self.marks[signref] = self.nvim.command(f'sign place {signref} line={blame.lineno} name={sign_id}') or True


"""The base class for deriving types that require schema support.

.. image:: images/ontic_property.jpg

.. contents::

======
Usage
======

The **meta_type** module contains three classes that are used in the
definition and instantiation of object instances.  The root class is
:class:`CoreType`, which is derived from the Python :class:`dict`. The
primary feature of the :class:`CoreType` is provide the means to access
properties of an object as a *dict* key or as object attribute.

Derived from :class:`CoreType` is :class:`MetaType`, which provides the
interface for retrieving a schema from an object schema definition via the
:meth:`MetaType.get_schema()`.

The :class:`PropertySchema` is utilized to define a schema for a single
property. This includes setting data type, required, and other such common
schema definition conventions. See :ref:`property-schema-settings-table` for
details on the :class:`PropertySchema` settings available.

It would be a rare case that requires the use of :class:`CoreType` or
:class:`MetaType`. For the majority of cases, the use of
:class:`PropertySchema` is sufficient. With that in mind, the remainder of
this section will focus on the use of :class:`PropertySchema`

Creating Property Schema
-------------------------

For general purposes, consider utilizing :class:`ontic.schema_type.SchemaType`,
for defining complete models. However, if you need validators for individual
properties, then direct use of :class:`PropertySchema` is a solution.

There are a number of ways to create a :class:`PropertySchema`. Take a look
at :class:`PropertySchema` class documentation for a complete exposition on
the means of instantiating an instance.

The most straight forward way to create an instance of a
:class:`PropertySchema`:

>>> prop_schema = OnticProperty(name='x', type='str', required=True, min=3)
>>> assert prop_schema == {'regex': None, 'enum': None, 'min': 3, 'default': None, 'max': None,
...     'required': True, 'member_type': None, 'member_min': None,
...     'type': str, 'member_max': None, 'name': 'x'}

Demonstrated above is the creation of a property schema of type string. In
addition the property schema forces the value of the property to required and
of minimum length of 3.

Along with the schema settings explicitly set in the constructor, there are a
number of other property schema settings that may be utilized. These
additional settings can be viewed in the output of the *prop_schema* object.
For the details on the property schema settings, see
:ref:`property-schema-settings-table`.

A :class:`PropertySchema` can be created or modified dynamically. If done so,
then the final schema instance should be validated with the use of the method
:meth:`validate_property_schema`.

Utilizing Property Schema
--------------------------

Validation of a value utilizing the *prop_schema* created, is done with the
:meth:`validate_value` method.

>>> prop_schema = OnticProperty(name='some_prop', type='str', required=True)
>>> some_value = 'The cat is on the roof.'
>>> prop_schema.validate_value(value=some_value)
[]

:class:`validate_value` returns an empty list if there are no errors.

The *name* parameter of the :meth:`validate_value`, is used to construct
friendly error messages. For example:

>>> meta.validate_value(prop_schema, None)
['The value for "some_prop" is required.']

The following example demonstrates how a :class:`PropertySchema` being
instantiated with a dictionary. Subsequently a bad value is passed with
multiple validation errors.

>>> other_schema = OnticProperty({
...     'name': 'other_prop',
...     'type': 'str',
...     'max': 3,
...     'enum': {'fish', 'dog', 'cat'}
... })
>>> ret = other_schema.validate_value('frog')
>>> assert len(ret) == 2
>>> assert ret[0] == '''The value "frog" for "other_prop" not in enumeration ['cat', 'dog', 'fish'].'''
>>> assert ret[1] == '''The value of "frog" for "other_prop" fails max of 3.'''

.. _property-schema-settings-table:

Available Property Schema Settings
-----------------------------------

The following table gives a listing of the property schema settings that can
be used to define properties. Details on the schema settings are provided
after the table.

.. table:: Property Schema Settings

    ============ ========= ======== ========  =================================
    Name         Type      Default  Required  Enumeration
    ============ ========= ======== ========  =================================
    type         str       None     False     bool, complex, date,
                 type                         datetime, dict, float, int, list,
                                              None, set, str, time
    default      None      None     False
    required     bool      False    False
    enum         set       None     False
    min          complex   None     False
                 date
                 datetime
                 float
                 int
                 time
    max          complex   None     False
                 date
                 datetime
                 float
                 int
                 time
    regex        str       None     False
    member_type  str       None     False     bool, complex, date,
                 type                         datetime, dict, float, int, list,
                                              None, set, str, time
    member_min   complex   None     False
                 date
                 datetime
                 float
                 int
                 time
    member_max   complex   None     False
                 date
                 datetime
                 float
                 int
                 time
    ============ ========= ======== ========  =================================

*type*
    The *type* settings restricts a property to a known type. If no type is
    defined, then any value type may be assigned to the property.

    The type definition can be by type or by string name. Both ``{type=int}``
    and ``{type='int'}`` are valid examples of type declaration.
*default*
    If no default is applied, then the default value will be ``None``. If a
    default value is supplied, it will only be applied under two conditions.
    A default value is applied during instantiation of an object of type
    :class:`PropertySchema`, :class:`~ontic.schema_type.SchemaType`,
    or :class:`~ontic.ontic_type.OnticType`. The other case is when an
    instance of on of the given types is perfected via the methods
    :func:`perfect_property_schema`, :func:`~ontic.schema_type.perfect_schema`,
    or :func:`~ontic.ontic_type.perfect_object`.

    The default is not applied during validation.

    For the collection types (dict, list, and set), the default value is deep
    copied. This is done to ensure that there is no sharing of collection
    instances or values.
*required*
    A *PropertySchema* with a required setting of *True*, will fail
    validation if the property value is *None*.
*enum*
    An *enum* setting is a set of values that the property value must adhere
    to. If the *type* setting is provided, then the choices provided by
    *enum* must be of that type. If no *type* is provided, then the choices
    in the *enum* set may be of any type, even mixed type.
*min*
    The *min* setting has differing behavior, based on the *type* setting. If
    no *type* setting is provided, then *min* test will not occur. For the
    boundable types (strings and collections) the *min* setting will test that
    the value length is not less than the minimum. For the comparable types
    (numeric and chronological) the *min* setting will test that the
    value is not less than the minimum.
*max*
    The *max setting has differing behavior, based on the *type* setting. If
    no *type* setting is provided, the *max* test will not occur. For the
    boundable types (strings and collections) the *max* setting will test that
    the value length is not more than the maximum. For the comparable types
    (numeric and chronological) the *max* setting will test that the
    value is not more than the maximum.
*regex*
    The *regex* setting is only tested if the *type* or *member_type* setting
    is 'str' and the *regex* setting is not None. When active, the *regex*
    setting will be used to test the given string value.  If the property
    value is 'None', then no regex testing will be done.
*member_type*
    The *member_type* setting is used to restrict the value type for property
    *type* 'list' or 'set'. It does so ensuring that each member of the
    collection is of the type designated by *member_type*.

    The type definition can be by type or by string name. Both
    ``{member_type=int}`` and ``{member_type='int'}`` are valid examples of
    type declaration.
*member_min*
    The *member_min* setting has differing behavior, based on the
    *member_type* setting. If no *member_type* setting is provided, then
    *member_min* test will not occur. For the boundable types
    (strings and collections), the *member_min* setting will test that the
    value length is not less than the minimum. For the comparable types
    (numeric and chronological) the *member_minimum* setting will test
    that the value is not less than the minimum.
*member_max*
    The *member_max* setting has differing behavior, based on the
    *member_max* setting. If no *member_type* setting is provided,
    then *member_max* test will not occur. For the boundable types
    (strings and collections), the *member_max* setting will test that the
    value length is not more than the maximum. For the comparable types
    (numeric and chronological) the *member_max* setting will test
    that the value is not more than the maximum.

"""
from typing import NoReturn, Any

from ontic import meta
from ontic import validation_exception

class OnticProperty(meta.Meta):
    """A class to define a schema for a property."""

    ONTIC_SCHEMA = meta.Meta({
        'name': meta.Meta({
            'name': 'name',
            'type': str,
            'default': None,
            'required': True,
            'enum': None,
            'min': 1,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'type': meta.Meta({
            'name': 'type',
            'type': (str, type),  # todo: raul - this could be restricted list
            'default': None,
            'required': False,
            'enum': meta.TYPE_SET + (None,),
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'default': meta.Meta({
            'name': 'default',
            'type': None,
            'default': None,
            'required': False,
            'enum': None,
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'required': meta.Meta({
            'name': 'required',
            'type': bool,
            'default': False,
            'required': False,
            'enum': None,
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'enum': meta.Meta({
            'name': 'enum',
            'type': set,
            'default': None,
            'required': False,
            'enum': None,
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'min': meta.Meta({
            'name': 'min',
            'type': tuple(meta.COMPARABLE_TYPES),
            'default': None,
            'required': False,
            'enum': None,
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'max': meta.Meta({
            'name': 'max',
            'type': tuple(meta.COMPARABLE_TYPES),
            'default': None,
            'required': False,
            'enum': None,
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'regex': meta.Meta({
            'name': 'regex',
            'type': str,
            'default': None,
            'required': False,
            'enum': None,
            'min': 1,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'member_type': meta.Meta({
            'name': 'member_type',
            'type': (str, type),  # todo: raul - this could be restricted list
            #  subclass testing.
            'default': None,
            'required': False,
            'enum': meta.TYPE_SET + (None,),
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'member_min': meta.Meta({
            'name': 'member_min',
            'type': tuple(meta.COMPARABLE_TYPES),
            'default': None,
            'required': False,
            'enum': None,
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
        'member_max': meta.Meta({
            'name': 'member_max',
            'type': tuple(meta.COMPARABLE_TYPES),
            'default': None,
            'required': False,
            'enum': None,
            'min': None,
            'max': None,
            'regex': None,
            'member_type': None,
            'member_min': None,
            'member_max': None,
        }),
    })

    def __init__(self, *args, **kwargs):
        super(OnticProperty, self).__init__(*args, **kwargs)

        self.perfect()
        self.validate()

    def perfect(self) -> NoReturn:
        """Method to ensure the completeness of a given schema property.

        :rtype: None
        :raises ValueError: If the candidate_property_type is None, or not
            of type *OnticProperty*.
        """
        perfect_property(self)

    def validate(
            self,
            raise_validation_exception: bool = True) -> (None, list[str]):
        """Method to validate a property schema definition.

        :param raise_validation_exception: If True, then
            *validate_property_type* will throw a *ValueException* upon
            validation failure. If False, then a list of validation errors is
            returned. Defaults to True.
        :return: If no validation errors are found, then *None* is returned.
            If validation fails, then a list of the errors is returned if the
            *raise_validation_exception* is set to True.
        :raises ValueError: *the_candidate_schema_property* is not an
            :class:`OnticProperty`.
        :raises ValidationException: A property of *candidate_property_type*
            does not meet schema requirements.
        """
        return validate_property(self, raise_validation_exception)

    def validate_value(self,
                       value: Any,
                       raise_validation_exception: bool = False) -> NoReturn:
        """Method that validates a value against the current schema."""
        value_errors = meta.validate_value(self, value)

        if value_errors and raise_validation_exception:
            raise validation_exception.ValidationException(value_errors)

        return value_errors

def perfect_property(ontic_property: 'OnticProperty') -> NoReturn:
    """Method to ensure the completeness of a given schema property.

    This method ensures completeness by stripping out any properties that
    are not defined by the schema definition. In addition, for any schema
    properties that are not included, the method will add those
    properties to the default value.

    :param ontic_property: The OnticProperty that is to be clean and restricted.
    :type ontic_property: :class:`OnticProperty`
    :rtype: None
    :raises ValueError: If the ontic_property is None, or not of type
        *OnticProperty*.
    """
    if ontic_property is None:
        raise ValueError('"ontic_property" must be provided.')
    if not isinstance(ontic_property, OnticProperty):
        raise ValueError('"ontic_property" must be OnticProperty type.')

    property_schema = ontic_property.get_schema()

    # remove un-necessary properties.
    extra_properties = set(ontic_property.keys()) - set(property_schema.keys())
    for property_name in extra_properties:
        del ontic_property[property_name]

    if 'type' in ontic_property:
        _perfect_type_setting(ontic_property)
    else:
        ontic_property.type = None

    if 'member_type' in ontic_property:
        _perfect_member_type_setting(ontic_property)
    else:
        ontic_property.member_type = None

    # set the default for the given property.
    for property_name, property_schema in property_schema.items():
        if property_name not in ontic_property:
            ontic_property[property_name] = None
        if ontic_property[property_name] is None:
            ontic_property[property_name] = property_schema.default

def validate_property(
        ontic_property: 'OnticProperty',
        raise_validation_exception: bool = True) -> (None, list[str]):
    """Method to validate a property schema definition.

    :param ontic_property: The schema property to be validated.
    :param raise_validation_exception: If True, then *validate_property_type*
        will throw a *ValueException* upon validation failure. If False,
        then a list of validation errors is returned. Defaults to True.
    :return: If no validation errors are found, then *None* is
        returned. If validation fails, then a list of the errors is returned
        if the *raise_validation_exception* is set to True.
    :raises ValueError: *the_candidate_schema_property* is not an
        :class:`~ontic.ontic_type.OnticType`.
    :raises ValidationException: A property of *ontic_property*
        does not meet schema requirements.
    """
    if ontic_property is None:
        raise ValueError('"ontic_property" must be provided.')
    if not isinstance(ontic_property, OnticProperty):
        raise ValueError('"ontic_property" must be OnticProperty type.')

    value_errors = []

    for property_schema in ontic_property.get_schema().values():
        property_value = ontic_property.get(property_schema.name, None)

        # todo: raul - for now skip validating compound schemas.
        if (isinstance(property_value, type) and
                issubclass(property_value, meta.Meta)):
            continue

        value_errors.extend(
            meta.validate_value(
                property_schema, property_value))

    if value_errors and raise_validation_exception:
        raise validation_exception.ValidationException(value_errors)

    return value_errors

def _perfect_type_setting(ontic_property: 'OnticProperty') -> None:
    """Perfect the type setting for a given candidate property schema."""
    if ontic_property.type is None:
        return

    candidate_type = ontic_property.type
    # coerce type declarations as string to base types.
    if isinstance(candidate_type, str):
        try:
            candidate_type = ontic_property.type = meta.TYPE_MAP[
                candidate_type]
        except KeyError:
            raise ValueError('Illegal type declaration: %s' %
                             ontic_property.type)

    # ensure that the type declaration is valid
    is_supported_type = candidate_type in meta.TYPE_SET
    is_meta_schema_type = issubclass(candidate_type, meta.Meta)
    if not (is_supported_type or is_meta_schema_type):
        raise ValueError('Illegal type declaration: %s' % str(candidate_type))

def _perfect_member_type_setting(ontic_property: 'OnticProperty') -> NoReturn:
    """Perfect the member_type setting for a given candidate property schema."""
    if ontic_property.member_type is None:
        return

    candidate_type = ontic_property.member_type
    # coerce type declarations as string to base types.
    if isinstance(candidate_type, str):
        try:
            candidate_type = ontic_property.member_type = meta.TYPE_MAP[
                candidate_type]
        except KeyError:
            raise ValueError('Illegal member_type declaration: %s' %
                             ontic_property.member_type)

    # ensure that the type declaration is valid
    is_supported_type = candidate_type in meta.TYPE_SET
    is_meta_schema_type = issubclass(candidate_type, meta.Meta)
    if not (is_supported_type or is_meta_schema_type):
        raise ValueError('Illegal type declaration: %s' % candidate_type)


"""
Tests for error conditions.
"""
import os
import platform
import struct
import tempfile
import unittest

import numpy as np

import kastore as kas
import kastore.store as store

IS_WINDOWS = platform.system() == "Windows"

class InterfaceMixin:
    """
    Exercise the low-level interface.
    """

    def setUp(self):
        fd, path = tempfile.mkstemp(prefix="kas_test_errors")
        os.close(fd)
        self.temp_file = path

    def tearDown(self):
        os.unlink(self.temp_file)

    def test_bad_dicts(self):
        for bad_dict in [[], "w34", None, 1]:
            self.assertRaises(
                TypeError, kas.dump, bad_dict, self.temp_file, engine=self.engine
            )
            self.assertRaises(
                TypeError, kas.dump, bad_dict, self.temp_file, engine=self.engine
            )

    def test_bad_filename_type(self):
        for bad_filename in [[], None, {}]:
            self.assertRaises(TypeError, kas.dump, {}, bad_filename, engine=self.engine)
            self.assertRaises(TypeError, kas.dump, {}, bad_filename, engine=self.engine)
            self.assertRaises(TypeError, kas.load, bad_filename, engine=self.engine)
            self.assertRaises(TypeError, kas.load, bad_filename, engine=self.engine)

    def test_bad_keys(self):
        a = np.zeros(1)
        for bad_key in [(1234,), b"1234", None, 1234]:
            with self.assertRaises(TypeError):
                kas.dump({bad_key: a}, self.temp_file, engine=self.engine)

    def test_bad_arrays(self):
        kas.dump({"a": []}, self.temp_file, engine=self.engine)
        for bad_array in [kas, lambda x: x, "1234", None, [[0, 1], [0, 2]]]:
            self.assertRaises(
                ValueError,
                kas.dump,
                {"a": bad_array},
                self.temp_file,
                engine=self.engine,
            )
        # TODO add tests for arrays in fortran order and so on.

    def test_file_not_found(self):
        a = np.zeros(1)
        for bad_file in ["no_such_file", "/no/such/file"]:
            self.assertRaises(FileNotFoundError, kas.load, bad_file, engine=self.engine)
        self.assertRaises(
            FileNotFoundError, kas.dump, {"a": a}, "/no/such/file", engine=self.engine
        )

    def test_file_is_a_directory(self):
        tmp_dir = tempfile.mkdtemp()
        try:
            exception = IsADirectoryError
            if IS_WINDOWS:
                exception = PermissionError
            self.assertRaises(
                exception, kas.dump, {"a": []}, tmp_dir, engine=self.engine
            )
            self.assertRaises(exception, kas.load, tmp_dir, engine=self.engine)
        finally:
            os.rmdir(tmp_dir)

class TestInterfacePyEngine(InterfaceMixin, unittest.TestCase):
    engine = kas.PY_ENGINE

class TestInterfaceCEngine(InterfaceMixin, unittest.TestCase):
    engine = kas.C_ENGINE

class TestEngines(unittest.TestCase):
    """
    Check that we correctly identify bad engines
    """

    bad_engines = [None, {}, "no such engine", b"not an engine"]

    def test_bad_engine_dump(self):
        for bad_engine in self.bad_engines:
            self.assertRaises(ValueError, kas.dump, {}, "", engine=bad_engine)

    def test_bad_engine_load(self):
        for bad_engine in self.bad_engines:
            self.assertRaises(ValueError, kas.load, "", engine=bad_engine)

class FileFormatsMixin:
    """
    Common utilities for tests on the file format.
    """

    def setUp(self):
        fd, path = tempfile.mkstemp(prefix="kas_malformed_files")
        os.close(fd)
        self.temp_file = path

    def tearDown(self):
        os.unlink(self.temp_file)

    def write_file(self, num_items=0):
        data = {}
        for j in range(num_items):
            data["a" * (j + 1)] = np.arange(j)
        kas.dump(data, self.temp_file)

class MalformedFilesMixin(FileFormatsMixin):
    """
    Tests for various types of malformed intput files.
    """

    def test_empty_file(self):
        with open(self.temp_file, "w"):
            pass
        self.assertEqual(os.path.getsize(self.temp_file), 0)
        self.assertRaises(
            EOFError,
            kas.load,
            self.temp_file,
            engine=self.engine,
            read_all=self.read_all,
        )

    def test_bad_magic(self):
        self.write_file()
        with open(self.temp_file, "rb") as f:
            buff = bytearray(f.read())
        before_len = len(buff)
        buff[0:8] = b"12345678"
        self.assertEqual(len(buff), before_len)
        with open(self.temp_file, "wb") as f:
            f.write(buff)
        self.assertRaises(
            kas.FileFormatError,
            kas.load,
            self.temp_file,
            engine=self.engine,
            read_all=self.read_all,
        )

    def test_bad_file_size(self):
        for num_items in range(10):
            for offset in [-2, -1, 1, 2**10]:
                self.write_file(num_items)
                file_size = os.path.getsize(self.temp_file)
                with open(self.temp_file, "rb") as f:
                    buff = bytearray(f.read())
                before_len = len(buff)
                buff[16:24] = struct.pack("<Q", file_size + offset)
                self.assertEqual(len(buff), before_len)
                with open(self.temp_file, "wb") as f:
                    f.write(buff)
                with self.assertRaises(kas.FileFormatError):
                    kas.load(self.temp_file, engine=self.engine, read_all=self.read_all)

    def test_truncated_file_descriptors(self):
        for num_items in range(2, 5):
            self.write_file(num_items)
            with open(self.temp_file, "rb") as f:
                buff = bytearray(f.read())
            with open(self.temp_file, "wb") as f:
                f.write(buff[: num_items * store.ITEM_DESCRIPTOR_SIZE - 1])
            with self.assertRaises(kas.FileFormatError):
                kas.load(self.temp_file, engine=self.engine, read_all=self.read_all)

    def test_truncated_file_data(self):
        for num_items in range(2, 5):
            self.write_file(num_items)
            with open(self.temp_file, "rb") as f:
                buff = bytearray(f.read())
            with open(self.temp_file, "wb") as f:
                f.write(buff[:-1])
            with self.assertRaises(kas.FileFormatError):
                # Must call dict to ensure all the keys are loaded.
                dict(
                    kas.load(self.temp_file, engine=self.engine, read_all=self.read_all)
                )

    def test_bad_item_types(self):
        items = {"a": []}
        descriptors, file_size = store.pack_items(items)
        num_types = len(store.np_dtype_to_type_map)
        for bad_type in [num_types + 1, 2 * num_types]:
            with open(self.temp_file, "wb") as f:
                descriptors[0].type = bad_type
                store.write_file(f, descriptors, file_size)
            with self.assertRaises(kas.FileFormatError):
                kas.load(self.temp_file, engine=self.engine, read_all=self.read_all)

    def test_bad_key_initial_offsets(self):
        items = {"a": np.arange(100)}
        # First key offset must be at header_size + n * (descriptor_size)
        for offset in [-1, +1, 2, 100]:
            # First key offset must be at header_size + n * (descriptor_size)
            descriptors, file_size = store.pack_items(items)
            descriptors[0].key_start += offset
            with open(self.temp_file, "wb") as f:
                store.write_file(f, descriptors, file_size)
            self.assertRaises(
                kas.FileFormatError,
                kas.load,
                self.temp_file,
                engine=self.engine,
                read_all=self.read_all,
            )

    def test_bad_key_non_sequential(self):
        items = {"a": np.arange(100), "b": []}
        # Keys must be packed sequentially.
        for offset in [-1, +1, 2, 100]:
            descriptors, file_size = store.pack_items(items)
            descriptors[1].key_start += offset
            with open(self.temp_file, "wb") as f:
                store.write_file(f, descriptors, file_size)
            self.assertRaises(
                kas.FileFormatError,
                kas.load,
                self.temp_file,
                engine=self.engine,
                read_all=self.read_all,
            )

    def test_bad_array_initial_offset(self):
        items = {"a": np.arange(100)}
        for offset in [-100, -1, +1, 2, 8, 16, 100]:
            # First key offset must be at header_size + n * (descriptor_size)
            descriptors, file_size = store.pack_items(items)
            descriptors[0].array_start += offset
            with open(self.temp_file, "wb") as f:
                store.write_file(f, descriptors, file_size)
            self.assertRaises(
                kas.FileFormatError,
                kas.load,
                self.temp_file,
                engine=self.engine,
                read_all=self.read_all,
            )

    def test_bad_array_non_sequential(self):
        items = {"a": np.arange(100), "b": []}
        for offset in [-1, 1, 2, -8, 8, 100]:
            descriptors, file_size = store.pack_items(items)
            descriptors[1].array_start += offset
            with open(self.temp_file, "wb") as f:
                store.write_file(f, descriptors, file_size)
            self.assertRaises(
                kas.FileFormatError,
                kas.load,
                self.temp_file,
                engine=self.engine,
                read_all=self.read_all,
            )

    def test_bad_array_alignment(self):
        items = {"a": np.arange(100, dtype=np.int8), "b": []}
        descriptors, file_size = store.pack_items(items)
        descriptors[0].array_start += 1
        descriptors[0].array_len -= 1
        with open(self.temp_file, "wb") as f:
            store.write_file(f, descriptors, file_size)
        with self.assertRaises(kas.FileFormatError):
            kas.load(self.temp_file, engine=self.engine, read_all=self.read_all)

    def test_bad_array_packing(self):
        items = {"a": np.arange(100, dtype=np.int8), "b": []}
        descriptors, file_size = store.pack_items(items)
        descriptors[0].array_start += 8
        descriptors[0].array_len -= 8
        with open(self.temp_file, "wb") as f:
            store.write_file(f, descriptors, file_size)
        with self.assertRaises(kas.FileFormatError):
            kas.load(self.temp_file, engine=self.engine, read_all=self.read_all)

class TestMalformedFilesPyEngine(MalformedFilesMixin, unittest.TestCase):
    read_all = False
    engine = kas.PY_ENGINE

class TestMalformedFilesCEngine(MalformedFilesMixin, unittest.TestCase):
    read_all = False
    engine = kas.C_ENGINE

class TestMalformedFilesPyEngineReadAll(MalformedFilesMixin, unittest.TestCase):
    read_all = True
    engine = kas.PY_ENGINE

class TestMalformedFilesCEngineReadAll(MalformedFilesMixin, unittest.TestCase):
    read_all = True
    engine = kas.C_ENGINE

class FileVersionsMixin(FileFormatsMixin):
    """
    Tests for the file major version.
    """

    def verify_major_version(self, version):
        self.write_file()
        with open(self.temp_file, "rb") as f:
            buff = bytearray(f.read())
        before_len = len(buff)
        buff[8:10] = struct.pack("<H", version)
        self.assertEqual(len(buff), before_len)
        with open(self.temp_file, "wb") as f:
            f.write(buff)
        kas.load(self.temp_file, engine=self.engine, read_all=self.read_all)

    def test_major_version_too_old(self):
        self.assertRaises(
            kas.VersionTooOldError, self.verify_major_version, store.VERSION_MAJOR - 1
        )

    def test_major_version_too_new(self):
        for j in range(1, 5):
            self.assertRaises(
                kas.VersionTooNewError,
                self.verify_major_version,
                store.VERSION_MAJOR + j,
            )

class TestFileVersionsPyEngine(FileVersionsMixin, unittest.TestCase):
    engine = kas.PY_ENGINE
    read_all = False

class TestFileVersionsPyEngineReadAll(FileVersionsMixin, unittest.TestCase):
    engine = kas.PY_ENGINE
    read_all = True

class TestFileVersionsCEngine(FileVersionsMixin, unittest.TestCase):
    engine = kas.C_ENGINE
    read_all = False

class TestFileVersionsCEngineReadAll(FileVersionsMixin, unittest.TestCase):
    engine = kas.C_ENGINE
    read_all = True


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

type_of_feature = "mwmd"

# per album heatmap
mwmd = pd.read_csv(f"general_results_mwmd.csv")
mse = pd.read_csv(f"general_results_mse.csv")
spec_flux = pd.read_csv(f"general_results_spectral_flux.csv")

feature_to_plot = "False Negatives"

mwmd["Accuracy"] = mwmd["True Positives"] / mwmd["Total beats"]
mwmd["F1-Score"] = 2*mwmd["True Positives"] / (2*mwmd["True Positives"] + mwmd["False Negatives"] + mwmd["False Positives"])
mwmd["Rush plus drag"] = mwmd["Drag mean"] + mwmd["Rush mean"]

mse["Accuracy"] = mse["True Positives"] / mse["Total beats"]
mse["F1-Score"] = 2*mse["True Positives"] / (2*mse["True Positives"] + mse["False Negatives"] + mse["False Positives"])
mse["Rush plus drag"] = mse["Drag mean"] + mse["Rush mean"]

spec_flux["Accuracy"] = spec_flux["True Positives"] / spec_flux["Total beats"]
spec_flux["F1-Score"] = 2*spec_flux["True Positives"] / (2*spec_flux["True Positives"] + spec_flux["False Negatives"] + spec_flux["False Positives"])
spec_flux["Rush plus drag"] = spec_flux["Drag mean"] + spec_flux["Rush mean"]

names = ["MSE", "Spectral Flux", "MWMD"]

mwmd_val = mwmd[feature_to_plot].mean()
mse_val = mse[feature_to_plot].mean()
spec_flux_val = spec_flux[feature_to_plot].mean()

plt.plot(names, [mse_val, spec_flux_val, mwmd_val])
plt.ylabel(feature_to_plot)
plt.savefig(f"plots/{feature_to_plot}_comparative.png")
plt.close()