#!/usr/bin/python3
r"""

Filters
=======

to do
"""

import numpy as np
import pycosmo.utils.settings as settings
from scipy.integrate import simpson
from typing import Any, Callable
from abc import ABC, abstractmethod

#################################################################################
# Base filter function class
#################################################################################

class Filter(ABC):
    r"""
    Base class representing a smoothing filter in k-space. A filter function object must be a subclass of this.
    """

    @abstractmethod
    def filter(self, x: Any, j: int = 0) -> Any:
        r"""
        Functional form of the filter.

        Parameters
        ----------
        x: array_like
            Argument.
        j: int, optional
            Specifies the n-the derivative. Default is 0, meaning the function itself.

        Returns
        -------
        wx: array_like
            Value of the function.
        """
        ...

    def __call__(self, x: Any, j: int = 0) -> Any:
        return self.filter(x, j)

    def convolution(self, f: Callable, r: Any, args: tuple = (), ) -> Any:
        r"""
        Convolve a function with the filter. i.e., smooth the function. The convolution of a function
        :math:`f(k)` with wthe filter is given by the integral

        .. math::
            F(r) = \int_0^\infty f(k) w(kr)^2 {\rm d}k

        Parameters
        ----------
        f: callable
            Function to convolve with the filter.
        r: array_like
            Smoothing radius or convolution argument.
        args: tuple, optional
            Other arguments to be passed to the function call.

        Returns
        -------
        F: array_like
            Value of the convolution.
        """

        def integrand(lnk: Any, r: Any, *args):
            k  = np.exp( lnk )
            kr = np.outer( r, k )
            return f( k, *args ) * self.filter( kr )**2

        r    = np.asfarray( r )
        args = ( r, *args )
        a, b = np.log( settings.ZERO ), np.log( settings.INF )
        out = _integrate( integrand, a, b, args = args )
        return out if np.ndim( r ) else out[0]

    def dcdr(self, f: Callable, r: Any, args: tuple = (), ) -> Any:
        r"""
        Compute the first derivative of the convolution.

        Parameters
        ----------
        f: callable
            Function to convolve with the filter.
        r: array_like
            Smoothing radius or convolution argument.
        args: tuple, optional
            Other arguments to be passed to the function call.

        Returns
        -------
        dF: array_like
            Value of the derivative of convolution.
        """

        def integrand(lnk: Any, r: Any, *args):
            k  = np.exp( lnk )
            kr = np.outer( r, k )
            return f( k, *args ) * k * self.filter( kr ) * self.filter( kr, 1 )

        args = ( r, *args )
        a, b = np.log( settings.ZERO ), np.log( settings.INF )
        y1   = 2 * _integrate( integrand, a, b, args = args )
        return y1 if np.ndim( r ) else y1[0]

    def d2cdr2(self, f: Callable, r: Any, args: tuple = (), ) -> Any:
        r"""
        Compute the second derivative of the convolution.

        Parameters
        ----------
        f: callable
            Function to convolve with the filter.
        r: array_like
            Smoothing radius or convolution argument.
        args: tuple, optional
            Other arguments to be passed to the function call.

        Returns
        -------
        d2F: array_like
            Value of the derivative of convolution.
        """

        def integrand(lnk: Any, r: Any, *args):
            k  = np.exp( lnk )
            kr = np.outer( r, k )

            wfactor = self.filter( kr ) * self.filter( kr, 2 ) + self.filter( kr, 1 )**2
            return f( k, *args ) * k**2 * self.filter( kr ) * wfactor

        args = ( r, *args )
        a, b = np.log( settings.ZERO ), np.log( settings.INF )
        y2   = 2 * _integrate( integrand, a, b, args = args )
        return y2 if np.ndim( r ) else y2[0]

###################################################################################
# Most used filter funcions
###################################################################################

class Tophat(Filter):
    r"""
    Spherical tophat filter in k-space.

    .. math::
        w(x) = 3 \frac{ \sin(x) - x\cos(x) }{ x^3 }

    """

    def filter(self, x: Any, j: int = 0) -> Any:
        x = np.asfarray( x )

        if j == 0:
            return ( np.sin( x ) - x * np.cos( x ) ) * 3.0 / x**3
        elif j == 1:
            return ( ( x**2 - 3.0 ) * np.sin( x ) + 3.0 * x * np.cos( x ) ) * 3.0 / x**4
        elif j == 2:
            return ( ( x**2 - 12.0 ) * x * np.cos( x ) - ( 5*x**2 - 12.0 ) * np.sin( x ) ) * 3.0 / x**5
        return NotImplemented

class Gaussian(Filter):
    r"""
    Gaussian filter in k-space.

    .. math::
        w(x) = e^{ -x^2 / 2 }

    """

    def filter(x: Any, j: int = 0) -> Any:
        x = np.asfarray( x )

        if j == 0:
            return np.exp( -0.5*x**2 )
        elif j == 1:
            return -x*np.exp( -0.5*x**2 )
        elif j == 2:
            return ( x**2 - 1 )*np.exp( -0.5*x**2 )
        return NotImplemented

class SharpK(Filter):
    r"""
    Sharp k filter in k-space. It is the Fourier transform of the spherical tophat filter
    in real space. It's value is 0 for x > 0 and 1 otherwise.

    Note: derivative of the convolution is not implemented.
    """

    def filter(self, x: Any, j: int = 0) -> Any:
        x = np.asfarray( x )

        if j == 0:
            return np.where( x > 1.0, 0.0, 1.0 )
        return NotImplemented

    def convolution(self, f: Callable, r: Any, args: tuple = ()) -> Any:

        def integrand(lnk: Any, *args):
            k  = np.exp( lnk )
            return f( k, *args )

        r    = np.asfarray( r )
        a, b = np.log( settings.ZERO ), np.log( 1/r )
        out  = _integrate( integrand, a, b, args = args )
        return out if np.ndim( r ) else out[0]

    def dcdr(self, f: Callable, r: Any, args: tuple = ()) -> Any:
        raise NotImplementedError()

    def d2cdr2(self, f: Callable, r: Any, args: tuple = ()) -> Any:
        raise NotImplementedError()

available = ['tophat', 'gauss', 'sharpk']

def get(key: str) -> Filter:

    if key == 'tophat':
        return Tophat()
    if key in ['gauss', 'gaussian']:
        return Gaussian()
    if key in ['sharpk', 'sharp-k']:
        return SharpK()

    raise ValueError(f"cannor find filter function '{key}'")

def j0convolution(f: Callable, r: Any, args: tuple = (), ) -> Any:
    r"""
    Compute the convolution with sinc function (spherical bessel function :math:`j_0`) filter.

    .. math::
        F(r) = \int_0^\infty f(k) \frac{ \sin(kr) }{ kr }

    Note: sinc function is highly oscillating for large arguments, it is exponentilly suppressed
    at x = 100. i.e., an additional factor of :math:`\exp[(x/100)^2]` is introduced.

    Parameters
    ----------
    f: callable
        Function to convolve with the filter.
    r: array_like
        Smoothing radius or convolution argument.
    args: tuple, optional
        Other arguments to be passed to the function call.

    Returns
    -------
    F: array_like
        Value of the convolution.
    """

    def integrand(lnk: Any, r: Any, *args):
        k  = np.exp( lnk )
        kr = np.outer( r, k )
        return f( k, *args ) * np.sinc( kr / np.pi ) * np.exp( -(kr / 100)**2 )

    r    = np.asfarray( r )
    args = ( r, *args )
    a, b = np.log( settings.ZERO ), np.log( settings.INF )
    out = _integrate( integrand, a, b, args = args )
    return out if np.ndim( r ) else out[0]

def _integrate(f: Callable, a: Any, b: Any, args: tuple = ()) -> Any:
    """
    Integrate a function using simpsons rule.
    """

    subdiv = settings.DEFAULT_SUBDIV

    pts  = int( 2**subdiv + 1 )
    x, h = np.linspace( a, b, pts, retstep = True, axis = -1 )
    y    = f( x, *args )
    return simpson(y, dx = h, axis = -1)


# register blueprint and start flask app
from flask import Flask, flash, url_for, redirect
from employee_management import EmployeeViews
from login import UserLoginAuth
from flask_login import LoginManager
from db.db import db
from models.login_models import User
from flask_bootstrap import Bootstrap

app = Flask(__name__, instance_relative_config=False)
login_manager = LoginManager()
login_manager.init_app(app)
bootstrap = Bootstrap(app)

app.secret_key = 'asrtarstaursdlarsn'
app.config.from_object('settings.Config')

# initialization
db.init_app(app)

app.register_blueprint(EmployeeViews.employee_app)
app.register_blueprint(UserLoginAuth.login_app)

@login_manager.user_loader
def load_user(user_id):
    """Check if user is logged-in on every page load."""
    if user_id is not None:
        return User.query.filter_by(id = user_id).first()
    return None

@login_manager.unauthorized_handler
def unauthorized():
    """Redirect unauthorized users to Login page."""
    flash('You must be logged in to view that page.')
    return redirect(url_for('login_auth.login'))

# run always put in last statement or put after all @app.route
if __name__ == '__main__':
    app.run(host='localhost')


import glob
import shutil
import os
import subprocess
import pathlib

zip_filename = "Bitmapflow"
dst_dir = "bin/Bitmapflow"

zip_filename_debug = "Bitmapflow_debug"
dst_dir_debug = "bin/Bitmapflow_debug"

# Delete folders
if os.path.exists(dst_dir):
    shutil.rmtree(dst_dir)
if os.path.exists(zip_filename):
    shutil.rmtree(zip_filename)
if os.path.exists(dst_dir_debug):
    shutil.rmtree(dst_dir_debug)
if os.path.exists(zip_filename_debug):
    shutil.rmtree(zip_filename_debug)

# Make folders
pathlib.Path(dst_dir).mkdir(parents=True, exist_ok=True)
pathlib.Path(dst_dir_debug).mkdir(parents=True, exist_ok=True)

# Copy all dlls
src_dir = "godot"
for dll in glob.iglob(os.path.join(src_dir, "*.dll")):
    shutil.copy(dll, dst_dir)
    shutil.copy(dll, dst_dir_debug)

# Export godot project (note: folder path is relative to the godot project folder, not current folder)
subprocess.run(["godot", "--no-window", "--export",
                "Windows Desktop", "../" + dst_dir + "/Bitmapflow.exe"], cwd="godot")
subprocess.run(["godot", "--no-window", "--export-debug",
                "Windows Desktop", "../" + dst_dir_debug + "/Bitmapflow_debug.exe"], cwd="godot")

# Zip it
print("Zipping...")
shutil.make_archive("bin/" + zip_filename, 'zip', dst_dir)

print("Success.")

print("Zipping debug...")
shutil.make_archive("bin/" + zip_filename_debug, 'zip', dst_dir_debug)

print("Success.")


import streamlit as st
import numpy as np
import pandas as pd
import math
from datetime import datetime, timedelta
import base64
import io

# configuration
st.set_option('deprecation.showfileUploaderEncoding', False)

# title of the app
st.title("Life Insurance Analytics ")

# Add a sidebar
st.sidebar.image('corplogo.PNG', use_column_width=True)
st.sidebar.subheader("Visualization Settings")

# Setup file upload
uploaded_file = st.sidebar.file_uploader(
    label="Upload your CSV or Excel file. (200MB max)",
    type=['csv', 'xlsx', 'xls']
)

if uploaded_file is not None:
    try:
        if uploaded_file.type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
            data_types = {'Policy No': str}
            df = pd.read_excel(uploaded_file, dtype = data_types , header = 4)

        elif uploaded_file.type == "text/csv":
            df = pd.read_csv(uploaded_file, header=4)

        df.dropna(subset=['Policy No'], inplace = True)

        df['Start Date'] = pd.to_datetime(df['Start Date'], format='%d/%m/%Y')

        # Change the format to year-month-day
        df['Start Date'] = df['Start Date'].dt.strftime('%Y-%m-%d')

        df['Start Date'] = pd.to_datetime(df['Start Date'])

        df['Maturity Date'] = pd.to_datetime(df['Maturity Date'], format='%d/%m/%Y')

        # Change the format to year-month-day
        df['Maturity Date'] = df['Maturity Date'].dt.strftime('%Y-%m-%d')

        df['Maturity Date'] = pd.to_datetime(df['Maturity Date'])

        #today = pd.Timestamp(datetime.today())
        today = datetime.today().strftime('%Y-%m-%d')
        df['Today'] = today
        df['Today'] = df['Today'].astype('datetime64[ns]')

        # Calculate the number of months
        df['Months(inception to maturity)'] = (df["Maturity Date"].dt.year - df["Start Date"].dt.year) * 12
        df["Monthly Premium"] = (df["Annual Premium"] / 12)
        df["Units Paid"] = df["Premium Received"]/df["Monthly Premium"]
        df["Units Outstanding"] = df['Months(inception to maturity)'] - df["Units Paid"]
        df["Total premium till maturity"] = df['Monthly Premium'] * df['Months(inception to maturity)']
        df["Premium Outstanding"] = df["Total premium till maturity"] - df["Premium Received"]

        df['Maturity Year'] = df['Maturity Date'].dt.year

        df['Maturity Month'] = df['Maturity Date'].dt.month_name()

    except Exception as e:
        st.write("Error:", e)

def create_download_link(df, filename):
    csv = df.to_csv(index=False)
    b64 = base64.b64encode(csv.encode()).decode()  # Convert DataFrame to CSV string
    href = f'Download {filename}'
    return href

# Define chart selection dropdown
chart_select = st.sidebar.selectbox(
            label="SELECT",
            options=["January 2023", "February 2023", "March 2023", "April 2023", "May 2023", "June 2023", "July 2023", "August 2023", "September 2023", "October 2023", "November 2023", "December 2023"]
        )

if uploaded_file is not None:
    if chart_select == "January 2023":
        # maturing in Jan 2023

        Jan = df[(df['Maturity Month'] == 'January') & (df['Maturity Year'] == 2023 )]
        Jan_number = len(Jan['Policy No'])

        # Select desired columns
        Jan = Jan.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Premium Outstanding']]
        # Save the DataFrame as a CSV file
        Jan.to_csv('january2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Jan, "january2023.csv"), unsafe_allow_html=True)

        Jan = Jan.to_html(index=False)
        # Add inline CSS to change font size
        Jan = Jan.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in January 2023")
        st.markdown (f"Total number of policies: **{Jan_number}**")
        st.markdown(Jan, unsafe_allow_html=True)

    elif chart_select == "February 2023":
        # maturing in Feb
        Feb = df[(df['Maturity Month'] == 'February') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Feb.to_csv('February2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Feb, "February2023.csv"), unsafe_allow_html=True)

        Feb_number = len(Feb['Policy No'])

        # Select desired columns
        Feb = Feb.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Premium Outstanding']]
        Feb = Feb.to_html(index=False)
        # Add inline CSS to change font size
        Feb = Feb.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in February 2023")
        st.markdown (f"Total number of policies: **{Feb_number}**")
        st.markdown(Feb, unsafe_allow_html=True)

    elif chart_select == "March 2023":
        # maturing in March
        Mar = df[(df['Maturity Month'] == 'March') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Mar.to_csv('March2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Mar, "March2023.csv"), unsafe_allow_html=True)
        Mar_number = len(Mar['Policy No'])

        # Select desired columns
        Mar = Mar.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Premium Outstanding']]
        Mar = Mar.to_html(index=False)
        # Add inline CSS to change font size
        Mar = Mar.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in March 2023")
        st.markdown (f"Total number of policies: **{Mar_number}**")
        st.markdown(Mar, unsafe_allow_html=True)

    elif chart_select == "April 2023":
        # maturing in April
        Apr = df[(df['Maturity Month'] == 'April') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Apr.to_csv('April2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Apr, "April2023.csv"), unsafe_allow_html=True)
        Apr_number = len(Apr['Policy No'])

        # Select desired columns
        Apr = Apr.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received','Premium Outstanding']]
        Apr = Apr.to_html(index=False)
        # Add inline CSS to change font size
        Apr = Apr.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in April 2023")
        st.markdown (f"Total number of policies: **{Apr_number}**")
        st.markdown(Apr, unsafe_allow_html=True)

    elif chart_select == "May 2023":
        # maturing in May
        May = df[(df['Maturity Month'] == 'May') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        May.to_csv('May2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(May, "May2023.csv"), unsafe_allow_html=True)
        May_number = len(May['Policy No'])

        # Select desired columns
        May = May.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received','Premium Outstanding']]
        May = May.to_html(index=False)
        # Add inline CSS to change font size
        May = May.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in May 2023")
        st.markdown (f"Total number of policies: **{May_number}**")
        st.markdown(May, unsafe_allow_html=True)

    elif chart_select == "June 2023":
        # maturing in June
        Jun = df[(df['Maturity Month'] == 'June') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Jun.to_csv('June2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Jun, "June2023.csv"), unsafe_allow_html=True)
        Jun_number = len(Jun['Policy No'])

        # Select desired columns
        Jun = Jun.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Months(inception to maturity)', 'Premium Outstanding']]
        Jun = Jun.to_html(index=False)
        # Add inline CSS to change font size
        Jun = Jun.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in June 2023")
        st.markdown (f"Total number of policies: **{Jun_number}**")
        st.markdown(Jun, unsafe_allow_html=True)

    elif chart_select == "July 2023":
        # maturing in July
        Jul = df[(df['Maturity Month'] == 'July') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Jul.to_csv('July2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Jul, "July2023.csv"), unsafe_allow_html=True)
        Jul_number = len(Jul['Policy No'])

        # Select desired columns
        Jul = Jul.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Premium Outstanding']]
        Jul = Jul.to_html(index=False)
        # Add inline CSS to change font size
        Jul = Jul.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in July 2023")
        st.markdown (f"Total number of policies: **{Jul_number}**")
        st.markdown(Jul, unsafe_allow_html=True)

    elif chart_select == "August 2023":
        # maturing in August
        Aug = df[(df['Maturity Month'] == 'August') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Aug.to_csv('August2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Aug, "August2023.csv"), unsafe_allow_html=True)
        Aug_number = len(Aug['Policy No'])

        # Select desired columns
        Aug = Aug.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Premium Outstanding']]
        Aug = Aug.to_html(index=False)
        # Add inline CSS to change font size
        Aug = Aug.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in August 2023")
        st.markdown (f"Total number of policies: **{Aug_number}**")
        st.markdown(Aug, unsafe_allow_html=True)

    elif chart_select == "September 2023":
        # maturing in September
        Sep = df[(df['Maturity Month'] == 'September') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Sep.to_csv('September2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Sep, "September2023.csv"), unsafe_allow_html=True)
        Sep_number = len(Sep['Policy No'])

        # Select desired columns
        Sep = Sep.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Units Paid', 'Premium Outstanding']]
        Sep = Sep.to_html(index=False)
        # Add inline CSS to change font size
        Sep = Sep.replace('<table', '<table style="font-size: 11px;"')

        # Display the DataFrame
        st.subheader(f"First Maturity in September 2023")
        st.markdown (f"Total number of policies: **{Sep_number}**")
        st.markdown(Sep, unsafe_allow_html=True)

    elif chart_select == "October 2023":
        # maturing in October
        Oct = df[(df['Maturity Month'] == 'October') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Oct.to_csv('October2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Oct, "October2023.csv"), unsafe_allow_html=True)
        Oct_number = len(Oct['Policy No'])

        Oct = Oct.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured','Annual Premium', 'Premium Received', 'Units Paid', 'Premium Outstanding']]
        Oct = Oct.to_html(index=False)
        # Add inline CSS to change font size
        Oct = Oct.replace('<table', '<table style="font-size: 11px;"')

         # Display the DataFrame
        st.subheader(f"First Maturity in October 2023")
        st.markdown (f"Total number of policies: **{Oct_number}**")
        st.markdown(Oct, unsafe_allow_html=True)

    elif chart_select == "November 2023":
        # maturing in November
        Nov = df[(df['Maturity Month'] == 'November') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Nov.to_csv('November2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Nov, "November2023.csv"), unsafe_allow_html=True)
        Nov_number = len(Nov['Policy No'])

        # Select desired colum
        Nov = Nov.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Premium Outstanding']]
        Nov = Nov.to_html(index=False)
        # Add inline CSS to change font size
        Nov = Nov.replace('<table', '<table style="font-size: 11px;"')

         # Display the DataFrame
        st.subheader(f"First Maturity in November 2023")
        st.markdown (f"Total number of policies: **{Nov_number}**")
        st.markdown(Nov, unsafe_allow_html=True)

    elif chart_select == "December 2023":
        # maturing in December
        Dec = df[(df['Maturity Month'] == 'December') & (df['Maturity Year'] == 2023 )]
        # Save the DataFrame as a CSV file
        Dec.to_csv('December2023.csv', index=False)

        # Create a download link for the CSV file
        st.markdown(create_download_link(Dec, "December2023.csv"), unsafe_allow_html=True)
        Dec_number = len(Dec['Policy No'])

        # Select desired column
        Dec = Dec.loc[:, ['Policy No', 'Insured', 'Status','Start Date', 'Maturity Date', 'Sum Insured', 'Annual Premium', 'Premium Received', 'Premium Outstanding']]
        Dec = Dec.to_html(index=False)

        # Add inline CSS to change font size
        Dec = Dec.replace('<table', '<table style="font-size: 11px;"')

         # Display the DataFrame
        st.subheader(f"First Maturity in December 2023")
        st.markdown (f"Total number of policies: **{Dec_number}**")
        st.markdown(Dec, unsafe_allow_html=True)

    else:
        st.write("Failed to load data from the uploaded file.")

else:
    st.write("Please upload a file to visualize.")


'''
Učitajte sliku “kamera1.png”, te detektirajte rubove pomoću Canny detektora rubova. Koje su
razlike u odnosu na sliku iz prethodnog zadatka u kojem smo koristili Sobel operator?
'''

import cv2

img = cv2.imread('kamera1.png')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

canny = cv2.Canny(gray, 150, 220)

cv2.imshow('Original', img)
cv2.imshow('Canny', canny)

cv2.waitKey(0)
cv2.destroyAllWindows()


import networkx as nx
import numpy as np

def parseInstanceFile(instanceFilePath):
	"""
		Reads a Pace instance file containing an instance of the Vertex cover problem.
		The file format is described here: https://pacechallenge.org/2019/vc/vc_format/
		The instance is converted into a networkx Digraph instance

		Arguments
		---------

		instanceFilePath : str
			Path of the instance file to parse.

		Returns
		-------

		instance : nx.Graph
			Networkx Graph structure containing the graph
	"""
	file = open(instanceFilePath,"r")

	graph = nx.Graph()

	# parse info about edges and vertices
	line = file.readline()
	numVertices = 0
	numEdges = 0
	if line.startswith("p td "):
		dat = line[len("p td "):-1].split()
		numVertices = int(dat[0])
		numEdges = int(dat[1])
	else:
		raise Exception('{} is not a proper STP file: does not contain proper first line identifier.'.format(instanceFilePath))

	# create nodes
	graph.add_nodes_from(range(numVertices))

	# parse edges
	line = file.readline()
	edges = np.empty((numEdges, 2), dtype=int)
	i=0
	while line:
		dat = line.split()
		edges[i][0]=int(dat[0])
		edges[i][1]=int(dat[1])
		i+=1
		line = file.readline()

	# add edges to the graph
	graph.add_edges_from([(data[0], data[1]) for data in edges])

	return graph


from typing import List

class Solution:
    def minSwaps(self, data: List[int]) -> int:
        cnt = sum(data)
        MAX = m = sum(data[0:cnt])
        for i in range(cnt, len(data)):
            m += data[i] - data[i - cnt]
            MAX = max(MAX, m)
        return cnt - MAX

if __name__ == '__main__':
    assert Solution().minSwaps([1, 0, 1, 0, 1]) == 1
    assert Solution().minSwaps([0, 0, 0, 1, 0]) == 0
    assert Solution().minSwaps([1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1]) == 3

LeetCode-Algorithm/1151. Minimum Swaps to Group All 1's Together/1151.py


from PIL import Image
import os
from django.conf import settings
from .models import *
from datetime import datetime as dt

def create_tumbnail(user, location, name):
    '''Creates thumbnails from originally upload files based on users cuirrent Plan'''
    size = UserDetail.objects.get(user=user).plan.size.all()
    new_path = dt.strftime(dt.now(), "images/%Y%m%d")
    for s in size:
        image = Image.open(os.path.join(settings.BASE_DIR)+"/"+location)
        image.thumbnail((s.thumbnail_width, s.thumbnail_height))
        save_dir = (os.path.join(settings.BASE_DIR, 'media/', new_path,
                                 'thumbnail'+str(s.thumbnail_width)+'x'+str(s.thumbnail_height)+'_'+str(user)+'_'+str(name))).replace("\\", "/")
        image.save(save_dir)
        file_uploaded = (os.path.join(new_path,
                                      'thumbnail'+str(s.thumbnail_width)+'x'+str(s.thumbnail_height)+'_'+str(user)+'_'+str(name))).replace("\\", "/")
        newTnail = UploadedFiles.create_new_thumbnail(
            data={
                "user": user,
                "file_uploaded": file_uploaded,
                "original": False
            })

        UploadedFiles.add_plan_to_file(
            UserDetail.objects.get(user=user).plan, newTnail)


# RBE 500 F01 Group Project
# November 16, 2022

import rclpy
import numpy as np
from scipy.spatial.transform import Rotation as R
from math import cos, sin, atan2, sqrt, acos
from rclpy.node import Node
from std_msgs.msg import Float32MultiArray
from geometry_msgs.msg import Pose
from sensor_msgs.msg import JointState

class Subscriber(Node):

    def __init__(self):

        super().__init__('Subscriber')

        # define the link lengths as given in the Gazebo setup
        self.p1=2
        self.p2=1
        self.p3=1

	# create the subscriber for calculating the end effector position as well as the publisher for publishing said pose
        self.listening = self.create_subscription(JointState, '/joint_states', self.calculate_forward_kinematics, 10)
        self.publishing = self.create_publisher(Pose, 'EndEffectorPose', 10)

    def calculate_forward_kinematics(self, mesgfrompub):

	# collect joint values from the published message
        joint_positions = mesgfrompub.position

	# define each joint value based on inputs
        theta1 = joint_positions[0]
        theta2 = joint_positions[1]
        disp3 = joint_positions[2]

        joints = [theta1, theta2, disp3]

        link_lengths = [self.p1, self.p2, self.p3]

	# create T30 transformation matrix. T30 = A1*A2*A3 which can be seen in report hand written work as well as Matlab code
        T30 = np.array([[((cos(theta1)*cos(theta2))-(sin(theta1)*sin(theta2))), ((-cos(theta1)*sin(theta2))-(cos(theta2)*sin(theta1))), 0, ((self.p2*cos(theta1)) + (self.p3*cos(theta1)*cos(theta2)) - (self.p3*sin(theta1)*sin(theta2)))],
        [((cos(theta1)*sin(theta2)) + (cos(theta2)*sin(theta1))), ((cos(theta1)*cos(theta2)) - (sin(theta1)*sin(theta1))), 0, ((self.p2*sin(theta1)) + (self.p3*cos(theta1)*sin(theta2)) + (self.p3*cos(theta2)*sin(theta1)))],
        [0, 0, 1, (self.p1 + disp3)],
        [0, 0, 0, 1]])

	# rotation matrix is the first 3 rows and columns of the overall transformation matrix
        rotation_matrix = np.array([[T30[0][0], T30[0][1], T30[0][2]], [T30[1][0], T30[1][1], T30[1][2]], [T30[2][0], T30[2][1], T30[2][2]]])
        # using python package to extract / calculate the quaternion from the transformation matrix. This is so that the full pose can be published
        # methods used reference: https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.transform.Rotation.html
        quaternion = R.from_matrix(rotation_matrix).as_quat()

    	# filling in the pose from calculated values
        pose = Pose()
        pose.position.x = T30[0][3]
        pose.position.y = T30[1][3]
        pose.position.z = T30[2][3]
        pose.orientation.x = quaternion[0]
        pose.orientation.y = quaternion[1]
        pose.orientation.z = quaternion[2]
        pose.orientation.w = quaternion[3]

 	# publish the Pose
        self.publishing.publish(pose)
        print(pose)

def main(args=None):

    rclpy.init(args=args)
    GroupProject_subscriber = Subscriber()
    rclpy.spin(GroupProject_subscriber)

if __name__ == '__main__':

    main()


#!/usr/bin/env python3
"""
This script takes data files from Wildfire, and make them into a Stracciatella mod.

The script expects the presence of all required SLF files, and the destination
directory to be clean.
"""

import argparse
import colorama
import os
import traceback
import sys
from colorama import Fore, Back, Style
from fs.mountfs import MountFS
from fs.osfs import OSFS
from pathlib import Path

from ja2py.fileformats import SlfFS
from ja2py.fileformats.Sti import load_16bit_sti

DIRS_TO_UNPACK = [
    "Maps",
    "RadarMaps",
    "TileSets"
]

FILES_TO_EXTRACT = [
    "JA2set.dat",
    "MilitiaMaps.sti",
    "b_map.sti",
    "069.npc", # VINCE
    "080.npc", # STEVE
    "093.npc", # SPIKE
    "117.npc", # HANS
    "132.npc", # JENNY
]

FILES_TO_EXCLUDE = [
    "Tilesets/0/smguns.sti",
    "Tilesets/0/SMITEMS.STI",
    "Tilesets/0/smp1items.sti",
    "Tilesets/0/smp2items.sti",
    "Tilesets/0/SMP3ITEMS.STI",
]

is_verbose = False

OK_LABEL    = Fore.GREEN  + Style.BRIGHT + '[OK]'    + Style.RESET_ALL
WARN_LABEL  = Fore.YELLOW + Style.BRIGHT + '[WARN]'  + Style.RESET_ALL
ERROR_LABEL = Fore.RED    + Style.BRIGHT + '[ERROR]' + Style.RESET_ALL

def main():
    global is_verbose
    colorama.init()

    """Checking the installation environment, and installs if OK"""
    parser = argparse.ArgumentParser(description='Install Wildfire mod')
    parser.add_argument(
        '--work_dir', help="path to the mod's Data directory", default='./Data', required=False
    )
    parser.add_argument(
        '--src_dir', help="path to the source Wildfire data directory", default='', required=False
    )
    parser.add_argument(
        '--verbose', help='display detailed error messages', action='store_true'
    )
    args = parser.parse_args()

    is_verbose = args.verbose
    work_dir = Path(args.work_dir)
    src_dir = determine_src_path(args.src_dir)

    preflight_checks(src_dir, work_dir)

    for slf in DIRS_TO_UNPACK:
        unpack_slf(src_dir, work_dir, slf)

    for res in FILES_TO_EXTRACT:
        extract_single_resource(src_dir, work_dir, res)

    for f in FILES_TO_EXCLUDE:
        delete_file(work_dir, f)

    convert_bmap(work_dir)
    replace_maps(work_dir)

    print(Fore.GREEN + "Installation finished" + Style.RESET_ALL)
    if os.name == 'nt':
        input("\nPress enter to close...")

def determine_src_path(src_dir):
    if src_dir:
        # given in command args
        return Path(src_dir)

    try:
        # not given, try opening a prompt dialog
        from tkinter import Tk
        from tkinter.filedialog import askdirectory
        tk = Tk()
        dir = askdirectory(title='Select Wildfire data folder', mustexist =True)
        tk.destroy()
        if dir:
            return Path(dir)
    except:
        # ignore errors, maybe there is no display
        pass

    print("You must provide the Wildfire source data directory.")
    sys.exit(1)

def preflight_checks(src_path, work_path):
    """Check we have all the data resource files are present in the right place"""
    has_errors = False
    print("Using " + str(src_path.resolve()) + " as source directory.")
    print("Checking for data files...")

    required_slf_files = DIRS_TO_UNPACK
    required_slf_files = set(required_slf_files)

    if not (work_path).is_dir():
        raise RuntimeError("Destination Data/ directory does not exist. You should run the installer from the mod directory")

    for slf in required_slf_files:
        file_name = slf + ".slf"
        file_name = resolve_case_insensitive_path(src_path, file_name)
        if (src_path / file_name).is_file():
            print("  * %s %s exists" % (OK_LABEL, file_name))
        else:
            has_errors = True
            print("  * %s %s does not exist in %s" % (ERROR_LABEL, file_name, str(src_path.resolve())))
        if (work_path / slf).is_dir():
            print("  * %s Destination directory %s/ already exists" % (WARN_LABEL, slf))

    if has_errors:
        raise RuntimeError("Some data files or directories are missing. Please check the README and make sure you" +
            " have all the required data files.")
    else:
        print("Pre-flight checks OK... Proceeding to installation")

def unpack_slf(src_path, work_path, slf_name):
    """Unpacks the entire SLF file at the current directory"""
    combined_fs = open_slf_for_copy(src_path, work_path, slf_name)

    print("  * Unpacking " + slf_name)
    combined_fs.copydir('/slf', '/out', overwrite=True)
    combined_fs.close()

def delete_file(work_path, file_path):
    """We are only interested in maps, delete any other un-needed files."""
    file_path = resolve_case_insensitive_path(work_path, file_path)
    print("  * Deleting " + str(file_path))
    (work_path / file_path).unlink()

def extract_single_resource(src_path, work_path, resource_path):
    """Extracts a single resource file from any SLF in the source path"""
    slf_name, resource_path = find_slf_that_contains_resource(src_path, resource_path)
    if slf_name is None or resource_path is None:
        raise RuntimeError("Could not find SLF file that contains " + resource_path)
    combined_fs = open_slf_for_copy(src_path, work_path, slf_name)

    print("  * Extracting " + slf_name + "/" + resource_path)
    combined_fs.copy('/slf/' + resource_path, '/out/' + resource_path, overwrite=True)
    combined_fs.close()

def find_slf_that_contains_resource(src_path, resource_path):
    """Finds the SLF that contains a resource_path and returns case sensitive slf file name and resource path"""
    slf_files = [f for f in os.listdir(src_path) if os.path.isfile(src_path / f) and f.lower().endswith(".slf")]
    for slf_filename in slf_files:
        slf_fs = SlfFS(str(src_path / slf_filename))
        for r in slf_fs.listdir("/"):
            if r.lower() == resource_path.lower():
                return slf_filename[:-4], r
    return None, None

def open_slf_for_copy(src_path, dest_path, slf_name):
    """Opens an SLF files for reading and returns a MountFS"""
    slf_filename = resolve_case_insensitive_path(src_path, slf_name + ".slf")
    slf_file = src_path / slf_filename
    slf_fs = SlfFS(str(slf_file))

    output_dir = dest_path / slf_name
    output_dir.mkdir(exist_ok=True)
    out_fs = OSFS(output_dir)

    combined_fs = MountFS()
    combined_fs.mountdir('slf', slf_fs)
    combined_fs.mountdir('out', out_fs)

    return combined_fs

def convert_bmap(work_path):
    """Converts the bmap.sti in Wildfire to bmap.pcx which is needed by Vanilla/JA2:S"""
    interface_dir = resolve_case_insensitive_path(work_path, "Interface")
    sti_filename = resolve_case_insensitive_path(work_path / interface_dir, "b_map.sti")
    sti_file = work_path / interface_dir / sti_filename
    output_file = work_path / interface_dir / "b_map.pcx"

    print("  * Converting InterFace/b_map.{sti => pcx}")
    with open(sti_file, 'rb') as file:
        sti = load_16bit_sti(file)
        image = sti.image.convert("P", colors=8)
        image.save(output_file)

def replace_maps(work_path):
    """Work-around some known issues with certain sectors."""
    # We want the alternate map for G6, but no logic in Vanilla codebase to switch
    print("  * Replacing G6.dat with alternate")
    maps_dir = resolve_case_insensitive_path(work_path, "maps")
    map_filename = resolve_case_insensitive_path(work_path / maps_dir, "g6_a.dat")
    (work_path / maps_dir / map_filename).replace(work_path / maps_dir / "g6.dat")

def resolve_case_insensitive_path(directory, path):
    parts = Path(path).parts
    current = directory
    path = ""
    for p in parts:
        listing = os.listdir(current)
        found = False
        for l in listing:
            if p.lower() == l.lower():
                current = os.path.join(current, l)
                path = os.path.join(path, l)
                found = True
                break;
        if not found:
            current = os.path.join(current, p)
            path = os.path.join(path, p)
    return Path(path)

if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        print("\n" + Fore.RED + str(e) + Fore.RESET)
        if is_verbose:
            print(Style.DIM + Fore.BLUE + "\nError traceback:\n")
            traceback.print_exc()
            print(Style.RESET_ALL)
        print("\nInstallation aborted with an error.")
        if os.name == 'nt':
            input("\nPress enter to close...")

        exit(1)


import os
import pandas as pd
from sklearn.metrics import accuracy_score, roc_auc_score, f1_score, precision_recall_curve, confusion_matrix, auc
import numpy as np
from imblearn.metrics import geometric_mean_score

import Confs as Confs
from BuildBaseModels import build_base_models
from BuildEnsembleNN import build_ensemble_model
from BuildInputTensors import extract_test_tensors, strat_shuffle_split_training, extract_tensor
from BuildInputTensors import extract_train_tensors, read_training_data_tensors, split_training
from Utils import compute_accuracy, compute_AUC
from Utils import load_model_and_compile, load_models_and_compile

def train_models(dataset_file, results_path, model_names, text_col_name,  label_col_name, aggressive_opt=True):
    csv_separator_char = Confs.parameters_general["csv_separator_char_training"]

    if not os.path.exists(results_path):
        os.makedirs(results_path)

    datatensors_path = os.path.join(results_path, 'input_tensors/')

    # Read input data
    if os.path.exists(os.path.join(datatensors_path, "data.npy")):
        data, labels = read_training_data_tensors(datatensors_path)
    else:
        extract_train_tensors(dataset_file, datatensors_path, csv_separator_char, text_col_name=text_col_name,
                              label_col_name=label_col_name)
        data, labels = read_training_data_tensors(datatensors_path)

    # Split input data
    x_train_base, y_train_base, _, _, x_test, y_test = strat_shuffle_split_training(data, labels)

    # Build base models
    valPerc = Confs.parameters_general["valPerc"]

    print("Building base models...")
    base_models = build_base_models(x_train_base, y_train_base, x_test, y_test, results_path, model_names,
                                    aggressive_opt, valPerc=valPerc)

def eval_single_model(y_test, y_pred, cut_off=0.5, decimal_numbers=5):
    num_classes = Confs.parameters_general['num_classes']
    if num_classes == 2:
        y_pred = np.ravel(y_pred)
        y_pred_round = y_pred >= cut_off
        acc = round(accuracy_score(y_test, y_pred_round), decimal_numbers)
        auc_score = round(roc_auc_score(y_test, y_pred), decimal_numbers)
        g_mean = round(geometric_mean_score(y_test, y_pred_round), decimal_numbers)
        f1 = round(f1_score(y_test, y_pred_round), decimal_numbers)
        pr1, rec1, thr1 = precision_recall_curve(y_test, y_pred)
        auc_score_pr = auc(rec1, pr1)
        print(confusion_matrix(y_test, y_pred_round))
        conf_matrix_file = Confs.parameters_general['conf_matrix_file']
        f_conf_matrix_file = open(conf_matrix_file, "a+")
        f_conf_matrix_file.write(str(confusion_matrix(y_test, y_pred_round)))
        f_conf_matrix_file.write('\n')
        f_conf_matrix_file.close()
    else:
        y_pred_vec = np.argmax(y_pred, axis=1)
        y_test_vec = np.argmax(y_test, axis=1)
        acc = round(accuracy_score(y_test_vec, y_pred_vec), decimal_numbers)
        auc_score = round(roc_auc_score(y_test, y_pred, multi_class='ovo'), decimal_numbers)
        g_mean = round(geometric_mean_score(y_test_vec, y_pred_vec, average='macro'), decimal_numbers)
        f1 = round(f1_score(y_test_vec, y_pred_vec, average='macro'), decimal_numbers)
        print(confusion_matrix(y_test_vec, y_pred_vec))
        conf_matrix_file = Confs.parameters_general['conf_matrix_file']
        f_conf_matrix_file = open(conf_matrix_file, "a+")
        f_conf_matrix_file.write(str(confusion_matrix(y_test_vec, y_pred_vec)))
        f_conf_matrix_file.write('\n')
        f_conf_matrix_file.close()
    return ''.join(["{:0.5f}".format(acc), "\t", "{:0.5f}".format(auc_score), "\t", "{:0.5f}".format(g_mean), "\t", "{:0.5f}".format(f1)])

def train_ensemble(dataset_file, results_path, combiner_mode, base_models_path=None, aggressive_opt=True,
                   text_col_name='SentimentText', label_col_name='label_col_name'):
    csv_separator_char = Confs.parameters_general["csv_separator_char_training"]

    if not os.path.exists(results_path):
        os.makedirs(results_path)

    if base_models_path is None:
        base_models_path = os.path.join(results_path, "base_models/")

    datatensors_path = os.path.join(results_path, 'input_tensors/')

    # Read input data
    if os.path.exists(os.path.join(datatensors_path, "data.npy")):
        data, labels = read_training_data_tensors(datatensors_path)
    else:
        extract_train_tensors(dataset_file, datatensors_path, csv_separator_char, text_col_name=text_col_name,
                              label_col_name=label_col_name)
        data, labels = read_training_data_tensors(datatensors_path)

    # Split input data
    _, _, x_train_ens, y_train_ens, x_test, y_test = strat_shuffle_split_training(data, labels)

    # Build base models
    valPerc = Confs.parameters_general["valPerc"]

    #print("Loading base models...")
    base_models = load_models_and_compile(base_models_path, Confs.parameters_general)

    output_file = "results_ensemble_" + combiner_mode + ".txt"

    name_results_file = os.path.join(results_path, output_file)
    f = open(name_results_file, "w")

    all_results_file = Confs.parameters_general['result_path_file']
    f_all_results = open(all_results_file, "a+")
    # ***** PRINT ACCURACY RESULTS: BASE MODELS
    f.write("model\tacc\tauc\tg-mean\tF1\n")
    for model in base_models:
        acc = compute_accuracy(model, x_test, y_test, verbose=2)
        print('Model ' + model.name)
        print('Accuracy ' + str(acc))
        f.write(model.name + '\t')

    # Build ensemble model
    print("Building the " + combiner_mode + " ensemble schema...")
    ensemble_model = build_ensemble_model(x_train_ens, y_train_ens, x_test, y_test, base_models, combiner_mode, results_path, valPerc,
                                          aggressive_opt)

    # reload saved models
    if (Confs.parameters_general["save_and_reload_models_for_accuracy_and_metalearner"]):
        ensemble_model_path = os.path.join(results_path, 'ensemble_model/')
        ensemble_model_filepath = os.path.join(ensemble_model_path, ensemble_model.name + '.h5')
        del ensemble_model
        ensemble_model = load_model_and_compile(ensemble_model_filepath, Confs.parameters_ensemble)

    # ***** PRINT ACCURACY RESULTS: ENSEMBLE
    f.write(ensemble_model.name + '\t')
    f_all_results.write(ensemble_model.name + '\t')
    conf_matrix_file = Confs.parameters_general['conf_matrix_file']
    f_conf_matrix_file = open(conf_matrix_file, "a+")
    f_conf_matrix_file.write(ensemble_model.name + '\n')
    f_conf_matrix_file.close()

    print("\n**********\nENSEMBLE accuracy: model " + ensemble_model.name)
    # make predictions on new text
    y_pred = ensemble_model.predict(x_test)
    score = eval_single_model(y_test, y_pred, cut_off=0.5, decimal_numbers=5)
    print(score)
    print('\n')
    f.write(score)
    f_all_results.write(score)
    f.write("\n")
    f_all_results.write("\n")
    f.close()
    f_all_results.close()

def train_base(dataset_file, results_path, base_models_path=None,
                   text_col_name='text_col_name', label_col_name='label_col_name'):
    csv_separator_char = Confs.parameters_general["csv_separator_char_training"]

    if not os.path.exists(results_path):
        os.makedirs(results_path)
    if base_models_path is None:
        base_models_path = os.path.join(results_path, "base_models/")
    datatensors_path = os.path.join(results_path, 'input_tensors/')
    # Read input data
    if os.path.exists(os.path.join(datatensors_path, "data.npy")):
        data, labels = read_training_data_tensors(datatensors_path)
    else:
        extract_train_tensors(dataset_file, datatensors_path, csv_separator_char, text_col_name=text_col_name,
                              label_col_name=label_col_name)
        data, labels = read_training_data_tensors(datatensors_path)
    _, _, x_train_ens, y_train_ens, x_test, y_test = strat_shuffle_split_training(data, labels)

    base_models = load_models_and_compile(base_models_path, Confs.parameters_general)
    all_results_file = Confs.parameters_general['result_path_file']
    f_all_results = open(all_results_file, "a+")
    for model in base_models:
        f_all_results.write(model.name + '\t')
        conf_matrix_file = Confs.parameters_general['conf_matrix_file']
        f_conf_matrix_file = open(conf_matrix_file, "a+")
        f_conf_matrix_file.write(model.name + '\n')
        f_conf_matrix_file.close()
        y_pred = model.predict(x_test)
        score = eval_single_model(y_test, y_pred, cut_off=0.5, decimal_numbers=5)
        f_all_results.write(score)
        f_all_results.write("\n")
    f_all_results.close()

def test(dataset_file, results_path, predictor_path, text_col_name, label_col_name='prediction'):
    csv_separator_char = Confs.parameters_general["csv_separator_char_test"]
    datatensors_path = os.path.join(results_path, 'input_tensors/')
    if ("ensemble" in predictor_path):
        parameters = Confs.parameters_ensemble
    elif ("lstm" in predictor_path):
        parameters = Confs.parameters['lstm']
    elif('cnn' in predictor_path):
        parameters = Confs.parameters['cnn']

    # Read new data
    x_test, texts = extract_test_tensors(dataset_file, datatensors_path, csv_separator_char, text_col_name)

    # Load the prediction model
    model = load_model_and_compile(predictor_path, parameters)

    # make predictions on new text
    pred_labels = model.predict(x_test)

    pred_labels = [pred for pred in pred_labels]

    # build a dataframe with two columns text, pred_label
    data_tuples = list(zip(texts, pred_labels))
    df = pd.DataFrame(data_tuples, columns=[text_col_name, label_col_name])

    # save dataframe on disk as a csv for future consultation
    output_file_name = model.name +"_preds.csv"
    output_file_path = os.path.join(results_path, output_file_name)
    df.to_csv(path_or_buf=output_file_path, index=False, sep='|', encoding=Confs.parameters_general["encoding"])

class TextPreprocessor(object):
    def __init__(self, datatensors_path, transform_single = True):
        self.datatensors_path = datatensors_path
        self.transform_single = transform_single
    def fit(self):  # comply with scikit-learn transformer requirement
        return self

    def transform(self, text):  # comply with scikit-learn transformer requirement
        tensor = extract_tensor(text, self.datatensors_path)
        if self.transform_single:
            tensor = np.array(tensor,)
        return tensor


import numpy as np

def load_TIRadarHeatmap(heatmap_path):
    '''
    read TI radar heatmap
    :param heatmap_path: str - TI radar heatmap path
    :return: dict(np.array)
    '''
    data = np.fromfile(heatmap_path, dtype='float32')
    print(data.shape)
    data = data.reshape((4*257, 232), order='F')
    data = data.reshape((4, 257, 232))
    res = {
        "heatmap_static": data[0, :, :],
        "heatmap_dynamic": data[1, :, :],
        "x_bins": data[2, :, :],
        "y_bins": data[3, :, :],
    }
    return res

pth = './Dataset/20211027_1_group0021/group0021_frame0000/TIRadar/1635319097.410.heatmap.bin'
res = load_TIRadarHeatmap(pth)
x_bins = res['x_bins']
y_bins = res['y_bins']
static = res['heatmap_static']
dynamic = res['heatmap_dynamic']
print(x_bins.shape)
print(static.shape)
print(dynamic.shape)

import matplotlib.pyplot as plt

print(np.arctan2(y_bins[20, :], x_bins[20, :]) / (np.pi) * 180)
# print(y_bins[-1, :], y_bins[-1, :].shape)
plt.imshow(dynamic)
plt.show()
print(np.max(y_bins))
n = 1
map = np.zeros((151*n, 76*n))
print(map.shape)
for i in range(x_bins.shape[0]):
    for j in range(x_bins.shape[1]):
       map[(int(np.ceil(x_bins[i,j])) + 75)*n, int(np.ceil(y_bins[i,j]))*n] = dynamic[i,j]

data = map.T[::-1,:]
print(data.shape)
# data_split = data[55:, 50:100]
data_split = data
print(np.max(data_split), np.min(data_split))
d = data_split / np.max(data_split)
# plt.imshow(d, interpolation='gaussian')
import cv2
# d = cv2.GaussianBlur(d, (3, 3), 1)
plt.imshow(d)
plt.show()
dd = np.where(d, 1, 0)
plt.imshow(dd)
plt.show()
# import seaborn as sns
# sns.heatmap(d)
# plt.show()


def deco(func):
  import time
  import functools
  import math

  @functools.wraps(func)
  def inner(*args, **kwargs):
    start = time.time()
    res = func(*args, **kwargs)
    finish = time.time()

    delta = (finish - start)
    print(f'{func.__name__} works for {delta:2.3}')
    return res
  return  inner

@deco
def twoSumSlice( nums, target):
    result=[]
    for i in range(len(nums)):
        if target-nums[i] in nums[i+1::]:
            result.append(i)
            result.append(nums[i+1::].index((target-nums[i]))+i+1)
    return result

@deco
def twoSumDict(nums, target):
    collected = {}
    for index, val in enumerate(nums):
        diff = target - val
        if diff in collected:
            return [collected[diff], index]
        collected[val] = index
        #print(collected)

if __name__ == "__main__":
  print(twoSumSlice((80,5,2,7,9,11), 9))
  print(twoSumDict((80,5,2,7,9,11), 9))
  assert twoSumSlice([2, 7, 11, 15], 9) == [0, 1], 'проверка методом со слайсом'
  assert twoSumDict([2, 7, 11, 15], 9) == [0, 1], 'проверка методом со словарём'


import json
import cherrypy
import json_rpc
import subprocess

class Module (object):

    # Expose secure when we get admin-secret working
    @json_rpc.expose_anonymous
    def run(self):
        if not cherrypy.request.method in ['POST', 'PUT']:
            raise cherrypy.HTTPError(405)

        command = cherrypy.request.body.read().decode()
        try:
            code = 0
            results = subprocess.check_output(command, shell=True)
        except subprocess.CalledProcessError as e:
            cherrypy.response.status = 500
            code = e.returncode
            results = e.output

        return {'code': code, 'output': results.decode().strip()}


#!/usr/local/bin/python3
import re
import sys
import pyperclip

template_file = sys.argv[1]

file = open(template_file,mode='r')
text = file.read()
file.close()

replacements = {}
placeholders = re.findall('\{(.*?)\}', text)
for placeholder in placeholders:
    try:
        namevariable,default_value = placeholder.split('|')
        if namevariable in replacements: continue
        user_input = input(f"{namevariable}: [{default_value}]?")
        if user_input == '':
            user_input = default_value

        text = text.replace(f"{{{placeholder}}}", user_input)

    except ValueError:
        namevariable = placeholder
        if namevariable in replacements: continue
        user_input = input(f"{namevariable}?")

        text = text.replace(f"{{{placeholder}}}", user_input)

    replacements[namevariable] = user_input

print(text)
pyperclip.copy(text)


#!/usr/bin/env python
from setuptools import setup

with open('README.md') as f:
    long_description = f.read()

setup(
    name='django-nucleus',
    description='Theme for Django admin',
    short_description='Theme for Django admin',
    long_description=long_description,
    long_description_content_type='text/markdown',
    version='0.1.2',
    url='http://github.com/lukasvinclav/django-nucleus',
    packages=[
        'nucleus',
    ],
    include_package_data=True,
    install_requires=('django', ),
    classifiers=[
        'Programming Language :: Python',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Operating System :: OS Independent',
        'Environment :: Web Environment',
        'Intended Audience :: Developers',
        'Framework :: Django',
    ],
    keywords="django admin theme clean minimal ui sidebar"
)


import math
import numpy as np
import matplotlib.pyplot as plt
from collections.abc import MutableMapping
from fractions import Fraction
from mpmath import quad
from mpl_toolkits.mplot3d import axes3d, Axes3D
from matplotlib import cm

class Stack(MutableMapping):
    '''
    Implements a stack holding variable bindings for execution of translated Maxima code.
    Provides access via Python dictionary like bindings.
    '''
    def __init__(self, data={}, sub={}):
        '''
        data parameter is a dictionary holding values for the current stack frame.
        sub is either an instance of Stack or dictionary, holding values at stack frames below the current one.
        '''
        self.mapping = data
        self.sub = sub

    def __getitem__(self, key):
        if key in self.mapping:
            return self.mapping[key]
        else:
            return self.sub[key]

    def __delitem__(self, key):
        if key in self.mapping:
            del self.mapping[key]
        else:
            del self.sub[key]

    def __setitem__(self, key, value):
        if key in self.mapping:
            self.mapping[key] = value
        else:
            self.sub[key] = value
        return(value)

    def __iter__(self):
        return iter({**self.sub, **self.mapping})

    def __len__(self):
        return len(self.mapping) + len(self.sub)

    def __repr__(self):
        return f"{self.mapping}, sub:{self.sub}"

    def ins(self, data):
        self.mapping={**self.mapping, **data}

# v contains the standard variable mapping from Maxima and is the base for all Stack instances for translated code
v = {
    'fpprec': 16,
    'pi': math.pi,
    'e': math.e,
    'ratepsilon': 2.0E-15
}

def plot2d(mapping, *constraints, v = v):
    '''
    provides functionality for plotting functions in 2D plane.
    mapping is either:
    1) A single function/lambda taking one input in the domain defined by constraints.
    2) List of functions/lambda as defined in 1 above.

    Example:
    plot2d(lambda x: x**2, ['x', 0, 1])
    where 0 and 1 are the lower and upper bounds respectively.
    '''
    plt.ion()
    if type(mapping) != list:
        mapping = [mapping]
    for expr in mapping:
        if len(constraints) == 1:
            X = np.arange(constraints[0][1],
                          constraints[0][2],
                          0.0001,
                          dtype = 'float')
            Y = np.array([expr(xi) for xi in X])
            plt.plot(X, Y)
    plt.draw()

def plot3d(mapping, *constraints, v = v):
    '''
    provides functionality for plotting functions in 3D plane.
    mapping is either:
    1) A single function/lambda taking two inputs in the domain defined by constraints.
    2) List of functions/lambda as defined in 1 above.

    Example:
    plot2d(lambda x, y: x**2 + y**2, ['x', 0, 1], ['y', 0, 2])
    where 0 and 1 are the lower and upper bounds respectively for first input and
    0 and 2 are the lower and upper bounds respectively for the second input.
    '''
    fig = plt.figure()
    ax = Axes3D(fig)
    plt.ion()
    if type(mapping) != list:
        mapping = [mapping]
    for expr in mapping:
        vexpr = np.vectorize(expr)
        if len(constraints) == 2:
            X = np.arange(constraints[0][1],
                          constraints[0][2],
                          0.01,
                          dtype = 'float')
            Y = np.arange(constraints[1][1],
                          constraints[1][2],
                          0.01,
                          dtype = 'float')
            X, Y = np.meshgrid(X, Y)
            Z = vexpr(X, Y)
            surf = ax.plot_surface(X, Y, Z, cmap=cm.coolwarm, linewidth=0, antialiased=False)
    plt.show()

# f contains the function mapping used by translated functions from Maxima
f = {
    'sin': math.sin,
    'pow': math.pow,
    'factorial': math.factorial,
    'floor': math.floor,
    'sqrt': math.sqrt,
    'num':lambda x:x,
    'denom': lambda x:1,
    'print': print,
    'listp': lambda x: type(x) == list,
    'numberp': lambda x: (type(x) == int or
                          type(x) == float or
                          type(x) == Fraction or
                          type(x) == np.float64 or
                          type(x) == np.float128 or
                          type(x) == np.float32 or
                          type(x) == np.float16 or
                          type(x) == np.float),
    'length': len,
     # As defined in "Concrete Mathematics", Section 3.4
    'mod': lambda x,y: (x if y == 0 else x - y * math.floor(x / y)),
    'emptyp': lambda x: (True if len(x) == 0 else False),
    'first': lambda x: x[0],
    'integerp': lambda x: type(x) == int,
    'append': lambda *x: [i for l in x for i in l],
    'plot2d': plot2d,
    'plot3d': plot3d,
    'map': lambda f, i: list(map(f, i)),
    'abs': abs,
    'every': lambda func, l: all(map(func, l)),
    'quad_qagi': quad,
    'cos': math.cos,
    'float': float,
    'signum': lambda x: 0 if x==0 else x/abs(x)
}

def assign(lhs, rhs, v = v):
    v[lhs] = rhs
    return rhs


from django.urls import include, path
from django.contrib.auth import views as auth_views

from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('read/<int:id>', views.read, name='read'),
    path('delete/<int:id>', views.delete, name='delete'),
    path('update/<int:id>', views.update, name='update'),
    path('read/<int:id>', views.read, name='read'),
    path('read/', views.index, name='index'),
    path('create/', views.create, name='create'),
    path('order/<int:id>', views.order, name='order'),
    # path('who/<int:who_id>',views.who, name='who'),
    # path('what/<int:what_id>',views.what, name='what'),
    # path('login/', views.login, name='login'),
    # path('logout/', views.logout, name='logout'),
    path('accounts/', include('django.contrib.auth.urls'))
]


import matplotlib.pyplot as plt
import umap
import pandas as pd
import glob

def main():
    feature_list = []
    subject_list = []

    directoryPath = '/Users/wangyan/Desktop/Research/Maze_Research/Maze-Navigation/Parsed_Data/'

    for file_name in glob.glob(directoryPath + '*.csv'):
        df = pd.read_csv(file_name)

        # read in the data on X and Z position plus the rotation
        feature = list(df['posX'].values[0:30000:10])
        feature2 = list(df['posZ'].values[0:30000:10])
        feature3 = list(df['rotX'].values[0:30000:10])
        feature4 = list(df['rotY'].values[0:30000:10])
        feature5 = list(df['rotZ'].values[0:30000:10])

        feature.extend(feature2)
        feature.extend(feature3)
        feature.extend(feature4)
        feature.extend(feature5)

        # Select the data with 15000 frames; the other subjects not with 15000 frames are not considered
        if len(feature) == 15000:
            feature_list.append(feature)
            # Each csv file is named with 4 letters, so take the 4 letters as their names.
            subject_list.append(file_name[74:78])

    standard_embedding = umap.UMAP().fit_transform(feature_list)
    umapDF = pd.DataFrame(standard_embedding)

    # This is a naive way to differentiate the two clusters. But try more times, it will work.
    cluster1 = []
    cluster2 = []
    for i in range(len(standard_embedding)):
        if standard_embedding[i][1] > 1:
            cluster1.append(subject_list[i])
        else:
            cluster2.append(subject_list[i])

    print(cluster1)
    print(cluster2)

    # display the scatter plot
    plt.scatter(umapDF[0], umapDF[1], alpha=1)
    plt.show()

main()


import os
import logging

from django.apps import apps
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError
from django.core.management.commands.makemessages import Command as BaseCommand

CONF_NAME = 'TRANSLATED_APPS'

class Command(BaseCommand):
    help = "Make messages, including apps as configured."

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument('--app', '-m', default=[], action='append',
            dest='apps', help='Only scan the contents of these app folders.')
        parser.add_argument('--configured-list', dest='configured_list',
            action='store_true', help='Load a list of apps from the settings.')

    def handle(self, apps=(), configured_list=False, **options):
        self.apps = list(self.parse_apps(apps, configured_list))
        if self.apps:
            print("Selecting apps: %s" % ", ".join(self.apps))
        super(Command, self).handle(**options)

    @staticmethod
    def parse_apps(in_apps, configured_list):
        for app in list(set(settings.INSTALLED_APPS) & set(in_apps)):
            in_apps.remove(app)
            yield app

        if in_apps:
            raise CommandError("App(s) not found %s" % ", ".join(in_apps))

        if configured_list:
            if hasattr(settings, CONF_NAME):
                for item in getattr(settings, CONF_NAME):
                    yield item
            else:
                raise CommandError("Config %s is not set." % CONF_NAME)

    def find_files(self, root):
        if self.apps:
            ab_root = os.path.abspath(root)
            files = []
            for app_name in self.apps:
                app = apps.get_app_config(app_name)
                path = app.path.replace(ab_root, root)
                files += super(Command, self).find_files(path)
            return files
        return super(Command, self).find_files(root)


import sys
import os
import logging
import traceback
import StringIO

import json
import simplejson

from services import constants, echoservice, shellservice, metadataservice, dataservice, securityservice, utils, datamodel, transformservice, requestcontext

from google.appengine.ext import db, search

class AppService(object):
    '''
    Provides access to data
    '''
    @staticmethod
    def get_view_maps(req, names=None):
        view_maps = []
        view_maps = req.metadata.view_maps(req)
        if names:
            view_maps = [ x for x in view_maps if x.name in names ]
        return view_maps

    @staticmethod
    def get_root_view_maps(req, limit, offset):
        view_maps = []
        view_maps = req.metadata.view_maps(req)
        view_maps = [ x for x in view_maps ]
        return view_maps

    @staticmethod
    def get_view_defs(req, names=None):
        view_defs = []
        view_defs = req.metadata.view_definitions(req)
        if names:
            view_defs = [ x for x in view_defs if x.name in names ]
        return view_defs

    @staticmethod
    def get_entity_metadatas(req, names):
        entity_metadatas = []
        entity_metadatas = req.metadata.entity_metadatas(req)
        if names:
            entity_metadatas = [ x for x in entity_metadatas if x.name in names ]
        return entity_metadatas

    @staticmethod
    def get_enumerations(req, names):
        enumerations = []
        enumerations = req.metadata.enumerations(req)
        if names:
            enumerations = [ x for x in enumerations if x.name in names ]
        return enumerations

    @staticmethod
    def get_type_infos(req, names):
        type_infos = []
        type_infos = req.metadata.type_infos(req)
        if names:
            type_infos = [ x for x in type_infos if x.name in names ]
        return type_infos

    @staticmethod
    def get_l10n_content(req, names):
        l10n_content = []
        l10n_content = req.metadata.l10n_content(req)
        if names:
            l10n_content = [ x for x in l10n_content if x.name in names ]
        return l10n_content

    @staticmethod
    def get_relations(req, names):
        relations = []
        relations = req.metadata.entity_relations(req)
        if names:
            relations = [ x for x in relations if x.name in names ]
        return relations

    @staticmethod
    def find_l10n_for_page(req, page, lang, offset=0, limit=128):
        ret = None
        query = datamodel.L10n.all()
        if page:
            query.filter('page =', page)
        if lang:
            query.filter('lang =', lang)
        #query.filter('metadata =', req.metadata.key())
        ret = query.fetch(limit, offset)
        return ret

    # these two methods could be deleted?
    @staticmethod
    def get_relations_for_entity_metadata(req, entity_name):
        relations = []
        relations = req.metadata.entity_relations(req)
        if names:
            relations = [ x for x in relations if x.parent_entity == entity_name ]
        return relations

    @staticmethod
    def get_page_metadata(req, view_map_name):
        page_metadata = {}
        page_metadata['view_map']           = metadataservice.find_view_map(req, view_map_name)
        page_metadata['view_definition']    = metadataservice.find_view_definition(req, page_metadata['view_map'].view_def)
        dataquery_names                     = metadataservice.find_data_query(req, page_metadata['view_definition'])
        dataquery_names                     = simplejson.JSONDecoder().decode(dataquery_names)
        dataqueries                         = [ metadataservice.find_data_query(req, name) for name in dataquery_names ]
        page_metadata['data_queries']       = data_queries

        return page_metadata

    @staticmethod
    def format_perm(req, app, name):
        pname = app.org.name + '.' + app.org.name + '-' + app.metadata.name + '.' + name
        return pname

    @staticmethod
    def create_perm(req, app, name):
        pname = AppService.format_perm(req, app, name)
        rp = utils.find_entity_by_name(datamodel.Permission, pname)
        if not rp:
            logging.info('create permission: ' + pname)
            rp = datamodel.Permission(key_name=pname)
            rp.name=pname
            rp.description = app.org.name + ' Organisations ' + app.name + ' App ' + name + ' Permission'
            rp.owner = req.user.name
            rp.org = app.org.name
            rp.put()

        return rp

    @staticmethod
    #"Parabay-User-Role","Users of parabay","ParabayOrg","ParabayOrg.ParabayOrg-Contacts.Read;ParabayOrg.ParabayOrg-Contacts.Write;ParabayOrg.ParabayOrg-Outlook.Read;ParabayOrg.ParabayOrg-Outlook.Write;ParabayOrg.ParabayOrg-Timmy.Read;ParabayOrg.ParabayOrg-Timmy.Write;ParabayOrg.ParabayOrg-Docs.Read;ParabayOrg.ParabayOrg-Docs.Write;ParabayOrg.ParabayOrg-Friends.Read;ParabayOrg.ParabayOrg-Friends.Write"
    def create_role(req, app, name):
        rname = app.name + '.' + name + '-Role'
        role = utils.find_entity_by_name(datamodel.Role, rname)
        if not role:
            logging.info('create role: ' + rname)
            role = datamodel.Role(key_name=rname)
            role.name=rname
            role.description = name + 's of ' + app.name
            role.owner = req.user.name
            role.org = app.org.name
            role.put()

        rp = utils.find_entity_by_name(datamodel.Permission, AppService.format_perm(req, app, 'Read'))
        wp = utils.find_entity_by_name(datamodel.Permission, AppService.format_perm(req, app, 'Write'))
        srp = utils.find_entity_by_name(datamodel.Permission, AppService.format_perm(req, app, 'Schema.Read'))
        swp = utils.find_entity_by_name(datamodel.Permission, AppService.format_perm(req, app, 'Schema.Write'))

        if rp and wp and srp and swp:
            logging.info('Setting role permissions..')
            role_perms = [rp.key(), wp.key(), srp.key()]
            if name == 'Admin':
                role_perms.append(swp.key())

            role.permissions = role_perms
            role.put()

        return role

    @staticmethod
    #"ParabayOrg-Friends.User-Group","Users of parabay friends app","ParabayOrg","Parabay-User-Role"
    def create_group(req, app, name):
        rname = app.name + '.' + name + '-Group'
        group = utils.find_entity_by_name(datamodel.Group, rname)
        if not group:
            logging.info('create group: ' + rname)
            group = datamodel.Group(key_name=rname)
            group.name=rname
            group.description = name + 's of ' + app.name
            group.owner = req.user.name
            group.org = app.org.name
            group.put()

        ur = utils.find_entity_by_name(datamodel.Role, app.name + '.User-Role')
        ar = utils.find_entity_by_name(datamodel.Role, app.name + '.Admin-Role')

        if ur and ar:
            logging.info('Setting group roles..')
            group_roles = [ur.key()]
            if name == 'Admin':
                group_roles.append(ar.key())

            group.roles = group_roles
            group.put()

        return group

    @staticmethod
    def create_app(req, appInfo, organisation):
        appName = appInfo['name']

        app = utils.find_entity_by_name(datamodel.App, appName)
        if not app:
            logging.info('creating app:' + appName)
            app = datamodel.App(key_name=appName)
            app.name = appName
        else:
            logging.info('found existing app:' + appName)
            if app.owner != req.user.name:
                return constants.STATUS_ACCESS_DENIED

        if 'description' in appInfo:
            app.description = appInfo['description']
        app.owner = req.user.name
        app.org = organisation
        app.put()

        if 'metadata' in appInfo:
            appMetadata = appInfo['metadata']
            m = utils.find_entity_by_name(datamodel.Metadata, appMetadata)
            if not m:
                m = datamodel.Metadata(key_name=appMetadata)
                m.name=appMetadata
            m.owner = req.user.name
            m.description = 'Metadata for ' + appName
            m.put()

            app.metadata = m
            app.put()

        AppService.create_perm(req, app, 'Read')
        AppService.create_perm(req, app, 'Write')
        AppService.create_perm(req, app, 'Schema.Read')
        AppService.create_perm(req, app, 'Schema.Write')

        AppService.create_role(req, app, 'User')
        AppService.create_role(req, app, 'Admin')

        AppService.create_group(req, app, 'User')
        AppService.create_group(req, app, 'Admin')

        securityservice.SecurityService.add_user_app_perms(req.user, appName, organisation.name)

        return constants.STATUS_OK

    @staticmethod
    def save_data_query(req, data):

        dquery = None
        if 'data_query' in data:
            dquery = data['data_query']
        if dquery:
            dq = transformservice.reverse_transform_metadata(req, dquery, datamodel.DataQuery)
            dq.owner = req.user.name
            dq.org = req.org.name
            dq.metadata = req.metadata
            dq.put();

        en_list = None
        if 'enumerations' in data:
            en_list = data['enumerations']
        if en_list and len(en_list)>0:
            for en in en_list:
                ev_list = en['enumeration_values']
                del en['enumeration_values']

                e =  transformservice.reverse_transform_metadata(req, en, datamodel.Enumeration)
                e.owner = req.user.name
                e.org = req.org.name
                e.metadata = req.metadata
                e.put();

                if ev_list:
                    for ev in ev_list:
                        v =  transformservice.reverse_transform_metadata(req, ev, datamodel.EnumerationValue)
                        v.owner = req.user.name
                        v.org = req.org.name
                        v.metadata = req.metadata
                        v.enumeration = e
                        v.put();

        em_list = None
        if 'entity_metadatas' in data:
            em_list = data['entity_metadatas']
        if em_list and len(em_list)>0:
            em_item = em_list[0]
            props_list = em_item['entity_property_metadatas']
            del em_item['entity_property_metadatas']
            logging.info(em_item)

            em = transformservice.reverse_transform_metadata(req, em_item, datamodel.EntityMetadata)
            em.owner = req.user.name
            em.org = req.org.name
            em.metadata = req.metadata
            logging.info(em.description)
            em.put()

            if props_list:
                for prop in props_list:
                    pp = prop['type_info']
                    del prop['type_info']
                    logging.info(pp)

                    p =  transformservice.reverse_transform_metadata(req, prop, datamodel.EntityPropertyMetadata)
                    p.type_info = utils.find_entity_by_name(datamodel.TypeInfo, pp)
                    logging.info(p.type_info)
                    p.owner = req.user.name
                    p.org = req.org.name
                    p.metadata = req.metadata
                    p.entity_metadata = em
                    p.put();

        return None

    @staticmethod
    def save_view_data(req, data):
        logging.info(data)

        viewdef = None
        if 'view_definition' in data:
            viewdef = data['view_definition']
        if viewdef:
            vd = transformservice.reverse_transform_metadata(req, viewdef, datamodel.ViewDefinition)
            vd.owner = req.user.name
            vd.org = req.org.name
            vd.metadata = req.metadata
            vd.put();

        viewmap = None
        if 'view_map' in data:
            viewmap = data['view_map']
        if viewmap:
            del viewmap['view_definition']
            vm = transformservice.reverse_transform_metadata(req, viewmap, datamodel.ViewMap)
            vm.view_definition = vd
            vm.owner = req.user.name
            vm.org = req.org.name
            vm.metadata = req.metadata
            vm.put();

        return None


import tempfile, shutil, sys, subprocess, json
import populate_xmeta as px

def ona_import(auth_token):
   token_test = subprocess.check_output(["curl", "-X", "GET", "https://api.ona.io/api/v1/forms", "-H", "Authorization: Token %s" % (auth_token)])
   if token_test == '{"detail":"Invalid token."}':
      print("Invalid authorization token")
      return
   else:
      ona_import_forms(auth_token)
      ona_import_data(auth_token)

def ona_import_forms(auth_token):
   forms = json.loads(subprocess.check_output(["curl", "-X", "GET", "https://api.ona.io/api/v1/forms", "-H", "Authorization: Token %s" % (auth_token)]))
   for form in forms:
      formid = form['formid']

      xlsform = subprocess.check_output(["curl", "-X", "GET", "https://api.ona.io/api/v1/forms/%s/form.xls" % (formid), "-H", "Authorization: Token %s" % (auth_token)])
      xform = json.loads(subprocess.check_output(["curl", "-X", "GET", "https://api.ona.io/api/v1/forms/%s/form.json" % (formid), "-H", "Authorization: Token %s" % (auth_token)]))
      fid = xform['id_string']
      vid = xform['version']
      ona_url = form['created_by']
      path = formid
      xform = json.dumps(xform, sort_keys=True,indent=4, separators=(',', ': '))
      px.populate_xmeta_ona(fid,vid,ona_url,path,xlsform,xform)

def ona_import_data(auth_token):
   forms = json.loads(subprocess.check_output(["curl", "-X", "GET", "https://api.ona.io/api/v1/forms", "-H", "Authorization: Token %s" % (auth_token)]))
   for form in forms:
      formid = form['formid']

      submissions = json.loads(subprocess.check_output(["curl", "-X", "GET", "https://api.ona.io/api/v1/data/%s" % (formid), "-H", "Authorization: Token %s" % (auth_token)]))
      for xdata in submissions:
         sid = str(xdata['_id'])
         fid = xdata['_xform_id_string']
         vid = xdata['_version']
         device_id = "ona"
         device_ip = "ona"
         device_type = "ona"
         xdata = json.dumps(xdata, sort_keys=True,indent=4, separators=(',', ': '))
         px.populate_xdata_ona(sid,fid,vid,device_id,device_ip,device_type,xdata)

if __name__ == '__main__':
   if len(sys.argv) == 2:
      ona_import(sys.argv[1])
   else:
      print("Usage: %s ONA API key" % sys.argv[0])


import ast
import glob
import logging
import os
import re
import shutil

log = logging.getLogger()

def validate_dir(dir_path):
    if not os.path.isdir(dir_path):
        raise ValueError(f"{dir_path} is not a valid directory.")
    return True

class DirectoryArchiver:
    """
    The DirectoryArchiver class provides functionality to create an archive of a specified directory.
    Attributes:
        _dir_path (str): The path to the directory that will be archived.
    Methods:
        get_parent_dir(): Returns the parent directory of _dir_path.
        _get_archive_name(): Generates and returns the name of the archive file.
        archive_folder(): Creates a zip archive of the directory.
    """

    def __init__(self, dir_path):
        validate_dir(dir_path)
        self._dir_path = dir_path

    def get_parent_dir(self) -> str:
        return os.path.dirname(self._dir_path)

    def _get_archive_name(self) -> str:
        dir_name = os.path.basename(self._dir_path)
        parent_dir = self.get_parent_dir()
        return os.path.join(parent_dir, f"{dir_name}_archive")

    def archive_folder(self):
        shutil.make_archive(self._get_archive_name(), "zip", self._dir_path)

class ConvertJobDirectoryProcessor:
    """
    The ConvertJobDirectoryProcessor class provides functionality to process a directory,
    archiving it, and storing specific file contents into a structured list.
    Attributes:
        _dir_path (str): The path to the directory that will be processed.
        _code_structure (list): A list to store the processed content of specific files.
        _removed_files (list): A list to store the names of files that have been removed.
    Methods:
        process_files(): Processes files in the directory, saving and removing specific files' content.
        cleanup(): Removes any remaining files that were supposed to be removed.
        get_code_structure() -> list: Returns the structured list of processed file contents.
        get_removed_files() -> list: Returns the list of removed file names.
    """

    def __init__(self, dir_path):
        validate_dir(dir_path)
        self._dir_path = dir_path
        self._code_structure = []
        self._removed_files = []

    def process_files(self):
        all_files = sorted(glob.glob(os.path.join(self._dir_path, "*")))

        for file in all_files:
            if file.endswith(".sql"):
                self._process_sql_files(file)
            elif file.endswith(".py"):
                self._process_python_files(file)

    @staticmethod
    def _has_run_function(file: str, content: str) -> bool:
        tree = ast.parse(content, file)
        for node in ast.iter_child_nodes(tree):
            if isinstance(node, ast.FunctionDef):
                if node.name == "run":
                    return True
        return False

    def _process_python_files(self, file):
        with open(file) as f:
            content = f.read()
        if self._has_run_function(file, content):
            self._code_structure.append(content)
            self._removed_files.append(os.path.basename(file))
            os.remove(file)

    def _process_sql_files(self, file):
        with open(file) as f:
            content = f.read()
        self._code_structure.append(f"%%vdksql\n{content}")
        self._removed_files.append(os.path.basename(file))
        os.remove(file)

    def cleanup(self):
        for file_name in self._removed_files:
            file_path = os.path.join(self._dir_path, file_name)
            if os.path.exists(file_path):
                os.remove(file_path)

    def get_code_structure(self):
        return self._code_structure

    def get_removed_files(self):
        return self._removed_files


from PyQt5.QtWidgets import (QMessageBox, QDialog, QPushButton, QTableView, QVBoxLayout,
                             QHeaderView, QHBoxLayout)
from PyQt5.QtSql import QSqlQuery, QSqlQueryModel
from PyQt5.QtCore import Qt, QVariant
from PyQt5.QtGui import QColor, QTextLength, QIcon
from PyQt5.QtPrintSupport import QPrintPreviewDialog, QPrintDialog
from ext.APM import DataError, QSqlAlignColorQueryModel
from ext import APM
from ext.Printing import ReportPrint
import poloresurce

class AvailableHorses(QDialog):

    def __init__(self, db, reportType, parent=None):
        super().__init__(parent=None)
        self.parent = parent
        self.reportType = reportType
        self.cdb = db
        self.setModal(True)
        self.setUI()
        self.table = None
        self.centerColumns = []
        self.printColumnWidths = []
        if self.reportType == APM.REPORT_TYPE_ALL_HORSES:
            self.setAllHorsesTable()
        elif self.reportType == APM.REPORT_TYPE_ALL_BREAKING_HORSES:
            self.setAllBreakingHorsesTable(APM.REPORT_TYPE_ALL_BREAKING_HORSES)
        elif self.reportType == APM.REPORT_TYPE_ALL_PLAYING_HORSES:
            self.setAllBreakingHorsesTable(APM.REPORT_TYPE_ALL_PLAYING_HORSES)

    def setUI(self):
        self.setWindowTitle("Unassigned Horses")
        self.horseTable = QTableView()
        self.horseTable.verticalHeader().setVisible(False)

        oKButton = QPushButton(QIcon(":/Icons8/exit/closeman.png"),"Exit", self)
        oKButton.clicked.connect(self.widgetClose)
        oKButton.setMaximumSize(100, 30)

        printButton = QPushButton(QIcon(":/Icons8/print/print.png"), "Print", self)
        printButton.setMaximumSize(100,30)
        printButton.clicked.connect(self.handlePrint)

        previewButton = QPushButton(QIcon(":/Icons8/print/printpreview.png"), "Preview", self)
        previewButton.setMaximumSize(100,30)
        previewButton.clicked.connect(self.handlePreview)

        pdfButton = QPushButton(QIcon(":/Icons8/print/savepdf.png"), "Save PDF", self)
        pdfButton.setMaximumSize(100, 30)
        pdfButton.clicked.connect(self.handlePdf)

        buttonsLayout = QHBoxLayout()
        buttonsLayout.addWidget(pdfButton)
        buttonsLayout.addWidget(printButton)
        buttonsLayout.addWidget(previewButton)
        buttonsLayout.addWidget(oKButton)

        layout = QVBoxLayout()
        layout.addWidget(self.horseTable)
        layout.addLayout(buttonsLayout)
        self.setLayout(layout)

    def setAllBreakingHorsesTable(self, mode):
        qry = QSqlQuery(self.cdb)
        qry.prepare("""SELECT
        ah.id ID,
        h.rp RP,
        h.name,
        CASE
            WHEN h.sexid = 1 THEN _ucs2 X'2642'
            WHEN h.sexid = 2 THEN _ucs2 X'2640'
            WHEN h.sexid = 3 THEN _ucs2 X'265E'
        END Sex,
        c.coat as Coat,
        TIMESTAMPDIFF(YEAR, h.dob, CURDATE()) Age,
        TIMESTAMPDIFF(MONTH, ah.dos, CURDATE()) Month,
        ah.agreementid AgrNo,
        ct.fullname as Contact
        FROM
        agreementhorses AS ah
        INNER JOIN horses AS h
        ON ah.horseid = h.id
        INNER JOIN coats as c
        ON h.coatid = c.id
        INNER JOIN agreements as a
        ON ah.agreementid = a.id
        INNER JOIN contacts as ct
        ON a.supplierid = ct.id
        WHERE ah.active
            AND a.agreementtype in (SELECT ?)
        ORDER BY a.agreementtype, a.id;""")
        qry.addBindValue(QVariant(0))if self.reportType == APM.REPORT_TYPE_ALL_BREAKING_HORSES \
            else qry.addBindValue(QVariant(1).value())
        qry.exec_()
        try:
            if qry.size() < 0:
                raise DataError('setAllHorsesTable', qry.lastError().text())
            elif qry.size() == 0:
                raise DataError('No Data', "There are not horses to show")
            self.centerColumns = [1, 3, 5, 6, 7]
            colorDict = {'column': (3),
                         u'\u2640': (QColor('pink'), QColor('black')),
                         u'\u2642': (QColor('lightskyblue'), QColor('black')),
                         u'\u265E': (QColor('lightgrey'), QColor('black'))}
            allHorsesModel = QSqlAlignColorQueryModel(self.centerColumns,[], colorDict)
            allHorsesModel.setQuery(qry)
            allHorsesModel.setHeaderData(0, Qt.Horizontal, "ID")
            allHorsesModel.setHeaderData(1, Qt.Horizontal, "RP")
            allHorsesModel.setHeaderData(2, Qt.Horizontal, "Name")
            allHorsesModel.setHeaderData(3, Qt.Horizontal, "Sex")
            allHorsesModel.setHeaderData(4, Qt.Horizontal, "Coat")
            allHorsesModel.setHeaderData(5, Qt.Horizontal, "Age")
            allHorsesModel.setHeaderData(6, Qt.Horizontal, "Month")
            allHorsesModel.setHeaderData(7, Qt.Horizontal, "AgrNo")
            allHorsesModel.setHeaderData(8, Qt.Horizontal, "With")
            self.horseTable.setModel(allHorsesModel)
            self.horseTable.hideColumn(0)
            header = self.horseTable.horizontalHeader()
            header.setStyleSheet("QHeaderView {font-size: 8pt;}")
            header.setSectionResizeMode(1, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(2, QHeaderView.Stretch)
            header.setSectionResizeMode(3, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(4, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(5, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(6, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(7, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(8, QHeaderView.ResizeToContents)
            self.horseTable.setRowHeight(0, 10)
            self.horseTable.verticalHeader().setDefaultSectionSize(
                self.horseTable.rowHeight(0))
            self.horseTable.setMinimumSize(700, 400)
            self.setWindowTitle("Horses On Breaking Agreements") if self.reportType == APM.REPORT_TYPE_ALL_BREAKING_HORSES \
                else self.setWindowTitle("Horses On Play & Sale Agreements")
            self.setMinimumWidth(800)
            # set the printing column widths

            col_1 = QTextLength(QTextLength.FixedLength, 45)
            col_2 = QTextLength()
            col_3 = QTextLength(QTextLength.FixedLength, 45)
            col_4 = QTextLength(QTextLength.FixedLength, 120)
            col_5 = QTextLength(QTextLength.FixedLength, 45)
            col_6 = QTextLength(QTextLength.FixedLength, 45)
            col_7 = QTextLength(QTextLength.FixedLength, 45)
            col_8 = QTextLength()

            self.printColumnWidths = [col_1, col_2, col_3, col_4, col_5,
                                      col_6, col_7, col_8]

        except DataError as err:
            QMessageBox.warning(self, "DataError", err.message)
            raise DataError(err.source, err.message)
            #self.widgetClose()

    def setAllPlayingHorsesTable(self):
        pass

    def setAllHorsesTable(self):
        """Gets the query for the model, uses a subclasesd SqlQueryModel including the posibility to
        pass information on centered columns and tex and background colors.
        Centered columns ares set in a list ([column#,....column#n]) where column referes to the column that
         will be centered.
         The color information is place in a dictionary with the following format:
            {'column' : value,
            key1:(QTextColor, QBackGroundCColor),
             key2: (QTextColor, QBackgroundColor),
             ....................................,
             keyn:(QTextColor, QBackgroundColor)}}
        where:
            column is the query field number,
            key is the comparison value to check agaisnt the query(column).value()
            and (QTextColor, QBackGroundColor) is a tuple wit099999999999999999999999999999o9h two QColor values, the first for
            QTextColorRole, and the second for QbackgroundColorrole"""
        colorDict = {'column':(5),
                        u'\u2640':(QColor('pink'), QColor('black')),
                        u'\u2642':(QColor('lightskyblue'), QColor('black')),
                        u'\u265E': (QColor('lightgrey'), QColor('black'))}
        try:
            qry = QSqlQuery(self.cdb)
            qry.exec_("""SELECT h.id,
                 h.rp,
                 h.name,
                 h.dob,
                 FLOOR(DATEDIFF(CURDATE(), h.dob)/365) Age,
                 CASE WHEN h.sexid = 1 THEN _ucs2 X'2642'
                 WHEN h.sexid = 2 THEN _ucs2 X'2640'
                 WHEN h.sexid = 3 THEN _ucs2 X'265E'
                 END Sex,
                 c.coat,
                 IF (h.isbroke = 1, _ucs2 X'2714', '') broke,
                 IF (h.isbreakable = 1, _ucs2 X'2714', '') breakable,
                 IF (h.active = 1, _ucs2 X'2714', '') active
                 FROM horses as h
                 INNER JOIN sexes as s
                 ON h.sexid = s.id
                 INNER JOIN coats as c
                 ON h.coatid = c.id
                 WHERE h.active
                 AND h.id NOT IN (SELECT ah.horseid FROM agreementhorses ah
                                  WHERE ah.active)
                 ORDER BY h.isbroke, h.sexid, h.name""")
            if qry.size() < 1:
                raise DataError('setAllHorsesTable', qry.lastError().text())
            self.centerColumns = [1, 4, 5, 7, 9]
            allHorsesModel = QSqlAlignColorQueryModel(self.centerColumns,[],colorDict)
            allHorsesModel.setQuery(qry)
            allHorsesModel.setHeaderData(1,Qt.Horizontal,"RP")
            allHorsesModel.setHeaderData(2, Qt.Horizontal, "Name")
            allHorsesModel.setHeaderData(3, Qt.Horizontal, "DOB")
            allHorsesModel.setHeaderData(4, Qt.Horizontal, "Age")
            allHorsesModel.setHeaderData(5, Qt.Horizontal, "Sex")
            allHorsesModel.setHeaderData(6, Qt.Horizontal, "Coat")
            allHorsesModel.setHeaderData(7, Qt.Horizontal, "Broke")
            allHorsesModel.setHeaderData(9,Qt.Horizontal, "Active")
            self.horseTable.setModel(allHorsesModel)
            self.horseTable.hideColumn(0)
            self.horseTable.hideColumn(8)
            header = self.horseTable.horizontalHeader()
            header.setStyleSheet("QHeaderView {font-size: 8pt;}")
            header.setSectionResizeMode(1, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(2, QHeaderView.Stretch)
            header.setSectionResizeMode(3, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(4, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(5, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(6, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(7, QHeaderView.ResizeToContents)
            header.setSectionResizeMode(9, QHeaderView.ResizeToContents)
            self.horseTable.setRowHeight(0, 10)
            self.horseTable.verticalHeader().setDefaultSectionSize(
                self.horseTable.rowHeight(0))
            self.horseTable.setMinimumSize(700, 400)

            #set the printing column widths

            col_1 = QTextLength(QTextLength.FixedLength, 45)
            col_2 = QTextLength()
            col_3 = QTextLength(QTextLength.FixedLength, 100)
            col_4 = QTextLength(QTextLength.FixedLength, 45)
            col_5 = QTextLength(QTextLength.FixedLength, 45)
            col_6 = QTextLength(QTextLength.FixedLength, 120)
            col_7 = QTextLength(QTextLength.FixedLength, 60)
            col_8 = QTextLength(QTextLength.FixedLength, 60)

            self.printColumnWidths = [col_1, col_2, col_3, col_4, col_5,
                                      col_6, col_7, col_8]

        except DataError as err:
            QMessageBox.warning(self,"DataError", err.message)

    def handlePrint(self):
        rep = ReportPrint(self.horseTable,self.window.title())
        rep.handlePrint()

    def handlePreview(self):
        rep = ReportPrint(self.horseTable, self.windowTitle())
        rep.handlePreview()

    def handlePdf(self):
        rep = ReportPrint(self.horseTable, self.windowTitle(),self.parent)
        rep.handlePdf()

    def widgetClose(self):
        self.done(QDialog.Rejected)