text:
# Basics Requirements
import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import numpy as np
import pandas as pd
from dash.dependencies import Input, Output, State

import json

# Recall app
from app import app
from library.elements_all import sidebar_maps
from library import def_data

##############################
# MAPS
##############################
# CONTENTS
# 1. Styles
# 2. SQL queries
# 3. Map
# 4. Ranking
# 5. DMU Groups
# 6. Slacks/Waste
# 7. Efficiency definition
# 8. Callback and DEA
# 9. Layout
# ------------------------------

# ------------------------------
# 1. Styles
# ------------------------------
# 1.1 Map Styles
MAP_MAPS_STYLE = {
     "position": "fixed",
     "width": "70%",
     "left": "17rem",
     "top": "140px",
     "border": "1px solid #e7eff6"
}

# ------------------------------
# 2. SQL Queries
# ------------------------------
# 2.1 Initial query
# ------------------------------
df_desertion = def_data.runQuery("""
    select code_dept, name_dept, avg(desertion_perc) as desertion_perc
    from master_table_by_municipio
    where year_cohort = 2019
    group by code_dept, name_dept; """)
df_desertion['desertion_perc'] = df_desertion['desertion_perc'].astype(np.float64)
df_vars = def_data.runQuery("""select * from public.var_definition order by group_id;""")

# 2.2 Format adjusment
# ------------------------------

# ------------------------------
# 3. MAP
# ------------------------------
# 3.1 Global variables
# ------------------------------
selected_var = 'desertion_perc'
selected_var_code = 181
selected_dpt = 'Antioquia'
label_fig = df_vars[df_vars['name'] == selected_var].reset_index()['label'][0]
# 3.2 Initial map
# ------------------------------
with open('data/MGN_MPIO_POLITICO.json') as geo:
     MUN_json = json.loads(geo.read())

with open('data/MGN_DPTO_POLITICO.json') as f:
     DEP_json = json.loads(f.read())
MUN2_json = DEP_json.copy()

map_fig = px.choropleth_mapbox(df_desertion,
            geojson=DEP_json,
            color="desertion_perc",
            locations="code_dept",
            featureidkey="properties.DPTO_CCDGO",
            color_continuous_scale = "Blues",
            center={"lat": 4.94, "lon": -73.77},
            hover_name="name_dept",
            mapbox_style="carto-positron", zoom=4)
map_fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0},
                       coloraxis_colorbar=dict(
                       title=label_fig))

# 3.2 Update map
# ------------------------------
input_array_map = np.append([Input('maps_dropdown_depto', 'value')],
                            [Input(f'mapLabel-{i}', 'n_clicks') for i in sidebar_maps.df_vars['var_id']])
@app.callback(Output('maps_fig1', 'figure'),list(input_array_map))
def build_chart(depto_val,*args):
    # 3.2.1 Determine is a variable is selected and which
    # ------------------------------
    global selected_var_code

    df_vars = sidebar_maps.df_vars.copy()
    changed_label_id = [p['prop_id'] for p in dash.callback_context.triggered][0]

    for i in df_vars['var_id']:
        if 'mapLabel-' + i + ".n_clicks" in changed_label_id:
            selected_var_code = i

    selected_var = df_vars.loc[df_vars['var_id'] == str(selected_var_code)]['name'].reset_index()['name'][0]

    single_qote = "'"

    if False: # If no change in either the dropdown or the labels, then do nothing.
        return map_fig
    else:
        # 3.2.2 Map by department (drop down mun is None)
        if depto_val is None:
            # 3.2.2.1 Query desired variable
            # ------------------------------
            selected_var="desertion_perc"
            sql_query = 'select code_dept, name_dept, avg(' + selected_var + ') as ' + selected_var + ' ' + \
                'from cluster_master_table_by_municipio ' + \
                'group by code_dept, name_dept;'
            df_var_all_dpto = def_data.runQuery(sql_query)
            df_var_all_dpto[selected_var] = df_var_all_dpto[selected_var].astype(np.float64)
            label_fig = df_vars[df_vars['name'] == selected_var].reset_index()['label'][0]
            label_tittle =  df_vars[df_vars['name'] == selected_var].reset_index()['description'][0]

            # 3.2.2.2 Define new map
            # ------------------------------
            new_map = px.choropleth_mapbox(df_var_all_dpto,
               geojson=DEP_json,
               color=selected_var,
               locations="code_dept",
               featureidkey="properties.DPTO_CCDGO",
               color_continuous_scale="Blues",
               center={"lat": 4.94, "lon": -73.77},
               hover_name="name_dept",
               mapbox_style="carto-positron", zoom=4)

            new_map.update_layout(margin={"r":0,"l":0,"b":0},
                                  title_text=label_tittle,
                                  coloraxis_colorbar=dict(
                                  title=label_fig))
            return new_map
        # 3.2.3 Map by municipality (drop down mun is not None)
        else:
            # 3.2.3.1 Query desired variable
            # ------------------------------

            mylist={'dane_alu_18_p','dane_tic_01','dane_alu_12_p','dane_tic_03_1_p','po_pob_rural_10mil'}

            if selected_var in mylist:
                selected_var_query = 'desertion_perc'
            else:
                selected_var_query = selected_var

            sql_query = 'select code_municip, code_dept, name_municip, dane_alu_18_p, dane_tic_03_1_p, ' + \
                        'po_pob_rural_10mil, dane_alu_12_p, dane_tic_01, ' + selected_var_query + ' ' + \
                        'from cluster_master_table_by_municipio ' + \
                        'where code_dept = ' + single_qote + depto_val + single_qote + ';'

            df_var_by_dpto = def_data.runQuery(sql_query)
            df_var_by_dpto[selected_var] = df_var_by_dpto[selected_var].astype(np.float64)

            # 3.2.3.2 Filtering the Departament
            # ------------------------------
            MUN2_json['features'] = [city for city in MUN_json['features'] if city['properties']['DPTO_CCDGO'] == depto_val]
            center_x=MUN2_json['features'][0]['geometry']['coordinates'][0][0][0]
            center_y=MUN2_json['features'][0]['geometry']['coordinates'][0][0][1]
            new_center=dict(lat=center_y, lon=center_x)

            # 3.2.3.3 MAp Layout
            # ------------------------------
            label_fig = df_vars[df_vars['name'] == selected_var].reset_index()['label'][0]
            label_tittle =  df_vars[df_vars['name'] == selected_var].reset_index()['description'][0]

            df_var_by_dpto['% Transferred students'] = pd.Series(["{0:.2f}%".format(val * 100) for val in df_var_by_dpto['dane_alu_12_p']],
                                                    index = df_var_by_dpto.index)
            df_var_by_dpto['% Students afternoon'] = pd.Series(["{0:.2f}%".format(val * 100) for val in df_var_by_dpto['dane_alu_18_p']],
                                                        index = df_var_by_dpto.index)
            df_var_by_dpto['% Schools with electricity'] = pd.Series(["{0:.2f}%".format(val * 100) for val in
                                                                      df_var_by_dpto['dane_tic_03_1_p']],index = df_var_by_dpto.index)
            df_var_by_dpto['% Transferred students'] = pd.Series(["{0:.2f}%".format(val * 100) for val in
                                                                  df_var_by_dpto['dane_alu_12_p']],index = df_var_by_dpto.index)
            df_var_by_dpto['% Habitants (towns-rural)'] = pd.Series([round(val, 2) for val in df_var_by_dpto['po_pob_rural_10mil']],
                                                        index = df_var_by_dpto.index)
            df_var_by_dpto['Avg computers x100 stu.'] = pd.Series([round(val, 2) for val in df_var_by_dpto['dane_tic_01']],
                                                        index = df_var_by_dpto.index)

            #df_var_by_dpto = df_var_by_dpto.rename({'dane_alu_18_p': '% Students afternoon',
            #                                        'dane_tic_01': 'Avg computers x100 stu.',
            #                                        'dane_alu_12_p': '% Transferred students',
            #                                        'dane_tic_03_1_p': '% Schools with electricity',
            #                                       'po_pob_rural_10mil': '% Habitants (towns-rural)'}, axis=1)

            # 3.2.3.4 Define new map
            # ------------------------------
            new_map = px.choropleth_mapbox(df_var_by_dpto,
                     geojson=MUN2_json,
                     locations='code_municip',
                     color=selected_var,
                     featureidkey="properties.MPIO_CCNCT",
                     hover_name="name_municip",
                     mapbox_style="carto-positron",
                     center=new_center,
                     hover_data=['% Students afternoon','% Habitants (towns-rural)', 'Avg computers x100 stu.',
                                 '% Schools with electricity',
                                 '% Transferred students'],
                     zoom=6,
                     color_continuous_scale="blues",
                     )
            new_map.update_layout(margin={"r":0,"l":0,"b":0},
                                  title_text=label_tittle,
                                  coloraxis_colorbar=dict(
                                  title=label_fig))
            return new_map

##############################
#Map Layout
##############################
map = html.Div([
    sidebar_maps.sidebar,
    html.Div([dcc.Graph(figure=map_fig, id='maps_fig1')],style = MAP_MAPS_STYLE)
    #dbc.Col(html.Div(dropdown.dropdown_mun)),
    #html.Div(dropdown.dropdown_mun)

], className="ds4a-body")


text:
#!/usr/bin/python3

"""
This module is supposed to take care of visualizational
concerns especially if it is about plotting data
"""

from gisme import (figure_path, lon_col, lat_col, de_load, bbox,
                   variable_dictionary, data_path, nuts3_01res_shape,
                   nuts0_shape, isin_path, log, demography_file)
from gisme.WeatherReader import WeatherReader
from gisme.LoadReader import LoadReader
from gisme.Predictions import ARMAXForecast
from gisme.Utility import Utility

import os
import itertools
import pandas as pd
import numpy as np
import shapefile as shp
from datetime import datetime,timedelta
from descartes import PolygonPatch
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
from matplotlib import pyplot as plt, colors, cm, markers, rc, rcParams

from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

# rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})
# ## for Palatino and other serif fonts use:
# #rc('font',**{'family':'serif','serif':['Palatino']})
# rc('text',usetex=True)
# plt.rc('text',usetex=True)
# plt.rc('font',family='serif')

class DataPlotter:
    """
    handles plotting for load and weather data

    Attributes
    ----------
    fmt     : string
              preferred output format of plots such as 'pdf', 'eps', 'jpg' or
              similar supported by matplotlib
    save    : boolean
              whether to save plot or not
    show    : boolean
              whether to display plot or not
    shape   : integer tuple
              for map multiplot, specify arrangement as tuple of length 2
    isin    : boolean
              for map plot, whether to filter by isinDE mask
    wreader : WeatherReader
              used to load weather data
    lreader : LoadReader
              used to load load data
    """
    def __init__(self, fmt='pdf', save=True, show=False, shape=None, isin=False):
        """Initializes WeatherPlot instance

        Parameters
        ----------
        fmt    : string
                 format that figure is saved to
        save  : boolean
                whether to save plots to file or not
        show  : boolean
                whether to show plots or not
        shape : tuple
                if multiplot should be plotted,specify shape
        isin  : boolean
                whether to filter values from outside germany

        Returns
        -------
        None
        """
        self.fmt = fmt
        self.save = save
        self.show = show
        self.shape = shape
        self.isin = isin
        self.wreader = WeatherReader(self.isin)
        self.lreader = LoadReader()

        # #change font size depending on size of plot
        # if self.shape is not None:
        #     rcParams.update({'font.size': 18./np.round(np.sqrt(self.shape[0]*self.shape[1]))})
        # else:
        #     rcParams.update({'font.size': 18.})

    def __save_show_fig(self, fig, dir_pth, file_name):
        """Save and show the passed figure if specified and finally close it

        Parameters
        ----------
        fig       : matplotlib.figure.Figure
                    the created figure
        dir_pth   : string
                    the path to the directory to save to
        file_name : string
                    the file name to save to

        Returns
        -------
        None
        """
        if self.save:
            log.info(f'saving plot in {file_name}')
            if not os.path.exists(dir_pth):
                os.makedirs(dir_pth)
            if type(self.fmt) is list:
                for f in self.fmt:
                    fig.savefig(f'{file_name}.{f}', bbox_inches='tight', format=f, optimize=True, dpi=150)
            else:
                fig.savefig(f'{file_name}.{self.fmt}', bbox_inches='tight', format=self.fmt, optimize=True, dpi=150)
        if self.show:
            plt.show()
        # close figures as they won't be closed automatically by python during runtime
        plt.close(fig)

    def __create_ax_map(self, ax, variable, time, norm, xlbl_true=None, ylbl_true=None):
        """Plot a map of germany on the given axis

        Parameters
        ----------
        ax        : matplotlib.axes.Axes
                    the axis used to plot
        variable  : string
                    the name of the weather variable
        time      : datetime.datetime
                    the time for which to plot the variable
        norm      : matplotlib.colors.BoundaryNorm
                    the norm for color distribution
        xlbl_true : bool
                    wether to plot a label for the x-axis or not
        ylbl_true : bool
                    wether to plot a label for the y-axis or not

        Returns
        -------
        None
        """
        if self.isin:
            data = self.wreader.vals4time(variable, time, isin=True)
            data.plot.imshow(ax=ax, cmap='jet', extent=bbox, norm=norm, add_colorbar=False)
        else:
            data = self.wreader.vals4time(variable, time)
            ax.imshow(data.values, cmap='jet', extent=bbox, interpolation='bilinear', norm=norm)

        # read shapefile
        eu_shape = shp.Reader(nuts0_shape)
        de_shape = None
        for record in eu_shape.shapeRecords():
            if 'DE' in record.record:
                de_shape = record
                break
        if de_shape is None:
            raise Exception('shape for germany could not be found!')

        # concatenate points so that single lines can be drawn
        state = de_shape.shape
        points = np.array(state.points)
        intervals = list(state.parts) + [len(state.points)]
        for (x, y) in zip(intervals[:-1], intervals[1:]):
            ax.plot(*zip(*points[x:y]), color='k', linewidth=2)

        ax.set_title(pd.to_datetime(time).strftime("%Y/%m/%d %HH"))
        # print x and y label only if is most left/lowest plot
        if xlbl_true:
            ax.set_xlabel(lon_col)
        else:
            ax.set_xlabel(None)
        if ylbl_true:
            ax.set_ylabel(lat_col)
        else:
            ax.set_ylabel(None)

        # set ticks for x and y in order to display the grid
        xticks = np.linspace(bbox[0], bbox[1], (bbox[1] - bbox[0]) * 4 + 1)
        ax.set_xticks(xticks, minor=True)
        yticks = np.linspace(bbox[2], bbox[3], (bbox[3] - bbox[2]) * 4 + 1)
        ax.set_yticks(yticks, minor=True)

        # plot own grid
        xgrid = np.linspace(bbox[0]+.125, bbox[1] - .125, num=(bbox[1] - bbox[0]) * 4)
        ygrid = np.linspace(bbox[2]+.125, bbox[3] - .125, num=(bbox[3] - bbox[2]) * 4)
        for xpoint in xgrid:
            ax.axvline(xpoint, alpha=.2, color='k', linewidth=.5, linestyle='--')
        for ypoint in ygrid:
            ax.axhline(ypoint, alpha=.2, color='k', linewidth=.5, linestyle='--')

    def __plot_days(self, days, fname):
        """Plot data for each day in days list and save file with specified format

        about file name format for single days:
            the leading number indicates the position in terms of min/max
            --> for min,0 means it's the smallest value,
                for max,the highest number corresponds to the highest value

        Parameters
        ----------
        days  : list
                list of days
        fname : string
                to name folder

        Returns
        -------
        None
        """
        assert (self.shape is None or self.shape[0]*self.shape[1] == len(days)),\
            "shape of plot must fit with number of plots"

        var = days.name
        cm_jet = cm.get_cmap('jet')

        vmin, vmax = self.wreader.get_minmax(var)
        norm = colors.BoundaryNorm(np.linspace(vmin, vmax, 256), ncolors=256)
        cbox_ticks = np.linspace(vmin, vmax, 8)
        smap = cm.ScalarMappable(norm=norm, cmap=cm_jet)

        if self.shape is not None:
            fig, axs = plt.subplots(*self.shape, constrained_layout=True)

            day_list = list(days['time'].values)

            for xcoord in range(0, self.shape[1]):
                for ycoord in range(0, self.shape[0]):
                    ax = axs[xcoord, ycoord]
                    day = day_list.pop()
                    self.__create_ax_map(ax, var, day, norm, xcoord == (self.shape[1] - 1), ycoord == 0)
            cbar = fig.colorbar(smap, ticks=cbox_ticks, ax=axs.ravel().tolist())
            cbar.set_label(self.wreader.get_long_name(var))

            dir_pth = os.path.join(figure_path, var, 'bundles')
            file_name = os.path.join(dir_pth, f'{fname}{len(days)}_maps{"_isin" if self.isin else ""}')

            self.__save_show_fig(fig, dir_pth, file_name)
        else:
            for day_num, day in enumerate(days["time"].values):
                fig, ax = plt.subplots()

                self.__create_ax_map(ax, var, day, norm, xlbl_true=True, ylbl_true=True)

                cbar = fig.colorbar(smap, ticks=cbox_ticks)
                cbar.set_label(self.wreader.get_long_name(var))

                dir_pth = os.path.join(figure_path, var, fname)
                file_name = os.path.join(dir_pth, f'{day_num}_map{"_isin" if self.isin else ""}')

                self.__save_show_fig(fig, dir_pth, file_name)

    def plot_nmin(self, var, n=4):
        """Plot/save the n days with the smallest values for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nmin_val_days(var, n)
        self.__plot_days(days, 'min')

    def plot_nmax(self, var, n=4):
        """Plot/save the n days with the largest values for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nmax_val_days(var, n)
        self.__plot_days(days, 'max')

    def plot_nmin_var(self, var, n=4):
        """Plot/save the n days with the smallest variance for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nminvar_val_days(var, n)
        self.__plot_days(days, 'minvar')

    def plot_nmax_var(self, var, n=4):
        """Plot/save the n days with the largest variance for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nmaxvar_val_days(var, n)
        self.__plot_days(days, 'maxvar')

    def plot_nmin_mean(self, var, n=4):
        """Plot/save the n days with the smallest mean for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nminmean_val_days(var, n)
        self.__plot_days(days, 'minmean')

    def plot_nmax_mean(self, var, n=4):
        """Plot/save the n days with the largest mean for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nmaxmean_val_days(var, n)
        self.__plot_days(days, 'maxmean')

    def plot_nmin_med(self, var, n=4):
        """Plot/save the n days with the smallest median for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nminmed_val_days(var, n)
        self.__plot_days(days, 'minmed')

    def plot_nmax_med(self, var, n=4):
        """Plot/save the n days with the largest median for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
           """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nmaxmed_val_days(var, n)
        self.__plot_days(days, 'maxmed')

    def plot_nmin_sum(self, var, n=4):
        """Plot/save the n days with the smallest sum for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f"variable '{var}' not found"

        days = self.wreader.nminsum_val_days(var, n)
        self.__plot_days(days, 'minsum')

    def plot_nmax_sum(self, var, n=4):
        """Plot/save the n days with the largest sum for the specified
           variable reduced over longitude and latitude

        Parameters
        ----------
        var : string
              name of variable
        n   : integer
              number of plots

        Returns
        -------
        None
        """
        assert var in self.wreader.get_vars(), f'variable "{var}" not found'

        days = self.wreader.nmaxsum_val_days(var, n)
        self.__plot_days(days, 'maxsum')

    def plot_isin(self):
        """Plot map showing which grid points are within germany

        Returns
        -------
        None
        """
        try:
            contained = np.load(os.path.join(isin_path, 'isinDE.npy'))
        except:
            log.info(f'isin file not found in {data_path}')
            util = Utility()
            contained = util.check_isinDE()

        fig, ax = plt.subplots()
        ax.imshow(contained, cmap=plt.cm.Greys, extent=bbox)

        ax.set_ylabel(lat_col)
        ax.set_xlabel(lon_col)

        file_name = os.path.join(figure_path, 'isinDE')
        self.__save_show_fig(fig, figure_path, file_name)

    def plot_isin_region(self, region_id):
        """Plot map showing which grid points are within specified region

        Parameters
        ----------
        region_id : string
                    the id of the region as string

        Returns
        -------
        None
        """
        util = Utility()
        try:
            contained = np.load(os.path.join(isin_path, f'isin{region_id}.npy'))
        except:
            log.info(f'isin file not found in {isin_path} for region {region_id}')
            contained = util.check_isin_region(region_id)

        fig, ax = plt.subplots()
        ax.imshow(contained, cmap=plt.cm.Greys, extent=bbox)

        ax.set_ylabel(lat_col)
        ax.set_xlabel(lon_col)

        file_name = os.path.join(figure_path, f'isin{util.get_region_name(region_id)}_{region_id}')
        self.__save_show_fig(fig, figure_path, file_name)

    def plot_isin_top_n(self, n, year):
        """Plot map showing which grid points are within n regions with highest population for specified year

        Parameters
        ----------
        n    : integer
               number of regions with highest population to plot
        year : integer
               specifies for which year population is checked

        Returns
        -------
        None
        """
        util = Utility()
        contained = util.demo_top_n_regions_map(n,2018)

        fig, ax = plt.subplots()
        ax.imshow(contained, cmap=plt.cm.Greys, extent=bbox)

        ax.set_ylabel(lat_col)
        ax.set_xlabel(lon_col)

        file_name = os.path.join(figure_path, f'isin_top{n}_year{year}')
        self.__save_show_fig(fig, figure_path, file_name)

    def plot_demo4year(self, year):
        """Plot a map of germany showing regional population data on NUTS 3 level

        Parameters
        ----------
        year : int
               name of variable

        Returns
        -------
        None
        """
        assert (year >= 2015 and year <=2018), "demography data only existing from 2015 to 2018"
        demo_df = pd.read_csv(demography_file, encoding='latin1', index_col='GEO')
        demo_df['Value'] = demo_df['Value'].map(lambda val: pd.NaT if val == ':' else float(val.replace(',', '')))
        df = demo_df[demo_df['TIME'] == year]
        with shp.Reader(nuts3_01res_shape) as nuts3_sf:
            regions = [rec for rec in nuts3_sf.shapeRecords() if rec.record['CNTR_CODE'] == 'DE']
        values = np.array([df.loc[region.record['NUTS_ID'], :]['Value'] for region in regions]) / 1000
        _min = values.min()
        _max = values.max()

        fig, ax = plt.subplots()
        plt.xlim([5.5, 15.5])
        plt.ylim([47, 55.5])
        ax.set_xlabel(lon_col)
        ax.set_ylabel(lat_col)

        # for logarithmic colorbar
        cbox_bound = np.exp(np.linspace(np.log(_min), np.log(_max), 256))
        norm = colors.BoundaryNorm(cbox_bound, ncolors=256)
        sm = cm.ScalarMappable(norm=norm, cmap=cm.get_cmap('jet'))
        cbar = plt.colorbar(sm)
        cbar.set_label('inhabitants (in 1k)')

        for value, region in zip(values, regions):
            ax.add_patch(PolygonPatch(region.shape.__geo_interface__, fc=sm.to_rgba(value), ec='none'))

        dir_pth = os.path.join(figure_path, 'demo')
        file_name = os.path.join(dir_pth, f'demo{year}_logscale')

        self.__save_show_fig(fig, dir_pth, file_name)

    def plot_load_acf(self, lags=48, hour_steps=1, ndiff=0):
        """Plot autocorrelation plot of load data within given time range
           and given lags, hour steps and number of differences

        Parameters
        ----------
        lags       : integer
                     specifies number of lags shown on plot
        hour_steps : integer
                     specifies time steps in data in hours
        ndiff      : integer
                     specifies how often to differentiate before plotting

        Returns
        -------
        None
        """
        data = self.lreader.vals4step(de_load, step=hour_steps).interpolate_na(dim='utc_timestamp', method='linear')\
                   .diff(dim='utc_timestamp', n=ndiff).values
        fig = plot_acf(data, fft=True, use_vlines=True, lags=lags)
        dir_pth = os.path.join(figure_path, 'ACF')
        file_name = os.path.join(dir_pth, f'load_{lags}lags_ndiff{ndiff}_hstep{hour_steps}')
        self.__save_show_fig(fig, dir_pth, file_name)

    def plot_load_pacf(self, lags=48, hour_steps=1, ndiff=0):
        """Plot partial autocorrelation plot of load data within given time
           range and given lags, hour steps and number of differneces

        Parameters
        ----------
        lags       : integer
                     specifies number of lags shown on plot
        hour_steps : integer
                     specifies time steps in data in hours
        ndiff      : integer
                     specifies how often to differentiate before plotting

        Returns
        -------
        None
        """
        data = self.lreader.vals4step(de_load, step=hour_steps).interpolate_na(dim='utc_timestamp', method='linear')\
                   .diff(dim='utc_timestamp', n=ndiff).values
        fig = plot_pacf(data, use_vlines=True, lags=lags)
        dir_pth = os.path.join(figure_path, 'PACF')
        file_name = os.path.join(dir_pth, f'load_{lags}lags_ndiff{ndiff}_hstep{hour_steps}')
        self.__save_show_fig(fig, dir_pth, file_name)

    def plot_load_time_func(self, var, start, stop, func, load_col=de_load,
                            freq=24, aspect=(12, 5), skip_bottom_labels=False):
        """Plot/save function of load and date with variable
           after applying given function to its data

        Parameters
        ----------
        var                : string
                             name of variable to plot
        start              : pandas.Timestamp
                             starting time (e.g. start = pandas.Timestamp(datetime(2015,1,1,12),tz='utc'))
        stop               : pandas.Timestamp
                             stopping time
        func               : function object
                             function applied to weather data to reduce over longitude and latitude
        load_col           : string
                             specifies column in load file that will be plotted
        freq               : integer (where freq mod 2 == 0, as resolution of data is 2H)
                             specifies in what frequency of hours points will be plotted
        aspect             : tuple of ints
                             defines the aspect ratio of the plot
        skip_bottom_labels : boolean
                             specifies whether to skip the bottom label or not

        Returns
        -------
        None
        """
        assert (freq % 2 == 0), "frequency must be dividable by 2 as resolution of data is 2h"

        fname = func.__name__

        rng = pd.date_range(start, stop, freq=f'{freq}H')
        load = self.lreader.vals4slice(load_col, start, stop, step=freq)
        ncval = self.wreader.reduce_lonlat(var, func).sel(time=rng)

        # select data for workdays
        ncweek = ncval.where((ncval['time.weekday'] != 5) & (ncval['time.weekday'] != 6), drop=True)
        loadweek = load.where((load['utc_timestamp.weekday'] != 5) & (load['utc_timestamp.weekday'] != 6), drop=True)
        rngweek = rng.where((rng.weekday != 5) & (rng.weekday != 6), other=pd.NaT).dropna()

        # select data for weekends
        ncwend = ncval.where((ncval['time.weekday'] == 5) | (ncval['time.weekday'] == 6), drop=True)
        loadwend = load.where((load['utc_timestamp.weekday'] == 5) | (load['utc_timestamp.weekday'] == 6), drop=True)
        rngwend = rng.where((rng.weekday == 5) | (rng.weekday == 6), other=pd.NaT).dropna()

        # use figsize to stretch
        fig, ax = plt.subplots(figsize=aspect)
        ax.scatter(rngwend, loadwend, s=24, c=ncwend, cmap='jet', edgecolors='none', label='weekend',
                   marker=markers.MarkerStyle(marker="^", fillstyle='none'))
        ax.scatter(rngweek, loadweek, s=12, c=ncweek, cmap='jet', edgecolors='none', label='workday')
        ax.set_ylabel(variable_dictionary[load_col])
        ax.legend()  # , markerscale=2)

        cmap = plt.get_cmap('jet', 256)
        norm = colors.Normalize(vmin=ncval.min(), vmax=ncval.max())
        scal_map = plt.cm.ScalarMappable(norm=norm, cmap=cmap)
        ticks = np.linspace(ncval.min(), ncval.max(), 8)

        cbar = fig.colorbar(scal_map, ticks=ticks, ax=ax)  # , pad=.04, shrink=.6) # shrink to fit if aspect is changed
        cbar.ax.set_ylabel(f'{self.wreader.get_long_name(var)} {fname} reduce over DE (K)',
                           rotation=90, rotation_mode='anchor')

        if not skip_bottom_labels:
            # rotate labels from x-axis by 30° for readability
            plt.setp(ax.get_xticklabels(), rotation=30, horizontalalignment='right')
        else:
            ax.set_xticklabels([])

        dir_pth = os.path.join(figure_path, 'plot_load_time_func')
        file_name = os.path.join(dir_pth, f'{var}_{fname}_{aspect[0]}A{aspect[1]}_'
                                 f'{start.strftime("%Y%m%d%H")}_{stop.strftime("%Y%m%d%H")}_{freq}F')

        self.__save_show_fig(fig, dir_pth, file_name)

    def plot_load(self, var, start, stop, freq=1, aspect=(12, 5), skip_bottom_labels=False):
        """Plot/save function of load variable/s

        Parameters
        ----------
        var                : list of strings
                             names of variables to plot
        start              : pandas.Timestamp
                             starting time (e.g. start = pandas.Timestamp(datetime(2015,1,1,12),tz='utc'))
        stop               : pandas.Timestamp
                             stopping time
        freq               : integer (where freq mod 2 == 0, as resolution of data is 2H)
                             specifies in what frequency of hours points will be plotted
        aspect             : tuple of ints
                             defines the aspect ratio of the plot
        skip_bottom_labels : boolean
                             specifies whether to skip the bottom label or not

        Returns
        -------
        None
        """
        # assert var in self.lreader.get_vars(), f'variable "{var}" not found' # assert not needed

        fig, ax = plt.subplots(figsize=aspect)

        drange = pd.date_range(start, stop, freq=f'{freq}H')

        for var_name in var:
            data = self.lreader.vals4time(var_name, drange)
            ax.plot(drange, data, '-', linewidth=1, label=variable_dictionary[var_name])

        ax.set_xlabel('UTC time')
        ax.set_ylabel('load (MW)')
        ax.set_ylim(-2000, 120000)
        ax.legend(loc='upper right')

        if not skip_bottom_labels:
            # rotate labels from x-axis by 30° for readability
            plt.setp(ax.get_xticklabels(), rotation=30, horizontalalignment='right')
        else:
            ax.set_xticklabels([])

        dir_pth = os.path.join(figure_path, 'load_plot')
        file_name = os.path.join(dir_pth, f'{"_".join([varname[:7] for varname in var])}_{aspect[0]}A{aspect[1]}_'
                                 f'{start.strftime("%Y%m%d%H")}_{stop.strftime("%Y%m%d%H")}_{freq}F')
        self.__save_show_fig(fig, dir_pth, file_name)

    def plot_armax_forecast(self, tstart, tstop, forecast_end, p, q, exog=None,
                            save_armax=False, plot_range=None):
        """Plot an ARMAX forecast for the given parameters

        Parameters
        ----------
        tstart       : datetime.datetime
                       start time
        tstop        : datetime.datetime
                       stop time
        forecast_end : datetime.datetime
                       stop time of forecast (=tstop + forecast length)
        p            : integer
                       specifies number of AR coefficients
        q            : integer
                       specifies number of MA coefficients
        exog         : list of strings
                       specifies the variables to include as exogenous variables
        save_armax   : bool
                       specifies whether to save the armax or not
        plot_range   : None or tuple of datetime
                       specifies wether to plot specific range of forecast

        Returns
        -------
        None
        """
        armax = ARMAXForecast(tstart, tstop, p, q, exog=exog, const=False)
        armax.train()
        armax.summary()
        if save_armax:
            armax.save()
        forecast = armax.predict_one_step_ahead(forecast_end)
        fig, ax = plt.subplots()
        # for i,hours in enumerate(hours_range):
        #     ax.plot(fc_range,forecast[i],label=f'{hours}H forecast')
        #     log.info(armax.forecasts[i])
        log.info(armax.forecasts[0])

        if plot_range is None:
            fc_range = pd.date_range(tstop+timedelta(hours=1), forecast_end, freq='1H')
            ax.plot(fc_range, forecast.forecast, label='1H forecast')
            ax.plot(fc_range, forecast.actual, label='actual value')
        else:
            fc_range = pd.date_range(plot_range[0], plot_range[1], freq='1H')
            df = forecast.sel(plot_range[0], plot_range[1])
            ax.plot(fc_range, df['forecast'].values, label='1H forecast')
            ax.plot(fc_range, df['actual'].values, label='actual value')

        ax.set_ylabel('load [MW]')
        plt.setp(ax.get_xticklabels(), rotation=30, horizontalalignment='right')
        plt.legend()

        dir_pth = os.path.join(figure_path, 'ARMAXfc')
        file_name = os.path.join(dir_pth,
                                 f'ARMAX_p{p}q{q}_data{tstart.year}to{tstop.year}_fcto{forecast_end.strftime("%Y%m%d%H")}'
                                 f'{"" if exog is None else "_" + exog[0] if len(exog) == 1 else "_" + "_".join(exog)}'
                                 f'{"" if plot_range is None else "_plot_range" + plot_range[0].strftime("%Y%m%d%H")}'
                                 f'{"" if plot_range is None else "_" + plot_range[1].strftime("%Y%m%d%H")}')
        self.__save_show_fig(fig, dir_pth, file_name)

ext: py

max_stars_repo_name: maGitty/GISME

max_issues_repo_name: maGitty/GISME

max_forks_repo_name: maGitty/GISME

reason: import numpy,from statsmodels


text:
# -*- coding: utf-8 -*-
"""
Spectrogram of sinus
@author: Kaw
"""
#!/usr/bin/env python
from pylab import subplot, plot, ylabel, title, xlabel, show, specgram, cm, figure
from numpy import arange, where, sin, pi, logical_and, logical_or, fft, linspace
from numpy.random import randn

Fs = 500                    # a frequência de amostragem em Hz
dt = 1./Fs
t = arange(0.0, 20.0, dt)
s1 = sin(2*pi*10*t)         # sinal de 10 Hz
s2 = 5*sin(2*pi*40*t)       # sinal de 40 Hz

# cria um zunido (chirp) transitório
mask1 = where(logical_or(t<10, t>15), 1.0, 0.0)
s1 = s1 * mask1
mask2 = where(logical_and(t>10, t<15), 1.0, 0.0)
s2 = s2 * mask2
nse = 0.01*randn(len(t))    # adiciona algum ruído na mistura
x = s1 + s2 #+ nse           # o sinal final
N = 256                  # o comprimento das janelas dos segmentos

X = fft.fft(x)
freq = linspace(0,Fs/2.,int(Fs/2.*20))

# Pxx é a matriz de 'segmentos' x 'freqs' da potência instantânea, 'freqs' é
# vetor frequência, 'bins' são os centros dos intervalos de tempo nos quais
# a potência é calculada, e 'im' é o objeto matplotlib.image.AxesImage
figure()
ax1 = subplot(311); plot(t,x); ylabel('Amplitude'); title('Espectrograma')
#subplot(312); plot(freq,2*abs(X[:len(X)/2]))
subplot(313,sharex=ax1)
Pxx,freqs,bins,im = specgram(x,NFFT=N,Fs=Fs,cmap=cm.gist_heat)
title(u'Soma de Senos de 10 e 40 Hz e ruído gaussiano')
xlabel('tempo (s)'); ylabel(u'frequência (Hz)')
show()

max_stars_repo_path: 08 Espectograma/spectrogram.py

max_issues_repo_path: 08 Espectograma/spectrogram.py

max_forks_repo_path: 08 Espectograma/spectrogram.py

reason: from numpy


text:
from sklearn.preprocessing import LabelEncoder
import numpy as np

class EloDataset:
    def __init__(
        self,
        valid_fration=0.2,
        test_fraction=0.2,
        time=None,
        valid_date=None,
        test_date=None,
    ):
        """
        Dataset object to train the Elo rating model. The data are split in three sets:
        the train set where the model is fitted, the validation set used for early stopping and the test set.
        Sets are split using the fraction of the total dataset size if no time and dates are provided.

        Parameters
        ----------

        test_fraction: double
            Define the size of the test data in % of the dataset total size.

        valid_fration: double
            Define the size of the validation data in % of the dataset total size.

        time: array of datetime (optional)
            Time index for matches.

        valid_date: date (optional)
            Date to split between train (before it) and validation set.

        test_date: date (optional)
            Date to split between validation (before it) and test set.

        """

        self.time = time
        self.test_date = test_date
        self.valid_date = valid_date
        self.test_fraction = test_fraction
        self.valid_fration = valid_fration

    def encode_teams(self, team_names):
        """Encode team name."""
        self.le_ = LabelEncoder().fit(list(team_names[:, 0]) + list(team_names[:, 1]))
        team_index = np.zeros_like(team_names)
        team_index[:, 0] = self.le_.transform(team_names[:, 0])
        team_index[:, 1] = self.le_.transform(team_names[:, 1])
        self.n_teams_ = len(self.le_.classes_)
        self.team_index_ = np.array(team_index).astype(np.int32)

    def split_train_test(self, team_names, scores):
        """
        Split the data for training.

        Parameters
        ----------
        team_names: string array of size (n_matches,2)
            Array of team names with names of team A in column 0 and names of team B in column 1.

        scores: array of size (n_matches,2)
            Array of goals score by Team A (column 0) and team B (column 1)

        Returns
        -------

        """
        self.scores_ = np.array(scores).astype(np.float32)
        self.encode_teams(team_names)

        if self.time is None:
            time = np.array(range(len(scores)))
        else:
            time = self.time

        if None not in (self.test_date, self.valid_date):
            assert time is not None
            test_date = self.test_date
            valid_date = self.valid_date
            split_type_ = "date"
        else:
            test_idx = int((1 - self.test_fraction) * len(time))
            train_idx = int((1 - self.test_fraction - self.valid_fration) * len(time))
            test_date = time[test_idx]
            valid_date = time[train_idx]
            split_type_ = "fraction"

        self.train_index_ = time < valid_date
        self.valid_index_ = (time > valid_date) & (time < test_date)
        self.test_index_ = time > test_date
        print(
            f"dataset split using {split_type_}: train size {int(sum(self.train_index_))}, validation size {int(sum(self.valid_index_))}, test size {int(sum(self.test_index_))}"
        )

    def get_train_split(self):
        return {
            "team_index": self.team_index_[self.train_index_, :],
            "scores": self.scores_[self.train_index_, :],
        }

    def get_dataset(self):
        assert hasattr(
            self, "scores_"
        ), "split_train_test(team_names, scores) needs to be call first."
        return {"team_index": self.team_index_, "scores": self.scores_}

reason: import numpy


text:
import sys
import os
import glob
import csv
import itertools
from pprint import pprint

import click
import rasterio
import numpy as np

from uncoverml import mpiops

def main(path, csvfile, recursive):
    """
    Will output some basic diagnostic information for geotiffs.
    If a directory is provided, all geotiffs in the direcotry (and
    subdirectories if '-r' option is used) will be processed. Providing
    a single filepath will only process that file.

    To store the printed output in a txt file, use output redirection:
    'covdiag /path/to/file >> output.txt'.
    """
    diags = []
    if os.path.isdir(path):
        if recursive:
            paths = mpiops.run_once(glob.glob, os.path.join(path, '**', '*.tif'), recursive=recursive)
        else:
            paths = mpiops.run_once(glob.glob, os.path.join(path, '*.tif'))
    else:
        paths = [path]
    if mpiops.chunk_index == 0:
        if not paths:
            print(f"No geotiffs found.")
        else:
            print(f"Found {len(paths)} geotiffs, retrieving information...")
    this_chunk_paths = np.array_split(paths, mpiops.chunks)[mpiops.chunk_index]
    for f in this_chunk_paths:
        diag = diagnostic(f)
        if diag is not None:
            diags.append(diag)
            print(f"Processed '{f}'")

    diags = mpiops.comm.gather(diags, root=0)
    mpiops.comm.barrier()

    if mpiops.chunk_index == 0:
        diags = list(itertools.chain.from_iterable(diags))

        fieldnames = ['name', 'driver', 'crs', 'dtype', 'width',
                      'height', 'bands', 'nodata', 'ndv_percent',
                      'min', 'max']
        if csvfile:
            with open(csvfile, 'w') as f:
                w = csv.DictWriter(f, fieldnames=fieldnames)
                w.writeheader()
                for diag in diags:
                    w.writerow(diag)
                    print(printer(diag))
        else:
            for diag in diags:
                print(printer(diag))

def diagnostic(filename):
    try:
        src = rasterio.open(filename)
    except rasterio.errors.RasterioIOError:
        print(f"Couldn't load '{filename}'\n")
        return None
    diag = {}
    diag['name'] = os.path.basename(filename)
    diag.update(src.meta)
    del diag['transform']
    diag['crs'] = diag['crs'].to_string()
    diag['bands'] = diag['count']
    del diag['count']
    diag['ndv_percent'] = []
    diag['min'] = []
    diag['max'] = []
    for i in range(1, diag['bands'] + 1):
        band = src.read(i)
        diag['ndv_percent'].append(_percentage(band, src.nodata, diag['width'] * diag['height']))
        diag['min'].append(np.min(band))
        diag['max'].append(np.max(band))
    src.close()
    return diag

def printer(diag):
    pretty_string = (
        f"Name:   {diag['name']}\n"
        f"Driver: {diag['driver']}\n"
        f"CRS:    {diag['crs']}\n"
        f"Dtype:  {diag['dtype']}\n"
        f"Width:  {diag['width']}\n"
        f"Height: {diag['height']}\n"
        f"Bands:  {diag['bands']}\n"
        f"NDV:    {diag['nodata']}\n"
        "Band stats:\n"
    )
    for i in range(diag['bands']):
        pretty_string += (
            f"\tBand {i + 1} NDV: {diag['ndv_percent'][i]}%\n"
            f"\tBand {i + 1} min: {diag['min'][i]}\n"
            f"\tBand {i + 1} max: {diag['max'][i]}\n"
        )
    return pretty_string

def _percentage(band, ndv, n_elements):
    return np.count_nonzero(band == ndv) / n_elements * 101

max_stars_repo_name: GeoscienceAustralia/uncoverml

max_issues_repo_name: GeoscienceAustralia/uncoverml

max_forks_repo_name: GeoscienceAustralia/uncoverml


text:
import sys
import os
import numpy as np
from sklearn.model_selection import GroupKFold, StratifiedKFold, KFold
import shutil
from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained('/mnt/model/scibert_scivocab_uncased', do_lower_case=True)

_, tsv_dir, group_tsv_dir, rad0_dir, rad1_dir, rad2_dir, k, output_dir = sys.argv

with open(os.path.join(tsv_dir, 'train.tsv'), 'r') as f:
    lines = f.read().strip().split('\n')
label_d = {'negative':0, 'mechanism':1, 'effect':2, 'advise':3, 'int':4}
labels = [label_d[l.split('\t')[1]] for l in lines]

with open(os.path.join(group_tsv_dir, 'train.tsv'), 'r') as f:
    g_lines = f.read().strip().split('\n')

print(len(lines), len(g_lines))

groups = []
#for l, gl in zip(lines, g_lines):
interval = 20
for l, gl, label in zip(lines, g_lines, labels):
    assert l.split('\t')[0] == gl.split('\t')[0]
    entity1_id = gl.split('\t')[4]
    sent = l.split('\t')[0]
    tokenized_sent = tokenizer.tokenize(sent)
    sentence_length = len(tokenized_sent)
    if sentence_length > 128:
        div = 128 // interval
    else:
        div = sentence_length // interval

    if entity1_id.split('.')[0] == 'DDI-MedLine':
        ctype = 0
    elif entity1_id.split('.')[0] == 'DDI-DrugBank':
        ctype = 1
    else:
        raise KeyError

    #fixed_label = label
    fixed_label = div * 5 + label
    #fixed_label = div * 10 + ctype * 5 + label
    groups.append(fixed_label)

print(groups)
k = int(k)
skf = StratifiedKFold(n_splits=k)

cnt = 1
for train_idx, test_idx in skf.split(lines, groups):

    if not os.path.exists(os.path.join(output_dir, str(cnt), 'tsv')):
        os.makedirs(os.path.join(output_dir, str(cnt), 'tsv'))
    tr_tsv_path = os.path.join(output_dir, str(cnt), 'tsv', 'train.tsv')
    f_tr = open(tr_tsv_path, 'w')
    for idx in train_idx:
        f_tr.write(lines[idx]+'\n')
    f_tr.close()

    te_tsv_path = os.path.join(output_dir, str(cnt), 'tsv', 'dev.tsv')
    f_te = open(te_tsv_path, 'w')
    for idx in test_idx:
        f_te.write(lines[idx]+'\n')
    f_te.close()

    for rad_dir, rad_name in zip((rad0_dir, rad1_dir, rad2_dir), ('rad0', 'rad1', 'rad2')):
        rad_npy = np.load(os.path.join(rad_dir, 'corpus_train.npy'), allow_pickle=True)

        if not os.path.exists(os.path.join(output_dir, str(cnt), rad_name)):
            os.makedirs(os.path.join(output_dir, str(cnt), rad_name))
        train_rad = rad_npy[train_idx]
        test_rad = rad_npy[test_idx]

        np.save(os.path.join(output_dir, str(cnt), rad_name, 'corpus_train.npy'), train_rad)
        np.save(os.path.join(output_dir, str(cnt), rad_name, 'corpus_dev.npy'), test_rad)

        shutil.copy(os.path.join(rad_dir, 'config.json'), os.path.join(output_dir, str(cnt), rad_name))

    cnt += 1


text:
# Este exemplo carrega a base Wine da UCI, treina uma Arvore de decisao usando
# holdout e outra usando validacao cruzada com 10 pastas.

# Importa bibliotecas necessarias
import numpy as np
import urllib
from sklearn import tree
from sklearn import  model_selection
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.externals.six import StringIO
from sklearn.tree import export_graphviz
from IPython.display import Image
from IPython.display import display
import pydotplus

#from sklearn.model_selection import StratifiedShuffleSplit
# Carrega uma base de dados do UCI
# Exemplo carrega a base Wine
url = "http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data"
raw_data = urllib.request.urlopen(url)

# Carrega arquivo como uma matriz
dataset = np.loadtxt(raw_data, delimiter=",")

# Imprime quantide de instancias e atributos da base
print("Instancias e atributos")
print(dataset.shape)

# Coloca em X os 13 atributos de entrada e em y as classes
# Observe que na base Wine a classe eh primeiro atributo
X = dataset[:,1:13]
y = dataset[:,0]

# EXEMPLO USANDO HOLDOUT
# Holdout -> dividindo a base em treinamento (70%) e teste (30%), estratificada
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=.3, random_state=42, stratify=y)

# declara o classificador
clfa = tree.DecisionTreeClassifier(criterion='entropy')

# treina o classificador
clfa = clfa.fit(X_train, y_train)

# testa usando a base de testes
predicted=clfa.predict(X_test)

# calcula a acuracia na base de teste (taxa de acerto)
score=clfa.score(X_test, y_test)

# calcula a matriz de confusao
matrix = confusion_matrix(y_test, predicted)

# apresenta os resultados
print("\nResultados baseados em Holdout 70/30")
print("Taxa de acerto = %.2f " % score)
print("Matriz de confusao:")
print(matrix)

# EXEMPLO USANDO VALIDACAO CRUZADA
clfb = tree.DecisionTreeClassifier(criterion='entropy')
folds=10
result = model_selection.cross_val_score(clfb, X, y, cv=folds)

print("\nResultados baseados em Validacao Cruzada")
print("Qtde folds: %d:" % folds)
print("Taxa de Acerto: %.2f" % result.mean())
print("Desvio padrao: %.2f" % result.std())

# matriz de confusão da validacao cruzada
Z = model_selection.cross_val_predict(clfb, X, y, cv=folds)
cm=confusion_matrix(y, Z)
print("Matriz de confusao:")
print(cm)

#imprime a arvore gerada
print("\nArvore gerada no experimento baseado em Holdout")
dot_data = StringIO()
export_graphviz(clfa, out_file=dot_data,
                filled=True, rounded=True,
                special_characters=True)

graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
im=Image(graph.create_png())
display(im)

max_stars_repo_name: Matheusqz/pucpr-ciencia-de-dados

max_issues_repo_name: Matheusqz/pucpr-ciencia-de-dados

max_forks_repo_name: Matheusqz/pucpr-ciencia-de-dados


text:
#!/usr/bin/env python
#coding=utf-8
import pickle
import sys, os, re, subprocess, math
from os.path import abspath, dirname, join
whereami = abspath(dirname(__file__))
sys.path.append(whereami)
import pandas as pd

import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder

def load_dataset(name):

	if name == 'hepatitis':
		fileName = 'hepatitis.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = None, sep = ',')
		df = df.applymap(lambda x: np.nan if '?' in str(x) else x)
		df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
		tempX = df.values[:,0:-2]
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'liver':
		fileName = 'Liver.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = 0, sep = ',')
		tempX = df.values[:,0:-1]
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'blood':
		fileName = 'Blood.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = 0, sep = ',')
		df = df.applymap(lambda x: np.nan if '?' in str(x) else x)
		df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
		tempX = df.values[:,1:-1]
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'mammographic':
		fileName = 'mammographic_masses_data.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = 0, sep = ',')
		df = df.applymap(lambda x: np.nan if '?' in str(x) else x)
		df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
		tempX = df.values[:,1:-1]
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'loan':
		fileName = 'loan.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = None, sep = ',')
		df = df.applymap(lambda x: np.nan if '?' in str(x) else x)
		df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
		tempX = df.values[:,0:-1]
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'tae':
		fileName = 'tae.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = None, sep = ',')
		df = df.applymap(lambda x: np.nan if '?' in str(x) else x)
		df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
		tempX = df.values[:,:-1]
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'cleveland':
		fileName = 'cleveland.data'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = None, sep = ',')
		df = df.applymap(lambda x: np.nan if '?' in str(x) else x)
		df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
		tempX = df.values[:,:-1]
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'cmc':
		fileName = 'cmc.data'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = None, sep = ',')
		df = df.applymap(lambda x: np.nan if '?' in str(x) else x)
		df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))
		df_sub_copy = df[ list(df.columns)[2:-1] ].copy()
		tempX1 = pd.get_dummies(df_sub_copy).values
		tempX2 = df.values[:,0:2]
		tempX = np.concatenate((tempX1, tempX2), axis=1)
		y = LabelEncoder().fit_transform(df.values[:,-1])

	if name == 'wine':
		fileName = 'wine_quality_red.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = 0, sep = ',')
		tempX = df.values[:,1:-2]
		y = LabelEncoder().fit_transform(df.values[:,-1])
		print(tempX[0], y[0], tempX.shape)

	if name == 'abalone':
		fileName = 'abalone.csv'
		df = pd.read_csv( join(whereami+'/dataset', fileName ), header = 0, sep = ',')
		tempX = df.values[:,2:-2]
		y = LabelEncoder().fit_transform(df.values[:,1])
		print(tempX[0], y[0], tempX.shape)

	X = tempX.astype(float)
	return X,y

def loadSeer(numberOfClass):

	def decide_range(time_range, target):
		if target == -1:
			import random
			print "target <= 0, please double check"
			raw_input()
			return random.randint(1,3)
		for i in range(len(time_range)):
			if time_range[i]<=target<=time_range[i+1]:
				return i

	records = pd.read_csv( join(whereami+'/dataset', 'seer.log' ), header = 0, sep = '\t')
	label = "SRV_TIME_MON"
	result = records.sort([label], ascending=True, axis = 0)[label]

	list_range = [0]
	for temp in range(1, numberOfClass):
		temp_p = result.values[ int(result.count()/numberOfClass * temp) ]
		list_range.append( temp_p )
	list_range.append( records[label].max(axis=0) )

	records[label + u'.outcome'] = records.apply(lambda x: decide_range( list_range , x[label]), axis=1)
	records.drop("SRV_TIME_MON", axis=1, inplace=True)

	tag = "SRV_TIME_MON.outcome"
	y = records[tag].values
	X = records[ list(set(records.columns) - set(tag)) ].values

	from sklearn.preprocessing import normalize
	X = normalize(X, axis=0, norm='l2')

	return X, y

if __name__ == '__main__':
	loadSeer(numberOfClass = 3)

max_stars_repo_name: superwaiwjia/lowRankForSeer

max_issues_repo_name: superwaiwjia/lowRankForSeer

max_forks_repo_name: superwaiwjia/lowRankForSeer


text:
"""
    Given a coordinate, find the closest vertex in the graph quickly
"""
import math
import numpy as np

def coord_match(vertices, target_coord: np.array, pos, minimum=0.0009):
    """
    Given a coordinate, find the closest vertex in the graph quickly
    Args:
        vertices: nx2 matrix, where n is the number of vertices. Each row is the x, y
        position of the vertex associated with that row
        target_coord: the coordinate in lat, long format
        pos: the definition of distance in our graph; typically x is long and y is lat
        minimum: the inital bounding box around the coord; the closest match is expected
                 to be within this distance of the coordinate
    Returns:
        closest vertex, target
    """
    # find all points within a box around the coordinate
    lower_left = np.array([target_coord[1] - minimum, target_coord[0] - minimum])
    upper_right = np.array([target_coord[1] + minimum, target_coord[0] + minimum])
    indices = np.all(
        np.logical_and(lower_left <= vertices, vertices <= upper_right), axis=1
    )
    inside_box = np.where(indices == 1)
    target = None
    # use euclidean distance to pick the closest point within the box
    for v in inside_box[0]:
        if (
            math.sqrt(
                np.sum(
                    np.square(pos[v].a - np.array([target_coord[1], target_coord[0]]))
                )
            )
            < minimum
        ):
            minimum = math.sqrt(
                np.sum(
                    np.square(pos[v].a - np.array([target_coord[1], target_coord[0]]))
                )
            )
            target = v
    if target is None:
        return coord_match(vertices, target_coord, pos, minimum * 2)
    return target

max_stars_repo_name: alan-turing-institute/urbanroute

max_issues_repo_name: alan-turing-institute/urbanroute

max_forks_repo_name: alan-turing-institute/urbanroute


text:
#!/usr/bin/python
# -*- encoding: utf-8 -*-

import sys
sys.path.insert(0, '.')
import os
import os.path as osp
import random
import logging
import time
import argparse
import numpy as np

import torch
import torch.nn as nn
import torch.distributed as dist
from torch.utils.data import DataLoader

from bisenetv2.bisenetv2 import BiSeNetV2
from bisenetv2.cityscapes_cv2 import get_data_loader
from bisenetv2.evaluatev2 import eval_model
from bisenetv2.ohem_ce_loss import OhemCELoss
from bisenetv2.lr_scheduler import WarmupPolyLrScheduler
from bisenetv2.meters import TimeMeter, AvgMeter
from bisenetv2.logger import setup_logger, print_log_msg

# apex
has_apex = True
try:
    from apex import amp, parallel
except ImportError:
    has_apex = False

## fix all random seeds
torch.manual_seed(123)
torch.cuda.manual_seed(123)
np.random.seed(123)
random.seed(123)
torch.backends.cudnn.deterministic = True
#  torch.backends.cudnn.benchmark = True
#  torch.multiprocessing.set_sharing_strategy('file_system')

lr_start = 5e-2
warmup_iters = 1000
max_iter = 150000  + warmup_iters
ims_per_gpu = 8

def parse_args():
    parse = argparse.ArgumentParser()
    parse.add_argument('--local_rank', dest='local_rank', type=int, default=-1,)
    parse.add_argument('--sync-bn', dest='use_sync_bn', action='store_true',)
    parse.add_argument('--fp16', dest='use_fp16', action='store_true',)
    parse.add_argument('--port', dest='port', type=int, default=44554,)
    parse.add_argument('--respth', dest='respth', type=str, default='./res',)
    return parse.parse_args()

args = parse_args()

def set_model():
    net = BiSeNetV2(19)
    if args.use_sync_bn: net = set_syncbn(net)
    net.cuda()
    net.train()
    criteria_pre = OhemCELoss(0.7)
    criteria_aux = [OhemCELoss(0.7) for _ in range(4)]
    return net, criteria_pre, criteria_aux

def set_syncbn(net):
    if has_apex:
        net = parallel.convert_syncbn_model(net)
    else:
        net = nn.SyncBatchNorm.convert_sync_batchnorm(net)
    return net

def set_optimizer(model):
    wd_params, non_wd_params = [], []
    for name, param in model.named_parameters():
        if param.dim() == 1:
            non_wd_params.append(param)
        elif param.dim() == 2 or param.dim() == 4:
            wd_params.append(param)
    params_list = [
        {'params': wd_params, },
        {'params': non_wd_params, 'weight_decay': 0},
    ]
    optim = torch.optim.SGD(
        params_list,
        lr=lr_start,
        momentum=0.9,
        weight_decay=5e-4,
    )
    return optim

def set_model_dist(net):
    if has_apex:
        net = parallel.DistributedDataParallel(net, delay_allreduce=True)
    else:
        local_rank = dist.get_rank()
        net = nn.parallel.DistributedDataParallel(
            net,
            device_ids=[local_rank, ],
            output_device=local_rank)
    return net

def set_meters():
    time_meter = TimeMeter(max_iter)
    loss_meter = AvgMeter('loss')
    loss_pre_meter = AvgMeter('loss_prem')
    loss_aux_meters = [AvgMeter('loss_aux{}'.format(i)) for i in range(4)]
    return time_meter, loss_meter, loss_pre_meter, loss_aux_meters

def save_model(states, save_pth):
    logger = logging.getLogger()
    logger.info('\nsave models to {}'.format(save_pth))
    for name, state in states.items():
        save_name = 'model_final_{}.pth'.format(name)
        modelpth = osp.join(save_pth, save_name)
        if dist.is_initialized() and dist.get_rank() == 0:
            torch.save(state, modelpth)

def train():
    logger = logging.getLogger()
    is_dist = dist.is_initialized()

    ## dataset
    dl = get_data_loader('./data/', ims_per_gpu, max_iter,
            mode='train', distributed=is_dist)

    ## model
    net, criteria_pre, criteria_aux = set_model()

    ## optimizer
    optim = set_optimizer(net)

    ## fp16
    if has_apex and args.use_fp16:
        net, optim = amp.initialize(net, optim, opt_level='O1')

    ## ddp training
    net = set_model_dist(net)

    ## meters
    time_meter, loss_meter, loss_pre_meter, loss_aux_meters = set_meters()

    ## lr scheduler
    lr_schdr = WarmupPolyLrScheduler(optim, power=0.9,
        max_iter=max_iter, warmup_iter=warmup_iters,
        warmup_ratio=0.1, warmup='exp', last_epoch=-1,)

    ## train loop
    for it, (im, lb) in enumerate(dl):
        im = im.cuda()
        lb = lb.cuda()

        lb = torch.squeeze(lb, 1)

        optim.zero_grad()
        logits, *logits_aux = net(im)
        loss_pre = criteria_pre(logits, lb)
        loss_aux = [crit(lgt, lb) for crit, lgt in zip(criteria_aux, logits_aux)]
        loss = loss_pre + sum(loss_aux)
        if has_apex:
            with amp.scale_loss(loss, optim) as scaled_loss:
                scaled_loss.backward()
        else:
            loss.backward()
        optim.step()
        torch.cuda.synchronize()
        lr_schdr.step()

        time_meter.update()
        loss_meter.update(loss.item())
        loss_pre_meter.update(loss_pre.item())
        _ = [mter.update(lss.item()) for mter, lss in zip(loss_aux_meters, loss_aux)]

        ## print training log message
        if (it + 1) % 100 == 0:
            lr = lr_schdr.get_lr()
            lr = sum(lr) / len(lr)
            print_log_msg(
                it, max_iter, lr, time_meter, loss_meter,
                loss_pre_meter, loss_aux_meters)

    ## dump the final model and evaluate the result
    save_pth = osp.join(args.respth, 'model_final.pth')
    logger.info('\nsave models to {}'.format(save_pth))
    state = net.module.state_dict()
    if dist.get_rank() == 0: torch.save(state, save_pth)

    logger.info('\nevaluating the final model')
    torch.cuda.empty_cache()
    eval_model(net, 4)

    return

def main():
    torch.cuda.set_device(args.local_rank)
    dist.init_process_group(
        backend='nccl',
        init_method='tcp://127.0.0.1:{}'.format(args.port),
        world_size=torch.cuda.device_count(),
        rank=args.local_rank
    )
    if not osp.exists(args.respth): os.makedirs(args.respth)
    setup_logger('BiSeNetV2-train', args.respth)
    train()

if __name__ == "__main__":
    main()

max_stars_repo_name: khsily/BiSeNet

max_issues_repo_name: khsily/BiSeNet

max_forks_repo_name: khsily/BiSeNet


text:
# This file is part of colorspacious
# Copyright (C) 2014 Nathaniel Smith <njs@pobox.com>
# See file LICENSE.txt for license information.

import numpy as np

from .util import stacklast, color_cart2polar, color_polar2cart

class LuoEtAl2006UniformSpace(object):
    """A uniform space based on CIECAM02.

    See :cite:`CAM02-UCS` for details of the parametrization.

    For most purposes you should just use one of the predefined instances of
    this class that are exported as module-level constants:

    * :data:`colorspacious.CAM02UCS`
    * :data:`colorspacious.CAM02LCD`
    * :data:`colorspacious.CAM02SCD`
    """

    def __init__(self, KL, c1, c2):
        self.KL = KL
        self.c1 = c1
        self.c2 = c2

    def __repr__(self):
        return "%s(%r, %r, %r)" % (
            self.__class__.__name__,
            self.KL, self.c1, self.c2)

    def JMh_to_Jpapbp(self, JMh):
        JMh = np.asarray(JMh, dtype=float)
        J = JMh[..., 0]
        M = JMh[..., 1]
        h = JMh[..., 2]
        Jp = (1 + 100 * self.c1) * J / (1 + self.c1 * J)
        Jp = Jp / self.KL
        Mp = (1. / self.c2) * np.log(1 + self.c2 * M)
        ap, bp = color_polar2cart(Mp, h)
        return stacklast(Jp, ap, bp)

    def Jpapbp_to_JMh(self, Jpapbp):
        Jpapbp = np.asarray(Jpapbp)
        Jp = Jpapbp[..., 0]
        ap = Jpapbp[..., 1]
        bp = Jpapbp[..., 2]
        Jp = Jp * self.KL
        J = - Jp / (self.c1 * Jp - 100 * self.c1 - 1)
        Mp, h = color_cart2polar(ap, bp)
        M = (np.exp(self.c2*Mp) - 1) / self.c2
        return stacklast(J, M, h)

CAM02UCS = LuoEtAl2006UniformSpace(1.00, 0.007, 0.0228)
CAM02LCD = LuoEtAl2006UniformSpace(0.77, 0.007, 0.0053)
CAM02SCD = LuoEtAl2006UniformSpace(1.24, 0.007, 0.0363)

def test_repr():
    # smoke test
    repr(CAM02UCS)


text:
#
#
# Use: encrypt("test") or encrypt("test", 50)
#
#
import numpy as np
import json
import base64
def encrypt(text, deg=50):
	l = len(text)
	x = np.arange(l)
	y = []
	for i in range(l):
		y.append(ord(text[i]))
	c = np.polyfit(x, y, deg).tolist()
	JSON = json.dumps(c, separators=(',',':'))
	return base64.b64encode(JSON.encode('ascii'))

max_stars_repo_name: arguiot/RegressionCipher

max_issues_repo_name: CrypTools/RegressionCipher

max_forks_repo_name: CrypTools/RegressionCipher


text:
import numpy as np
import cv2
import imutils

height_map = 250
width_map = 400
clr_robot = 5

map = np.zeros((height_map, width_map, 3), dtype='uint8')

for x in range(width_map):
    for y in range(height_map):
        # co-ordinates for 'map' are interchanged due to OpenCV convention

        # For the Convex circular region
        if ((x-300)**2 + (y-65)**2 <= ((40+clr_robot)**2)):
            map[y][x] = [255,255,255]

        # For the Convex hexagonal region
        if (y+(0.57*x)) >= 218.53 and (y-(0.57*x))>= -10.04 and (x<=240) and (y+(0.57*x))<=310.04 and (y-(0.57*x)) <= 81.465 and (x >= 160):
        # if (j+(0.57*i)-218.53)>=0 and (j-(0.57*i)+10.04)>=0 and (i-240)<=0 and (j+(0.57*i)-310.04)<=0 and (j-(0.57*i)-81.465)<=0 and (i-160)>=0:
            map[y][x] = [255,255,255]

        # For the Non-Convex triangular region
        if ((y+(0.316*x) >= 71.1483) and (y+(0.857*x) <= 145.156) and (y-(0.114*x) <= 60.909)) or ((y-(1.23*x)) <= 28.576 and (y-(3.2*x)) >= -202.763 and (y-(0.114*x)) >= 60.909):
            map[y][x] = [255,255,255]

        # For the outer boundaries (Non-Convex shape)
        if (x<=clr_robot) or (x>=(400-clr_robot)) or (y<=clr_robot) or (y>=(250-clr_robot)):
            map[y][x] = [255,255,255]

# Resizing the map to 800x500 resolution for better visibilty
map = imutils.resize(map, height=2*height_map, width=2*width_map)

cv2.imshow("Map",map)
cv2.waitKey(0)
cv2.destroyAllWindows()

max_stars_repo_name: adarshmalapaka/dijkstra-point-robot

max_issues_repo_name: adarshmalapaka/dijkstra-point-robot

max_forks_repo_name: adarshmalapaka/dijkstra-point-robot


text:
import math
import numpy as np
import pandas as pd

from influx_base import InfluxBase

class FloatQueries(InfluxBase):
    number_of_humidity_temperature_sensors = 30
    IMU_data_identifier = "t1"
    orientation_identifier = "t2"
    temperature_identifier = "t3"

    installation_id_to_float_ids = {
        "proto3": ["f6", "f18"],
        "proto5": ["f14", "f20", "f21", "f8"],
        "proto6": ["f1", "f2", "f3", "f5"],
        "testRig": ["f91", "f92", "f93", "f95"],
        "DAC1": ["f12"],
        "STT": ["f24", "f17", "f23"]

    }

    def __init__(self, token, org="Kodeworks", url="https://ts.sunlitsea.no/"):
        super().__init__(token, org, url)
        self.calculate_solar_angles = True

    # Helper methods
    ######################################################
    def float_id_to_installation_id(self, float_id):
        for installation_id, float_ids in self.installation_id_to_float_ids.items():
            if float_id in float_ids:
                return installation_id

        return "-1"

    @staticmethod
    def to_query_date_string(day, month, year, hour=0, minutes=0, seconds=0):
        return f"{year:04d}-{month:02d}-{day:02d}T{hour:02d}:{minutes:02d}:{seconds:02d}Z"

    @staticmethod
    def custom_sort_columns(arr):
        with_digits: list = list(element for element in arr if any(map(str.isdigit, element)))
        without_digits: list = list(element for element in arr if element not in with_digits)

        # Sort by digits first, then by the first letter
        with_digits.sort(key=lambda x: int(x[1:]))
        with_digits.sort(key=lambda x: x[0])

        return np.array(without_digits + with_digits)

    @staticmethod
    def pitch_roll_to_azimuth_tilt(pitch, roll):
        azimuth = - math.atan2(math.sin(roll), math.tan(pitch))
        tilt = math.atan2(math.sqrt(math.sin(pitch) ** 2 + math.cos(pitch) ** 2 * math.sin(roll) ** 2),
                          math.cos(pitch) * math.cos(roll))
        return azimuth, tilt

    def convert_data_frame(self, df: pd.DataFrame):
        # Converts to the following columns:
        # Unix time, installation_id, string_id, Float_id, orientation_field1, ... orientation_fieldN,
        # humidityTempField1, ... humidityTempFieldM
        if df.empty:
            return df

        df.rename(columns={c: c.replace("_", "") for c in df.columns}, inplace=True)

        df.sort_values(["time", "measurement"], inplace=True)

        # Remove measurement series names, keep float id
        df['measurement'] = df['measurement'].apply(lambda x: x.split("_")[0])
        df.rename(columns={"measurement": "float_id"}, inplace=True)

        measurement_names: np.array = df.get("field").unique()
        convert_angles = self.calculate_solar_angles and all(x in measurement_names for x in ['a_x', 'a_y'])

        if convert_angles:
            measurement_names = list(measurement_names) + ['azimuth', 'tilt']
        measurement_names = self.custom_sort_columns(measurement_names)

        field_names = ["time", "installation_id", "string_id", "float_id"] + measurement_names.tolist()

        df_grouped = df.groupby(by=["time", "float_id"])

        # Initialize a numpy array to hold the transformed data
        data_types = ['i8', 'U8', 'i4', 'i4'] + np.full(shape=measurement_names.shape[0], fill_value='f8').tolist()
        dtype = list(zip(field_names, data_types))
        transformed_arr = np.full(shape=(df_grouped.ngroups,), fill_value=np.nan, dtype=dtype)

        row = 0
        for name, group in df_grouped:
            head = group.iloc[0]
            unix_time = (head['time'] - pd.Timestamp("1970-01-01", tz="UTC")) // pd.Timedelta("1ms")
            transformed_arr[row]['time'] = unix_time
            transformed_arr[row]['installation_id'] = self.float_id_to_installation_id(
                head['float_id'])
            transformed_arr[row]['string_id'] = 0
            transformed_arr[row]['float_id'] = ''.join(filter(str.isdigit, head['float_id']))

            for idx, entry in group.iterrows():
                transformed_arr[row][entry['field']] = entry['value']

            if convert_angles:
                azimuth, tilt = self.pitch_roll_to_azimuth_tilt(transformed_arr[row]['a_x'],
                                                                transformed_arr[row]['a_y'])
                transformed_arr['azimuth'] = azimuth
                transformed_arr['tilt'] = tilt

            row += 1
        return pd.DataFrame(data=transformed_arr, columns=field_names)

    # Create queries
    ######################################################
    def create_temperature_query(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        if not float_ids:
            float_ids = self.installation_id_to_float_ids[installation_id]

        measurements = list(f"{float_id}_{self.temperature_identifier}" for float_id in float_ids)
        fields = list(f"t{numb}" for numb in range(self.number_of_humidity_temperature_sensors))

        return self.create_query(installation_id, measurements, fields,
                                 start_time, end_time, window_period)

    def create_humidity_query(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        if not float_ids:
            float_ids = self.installation_id_to_float_ids[installation_id]

        measurements = list(f"{float_id}_{self.temperature_identifier}" for float_id in float_ids)
        fields = list(f"h{numb}" for numb in range(self.number_of_humidity_temperature_sensors))

        return self.create_query(installation_id, measurements, fields,
                                 start_time, end_time, window_period)

    def create_orientation_query(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        if not float_ids:
            float_ids = self.installation_id_to_float_ids[installation_id]
        measurements = list(f"{float_id}_{self.orientation_identifier}" for float_id in float_ids)

        return self.create_query(installation_id, measurements, [],
                                 start_time, end_time, window_period)

    def create_IMU_data_query(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        if not float_ids:
            float_ids = self.installation_id_to_float_ids[installation_id]

        measurements = list(f"{float_id}_{self.IMU_data_identifier}" for float_id in float_ids)

        return self.create_query(installation_id, measurements, [],
                                 start_time, end_time, window_period)

    def create_floats_query(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        if not float_ids:
            float_ids = self.installation_id_to_float_ids[installation_id]

        measurements = list(f"{float_id}_{self.IMU_data_identifier}" for float_id in float_ids)
        measurements += list(f"{float_id}_{self.orientation_identifier}" for float_id in float_ids)
        measurements += list(f"{float_id}_{self.temperature_identifier}" for float_id in float_ids)

        return self.create_query(installation_id, measurements, [],
                                 start_time, end_time, window_period)

    # Create and send queries
    ######################################################
    def send_query(self, query):
        result = super().send_query(query)
        if self.result_format == "standard":
            return self.convert_data_frame(result)

        return result

    def query_temperature(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        query = self.create_temperature_query(installation_id, float_ids, start_time, end_time, window_period)
        return self.send_query(query)

    def query_humidity(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        query = self.create_humidity_query(installation_id, float_ids, start_time, end_time, window_period)
        return self.send_query(query)

    def query_orientation(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        query = self.create_orientation_query(installation_id, float_ids, start_time, end_time, window_period)
        return self.send_query(query)

    def query_IMU_data(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        query = self.create_IMU_data_query(installation_id, float_ids, start_time, end_time, window_period)
        return self.send_query(query)

    def query_floats(self, installation_id, float_ids, start_time, end_time="", window_period=""):
        query = self.create_floats_query(installation_id, float_ids, start_time, end_time, window_period)
        return self.send_query(query)

max_stars_repo_name: Kodeworks/sunlitsea-timeseries

max_issues_repo_name: Kodeworks/sunlitsea-timeseries

max_forks_repo_name: Kodeworks/sunlitsea-timeseries


text:
"""Top-level functions for inference from causal diagrams."""

import networkx as nx
from oyster.utils.graph_utils import NA_pairs, De, Pa
from oyster.utils.set_utils import minimal_sets
from oyster.adjust import d_separated, d_separator_search
from oyster.viz.viz import draw

def implied_independencies(DAG):
    print('Implied independencies:')
    ind = {(X, Y): minimal_sets(d_separator_search(DAG, X, Y))
           for X, Y in NA_pairs(DAG)}
    for (X, Y), minsets in ind.items():
           print(f'{X} ⫫ {Y} | {minsets}')

def basis_set(DAG):
    for v in DAG.nodes:
        pa = Pa(DAG, v)
        nondesc = DAG.nodes - {v} - De(DAG, v) - pa
        if nondesc: print(f'{v} ⫫ {nondesc} | {pa}')

max_stars_repo_name: BrennanBarker/oyster

max_issues_repo_name: BrennanBarker/oyster

max_forks_repo_name: BrennanBarker/oyster

reason: import networkx


text:
"""
============
Moving Digit (Experimental - Hierarchical SFA)
============
An example of :class:`sksfa.HSFA` applied to a simple image time-series:
a one-digit version of the moving MNIST dataset. Each data point is 4096-dimensional.

.. image:: ../images/moving_mnist.gif
   :align: center

If the change in x is not significantly faster or slower than the change in y, HSFA with only two output features
successfully extracts a smooth (and possibly flipped) representation of the position of the digit in the image.

Ground truth is only added for the comparison, not during training.

(Note that a problem like this can also be solved with linear SFA. This example serves the purpose of providing an
example on how an HSFA network is initialized and how it can be directly applied to image data without flattening.)
"""

import numpy as np
from sksfa import HSFA
import matplotlib.pyplot as plt
import os

# Loading and preparing the data
# - HSFA requires a colorchannel, even for grayscale images
split_ratio = 0.7
data = np.load("data/mmnist_data.npy").squeeze()[..., None]
ground_truth = np.load("data/mmnist_positions.npy").squeeze()
n_points = data.shape[0]
split_idx = int(split_ratio * n_points)

training_data = data[:split_idx]
training_gt = ground_truth[:split_idx]
test_data = data[split_idx:]
test_gt = ground_truth[split_idx:]

# Preparing the HSFA-network:
# - each layer needs a 6-tuple for configuration
# - each 6-tuple contains (kernel_width, kernel_height, stride_width, stride_height, n_features, expansion_degree)
# The final layer will always be a full connected SFA layer
layer_configurations = [(8, 8, 8, 8, 8, 1),
                        (2, 2, 2, 2, 8, 2)]

hsfa = HSFA(n_components=2,
            input_shape=data.shape[1:],
            layer_configurations=layer_configurations,
            internal_batch_size=100,
            noise_std=0.01)

hsfa.summary()

hsfa.fit(training_data)
output = hsfa.transform(test_data)

gt_delta = np.var(test_gt[1:] - test_gt[:-1], axis=0)
gt_order = np.argsort(gt_delta)
gt_labels = ["x", "y"]

fig, ax = plt.subplots(2, 2, sharex=True)
cutoff = 60
ax[0, 0].plot(output[:cutoff, 0])
ax[1, 0].plot(output[:cutoff, 1])
ax[0, 1].plot(test_gt[:cutoff, gt_order[0]])
ax[1, 1].plot(test_gt[:cutoff, gt_order[1]])
ax[0, 0].set_title("Extracted features")
ax[0, 1].set_title("True position")

plt.tight_layout()
plt.show()

max_stars_repo_name: wiskott-lab/sklearn-sfa

max_issues_repo_name: wiskott-lab/sklearn-sfa

max_forks_repo_name: wiskott-lab/sklearn-sfa


text:
import numpy as np
from copy import deepcopy

def Rad2_FFT_DIF_Fix16(In_Data, Twiddles, N_fft):
    Data = deepcopy(In_Data)
    iLog2N = int(np.log2(N_fft))
    iL = 1
    iM = N_fft // 2
    for iCnt1 in range(iLog2N-3):
        iQ = 0
        for iCnt2 in range(0, iM):
            Wr = np.int64(Twiddles[0, iQ])
            Wi = np.int64(Twiddles[1, iQ])
            iA = iCnt2
            for iCnt3 in range(iL):
                iB = iA + iM
                Tmpr = np.int64(Data[0, iA]) - Data[0, iB]
                Tmpi = np.int64(Data[1, iA]) - Data[1, iB]
                Data[0, iA] = np.int16((np.int32(Data[0, iA]) + Data[0, iB]) >> 1)
                Data[1, iA] = np.int16((np.int32(Data[1, iA]) + Data[1, iB]) >> 1)
                Data[0, iB] = np.int16((((Tmpr*Wr) - (Tmpi*Wi)) >> 15) >> 1)
                Data[1, iB] = np.int16((((Tmpr*Wi) + (Tmpi*Wr)) >> 15) >> 1)
                iA = iA + 2 * iM
            iQ = iQ + iL
        iL *= 2
        iM //= 2

    iQ = 0
    iM = 4
    iL = int(N_fft) >> 3
    for iCnt2 in range(0, iM):
        Wr = Twiddles[0, iQ]
        Wi = Twiddles[1, iQ]
        iA = iCnt2
        for iCnt3 in range(iL):
            iB = iA + iM
            Tmpr = np.int64(Data[0, iA]) - Data[0, iB]
            Tmpi = np.int64(Data[1, iA]) - Data[1, iB]
            Data[0, iA] = np.int16((np.int32(Data[0, iA]) + Data[0, iB]))
            Data[1, iA] = np.int16((np.int32(Data[1, iA]) + Data[1, iB]))
            Data[0, iB] = np.int16((((Tmpr*Wr) - (Tmpi*Wi)) >> 15))
            Data[1, iB] = np.int16((((Tmpr*Wi) + (Tmpi*Wr)) >> 15))
            iA = iA + 2 * iM
        iQ += iL

    iQ = 0
    iM = 2
    iL = int(N_fft) >> 2
    for iCnt2 in range(0, iM):
        Wr = Twiddles[0, iQ]
        Wi = Twiddles[1, iQ]
        iA = iCnt2
        for iCnt3 in range(iL):
            iB = iA + iM
            Tmpr = np.int64(Data[0, iA]) - Data[0, iB]
            Tmpi = np.int64(Data[1, iA]) - Data[1, iB]
            Data[0, iA] = np.int16((np.int32(Data[0, iA]) + Data[0, iB]))
            Data[1, iA] = np.int16((np.int32(Data[1, iA]) + Data[1, iB]))
            Data[0, iB] = np.int16((((Tmpr*Wr) - (Tmpi*Wi)) >> 15))
            Data[1, iB] = np.int16((((Tmpr*Wi) + (Tmpi*Wr)) >> 15))
            iA = iA + 2 * iM
        iQ += iL

    iM = 1
    iL = N_fft // 2
    iA = 0
    for iCnt3 in range(iL):
        iB = iA + iM
        Tmpr = np.int32(Data[0, iA]) - Data[0, iB]
        Tmpi = np.int32(Data[1, iA]) - Data[1, iB]
        Data[0, iA] = np.int16((np.int32(Data[0, iA]) + Data[0, iB]))
        Data[1, iA] = np.int16((np.int32(Data[1, iA]) + Data[1, iB]))
        Data[0, iB] = np.int16(Tmpr)
        Data[1, iB] = np.int16(Tmpi)
        iA = iA + 2 * iM

    return Data

def Rad4_FFT_DIF_Fix16(In_Data, Twiddles, N_fft):
    Data = deepcopy(In_Data)
    iLog4N = int(np.log2(N_fft)) >> 1
    iL = 1
    iM = N_fft // 4
    for iCnt1 in range(iLog4N-2):
        iQ = 0
        for iCnt2 in range(0, iM):
            W1r = Twiddles[0, iQ]
            W1i = Twiddles[1, iQ]
            W2r = Twiddles[0, iQ*2]
            W2i = Twiddles[1, iQ*2]
            W3r = Twiddles[0, iQ*3]
            W3i = Twiddles[1, iQ*3]
            iA = iCnt2
            for iCnt3 in range(iL):
                A1r = ((np.int32(Data[0, iA]) + Data[0, iA+2*iM]) + (Data[0, iA+iM]  +  Data[0, iA+3*iM]))
                A1i = ((np.int32(Data[1, iA]) + Data[1, iA+2*iM]) + (Data[1, iA+iM]  +  Data[1, iA+3*iM]))
                B1r = ((np.int32(Data[0, iA]) - Data[0, iA+2*iM]) + (Data[1, iA+iM]  -  Data[1, iA+3*iM]))
                B1i = ((np.int32(Data[1, iA]) - Data[1, iA+2*iM]) - (Data[0, iA+iM]  -  Data[0, iA+3*iM]))
                C1r = ((np.int32(Data[0, iA]) + Data[0, iA+2*iM]) - (Data[0, iA+iM]  +  Data[0, iA+3*iM]))
                C1i = ((np.int32(Data[1, iA]) + Data[1, iA+2*iM]) - (Data[1, iA+iM]  +  Data[1, iA+3*iM]))
                D1r = ((np.int32(Data[0, iA]) - Data[0, iA+2*iM]) - (Data[1, iA+iM]  -  Data[1, iA+3*iM]))
                D1i = ((np.int32(Data[1, iA]) - Data[1, iA+2*iM]) + (Data[0, iA+iM]  -  Data[0, iA+3*iM]))

                Tmp = B1r
                B1r = np.int64(B1r*W1r - B1i*W1i) >> 15
                B1i = np.int64(Tmp*W1i + B1i*W1r) >> 15
                Tmp = C1r
                C1r = np.int64(C1r*W2r - C1i*W2i) >> 15
                C1i = np.int64(Tmp*W2i + C1i*W2r) >> 15
                Tmp = D1r
                D1r = np.int64(D1r*W3r - D1i*W3i) >> 15
                D1i = np.int64(Tmp*W3i + D1i*W3r) >> 15

                Data[0, iA     ] = np.int16(A1r >> 2)
                Data[1, iA     ] = np.int16(A1i >> 2)
                Data[0, iA+  iM] = np.int16(B1r >> 2)
                Data[1, iA+  iM] = np.int16(B1i >> 2)
                Data[0, iA+2*iM] = np.int16(C1r >> 2)
                Data[1, iA+2*iM] = np.int16(C1i >> 2)
                Data[0, iA+3*iM] = np.int16(D1r >> 2)
                Data[1, iA+3*iM] = np.int16(D1i >> 2)

                iA = iA + 4 * iM
            iQ += iL
        iL = iL * 4
        iM = iM // 4

    iQ = 0
    iM = 4
    iL = N_fft // 16
    for iCnt2 in range(0, iM):
        W1r = Twiddles[0, iQ]
        W1i = Twiddles[1, iQ]
        W2r = Twiddles[0, iQ*2]
        W2i = Twiddles[1, iQ*2]
        W3r = Twiddles[0, iQ*3]
        W3i = Twiddles[1, iQ*3]
        iA = iCnt2
        for iCnt3 in range(iL):
            A1r = ((np.int32(Data[0, iA]) + Data[0, iA+2*iM]) + (Data[0, iA+iM]  +  Data[0, iA+3*iM]))
            A1i = ((np.int32(Data[1, iA]) + Data[1, iA+2*iM]) + (Data[1, iA+iM]  +  Data[1, iA+3*iM]))
            B1r = ((np.int32(Data[0, iA]) - Data[0, iA+2*iM]) + (Data[1, iA+iM]  -  Data[1, iA+3*iM]))
            B1i = ((np.int32(Data[1, iA]) - Data[1, iA+2*iM]) - (Data[0, iA+iM]  -  Data[0, iA+3*iM]))
            C1r = ((np.int32(Data[0, iA]) + Data[0, iA+2*iM]) - (Data[0, iA+iM]  +  Data[0, iA+3*iM]))
            C1i = ((np.int32(Data[1, iA]) + Data[1, iA+2*iM]) - (Data[1, iA+iM]  +  Data[1, iA+3*iM]))
            D1r = ((np.int32(Data[0, iA]) - Data[0, iA+2*iM]) - (Data[1, iA+iM]  -  Data[1, iA+3*iM]))
            D1i = ((np.int32(Data[1, iA]) - Data[1, iA+2*iM]) + (Data[0, iA+iM]  -  Data[0, iA+3*iM]))

            Tmp = B1r
            B1r = np.int64(B1r*W1r - B1i*W1i) >> 15
            B1i = np.int64(Tmp*W1i + B1i*W1r) >> 15
            Tmp = C1r
            C1r = np.int64(C1r*W2r - C1i*W2i) >> 15
            C1i = np.int64(Tmp*W2i + C1i*W2r) >> 15
            Tmp = D1r
            D1r = np.int64(D1r*W3r - D1i*W3i) >> 15
            D1i = np.int64(Tmp*W3i + D1i*W3r) >> 15

            Data[0, iA     ] = np.int16(A1r)
            Data[1, iA     ] = np.int16(A1i)
            Data[0, iA+  iM] = np.int16(B1r)
            Data[1, iA+  iM] = np.int16(B1i)
            Data[0, iA+2*iM] = np.int16(C1r)
            Data[1, iA+2*iM] = np.int16(C1i)
            Data[0, iA+3*iM] = np.int16(D1r)
            Data[1, iA+3*iM] = np.int16(D1i)

            iA = iA + 4 * iM
        iQ += iL

    iM = 1
    iL = N_fft // 4
    iA = 0
    for iCnt3 in range(iL):
        Ar = np.int32(Data[0, iA])
        Ai = np.int32(Data[1, iA])
        Br = np.int32(Data[0, iA+iM])
        Bi = np.int32(Data[1, iA+iM])
        Cr = np.int32(Data[0, iA+2*iM])
        Ci = np.int32(Data[1, iA+2*iM])
        Dr = np.int32(Data[0, iA+3*iM])
        Di = np.int32(Data[1, iA+3*iM])

        Data[0, iA     ] = np.int16(Ar + Cr + Br + Dr)
        Data[1, iA     ] = np.int16(Ai + Ci + Bi + Di)
        Data[0, iA+  iM] = np.int16(Ar - Cr + Bi - Di)
        Data[1, iA+  iM] = np.int16(Ai - Ci - Br + Dr)
        Data[0, iA+2*iM] = np.int16(Ar + Cr - Br - Dr)
        Data[1, iA+2*iM] = np.int16(Ai + Ci - Bi - Di)
        Data[0, iA+3*iM] = np.int16(Ar - Cr - Bi + Di)
        Data[1, iA+3*iM] = np.int16(Ai - Ci + Br - Dr)

        iA = iA + 4 * iM
    return Data

def SwapSamples(In_Data, SwapTable):
    Data = deepcopy(In_Data)
    for i, swap_idx in enumerate(SwapTable):
        if i < swap_idx:
            Sr = Data[0, i]
            Si = Data[1, i]
            Data[0, i] = Data[0, swap_idx]
            Data[1, i] = Data[1, swap_idx]
            Data[0, swap_idx] = Sr
            Data[1, swap_idx] = Si
    return Data

def RFFT_Step_Fix16(In_Data, RTwiddles, N_fft):
    RFFT_Out = np.zeros((2, N_fft//2+1))

    xBR = np.int32(In_Data[0, 0]) >> 2
    xBI = np.int32(In_Data[1, 0]) >> 2
    xAR = np.int32(In_Data[0, 0]) >> 2
    xAI = np.int32(In_Data[1, 0]) >> 2
    RFFT_Out[0, 0] = (xBR + xAR + xBI + xAI)
    RFFT_Out[1, 0] = (xAI - xBI + xBR - xAR)

    k = N_fft // 2 - 1
    for i in range(k):
        twr, twi = np.int64(RTwiddles[0, 1+i]), np.int64(RTwiddles[1, 1+i])
        xar, xai = np.int32(In_Data  [0, 1+i]), np.int32(In_Data  [1, 1+i])
        xbr, xbi = np.int32(In_Data  [0, k-i]), np.int32(In_Data  [1, k-i])

        t1r = np.int32((-xar + xbr) >> 2)
        t1i = np.int32((-xai - xbi) >> 2)
        t2r = np.int32(( xar + xbr) >> 2)
        t2i = np.int32(( xai - xbi) >> 2)

        RFFT_Out[0, i+1] = np.int16(((twr*t1r - twi*t1i) >> 15) + t2r)
        RFFT_Out[1, i+1] = np.int16(((twr*t1i + twi*t1r) >> 15) + t2i)

    xBR = np.int32(In_Data[0, k-(k-1)]) >> 2
    xBI = np.int32(In_Data[1, k-(k-1)]) >> 2
    xAR = np.int32(In_Data[0, 1+(k-1)]) >> 2
    xAI = np.int32(In_Data[1, 1+(k-1)]) >> 2
    RFFT_Out[0, k+1] = np.int16( xBR + xAR - xBI - xAI )
    RFFT_Out[1, k+1] = np.int16(0) #( xAI - xBI - xBR + xAR )
    return RFFT_Out


text:
#Author: Matt Williams
#Version: 06/24/2022

from enum import Enum
import os
import pandas as pd
import numpy as np
import re

# A Simple Python File that contains constant values and methods important to the project.
# Most of these values are file path related.

class Datasets(Enum):
    TRAIN = "Train"
    TEST = "Test"

class WordVectorModels(Enum):
    FASTTEXT = "fasttext"
    GLOVE = "glove"
    WORD2VEC = "word2vec"

    @classmethod
    def get_values_as_list(cls):
        return [model.value for model in WordVectorModels]

class ClassificationModels(Enum):
    KNN = "K-Nearest Neighbor"
    SVM = "Support Vector Machine"
    RF = "Random Forest"
    LOG_REG = "Logistic Regression"
    ADA = "Ada-Boost"
    BAG = "Bagging"
    MLP = "Multi-Layer Perceptron"
    CENT = "Nearest Centroid"
    RAD = "Radius Neighbors"
    CNB = "Complement Naive Bayes"
    GRAD = "Gradient Boosted Trees"

    @classmethod
    def get_values_as_list(cls):
        return [model.value for model in ClassificationModels]

class Categories(Enum):
    MEDIA = "MEDIA"
    WEIRD = "WEIRD NEWS"
    GREEN =  "GREEN"
    POST =  "WORLDPOST"
    RELIGION = "RELIGION"
    STYLE =  "STYLE"
    SCIENCE =  "SCIENCE"
    NEWS =  "WORLD NEWS"
    TASTE = "TASTE"
    TECH =  "TECH"

    @classmethod
    def get_values_as_list(cls):
        return [cat.value for cat in Categories]

CV_BEST_DICT_KEY = "best params"

#Dataset: https://www.kaggle.com/rmisra/news-category-dataset
CWD_PATH = os.path.abspath(os.getcwd())
DATA_DIR_PATH = os.path.join(CWD_PATH, "Data")

#This file contains the original data set.
DATA_SET_PATH = os.path.join(DATA_DIR_PATH, "News_Category_Dataset_v2.json")

#This file contains the original data set sorted by article category
SORTED_DATA_PATH = os.path.join(DATA_DIR_PATH, "Sorted_Data_Set.json")

#This file contains the articles to be used for the project.
ARTICLE_SET_PATH = os.path.join(DATA_DIR_PATH, "Article_Set.json")

#This file contains some text to test text_cleaner.py on before cleaning
#all the articles.
TEST_TEXT_PATH = os.path.join(DATA_DIR_PATH, "test_text.txt")

#This file contains a dictionary where each key is a category and each value a list of strings,
# where each string is the scrapped text of an article.
SCRAPED_TEXT_PATH = os.path.join(DATA_DIR_PATH, "Scraped_Text.json")

# This file contains the cleaned text from the Scrapped Text file.
# Contains a dictionary where each key is a category and each value is a 3d list:
# dict[category][i][j][k] - kth word in jth sentence in ith article for the given category
CLEANED_TEXT_PATH = os.path.join(DATA_DIR_PATH, "Cleaned_Text.json")

TEST_SET_PERC = 0.1
TRAIN_SET_PERC = 0.9
K_FOLDS = 5

#File Path to the directory that contains our Models related to generating word embeddings.
MODEL_DIR_PATH = os.path.join(CWD_PATH, "Models")

def get_model_path(name):
    '''Given the name of a file for a vector model, return the full file path for that file to be stored at.'''
    if not os.path.exists(MODEL_DIR_PATH):
        os.mkdir(MODEL_DIR_PATH)
    return os.path.join(MODEL_DIR_PATH, name)

#File Path to the directory where the article vectors are to be stored.
ARTICLE_VECS_DIR_PATH = os.path.join(CWD_PATH, "Article_Vectors")

def get_article_vecs_path(subfolder, name):
    '''Given subfolder(Train or Test) and the name of a file, return the full file path for that file to be stored at'''
    if not os.path.exists(ARTICLE_VECS_DIR_PATH):
        os.mkdir(ARTICLE_VECS_DIR_PATH)
    res_path = os.path.join(ARTICLE_VECS_DIR_PATH, subfolder)
    return os.path.join(res_path, name)

#File Path to the directory where classification or clustering results are to be stored.
RESULTS_DIR_PATH = os.path.join(CWD_PATH, "Results")

def find_json_file_name_number(folder):
    '''Given a file path to a folder containing .json files, find the highest number associated with a filename in that
    folder, then increment that number by 1 and return it '''
    highest_num = 0
    num_pattern = re.compile(r"[0-9]+\.json")
    for file in os.listdir(folder):
        match = num_pattern.search(file).group()
        num = int(match.split(".")[0])
        if num > highest_num:
            highest_num = num

    return highest_num + 1

def make_result_path(subfolder, vec_model):
    '''Given the name of a subfolder(Name of the Classification/Clustering algorithm) and the name
        of the vector model, generate a name for the file and return the full file path for the file to be stored at.'''

    if not os.path.exists(RESULTS_DIR_PATH):
        os.mkdir(RESULTS_DIR_PATH)

    updated_path = os.path.join(RESULTS_DIR_PATH, subfolder)

    if not os.path.exists(updated_path):
        os.mkdir(updated_path)

    num = find_json_file_name_number(updated_path)
    file_name = vec_model + "_results_{}.json".format(num)
    return os.path.join(updated_path, file_name)

def get_result_path(subfolder, file_name):
    '''Get the full file path for the result with the given filename and the given subfolder.'''
    res_path = os.path.join(RESULTS_DIR_PATH, subfolder)
    return os.path.join(res_path, file_name)

#File Path to the directory where cross validation results are to be stored.
CV_RESULTS_DIR_PATH = os.path.join(CWD_PATH,"CV_Results")

def make_cv_result_path(subfolder, vec_model):
    '''Given the name of a subfolder(Name of the Classification/Clustering algorithm), generate a name
        for the file and return the full file path for the file to be stored at. '''
    if not os.path.exists(CV_RESULTS_DIR_PATH):
        os.mkdir(CV_RESULTS_DIR_PATH)

    updated_path = os.path.join(CV_RESULTS_DIR_PATH, subfolder)

    if not os.path.exists(updated_path):
        os.mkdir(updated_path)

    num = find_json_file_name_number(updated_path)
    file_name = vec_model + "_cv_results_{}.json".format(num)
    return os.path.join(updated_path, file_name)

def get_cv_result_path(subfolder, file_name):
    '''Get the full file path for the cross validation result with the given filename in the given subfolder.'''
    res_path = os.path.join(CV_RESULTS_DIR_PATH, subfolder)
    return os.path.join(res_path, file_name)

def convert_categories_to_numbers(labels):
    '''given a pandas series that contains the different categories, convert those
    categories into integers. Integers represent location in CATEGORIES list.'''

    if isinstance(labels, pd.Series):
        for i,category in enumerate(Categories.get_values_as_list()):
            labels = labels.replace(category, i)

    elif isinstance(labels, np.ndarray):
        for i,category in enumerate(Categories.get_values_as_list()):
            labels[labels == category] = i

    return labels