from __future__ import print_function

import functools
import io
import os
import re
import sys

# pylint: disable=deprecated-module
import distutils.core
from distutils.command.build import build as distutils_build
from distutils.command.install import install as distutils_install
from distutils.command.install_data import install_data as distutils_install_data
from distutils.command.sdist import sdist as distutils_sdist
# pylint: enable=deprecated-module

# pylint: disable=import-error
if sys.version_info >= (3, 0):
    import configparser
else:
    import ConfigParser as configparser
# pylint: enable=import-error

try:
    from wheel.bdist_wheel import bdist_wheel
except ImportError:
    bdist_wheel = None

try:
    import distutils644
except ImportError:
    pass
else:
    distutils644.install()

type({0})  # Python >= 2.7 is required

def uopen(*args):
    if str is bytes:
        return open(*args)  # pylint: disable=consider-using-with,unspecified-encoding
    else:
        return open(*args, encoding='UTF-8')  # pylint: disable=consider-using-with

def get_readme():
    with io.open('doc/README', encoding='ASCII') as file:
        content = file.read()
    content = re.compile('^[.][.] vim:.*', re.MULTILINE).sub('', content)
    return '\n' + content.strip() + '\n'

def get_version():
    with io.open('doc/changelog', encoding='UTF-8') as file:
        line = file.readline()
    return line.split()[1].strip('()')

class cmd_build_doc(distutils_build):

    description = 'build documentation'

    def make_tags_rst(self, data_path, rst_path):
        cp = configparser.RawConfigParser()
        options = {}
        if str is not bytes:
            options.update(encoding='UTF-8')
        cp.read(data_path, **options)
        with uopen(rst_path + '.tmp', 'w') as rst_file:
            self._make_tags_rst(cp, print=functools.partial(print, file=rst_file))
        os.rename(rst_path + '.tmp', rst_path)

    def _make_tags_rst(self, data, print):  # pylint: disable=redefined-builtin
        _strip_leading_dot = functools.partial(
            re.compile('^[.]', re.MULTILINE).sub,
            ''
        )
        def _parse_multiline(value):
            for s in value.splitlines():
                if not s or s.isspace():
                    continue
                yield _strip_leading_dot(s)
        def parse_multiline(tag, key):
            value = tag.get(key, '')
            return _parse_multiline(value)
        for tagname in data.sections():
            tag = dict(data.items(tagname))
            print(tagname)
            print('~' * len(tagname))
            description = str.join('\n',
                parse_multiline(tag, 'description')
            )
            if not description.strip():
                raise ValueError('missing description for {0}'.format(tagname))
            print(description)
            print()
            references = list(parse_multiline(tag, 'references'))
            if references:
                print('References:', end='\n\n')
                for ref in references:
                    match = re.match(r'\A(?P<name>[\w-]+)[(](?P<section>[0-9])[)]\Z', ref)
                    if match is not None:
                        ref = r'**{name}**\ ({section})'.format(**match.groupdict())
                    print(' |', ref)
                print()
            print('Severity, certainty:', end='\n\n')
            print(' {0}, {1}'.format(tag['severity'], tag['certainty']))
            print()

    def make_man(self, rst_path, man_path):
        # pylint: disable=import-outside-toplevel
        import docutils.core
        import docutils.writers.manpage
        # pylint: enable=import-outside-toplevel
        if str is bytes:
            tmp_file = io.BytesIO()
        else:
            tmp_file = io.StringIO()
        tmp_file.close = object  # prevent docutils from closing the file
        docutils.core.publish_file(
            source_path=rst_path,
            destination=tmp_file,
            writer=docutils.writers.manpage.Writer(),
            settings_overrides=dict(input_encoding='UTF-8', halt_level=1),
        )
        tmp_file.seek(0)
        with uopen(man_path, 'w') as man_file:
            for line in tmp_file:
                if line.startswith(r'.\" vim:'):
                    continue
                if line.startswith('.BI'):
                    # work-around for <https://bugs.debian.org/806601>:
                    line = line.replace(r'\fP', r'\fR')
                line = re.sub(r'(?<=[a-zA-Z])\\\(aq(?=[a-z])', "'", line)
                man_file.write(line)

    def run(self):
        data_path = 'pydiatra/data/tags'
        tags_rst_path = 'doc/tags.rst'
        man_rst_path = 'doc/manpage.rst'
        man_path = 'doc/pydiatra.1'
        self.make_file([data_path], tags_rst_path, self.make_tags_rst, [data_path, tags_rst_path])
        self.make_file([tags_rst_path, man_rst_path], man_path, self.make_man, [man_rst_path, man_path])

distutils_build.sub_commands[:0] = [('build_doc', None)]

class cmd_install_doc(distutils_install_data):

    description = 'install documentation'

    def run(self):
        man_dir = os.path.join(self.install_dir, 'share/man/man1')
        self.mkpath(man_dir)
        path = '{dir}/{prog}.1'.format(dir=man_dir, prog=script_name)
        msg = 'writing {path}'.format(path=path)
        data = ['.so pydiatra.1']
        self.execute(distutils.file_util.write_file, (path, data), msg)
        self.outfiles += [path]  # pylint: disable=no-member
        (path, _) = self.copy_file('doc/pydiatra.1', man_dir)
        self.outfiles += [path]  # pylint: disable=no-member

distutils_install.sub_commands[:0] = [('install_doc', None)]

class cmd_sdist(distutils_sdist):

    def run(self):
        self.run_command('build_doc')
        return distutils_sdist.run(self)

    def maybe_move_file(self, base_dir, src, dst):
        src = os.path.join(base_dir, src)
        dst = os.path.join(base_dir, dst)
        if os.path.exists(src):
            self.move_file(src, dst)

    def make_release_tree(self, base_dir, files):
        distutils_sdist.make_release_tree(self, base_dir, files)
        self.maybe_move_file(base_dir, 'LICENSE', 'doc/LICENSE')
        # distutils doesn't seem to handle symlinks-to-directories
        # out of the box, so let's take care of them manually:
        target = os.readlink('data')
        dest = os.path.join(base_dir, 'data')
        distutils.log.info('linking %s -> %s', dest, target)
        if not self.dry_run:
            os.symlink(target, dest)

classifiers = '''
Development Status :: 4 - Beta
Environment :: Console
Intended Audience :: Developers
License :: OSI Approved :: MIT License
Operating System :: OS Independent
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 3
Topic :: Software Development :: Quality Assurance
'''.strip().splitlines()

pkg_name = 'pydiatra'
script_name = 'py{0}diatra'.format(*sys.version_info)

def d(**kwargs):
    return {
        k: v for k, v in kwargs.items()
        if v is not None
    }

setup_options = dict(
    name='pydiatra',
    version=get_version(),
    license='MIT',
    description='yet another static checker for Python code',
    long_description=get_readme(),
    classifiers=classifiers,
    url='https://jwilk.net/software/pydiatra',
    author='Jakub Wilk',
    author_email='jwilk@jwilk.net',
    packages=[pkg_name],
    package_dir={pkg_name: 'pydiatra'},
    package_data={pkg_name: ['data/*']},
    py_modules=['pydiatra'],
    scripts=[script_name],
    cmdclass=d(
        bdist_wheel=bdist_wheel,
        build_doc=cmd_build_doc,
        install_doc=cmd_install_doc,
        sdist=cmd_sdist,
    ),
)

if __name__ == '__main__':
    distutils.core.setup(**setup_options)

# vim:ts=4 sts=4 sw=4 et


"""
Written by KP_Hong <kphong16@daum.net>
2021, KP_Hong

INDEX

Modules
-------
Index : Create and manage an array of dates
PrjtIndex : Create and manage index objects

Methods
-------
booleanloc : Return boolean array of data which is in array
"""

import pandas as pd
import numpy as np

#from pandas import (
#    date_range,
#    DatetimeIndex,
#    PeriodIndex,
    #RangeIndex,
#)

from datetime import (
    date,
    timedelta,
)
from dateutil.relativedelta import relativedelta

#from .genfunc import is_iterable
__all__ = ['RangeIndex', 'DateIndex', 'date_range', 'Index'] #, 'booleanloc', 'PrjtIndex']

def is_scalar(value):
    if isinstance(value, int):
        return True
    else:
        return False

def str_to_date(value):
    # date
    if isinstance(value, date):
        return value

    # date('-')
    value_lst = value.split('-')
    if len(value_lst) == 3:
        int_lst = [int(x) for x in value_lst]
        return date(*int_lst)
    elif len(value_lst) == 2:
        int_lst = [int(x) for x in value_lst]
        return date_monthend(*int_lst)

    # date('.')
    value_lst = value.split('.')
    if len(value_lst) == 3:
        int_lst = [int(x) for x in value_lst]
        return date(*int_lst)
    elif len(value_lst) == 2:
        int_lst = [int(x) for x in value_lst]
        return date_monthend(*int_lst)

    raise TypeError("Type of the data is not suitable.")

def date_monthend(year, month=None):
    if isinstance(year, date):
        dt = year
        return date_monthend(dt.year, dt.month)

    if isinstance(year, str):
        dt = year
        return date_monthend(str_to_date(dt))

    if month in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]:
        month_next = month + 1
        dt_next = date(year, month_next, 1)
        dt_monthend = dt_next - timedelta(days=1)
    elif month == 12:
        year_next = year + 1
        dt_next = date(year_next, 1, 1)
        dt_monthend = dt_next - timedelta(days=1)
    return dt_monthend

def date_next(dt, num=1, freq='M'):
    if isinstance(dt, str):
        dt = str_to_date(dt)
    assert isinstance(dt, date)
    if freq=='M':
        month_next = dt.month + num - 1
        year_next, month_next = divmod(month_next, 12)
        month_next += 1
        year_next = dt.year + year_next
        dt_next = date_monthend(year_next, month_next)
        return dt_next
    if freq=='Y':
        dt_next = dt + relativedelta(years=num)
        return dt_next
    if freq=='D':
        dt_next = dt + timedelta(days=num)
        return dt_next

def is_samemonth(*args):
    if len(args) < 2:
        raise ValueError("Number of values are not enough.")
    year_crit = args[0].year
    month_crit = args[0].month

    for val in args[1:len(args)]:
        if val.year != year_crit:
            return False
        if val.month != month_crit:
            return False
    return True

def all_none(*args):
    tmp_lst = [x==None for x in args]
    return all(tmp_lst)

def any_none(*args):
    tmp_lst = [x==None for x in args]
    return any(tmp_lst)

def allnot_none(*args):
    tmp_lst = [x!=None for x in args]
    return all(tmp_lst)

def anynot_none(*args):
    tmp_lst = [x!=None for x in args]
    return any(tmp_lst)

class RangeIndex():
    """
    Immutable sequence used for indexing a monotonic integer range.

    Parameters
    ----------
    start : int(default:0), range, RangeIndex instance
        If 'start' is an int and 'stop' is not given, interpreted as 'stop' instead.
    stop : int(default:0)
    step : int(default:1)
    name : str(default None)
        Name of the index.

    Attributes
    ----------
    name : The name of index
    len : The length of the index
    data : Raw data of index
    values : Raw data of index

    Methods
    -------
    copy : Deep copy of the index

    Examples
    --------
    Input one integer.
    >>> idxa = RangeIndex(10, name='IdxA')
    >>> idxa
        RangeIndex(range(0, 10))
    >>> idxa.name
        'IdxA'
    >>> len(idxa)
        10
    >>> idxa.len
        10
    >>> idxa.copy(name='NewIdxA')
        RangeIndex(range(0, 10))
    Input a RangeIndex.
    >>> RangeIndex(idxa)
        RangeIndex(range(0, 10))
    Input a range.
    >>> RangeIndex(range(0, 20))
        RangeIndex(range(0, 20))
    Input two integer(start, stop).
    >>> RangeIndex(10, 20)
        RangeIndex(range(10, 20))
    """
    def __new__(
        cls,
        start=None,
        stop=None,
        step=None,
        name=None,
        ):

        # RangeIndex, range
        if isinstance(start, RangeIndex):
            if name is None:
                name = start.name
            return start.copy(name=name)
        elif isinstance(start, range):
            return cls._simple_new(start, name)

        # validate the arguments
        if all_none(start, stop, step):
            raise TypeError("RangeIndex(...) must be called with integers.")

        if start is not None:
            start = cls._ensure_int(start)
        else:
            start = 0

        if stop is None:
            (start, stop) = (0, start)
        else:
            stop = cls._ensure_int(stop)

        step = cls._ensure_int(step) if step is not None else 1
        if step == 0:
            raise ValueError("Step must not be zero.")

        rng = range(start, stop, step)

        return cls._simple_new(rng, name=name)

    @classmethod
    def _simple_new(cls, values: range, name=None):
        result = object.__new__(cls)

        assert isinstance(values, (range, RangeIndex))

        result._data = values
        result._range = values
        result._name = name
        result._cache = {}
        return result

    @classmethod
    def _ensure_int(cls, value):
        new_value = int(value)
        assert new_value == value
        return new_value

    def copy(self, name=None):
        try:
            casted = self._data
        except (TypeError, ValueError) as err:
            raise TypeError(
                f"Cannot cast {type(self).__name__}"
                ) from err
        if name is None:
            name = self.name
        return RangeIndex(casted, name=name)

    def __len__(self) -> int:
        """
        Return the length of the Index.
        """
        return len(self._data)

    def __setitem__(self, key, value):
        raise TypeError("Index does not support mutable operations")

    def __getitem__(self, key):
        getitem = self._data.__getitem__

        if is_scalar(key):
            return getitem(key)

        if isinstance(key, slice):
            result = getitem(key)
            return type(self)._simple_new(result, name=self.name)

    def __repr__(self):
        """
        Return a string representation for this object.
        """
        klass_name = type(self).__name__
        data = str(self._data)
        return f"{klass_name}({data})"

    @property
    def values(self):
        return self._data

    @property
    def data(self):
        return self._data

    @property
    def name(self):
        return self._name
    @name.setter
    def name(self, value):
        self._name = value

    @property
    def len(self):
        return self.__len__()

class DateIndex():
    """
    Immutable sequence used for indexing date.

    Parameters
    ----------
    data : date string, datetime.date, list(...)
    name : str(default None)
        Name of the index.

    Attributes
    ----------
    name : The name of index
    len : The length of the index
    data : Raw data of index
    values : Raw data of index

    Methods
    -------
    copy : Deep copy of the index

    Examples
    --------
    Input one date string.
    >>> DateIndex('2022.01')
        DateIndex(['2022.01.31'])
    >>> DateIndex("2022-3-27")
        DateIndex(['2022.03.27'])
    Input one date type data.
    >>> DateIndex(datetime.date(2022, 1, 1))
        DateIndex(['2022.01.01'])
    Input a list of date string.
    >>> DateIndex(['2022.01', '2022.02', '2022.03'])
        DateIndex(['2022.01.31', '2022.02.28', '2022.03.31'])
    Input a list of date type data.
    >>> DateIndex([date(2022, 1, 31), date(2022, 2, 28)])
        DateIndex(['2022.01.31', '2022.02.28'])
    """
    def __new__(
        cls,
        data=None,
        name=None,
        ):

        # DateIndex
        if isinstance(data, DateIndex):
            dtarr = data.copy(name=name)

        # string
        elif isinstance(data, str):
            dtarr = [str_to_date(data)]

        # date
        elif isinstance(data, date):
            dtarr = [data]

        # list
        elif isinstance(data, list):
            new_lst = []
            for val in data:
                if isinstance(val, date):
                    new_lst.append(val)
                elif isinstance(val, str):
                    new_lst.append(str_to_date(val))
            dtarr = new_lst
        else:
            raise TypeError("There is a problem with the entered data type.")

        subarr = cls._simple_new(dtarr, name=name)
        return subarr

    @classmethod
    def _simple_new(cls, values: list, name=None):
        result = object.__new__(cls)

        assert isinstance(values, (list, DateIndex))

        result._data = values
        result._name = name
        result._cache = {}
        return result

    def copy(self, name=None):
        try:
            casted = self._data
        except (TypeError, ValueError) as err:
            raise TypeError(
                f"Cannot cast {type(self).__name__}"
                ) from err
        if name is None:
            name = self.name
        return DateIndex(casted, name=name)

    def __len__(self) -> int:
        """
        Return the length of the Index.
        """
        return len(self._data)

    def __setitem__(self, key, value):
        raise TypeError("Index does not support mutable operations")

    def __getitem__(self, key):
        getitem = self._data.__getitem__

        if is_scalar(key):
            return getitem(key)

        if isinstance(key, slice):
            result = getitem(key)
            return type(self)._simple_new(result, name=self.name)

    def __repr__(self):
        """
        Return a string representation for this object.
        """
        klass_name = type(self).__name__
        data = str([val.strftime("%Y.%m.%d") for val in self._data])
        return f"{klass_name}({data})"

    @property
    def values(self):
        return self._data

    @property
    def data(self):
        return self._data

    @property
    def name(self):
        return self._name
    @name.setter
    def name(self, value):
        self._name = value

    @property
    def len(self):
        return self.__len__()

def date_range(
    start=None,
    stop=None,
    periods=None,
    freq=None,
    name=None,
    ) -> DateIndex:
    """
    Return a fixed frequency DateIndex.

    Parameters
    ----------
    start : str or datetime.date
        Left bound for generating dates.
    stop : str or datetime.date
        Right bound for generating dates.
    periods : int(>0), optional
        Number of periods to generate.
    freq : str or DateOffset, default 'M'

    Returns
    -------
    DateIndex

    Attributes
    ----------
    name : The name of index
    len : The length of the index
    data : Raw data of index
    values : Raw data of index

    Methods
    -------
    copy : Deep copy of the index

    Examples
    --------
    Input a start date and periods.
    >>> date_range("2022.01", 3)
        DateIndex(['2022.01.31', '2022.02.28', '2022.03.31'])
    >>> date_range(date(2022, 1, 1), 3)
        DateIndex(['2022.01.01', '2022.02.28', '2022.03.31'])
    >>> idx = date_range("2022.01", 3)
    >>> idx.len
        3
    >>> idx.data
        [datetime.date(2022, 1, 1), datetime.date(2022, 2, 28), datetime.date(2022, 3, 31)]
    >>> idx.values
        [datetime.date(2022, 1, 1), datetime.date(2022, 2, 28), datetime.date(2022, 3, 31)]
    Input an stop date and periods.
    >>> date_range(stop='2022.12', periods=3)
        DateIndex(['2022.10.31', '2022.11.30', '2022.12.31'])
    Input a start date and an stop date.
    >>> date_range('2022.01', '2022.03')
        DateIndex(['2022.01.31', '2022.02.28', '2022.03.31'])
    """

    if freq is None and any_none(periods, start, stop):
        freq = "M"

    if allnot_none(start, stop, periods):
        raise ValueError("There are too many data.")
    elif allnot_none(start, periods):
        dtarr = []
        if periods > 0:
            start = str_to_date(start)
            dtarr.append(start)
            dt_next = start
        else:
            raise ValueError(
                "'periods' should be larger than 0."
            )
        for no in range(1, periods):
            dt_next = date_next(dt_next, freq=freq)
            dtarr.append(dt_next)
    elif allnot_none(stop, periods):
        stop = str_to_date(stop)
        start = stop - relativedelta(months=(periods-1))
        return date_range(start, periods=periods)
    elif allnot_none(start, stop):
        if isinstance(stop, int):
            periods = stop
            return date_range(start, periods=periods)
        else:
            dtarr = []
            start = str_to_date(start)
            stop = str_to_date(stop)
            if stop == start:
                dtarr.append(start)
            elif stop > start:
                dtarr.append(start)
                dt_next = start
                while True:
                    if dt_next >= stop:
                        break
                    dt_next = date_next(dt_next, freq=freq)
                    dtarr.append(dt_next)
            elif stop < start:
                raise ValueError("The stop date is before the start date.")

    subarr = DateIndex._simple_new(dtarr, name=name)
    return subarr

class Index():
    """
    Immutable sequence used for indexing.

    Parameters
    ----------
    start : int, str or datetime.date, list(date)
    stop : int, str or datetime.date
    step : int
    periods : int(>0), optional
        Number of periods to generate.
    freq : str or DateOffset, default 'M'
    name : name of index

    Returns
    -------
    RangeIndex or DateIndex

    Attributes
    ----------
    name : The name of index
    len : The length of the index
    data : Raw data of index
    values : Raw data of index

    Methods
    -------
    copy : Deep copy of the index

    Examples
    --------
    Input a start date and periods.
    >>> Index("2022.01", 3)
        DateIndex(['2022.01.31', '2022.02.28', '2022.03.31'])
    >>> Index(date(2022, 1, 1), 3)
        DateIndex(['2022.01.01', '2022.02.28', '2022.03.31'])
    >>> idx = Index("2022.01", 3)
    >>> idx.len
        3
    >>> idx.data
        [datetime.date(2022, 1, 1), datetime.date(2022, 2, 28), datetime.date(2022, 3, 31)]
    >>> idx.values
        [datetime.date(2022, 1, 1), datetime.date(2022, 2, 28), datetime.date(2022, 3, 31)]
    Input an stop date and periods.
    >>> Index(stop='2022.12', periods=3)
        DateIndex(['2022.10.31', '2022.11.30', '2022.12.31'])
    Input a start date and an stop date.
    >>> Index('2022.01', '2022.03')
        DateIndex(['2022.01.31', '2022.02.28', '2022.03.31'])
    Input one date string.
    >>> Index('2022.01')
        DateIndex(['2022.01.31'])
    >>> Index("2022-3-27")
        DateIndex(['2022.03.27'])
    Input one date type data.
    >>> Index(datetime.date(2022, 1, 1))
        DateIndex(['2022.01.01'])
    Input a list of date string.
    >>> Index(['2022.01', '2022.02', '2022.03'])
        DateIndex(['2022.01.31', '2022.02.28', '2022.03.31'])
    Input a list of date type data.
    >>> Index([date(2022, 1, 31), date(2022, 2, 28)])
        DateIndex(['2022.01.31', '2022.02.28'])
    Input one integer.
    >>> Index(10, name='IdxA')
        RangeIndex(range(0, 10))
    >>> idxa.name
        'IdxA'
    >>> idxa.copy(name='NewIdxA')
        RangeIndex(range(0, 10))
    Input a RangeIndex.
    >>> Index(idxa)
        RangeIndex(range(0, 10))
    Input a range.
    >>> Index(range(0, 20))
        RangeIndex(range(0, 20))
    Input two integer(start, stop).
    >>> Index(10, 20)
        RangeIndex(range(10, 20))
    """
    def __new__(
        cls,
        start=None,
        stop=None,
        step=None,
        periods=None,
        freq=None,
        name=None,
        ):

        # RangeIndex
        if isinstance(start, RangeIndex):
            if name is None:
                name = start.name
            return RangeIndex(start, name=name)
        if isinstance(start, range):
            return RangeIndex(start, name=name)
        if isinstance(start, int):
            return RangeIndex(start, stop, step, name)
        if start is None:
            if isinstance(stop, int):
                return RangeIndex(start, stop, step, name)

        # DateIndex
        if isinstance(start, DateIndex):
            if name is None:
                name = start.name
            return DateIndex(start, name=name)
        if isinstance(start, str):
            if all_none(stop, step, periods):
                return DateIndex(start, name=name)
        if isinstance(start, date):
            if all_none(stop, step, periods):
                return DateIndex(start, name=name)
        if isinstance(start, list):
            if all_none(stop, step, periods):
                return DateIndex(start, name=name)

        # date_range
        if isinstance(start, (str, date)):
            if isinstance(stop, (int, str, date)):
                return date_range(start, stop, periods=periods, freq=freq, name=name)
        if start is None:
            if isinstance(stop, (str, date)):
                return date_range(start, stop, periods=periods, freq=freq, name=name)


"""empty message

Revision ID: 19cf818b3afc
Revises: 2a38af3ad8c0
Create Date: 2022-02-10 19:35:15.487454

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '19cf818b3afc'
down_revision = '2a38af3ad8c0'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('tag_post_map', 'tag_id',
               existing_type=mysql.CHAR(length=32),
               nullable=True)
    op.alter_column('tag_post_map', 'post_id',
               existing_type=mysql.CHAR(length=32),
               nullable=True)
    op.drop_constraint('tag_post_map_ibfk_2', 'tag_post_map', type_='foreignkey')
    op.drop_constraint('tag_post_map_ibfk_1', 'tag_post_map', type_='foreignkey')
    op.create_foreign_key(None, 'tag_post_map', 'posts', ['post_id'], ['id'], ondelete='CASCADE')
    op.create_foreign_key(None, 'tag_post_map', 'tags', ['tag_id'], ['id'], ondelete='CASCADE')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'tag_post_map', type_='foreignkey')
    op.drop_constraint(None, 'tag_post_map', type_='foreignkey')
    op.create_foreign_key('tag_post_map_ibfk_1', 'tag_post_map', 'posts', ['post_id'], ['id'])
    op.create_foreign_key('tag_post_map_ibfk_2', 'tag_post_map', 'tags', ['tag_id'], ['id'])
    op.alter_column('tag_post_map', 'post_id',
               existing_type=mysql.CHAR(length=32),
               nullable=False)
    op.alter_column('tag_post_map', 'tag_id',
               existing_type=mysql.CHAR(length=32),
               nullable=False)
    # ### end Alembic commands ###


import json
import tensorflow as tf
import tensorflow_datasets as tfds
import tensorflow_hub as hub

import warnings
warnings.filterwarnings('ignore')
import logging
logger = tf.get_logger()
logger.setLevel(logging.ERROR)

import numpy as np
import matplotlib.pyplot as plt
import json
import glob
from PIL import Image

from tensorflow.keras import layers

import argparse

parser = argparse.ArgumentParser(description='Image Classifier - Prediction Part')
parser.add_argument('--input', default='./test_images/hard-leaved_pocket_orchid.jpg', action="store", type = str, help='image path')
parser.add_argument('--model', default='./modell.h5', action="store", type = str, help='checkpoint file path/name')
parser.add_argument('--top_k', default=5, dest="top_k", action="store", type=int, help='return top K most likely classes')
parser.add_argument('--category_names', dest="category_names", action="store", default='label_map.json', help='mapping the categories to real names')

arg_parser = parser.parse_args()

image_path = arg_parser.input
model_path = arg_parser.model
topk = arg_parser.top_k
category_names = arg_parser.category_names

def process_image(image):
    image = tf.convert_to_tensor(image, dtype=tf.float32)
    image = tf.image.resize(image, (224, 224))
    image /= 255
    image=image.numpy()
    return image

def predict(image_path, model, top_k):
    img = Image.open(image_path)
    test_img = np.asarray(img)
    trandform_img = process_image(test_img)
    redim_img = np.expand_dims(trandform_img, axis=0)
    prob_pred = model.predict(redim_img)
    prob_pred = prob_pred.tolist()

    probs, classes = tf.math.top_k(prob_pred, k=top_k)
    probs=probs.numpy().tolist()[0]
    classes=classes.numpy().tolist()[0]
    return probs,classes

if __name__== "__main__":

    print ("start Prediction ...")
    with open(category_names, 'r') as f:
        class_names = json.load(f)

    reloaded_model = tf.keras.models.load_model(model_path,custom_objects={'KerasLayer':hub.KerasLayer})
    probs, classes = predict(image_path, reloaded_model, topk)
    label_names = [class_names[str(int(idd)+1)] for idd in classes]
    print(probs)
    print(classes)
    print(label_names)
print ("End Prediction ")


from collections import Counter
def solution(a, b, c, d):
    answer = 0
    cnt = Counter([a, b, c, d]).most_common()
    l = len(cnt)
    if l == 1:
        return 1111 * cnt[0][0]

    if l == 2:
        p, q = cnt[0][0], cnt[1][0]
        if cnt[0][1] == 2:
            return (p + q) * abs(q - p)
        else:
            return (10 * p + q) ** 2

    if l == 3:
        p, q, r = cnt[0][0], cnt[1][0], cnt[2][0]
        return q * r

    return min([a, b, c, d])


# General helping functions used throughout the project
import os
import shutil
import subprocess
import sys
from datetime import datetime
from pprint import pprint

import pypandoc
import validators
import yaml

import main

class Color:
    PURPLE = '\033[95m'
    CYAN = '\033[96m'
    DARKCYAN = '\033[36m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    END = '\033[0m'

def copy_file(source_filepath, dest_path):
    """Copies file from source path to destination.
    Replaces the file if it already exists
    """

    if os.path.isfile(dest_path):
        print(Color.RED, f"Warning: File with same name already exists at destination: {source_filepath}", Color.END)
    else:
        try:
            print(f"\nCopying: {source_filepath}")
            shutil.copy2(source_filepath, dest_path)
            print(f"Copied Successfully")
        except PermissionError:
            print(Color.RED, "Error: Permission denied.", Color.END)
            sys.exit(1)
        except:
            print(Color.RED, "Error: Unable to copy file.", Color.END)
            sys.exit(1)

def copy_dir_files(source_folder_path, dest_folder_path):
    """Copies all files from source directory to destination directory
    Replaces the files if they already exist
    """

    try:
        if os.path.isdir(source_folder_path):
            files = os.listdir(source_folder_path)
            for fname in files:
                if os.path.isfile(os.path.join(dest_folder_path, fname)):
                    print(Color.RED,
                          f"File with same name already exists at destination: {os.path.join(source_folder_path, fname)}",
                          Color.END)
                else:
                    copy_file(os.path.join(source_folder_path, fname), dest_folder_path)
        else:
            print(Color.RED, f"Warning: Source directory does not exist: {source_folder_path}", Color.END)
    except NotADirectoryError:
        print(Color.RED, f"Error: Source path is not a directory :{source_folder_path}", Color.END)
        sys.exit(1)
    except:
        print(Color.RED, f"Error: Unable to list files in :{source_folder_path}", Color.END)
        sys.exit(1)

def convert_md2tex(md_filename, latex_filename):
    """Converts specified markdown file to LaTeX"""

    print(f"Converting {md_filename} file to LaTeX")
    output = pypandoc.convert_file(md_filename, 'latex', outputfile=latex_filename, extra_args=['-f', 'gfm'])
    assert output == ""
    print(f"Created successfully: {latex_filename}")

def clean_directory(folder_path):
    """Deletes the specified directory"""

    try:
        if os.path.isdir(folder_path):
            print(f"\nCleaning directory: {folder_path}")
            shutil.rmtree(folder_path)
            print("Directory cleaned Successfully")
    except:
        print(Color.RED, f"\nWarning: Unable to clean directory: {folder_path}", Color.END)

def load_yaml(file_path):
    """Loads data from given YAML file in the form of dictionary"""

    try:
        with open(file_path) as stream:
            data = yaml.safe_load(stream)
            pprint(data, sort_dicts=False)
            return data
    except yaml.YAMLError as exc:
        print(exc)
        print(Color.RED, "Error: Unable to load data from YAML file.", Color.RED)
        sys.exit(1)

def decrease_level(metric_path):
    """Replaces '# ' with '### ' anywhere in the file"""

    try:
        print(f"\nDecreasing heading levels by 2 in metric: {metric_path}")
        cmd = 'sed -i "s/^\#/###/g" ' + metric_path
        os.system(cmd)
    except:
        print(Color.RED, f"Error: Unable to decrease heading levels in metric: {metric_path}.")
        print("Make sure the metric follows the template.", Color.END)
        sys.exit(1)

    """Convert absolute to relative links for every image (if any) in the metric"""
    try:
        print(f"\nConverting absolute to relative links in metric: {metric_path}")
        subprocess.check_call(["sed","-i",r"s|\(!\[[^]]\+](\)[^)]\+/\([^)]\+/[^)]\+)\)|\1\2|",metric_path])
    except:
        pprint(Color.RED, f"Error: Unable to convert abs to relative in metric: {metric_path}.")
        print("Make sure the metric follows the template.", Color.END)
        sys.exit(1)

def delete_dictkey(key, dictionary):
    """Deletes given key from given dictionary"""

    if key in dictionary:
        del dictionary[key]
    else:
        print(Color.RED, f"Warning: Key- {key} not found in {dictionary}", Color.RED)

def is_url(string):
    """Checks if the given string is a valid URL"""

    if validators.url(string):
        return True
    else:
        return False

def clone_repo(url, name, branch):
    """Clones repository and checkout the given branch"""

    try:
        subprocess.check_call(['git', 'clone', '-b', branch, url, name])
    except subprocess.CalledProcessError:
        print(Color.RED, f"Error: Repository with name: {name} already exists")
        print("Please ensure only one link is present in the YAML file")

    except:
        print(Color.RED, f"Error: Unable to clone/checkout repository from {url}")
        print("Verify the repository details specified in YAML file.", Color.END)
        sys.exit(1)

def add_front_matter(yaml_data):
    """Creates the latex file to add content just after
    the table of contents in PDF
    """

    # Create and include front matter files
    with open("front-matter.tex", "w") as front_matter:
        if yaml_data["front-matter"] is not None:
            for page in yaml_data["front-matter"]:
                if is_url(page):
                    print(f"\nDownloading file: {page}")
                    os.system(f"wget {page}")
                    filename = os.path.basename(page)
                    name, extension = os.path.splitext(filename)
                    if extension == ".md":
                        convert_md2tex(filename, name + ".tex")
                        front_matter.write(f"\input{{{name}}} \n")
                    elif extension == ".tex":
                        front_matter.write(f"\input{{{name}}} \n")
                    else:
                        print(Color.RED, f"Error: Could not incorporate {page} in front-matter.")
                        print("Please make sure that the URL is valid. Only Markdown/LaTeX file format is supported.", Color.END)
                        sys.exit(1)
                elif os.path.splitext(page)[1] == ".md":
                    convert_md2tex(page, os.path.splitext(page)[0] + ".tex")
                    front_matter.write(f"\input{{{os.path.splitext(page)[0]}}} \n")
                elif os.path.splitext(page)[1] == ".tex":
                    front_matter.write(f"\input{{{os.path.splitext(page)[0]}}}" + "\n")
                else:
                    print(Color.RED, f"Error: Could not incorporate {page} in front-matter.")
                    print("Please make sure that the filename is valid. Only Markdown/LaTeX file format is supported.",Color.END)
                    sys.exit(1)
        else:
            print(Color.RED, "Warning: No documents detected for the front-matter", Color.END)

    with open(main.master_file_path, "a") as master_file:
        master_file.write("\n\include{front-matter}")

def add_end_matter(yaml_data):
    """Creates latex file to add content at the end of PDF"""

    # Create and include end matter files
    with open("end-matter.tex", "w") as end_matter:
        if yaml_data["end-matter"] is not None:
            for page in yaml_data["end-matter"]:
                if is_url(page):
                    print(f"\nDownloading file: {page}")
                    os.system(f"wget {page}")
                    filename = os.path.basename(page)
                    name, extension = os.path.splitext(filename)
                    if name == "LICENSE":
                        os.rename("LICENSE", "LICENSE.md")
                        convert_md2tex("LICENSE.md", "LICENSE.tex")
                        end_matter.write("\clearpage\n\section{LICENSE}\n\input{LICENSE}\n")
                    elif extension == ".md":
                        convert_md2tex(filename, name + ".tex")
                        end_matter.write(f"\input{{{name}}} \n")
                    elif extension == ".tex":
                        end_matter.write(f"\input{{{name}}} \n")
                    else:
                        print(Color.RED,
                              f"Error: Could not incorporate {page} in end matter.")
                        print("Please make sure that the URL is valid. Only Markdown/LaTeX file format is supported.",
                              Color.END)
                        sys.exit(1)

                elif os.path.splitext(page)[1] == ".md":
                    convert_md2tex(page, os.path.splitext(page)[0] + ".tex")
                    end_matter.write(f"\input{{{os.path.splitext(page)[0]}}} \n")
                elif os.path.splitext(page)[1] == ".tex":
                    end_matter.write(f"\input{{{os.path.splitext(page)[0]}}}" + "\n")
                elif os.path.splitext(page)[0] == "LICENSE" and os.path.splitext(page)[1] == "":
                    os.rename("LICENSE", "LICENSE.md")
                    convert_md2tex("LICENSE.md", "LICENSE.tex")
                    end_matter.write("\clearpage\n\section{LICENSE}\n\input{LICENSE}\n")
                else:
                    print(Color.RED, f"Error: Could not incorporate {page} in end matter.")
                    print(f"Please make sure that the filename is valid. Only Markdown/LaTeX file format is supported.", Color.END)

                    sys.exit(1)

        else:
            print(Color.RED, "Warning: No documents detected for the end-matter", Color.END)

def spilt_by_colon(string):
    """Splits the string in two using priority list of colon delimiters.
    Returns only the latter half
    """

    colon_list = [":**", "：**", ":", "：", "ː", "˸", "᠄", "⍠", "꞉", "︓", " "]
    i = 0
    while (i < len(colon_list)):
        try:
            a, b = string.split(colon_list[i], maxsplit=1)
            return b.strip()
        except ValueError:
            i += 1
        except:
            print(Color.RED, f"Error: Unexpected error while extracting data from string - {string}", Color.END)
            break
    print(Color.RED,
          "Error: No colon delimiter found. Please make sure that metric/focus_area README follow the template.",
          Color.END)
    sys.exit(1)

def extract_question(metric):
    """Extracts the name and question from the given metric"""

    with open(metric) as f:
        data = f.readlines()
    data = [x.strip() for x in data]

    # filter out empty strings
    data = list(filter(None, data))

    # data[0] = '# Technical Fork'
    metric_name = data[0].split(maxsplit=1)[1]

    # data[1] = 'Question: question part of the metric'
    metric_question = spilt_by_colon(data[1])

    # Replace '&' to 'and' to prevent breaking of tables in pandoc
    metric_name = metric_name.replace('&', 'and')
    metric_question = metric_question.replace('&', 'and')

    return metric_name, metric_question

def extract_goal(focus_area_README):
    """Extracts the name and goal from given focus area README"""

    with open(focus_area_README) as f:
        data = f.readlines()
    data = [x.strip() for x in data]

    # filter out empty strings
    data = list(filter(None, data))

    focus_area_name = data[0].split(maxsplit=1)[1]
    focus_area_goal = spilt_by_colon(data[1])

    # Replace '&' to 'and'
    focus_area_name = focus_area_name.replace('&', 'and')
    focus_area_goal = focus_area_goal.replace('&', 'and')

    return focus_area_name, focus_area_goal

def read_file(filename):
    """Returns data from given file in the form of a string"""

    with open(filename, "r") as f:
        return f.read()

def replace_metric_table_keywords(table_head, focus_area_README, word_translation_yaml_data, language):
    """Replaces specific keywords from the metric table templates.
     Also adds focus area name and goal to the file
     """

    focus_area_name, focus_area_goal = extract_goal(focus_area_README)

    keywords_dict = {
        "$FOCUS_AREA_NAME$": focus_area_name,
        "$FOCUS_AREA_GOAL$": focus_area_goal,
        "$FOCUS_AREA$": word_translation_yaml_data[language]["focus-area"],
        "$GOAL$": word_translation_yaml_data[language]["goal"],
        "$METRIC$": word_translation_yaml_data[language]["metric"],
        "$QUESTION$": word_translation_yaml_data[language]["question"]
    }

    for k, v in keywords_dict.items():
        table_head = table_head.replace(k, v)

    return table_head

def generate_metric_table(table_head, table_tail, focus_area_filename, metric_list):
    """Creates the metric tables for a particular focus area """

    for metric in metric_list:
        metric_name, metric_question = extract_question(metric)
        table_head += '\t\t' + metric_name + ' & ' + metric_question + ' \\\\ \n\t\t\hline\n'

    table_head += table_tail

    with open(focus_area_filename, 'w') as f:
        f.write(table_head)

    print(f"\nGenerating focus-area file = {focus_area_filename}")

def replace_fa_table_keywords(table_head, section_name, word_translation_yaml_data, language):
    """Replaces specific keywords from the focus area table templates.
     Also adds WG heading as section_name
     """

    table_head = table_head.replace("$SECTION_NAME$", section_name)
    table_head = table_head.replace("$FOCUS_AREA$", word_translation_yaml_data[language]["focus-area"])
    table_head = table_head.replace("$GOAL$", word_translation_yaml_data[language]["goal"])

    return table_head

def generate_fa_table(table_head, table_tail, wg_filename, focus_area_README_list):
    """Creates the table of focus areas starting from a blank WG.tex file"""

    for FA in focus_area_README_list:
        # FA[0] = focus_area_name
        # FA[1] = focus_area_README.md

        focus_area_name, focus_area_goal = extract_goal(FA[1])
        table_head += '\t\t' + focus_area_name + ' & ' + focus_area_goal + ' \\\\ \n\t\t\hline\n'

    table_head += table_tail

    with open(wg_filename, 'w') as f:
        f.write(table_head)

def convert_tex2pdf(tex_filename, word_translation_yaml_data, language, cover_filename):
    """Converts master latex file to PDF. Adds the toc headings and
    cover page depending on language
    """

    toc_heading = word_translation_yaml_data[language]["toc-heading"]

    pdf_filename = f"{language.title()}-Release-" + datetime.today().strftime('%Y-%m-%d') + ".pdf"

    print(f"\nConverting {tex_filename} file to PDF")
    output = pypandoc.convert_file(tex_filename, 'pdf', outputfile=pdf_filename, extra_args=['-f', 'latex',
                                                                                '--pdf-engine=xelatex',
                                                                                '--include-in-header', 'header_1.tex',
                                                                                '--highlight-style', 'zenburn',
                                                                                '-V', 'geometry:margin=0.8in',
                                                                                '-V', 'monofont:DejaVuSansMono.ttf',
                                                                                '-V', 'mathfont:texgyredejavu-math.otf',
                                                                                '-V', 'geometry:a4paper',
                                                                                '-V', 'colorlinks=true',
                                                                                '-V', f'toc-title:{toc_heading}',
                                                                                '-V', 'linkcolour:blue',
                                                                                '-V', 'fontsize=12pt',
                                                                                '--toc', '--toc-depth= 3',
                                                                                '--include-before-body', cover_filename,
                                                                                '--include-after-body', 'end-matter.tex'
                                                                                ]
                                   )

    return pdf_filename

def print_summary(wg_count, focus_area_count, metric_count):
    print()
    print(Color.CYAN + "\n" + "=" * 10 + Color.YELLOW + " SUMMARY " + Color.CYAN + "=" * 10 + Color.GREEN)
    print(f"Total working groups: {wg_count}")
    print(f"Total focus areas: {focus_area_count}")
    print(f"Total metrics: {metric_count}", Color.CYAN)
    print("=" * 29, Color.END)

def print_final_msg(pdf_filename):
    print(Color.CYAN)
    print("Created the final PDF ->", Color.GREEN, pdf_filename)
    print(Color.CYAN)

    final_path = "./output/" + pdf_filename
    print("\nLogs are saved in", Color.GREEN, "logs.txt", Color.CYAN)
    print("Output PDF is saved in", Color.GREEN, final_path, Color.END)


#-----# Importing packages #-----#
# Import packages
import numpy as np # Matrix maths
import tensorflow as tf # NN functions
import matplotlib.pyplot as plt # For drawing graph
from sklearn.metrics import classification_report
import keras

from pathlib import Path

#-----# Project desctiption #-----#

# Defining main function
def main():

    # Instantiate class
    cnn = legoCNN()

# Defining class
class legoCNN:
    def __init__(self):

        # Setting output directory
        self.out_dir = self.setting_output_directory()

        # Setting output directory
        self.data_dir = self.setting_data_directory()

        # Preproces data
        self.preprocess_data()

         # Build model
        self.build_model()

        # Train and evaluate model
        self.train_and_evaluate()

        # Create plot over model learning history
        self.plot_history()

        return

    #-----# Utility functions #-----#

    # Defining function for setting directory for the raw data
    def setting_output_directory(self):

        root_dir = Path.cwd()  # Setting root directory

        out_dir = root_dir / 'output' / 'model_outputs' # Setting data directory

        return out_dir

    # Defining function for setting directory for the raw data
    def setting_data_directory(self):

        root_dir = Path.cwd()  # Setting root directory

        data_dir = root_dir / 'data'  # Setting data directory

        return data_dir

    # Defining a finction for loading and preprocesisng data
    def preprocess_data(self):

        # Loading in data directly from folders and turning them grayscale
        self.train_data = tf.keras.preprocessing.image_dataset_from_directory(self.data_dir / 'train',
                                                                            image_size=(132, 132),
                                                                            batch_size=16,
                                                                            color_mode='grayscale')

        self.test_data = tf.keras.preprocessing.image_dataset_from_directory(self.data_dir / 'test',
                                                                          image_size=(132, 132),
                                                                          batch_size=16,
                                                                          color_mode='grayscale')

        # Gettiing names of each class (brick names)
        self.class_names = self.train_data.class_names

        name_of_classes = open(str(self.out_dir) + "/name_of_classes.txt", "w")

        name_of_classes.write(str(self.class_names))

        name_of_classes.close()

        # Number of classes
        self.num_classes = len(self.class_names)

    # Defining cnn in a single function
    def build_model(self):

        # Image input shape
        input_shape = (132, 132, 1)

        # Build the model
        self.model = keras.models.Sequential()

        # Compressing numbers into a smaller vectorspace for better convergences
        self.model.add(keras.layers.experimental.preprocessing.Rescaling(1. / 255, input_shape=input_shape))

        self.model.add(keras.layers.Conv2D(32, kernel_size=(5, 5), activation='relu'))

        self.model.add(keras.layers.Conv2D(64, (3, 3), activation='relu'))

        self.model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))

        self.model.add(keras.layers.Dropout(0.25))

        self.model.add(keras.layers.Flatten())

        self.model.add(keras.layers.Dense(32, activation='relu'))

        self.model.add(keras.layers.Dense(16, activation='relu'))

        self.model.add(keras.layers.Dense(self.num_classes, activation='softmax'))

        # Compile the layers into one model
        # Loss function and optimizer needed (using SparseCategoricalCrossentropy and Adam)
        self.model.compile(
            optimizer = 'adam',
            loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), # This allows more than two classes
            metrics = ['accuracy']
        )

        print(self.model.summary())

    # Defining function for trainings
    def train_and_evaluate(self):

        self.epochs = 2

        # Train the model
        self.history = self.model.fit(
            self.train_data,
            validation_data=self.test_data,
            epochs = self.epochs # Number of iterations over the entire training dataset
        )

        # Saving model
        self.model.save(self.out_dir / f"lego-CNN.model")

        # Evaluating model
        loss, acc = self.model.evaluate(self.test_data, verbose=1)

        # Print loss and accuracy
        print("loss: %.2f" % loss)

        print("acc: %.2f" % acc)

    #Function for plotting the models performance
    def plot_history(self):

        # Visualize training history
        plt.style.use("fivethirtyeight")

        fig = plt.figure()

        plt.plot(np.arange(0, self.epochs), self.history.history["loss"], label="train_loss")

        plt.plot(np.arange(0, self.epochs), self.history.history["val_loss"], label="val_loss")

        plt.plot(np.arange(0, self.epochs), self.history.history["accuracy"], label="train_acc")

        plt.plot(np.arange(0, self.epochs), self.history.history["val_accuracy"], label="val_acc")

        plt.title("Training Loss and Accuracy")

        plt.xlabel("Epoch #")

        plt.ylabel("Loss/Accuracy")

        plt.legend()

        plt.tight_layout()
        out_file = self.out_dir / f"train_val_history_{self.epochs}_epochs.png"

        plt.savefig(out_file)

# Executing main function when script is run
if __name__ == '__main__':
    main()


import csv
import gensim
import numpy
from nltk.tokenize import RegexpTokenizer
tokenizer = RegexpTokenizer(r'\w+')
from nltk.corpus import stopwords
stop = stopwords.words('english')

model = gensim.models.Word2Vec.load_word2vec_format(
    'GoogleNews-vectors-negative300.bin', binary=True)
reader = csv.reader(open('filtered_data.csv'),
                    delimiter='\t', quoting=csv.QUOTE_NONE)
dict_data = {}

def clean_query(query):
    query = query.lower()  # converted to lowercase alphabet
    query = tokenizer.tokenize(query)  # tokenized
    query = [q for q in query if q not in stop]  # removed stop words
    return query

def wordvec(word):
    try:
        return model[word]
    except KeyError:
        pass
    return None

def get_centroid(query):
    query = clean_query(query)
    s = numpy.zeros(len(model["one"]))  # just a hack
    length = 0
    for a in query:
        if (a in model):
            s += model[a]
            length += 1
    if (length == 0):
        return s
    s = s / length
    return s

for line in reader:
    dict_data[line[0]] = line[1:]

TITLE = 0
AUTHORS = 1
YEAR = 2
VENUE = 3
ABSTRACT = 4
REFS = 5

# Get the Word Centroid Difference

def get_knn_wcd(query_id, num):
    dic={}
    vec_A = get_centroid(dict_data[query_id][ABSTRACT])
    for i in dict_data:
        if (i == query_id):
            continue
        vec_B = get_centroid(dict_data[i][ABSTRACT])
        val = numpy.linalg.norm(vec_A - vec_B)
        if(len(dic)<num):
            dic[i]=val
        else:
            m=max(dic,key=dic.get)
            if(dic[m]>val):
                del (dic[m])
                dic[i]=val
    return list(dic.keys())


from asyncio.windows_events import NULL
from django.shortcuts import render
from django.http import HttpResponse

from .models import Ticket

#create a tree like [{"head" : gemini, "children" : [{"head" : payload, "children" : [...]}]}]

def index(request):
    if (request.method == 'POST'):
        Ticket.objects.get(urlName= request.POST['nodeName']).toggleComplete()
    tickets=[]
    connectionList = []
    for ticket in Ticket.objects.filter(parentTicket=None):
        tickets.append(ticket.addNodeToTree(connectionList))

    context = {'title': "RocketTicket",'Tickets' : tickets, 'ConnectionList' : connectionList}
    parentTickets = Ticket.objects.filter(parentTicket=None)
    return render(request, 'rocketTicket/index.html', context)


import streamlit as st
import pandas as pd

def app():

    data = pd.read_excel("./2017_Batch.xlsx")

    # Define the number of rows to display per page
    rows_per_page = st.slider('Rows per Page', min_value=1, max_value=100, value=50)

    # Get the current page number from the user
    current_page = st.number_input('Current Page', min_value=1, value=1)

    # Calculate the start and end indices for the current page
    start_idx = (current_page - 1) * rows_per_page
    end_idx = start_idx + rows_per_page

    # Display the data for the current page
    st.title('All Students')
    st.dataframe(data.iloc[start_idx:end_idx], height = 700,width = 1350)
    st.markdown(
    """
    <style>
    .dataframe { width: 1000px; }
    </style>
    """,
    unsafe_allow_html=True
    )

    # Calculate the total number of pages
    total_pages = len(data) // rows_per_page + (len(data) % rows_per_page > 0)

    # Display pagination information
    st.write(f'Page {current_page} of {total_pages}')

    # Store the current page in the session state
    st.session_state.current_page = current_page


"""Read in nev file from cerebus and spit out an ASCII file that can be read by
nev2lkit"""
import logging
from neurapy.cerebus import nev

LOG_FILENAME = 'nev2ascii.log'
logging.basicConfig(filename=LOG_FILENAME,
                    format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
                    level=logging.DEBUG)
logger = logging.getLogger('Nev2ASCII')

from optparse import OptionParser #For command line arguments

def write_header(fout, basic_header, extended_header, channel, total_channels):
  """Given a (ascii) file handle and the basic and extended headers, write the
  header for the ascii file. nev2lkit is fussy about the formatting. Specifically
  it needs the header string to be exact (it's hard coded in their src)
  http://nev2lkit.cvs.sourceforge.net/viewvc/nev2lkit/nev2lkit/lib/ascii.cpp?revision=1.11&view=markup
  """
  Fs = basic_header['neural sample resolution Hz']
  channel_info_dict = extended_header['neural event waveform'][channel]
  bytes_in_data_packets = basic_header['bytes in data packets']
  bytes_per_waveform_sample = channel_info_dict['bytes per waveform sample']
  waveform_size = (bytes_in_data_packets - 8)/bytes_per_waveform_sample
  fout.write('%d\n%d\n%d\n' %(int(Fs), waveform_size, total_channels))#Needs all ints
  #Set to the max number of channels possible (255)
  fout.write('Electrode.Unit\tTime stamp\tWave form\n')#Has to be exactly like this

def write_data(fout, channel, unit, data, threshold):
  spike_time_ms = data['spike time ms']
  waveform = data['waveform mV']
  for n in range(spike_time_ms.size):
    if waveform[n,:].max() < threshold:
      fout.write('%d.%d %f ' %(channel, unit, spike_time_ms[n] * 1000))#Need spike time in s
      for m in range(waveform[n,:].size):
        fout.write('%3.2f\t' %(waveform[n,m]))
    fout.write('\n')

parser = OptionParser()
parser.add_option("-f", "--nevfile",
                  dest = "nevfile",
                  default = '/Users/kghose/Research/2008-20XX (Monkeys)/Data/Mstim1/DOBBY2010/Neural/20100712/afc005.nev',
                  help = 'nev file to read in [%default]')
parser.add_option("--fragdir",
                  dest = "fragdir",
                  default = '/Users/kghose/Research/2008-20XX (Monkeys)/Data/Mstim1/DOBBY2010/NeuralProcessed/NevFrag/2010-07-12-05/',
                  help = 'directory to store the fragmented files [%default]')
parser.add_option("-o", "--outdir",
                  dest = "outdir",
                  default = '/Users/kghose/Research/2008-20XX (Monkeys)/Data/Mstim1/DOBBY2010/NeuralProcessed/ASCII/2010-07-12-05/',
                  help = 'Output file [%default]')
parser.add_option("--threshold",
                  dest = "threshold",
                  default = 500,
                  type = 'float',
                  help = '[%default] Reject spikes above this as artefacts.')

(options, args) = parser.parse_args()

fragment = False
read_non_neural = False
load_waveform = True

logger.info('Reading headers')
f = open(options.nevfile)
basic_header = nev.read_basic_header(f)
extended_header = nev.read_extended_header(f, basic_header)
unit = 0#Unsorted

for channel in range(1,97):
  fout_name =  options.outdir + '/channel%02d.asc' %(channel)
  fout = open(fout_name,'w')
  write_header(fout, basic_header, extended_header, 1, 1)#each file gets a separate channel
  data, data_ok = \
    nev.read_frag_unit(options.fragdir, basic_header, extended_header,
                 channel = channel,
                 unit = unit,
                 tstart_ms = 0.0,
                 tdur_ms = -1.0,#Read all of it
                 load_waveform = True)
  if data_ok:
    write_data(fout, 1, unit, data, options.threshold)
  fout.close()

f.close()


import unittest, os, logging, sys, shutil, logging, time, glob

sys.path.append(os.path.join(os.path.join(os.path.dirname(__file__)), ".."))
import warnings

with warnings.catch_warnings():
    warnings.filterwarnings("ignore", category=DeprecationWarning)

import qgis, qgis.core
from qgis.PyQt.QtWidgets import QApplication

from Lamia.api.dbasemanager.dbaseparserfactory import DBaseParserFactory
from Lamia.api.libslamia.lamiacost.lamiacost import CostCore
from Lamia.qgisiface.iface.qgsconnector.ifaceqgisconnector import QgisConnector

from settings import *

class DBaseTest(unittest.TestCase):

    """Test case utilisé pour tester les fonctions du module 'random'."""

    def setUp(self):
        """Initialisation des tests."""
        # TESTDIR = os.path.join(os.path.join(os.path.dirname(__file__)), 'temp')
        self.connector = QgisConnector()
        self.testcdir = os.path.join(TESTDIR, "c_creation")
        self.costtdir = os.path.join(TESTDIR, "cost")
        if not os.path.isdir(self.costtdir):
            os.mkdir(self.costtdir)

    def test_a_generateCost(self):

        testcdir = os.path.join(TESTDIR, "c_creation")

        slfile = SLFILE = os.path.join(
            os.path.dirname(__file__), "datas", "lamia_digue", "test01.sqlite"
        )
        sqlitedbase = DBaseParserFactory("spatialite").getDbaseParser()
        sqlitedbase.loadDBase(slfile=SLFILE)
        basename = os.path.basename(SLFILE).split(".")[0]
        sqlitedbase.variante = (
            "test" if sqlitedbase.variante is None else sqlitedbase.variante
        )
        self.makeTests(
            basename, sqlitedbase.worktype, sqlitedbase.variante, sqlitedbase
        )
        sqlitedbase.disconnect()

    def makeTests(self, dbtype, work, variante, dbase):

        costtool = CostCore(dbase, messageinstance=self.connector)

        conftotest = []

        for filename in glob.glob(os.path.join(costtool.confdatadirplugin, "*.csv")):
            basename = os.path.basename(filename).split(".")[0]
            if basename == "README":
                continue
            conftotest.append(basename)

        # conftotest = ['Infralineaire']        #uncomment to test only one conf
        for confname in conftotest:
            logging.getLogger("Lamia_unittest").debug(
                "********************** %s %s, %s - exporting conf : %s ...",
                dbtype,
                work,
                variante,
                confname,
            )
            # destfile = os.path.join(self.reportdir, dbtype + '_' + work + '_' + variante + '_' + confname  + '.pdf')
            costtool.runCost(costfilepath=confname, pkzonegeos=[2])

def main():
    app = initQGis()
    logging.basicConfig(
        format="%(asctime)s :: %(levelname)s :: %(module)s :: %(funcName)s :: %(message)s",
        datefmt="%H:%M:%S",
    )
    logging.getLogger("Lamia_unittest").setLevel(logging.DEBUG)
    unittest.main()
    exitQGis()

if __name__ == "__main__":
    main()
    """
    logging.basicConfig( stream=sys.stderr )
    logging.getLogger( "Lamia_unittest" ).setLevel( logging.DEBUG )
    unittest.main()
    """


import cv2
import numpy as np
from model import EMR

# prevents opencl usage and unnecessary logging messages
cv2.ocl.setUseOpenCL(False)

EMOTIONS = ['Angry', 'Disgusted', 'Fearful', 'Happy', 'Sad', 'Surprised', 'Neutral']

# Initialize object of EMR class
network = EMR()
network.build_network()

# In case you want to detect emotions on a video, provide the video file path instead of 0 for VideoCapture.
cap = cv2.VideoCapture('r.jpg')
font = cv2.FONT_HERSHEY_SIMPLEX

while True:
    # Again find haar cascade to draw bounding box around face
    ret, frame = cap.read()
    if not ret:
        break
    facecasc = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    faces = facecasc.detectMultiScale(gray, 1.3, 5)
    if len(faces) > 0:
        # draw box around faces
        for face in faces:
            (x,y,w,h) = face
            frame = cv2.rectangle(frame,(x,y-30),(x+w,y+h+10),(255,0,0),2)
            newimg = cv2.cvtColor(frame[y:y+h,x:x+w], cv2.COLOR_BGR2GRAY)
            cv2.imshow('naming',newimg)
            newimg = cv2.resize(newimg, (48,48), interpolation = cv2.INTER_CUBIC) / 255.
            result = network.predict(newimg)
            if result is not None:
                maxindex = np.argmax(result[0])
                font = cv2.FONT_HERSHEY_SIMPLEX
                cv2.putText(frame,EMOTIONS[maxindex],(x,y), font, 1,(255,255,255),2,cv2.LINE_AA)

    asd=cv2.resize(frame,None,fx=2,fy=2,interpolation = cv2.INTER_CUBIC)
    abc=cv2.resize(asd,(900,600))
    cv2.imshow('Video',abc )
    cv2.waitKey(0)
cap.release()
cv2.destroyAllWindows()


from __future__ import annotations

from typing import List, Optional

from cards import Card, Queen, CardType
from positions import HandPosition, SleepingQueenPosition, AwokenQueenPosition, Position, QueenCollectionInterface
from hand import HandInterface
from evaluate import EvaluateAttackInterface, MoveQueenInterface

class PlayerState:
    def __init__(self) -> None:
        self.cards: List[Card] = []
        self.awoken_queens: List[Optional[Queen]] = []

class Player:
    def __init__(self, hand: HandInterface, awoken_queens: QueenCollectionInterface,
                 move_queen: MoveQueenInterface, eval_attack: EvaluateAttackInterface,
                 player_state: PlayerState) -> None:
        self.hand = hand
        self.awoken_queens = awoken_queens
        self.move_queen = move_queen
        self.evaluate = eval_attack.evaluate
        self.player_state = player_state
        self.picked_numbered_cards: List[Card] = []

    def update_player_state(self) -> None:
        self.player_state.cards = self.hand.get_cards()
        self.player_state.awoken_queens = self.awoken_queens.get_queens()

    def play(self, positions: List[Position]) -> Optional[bool]:
        from_hand: List[HandPosition] = [
            pos for pos in positions if type(pos) == HandPosition]
        sleeping_queens: List[SleepingQueenPosition] = [
            pos for pos in positions if type(pos) == SleepingQueenPosition]
        awoken_queens: List[AwokenQueenPosition] = [
            pos for pos in positions if type(pos) == AwokenQueenPosition]

        # attack on somebody else's awoken queen
        if len(awoken_queens) == 1 and len(from_hand) == 1 and not sleeping_queens:
            attack_pos = from_hand[0]
            target_pos = awoken_queens[0]
            result = self.evaluate(attack_pos, target_pos)
            if result is not None:
                self.hand.pick_cards(from_hand)
                self.hand.remove_picked_cards_and_redraw()
                self.update_player_state()
            return result

        # trying to wake up a sleeping queen using a king
        if len(sleeping_queens) == 1 and len(from_hand) == 1 and not awoken_queens:
            king = from_hand[0].get_card()
            target_queen = sleeping_queens[0]
            if king not in self.hand.get_cards():
                return None
            if king.get_type() == 'KING':
                self.move_queen.wake_up(target_queen)
                self.hand.pick_cards(from_hand)
                self.hand.remove_picked_cards_and_redraw()
                self.update_player_state()
                return True
            return None

        cards_from_hand = self.hand.pick_cards(from_hand)
        self.picked_numbered_cards.clear()
        for card in cards_from_hand or []:
            if card.type == CardType.NUMBER:
                self.picked_numbered_cards.append(card)
        if len(self.picked_numbered_cards) == len(positions):       # all picked cards are numbered
            if self.evaluate_numbered_cards():
                self.hand.remove_picked_cards_and_redraw()
                self.update_player_state()
                return True
        return None

    def evaluate_numbered_cards(self) -> bool:
        """
        Checks if current move is valid according to game rules.
        """
        cards = self.picked_numbered_cards
        count = len(cards)
        if count == 1 or (count == 2 and cards[0].get_points() == cards[1].get_points()):
            return True
        values = sorted(list(map(lambda x: x.get_points(), cards)))
        if sum(values[:-1], ) == values[-1]:
            return True
        return False

    def remove_queen(self, queen: Queen) -> None:
        self.awoken_queens.remove_queen(queen)

    def add_queen(self, queen: Queen) -> None:
        self.awoken_queens.add_queen(queen)

    def count_points(self) -> int:
        return self.awoken_queens.count_points()

    def count_queens(self) -> int:
        return self.awoken_queens.count_queens()

    def has_card_of_type(self, card_type: CardType) -> Optional[HandPosition]:
        return self.hand.has_card_of_type(card_type)


#!/usr/bin/env python
import numpy as np
import base64

class LFSR():
    def __init__(self, fpoly=[5,2], initstate='ones', verbose=False):
        if isinstance(initstate, str):
            if initstate=='ones':
                initstate = np.ones(np.max(fpoly))
            elif initstate=='random':
                initstate = np.random.randint(0,2,np.max(fpoly))
            else:
                raise Exception('Unknown initial state')

        self.initstate = initstate
        self.fpoly  = fpoly
        self.state  = initstate.astype(int)
        self.count  = 0
        self.seq    = np.array(-1)
        self.outbit = -1
        self.feedbackbit = -1
        self.verbose = verbose
        self.M = self.initstate.shape[0]
        self.expectedPeriod = 2**self.M -1
        self.fpoly.sort(reverse=True)
        feed = ' '
        for i in range(len(self.fpoly)):
            feed = feed + 'x^'+str(self.fpoly[i])+' + '
        feed = feed + '1'
        self.feedpoly = feed

    def set(self,fpoly,state='ones'):
        self.__init__(fpoly=fpoly,initstate=state)

    def reset(self):
        self.__init__(initstate=self.initstate,fpoly=self.fpoly)

    def changeFpoly(self, newfpoly, reset=False):
        newfpoly.sort(reverse=True)
        self.fpoly =newfpoly
        feed = ' '
        for i in range(len(self.fpoly)):
            feed = feed + 'x^'+str(self.fpoly[i])+' + '
        feed = feed + '1'
        self.feedpoly = feed

        self.check()
        if reset:
            self.reset()

    def next(self):
        b = np.logical_xor(self.state[self.fpoly[0]-1],self.state[self.fpoly[1]-1])
        if len(self.fpoly)>2:
            for i in range(2,len(self.fpoly)):
                b = np.logical_xor(self.state[self.fpoly[i]-1],b)

        self.state = np.roll(self.state, 1)
        self.state[0] = b*1
        self.feedbackbit = b*1
        if self.count==0:
            self.seq = self.state[-1]
        else:
            self.seq = np.append(self.seq,self.state[-1])
        self.outbit = self.state[-1]
        self.count +=1
        if self.verbose:
            print('S: ',self.state)
        return self.state[-1]

    def runFullCycle(self):
        for i in range(self.expectedPeriod):
            self.next()
        return self.seq

    def runKCycle(self,k):
        tempseq =np.ones(k)*-1
        for i in range(k):
            tempseq[i] = self.next()

        return tempseq

def main():
    print("Reading flag...")
    with open("flag.txt") as f:
        flag = f.read().strip()

    flag = f"Well done! You got the flag. The flag is {flag}".encode('ascii')

    with open("state.txt") as f:
        state1 = np.array(list(map(int, f.readline().split(','))))
        state2 = np.array(list(map(int, f.readline().split(','))))

    print(f"state 1 size: {len(state1)}")
    print(f"state 2 size: {len(state2)}")

    fpoly1 = [8,4,1]
    fpoly2 = [13,6,3]

    A = LFSR(fpoly=fpoly1, initstate =state1)
    B = LFSR(fpoly=fpoly2, initstate =state2)

    size = len(flag)
    xord_byte_array = bytearray(len(flag))

    # Flag is encrypted by xor-ing with keystream
    # Keystream = (LFSR A output + LFSR B output) % 256
    print("Encrypting flag...")
    for i in range(size):
        newbyte1=0
        newbyte2=0
        for x in range(8):
            A.next()
            B.next()
            newbyte1 += A.outbit*pow(2,x)
            newbyte2 += B.outbit*pow(2,x)

        xorbyte=(newbyte1+newbyte2)%256
        xord_byte_array[i] = flag[i] ^ xorbyte

    encoded = base64.b64encode(xord_byte_array)
    print(f"Safely encrypted flag: {encoded.decode('ascii')}")

if __name__ == '__main__':
    main()


from sagemaker.huggingface import HuggingFaceModel
import os

# role should be codespace secret called AWS_ROLE

role = os.environ["AWS_ROLE"]

# Hub Model configuration. https://huggingface.co/models
hub = {
    "HF_MODEL_ID": "microsoft/swin-tiny-patch4-window7-224",
    "HF_TASK": "image-classification",
}

# create Hugging Face Model Class
huggingface_model = HuggingFaceModel(
    transformers_version="4.17.0",
    pytorch_version="1.10.2",
    py_version="py38",
    env=hub,
    role=role,
)

# deploy model to SageMaker Inference
print("Deploying model to SageMaker Inference")
predictor = huggingface_model.deploy(
    initial_instance_count=1,  # number of instances
    instance_type="ml.m5.xlarge",  # ec2 instance type
)


from __future__ import annotations

import dataclasses
from abc import abstractmethod
from dataclasses import dataclass
from functools import partial
from typing import Any, Callable, Dict, Generic, List, Optional, Tuple, TypeVar
from uuid import uuid4

import numpy as np
import pandas as pd
import pyarrow as pa
from pyarrow import dataset as pada

from daft.expressions import ColID, ColumnExpression, Expression, ExpressionExecutor
from daft.logical.schema import ExpressionList
from daft.runners.blocks import ArrowArrType, DataBlock, PyListDataBlock

from ..execution.operators import OperatorEnum, PythonExpressionType

PartID = int

@dataclass(frozen=True)
class PyListTile:
    column_id: ColID
    column_name: str
    partition_id: PartID
    block: DataBlock

    def __len__(self) -> int:
        return len(self.block)

    def apply(self, func: Callable[[DataBlock], DataBlock]) -> PyListTile:
        return dataclasses.replace(self, block=func(self.block))

    def split_by_index(
        self,
        num_partitions: int,
        pivots: np.ndarray,
        target_partitions: np.ndarray,
        argsorted_target_partition_indices: DataBlock,
    ) -> List[PyListTile]:
        assert len(argsorted_target_partition_indices) == len(self)
        new_blocks = self.block.partition(
            num_partitions,
            pivots=pivots,
            target_partitions=target_partitions,
            argsorted_targets=argsorted_target_partition_indices,
        )
        assert len(new_blocks) == num_partitions
        return [dataclasses.replace(self, block=nb, partition_id=i) for i, nb in enumerate(new_blocks)]

    @classmethod
    def merge_tiles(cls, to_merge: List[PyListTile], verify_partition_id: bool = True) -> PyListTile:
        assert len(to_merge) > 0

        if len(to_merge) == 1:
            return to_merge[0]

        column_id = to_merge[0].column_id
        partition_id = to_merge[0].partition_id
        column_name = to_merge[0].column_name
        # first perform sanity check
        for part in to_merge[1:]:
            assert part.column_id == column_id
            assert not verify_partition_id or part.partition_id == partition_id
            assert part.column_name == column_name

        merged_block = DataBlock.merge_blocks([t.block for t in to_merge])
        return dataclasses.replace(to_merge[0], block=merged_block)

    def replace_block(self, block: DataBlock) -> PyListTile:
        return dataclasses.replace(self, block=block)

@dataclass(frozen=True)
class vPartition:
    columns: Dict[ColID, PyListTile]
    partition_id: PartID

    def __post_init__(self) -> None:
        size = None
        for col_id, tile in self.columns.items():
            if tile.partition_id != self.partition_id:
                raise ValueError(f"mismatch of partition id: {tile.partition_id} vs {self.partition_id}")
            if size is None:
                size = len(tile)
            if len(tile) != size:
                raise ValueError(f"mismatch of tile lengths: {len(tile)} vs {size}")
            if col_id != tile.column_id:
                raise ValueError(f"mismatch of column id: {col_id} vs {tile.column_id}")

    def __len__(self) -> int:
        assert len(self.columns) > 0
        return len(next(iter(self.columns.values())))

    def eval_expression(self, expr: Expression) -> PyListTile:
        expr_col_id = expr.get_id()
        expr_name = expr.name()

        assert expr_col_id is not None
        assert expr_name is not None
        if not expr.has_call():
            return PyListTile(
                column_id=expr_col_id,
                column_name=expr_name,
                partition_id=self.partition_id,
                block=self.columns[expr_col_id].block,
            )

        required_cols = expr.required_columns()
        required_blocks = {}
        for c in required_cols:
            col_id = c.get_id()
            assert col_id is not None
            block = self.columns[col_id].block
            name = c.name()
            assert name is not None
            required_blocks[name] = block
        exec = ExpressionExecutor()
        result = exec.eval(expr, required_blocks)
        expr_col_id = expr.get_id()
        expr_name = expr.name()
        assert expr_col_id is not None
        assert expr_name is not None
        return PyListTile(column_id=expr_col_id, column_name=expr_name, partition_id=self.partition_id, block=result)

    def eval_expression_list(self, exprs: ExpressionList) -> vPartition:
        tile_list = [self.eval_expression(e) for e in exprs]
        new_columns = {t.column_id: t for t in tile_list}
        return vPartition(columns=new_columns, partition_id=self.partition_id)

    @classmethod
    def from_arrow_table(cls, table: pa.Table, column_ids: List[ColID], partition_id: PartID) -> vPartition:
        names = table.column_names
        assert len(names) == len(column_ids)
        tiles = {}
        for i, (col_id, name) in enumerate(zip(column_ids, names)):
            arr = table[i]
            block: DataBlock[ArrowArrType] = DataBlock.make_block(arr)
            tiles[col_id] = PyListTile(column_id=col_id, column_name=name, partition_id=partition_id, block=block)
        return vPartition(columns=tiles, partition_id=partition_id)

    @classmethod
    def from_pydict(cls, data: Dict[str, List[Any]], schema: ExpressionList, partition_id: PartID) -> vPartition:
        column_exprs = schema.to_column_expressions()
        tiles = {}
        for col_expr in column_exprs:
            col_id = col_expr.get_id()
            col_name = col_expr.name()
            arr = (
                data[col_name]
                if isinstance(col_expr.resolved_type(), PythonExpressionType)
                else pa.array(data[col_expr.name()])
            )
            block: DataBlock = DataBlock.make_block(arr)
            tiles[col_id] = PyListTile(column_id=col_id, column_name=col_name, partition_id=partition_id, block=block)
        return vPartition(columns=tiles, partition_id=partition_id)

    def to_pandas(self, schema: Optional[ExpressionList] = None) -> pd.DataFrame:
        if schema is not None:
            output_schema = [(expr.name(), expr.get_id()) for expr in schema]
        else:
            output_schema = [(tile.column_name, id) for id, tile in self.columns.items()]
        return pd.DataFrame(
            {
                name: pd.Series(self.columns[id].block.data)
                if isinstance(self.columns[id].block, PyListDataBlock)
                else self.columns[id].block.data.to_pandas()
                for name, id in output_schema
            }
        )

    def for_each_column_block(self, func: Callable[[DataBlock], DataBlock]) -> vPartition:
        return dataclasses.replace(self, columns={col_id: col.apply(func) for col_id, col in self.columns.items()})

    def head(self, num: int) -> vPartition:
        # TODO make optimization for when num=0
        return self.for_each_column_block(partial(DataBlock.head, num=num))

    def sample(self, num: int) -> vPartition:
        if len(self) == 0:
            return self
        sample_idx: DataBlock[ArrowArrType] = DataBlock.make_block(data=np.random.randint(0, len(self), num))
        return self.for_each_column_block(partial(DataBlock.take, indices=sample_idx))

    def filter(self, predicate: ExpressionList) -> vPartition:
        mask_list = self.eval_expression_list(predicate)
        assert len(mask_list.columns) > 0
        mask = next(iter(mask_list.columns.values())).block
        for to_and in mask_list.columns.values():
            mask = mask.run_binary_operator(to_and.block, OperatorEnum.AND)
        return self.for_each_column_block(partial(DataBlock.filter, mask=mask))

    def sort(self, sort_key: Expression, desc: bool = False) -> vPartition:
        sort_tile = self.eval_expression(sort_key)
        argsort_idx = sort_tile.apply(partial(DataBlock.argsort, desc=desc))
        return self.take(argsort_idx.block)

    def take(self, indices: DataBlock) -> vPartition:
        return self.for_each_column_block(partial(DataBlock.take, indices=indices))

    def agg(self, to_agg: List[Tuple[Expression, str]], group_by: Optional[ExpressionList] = None) -> vPartition:
        evaled_expressions = self.eval_expression_list(ExpressionList([e for e, _ in to_agg]))
        ops = [op for _, op in to_agg]
        if group_by is None:
            agged = {}
            for op, (col_id, tile) in zip(ops, evaled_expressions.columns.items()):
                agged[col_id] = tile.apply(func=partial(tile.block.__class__.agg, op=op))
            return vPartition(partition_id=self.partition_id, columns=agged)
        else:
            grouped_blocked = self.eval_expression_list(group_by)
            assert len(evaled_expressions.columns) == len(ops)
            gcols, acols = DataBlock.group_by_agg(
                list(tile.block for tile in grouped_blocked.columns.values()),
                list(tile.block for tile in evaled_expressions.columns.values()),
                agg_ops=ops,
            )
            new_columns = {}

            for block, (col_id, tile) in zip(gcols, grouped_blocked.columns.items()):
                new_columns[col_id] = dataclasses.replace(tile, block=block)

            for block, (col_id, tile) in zip(acols, evaled_expressions.columns.items()):
                new_columns[col_id] = dataclasses.replace(tile, block=block)
            return vPartition(partition_id=self.partition_id, columns=new_columns)

    def split_by_hash(self, exprs: ExpressionList, num_partitions: int) -> List[vPartition]:
        values_to_hash = self.eval_expression_list(exprs)
        keys = list(values_to_hash.columns.keys())
        keys.sort()
        hsf = None
        assert len(keys) > 0
        for k in keys:
            block = values_to_hash.columns[k].block
            hsf = block.array_hash(seed=hsf)
        assert hsf is not None
        target_idx = hsf.run_binary_operator(num_partitions, OperatorEnum.MOD)
        return self.split_by_index(num_partitions, target_partition_indices=target_idx)

    def split_by_index(self, num_partitions: int, target_partition_indices: DataBlock) -> List[vPartition]:
        assert len(target_partition_indices) == len(self)
        new_partition_to_columns: List[Dict[ColID, PyListTile]] = [{} for _ in range(num_partitions)]
        argsort_targets = target_partition_indices.argsort()
        sorted_targets = target_partition_indices.take(argsort_targets)
        sorted_targets_np = sorted_targets.to_numpy()
        pivots = np.where(np.diff(sorted_targets_np, prepend=np.nan))[0]
        target_partitions = sorted_targets_np[pivots]

        for col_id, tile in self.columns.items():
            new_tiles = tile.split_by_index(
                num_partitions=num_partitions,
                pivots=pivots,
                target_partitions=target_partitions,
                argsorted_target_partition_indices=argsort_targets,
            )
            for part_id, nt in enumerate(new_tiles):
                new_partition_to_columns[part_id][col_id] = nt

        return [vPartition(partition_id=i, columns=columns) for i, columns in enumerate(new_partition_to_columns)]

    def join(
        self,
        right: vPartition,
        left_on: ExpressionList,
        right_on: ExpressionList,
        output_schema: ExpressionList,
        how: str = "inner",
    ) -> vPartition:
        assert how == "inner"
        left_key_part = self.eval_expression_list(left_on)
        left_key_ids = list(left_key_part.columns.keys())

        right_key_part = right.eval_expression_list(right_on)
        left_key_list = [left_key_part.columns[le.get_id()].block for le in left_on]
        right_key_list = [right_key_part.columns[re.get_id()].block for re in right_on]

        left_nonjoin_ids = [i for i in self.columns.keys() if i not in left_key_part.columns]
        right_nonjoin_ids = [i for i in right.columns.keys() if i not in right_key_part.columns]

        left_nonjoin_blocks = [self.columns[i].block for i in left_nonjoin_ids]
        right_nonjoin_blocks = [right.columns[i].block for i in right_nonjoin_ids]

        joined_blocks = DataBlock.join(left_key_list, right_key_list, left_nonjoin_blocks, right_nonjoin_blocks)

        result_keys = left_key_ids + left_nonjoin_ids + right_nonjoin_ids

        assert len(joined_blocks) == len(result_keys)
        joined_block_idx = 0
        result_columns = {}
        for k in left_key_ids:
            result_columns[k] = left_key_part.columns[k].replace_block(block=joined_blocks[joined_block_idx])
            joined_block_idx += 1

        for k in left_nonjoin_ids:
            result_columns[k] = self.columns[k].replace_block(block=joined_blocks[joined_block_idx])
            joined_block_idx += 1

        for k in right_nonjoin_ids:
            result_columns[k] = right.columns[k].replace_block(block=joined_blocks[joined_block_idx])
            joined_block_idx += 1

        assert joined_block_idx == len(result_keys)

        output = vPartition(columns=result_columns, partition_id=self.partition_id)
        return output.eval_expression_list(output_schema)

    def _to_file(
        self,
        file_format: str,
        root_path: str,
        partition_cols: Optional[ExpressionList] = None,
        compression: Optional[str] = None,
    ) -> List[str]:
        keys = [col_id for col_id in self.columns.keys()]
        names = [self.columns[k].column_name for k in keys]
        data = [self.columns[k].block.to_arrow() for k in keys]
        arrow_table = pa.table(data, names=names)
        partition_col_names = []
        if partition_cols is not None:
            for col in partition_cols:
                assert isinstance(
                    col, ColumnExpression
                ), "we can only support ColumnExpressions for partitioning parquet"
                col_id = col.get_id()
                assert col_id is not None
                assert col_id in keys
                partition_col_names.append(self.columns[col_id].column_name)

        visited_paths = []

        def file_visitor(written_file):
            visited_paths.append(written_file.path)

        format: pada.FileFormat
        opts = None

        if file_format == "parquet":
            format = pada.ParquetFileFormat()
            opts = format.make_write_options(compression=compression)
        elif file_format == "csv":
            format = pada.CsvFileFormat()
            assert compression is None

        pada.write_dataset(
            arrow_table,
            base_dir=root_path,
            basename_template=str(uuid4()) + "-{i}." + format.default_extname,
            format=format,
            partitioning=partition_col_names,
            file_options=opts,
            file_visitor=file_visitor,
            use_threads=False,
            existing_data_behavior="overwrite_or_ignore",
        )
        return visited_paths

    def to_parquet(
        self, root_path: str, partition_cols: Optional[ExpressionList] = None, compression: Optional[str] = None
    ) -> List[str]:
        return self._to_file("parquet", root_path=root_path, partition_cols=partition_cols, compression=compression)

    def to_csv(
        self, root_path: str, partition_cols: Optional[ExpressionList] = None, compression: Optional[str] = None
    ) -> List[str]:
        return self._to_file("csv", root_path=root_path, partition_cols=partition_cols, compression=compression)

    @classmethod
    def merge_partitions(cls, to_merge: List[vPartition], verify_partition_id: bool = True) -> vPartition:
        assert len(to_merge) > 0

        if len(to_merge) == 1:
            return to_merge[0]

        pid = to_merge[0].partition_id
        col_ids = set(to_merge[0].columns.keys())
        # first perform sanity check
        for part in to_merge[1:]:
            assert not verify_partition_id or part.partition_id == pid
            assert set(part.columns.keys()) == col_ids

        new_columns = {}
        for col_id in to_merge[0].columns.keys():
            new_columns[col_id] = PyListTile.merge_tiles(
                [vp.columns[col_id] for vp in to_merge], verify_partition_id=verify_partition_id
            )
        return dataclasses.replace(to_merge[0], columns=new_columns)

PartitionT = TypeVar("PartitionT")

class PartitionSet(Generic[PartitionT]):
    @abstractmethod
    def to_pandas(self, schema: Optional[ExpressionList] = None) -> pd.DataFrame:
        raise NotImplementedError()

    @abstractmethod
    def get_partition(self, idx: PartID) -> PartitionT:
        raise NotImplementedError()

    @abstractmethod
    def set_partition(self, idx: PartID, part: PartitionT) -> None:
        raise NotImplementedError()

    @abstractmethod
    def delete_partition(self, idx: PartID) -> None:
        raise NotImplementedError()

    @abstractmethod
    def has_partition(self, idx: PartID) -> bool:
        raise NotImplementedError()

    @abstractmethod
    def __len__(self) -> int:
        return sum(self.len_of_partitions())

    @abstractmethod
    def len_of_partitions(self) -> List[int]:
        raise NotImplementedError()

    @abstractmethod
    def num_partitions(self) -> int:
        raise NotImplementedError()

class PartitionManager:
    def __init__(self, pset_default: Callable[[], PartitionSet]) -> None:
        self._nid_to_partition_set: Dict[int, PartitionSet] = {}
        self._pset_default_list = [pset_default]

    def new_partition_set(self) -> PartitionSet:
        func = self._pset_default_list[0]
        return func()

    def get_partition_set(self, node_id: int) -> PartitionSet:
        assert node_id in self._nid_to_partition_set
        return self._nid_to_partition_set[node_id]

    def put_partition_set(self, node_id: int, pset: PartitionSet) -> None:
        self._nid_to_partition_set[node_id] = pset

    def rm(self, node_id: int, partition_id: Optional[int] = None):
        if partition_id is None:
            del self._nid_to_partition_set[node_id]
        else:
            self._nid_to_partition_set[node_id].delete_partition(partition_id)
            if self._nid_to_partition_set[node_id].num_partitions() == 0:
                del self._nid_to_partition_set[node_id]

    def clear(self) -> None:
        del self._nid_to_partition_set
        self._nid_to_partition_set = {}


import pygame
import funciones
import clases_genericas
import colores

def pantalla_pausa(ventana):

    flag_inicio = True

    foto_titulo = clases_genericas.Imagen('imagenes\\pausa.png',400,70,135,150)

    while flag_inicio: #pantalla de inicio

        lista_eventos = pygame.event.get()

        for evento in lista_eventos:

            if evento.type == pygame.QUIT:

                exit()

            if evento.type == pygame.KEYDOWN:

                if evento.key == pygame.K_ESCAPE:

                    flag_inicio = False

                elif evento.key == pygame.K_z:

                    exit()

        ventana.fill((0,0,0))
        foto_titulo.update(ventana)
        funciones.crear_titulo_pantalla(ventana,'"ESC" para renaudar',27,200,460,colores.BLANCO_BEIGE)
        funciones.crear_titulo_pantalla(ventana,'"Z"Salir sin guardar',27,200,560,colores.BLANCO_BEIGE)
        pygame.display.flip()


# coding: utf-8

import os

import boto3

from typing import Dict

# CONSTANT
CLIENT = boto3.client('lambda')

class ScriptSettings(object):
    """Settings for the files in scripts folder."""

    def __init__(self, func_name: str):
        """Constructor of the class.

        :param func_name: Name of the lambda function whose settings are to be
                            constructed here.
        """
        self.func_name = func_name
        self.env = os.environ['env']
        self.env_variables = self.create_env_vairables()

    def create_env_vairables(self) -> Dict[str, str]:
        """Create environment variables based on the given env value.

        Different lambda function results in different environment variables.

        :return: The environment variables as a dict.
        """
        return {
            'IOT_PREFIX': os.environ['IOT_PREFIX'],
            'S3_PREFIX': os.environ['S3_PREFIX'],
        }