from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch
import numpy as np

from models.losses import FocalLoss, RegL1Loss, RegLoss, RegWeightedL1Loss, BinRotLoss, AutoShape_Position_loss, RegWeightedL1Loss2
from models.utils import _sigmoid
from .base_trainer import BaseTrainer

class AutoShapeLoss(torch.nn.Module):
    def __init__(self, opt):
        super(AutoShapeLoss, self).__init__()
        self.crit = FocalLoss()
        self.crit_hm_hp = torch.nn.MSELoss() if opt.mse_loss else FocalLoss()
        self.crit_kp = RegWeightedL1Loss() if not opt.dense_hp else \
            torch.nn.L1Loss(reduction='sum')
        self.crit_p3d = RegWeightedL1Loss2()
        self.crit_reg = RegL1Loss() if opt.reg_loss == 'l1' else \
            RegLoss() if opt.reg_loss == 'sl1' else None
        self.crit_rot = BinRotLoss()
        self.opt = opt
        is_kitti = False
        if 'kitti' in opt.dataset: is_kitti = True
        self.position_loss = AutoShape_Position_loss(opt, is_kitti)

    def forward(self, outputs, batch, phase=None):

        opt = self.opt
        hm_loss, wh_loss, off_loss = 0, 0, 0
        hp_loss, off_loss, hm_hp_loss, hp_offset_loss = 0, 0, 0, 0
        dim_loss, rot_loss, prob_loss = 0, 0, 0
        p3d_loss = 0
        coor_loss =0
        box_score=0
        output = outputs[0]
        output['hm'] = _sigmoid(output['hm'])
        if opt.hm_hp and not opt.mse_loss:
            output['hm_hp'] = _sigmoid(output['hm_hp'])
        hm_loss = self.crit(output['hm'], batch['hm'])
        hp_loss = self.crit_kp(output['hps'],batch['hps_mask'], batch['ind'], batch['hps'],batch['dep'])
        if opt.wh_weight > 0:
            wh_loss = self.crit_reg(output['wh'], batch['reg_mask'],batch['ind'], batch['wh'])
        if opt.dim_weight > 0:
            dim_loss = self.crit_reg(output['dim'], batch['reg_mask'],batch['ind'], batch['dim'])
            p3d_loss += self.crit_reg(output['p3d'], batch['reg_mask'],
                                     batch['ind'], batch['p3d'])
        if opt.rot_weight > 0:
            rot_loss = self.crit_rot(output['rot'], batch['rot_mask'], batch['ind'], batch['rotbin'], batch['rotres'])
        if opt.reg_offset and opt.off_weight > 0:
            off_loss = self.crit_reg(output['reg'], batch['reg_mask'], batch['ind'], batch['reg'])
        if opt.reg_hp_offset and opt.off_weight > 0:
            hp_offset_loss = self.crit_reg(output['hp_offset'], batch['hp_mask'], batch['hp_ind'], batch['hp_offset'])
        if opt.hm_hp and opt.hm_hp_weight > 0:
            hm_hp_loss = self.crit_hm_hp(output['hm_hp'], batch['hm_hp'])
        coor_loss, prob_loss, box_score = self.position_loss(output, batch,phase)
        loss_stats = {'loss': box_score, 'hm_loss': hm_loss, 'hp_loss': hp_loss,
                      'hm_hp_loss': hm_hp_loss, 'hp_offset_loss': hp_offset_loss,
                      'wh_loss': wh_loss, 'off_loss': off_loss, 'dim_loss': dim_loss,
                      'rot_loss': rot_loss, 'prob_loss': prob_loss, 'box_score': box_score, 'coor_loss': coor_loss,
                      'p3d_loss': p3d_loss
                      }

        return loss_stats, loss_stats

class AutoShapeTrainer(BaseTrainer):
    def __init__(self, opt, model, optimizer=None):
        super(AutoShapeTrainer, self).__init__(opt, model, optimizer=optimizer)

    def _get_losses(self, opt):
        loss_states = ['loss', 'hm_loss', 'hp_loss', 'hm_hp_loss',
                       'hp_offset_loss', 'wh_loss', 'off_loss', 'dim_loss', 'rot_loss', 'prob_loss', 'coor_loss',
                       'box_score',
                       'p3d_loss'
                       ]
        loss = AutoShapeLoss(opt)
        return loss_states, loss

    def debug(self, batch, output, iter_id):
        pass

    def save_result(self, output, batch, results):
        pass


import logging
import os
import pickle

import numpy as np
from sklearn.model_selection import train_test_split

def get_training_files_for_params(full_dataset, prefix, training_files_full, training_files_partial):
    '''
    Function that returns the correct training files, for both cases we want to use the small dataset or
    the big dataset.
    :param full_dataset: boolean parameter which specifies if full dataset is used or small one
    :param prefix: the prefix of the file (might have been processed before, look into process_input)
    :param training_files_full: the dictionary with the full files and associated label for each tweet in the file
    :param training_files_partial: the dictionary with the partial files and associated label for each tweet in the file
    :return: the dictionary with the correct files
    '''
    if full_dataset:
        return {os.path.join('data', prefix + base_file): label for base_file, label in training_files_full.items()}

    return {os.path.join('data', prefix + base_file): label for base_file, label in training_files_partial.items()}

def shuffle_dataset(X, y):
    '''
    Function that shuffles a dataset.
    :param X: the instances of the dataset
    :param y: the labels of the dataset
    :return: the shuffled dataset, as a pair (X_suhffled, y_shuffled)
    '''
    np.random.seed(0)
    p = np.random.permutation(len(y))

    X = X[p]
    y = y[p]

    return X, y

def construct_dataset_from_files(filenames, split_size=0.1):
    '''
    Function that constructs a dataset from files. If split_size is None, then no split is performed and
    function returns a pair (X, y). If split_size is a float value, then the dataset is splitted into train
    set and test set, with test set size being equal to split_size * (full_dataset_size).
    :param filenames: the dictionary which has filenames as the keys and as values the labels for the tweets
        in each filen.
    :param split_size: the split size of the set
    :return: (X, y) is split_size is None, the full dataset, otherwise (X_train, y_train, X_test, y_test)
    '''
    logging.info("Loading data from files: " + str(filenames))
    print("Loading data from files: " + str(filenames))

    X = []
    y = []

    for filename, label in filenames.items():
        with open(filename, 'r', encoding='utf8') as f:
            for tweet in f:
                X.append(tweet)
                y.append(int(label))

    X = np.array(X)
    y = np.array(y)

    X, y = shuffle_dataset(X, y)

    if split_size is not None:
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=split_size, random_state=100)
        return X_train, y_train, X_test, y_test

    return X, y

def construct_test_from_file(filename):
    '''
    Function that constructs a dataset representing the final test set used for predictions from the specified file.
    :param filename: the filename where the test set is stored
    :return: (X_test, ids), an pair of arrays representing the test set and associated id for each tweet
    '''
    tweets = []
    ids = []

    with open(filename, 'r', encoding='utf8') as f:
        for line in f:
            id, tweet = line.split(',', 1)
            ids.append(id)
            tweets.append(tweet)

    return np.array(tweets), np.array(ids)

def create_submission(filename, predictions, ids):
    '''
    Function that creates a submission, given the predictions and associated ids.
    :param filename: the filename where we store the submission
    :param predictions: the predictions for the test set
    :param ids: the ids of the tweets
    :return: None
    '''
    with open(filename, 'w', encoding='utf8') as f:
        f.write('Id,Prediction\n')
        for id, prediction in zip(ids, predictions):
            f.write(str(id) + ',' + str(prediction) + '\n')

def construct_predictions_from_pickles(path_train, path_test):
    '''
    Function that creates the prediction of the neural networks models to be used for the second tier classifier
    '''
    with open(path_train, 'rb') as f:
        [X_train, y_train] = pickle.load(f)

    with open(path_test, 'rb') as f:
        X_test = pickle.load(f)

    return X_train, y_train, X_test


import csv
import random
import os
import uuid
import xml.etree.ElementTree as ET
from argparse import ArgumentParser
from multiprocessing import cpu_count
from multiprocessing.pool import ThreadPool
from typing import List, Tuple
from zipfile import ZipFile

DEFAULT_ARC_COUNT = 50
DEFAULT_ARC_FILES = 100
DEFAULT_PROCESSES_COUNT = cpu_count()

class XmlCsvProcessor:
    def __init__(self, zip_path: str, csv_path: str) -> None:
        self._zip_path = zip_path
        self._csv_path = csv_path

    def gen_xml_archives(self, arc_count: int = DEFAULT_ARC_COUNT,
                         files_in_arc: int = DEFAULT_ARC_FILES) -> None:
        '''
        Generates archives with XML files
        :param arc_count: number of result archives
        :param files_in_arc: number of files in one archive
        '''
        if not os.path.exists(self._zip_path):
            os.mkdir(self._zip_path)

        for _ in range(arc_count):
            self._gen_archive(files_in_arc)

    def _gen_archive(self, files_in_arc: int) -> None:
        rand_zip_path = self._rand_zip_path()

        with ZipFile(rand_zip_path, mode='w') as arc:
            for i in range(files_in_arc):
                root = XmlCsvProcessor._gen_single_xml()
                self._write_xml_archive(root, arc, i)

    def _rand_zip_path(self) -> str:
        while True:
            zip_name = XmlCsvProcessor._rand_str() + '.zip'
            full_zip_path = os.path.join(self._zip_path, zip_name)
            if not os.path.exists(full_zip_path):
                return full_zip_path

    @staticmethod
    def _gen_single_xml() -> ET.Element:
        root = ET.Element('root')

        var_id = ET.Element(
            'var', attrib={'name': 'id', 'value': str(uuid.uuid4())})
        root.append(var_id)
        var_level = ET.Element(
            'var',
            attrib={'name': 'level', 'value': str(random.randint(1, 100))}
        )
        root.append(var_level)

        objs = ET.Element('objects')
        for _ in range(random.randint(1, 10)):
            obj = ET.Element(
                'object', attrib={'name': XmlCsvProcessor._rand_str()})
            objs.append(obj)

        root.append(objs)
        return root

    @staticmethod
    def _rand_str() -> str:
        result = []

        for _ in range(random.randint(4, 10)):
            ch = chr(ord('a') + random.randint(0, 25))
            result.append(ch)

        return ''.join(result)

    def _write_xml_archive(self, root: ET.Element, arc: ZipFile, i: int) -> None:
        xml_str = ET.tostring(root)
        arc.writestr(f'{i}.xml', xml_str)

    def gen_csv_files(self,
                      processes_count: int = DEFAULT_PROCESSES_COUNT) -> None:
        '''
        Generates CSV files from prepared XMLs in archives
        :param processes_count: parallelism level
        '''
        if not os.path.exists(self._csv_path):
            os.mkdir(self._csv_path)

        with ThreadPool(processes=processes_count) as pool:
            results = pool.map(
                self._process_single_zip, os.listdir(self._zip_path))

        levels_path = os.path.join(self._csv_path, 'levels.csv')
        names_path = os.path.join(self._csv_path, 'names.csv')

        with open(levels_path, 'w') as lf, open(names_path, 'w') as nf:
            levels_writer = csv.DictWriter(lf, fieldnames=('id', 'level'))
            levels_writer.writeheader()
            names_writer = csv.DictWriter(nf, fieldnames=('id', 'object_name'))
            names_writer.writeheader()

            for res in results:
                for id_, level, object_ids_names in res:
                    levels_writer.writerow({'id': id_, 'level': level})
                    for name in object_ids_names:
                        names_writer.writerow({'id': id_, 'object_name': name})

    def _process_single_zip(self, zip_name: str) -> List[Tuple[str, int, List[str]]]:
        zip_path = os.path.join(self._zip_path, zip_name)
        result = []

        with ZipFile(zip_path) as zip_file:
            for name in zip_file.namelist():
                buf = zip_file.read(name)
                root = ET.fromstring(buf.decode())

                vars = root.findall('var')
                if vars[0].get('name') == 'id':
                    id_, level = vars[0].get('value'), int(vars[1].get('value'))
                else:
                    id_, level = vars[1].get('value'), int(vars[0].get('value'))

                objects = root.find('objects')
                object_ids_names = [obj.get('name') for obj in objects]

                result.append((id_, level, object_ids_names))

        return result

def main():
    parser = ArgumentParser()
    parser.add_argument(
        'arch_path', type=str, help='path for arhcives generation')
    parser.add_argument('csv_path', type=str, help='path for csv generation')
    args = parser.parse_args()

    processor = XmlCsvProcessor(
        zip_path=args.arch_path, csv_path=args.csv_path)

    processor.gen_xml_archives()
    processor.gen_csv_files()

if __name__ == '__main__':
    main()


from django.conf import settings
from django.shortcuts import redirect, render

class login_required_middleware:
    """ This middleware takes care of anonymous users trying to visit links that requires an user to be logged-in.
        Also, non-admin users trying to access the admin panel
    """

    admin_base_path = '/admin/'

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        response = self.get_response(request)
        return response

    def process_view(self, request, view_func, view_args, view_kwargs):
        """ process_view() is called just before Django calls the view """

        assert hasattr(request, 'user'), "assertion error in login_required_middleware()"
        path = request.path_info
        if request.user.is_authenticated:
            if not request.user.is_admin():
                if path == self.admin_base_path:
                    return render(request, 'trespass.html', {})
            if path == '/':
                return redirect('/home/')
        else:
            if path not in settings.LOGIN_EXEMPT_URL:
                return redirect(settings.LOGIN_URL)


import io
import logging
import re
import sys
from collections import UserDict, UserList
from collections.abc import Mapping, Sequence

logger = logging.getLogger(__name__)

def deep_update(
    target: Mapping,
    other: Mapping,
    overwrite=True,
    list_insert_index: dict = None,
) -> None:
    if other.get("_do_not_overwrite", False):
        overwrite = False
    for k, v in other.items():
        if isinstance(v, Mapping):
            if k not in target:
                target[k] = dict(v)  # use a copy
            elif isinstance(target[k], Mapping):
                deep_update(target[k], v, overwrite=overwrite)
            else:
                raise ValueError(
                    f"trying to merge key {k} map {v} into non-map {target[k]}"
                )
        elif isinstance(v, Sequence) and not isinstance(v, str):
            if k not in target:
                target[k] = list(v)  # use a copy
            elif isinstance(target[k], Sequence) and not isinstance(target[k], str):
                if list_insert_index and k in list_insert_index:
                    idx = list_insert_index[k]
                    target[k][idx:idx] = v
                else:
                    for item in v:
                        target[k].append(item)
            else:
                raise ValueError(
                    f"trying to merge key {k} sequence {v}"
                    f" into non-sequence {target[k]}"
                )
        else:
            if overwrite or k not in target:
                # this key is special, should find better way
                if k != "_do_not_overwrite":
                    target[k] = v

class DictWrapper(UserDict):
    def __init__(self, dict):
        # do not copy the original dict as the normal UserDict does
        # but wrap the original so that updates go to the original
        self.data = dict

    def __getattr__(self, attr):
        if attr not in self.data:
            raise AttributeError(
                f"could not find attribute {attr} in {self}, {self.data}"
            )
        else:
            return wrap(self.data[attr])

    def __repr__(self):
        return f"DictWrapper({self.data})"

    def pprint_str(self, indent=""):
        output = io.StringIO()
        pprint_map(self, indent=indent, file=output)
        result = output.getvalue()
        output.close
        return result

    def deep_update_path(self, path: str, value):
        keys = path.split(".")
        data = self.data
        for key in keys[:-1]:
            key = key.replace("_dot_", ".")
            if key not in data:
                data[key] = {}
            data = data[key]
        final_key = keys[-1]
        final_key = final_key.replace("_dot_", ".")
        if final_key in data:
            if isinstance(data[final_key], Mapping):
                # Try to merge two Mappings
                if not isinstance(value, Mapping):
                    raise ValueError(
                        f"Can not assign non-dict {value} to"
                        f"dict {data[final_key]} for path {path}"
                    )
                deep_update(data[final_key], value)
            else:
                data[final_key] = value
        elif final_key == "[0]" and isinstance(data, Sequence):
            deep_update(data[0], value)
        else:
            data[final_key] = value

    # TODO: Will be removed in 2.0
    def _set_path(self, path: str, value):
        self.set_path(path, value)

    def set_path(self, path: str, value):
        keys = path.split(".")
        data = self.data
        for key in keys[:-1]:
            key = key.replace("_dot_", ".")
            if key not in data:
                data[key] = {}
            data = data[key]
        final_key = keys[-1]
        final_key = final_key.replace("_dot_", ".")
        if final_key in data:
            if isinstance(data[final_key], Mapping):
                # Try to merge two Mappings
                if not isinstance(value, Mapping):
                    raise ValueError(
                        f"Can not assign non-dict {value} to"
                        f"dict {data[final_key]} for path {path}"
                    )
                data[final_key].update(value)
            else:
                data[final_key] = value
        elif final_key == "[0]" and isinstance(data, Sequence):
            deep_update(data[0], value)
        else:
            data[final_key] = value

    # TODO: Will be removed in 2.0
    def _del_path(self, path: str):
        self.del_path(path)

    def del_path(self, path: str):
        keys = path.split(".")
        data = self.data
        for key in keys[:-1]:
            key = key.replace("_dot_", ".")
            if key not in data:
                logger.warning(f"non existent key {key} in del_path {path}")
                return
            data = data[key]
            while isinstance(data, Sequence):
                # get first and only item of list
                if len(data) == 1:
                    data = data[0]
                else:
                    logger.warning(f"list at {key} in del_path {path}")
                    return
        final_key = keys[-1]
        final_key = final_key.replace("_dot_", ".")
        if final_key in data:
            del data[final_key]
        else:
            logger.warning(f"non existent key {final_key} in del_path {path}")

    def get(self, path: str, default=None):
        return self.get_path(path, default)

    def set(self, path: str, val):
        return self.set_path(path, val)

    # TODO: Will be removed in 2.0
    def _get_path(self, path: str, default=None, mandatory=False):
        return self.get_path(path, default, mandatory)

    def get_path(self, path: str, default=None, mandatory=False):
        keys = path.split(".")
        data = self.data
        for key in keys:
            key = key.replace("_dot_", ".")
            if key == "[0]":
                if isinstance(data, Sequence):
                    if len(data) > 0:
                        data = data[0]
                    elif mandatory:
                        raise ValueError(f"could not find mandatory field {path}")
                    else:
                        return default
                else:
                    raise ValueError(
                        f"trying to get [0] in path {path} of non-sequence {type(data)}"
                    )
            elif key not in data:
                if mandatory:
                    raise ValueError(f"could not find mandatory field {path}")
                return default
            else:
                data = data[key]
        return data

class ListWrapper(UserList):
    def __init__(self, seq) -> None:
        # do not copy the original list as the normal UserList does
        # but wrap the original so that updates go to the original
        self.data = seq

    def __getitem__(self, idx):
        # Wrap the returned value
        return wrap(self._seq[idx])

def wrap(obj):
    if isinstance(obj, str):
        return obj
    if isinstance(obj, Sequence) and not isinstance(obj, ListWrapper):
        return ListWrapper(obj)
    if isinstance(obj, Mapping) and not isinstance(obj, DictWrapper):
        return DictWrapper(obj)
    return obj

def pprint_map(map, indent="", file=None, use_quotes=False):
    if file is None:
        file = sys.stdout
    indent_step = "  "
    if isinstance(map, str):
        print(f"{indent}{map}", file=file)
        return
    elif isinstance(map, Sequence):
        for v in map:
            print(f"{indent}- {v}", file=file)
        return

    if map is None:
        logger.warning("No values present to print")
        return

    for key in sorted(map.keys()):
        val = map.get(key, None)
        if isinstance(val, Mapping):
            if len(val) == 0:
                print(f"{indent}{key}: " + "{}", file=file)
            else:
                print(f"{indent}{key}:", file=file)
                pprint_map(val, indent=indent + indent_step, file=file, use_quotes=use_quotes)
        elif isinstance(val, str):
            if use_quotes:
                print(f'{indent}{key}: "{val}"', file=file)
            else:
                print(f"{indent}{key}: {val}", file=file)
        elif isinstance(val, Sequence):
            print(f"{indent}{key}:", file=file)
            for v in val:
                print(f"{indent}- {v}", file=file)
        else:
            print(f"{indent}{key}: {val}", file=file)

def pprint_tuple(tup, prefix=None, delimiter=".", pattern=None):
    for item in sorted(tup, key=lambda x: x[0]):
        if prefix is None:
            print_filtered(f"{item[0]}={item[1]}", pattern)
        else:
            print_filtered(f"{prefix}{delimiter}{item[0]}={item[1]}", pattern)

def print_filtered(text, pattern):
    if pattern is not None:
        regex_pattern = re.compile(pattern, re.IGNORECASE)
        if regex_pattern.search(text):
            print(text)
    else:
        print(text)


from django.contrib import admin
from django.urls import path, include
from django.conf.urls import url
from booker import views
from django.conf.urls.static import static
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseNotFound
from django.shortcuts import render

urlpatterns = [
    path('admin', admin.site.urls),
    path('', views.Home.as_view(), name='home'),
    path('upcomingcampaigns/', views.BookvenueList.as_view(),
         name='upcoming_campaigns'),
    path('createcampaign/', login_required(views.CreateCampaign.as_view()),
         name='create_campaign'),
    path('dashboard/', views.CampaignList.as_view(), name='dashboard'),
    path('venues/', views.Venue.as_view(), name='venues'),
    path('signup/', views.register_request, name="signup"),
    path('login/', views.login_request, name="login"),
    path('logout/', views.logout_request, name="logout"),
    path('profile/', views.Profile.as_view(), name="profile"),
    path('summernote/', include('django_summernote.urls')),
    path(
        'delete_campaign/<int:id>/',
        login_required(views.DeleteCampaign.as_view()),
        name='delete_campaign'
    ),
    path(
        'edit_campaign/<int:id>/',
        login_required(views.EditCampaign.as_view()),
        name='edit_campaign'
    ),
    path(
        'delete_venue/<int:id>/',
        login_required(views.DeleteVenue.as_view()),
        name='delete_venue'
    ),
    path(
        'edit_venue/<int:id>/',
        login_required(views.EditVenue.as_view()),
        name='edit_venue'
    ),
    path('<path:invalid_path>', views.handler404),  # Catch all invalid URLs
]

handler404 = 'booker.views.handler404'

if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import os.path
from datetime import datetime
from lib.Bocsar import Bocsar
from lib.ExcelParser import ExcelParser

class ImportResponse:
    def __init__(self):
        self.USED_TO_UPDATED_BEFORE = 10
        self.UPDATE_SUCCESSFULLY = 11
        self.LGA_HAS_NO_DATA = 20

    def get(self, lga):
        assumedDirectory = './storage/' + lga.replace(' ', '').lower() + 'lga.xlsx'
        if os.path.isfile(assumedDirectory):
            return self.USED_TO_UPDATED_BEFORE, 'LGA data used to be updated before', \
                   datetime.fromtimestamp(os.path.getmtime(assumedDirectory))
        else:
            bocsar = Bocsar()
            file = bocsar.downloadLgaExcel(lga.replace(' ', '').lower())
            if file:
                parser = ExcelParser(file)
                parser.write()
                return self.UPDATE_SUCCESSFULLY, 'LGA data update successfully', \
                       datetime.fromtimestamp(os.path.getmtime(file))
            else:
                return self.LGA_HAS_NO_DATA, 'LGA has no data', datetime.now()


from time import gmtime, strftime
from bottle import route, get, run, post, request

@post('/get')
def index():
    rulename = request.forms.get('rulename')
    f = open(rulename, 'rb')
    rule = f.read()
    f.close()
    return rule

@post('/put')
def index():
    recivedt = strftime('%Y-%m-%d %H:%M:%S', gmtime())
    rulename = request.forms.get('rulename')
    filename = request.forms.get('filename')
    hostname = request.forms.get('hostname')
    f = open("results.txt", "a")
    f.write("%s,%s,%s,%s\r\n" % (recivedt, hostname, rulename, filename))
    f.close()
    return ""

run(host='192.168.1.68', port=8080)


from db import dbtypes as dbt
from typing import Any, Dict, List, Iterator, Sequence, Type, Union, Optional, get_type_hints

class TableMetadata:
    def __init__(self, table_class):

        type_hints:Dict[str, Any] = get_type_hints(table_class)
        self.table_name = type_hints['__table_name__'] if '__table_name__' in type_hints else table_class.__name__

        # generate columns and foregin keys
        self.columns:Dict[str, dbt.DataTypeBase] = {}
        _key_name:Optional[str] = None
        self.foreign_keys_table:Dict[str, TableBase] = {}
        for name, _type in type_hints.items():
            if name.startswith('__'): continue
            datatype = to_data_type(_type)
            self.columns[name] = datatype

            # If this column is primary key, save this key's name
            if datatype.info.primary:
                if _key_name is not None:
                    raise RuntimeError('Multiple primary key is not allowed.')
                _key_name = name

            # If this column is foreign key, append to list
            if datatype.info.link_table:
                self.foreign_keys_table[name] = datatype.info.link_table

        if _key_name is None:
            raise RuntimeError('Primary key not found.')

        self.key_column_name = _key_name

class TableBase:
    """ Base class of a Table in the database """

    @classmethod
    def metadata(cls):
        if not hasattr(cls, '__metadata__'):
            cls.__metadata__ = TableMetadata(cls)
        return cls.__metadata__

    @classmethod
    def show_create_table(cls) -> str:
        metadata = cls.metadata()
        table_name = metadata.table_name
        columns = metadata.columns
        fk_table = metadata.foreign_keys_table
        return (
            f'CREATE TABLE `{table_name}`(\n  ' + ', \n  '.join((
                *(f'`{name}` {dt.sql_expr}' for name, dt in columns.items()),
                *(
                    f'FOREIGN KEY `fk_{name}` (`{name}`) REFERENCES `{t.metadata().table_name}`(`{t.metadata().key_column_name}`)'
                    for name, t in fk_table.items()
                ),
            )) + '\n);'
        )

    @classmethod
    def select(cls, where, order, count, offset) -> Iterator:
        """ Select records from the database """

    @classmethod
    def select_one(cls, where) -> Iterator:
        """ Select just one record from the database
            If more than one records are found, throw a exception.
        """

    """ Instance methods """

    def load(self) -> None:
        """ Select (the latest version of) this record from the database """

    def save(self) -> None:
        """ Insert or update this record to the database """

    def insert(self) -> None:
        """ Insert this record as new record
            If the record already exists on the database, throw a exception.
        """

    def delete(self) -> None:
        """ Delete this record from the database """

class RawTable(TableBase):
    """ """

class Table(RawTable):
    """ """
    id: dbt.IDKey(auto_increment=True)

class UniqueTable(Table):
    """ """

    @classmethod
    def show_selsert(cls, arg_colnames:Optional[List[str]] = None) -> str:
        metadata = cls.metadata()
        table_name = metadata.table_name
        columns = metadata.columns
        key_name = metadata.key_column_name

        _v_colnames = {name for name in columns.keys() if name != key_name}

        if arg_colnames is not None:
            if len(arg_colnames) != len(set(arg_colnames)):
                raise RuntimeError('Duplicate argument column names.')
            if not all(colname in _v_colnames for colname in arg_colnames):
                raise RuntimeError('Unknown column(s) exist in argument column names.')
            v_colnames = arg_colnames
        else:
            v_colnames = list(_v_colnames)

        return (
             'DELIMITER //\n' +
            f'CREATE FUNCTION `selsert_{table_name}`(  \n' + ', '.join(
                f'v_{name} {columns[name].info.dbtype}' for name in v_colnames
            ) +
             ')\n' +
            f'RETURNS {columns[key_name].info.dbtype} DETERMINISTIC\n' +
             'BEGIN\n' +
               '@ret = (\n' +
                 f'SELECT `{key_name}` FROM `{table_name}`\n' +
                 f'WHERE ' + ' AND '.join(
                f'`{name}` = v_{name}' for name in v_colnames
            ) +
             '  );\n' +
             '  IF @ret IS NOT NULL THEN\n' +
             '    RETURN @ret;\n' +
             '  END IF;\n' +
            f'  INSERT INTO `{key_name}`(' + ', '.join(f'`{name}`' for name in v_colnames) + ') \n' +
             '    VALUES(' + ', '.join(f'v_{name}' for name in v_colnames) + ');\n' +
             '  RETURN LAST_INSERT_ID();\n' +
             'END//\n' +
             'DELIMITER ;\n'
        )
#
#         @classmethod
#         def show_tables_selsert_st(cls, args:Sequence[Union[str, Tuple[str, Any]]]) -> str:
#             metadata = cls.metadata()
#             table_name = metadata.table_name
#             arg_sts = []
#             for arg in args:
#                 if isinstance(arg, str):
#                     st.append(f'v_{arg}')
#                 elif isinstance(arg, tuple) and len(arg) == 2:
#                     st.append()
#
#             return f'selsert_{table_name}(\n  ' + ', \n  '.join(arg_sts) + ');\n'
#

def create_tables(*tables:List[Type]):
    for table in tables:
        # print(f'# {table.__name__}')
        # type_hints = get_type_hints(table)
        # for k, v in type_hints.items():
        #     print(f'    {k}: {v}')
        print(table.show_create_table())
        if issubclass(table, UniqueTable):
            print(table.show_selsert())
        print('')

def to_data_type(t:Any, **options):

    if isinstance(t, type):

        # if `t` is already DataType
        if issubclass(t, dbt.DataTypeBase):
            if options:
                return dbt.DataType(t, **options)
            return t

        if issubclass(t, TableBase):
            return dbt.ForeignKey(t)

    # assume typing.Optional type
    if getattr(t, '__origin__') == Union:
        if len(t.__args__) != 2:
            raise RuntimeError('This union type is not available.')
        if t.__args__[0] == type(None):
            return to_data_type(t.__args__[1], nullable=True)
        elif t.__args__[1] == type(None):
            return to_data_type(t.__args__[0], nullable=True)
        else:
            raise RuntimeError('This union type is not available.')

    raise RuntimeError('Failed to convert to data type.')


import algorithms.linear_reg.linearreg as lr
import algorithms.linear_reg.plotdata as pd
import numpy as np
import matplotlib.pyplot as plt

"""This is the main driver file to test the rest machine learning modules."""

#loads the data1_1.txt into data variable
#The data is a 97 x 2 matrix
data = np.genfromtxt("datas/data1_1.txt", delimiter=',')

#loading data to X and y - the first column corresponds to input and the second column corresponds to output
#X becomes a 97x2 matrix with first column as 1s and y becomes a 97x1 matrix
X=np.c_[np.ones(data.shape[0]),data[:,0]]
y=np.c_[data[:,1]]
print(data)

#plotting X and y
pd.plotData(X, y, 'x-label', 'y-label')
plt.close()

#initialise theta to zeros - theta is a 2 x 1 matrix
#ComputeCost computes the cost of the linear regression
theta =[[0],[0]]
J = lr.ComputeCost(X, y, theta)
print('The cost of the hypothesis created with theta as 0,0 : ')
print(J)

iteration = 1500
alpha = 0.01

#theta is the optimized theta after 1500 iterations and J_plot is the values of 1500 cost with theta after every iteration
#theta is the same 2 x 1 matrix and J_plot has 1500 values in a array
theta, J_plot= lr.gradientDescent(X, y, theta, iteration, alpha)

#plotting the cost of hypothesis in 1500 iterationset
pd.plotDataJ('x-label', 'y-label', J_plot)
#the optimised theta is
print('The optimised theta is : ')
print(theta)

#new cost must decrease
J = lr.ComputeCost(X,y,theta)
print('The new cost after the optimised theta is : ')
print(J)

#plotting the predicted function i.e. theta0*x0 + theta1*x1
#set the range for which hypothesis is to be plotted
range_x = np.arange(30)
#calculating the formula of hypothesis with the new theta
hx = theta[0]+theta[1]*range_x
pd.plotDataH(X, y, 'x-label', 'y-label', hx, range_x)

#using scikit-learn to plot the same and comapring both the values
pd.plotDataSK(X, y, range_x, hx)

#predicting profits - theta(T).x
print(' The predicted value for custom x is : ')
print(theta.T.dot([1, 3.5])*10000)

#plotting cost for each value of theta and plotting a 3D graph of the same
pd.plotDataCOST(X, y)

""" Contour plots are still to be completed"""


import tensorflow as tf
import numpy as np
import core as cr
from buffer import replay_buffer
from tensorboardX import SummaryWriter
from ou_noise import OU_noise
import gym

class SAC:
    def __init__(self):
        self.sess = tf.Session()
        self.state_size = 33
        self.output_size = 4
        self.tau = 0.995
        self.gamma = 0.99
        self.hidden = [400, 300]
        self.batch_size = 64
        self.pi_lr = 1e-3
        self.q_lr = 1e-3
        self.action_limit = 1.0
        self.memory = replay_buffer(1e5)
        self.target_noise = 0.2
        self.noise_clip = 0.1
        self.alpha = 1e-5
        self.num_worker = 20
        self.noise = OU_noise(self.output_size, self.num_worker)

        self.x_ph, self.a_ph, self.x2_ph, self.r_ph, self.d_ph = \
            cr.placeholders(self.state_size, self.output_size, self.state_size, None, None)

        with tf.variable_scope('main'):
            self.mu, self.pi, self.logp_pi, self.q1, self.q2, self.q1_pi, self.q2_pi, self.v = \
                cr.sac_mlp_actor_critic(
                    x=self.x_ph,
                    a=self.a_ph,
                    hidden=self.hidden,
                    activation=tf.nn.relu,
                    output_activation=tf.tanh,
                    output_size=self.output_size,
                    action_limit=self.action_limit
                )
        with tf.variable_scope('target'):
            _, _, _, _, _, _, _, self.v_targ = \
                cr.sac_mlp_actor_critic(
                    x=self.x2_ph,
                    a=self.a_ph,
                    hidden=self.hidden,
                    activation=tf.nn.relu,
                    output_activation=tf.tanh,
                    output_size=self.output_size,
                    action_limit=self.action_limit
                )

        self.pi_params = cr.get_vars('main/pi')
        self.value_params = cr.get_vars('main/q') + cr.get_vars('main/v')

        self.min_q_pi = tf.minimum(self.q1_pi, self.q2_pi)
        self.q_backup = tf.stop_gradient(self.r_ph + self.gamma * (1 - self.d_ph) * self.v_targ)
        self.v_backup = tf.stop_gradient(self.min_q_pi - self.alpha * self.logp_pi)

        self.pi_loss = tf.reduce_mean(self.alpha * self.logp_pi - self.q1_pi)
        self.q1_loss = 0.5 * tf.reduce_mean((self.q_backup - self.q1) ** 2)
        self.q2_loss = 0.5 * tf.reduce_mean((self.q_backup - self.q2) ** 2)
        self.v_loss  = 0.5 * tf.reduce_mean((self.v_backup - self.v) ** 2)
        self.value_loss = self.q1_loss + self.q2_loss + self.v_loss

        self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
        self.train_pi_op = self.pi_optimizer.minimize(self.pi_loss, var_list=self.pi_params)

        self.value_optimizer = tf.train.AdamOptimizer(self.q_lr)
        with tf.control_dependencies([self.train_pi_op]):
            self.train_value_op = self.value_optimizer.minimize(self.value_loss, var_list=self.value_params)

        with tf.control_dependencies([self.train_value_op]):
            self.target_update = tf.group([tf.assign(v_targ, self.tau*v_targ + (1-self.tau)*v_main)
                                  for v_main, v_targ in zip(cr.get_vars('main'), cr.get_vars('target'))])

        self.step_ops = [self.pi_loss, self.q1_loss, self.q2_loss, self.v_loss, self.q1, self.q2,
                         self.v,       self.logp_pi, self.train_pi_op, self.train_value_op, self.target_update]

        self.target_init = tf.group([tf.assign(v_targ, v_main)
                              for v_main, v_targ in zip(cr.get_vars('main'), cr.get_vars('target'))])

        self.sess.run(tf.global_variables_initializer())
        self.sess.run(self.target_init)

    def update(self):
        data = self.memory.get_sample(sample_size=self.batch_size)
        feed_dict = {
            self.x_ph : data['state'],
            self.x2_ph : data['next_state'],
            self.a_ph : data['action'],
            self.r_ph : data['reward'],
            self.d_ph : data['done']
        }

        self.sess.run(self.step_ops, feed_dict=feed_dict)

    def get_action(self, state, deterministic=False):
        act_op = self.mu if deterministic else self.pi
        return self.sess.run(act_op, feed_dict={self.x_ph: [state]})[0]

    def test(self):
        env = gym.make('Pendulum-v0')
        while True:
            state = env.reset()
            done = False
            while not done:
                env.render()
                action = self.get_action(state, 0)
                state, _, done,_ = env.step(action)

    def run(self):
        from mlagents.envs import UnityEnvironment

        writer = SummaryWriter('runs/sac')
        num_worker = self.num_worker
        state_size = self.state_size
        output_size = self.output_size
        ep = 0
        train_size = 5

        env = UnityEnvironment(file_name='env/training', worker_id=1)
        default_brain = env.brain_names[0]
        brain = env.brains[default_brain]
        initial_observation = env.reset()

        step = 0
        start_steps = 100000

        states = np.zeros([num_worker, state_size])
        for i in range(start_steps):
            actions = np.clip(np.random.randn(num_worker, output_size), -self.action_limit, self.action_limit)
            actions += self.noise.sample()
            env_info = env.step(actions)[default_brain]
            next_states = env_info.vector_observations
            rewards = env_info.rewards
            dones = env_info.local_done
            for s, ns, r, d, a in zip(states, next_states, rewards, dones, actions):
                self.memory.append(s, ns, r, d, a)
            states = next_states
            if dones[0]:
                self.noise.reset()
            if i % train_size == 0:
                if len(self.memory.memory) > self.batch_size:
                    self.update()
            print('data storing :', float(i / start_steps))

        while True:
            ep += 1
            states = np.zeros([num_worker, state_size])
            terminal = False
            score = 0
            while not terminal:
                step += 1
                '''
                if step > start_steps:
                    actions = [self.get_action(s) for s in states]
                    action_random = 'False'
                else:
                    actions = np.clip(np.random.randn(num_worker, output_size), -self.action_limit, self.action_limit)
                    action_random = 'True'
                '''
                actions = [self.get_action(s) for s in states]
                action_random = 'False'

                env_info = env.step(actions)[default_brain]

                next_states = env_info.vector_observations
                rewards = env_info.rewards
                dones = env_info.local_done

                terminal = dones[0]

                for s, ns, r, d, a in zip(states, next_states, rewards, dones, actions):
                    self.memory.append(s, ns, r, d, a)

                score += sum(rewards)

                states = next_states

                if len(self.memory.memory) > self.batch_size:
                    if step % train_size == 0:
                        self.update()

            if ep < 1000:
                print('step : ', step, '| start steps : ', start_steps, '| episode :', ep, '| score : ', score, '| memory size', len(self.memory.memory), '| action random : ', action_random)
                writer.add_scalar('data/reward', score, ep)
                writer.add_scalar('data/memory_size', len(self.memory.memory), ep)

if __name__ == '__main__':
    agent = SAC()
    agent.run()


from random import randint
import pdb

from config import NUM_PERSONS, MAX_LOCATION, NUM_ITERATION_PER_DAY, NUM_HOSPITALS, NUM_BEDS_PER_HOSPITAL, NUM_INITIAL_INFECTED, NUM_ITERATIONS_TO_RUN

from living_state import *
from person import Person
from hospital import Hospital
from covid19 import Covid19
from get_stats import get_people_stats
from contact import contact

def create_hospitals():
    all_hospitals = []
    for j in range(NUM_HOSPITALS):
        locations_x = []
        locations_y = []

        hospital_location_x = randint(10, MAX_LOCATION)
        hospital_location_y = randint(10, MAX_LOCATION)

        while (hospital_location_x in locations_x and hospital_location_y in locations_y):
            hospital_location_x = randint(10, MAX_LOCATION)
            hospital_location_y = randint(10, MAX_LOCATION)
        all_hospitals.append(Hospital(j, NUM_BEDS_PER_HOSPITAL, [hospital_location_x, hospital_location_y]))
    return all_hospitals

def create_people():
    all_people = []
    total_infected = 5
    infected_indices = []
    for j in range(total_infected):
        next_infected = randint(0, NUM_PERSONS)
        while(next_infected in infected_indices):
            next_infected = randint(0, NUM_PERSONS)
        infected_indices.append(next_infected)

    for j in range(NUM_PERSONS):
        if j in infected_indices:
            next_person = Person(j, MAX_LOCATION, INFECTED, active_probab = 0.6)
            next_person.infect(Covid19(), 0)
            all_people.append(next_person)
        else:
            next_person = Person(j, MAX_LOCATION, UNINFECTED, active_probab = 0.6)
            all_people.append(next_person)

    return all_people

def run_iters(num_iterations):
    all_hospitals = create_hospitals()
    all_people = create_people()
    all_hospital_full = False

    for i in range(num_iterations):
        for j in range(NUM_PERSONS):
            all_people[j].move()
            all_people[j].are_symptoms_visible(i)
            if all_people[j].shows_symptom:

                # Checking if there is any space left in hospital
                all_hospital_full = True
                for a_hospital in all_hospitals:
                    if not a_hospital.is_full:
                        all_hospital_full = False
                        break

                # If all hospitals are not full then
                if not all_hospital_full:
                    for a_hospital in all_hospitals:
                        if not a_hospital.is_full:
                            hospital_location = a_hospital.location
                            a_hospital.occupy_bed()
                            break
                    all_people[j].hospitalize(hospital_location)

                # otherwise quarantine the person
                else:
                    all_people[j].no_hospital_quarantine()

        for j in range(NUM_PERSONS):
            for k in range(NUM_PERSONS):
                if j != k:
                    contact(all_people[j], all_people[k], i)

        for j in range(NUM_PERSONS):
            if all_people[j].status == NO_HOSPITAL_QUARANTINED or all_people[j].is_hospitalized:
                all_people[j].dead_immune_delay(i)

        get_people_stats(all_people, i)

if __name__ == "__main__":
    run_iters(NUM_ITERATIONS_TO_RUN)


def getProperty(configFile, section, prop):
	try:
		import configparser
		config = configparser.ConfigParser()

		config.read(configFile)

		return config[section][prop]

	except KeyError:
		print ('Configuration file does not exist, or is corrupted. Please create it using helpers/makeconfig.py');
		return ''

import socket

savedir = getProperty('config.ini', 'files', 'savedir')
s = socket.socket()
host = '' # Means all available interfaces
port = 13131
s.bind((host, port))

s.listen(1)

import connection
while True:
	con, addr = s.accept()

	current = connection.Client(con, addr, savedir)
	current.start()

con.close()


from environments.music_world import MusicWorld
from visualizer.music_visualizer import InteractiveComposer

from argparse import ArgumentParser

def main():
  parser: ArgumentParser = ArgumentParser()
  parser.add_argument('--epsilon', type=float, default=0.1, help="randomness in action")
  parser.add_argument('--discount', type=float, default=1, help="learning rate")
  parser.add_argument('--episodes', type=int, default=5000, help="number of training episodes")
  parser.add_argument('--model', type=str, default="", help="loads and persists model in file")
  parser.add_argument('--step', type=int, default=100, help="visualize results after a number of steps")
  parser.add_argument('--interactive_mode', action="store_true", help="interact with user for learning")
  parser.add_argument('--aprox_q_learning', action="store_true", help="use NN for aproximate q-learning")
  parser.add_argument('--batch_size', type=int, default=100, help="size of each NN batch")
  parser.add_argument('--results', action="store_true", help="plays the best result so far")

  args = parser.parse_args()

  env : MusicWorld = MusicWorld(args.interactive_mode)

  viz: InteractiveComposer = InteractiveComposer(env, args.model)

  if(not args.results):
    if (args.aprox_q_learning):
      viz.deep_q_learning(args.epsilon, args.discount, args.batch_size, args.episodes, args.step)
    else:
      viz.q_learning(args.epsilon, args.discount, args.episodes, args.step)
  else:
    viz.greedy_policy_vis(8)

if __name__ == "__main__":
  main()


from numbers import Number
from typing import Dict, Hashable, List, Tuple

class IndexedPriorityQueue:
    def __init__(self):
        self.queue: List[Number] = []
        self.key_index: Dict[Hashable, int] = {}  # key -> index in heap
        self.index_key: Dict[int, Hashable] = {}  # index in heap -> key

    def __bool__(self) -> bool:
        return bool(self.queue)

    def __len__(self) -> int:
        return len(self.queue)

    def __contains__(self, key: Hashable) -> bool:
        return key in self.key_index

    def index(self, key: Hashable) -> int:
        return self.key_index[key]

    def key(self, index: int) -> Hashable:
        return self.index_key[index]

    def priority(self, key: Hashable) -> Number:
        index = self.index(key)
        return self.queue[index]

    def peek(self) -> Tuple[Hashable, Number]:
        if len(self.queue) == 0:
            raise IndexError()

        return self.key(0), self.queue[0]

    def push(self, key: Hashable, priority: Number) -> None:
        if key in self.key_index:
            raise KeyError("Key already exists")

        self.queue.append(priority)

        index = len(self.queue) - 1

        self.key_index[key] = index
        self.index_key[index] = key

        self._maintain_invariant(index)

    def pop(self) -> Tuple[Hashable, Number]:
        if len(self.queue) == 0:
            raise IndexError()

        if len(self.queue) == 1:
            index = 0
            key = self.index_key[index]

            priority = self.queue.pop()

            del self.index_key[index]
            del self.key_index[key]

            return key, priority

        if len(self.queue) > 1:
            index = 0
            key = self.index_key[index]
            priority = self.queue[index]

            last_index = len(self.queue) - 1
            last_key = self.index_key[last_index]
            last_priority = self.queue.pop()

            self.queue[index] = last_priority

            del self.key_index[key]
            del self.index_key[last_index]

            self.index_key[index] = last_key
            self.key_index[last_key] = index

            self._maintain_invariant(0)

            return key, priority

    def delete(self, key: Hashable) -> Tuple[Hashable, Number]:
        index = self.index(key)

        if len(self.queue) == 1:
            return self.pop()

        priority = self.queue[index]

        last_index = len(self.queue) - 1
        last_key = self.index_key[last_index]
        last_priority = self.queue.pop()

        del self.key_index[key]
        del self.index_key[last_index]

        if index != last_index:
            self.queue[index] = last_priority

            self.key_index[last_key] = index
            self.index_key[index] = last_key

            self._maintain_invariant(index)

        return key, priority

    def update(self, key: Hashable, new_priority: Number) -> None:
        index = self.index(key)

        self.queue[index] = new_priority

        self._maintain_invariant(index)

    def _maintain_invariant(self, index: int) -> None:
        self._move_down(index)
        self._move_up(index)

    def _move_up(self, index) -> None:
        parent_index = (index - 1) // 2

        if index > 0 and self.queue[parent_index] > self.queue[index]:
            self._swap(index, parent_index)
            self._move_up(parent_index)

    def _swap(self, index_a: int, index_b: int) -> None:
        key_a = self.index_key[index_a]
        key_b = self.index_key[index_b]

        # Swap priorities in heap
        self.queue[index_a], self.queue[index_b] = (
            self.queue[index_b],
            self.queue[index_a],
        )

        # Swap mappings
        self.key_index[key_a], self.key_index[key_b] = index_b, index_a
        self.index_key[index_a], self.index_key[index_b] = key_b, key_a

    def _move_down(self, index: int) -> None:
        left_child_index = index * 2 + 1

        if (
            left_child_index < len(self.queue)
            and self.queue[left_child_index] < self.queue[index]
        ):
            self._swap(index, left_child_index)
            self._move_down(left_child_index)

        right_child_index = index * 2 + 2

        if (
            right_child_index < len(self.queue)
            and self.queue[right_child_index] < self.queue[index]
        ):
            self._swap(index, right_child_index)
            self._move_down(right_child_index)


from selenium import webdriver
from selenium.webdriver.common.by import By

driver=webdriver.Chrome(executable_path="D:\\Python_Atul\\chromedriver\\chromedriver.exe")

driver.get("https://rahulshettyacademy.com/AutomationPractice/")
validatetext="ATUL"
driver.find_element(by=By.CSS_SELECTOR,value='#name').send_keys(validatetext)
driver.find_element(by=By.CSS_SELECTOR,value="#alertbtn").click()

alert=driver.switch_to.alert
alerttext=alert.text
assert validatetext in alerttext
alert.accept()


import argparse
import re
import logging

"""
This script checks whether the results format for Task 5 is correct.
It also provides some warnings about possible errors.

The correct format of the Task 5 results file is the following:
<line_number> <TAB> <score>

where <line_number> is the number of the claim in the debate
and <score> indicates the degree of 'check-worthiness' of the given line.
"""

_LINE_PATTERN_A = re.compile('^[1-9][0-9]{16,22}\t([-+]?\d*\.\d+|\d+)$')
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)

def check_format(file_path):
    with open(file_path, encoding='UTF-8') as out:
        file_content = out.read().strip()
        for i, line in enumerate(file_content.split('\n')):
            topic_id, tweet_id, score, run_id = line.strip().split('\t')

            if not _LINE_PATTERN_A.match("%s\t%s"%(tweet_id, score)):
                # 1. Check line format.
                logging.error("Wrong line format: {}".format(line))
                return False
            tweet_id = int(tweet_id)
            score = float(score.strip())

    return True

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--pred_file_path", "-p", required=True, help="The absolute path to the file you want to check.", type=str)
    args = parser.parse_args()
    logging.info("Task 5: Checking file: {}".format(args.pred_file_path))
    check_format(args.pred_file_path)


'''
File to be used to test CRUD operations on the api.

The data for the Create operation can be passed as command line
arguments, or simply ignored as defaults are put in place.
'''

import requests, argparse
from json import dumps as _

parser = argparse.ArgumentParser()
parser.add_argument('-n', '--name', help='The name of the Person object to be created through the api')
parser.add_argument('-e', '--email', help = 'The email of the Person object to be created through the api')
parser.add_argument('-b', '--bio', help='The bio of the Person object to be created through the api')

args = parser.parse_args()

if not args.name and not args.email and not args.bio:
	import sys
	print(f"\nThis file comes with a CLI, run `python {sys.argv[0]} -h` to view the help option\n")

# provided name or default value
name = args.name or 'Jonathan Ma'
email = args.email or 'Joma@gmail.com'
bio = args.bio or 'Tech youtuber.'

# endpoints
list_create = 'http://tegarorobi.pythonanywhere.com/api/'
retrieve_update_destroy = f'http://tegarorobi.pythonanywhere.com/api/{name}/'

print(f"Create Person object: {name}")
persons_create = requests.post(list_create, {
	'name':name,
	'email':args.email or 'Joma@gmail.com',
	'bio':args.bio or 'Tech youtuber.'})
if persons_create.status_code == 400:
	import sys
	sys.exit(_(persons_create.json(), indent=4))
print(_(persons_create.json(), indent=4), '\n\n')

print("Person objects' list:")
persons_list = requests.get(list_create)
print(_(persons_list.json(), indent=4), '\n\n')

print(f"Retrieve Person object: {name}")
person_retrieve = requests.get(retrieve_update_destroy)
print(_(person_retrieve.json(), indent=4), '\n\n')

print(f"Update Person object: {name}")
person_update = requests.patch(retrieve_update_destroy, {'email':'updated@domain.com'})
print(_(person_update.json(), indent=4), '\n\n')

print(f"Delete Person obect: {name}")
person_delete = requests.delete(retrieve_update_destroy)
print('\n')

print("Person objects' list:")
persons_list = requests.get(list_create)
print(_(persons_list.json(), indent=4), '\n\n')


from aiogram import types, Dispatcher
from telegram import markups
from aiogram.dispatcher import FSMContext

async def start_command(message: types.Message, state: FSMContext):
    if message.chat.type != 'private':  # start only in private messages
        return
    await state.finish()
    keyboard = markups.get_start_menu()
    await message.answer("👋Добро пожаловать в бот для сбора данных с телеграмм груп!\n"
                         "🧐Через меня вы сможете получить информацию о пользователях",
                         reply_markup=keyboard)

def register_handlers(dp: Dispatcher):
    """Register message handlers"""
    dp.register_message_handler(start_command, commands="start")


from fastapi import APIRouter, Depends, HTTPException, status
import database, models, schemas
from sqlalchemy.orm import Session

get_db = database.get_db

router = APIRouter(
    prefix='/dataentry',
    tags=['dataentry']
)

@router.post('/')
async def data_entry(request: schemas.Data, db: Session = Depends(get_db)):
    new_form_data = models.DataEntry(writer=request.writer, formId=request.formId,
                                   answer=request.answer)
    db.add(new_form_data)
    db.commit()
    db.refresh(new_form_data)
    return HTTPException(status_code=status.HTTP_201_CREATED)

@router.get('/answers')
async def send_all_answers(db: Session = Depends(get_db)):
    answers = db.query(models.DataEntry).all()
    return answers


#! *-* coding: utf-8 *-*
"""reconfig formula after brew upgrade"""

from fabric.api import env
from fabric.operations import put, run
import os

LOCAL_DIR = '/usr/local'
CELLAR_DIR = '/usr/local/Cellar'
SOFTWARE_DIR = '/Users/hqlgree2/Downloads/software'
CONNECTOR_MYSQL = 'mysql-connector-java-5.1.38-bin.jar'

env.hosts = ['localhost']

def check():
    """check /usr/local                => fab check"""
    # ln -s /usr/local/Cellar/apache-drill/version /usr/local/apache-drill
    run('ls -l {}'.format(LOCAL_DIR))

def apache_drill(version):
    """config /usr/local/apache-drill  => fab apache_drill:'1.4.0'"""
    # ln -s /usr/local/Cellar/apache-drill/version /usr/local/apache-drill
    link_from = os.path.join(CELLAR_DIR, 'apache-drill', version)
    link_to = os.path.join(LOCAL_DIR, 'apache-drill')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))
    # copy config files
    cfg_from = './apache-drill'
    cfg_to = os.path.join(link_to, 'libexec/conf')
    for cfg in os.listdir(cfg_from):
        file_i = os.path.join(cfg_from, cfg)
        file_o = os.path.join(cfg_to, cfg)
        put(file_i, file_o)

def apache_spark(version):
    """config /usr/local/apache-spark  => fab apache_spark:'1.6.0'"""
    # ln -s /usr/local/Cellar/apache-drill/version /usr/local/apache-drill
    link_from = os.path.join(CELLAR_DIR, 'apache-spark', version)
    link_to = os.path.join(LOCAL_DIR, 'apache-spark')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))
    # copy config files
    cfg_from = './apache-spark'
    cfg_to = os.path.join(link_to, 'libexec/conf')
    for cfg in os.listdir(cfg_from):
        file_i = os.path.join(cfg_from, cfg)
        file_o = os.path.join(cfg_to, cfg)
        put(file_i, file_o)

def hadoop(version, use_yarn):
    """config /usr/local/hadoop        => fab hadoop:'2.7.1',True"""
    # ln -s /usr/local/Cellar/hadoop/version /usr/local/hadoop
    link_from = os.path.join(CELLAR_DIR, 'hadoop', version)
    link_to = os.path.join(LOCAL_DIR, 'hadoop')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))
    # copy config files
    cfg_from = './hadoop/' + ('yarn' if use_yarn else 'mapred')
    cfg_to = os.path.join(link_to, 'libexec/etc/hadoop')
    for cfg in os.listdir(cfg_from):
        file_i = os.path.join(cfg_from, cfg)
        file_o = os.path.join(cfg_to, cfg)
        put(file_i, file_o)

def hbase(version):
    """config /usr/local/hbase         => fab hbase:'1.1.2'"""
    # ln -s /usr/local/Cellar/hbase/version /usr/local/hbase
    link_from = os.path.join(CELLAR_DIR, 'hbase', version)
    link_to = os.path.join(LOCAL_DIR, 'hbase')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))
    # copy config files
    cfg_from = './hbase/'
    cfg_to = os.path.join(link_to, 'libexec/conf')
    for cfg in os.listdir(cfg_from):
        file_i = os.path.join(cfg_from, cfg)
        file_o = os.path.join(cfg_to, cfg)
        put(file_i, file_o)

def hive(version):
    """config /usr/local/hive          => fab hive:'1.2.1'"""
    # ln -s /usr/local/Cellar/hive/version /usr/local/hive
    link_from = os.path.join(CELLAR_DIR, 'hive', version)
    link_to = os.path.join(LOCAL_DIR, 'hive')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))
    # copy config files
    cfg_from = './hive/'
    cfg_to = os.path.join(link_to, 'libexec/conf')
    for cfg in os.listdir(cfg_from):
        file_i = os.path.join(cfg_from, cfg)
        file_o = os.path.join(cfg_to, cfg)
        put(file_i, file_o)
    # copy connector(mysql)
    copy_to = os.path.join(link_to, 'libexec/lib', 'mysql.*.jar')
    run('rm -f {}'.format(copy_to))
    copy_form = os.path.join(SOFTWARE_DIR, CONNECTOR_MYSQL)
    copy_to = os.path.join(link_to, 'libexec/lib', CONNECTOR_MYSQL)
    put(copy_form, copy_to)

def kafka(version):
    """config /usr/local/kafka         => fab kafka:'0.8.2.2'"""
    # ln -s /usr/local/Cellar/kafka/version /usr/local/kafka
    link_from = os.path.join(CELLAR_DIR, 'kafka', version)
    link_to = os.path.join(LOCAL_DIR, 'kafka')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))
    # copy config files
    cfg_from = './kafka/'
    cfg_to = os.path.join(link_to, 'libexec/conf')
    for cfg in os.listdir(cfg_from):
        file_i = os.path.join(cfg_from, cfg)
        file_o = os.path.join(cfg_to, cfg)
        put(file_i, file_o)

def solr(version):
    """config /usr/local/solr          => fab solr:'5.4.0'"""
    # ln -s /usr/local/Cellar/solr/version /usr/local/solr
    link_from = os.path.join(CELLAR_DIR, 'solr', version)
    link_to = os.path.join(LOCAL_DIR, 'solr')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))

def sqoop(version):
    """config /usr/local/sqoop         => fab sqoop:'1.4.6'"""
    # ln -s /usr/local/Cellar/sqoop/version /usr/local/sqoop
    link_from = os.path.join(CELLAR_DIR, 'sqoop', version)
    link_to = os.path.join(LOCAL_DIR, 'sqoop')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))

def tomcat(version):
    """config /usr/local/tomcat        => fab tomcat:'8.0.30'"""
    # ln -s /usr/local/Cellar/tomcat/version /usr/local/tomcat
    link_from = os.path.join(CELLAR_DIR, 'tomcat', version)
    link_to = os.path.join(LOCAL_DIR, 'tomcat')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))

def zookeeper(version):
    """config /usr/local/zookeeper     => fab zookeeper:'3.4.7'"""
    # ln -s /usr/local/Cellar/zookeeper/version /usr/local/zookeeper
    link_from = os.path.join(CELLAR_DIR, 'zookeeper', version)
    link_to = os.path.join(LOCAL_DIR, 'zookeeper')
    run('rm -f {}'.format(link_to))
    run('ln -s {} {}'.format(link_from, link_to))
    # copy config files
    cfg_from = './zookeeper/'
    cfg_to = os.path.join(LOCAL_DIR, 'etc/zookeeper')
    for cfg in os.listdir(cfg_from):
        file_i = os.path.join(cfg_from, cfg)
        file_o = os.path.join(cfg_to, cfg)
        put(file_i, file_o)


from game_classes import Character, Weapon
import json, os

# This will save the players character data (Only to be used for one player it will overwrite all data in the JSON file.) light_damage, _heavy_damage, level, xp
def save_character(player):
    player_stats = {"Name": player.name, "Health": player.health, "Race": player.race, "Level": player.level, "Weapon": {"Light": player.weapon.light_damage, "Heavy": player.weapon.heavy_damage, "Level": player.weapon.level, "XP": player.weapon.xp}}
    with open("player_data.json", "w") as player_data:
        json.dump(player_stats, player_data)

# This will load the player character data from a JSON file.
def get_saved_character():
    choice = ''
    character = None
    if os.path.isfile("player_data.json"):
        with open("player_data.json", "r") as player_data:
            character = json.load(player_data)

        while choice != '1' and choice != '3':
            choice = input(f'Available saved charater: {character["Name"]}, Level {character["Level"]}.\nPress 1 to play a new game, Press 3 to load this character.')
    else:
        choice = '1'
        print('No saved character data found.\nStarting a new game.......')
    return choice, character

# This function will Load a players character and weapon from a dictionary of characters data.
def load_character(data):
    saved_weapon = Weapon(data['Weapon']['Light'], data['Weapon']['Heavy'], data['Weapon']['Level'], data['Weapon']['XP'])
    saved_player = Character(data['Name'], data['Health'], data['Race'], data['Level'], saved_weapon)
    return saved_player


from datetime import datetime

from ..call_logs import CallLogsCommand
from hamcrest import assert_that
from hamcrest import equal_to
from xivo_lib_rest_client.tests.command import RESTCommandTestCase

class TestCallLogs(RESTCommandTestCase):

    Command = CallLogsCommand

    csvdata = u"Call Date,Caller,Called,Period,user Field\r\n2015-06-29T12:01:00.725871,John (1000),1234567890,0,\r\n"

    def test_list(self):
        self.session.get.return_value = self.new_response(200, body=self.csvdata)

        result = self.command.list()

        self.session.get.assert_called_once_with(self.base_url,
                                                 params={},
                                                 headers={'Accept': 'text/csv'})
        assert_that(result, equal_to(self.csvdata))

    def test_list_with_dates(self):
        self.session.get.return_value = self.new_response(200, body=self.csvdata)

        expected_params = {
            'start_date': '2015-01-01T12:13:14',
            'end_date': '2015-01-02T12:13:14',
        }

        self.command.list(start_date=datetime(2015, 1, 1, 12, 13, 14),
                          end_date=datetime(2015, 1, 2, 12, 13, 14))

        self.session.get.assert_called_once_with(self.base_url,
                                                 params=expected_params,
                                                 headers={'Accept': 'text/csv'})

    def test_when_not_200(self):
        self.session.get.return_value = self.new_response(404)

        self.assertRaisesHTTPError(self.command.list)


import config
import requests

urlLine = 'https://notify-api.line.me/api/notify'
token = config.LINE_NOTIFY_TOKEN
headers = {
            'content-type':
            'application/x-www-form-urlencoded',
            'Authorization':'Bearer '+token
           }

def send_alert(msg = 'ทดสอบ'):
    r = requests.post(urlLine, headers=headers , data = {'message':msg})
    print(r.text)

def send_pic(path='',msg ='BUY'):
    file = {'imageFile':open(path+'.png','rb')}
    data = ({
            'message':msg
        })
    LINE_HEADERS = {"Authorization":"Bearer "+token}
    session = requests.Session()
    r=session.post(urlLine, headers=LINE_HEADERS, files=file, data=data)
    print(r.text)


import os
import shutil
import math
import warnings
import numpy as np
import pandas as pd
from GlobalParameters import DefaultParam
from Preprocess import Utilities
from python_speech_features import sigproc
from FeatureExtraction import ITheoryDomain, WaveletDomain, FrequencyDomain, TimeDomain
warnings.filterwarnings("ignore")
# Overriding the paths
DefaultParam.DATA_PATH = os.path.join(os.path.dirname(os.getcwd()), 'Storage', 'RawDataGenuine')
DefaultParam.FEATURE_PATH = os.path.join(os.path.dirname(os.getcwd()), 'Storage', 'FeatureFilesGenuine')

USER_LIST = os.listdir(DefaultParam.DATA_PATH)
USER_LIST = sorted(USER_LIST)
INVALID_USER_LIST = []
VALID_USER_LIST = set(USER_LIST) - set(INVALID_USER_LIST)
# print('Total user in VALID_USER_LIST: ', len(VALID_USER_LIST))
# Sorting the list of users for the sake of keeping track how many finished
VALID_USER_LIST = sorted(VALID_USER_LIST)
print('VALID_USER_LIST',VALID_USER_LIST)
for USER in VALID_USER_LIST:  # For all user
    # print('Working on ', USER)
    # Creatin a folder for the current user in thr feature_files location
    USER_PATH_IN = os.path.join(DefaultParam.DATA_PATH, USER)
    USER_PATH_OUT = os.path.join(DefaultParam.FEATURE_PATH, USER)
    if os.path.exists(USER_PATH_OUT):
        raise FileExistsError("The output directory already exists.. delete manually")
    else:
        os.mkdir(USER_PATH_OUT)
        print(f'###############################Created a new directory!')

    DefaultParam.MODES = ['Training', 'Testing']
    for MODE in DefaultParam.MODES:  # For training and testing
        MODE_PATH_IN = os.path.join(DefaultParam.DATA_PATH, USER, MODE)
        MODE_PATH_OUT = os.path.join(DefaultParam.FEATURE_PATH, USER, MODE)
        FREQ_PATH = os.path.join(DefaultParam.FEATURE_PATH, USER, MODE, 'FrequencyFeatures')
        IT_PATH = os.path.join(DefaultParam.FEATURE_PATH, USER, MODE, 'ITheoryFeatures')
        TIME_PATH = os.path.join(DefaultParam.FEATURE_PATH, USER, MODE, 'TimeFeatures')
        WAVE_PATH = os.path.join(DefaultParam.FEATURE_PATH, USER, MODE, 'WaveletFeatures')
        # Create the feature director if it does not exists
        try:  # Create target Directory
            os.mkdir(MODE_PATH_OUT)
            os.mkdir(FREQ_PATH)
            os.mkdir(IT_PATH)
            os.mkdir(TIME_PATH)
            os.mkdir(WAVE_PATH)
            # print("Feature directory created at " + FREQ_PATH)
        except FileExistsError:
            print("Directories already exists")
        for SENSOR in DefaultParam.SENSOR_FILE_LIST:  # for all sensors
            CURR_SENSOR_PATH_IN = os.path.join(MODE_PATH_IN, SENSOR)
            CS_FREQ_PATH = os.path.join(FREQ_PATH, 'feat_' + SENSOR.replace(".txt",".csv"))
            CS_IT_PATH = os.path.join(IT_PATH, 'feat_' + SENSOR.replace(".txt",".csv"))
            CS_TIME_PATH = os.path.join(TIME_PATH, 'feat_' + SENSOR.replace(".txt",".csv"))
            CS_WAVE_PATH = os.path.join(WAVE_PATH, 'feat_' + SENSOR.replace(".txt",".csv"))
            # print('CURR_SENSOR_PATH',CURR_SENSOR_PATH_IN)

            # The old data had different sampling rates.. here computing the sampling rate dynamically
            if USER in DefaultParam.OLD_USER_LIST:
                RAW_DATA = pd.read_csv(CURR_SENSOR_PATH_IN, sep=',', header=None, names=DefaultParam.OLD_COLUMN_NAMES)
                samplerate = Utilities.get_sampling_rate(RAW_DATA)
            else:
                RAW_DATA = pd.read_csv(CURR_SENSOR_PATH_IN, sep=',', header=None, names=DefaultParam.NEW_COLUMN_NAMES)
                samplerate = DefaultParam.NEW_SAMPLING_RATE
            print('sampling rate for ' + CURR_SENSOR_PATH_IN + ': ' + str(samplerate))

            ######################################################################
            signalX = RAW_DATA['x']
            signalY = RAW_DATA['y']
            signalZ = RAW_DATA['z']

            # Sliding window based cutting of the signal
            framesX = sigproc.framesig(signalX, DefaultParam.WINLENGTH * samplerate, DefaultParam.WINSTEP * samplerate)
            framesY = sigproc.framesig(signalY, DefaultParam.WINLENGTH * samplerate, DefaultParam.WINSTEP * samplerate)
            framesZ = sigproc.framesig(signalZ, DefaultParam.WINLENGTH * samplerate, DefaultParam.WINSTEP * samplerate)

            if framesX.shape[1] > 64 and framesX.shape[1] < 128:
                FFTLENGTH = 128
            elif framesX.shape[1] > 128 and framesX.shape[1] < 256:
                FFTLENGTH = 256
            else:
                FFTLENGTH = 512

            # The  sigproc.framesig function fills zeroes in the last unavailable points
            # we dont want that
            framesX = framesX[:-1, :]
            framesY = framesY[:-1, :]
            framesZ = framesZ[:-1, :]

            fmatrix_freq = []
            fmatrix_itheory = []
            fmatrix_time = []
            fmatrix_wavelet = []

            for frameX, frameY, frameZ in zip(framesX, framesY, framesZ):
                ############SMOOTHING THE DATA BEFORE ANY FEATURE EXTRACTION
                ############EVEN BEFORE COMPUTATION OF THE M -- ELSE ERROR PROPOGATE
                # smoothing the signal with 5% of total obtained in every second
                percent_smooth = 0.05
                smoothing_span = math.ceil(samplerate * percent_smooth + 1)
                framesX = Utilities.moving_average(frameX, smoothing_span)
                framesY = Utilities.moving_average(frameY, smoothing_span)
                framesZ = Utilities.moving_average(frameZ, smoothing_span)

                frameM = np.sqrt(frameX*frameX + np.multiply(frameY, frameY) + np.multiply(frameZ, frameZ))

                fdnamesX, fdfvectorX = FrequencyDomain.getall_fd_features(frameX, DefaultParam.NUM_BINS_FOR_FFT,
                                                                          samplerate, DefaultParam.WINLENGTH,
                                                                          DefaultParam.WINSTEP, FFTLENGTH)
                fdnamesY, fdfvectorY = FrequencyDomain.getall_fd_features(frameY, DefaultParam.NUM_BINS_FOR_FFT,
                                                                          samplerate, DefaultParam.WINLENGTH,
                                                                          DefaultParam.WINSTEP, FFTLENGTH)
                fdnamesZ, fdfvectorZ = FrequencyDomain.getall_fd_features(frameZ, DefaultParam.NUM_BINS_FOR_FFT,
                                                                          samplerate, DefaultParam.WINLENGTH,
                                                                          DefaultParam.WINSTEP, FFTLENGTH)
                fdnamesM, fdfvectorM = FrequencyDomain.getall_fd_features(frameM, DefaultParam.NUM_BINS_FOR_FFT,
                                                                          samplerate, DefaultParam.WINLENGTH,
                                                                          DefaultParam.WINSTEP, FFTLENGTH)

                itdnamesX, itdfvectorX = ITheoryDomain.getall_itd_features(frameX)
                itdnamesY, itdfvectorY = ITheoryDomain.getall_itd_features(frameY)
                itdnamesZ, itdfvectorZ = ITheoryDomain.getall_itd_features(frameZ)
                itdnamesM, itdfvectorM = ITheoryDomain.getall_itd_features(frameM)

                tdnamesX, tdfvectorX = TimeDomain.getall_td_feature(frameX)
                tdnamesY, tdfvectorY = TimeDomain.getall_td_feature(frameY)
                tdnamesZ, tdfvectorZ = TimeDomain.getall_td_feature(frameZ)
                tdnamesM, tdfvectorM = TimeDomain.getall_td_feature(frameM)

                wdfnamesX, wdfvectorX = WaveletDomain.wavelet_features(frameX, DefaultParam.MOTHER_WAVELET,
                                                                       DefaultParam.WAVEDEC_LEVEL)
                wdfnamesY, wdfvectorY = WaveletDomain.wavelet_features(frameY, DefaultParam.MOTHER_WAVELET,
                                                                       DefaultParam.WAVEDEC_LEVEL)
                wdfnamesZ, wdfvectorZ = WaveletDomain.wavelet_features(frameZ, DefaultParam.MOTHER_WAVELET,
                                                                       DefaultParam.WAVEDEC_LEVEL)
                wdfnamesM, wdfvectorM = WaveletDomain.wavelet_features(frameM, DefaultParam.MOTHER_WAVELET,
                                                                       DefaultParam.WAVEDEC_LEVEL)

                finalfdfvs = fdfvectorX + fdfvectorY + fdfvectorZ + fdfvectorM
                finalitdfvs = itdfvectorX + itdfvectorY + itdfvectorZ + itdfvectorM
                finaltdfvs = tdfvectorX + tdfvectorY + tdfvectorZ + tdfvectorM
                finalwfdfvs = wdfvectorX + wdfvectorY + wdfvectorZ + wdfvectorM

                fmatrix_freq.append(finalfdfvs)
                fmatrix_itheory.append(finalitdfvs)
                fmatrix_time.append(finaltdfvs)
                fmatrix_wavelet.append(finalwfdfvs)

            fdnames = fdnamesX + fdnamesY + fdnamesZ + fdnamesM
            itdnames = itdnamesX + itdnamesY + itdnamesZ + itdnamesM
            tdnames = tdnamesX + tdnamesY + tdnamesZ + tdnamesM
            wdfnames = wdfnamesX + wdfnamesY + wdfnamesZ + wdfnamesM
            feature_names = fdnames + itdnames + tdnames + wdfnames

            fmatrix_freq = pd.DataFrame(fmatrix_freq, columns=fdnames)
            fmatrix_itheory = pd.DataFrame(fmatrix_itheory, columns=itdnames)
            fmatrix_time = pd.DataFrame(fmatrix_time, columns=tdnames)
            fmatrix_wavelet = pd.DataFrame(fmatrix_wavelet, columns=wdfnames)
            #
            # fmatrix_freq.to_csv(CS_FREQ_PATH)
            # fmatrix_itheory.to_csv(CS_IT_PATH)
            # fmatrix_time.to_csv(CS_TIME_PATH)
            # fmatrix_wavelet.to_csv(CS_WAVE_PATH)


import os
import warnings
from collections import namedtuple

import numpy as np
import torch
from PIL import Image

from kaolin.io.materials import MaterialLoadError, MaterialFileError, MaterialNotFoundError, \
    process_materials_and_assignments
from kaolin.io import utils

__all__ = [
    'ignore_error_handler',
    'skip_error_handler',
    'create_missing_materials_error_handler',
    'default_error_handler',
    'import_mesh'
]

return_type = namedtuple('return_type',
                         ['vertices', 'faces', 'uvs', 'face_uvs_idx', 'materials',
                          'material_assignments', 'normals', 'face_normals_idx'])

def ignore_error_handler(error, **kwargs):
    """Simple error handler to use in :func:`load_obj` that ignore all errors"""
    pass

def skip_error_handler(error, **kwargs):
    """Simple error handler to use in :func:`load_obj` that skips all errors
    and logs them as warnings."""
    warnings.warn(error.args[0], UserWarning)

def create_missing_materials_error_handler(error, **kwargs):
    """Error error_handler to be provided to obj.read_mesh that can handle MaterialNotFound error,
    returning a dummy material with a random diffuse color instead. Material will contain
    an additional "error" field. MaterialFileError and MaterialLoadError will print a warning
    and be ignored."""
    if type(error) == MaterialNotFoundError:
        warnings.warn(f'{error.args[0]}, creating dummy material instead', UserWarning)
        return {'Ka': torch.rand((3,)), 'error': f'Dummy material created for missing material: {error}'}
    elif type(error) in [MaterialFileError, MaterialLoadError]:
        warnings.warn(error.args[0], UserWarning)
    else:
        raise error

def default_error_handler(error, **kwargs):
    """Simple error handle to use in :func:`load_obj` that raises all errors."""
    raise error

def flatten_feature(feature):
    """Flatten the nested list of a feature.
    """
    if feature is None or len(feature) == 0:
        return None
    else:
        return [item for sublist in feature for item in sublist]

# TODO(cfujitsang): support https://en.wikipedia.org/wiki/Wavefront_.obj_file#Geometric_vertex ?
def import_mesh(path, with_materials=False, with_normals=False,
                error_handler=None, heterogeneous_mesh_handler=None,
                triangulate=False):
    r"""Load data from an obj file as a single mesh.

    With limited materials support to Kd, Ka, Ks, map_Kd, map_Ka and map_Ks.
    Followed format described in: http://paulbourke.net/dataformats/obj/

    Args:
        path (str): path to the obj file (with extension).
        with_materials (bool): if True, load materials. Default: False.
        with_normals (bool): if True, load vertex normals. Default: False.
        error_handler (Callable, optional):
            function that handles errors that can be raised (see raised errors, except `NonHomogeneousMeshError`
            handled separately), with the signature ``error_handler(error: Exception, **kwargs)``.
            Handler can provide special treatment of :class:`MaterialNotFoundError`,
            returning a dummy material dictionary instead (if this is not the case, assignments to
            non-existent materials will be lost). For options see:
            :func:`create_missing_materials_error_handler`, :func:`skip_error_handler`, :func:`ignore_error_handler`,
            and :func:`default_error_handler` (**Default** is to raise all errors).
        heterogeneous_mesh_handler (Callable, optional):
            function that handles a heterogeneous mesh, homogenizing, returning None or throwing error,
            with the following signature:
            ``heterogeneous_mesh_handler(vertices, face_vertex_counts, *args, face_assignments)``
            for example, see :func:`mesh_handler_naive_triangulate <kaolin.io.utils.mesh_handler_naive_triangulate>`
            and :func:`heterogeneous_mesh_handler_skip <kaolin.io.utils.heterogeneous_mesh_handler_skip>`.
            Default: will raise a NonHomogeneousMeshError.
        triangulate: if True, will triangulate all non-triangular meshes using same logic as
            :func:`mesh_handler_naive_triangulate <kaolin.io.utils.mesh_handler_naive_triangulate>`.

    Returns:
        (obj.return_type):
            namedtuple of:

            - **vertices** (torch.Tensor): vertex locations of shape :math:`(\text{num_vertices}, 3)`.
            - **faces** (torch.LongTensor): indices into vertex array
              of shape :math:`(\text{num_faces}, \text{face_size})`.
            - **uvs** (torch.Tensor): UV map coordinates of shape :math:`(\text{num_uvs}, 2)`.
            - **face_uvs_idx** (torch.LongTensor): indices into UVmap for every vertex of every face
              of shape :math:`(\text{num_faces}, \text{face_size})`.
            - **materials** (list of dict):
              a list of materials (see return values of :func:`load_mtl`) sorted by their `material_name`.
            - **material_assignments** (dict of torch.LongTensor): (torch.ShortTensor): of shape `(\text{num_faces},)`
                containing index of the material (in the `materials` list) assigned to the corresponding face,
                or `-1` if no material was assigned.
            - **normals** (torch.Tensor): normal values of shape :math:`(\text{num_normals}, 3)`.
            - **face_normals_idx** (torch.LongTensor): indices into the normal array for every vertex
              of every face, of shape :math:`(\text{num_faces}, \text{face_size})`.

    Raises:
        MaterialNotFoundError:
            The .obj is using a material not parsed from material libraries (set `error_handler` to skip).
        MaterialFileError:
            From :func:`load_mtl`: Failed to open material path (set `error_handler` to skip).
        MaterialLoadError:
            From :func:`load_mtl`: Failed to load material, very often due to path to
            map_Kd/map_Ka/map_ks being invalid (set `error_handler` to skip).
        NonHomogeneousMeshError:
            The number of vertices were not equal for all faces (set `heterogeneous_mesh_handler` to handle).
    """
    triangulate_handler = None if not triangulate else utils.mesh_handler_naive_triangulate

    if error_handler is None:
        error_handler = default_error_handler
    vertices = []
    faces = []
    uvs = []
    # 3 values per face
    face_uvs_idx = []
    normals = []
    # 3 values per face
    face_normals_idx = []

    # materials_dict contains:
    #   {material_name: {properties dict}}
    materials_dict = {}

    # material_assignments contain:
    #    {material_name: [(face_idx_start, face_idx_end], (face_idx_start, face_idx_end])
    material_assignments_dict = {}
    material_faceidx_start = None
    active_material_name = None

    def _maybe_complete_material_assignment():
        if active_material_name is not None:
            if material_faceidx_start != len(face_uvs_idx):  # Only add if at least one face is assigned
                material_assignments_dict.setdefault(active_material_name, []).append(
                    torch.LongTensor([material_faceidx_start, len(face_uvs_idx)]))

    with open(path, 'r', encoding='utf-8') as f:
        for line in f:
            data = line.split()
            if len(data) == 0:
                continue
            if data[0] == 'v':
                vertices.append(data[1:])
            elif with_materials and data[0] == 'vt':
                uvs.append(data[1:3])
            elif with_normals and data[0] == 'vn':
                normals.append(data[1:])
            elif data[0] == 'f':
                data = [da.split('/') for da in data[1:]]
                faces.append([int(d[0]) for d in data])
                if with_materials:
                    if len(data[1]) > 1 and data[1][1] != '':
                        face_uvs_idx.append([int(d[1]) for d in data])
                    else:
                        face_uvs_idx.append([0] * len(data))
                if with_normals:
                    if len(data[1]) > 2:
                        face_normals_idx.append([int(d[2]) for d in data])
                    else:
                        face_normals_idx.append([0] * len(data))
            elif with_materials and data[0] == 'usemtl':
                _maybe_complete_material_assignment()
                active_material_name = data[1]
                material_faceidx_start = len(face_uvs_idx)
            elif with_materials and data[0] == 'mtllib':
                mtl_path = os.path.join(os.path.dirname(path), data[1])
                materials_dict.update(load_mtl(mtl_path, error_handler))

    _maybe_complete_material_assignment()

    vertices = torch.FloatTensor([float(el) for sublist in vertices for el in sublist]).view(-1, 3)
    face_vertex_counts = torch.IntTensor([len(f) for f in faces])
    # key: (Nx2) tensor of (start, end faceidx]
    material_assignments_dict = {k: torch.stack(v) for k, v in material_assignments_dict.items()}

    def _apply_handler(handler):
        all_features = [faces, face_uvs_idx, face_normals_idx]
        # Flatten all features
        all_features = [flatten_feature(f) for f in all_features]
        return handler(vertices, face_vertex_counts, *all_features, face_assignments=material_assignments_dict)

    # Handle non-homogeneous meshes
    is_heterogeneous = not torch.all(face_vertex_counts == face_vertex_counts[0])
    if is_heterogeneous:
        if heterogeneous_mesh_handler is None:
            raise utils.NonHomogeneousMeshError(f'Mesh is non-homogeneous '
                                                f'and cannot be imported from {path}.'
                                                f'User can set heterogeneous_mesh_handler.'
                                                f'See kaolin.io.utils for the available options')

        mesh = _apply_handler(heterogeneous_mesh_handler)
        if mesh is None:
            warnings.warn(f'Heterogeneous mesh at path {path} not converted by the handler; returning None.')
            return None
        vertices, face_vertex_counts, faces, face_uvs_idx, face_normals_idx, material_assignments_dict = mesh

    if triangulate_handler is not None and not torch.all(face_vertex_counts == 3):
        mesh = _apply_handler(triangulate_handler)
        if mesh is None:
            warnings.warn(f'Non-triangular mesh at path {path} not triangulated; returning None.')
            return None
        vertices, face_vertex_counts, faces, face_uvs_idx, face_normals_idx, material_assignments_dict = mesh

    faces = torch.LongTensor(faces) - 1

    if with_materials:
        uvs = torch.FloatTensor([float(el) for sublist in uvs
                                 for el in sublist]).view(-1, 2)
        face_uvs_idx = torch.LongTensor(face_uvs_idx) - 1
        materials, material_assignments = process_materials_and_assignments(
            materials_dict, material_assignments_dict, error_handler, faces.shape[0], error_context_str=path)
    else:
        uvs = None
        face_uvs_idx = None
        materials = None
        material_assignments = None

    if with_normals:
        normals = torch.FloatTensor(
            [float(el) for sublist in normals
             for el in sublist]).view(-1, 3)
        face_normals_idx = torch.LongTensor(face_normals_idx) - 1
    else:
        normals = None
        face_normals_idx = None

    return return_type(vertices, faces, uvs, face_uvs_idx, materials,
                       material_assignments, normals, face_normals_idx)

def load_mtl(mtl_path, error_handler):
    """Load and parse a Material file.

    Followed format described in: https://people.sc.fsu.edu/~jburkardt/data/mtl/mtl.html.
    Currently only support diffuse, ambient and specular parameters (Kd, Ka, Ks)
    through single RGB values or texture maps.

    Args:
        mtl_path (str): Path to the mtl file.

    Returns:
        (dict):
            Dictionary of materials, which are dictionary of properties with optional torch.Tensor values:

            - **Kd**: diffuse color of shape (3)
            - **map_Kd**: diffuse texture map of shape (H, W, 3)
            - **Ks**: specular color of shape (3)
            - **map_Ks**: specular texture map of shape (H', W', 3)
            - **Ka**: ambient color of shape (3)
            - **map_Ka**: ambient texture map of shape (H'', W'', 3)
            - **material_name**: string name of the material

    Raises:
        MaterialFileError:
            Failed to open material path.
        MaterialLoadError:
            Failed to load material, very often due to path to map_Kd/map_Ka/map_Ks being invalid.
    """
    mtl_data = {}
    root_dir = os.path.dirname(mtl_path)

    try:
        f = open(mtl_path, 'r', encoding='utf-8')
    except Exception as e:
        error_handler(MaterialFileError(
            f"Failed to load material at path '{mtl_path}':\n{e}"),
            mtl_path=mtl_path, mtl_data=mtl_data)
    else:
        for line in f.readlines():
            data = line.split()
            if len(data) == 0:
                continue
            try:
                if data[0] == 'newmtl':
                    material_name = data[1]
                    mtl_data[material_name] = {'material_name': material_name}
                elif data[0] in {'map_Kd', 'map_Ka', 'map_Ks'}:
                    texture_path = os.path.join(root_dir, data[1])
                    img = Image.open(texture_path)
                    if img.mode != 'RGB':
                        img = img.convert('RGB')
                    mtl_data[material_name][data[0]] = torch.from_numpy(
                        np.array(img))
                elif data[0] in {'Kd', 'Ka', 'Ks'}:
                    mtl_data[material_name][data[0]] = torch.tensor(
                        [float(val) for val in data[1:]])
            except Exception as e:
                error_handler(MaterialLoadError(
                    f"Failed to load material at path '{mtl_path}':\n{e}"),
                    data=data, mtl_data=mtl_data)
        f.close()
    return mtl_data


import pandas as pd
import torch

from dataset import Dataset
from model import LSTMStockPriceModel

if __name__ == '__main__':
    batch_size = 16
    lr = 0.0001
    epochs = 100

    # set device to cuda if available
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

    print('Set up Data Loader...')
    df = pd.read_csv('../data/stocks_prices_prep.csv', sep=';')#.sample(frac=1).head(4)
    train_set = Dataset(df, seq_len=30)
    train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=False, drop_last=True)
    print(f'Series length: {len(train_set)}')

    print('Loaded model to device...')
    model = LSTMStockPriceModel().float()
    model = model.to(device)

    total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
    print(f'Trainable model parameters: {total_params}')

    print('Setup Adam optimizer...')
    optimizer = torch.optim.Adam(
        model.parameters(),
        lr=lr,
    )

    print('Setup loss function...')
    loss = torch.nn.MSELoss(reduction='sum').to(device)
    monitor_loss = torch.nn.L1Loss()

    print('Start train loop...')
    for epoch in range(1, epochs+1):
        epoch_loss = 0
        epoch_monitor_loss = 0

        # reset state every epoch
        state = None

        # iter over batches
        for x, y in train_loader:
            # move data to device
            x = x.to(device)
            y = y[:, 0, :].to(device)

            # get prediction
            state = None
            y_pred, state = model(x, state)

            # compute loss
            batch_loss = loss(y_pred, y)
            epoch_loss += batch_loss
            batch_monitor_loss = monitor_loss(y_pred, y)
            epoch_monitor_loss += batch_monitor_loss

            # perform gradient step
            model.zero_grad()
            batch_loss.backward()
            optimizer.step()

        print(f'EPOCH: {epoch} of {epochs} with MSELoss: {epoch_loss/len(train_set):.5f} and MAELoss: {epoch_monitor_loss/len(train_set):.5f}')

    print('Save model...')
    torch.save(model.state_dict(), 'lstm.t7')


import json

# with open('worldpopulation.json','r') as f:
with open(input('File Name: '),'r') as f:
    data = json.load(f)
    # print(data)

def c1():
    nCountry = 0
    for i in data:
        nCountry += 1
    print(nCountry)

def c2():
    pop = 0
    for i in range(len(data)):
        pop += int(data[i]['population'])
    print(pop)

def c3():
    cLetter = 0
    moreThanFive = 0
    for i in range(len(data)):
        ac = data[i]['country']
        if ac[0] == "C":
            cLetter += 1
        if len(ac) > 5:
            moreThanFive += 1
    print(cLetter)
    print(moreThanFive)

def c4():
    gt500M = 0        # 1)มากกว่า 500 ล้านคน
    mid = 0           # 2)ระหว่าง 250 ถึง 750 ล้านคน
    lt10M = 0         # 3)น้อยกว่า 10 ล้าน
    for i in range(len(data)):
        p = int(data[i]['population'])
        if p > 500000000:
            gt500M += 1
        if p > 250000000 and p < 750000000:
            mid += 1
        if p < 10000000:
            lt10M += 1
    print(gt500M)
    print(mid)
    print(lt10M)

def c5():
    topPop = sorted([float(i['World']) for i in data],reverse=True)
    top20 = sum([topPop[i] for i in range(20)])*100
    top50to150 = sum([topPop[i] for i in range(49,150)])*100
    print(f'{top20:.2f}')
    print(f'{top50to150:.2f}')

# def c5():
#     topPop = sorted([int(i['population']) for i in data],reverse=True)
#     sumTop20 = sum([topPop[i] for i in range(20)])
#     sum50to150 = sum([topPop[i] for i in range(49,150)])
#     sumPop = sum(topPop)
#     print(f"{(sumTop20/sumPop)*100:.2f}")
#     print(f"{(sum50to150/sumPop)*100:.2f}")
#     # print(topPop[49]-topPop[149])

x = int(input('Input : '))
if x == 1:
    c1()
elif x == 2:
    c2()
elif x == 3:
    c3()
elif x == 4:
    c4()
elif x == 5:
    c5()