""" Данные по Google-рекламе (обновленная выгрузка на новом API)
Описание полей можно найти в документации https://developers.google.com/google-ads/api/fields/v7/ad_group_ad
"""

import pandas as pd
import logging
from datetime import datetime, timedelta, date
from google.ads.googleads.client import GoogleAdsClient

def get_creeds(login_customer_id=None) -> dict:
    """
    Функция возвращает креды для создания клиента
    @param login_customer_id: Идентификатор зависит от аккаунта к которому осуществляются запросы
    @type login_customer_id: str
    @return: Словарь кредов
    @rtype: dict
    """
    hook = HttpHook(http_conn_id='google_ads_new', method='GET')
    conn = hook.get_connection('google_ads_new')

    cred_dict = {
        'developer_token': conn.extra_dejson.get('developer_token'),
        'refresh_token': conn.extra_dejson.get('refresh_token'),
        'client_id': conn.extra_dejson.get('client_id'),
        'client_secret': conn.extra_dejson.get('client_secret'),
        'access_token': conn.extra_dejson.get('access_token'),
        'use_proto_plus': True,
        'login_customer_id': login_customer_id
    }
    return cred_dict

def create_client(cred_dict, _version="v9") -> object:
    """
    Функция для создания класса клиента, по которому в дальнейшим осуществляется запросы
    @param cred_dict: Словарь кредов получается функцией get_creeds
    @type cred_dict: dict
    @param _version: Версия API
    @type _version: str
    @return: Класс для обращений к API
    @rtype: object
    """
    try:
        client = GoogleAdsClient.load_from_dict(cred_dict, version=_version)
        return client
    except Exception as e:
        logging.error(f"Create client Failed \nPlease check function 'create_client'\n{e}")

def _account_hierarchy(customer_client, customer_ids_to_child_accounts) -> dict:
    """
    Функция для парсинга иерархии аккаунта
    @param customer_client: Класс клиента
    @type customer_client: object
    @param customer_ids_to_child_accounts: Класс кабинетов клиента
    @type customer_ids_to_child_accounts: object
    @return: Словарь клиента со списком всех входящих кабинетов
    @rtype: dict
    """
    # проверка на наличие клиентов у аккаунта
    if len(customer_ids_to_child_accounts) > 0:
        dict_customer_client = {
            'client_id': customer_client.id,
            'client_name': customer_client.descriptive_name,
            'customers_client': []
        }
        for child_account in customer_ids_to_child_accounts[customer_client.id]:
            dict_child_account = {
                'id': child_account.id,
                'name': child_account.descriptive_name,
                'currency_code': child_account.currency_code,
                'time_zone': child_account.time_zone
            }
            dict_customer_client['customers_client'].append(dict_child_account)
    # Если у аккаунта нет клиентов - в качестве клиента возвращаем сам аккаунт
    else:
        dict_customer_client = {
            'client_id': customer_client.id,
            'client_name': customer_client.descriptive_name,
            'customers_client': [
                {
                    'id': customer_client.id,
                    'name': customer_client.descriptive_name,
                    'currency_code': customer_client.currency_code,
                    'time_zone': customer_client.time_zone
                }
            ]
        }
    return dict_customer_client

def get_account_list(client, login_customer_id=None) -> list:
    """
    Функция для получения иерархии аккаунта
    @param client: Клиент к которому осуществляются запросы, получается из create_client
    @type client: object
    @param login_customer_id: Идентификатор клиента, если нужно получить по конкретному,
     если None - возвращает по всем доступным
    @type login_customer_id: str
    @return: Список словарей с иерархией аккаунта
    @rtype: list
    """

    # Получает экземпляры клиентов GoogleAdsService и CustomerService.
    googleads_service = client.get_service("GoogleAdsService")
    customer_service = client.get_service("CustomerService")

    # Лист идентификаторов клиентов для обработки.
    seed_customer_ids = []

    # Создает запрос, который извлекает все дочерние учетные записи менеджера.
    query = """
        SELECT
          customer_client.client_customer,
          customer_client.level,
          customer_client.manager,
          customer_client.descriptive_name,
          customer_client.currency_code,
          customer_client.time_zone,
          customer_client.id
        FROM customer_client
        WHERE customer_client.level <= 1"""

    # Если идентификатор менеджера был указан в параметре customerId, он будет единственным идентификатором в списке.
    # В противном случае мы отправим запрос для всех клиентов,
    # доступных для этой аутентифицированной учетной записи Google.
    if login_customer_id is not None:
        seed_customer_ids = [login_customer_id]
    else:
        logging.info(
            "No manager ID is specified. The example will print the "
            "hierarchies of all accessible customer IDs."
        )

        customer_resource_names = (
            customer_service.list_accessible_customers().resource_names
        )

        for customer_resource_name in customer_resource_names:
            customer_id = googleads_service.parse_customer_path(
                customer_resource_name
            )["customer_id"]
            logging.info(customer_id)
            seed_customer_ids.append(customer_id)
    list_hierarchy = []
    for seed_customer_id in seed_customer_ids:
        unprocessed_customer_ids = [seed_customer_id]
        customer_ids_to_child_accounts = dict()
        root_customer_client = None

        while unprocessed_customer_ids:
            customer_id = int(unprocessed_customer_ids.pop(0))
            response = googleads_service.search(
                customer_id=str(customer_id), query=query
            )

            # Выполняет итерацию по всем строкам на всех страницах,
            # чтобы получить все кабинеты клиентов
            for googleads_row in response:
                customer_client = googleads_row.customer_client

                # Кабинет клиента, который с уровнем 0 является указанным Кабинет
                if customer_client.level == 0:
                    if root_customer_client is None:
                        root_customer_client = customer_client
                    continue

                if customer_id not in customer_ids_to_child_accounts:
                    customer_ids_to_child_accounts[customer_id] = []

                customer_ids_to_child_accounts[customer_id].append(
                    customer_client
                )

                if customer_client.manager:
                    # Клиентом могут управлять несколько менеджеров, поэтому
                    # предотвратить посещение одного и того же клиента много раз, мы
                    # проверяем, есть ли он уже в словаре.
                    if (
                            customer_client.id not in customer_ids_to_child_accounts
                            and customer_client.level == 1
                    ):
                        unprocessed_customer_ids.append(customer_client.id)

        if root_customer_client is not None:
            logging.info(
                "The hierarchy of customer ID "
                f"{root_customer_client.id} is printed below:"
            )
            list_hierarchy.append(_account_hierarchy(root_customer_client, customer_ids_to_child_accounts))
        else:
            logging.info(
                f"Customer ID {login_customer_id} is likely a test "
                "account, so its customer client information cannot be "
                "retrieved."
            )
    return list_hierarchy

def get_ads_data(client, account_id, start_date, end_date) -> pd.DataFrame:
    """
    Функция для получения данных по аккаунту. Для добавление новых метрик нужно
    1. Добавить их в запрос (query)
    2. Добавить в словарь (tmp_dict)
    Документация https://developers.google.com/google-ads/api/fields/v7/ad_group_ad

    @param client: Объект клиента из функции create_client
    @type client: object
    @param account_id: Идентификатор РК
    @type account_id: str
    @param start_date: Дата начала в формате 'YYYY-MM-DD'
    @type start_date: str
    @param end_date: Дата окончания в формате 'YYYY-MM-DD'
    @type end_date: str
    @return: Возвращает pd.DataFrame с данными в разрезе объявлений
    @rtype: pd.DataFrame
    """
    ga_service = client.get_service("GoogleAdsService")

    query = """
    SELECT
      -- Кампании
      campaign.id,
      campaign.name,
      campaign.status,
      -- Группа объявлений
      ad_group.name,
      ad_group.id,
      ad_group.status,
      ad_group_ad.labels,
      -- Объявления
      ad_group_ad.ad.id,
      ad_group_ad.ad.type,
      ad_group_ad.ad.tracking_url_template,
      ad_group_ad.ad.device_preference,
      ad_group_ad.ad.expanded_text_ad.description,
      ad_group_ad.ad.expanded_text_ad.description2,
      ad_group_ad.ad.display_url,
      ad_group_ad.ad.expanded_text_ad.headline_part1,
      ad_group_ad.ad.expanded_text_ad.headline_part2,
      ad_group_ad.ad.expanded_text_ad.headline_part3,
      -- Метрики
      metrics.impressions,
      metrics.clicks,
      metrics.cost_micros,
      metrics.all_conversions,
      metrics.view_through_conversions,
      -- Сегменты
      segments.date,
      segments.ad_network_type,
      segments.device
    FROM ad_group_ad
    WHERE segments.date BETWEEN '""" + str(start_date) + """' AND '""" + str(end_date) + """' -- AND ad_group_ad.status != 'REMOVED'
    ORDER BY campaign.id
        """
    # Получение данных
    search_request = client.get_type("SearchGoogleAdsStreamRequest")
    search_request.customer_id = account_id
    search_request.query = query
    response = ga_service.search_stream(search_request)

    data_df = pd.DataFrame()
    try:
        for batch in response:
            for row in batch.results:
                tmp_dict = {'campaign_name': row.campaign.name,
                            'campaign_id': row.campaign.id,
                            'campaign_status': row.campaign.status.name,
                            'ad_group_id': row.ad_group.id,
                            'ad_group_name': row.ad_group.name,
                            'ad_group_status': row.ad_group.status.name,
                            'labels': row.ad_group.labels,
                            'ad_id': row.ad_group_ad.ad.id,
                            'ad_type': row.ad_group_ad.ad.type_.name,
                            'tracking_url_template': row.ad_group_ad.ad.tracking_url_template,
                            'description': row.ad_group_ad.ad.expanded_text_ad.description,
                            'description2': row.ad_group_ad.ad.expanded_text_ad.description2,
                            'display_url': row.ad_group_ad.ad.display_url,
                            'headline_part1': row.ad_group_ad.ad.expanded_text_ad.headline_part1,
                            'headline_part2': row.ad_group_ad.ad.expanded_text_ad.headline_part2,
                            'headline_part3': row.ad_group_ad.ad.expanded_text_ad.headline_part3,
                            'clicks': row.metrics.clicks,
                            'impressions': row.metrics.impressions,
                            'cost': row.metrics.cost_micros / 1000000,
                            'all_conversions': row.metrics.all_conversions,
                            'view_through_conversions': row.metrics.view_through_conversions,
                            'start_date': row.segments.date,
                            'ad_network_type': row.segments.ad_network_type.name,
                            'device': row.segments.device.name
                            }
                data_df = data_df.append(tmp_dict, ignore_index=True)

        # Приведение к нужным типам
        data_df['campaign_id'] = data_df['campaign_id'].astype(int)
        data_df['ad_group_id'] = data_df['ad_group_id'].astype(int)
        data_df['ad_id'] = data_df['ad_id'].astype(int)
        data_df['clicks'] = data_df['clicks'].astype(int)
        data_df['impressions'] = data_df['impressions'].astype(int)
        data_df['cost'] = data_df['cost'].astype(float)
        data_df['all_conversions'] = data_df['all_conversions'].astype(int)
        data_df['view_through_conversions'] = data_df['view_through_conversions'].astype(int)

        return data_df
    except:
        return None

def get_ads_data_performance(client, account_id, start_date, end_date) -> pd.DataFrame:
    """
    Функция для получения данных по аккаунту. Для добавление новых метрик нужно
    1. Добавить их в запрос (query)
    2. Добавить в словарь (tmp_dict)
    Документация https://developers.google.com/google-ads/api/fields/v7/ad_group_ad

    Это отдельная функция для выгрузки Performance Max т.к. они выгружаются только в разрезе кампаний
    поля с ad_group_id и ad_id сохранены, но будут пустые

    @param client: Объект клиента из функции create_client
    @type client: object
    @param account_id: Идентификатор РК
    @type account_id: str
    @param start_date: Дата начала в формате 'YYYY-MM-DD'
    @type start_date: str
    @param end_date: Дата окончания в формате 'YYYY-MM-DD'
    @type end_date: str
    @return: Возвращает pd.DataFrame с данными в разрезе объявлений
    @rtype: pd.DataFrame
    """
    ga_service = client.get_service("GoogleAdsService")

    query = """
    SELECT
      -- Кампании
      campaign.id,
      campaign.name,
      campaign.status,
      -- Метрики
      metrics.impressions,
      metrics.clicks,
      metrics.cost_micros,
      metrics.all_conversions,
      metrics.view_through_conversions,
      -- Сегменты
      segments.date,
      segments.ad_network_type,
      segments.device
    FROM campaign
    WHERE segments.date BETWEEN '""" + str(start_date) + """' AND '""" + str(end_date) + """'
    and campaign.advertising_channel_type = PERFORMANCE_MAX
    ORDER BY campaign.id
        """
    # Получение данных
    search_request = client.get_type("SearchGoogleAdsStreamRequest")
    search_request.customer_id = account_id
    search_request.query = query
    response = ga_service.search_stream(search_request)

    data_df = pd.DataFrame()
    try:
        for batch in response:
            for row in batch.results:
                tmp_dict = {'campaign_name': row.campaign.name,
                            'campaign_id': row.campaign.id,
                            'campaign_status': row.campaign.status.name,
                            'ad_group_id': row.ad_group.id,
                            'ad_group_name': row.ad_group.name,
                            'ad_group_status': row.ad_group.status.name,
                            'labels': row.ad_group.labels,
                            'ad_id': row.ad_group_ad.ad.id,
                            'ad_type': row.ad_group_ad.ad.type_.name,
                            'tracking_url_template': row.ad_group_ad.ad.tracking_url_template,
                            'description': row.ad_group_ad.ad.expanded_text_ad.description,
                            'description2': row.ad_group_ad.ad.expanded_text_ad.description2,
                            'display_url': row.ad_group_ad.ad.display_url,
                            'headline_part1': row.ad_group_ad.ad.expanded_text_ad.headline_part1,
                            'headline_part2': row.ad_group_ad.ad.expanded_text_ad.headline_part2,
                            'headline_part3': row.ad_group_ad.ad.expanded_text_ad.headline_part3,
                            'clicks': row.metrics.clicks,
                            'impressions': row.metrics.impressions,
                            'cost': row.metrics.cost_micros / 1000000,
                            'all_conversions': row.metrics.all_conversions,
                            'view_through_conversions': row.metrics.view_through_conversions,
                            'start_date': row.segments.date,
                            'ad_network_type': row.segments.ad_network_type.name,
                            'device': row.segments.device.name
                            }
                data_df = data_df.append(tmp_dict, ignore_index=True)

        # Приведение к нужным типам
        data_df['campaign_id'] = data_df['campaign_id'].astype(int)
        data_df['ad_group_id'] = data_df['ad_group_id'].astype(int)
        data_df['ad_id'] = data_df['ad_id'].astype(int)
        data_df['clicks'] = data_df['clicks'].astype(int)
        data_df['impressions'] = data_df['impressions'].astype(int)
        data_df['cost'] = data_df['cost'].astype(float)
        data_df['all_conversions'] = data_df['all_conversions'].astype(int)
        data_df['view_through_conversions'] = data_df['view_through_conversions'].astype(int)

        return data_df
    except:
        return None

def get_date_range(start_days=5, stop_days=1) -> list:
    """
    Генерация списка дат для итерации
    @param start_days: Выбирается начало относительно сегодня в количестве дней
    @type start_days: int
    @param stop_days: Выбираем конец, по дефолту - до вчера
    @type stop_days: int
    @return: Список дат
    @rtype: list
    """
    start = (datetime.now() - timedelta(days=start_days)).date()
    end = (datetime.now() - timedelta(days=stop_days)).date()
    date_generated = [(start + timedelta(days=x)).strftime("%Y-%m-%d") for x in range(0, (end - start).days)]
    return date_generated

def main():
    # Итоговый фрейм для добавления данных
    df = pd.DataFrame()
    # Получаем креды
    creds = get_creeds()
    # Создаем клиента для получения списка кабинетов
    client = create_client(creds)
    # Получаем кабинеты
    account_hierarchy = get_account_list(client)
    # Создаем лист дат для итерации
    list_date = get_date_range(start_days=3)
    # Итерация по датам
    for day in list_date:
        logging.info(f'Start day {day}')
        # Итерация по клиентам
        for account in account_hierarchy:
            logging.info(f"Start account {account['client_id']}")
            # Создаем клиента для конкретного кабинета
            account_creeds = get_creeds(login_customer_id=account['client_id'])
            account_client = create_client(account_creeds)
            # Итерация по кабинетам
            for cabinet in account['customers_client']:
                logging.info(f"Start cabinet {cabinet['id']}")
                # Датафрейм для записи результатов одной итерации
                cabinet_df = get_ads_data(account_client, str(cabinet['id']), day, day)
                # Отдельно выгружаются кампании "максимальной эффективности"
                cabinet_perf_max_df = get_ads_data_performance(account_client, str(cabinet['id']), day, day)
                # Объединяем данные
                if cabinet_df is not None or cabinet_perf_max_df is not None:
                    cabinet_all_df = pd.concat([cabinet_df, cabinet_perf_max_df])
                else:
                    continue
                if cabinet_all_df is not None:
                    cabinet_all_df['account_name'] = account['client_name']
                    cabinet_all_df['account_id'] = account['client_id']
                    cabinet_all_df['cabinet_name'] = cabinet['name']
                    cabinet_all_df['cabinet_id'] = cabinet['id']
                    cabinet_all_df['date'] = day
                    cabinet_all_df['currency_code'] = cabinet['currency_code']
                    cabinet_all_df['time_zone'] = cabinet['time_zone']

                    # Добавляем в финальный фрейм
                    df = pd.concat([df, cabinet_all_df])
                else:
                    continue
    return df

_target_sql = '''
select
    md5(account_id+cabinet_id+date::varchar+campaign_id+ad_group_id+ad_id) as upd_key,
    account_name,
    account_id,
    cabinet_name,
    cabinet_id,
    date,
    currency_code,
    time_zone,
    campaign_name,
    campaign_id,
    campaign_status,
    ad_group_id,
    ad_group_name,
    ad_group_status,
    labels,
    ad_id,
    ad_type,
    tracking_url_template,
    description,
    description2,
    display_url,
    headline_part1,
    headline_part2,
    headline_part3,
    clicks,
    impressions,
    cost,
    all_conversions,
    view_through_conversions,
    start_date,
    ad_network_type,
    device
from [[ads]]
;'''

fields_types = {
    'upd_key': Type.VARCHAR,
    'account_name': Type.VARCHAR,
    'account_id': Type.VARCHAR,
    'cabinet_name': Type.VARCHAR,
    'cabinet_id': Type.VARCHAR,
    'date': Type.DATE,
    'currency_code': Type.VARCHAR,
    'time_zone': Type.VARCHAR,
    'campaign_name': Type.VARCHAR,
    'campaign_id': Type.VARCHAR,
    'campaign_status': Type.VARCHAR,
    'ad_group_id': Type.VARCHAR,
    'ad_group_name': Type.VARCHAR,
    'ad_group_status': Type.VARCHAR,
    'labels': Type.VARCHAR,
    'ad_id': Type.VARCHAR,
    'ad_type': Type.VARCHAR,
    'tracking_url_template': Type.VARCHAR(32768),
    'description': Type.VARCHAR(32768),
    'description2': Type.VARCHAR(32768),
    'display_url': Type.VARCHAR(32768),
    'headline_part1': Type.VARCHAR(32768),
    'headline_part2': Type.VARCHAR(32768),
    'headline_part3': Type.VARCHAR(32768),
    'clicks': Type.INTEGER,
    'impressions': Type.INTEGER,
    'cost': Type.FLOAT,
    'all_conversions': Type.INTEGER,
    'view_through_conversions': Type.INTEGER,
    'start_date': Type.DATE,
    'ad_network_type': Type.VARCHAR,
    'device': Type.VARCHAR
}

tpl = EntityBuilderTemplate(
    User.Efremov,
    is_incremental=True,
    update_key='upd_key',
    schedule_interval='0 5 * * *',
    fields=fields_types,
    query=_target_sql
)

tpl.add_stage(Stage([
    FetchListSourceOperator(name='ads',
                            python_callable=main,
                            types=fields_types
                            )
]))
dag = tpl.DAG()


import pandas as pd
import sys
import datetime

def create_url(date):
    """From a date, create the url pointing to a csv of all thesis defended at that date"""
    url = "https://www.theses.fr/?q=&zone1=titreRAs&val1=&op1=AND&zone2=auteurs&val2=&op2=AND&zone3=etabSoutenances&val3=&op3=AND&zone4=dateSoutenance&val4a={d}%2F{m}%2F{y}&val4b={d}%2F{m}%2F{y}&lng=fr/&type=avancee&checkedfacets=&format=csv"
    url = url.format(
        d=date.strftime("%d"),
        m=date.strftime("%m"),
        y=date.strftime("%Y"),
    )
    return url

def getDefendedThesisList(start_date, end_date):
    """Download all the general informations of all the theses defended between
    start_date and end_date (end_date excluded)"""
    current_date = start_date
    list_of_all_data_frames = []

    while current_date < end_date:
        a = None
        a = pd.read_csv(create_url(current_date), sep=";")
        current_date += datetime.timedelta(days=1)
        if a.shape[0] != 0:
            list_of_all_data_frames.append(a)
    return pd.concat(list_of_all_data_frames)

def getLimitDates():
    """Treat the script entry arguments into proper datetime objects"""
    s_date = datetime.datetime.strptime(sys.argv[1], "%Y-%m-%d")
    e_date = datetime.datetime.strptime(sys.argv[2], "%Y-%m-%d")
    return (s_date, e_date)

if __name__ == "__main__":
    s_date, e_date = getLimitDates()
    results = getDefendedThesisList(s_date, e_date)
    outFilePath = sys.argv[3]
    results.to_csv(outFilePath, sep=";", index=False)


#!/usr/bin/env python3

import pandas as pd
import plotly.express as px

if __name__ == '__main__':
    df = pd.read_csv("data/6fadff46-9efd-4c53-942a-54aca783c30c", sep=";")
    france_met = df[df.dep.str.len() <= 2].groupby("jour", as_index=False).sum()
    france_met["dep"] = "France"
    df2 = pd.concat((france_met, df[df.dep.isin(("01","69"))]), sort=False)
    plot = px.line(df2, x="jour", y="incid_hosp", color="dep", facet_row="dep",
            labels = {"jour":"Date", "incid_hosp":"Nouvelles hospitalisations",
                "dep":"Département"})
    plot.update_layout(title="Nouvelles hospitalisations dues au Covid-19")
    plot.update_yaxes(matches=None)
    plot.write_html("/home/joel/public_html/covid.html")


#jednostavan python skript za download kompozicija

#!pip install bs4
from urllib.request import urlopen, urlretrieve
from bs4 import BeautifulSoup
import time

#definisemo direktorijum u kojem cemo sacuvati kompozicije
save_dir = './guitar/'

#definisemo url komponente
url0 = 'https://www.mutopiaproject.org/cgibin/make-table.cgi?startat='
url1 = '&searchingfor=&Composer=&Instrument=Guitar&Style=&collection=&id=&solo=&recent=&timelength=&timeunit=&lilyversion=&preview='

#incijalizacija
songNumber = 0
i = 0

#pronalazimo i download-ujemo sve midi fajlove
while i < 380:
    #spajamo putanju
    url = url0 + str(songNumber) + url1
    html = urlopen(url)
    #parsiramo html
    soup = BeautifulSoup(html.read())
    #pronalazimo sve linkove
    links = soup.find_all('a')

    for link in links:
        href = link['href']

        #ako link sadrzi .mid
        if href.find('.mid') >= 0:
            #download-ujemo ga
            urlretrieve(href, 'guitar'+str(i)+'.mid')
            i += 1

    #na svakoj stranici se nalazi po 10 kompozicija
    songNumber += 10


from enum import Flag
from typing import cast
from media import Media
from pyfiglet import Figlet
from film import Film
from series import Series
from clip import Clip
from documentary import Documentry

class Main:
    def __init__(self):

        try:
            myfile=open('database.txt', 'r')
        except:
            print("Database doesn't found")

        self.movielist=[]

        data = myfile.read()
        movie_list = data.split('\n')

        for i in range(len(movie_list)):

            movie_info=movie_list[i].split(',')

            if movie_info[6]=='movie':
                self.movielist.append(Film(movie_info))

            elif movie_info[6]=='series':
                self.movielist.append(Series(movie_info))

            elif movie_info[6]=='clip':
                self.movielist.append(Clip(movie_info))

            elif movie_info[6]=='documentory':
                self.movielist.append(Documentry(movie_info))

            myfile.close()
        self.ShowMenu()

    def ShowMovielist(self):
        for i in self.movielist:
            i.showinfo()

    def AddMovie(self):
        name =input('Enter name :')
        director = input('Enter director : ')
        IMDB = input('Enter IMDB : ')
        url = input('Enter url : ')
        duration = input('Enter duration : ')
        casts = input('Enter casts : ')
        category =input('Enter category : ')

        if category == 'series':
            epnum = input('Enter the number of series episodes : ')
            movie_info = [name,director,IMDB,url,duration,casts,category,epnum]
        else:
            movie_info = [name,director,IMDB,url,duration,casts,category]

        if category=='movie':
            self.movielist.append(Film(movie_info))

        elif category=='series':
            self.movielist.append(Series(movie_info))

        elif category=='clip':
            self.movielist.append(Clip(movie_info))

        elif category=='documentory':
            self.movielist.append(Documentry(movie_info))

        else:
            print("This type of category doesn't exist!")

        self.ShowMovielist()
        self.ShowMenu()

    def EditMovie(self):
        flag=0
        choise = input("\nEnter the name of the movie that you wanna edit : ")

        for media in self.movielist:

            if choise == media.name:

                if media.category == 'series':
                    media.edit_series()

                if media.category == 'movie':
                    media.edit_film()

                if media.category == 'clip':
                    media.edit_clip()

                if media.category == 'documentory':
                    media.edit_documentory()

            else:
                flag+=1

            if flag>=5:
                print("This movie doesn't exist")

        self.ShowMenu()

    def Deletemovie(self):

        flag=0
        choise = input("\nEnter the name of the movie that you wanna delete : ")

        for media in self.movielist:
            if choise == media.name:
                self.movielist.remove(media)
                print("\nThis movie deleted succssesfuly\n")
            else:
                flag+=1

            if flag>=5:
                print("This movie doesn't exist")

        self.ShowMenu()

    def Download(self):
        flag = 0
        answer = input('\ngive me the name of the movie you wanna download: ')
        for media in self.movielist:
            if answer == media.name:
                media.download()
            else:
                flag += 1

            if flag >= len(self.movielist):
                print("\nThis movie doesn't exist in our list")
        self.ShowMenu()

    def Searchbydu(self):
        firstdu = input("Give me the first duration like this [00:00]: ")
        secdu = input("Give me the second duration like this [00:00]: ")
        firstdu = firstdu.split(':')
        h1 = int(firstdu[0])
        min1 = int(firstdu[1])
        du1 = h1*3600 + min1*60
        secdu = secdu.split(':')
        h2 = int(secdu[0])
        min2 = int(secdu[1])
        du2 = h2*3600 + min2*60

        for media in self.movielist:
            moviedu = media.duration.split(':')
            h3 = int(moviedu[0])
            m3 = int(moviedu[1])
            moviedu3 = h3*3600 + m3*60

            if du1 <= moviedu3 <= du2:
                print("This movie seems near to your input duration : ",media.name)

    def Savetodatabase(self):
        myfile=open('database.txt', 'w')

        for media in self.movielist:
            if media.category == 'series':
                myfile.write((media.name)+','+(media.director)+','+(media.IMDB)+','+(media.url)+','+(media.duration)+','+(media.casts)+','+(media.category)+','+(media.numberofep))
            else:
                myfile.write((media.name)+','+(media.director)+','+(media.IMDB)+','+(media.url)+','+(media.duration)+','+(media.casts)+','+(media.category))
            if media != self.movielist[-1]:
                myfile.write('\n')

        myfile.close()
        print("Database updated succssefully")
        exit()

    def ShowMenu(self):

        print('1-Add movie\n2-Edit movie\n3-Delete movie\n4-Search by duration\n5-Show movielist\n6-Download movie\n7-Save and Exit ')

        choice = input('Please choose a number : ')

        if choice == '1':
            self.AddMovie()

        elif choice == '2':
            self.EditMovie()

        elif choice == '3':
            self.Deletemovie()

        elif choice == '4':
            self.Searchbydu()

        elif choice == '5':
            self.ShowMovielist()

        elif choice == '6':
            self.Download()

        elif choice == '7':
            self.Savetodatabase()

        else:
            print('Wrong input')

f = Figlet(font='standard')
print(f.renderText('2 4 M o v i e s'))

main = Main()


#!/usr/bin/env python3
import sys
import logging
from timeit import default_timer as timer

import util

LOG_LEVEL = logging.INFO

def main(input_file):

    # Graph where a edge A->B indicates A orbits B
    orbit_map = util.Graph()

    with open(input_file) as f:
        while True:
            line = f.readline().strip()
            if not line:
                break
            target, orbitor = line.split(')')
            logging.debug(f'{orbitor} orbits {target}')

            orbit_map.add_edge([orbitor, target], bidirectional=True)

        current_orbital = orbit_map.neighbors("YOU")[0]
        target = orbit_map.neighbors("SAN")[0]

        distance_map, _ = orbit_map.dijkstra(current_orbital)
        logging.debug(f'Distance Map {distance_map}')

        print("Orbit Transfers: " + str(distance_map[target]))

if __name__ == "__main__":

    logging.basicConfig(format='%(levelname)s: %(message)s', level=LOG_LEVEL)

    if len(sys.argv) < 2:
        logging.critical("Missing input file")
    else:
        start = timer()
        main(sys.argv[1])
        end = timer()

        logging.info(f'Executed in {(end-start):.5f} seconds')


import colorsys # for get_N_HexCol
import os
import json
import csv
import sys
import subprocess
from multiprocessing import Process, Queue
import subprocess # copy text to clipboard
import time

def pd(what, start_time):
    time_in_min = (time.time() - start_time) / 60.0
    time_in_h = time_in_min / 60.0
    print("%s took ~%0.2f min, ~%0.2f h"%(what, time_in_min, time_in_h))

def copy2clip(txt):
    cmd="echo '"+txt.strip()+"'| xsel --clipboard"
    return subprocess.check_call(cmd, shell=True)

# https://stackoverflow.com/questions/4760215/running-shell-command-from-python-and-capturing-the-outputhttps://stackoverflow.com/questions/4760215/running-shell-command-from-python-and-capturing-the-output
def execute_command(command_str):
    command_pieces = command_str.split(" ")
    return subprocess.check_output(command_pieces)

# saves a file to the experiment directory
def save_to_json(data, outfile_name):
    with open(outfile_name, "w") as f:
        f.write( json.dumps( data ) )
    print("Saved to json: %s."%outfile_name)

# dump data array to file
def save_to_csv(data_rows, outfile_name):
    with open(outfile_name, "w") as f:
        cw =  csv.writer(f,delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
        for data_row in data_rows:
            cw.writerow(data_row)
    print("Saved to csv: %s."%outfile_name)

# load data from vocabulary
def load_from_json(infile_name):
    with open(infile_name, "r") as f:
        print("Loaded from json: %s"%infile_name)
        json_str = f.read()
        return json.loads(json_str)

# load data from csv
def load_from_csv(infile_name):
    with open(infile_name, "r") as f:
        read_rows =  csv.reader(f,delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
        rows = []
        for row in read_rows:
            rows.append(row)
        print("Loaded from csv: %s."%infile_name)
        return rows

def create_if_not_exists(dir_path):
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

def num_lines(file_name):
    return int(execute_command("wc -l %s"%file_name).split()[0])

# get a hex color range for number of parts
def get_N_HexCol(N=5):
    HSV_tuples = [(x*1.0/N, 1, 1) for x in xrange(N)]
    hex_out = []
    for rgb in HSV_tuples:
        rgb = map(lambda x: int(x*255),colorsys.hsv_to_rgb(*rgb))
        hex_out.append("#"+ "".join(map(lambda x: chr(x).encode('hex'),rgb)).upper() )
    return hex_out
"""
def process_one_line(line_q, result_q, ):
    # get item blocking
    new_line = line_q.get(timeout=5)
    # do something
    result =  experiment_lib.extract_pattern_id(new_line)
    # get back result
    result_q.put(result)
"""
def multiprocess_file(file_name, process_one_line,  num_processes=8, max_size=10000):
    # define multithreading
    line_q = Queue(maxsize=max_size)
    result_q = Queue()

    # process for scooping lines to process on input q
    def load_line(line_q, file_name):
        for l in open(file_name, 'r'):
            line_q.put(l)
    # wrapper for processing the line in one loop
    def proccess_one_line_loop(line_q,result_q,pid):
        try:
            while True:
                process_one_line(line_q,result_q)
        except Exception as e:
            print(e)
            print("Shutting down processing thread %i"%pid)

    # define processes
    processes = []
    for pid in xrange(num_processes):
        processes.append(Process(target=proccess_one_line_loop, args=(line_q,result_q,pid)))

    line_load_p = Process(target=load_line, args=(line_q,file_name))

    # start threads
    [p.start() for p in processes]
    line_load_p.start()

    return result_q, line_q, processes, line_load_p

# Print iterations progress
def print_progress (iteration, total, prefix = '', suffix = '', decimals = 2, barLength = 100):
    filledLength    = int(round(barLength * iteration / float(total)))
    percents        = round(100.00 * (iteration / float(total)), decimals)
    bar             = '#' * filledLength + '-' * (barLength - filledLength)
    sys.stdout.write('%s [%s] %s%s %s\r' % (prefix, bar, percents, '%', suffix)),
    sys.stdout.flush()
    if iteration == total:
        print("\n")


from backend.model import getPrice, addItemToNewOrder, getProductsFromOrder, addNewItemToOrder, updateExistingItem, getOrderID, deleteItemFromOrder, readOrder, validateOrderId, getImageUrl, readOrderForHeaderCart, getQuantity, readOrderByOrderId, getStock, updateOrderStatus, readPlacedOrder, getUserRating, getSellerById, readOffers
from flask import session
import random
from datetime import datetime

def addItemToCart(productid, quantity):
    userid = session["user_id"][0]
    if session["order_id"] == 'None':
        orderid = 'None'
    else:
        orderid = int(session["order_id"])
    sp = getPrice(productid)
    for i in sp:
        product_price = i['product_price']
        offer_flag = i['offer_flag']
        offer_id = i['offer_id']
    if offer_flag == 1:
        if offer_flag != 0:
            offers = readOffers(productid, offer_id)
            for item in offers:
                offer_percent = item['offer_percent']
        selling_price = round((product_price - (product_price * offer_percent) / 100),2)
    else:
        selling_price = round(product_price, 2)

    if orderid == "None":
        orderid = 0
        orderExists = "True"
        while orderExists == "True":
            orderid = random.randint(1111,9999)
            val = validateOrderId(orderid)
            for i in val:
                count = i[0]
            if count == 0:
                orderExists = "False"
        addItemToNewOrder(orderid, userid, productid, selling_price, quantity)
        session["order_id"] = orderid
        session["total_items"] = 1
        refreshCart()
    else:
        count = getProductsFromOrder(orderid, productid)
        for i in count:
            count = i[0]
        if int(count) == 1:
            updateExistingItem(orderid, productid, selling_price, quantity)
        else:
            addNewItemToOrder(orderid, userid, productid, selling_price, quantity)

        session["order_id"] = orderid
        session["total_items"] = count
        refreshCart()
    return "True"

def getOrder(userid):
    order = readOrder(userid[0])
    keyList = ["order_id", "user_id", "ship_address", "order_date", "quantity", "selling_price", "order_status", "product_serial_number", "image_id", "item_total", "order_total", "product_name", "product_url"]
    cart = []
    i = 1
    order_total = 0
    for row in order:
        item = {key: [] for key in keyList}
        item['order_id'].append(row["order_id"])
        item['user_id'].append(row["user_id"])
        item['ship_address'].append(row["ship_address"])
        item['order_date'].append(row["order_date"])
        item['quantity'].append(row["quantity"])
        item['selling_price'].append(row["selling_price"])
        item['order_status'].append(row["order_status"])
        item['product_serial_number'].append(row["product_serial_number"])
        item_total = round(float(row["quantity"]) * float(row["selling_price"]), 2)
        item['item_total'].append(item_total)
        image_id = getImageUrl(row["product_serial_number"])
        for url in image_id:
            item['image_id'].append(url["image_id"])
            item['product_name'].append(url["product_name"])
            p_url = "/product?categoryid=" + str(url["category_id"]) + "&subcategoryid=" +  str(url["sub_category_id"]) + "&product_serial_number=" + str(url["product_serial_number"])
            item['product_url'].append(p_url)
        cart.append(item)
        i += 1
        order_total += item_total
        item
    if cart is not None:
        return cart, order_total
    else:
        return "ERROR"

def deleteItemFromCart(productid):
    userid = session["user_id"][0]
    print("User :")
    print(userid)
    orderid = int(session["order_id"])
    deleteItemFromOrder(orderid, userid, productid)
    refreshCart()
    return "True"

def refreshCart():
    # cart = readOrderItemsForProductPage(orderid)
    # order = []
    # for item in cart:
    #     items = {}
    #     items["product_serial_number"] = item['product_serial_number']
    #     items["quantity"] = item['quantity']
    #     order.append(items)
    order = readOrderForHeaderCart(session["user_email"][0])
    order_id = 'None'
    item_count = '0'
    if order.rowcount != 0:
        for i in order:
            order_id = i['order_id']
            item_count = i['item_count']
            print("order id is")
            print(order_id)
            session["order_id"] = i['order_id']
    session["total_items"] = item_count

def getCurrentQuantityForAProduct(productid):
    quantity = getQuantity(productid, session['order_id'])
    return quantity

def updateOrder():
    orderid = session["order_id"]
    order = readOrderByOrderId(orderid)
    new_stock = {}
    for item in order:
        print("order id :", item["order_id"])
        print("product_serial_number :", item["product_serial_number"])
        print("quantity :", item["quantity"])
        stock = getStockValue(item["product_serial_number"])
        new_stock[item["product_serial_number"]] = int(stock) - int(item["quantity"])
        if int(stock) < int(item["quantity"]):
            return "incomplete"
    date = datetime.now().strftime("%d-%m-%Y")
    status = updateOrderStatus(orderid, new_stock, date, session['user_address'][0])
    if status is True:
        session.pop("order_id")
        session["order_id"] = 'None'
        refreshCart()
        order = readOrderByOrderId(orderid)
        return order, status
    order = readOrderByOrderId(orderid)
    return order, status

def getStockValue(product_serial_number):
    stock = getStock(product_serial_number)
    for item in stock:
        value = int(item["stock"])
        return value
    return False

def getPlacedOrder(orderid):
    order = readPlacedOrder(orderid)
    keyList = ["order_id", "user_id", "ship_address", "order_date", "quantity", "selling_price", "order_status", "product_serial_number", "image_id", "item_total", "order_total", "product_name", "product_url", "rating", "seller_name", "seller_email", "seller_address", "seller_id"]
    cart = []
    i = 1
    order_total = 0
    for row in order:
        item = {key: [] for key in keyList}
        item['order_id'].append(row["order_id"])
        item['user_id'].append(row["user_id"])
        item['ship_address'].append(row["ship_address"])
        item['order_date'].append(row["order_date"])
        item['quantity'].append(row["quantity"])
        item['selling_price'].append(row["selling_price"])
        item['order_status'].append(row["order_status"])
        item['product_serial_number'].append(row["product_serial_number"])
        item_total = round(float(row["quantity"]) * float(row["selling_price"]), 2)
        item['item_total'].append(item_total)
        image_id = getImageUrl(row["product_serial_number"])
        for url in image_id:
            item['seller_id'].append(url["seller_id"])
            item['image_id'].append(url["image_id"])
            item['product_name'].append(url["product_name"])
            p_url = "/product?categoryid=" + str(url["category_id"]) + "&subcategoryid=" +  str(url["sub_category_id"]) + "&product_serial_number=" + str(url["product_serial_number"])
            item['product_url'].append(p_url)
        seller = getSellerById(item["seller_id"][0])
        for ss in seller:
            print("ss['seller_name']", ss["seller_name"])
            item["seller_name"].append(ss["seller_name"])
            item["seller_email"].append(ss["seller_email"])
            item["seller_address"].append(ss["seller_address"])
        rating = getUserRating(row["product_serial_number"], row["user_id"])
        for rr in rating:
            item['rating'].append(rr["rating_score"])
        cart.append(item)
        i += 1
        order_total += item_total
        item
    if cart is not None:
        return cart, order_total
    else:
        return "ERROR"


import tensorflow as tf
from konlpy.tag import Okt
from sklearn.feature_extraction.text import TfidfVectorizer
tf.set_random_seed(0)

text_data = ["이 영화는 별로다",
             "이 영화는 좋다",
             "이 영화는 글쎄",
             "진짜 이 영화는 좋다",
             "싫다",
             "나는 좋다",
             "진짜 별로다",
             "나는 영화는 좋다",
             "정말 싫다",
             "나는 진짜 좋다"]

target_data = [[0.], [1.], [0.], [1.], [0.], [1.], [0.], [1.], [0.], [1.]]

def tokenizer(text):
    okt = Okt()
    words = okt.morphs(text)
    return words

stopword_list = ['나','는','다','이']
max_features = 7
tfidf = TfidfVectorizer(tokenizer=tokenizer, stop_words=stopword_list, max_features=max_features)
sparse_tfidf_texts = tfidf.fit_transform(text_data)
dense_tfidf_texts = sparse_tfidf_texts.todense()

X = tf.placeholder(dtype=tf.float32, shape=[None, max_features])
Y = tf.placeholder(dtype=tf.float32, shape=[None, 1])
W = tf.get_variable('weight', shape=[max_features, 1], initializer=tf.random_normal_initializer())
b = tf.get_variable('bias',  shape=[1])
model_output = tf.nn.sigmoid(tf.nn.bias_add(tf.matmul(X, W), b))

loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=model_output, labels=Y))
prediction = tf.cast(tf.reshape(model_output, [-1]) > 0.5, tf.float32)
optim = tf.train.AdamOptimizer(0.01).minimize(loss)

sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)

print(tfidf.get_feature_names())

print("\n> Start train")
for i in range(1000):
    _, l = sess.run([optim, loss], feed_dict={X: dense_tfidf_texts, Y: target_data})
    if (i+1)%100 == 0:
        print("Epoch : {:3d}, Loss : {:.6f}".format(i+1, l))
print("> Done")

print("\n> Predict")
for ix in range(10):
    print(text_data[ix], end="")
    print(sess.run(prediction, feed_dict={X: dense_tfidf_texts[ix]}))

'''
['글쎄', '별로', '싫다', '영화', '정말', '좋다', '진짜']

> Start train
Epoch : 100, Loss : 0.617542
Epoch : 200, Loss : 0.564370
Epoch : 300, Loss : 0.540849
Epoch : 400, Loss : 0.528785
Epoch : 500, Loss : 0.521805
Epoch : 600, Loss : 0.517394
Epoch : 700, Loss : 0.514419
Epoch : 800, Loss : 0.512309
Epoch : 900, Loss : 0.510754
Epoch : 1000, Loss : 0.509571
> Done

> Predict
이 영화는 별로다[0.]
이 영화는 좋다[1.]
이 영화는 글쎄[0.]
진짜 이 영화는 좋다[1.]
싫다[0.]
나는 좋다[1.]
진짜 별로다[0.]
나는 영화는 좋다[1.]
정말 싫다[0.]
나는 진짜 좋다[1.]
'''


import numpy as np
import numpy.polynomial.chebyshev as chebyshev
import numba
from stationary_functions import stationary

def backward_iterate_linear(kplus,Pi,Phi,beta,alpha,delta,z,k):
    """
    Uses backwards iteration to update the capital policy function
    """
    k_endogenous = (1/Phi)*(1+Phi*k-beta*((1-delta)+Pi@(alpha*z[:,np.newaxis]*k**(alpha-1)+Phi*(kplus-k))))

    k_new = np.empty_like(kplus)
    for zi in range(len(z)):
        k_new[zi, :] = np.interp(k,k_endogenous[zi,:],k)
    return k_new

def ss_policy_linear(alpha,beta,delta, Phi,Pi,z,klow,khigh, N,tol = 1E-9):
    k = np.linspace(klow,khigh,num=N,endpoint=True)
    kplus = np.broadcast_to(k, (len(z), len(k)))
    for it in range(1000):
        kplus = backward_iterate_linear(kplus,Pi,Phi,beta,alpha,delta,z,k)
        if it % 10 == 1 and  np.max(np.abs(kplus - kold)) < tol:
            print(f'convergence in {it} iterations!')
            return kplus
        kold = kplus

def chebyshev_nodes(xlow, xhigh, N):
    standard_nodes = -np.cos((2*np.arange(1, N+1)-1)/(2*N)*np.pi)
    x = xlow + (xhigh-xlow)/2*(1+standard_nodes)
    return x

def chebyshev_interp(x, y):
    X = chebyshev.chebvander(x, len(x)-1)
    return np.linalg.solve(X, y)

def backward_iterate_chebyshev(kplus,Pi,Phi,beta,alpha,delta,z,k):
    """
    Uses backwards iteration to update the capital policy function
    """
    k_endogenous = (1/Phi)*(1+Phi*k-beta*((1-delta)+Pi@(alpha*z[:,np.newaxis]*k**(alpha-1)+Phi*(kplus-k))))

    q = np.empty_like(kplus)
    k_new = np.empty_like(kplus)
    for zi in range(len(z)):
        q = chebyshev_interp(k_endogenous[zi,:], k)
        k_new[zi, :] = chebyshev.chebval(k, q)
    return k_new

def ss_policy_chebyshev(alpha,beta,delta, Phi,Pi,z,klow,khigh, N):
    k = chebyshev_nodes(klow,khigh, N)
    kplus = np.broadcast_to(k, (len(z), len(k)))
    for it in range(1000):
        kplus = backward_iterate_chebyshev(kplus,Pi,Phi,beta,alpha,delta,z,k)
        if it % 10 == 1 and  np.max(np.abs(kplus - kold)) < 1E-10:
            print(f'convergence in {it} iterations!')
            return kplus
        kold = kplus

def ss_policy_chebyshev_remapped(alpha,beta,delta, Phi,Pi,z,klow,khigh, N):
    k = chebyshev_nodes(klow,khigh, N)
    kplus = np.broadcast_to(k, (len(z), len(k)))
    for it in range(1000):
        kplus = backward_iterate_chebyshev_remapped(kplus,Pi,Phi,beta,alpha,delta,z,k,klow,khigh)
        if it % 10 == 1 and  np.max(np.abs(kplus - kold)) < 1E-10:
            print(f'convergence in {it} iterations!')
            return kplus
        kold = kplus

def euler_equation_errors_linear(kplus,alpha,beta,delta, Phi,Pi,z,k,klow,khigh, N):
    k_grid = np.linspace(klow,khigh,num=N,endpoint=True)
    kplus_dense = np.empty((len(z),len(k)))
    for i in range(len(z)):
        kplus_dense[i,:] = np.interp(k,k_grid,kplus[i,:])
    LHS = 1+Phi*(kplus_dense-k)
    RHS = np.empty_like(LHS)
    k2 = np.empty_like(kplus_dense)
    for i, zi in enumerate(z):
        for j in range(len(z)):
            k2[j,:] = np.interp(kplus_dense[i,:],k,kplus_dense[j,:])
        RHS[i,:]=beta*((1-delta)+Pi[i,:]@(alpha*z[:,np.newaxis]*kplus_dense[i,:]**(alpha-1)+Phi*(k2-kplus_dense[i,:])))
    return np.log10(np.abs(LHS-RHS))

def euler_equation_errors(kplus,alpha,beta,delta, Phi,Pi,z,klow,khigh, N):
    k = np.linspace(klow,khigh,num=N,endpoint=True)
    LHS = 1+Phi*(kplus-k)
    RHS = np.empty_like(LHS)
    k2 = np.empty_like(kplus)
    for i, zi in enumerate(z):
        for j in range(len(z)):
            k2[j,:] = np.interp(kplus[i,:],k,kplus[j,:])
        RHS[i,:]=beta*((1-delta)+Pi[i,:]@(alpha*z[:,np.newaxis]*kplus[i,:]**(alpha-1)+Phi*(k2-kplus[i,:])))
    return np.log10(np.abs(LHS-RHS))

def ee_error_chebyshev(q, k,alpha,beta,delta, Phi,Pi,z):
    kplus = chebyshev.chebval(k, q)
    LHS = 1+Phi*(kplus-k)
    RHS = np.empty_like(LHS)
    for i, zi in enumerate(z):
        k2 = chebyshev.chebval(kplus[i,:], q)
        c_inner = chebyshev.chebval(kplus[i, :], q.T)
        RHS[i,:]=beta*((1-delta)+Pi[i,:]@(alpha*z[:,np.newaxis]*kplus[i,:]**(alpha-1)+Phi*(k2-kplus[i,:])))
    return np.log10(np.abs(LHS-RHS))

@numba.njit
def interpolate_coord(x, xq):
    """Get representation xqi, xqpi of xq interpolated against x:
    xq = xqpi * x[xqi] + (1-xqpi) * x[xqi+1]

    Parameters
    ----------
    x    : array (n), ascending data points
    xq   : array (nq), ascending query points

    Returns
    ----------
    xqi  : array (nq), indices of lower bracketing gridpoints
    xqpi : array (nq), weights on lower bracketing gridpoints
    """
    nxq, nx = xq.shape[0], x.shape[0]
    xqi = np.empty(nxq, dtype=np.int64)
    xqpi = np.empty(nx)

    xi = 0
    x_low = x[0]
    x_high = x[1]
    for xqi_cur in range(nxq):
        xq_cur = xq[xqi_cur]
        while xi < nx - 2:
            if x_high >= xq_cur:
                break
            xi += 1
            x_low = x_high
            x_high = x[xi + 1]

        xqpi[xqi_cur] = (x_high - xq_cur) / (x_high - x_low)
        xqi[xqi_cur] = xi
    return xqi, xqpi

@numba.njit
def forward_iterate(D, Pi_T, kplus_i, kplus_pi):
    # first step: update using endogenous capital policy
    # initialize Dnew with zeros because we're looping through D, not Dnew
    Dnew = np.zeros_like(D)
    for zi in range(D.shape[0]):
        for ki in range(D.shape[1]):
            i = kplus_i[zi, ki]
            pi = kplus_pi[zi, ki]
            d = D[zi, ki]

            Dnew[zi, i] += d*pi
            Dnew[zi, i+1] += d*(1-pi)

    # second step: update using transpose of Markov matrix for exogenous state z
    # take Pi_T itself as input for memory efficiency
    return Pi_T @ Dnew

def ergodic_dist(Pi, kplus_i, kplus_pi):
    # start by getting stationary distribution of z
    pi = stationary(Pi)

    # need to initialize joint distribution of (z, k), assume uniform on k
    nK = kplus_i.shape[1]
    D = np.outer(pi, np.full(nK, 1/nK))

    # Pi.T is a "view" on Pi with the wrong memory order, copy this to get right memory order (faster operations)
    Pi_T = Pi.T.copy()

    # now iterate forward until convergence
    for it in range(100_000):
        Dnew = forward_iterate(D, Pi_T, kplus_i, kplus_pi)

        # only check convergence every 20 iterations for efficiency
        if it % 20 == 0 and np.max(np.abs(Dnew-D)) < 1E-10:
            print(f'Convergence after {it} forward iterations!')
            break
        D = Dnew
    else:
        raise ValueError(f'No convergence after {maxit} forward iterations!')

    return D

import numba
@numba.njit
def chebval_matt(x, q):
    b_2 = 0.
    b_1 = 0.
    for k in range(len(q)-1, 0, -1):
        b = q[k] + 2*x*b_1 - b_2
        b_2 = b_1
        b_1 = b
    return q[0] + x*b_1 - b_2


#!/usr/bin/env python3

import serial
import argparse

def mavsend(command):
   ser = serial.Serial('/dev/ttyUSB0', 9600)
   ser.write(command.encode())
   mavrecv = ser.readline()
   ser.close()
   return mavrecv.decode()

parser = argparse.ArgumentParser()
parser.add_argument('--tieav', '-tav',nargs=2,type=int,help='Tie' )
parser.add_argument('--tieaudio', '-ta',nargs=2,help='Tie Audio' )
parser.add_argument('--tievideo', '-tv',nargs=2,help='Tie Video' )
parser.add_argument('--viewav', '-vav',nargs=1,type=int,help='View' )
parser.add_argument('--viewaudio', '-va',nargs=1,help='View Audio' )
parser.add_argument('--viewvideo', '-vv',nargs=1,help='View Video' )
parser.add_argument('--preset', '-p',nargs=1,help='Tie Presets' )
parser.add_argument('--raw', '-r',nargs=1,help='Send RAW command' )
parser.add_argument('--curaudio', '-ca',action='store_true',help='Current Audio Ties' )
parser.add_argument('--curvideo', '-cv',action='store_true',help='Current Video Ties' )

args = parser.parse_args()

if args.tieav:
   mavsend(str(args.tieav[0]) + "*" + str(args.tieav[1]) + "!")
elif args.tieaudio:
   mavsend(str(args.tieaudio[0]) + "*" + str(args.tieaudio[1]) + "$")
elif args.tievideo:
   mavsend(str(args.tievideo[0]) + "*" + str(args.tievideo[1]) + "&")
elif args.viewav:
   print(mavsend("v" + str(args.viewav[0]) + "!"))
elif args.viewaudio:
   print(mavsend("v" + str(args.viewaudio[0]) + "$"))
elif args.viewvideo:
   print(mavsend("v" + str(args.viewvideo[0]) + "&"))
elif args.raw:
  print(mavsend(str(args.raw)))
elif args.preset:
   if str(args.preset[0]) == "1": # BT to All
      print("BT Preset")
      mavsend("1*1$")
      mavsend("1*2$")
      mavsend("1*3$")
      mavsend("1*14$")
      mavsend("1*16$")
   elif str(args.preset[0]) == "2": # TV to All
      print("Reciever Preset")
      mavsend("2*1$")
      mavsend("2*2$")
      mavsend("2*3$")
      mavsend("2*14$")
      mavsend("2*16$")
   elif str(args.preset[0]) == "3": # TV to All
      print("TV Preset")
      mavsend("3*1$")
      mavsend("3*2$")
      mavsend("3*3$")
      mavsend("3*14$")
      mavsend("3*16$")
elif args.curaudio:
   data = ""
   for i in range(1,17,1):
      results = (mavsend("v" + str(i) + "$"))
      results = "\"Out" + "{0:02d}".format(i) + "\": \"" + results[8:10] + "\""
      if i < 16:
         data = data + results + ", "
      else:
         data = data + results
#      print(results)
   print("{" + data + "}")
elif args.curvideo:
   for i in range(1,17,1):
      results = (mavsend("v" + str(i) + "&"))
      results = results[8:10] + ":" + "{0:02d}".format(i)
      print(results)
else:
   print("~ No Flag")

#ser = serial.Serial('/dev/ttyUSB0', 9600)
#ser.write(b'16!')
#print(ser.readline())
#ser.close()             # close port


from bs4 import BeautifulSoup
import pandas as pd
import re

#
# Sample format of file I want to extract
"""

Date1

Part1_of_row1: Part2_of row1

        .
        .
        .

Part1_of_rowN: Part2_of rown

Date2

Part1_of_rowNplus1: Part2_of rowNplus1</aL

.
.

"""
# Desired output is a .csv where each line has ["date","part1_of_rowN","Part2_of_rowN","url_for_rowN"]

with open("myfile.html", "r") as og_file:
    page = str(og_file.read())

soup = BeautifulSoup(page, "html5lib")

list_items = soup.find_all('li')

#separate each li in the soup object into columns
list_output = []
for li in list_items:
    date = li.find_previous("h2")
    date = date.text if date else "-"

    halfway = str(li.text)
    if ":" in halfway:
        halfway = halfway.split(":")
        company = halfway[0]
        role = halfway[1]
    elif "," in halfway:
        halfway = halfway.split(",", maxsplit=1)
        company = halfway[0]
        role = halfway[1]
    elif "at" in halfway:
        halfway = halfway.split("at", maxsplit=1)
        company = halfway[1]
        role = halfway[0]
    else:
        role = halfway
        company = ""

    a= li.a
    if a:
        url = a["href"]
        url = url.replace("https://www.google.com/url?q=","")

    list_output.append([date,company,role,url])
df = pd.DataFrame(list_output, columns=["Date","Company", "Role","URL"])

#clean up some crap in the Company column
df["Company"] = df["Company"].str.replace(':','')
df["Company"] = df["Company"].str.strip()

#filter out the non-dates
df = df[~df['Date'].str.contains('Knack|Applications')]

#remove days of the week from the date column, this should probably be done above on each date row individually
dirty_date = df["Date"]
clean_date = dirty_date.str.replace("\(.*\)",'',regex=True)
df["Date"] = clean_date

with open('scrape.csv', 'w', newline='') as file:
    df.to_csv('scrape.csv')


from unittest import case
from urllib import response
from flask import Blueprint, render_template, request
import requests
from api import get_data

views = Blueprint('views', __name__)

@views.route('/', methods=["GET", "POST"])
def base():
    if request.method == "POST":
        search_location = request.form.get('searchTerm')
        longitude, latitude, curr_weather, forecast, poll_data = get_data(search_location)

        return render_template("base.html",longitude = longitude, latitude = latitude, curr_weather = curr_weather, forecast = forecast, location_name = search_location, poll_data = poll_data)
    else:
        print("Invalid location or an error has occured.")

    #If not loading by POST supply default values/objects
    #Default Cape Town, South Africa
    search_location = "cape town, south africa"
    longitude, latitude, curr_weather, forecast, poll_data = get_data(search_location)
    return render_template("base.html",longitude = longitude, latitude = latitude, curr_weather = curr_weather, forecast = forecast, location_name = search_location, poll_data = poll_data)


# coding: utf-8

"""
    FIX Gateway Commander API

    # FIX Gateway Commander API REST API to get status and to send control commands to FIX Gateway.  Configuration API allows to view and update FIX Gateway session configuration.                         # noqa: E501

    OpenAPI spec version: 0.1
    Contact: dmtsyganov@devexperts.com
    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

import pprint
import re  # noqa: F401

import six

class AgentsResponseInner(object):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """
    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'agent_type': 'str',
        'name': 'str',
        'time_updated': 'int',
        'sessions_count': 'int',
        'status': 'str'
    }

    attribute_map = {
        'agent_type': 'agentType',
        'name': 'name',
        'time_updated': 'timeUpdated',
        'sessions_count': 'sessionsCount',
        'status': 'status'
    }

    def __init__(self, agent_type=None, name=None, time_updated=None, sessions_count=None, status=None):  # noqa: E501
        """AgentsResponseInner - a model defined in Swagger"""  # noqa: E501
        self._agent_type = None
        self._name = None
        self._time_updated = None
        self._sessions_count = None
        self._status = None
        self.discriminator = None
        if agent_type is not None:
            self.agent_type = agent_type
        if name is not None:
            self.name = name
        if time_updated is not None:
            self.time_updated = time_updated
        if sessions_count is not None:
            self.sessions_count = sessions_count
        if status is not None:
            self.status = status

    @property
    def agent_type(self):
        """Gets the agent_type of this AgentsResponseInner.  # noqa: E501

        Agent type - SERVER or CLIENT  # noqa: E501

        :return: The agent_type of this AgentsResponseInner.  # noqa: E501
        :rtype: str
        """
        return self._agent_type

    @agent_type.setter
    def agent_type(self, agent_type):
        """Sets the agent_type of this AgentsResponseInner.

        Agent type - SERVER or CLIENT  # noqa: E501

        :param agent_type: The agent_type of this AgentsResponseInner.  # noqa: E501
        :type: str
        """

        self._agent_type = agent_type

    @property
    def name(self):
        """Gets the name of this AgentsResponseInner.  # noqa: E501

        Agent name  # noqa: E501

        :return: The name of this AgentsResponseInner.  # noqa: E501
        :rtype: str
        """
        return self._name

    @name.setter
    def name(self, name):
        """Sets the name of this AgentsResponseInner.

        Agent name  # noqa: E501

        :param name: The name of this AgentsResponseInner.  # noqa: E501
        :type: str
        """

        self._name = name

    @property
    def time_updated(self):
        """Gets the time_updated of this AgentsResponseInner.  # noqa: E501

        Unix time since Agent status was updated  # noqa: E501

        :return: The time_updated of this AgentsResponseInner.  # noqa: E501
        :rtype: int
        """
        return self._time_updated

    @time_updated.setter
    def time_updated(self, time_updated):
        """Sets the time_updated of this AgentsResponseInner.

        Unix time since Agent status was updated  # noqa: E501

        :param time_updated: The time_updated of this AgentsResponseInner.  # noqa: E501
        :type: int
        """

        self._time_updated = time_updated

    @property
    def sessions_count(self):
        """Gets the sessions_count of this AgentsResponseInner.  # noqa: E501

        Number of active FIX sessions  # noqa: E501

        :return: The sessions_count of this AgentsResponseInner.  # noqa: E501
        :rtype: int
        """
        return self._sessions_count

    @sessions_count.setter
    def sessions_count(self, sessions_count):
        """Sets the sessions_count of this AgentsResponseInner.

        Number of active FIX sessions  # noqa: E501

        :param sessions_count: The sessions_count of this AgentsResponseInner.  # noqa: E501
        :type: int
        """

        self._sessions_count = sessions_count

    @property
    def status(self):
        """Gets the status of this AgentsResponseInner.  # noqa: E501

        Agent status  # noqa: E501

        :return: The status of this AgentsResponseInner.  # noqa: E501
        :rtype: str
        """
        return self._status

    @status.setter
    def status(self, status):
        """Sets the status of this AgentsResponseInner.

        Agent status  # noqa: E501

        :param status: The status of this AgentsResponseInner.  # noqa: E501
        :type: str
        """

        self._status = status

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value
        if issubclass(AgentsResponseInner, dict):
            for key, value in self.items():
                result[key] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, AgentsResponseInner):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        return not self == other


from wpilib.command import Command
from utilities.drivetrain_mp_controller import DrivetrainMPController
import logging
from constants import LOGGER_LEVEL
logger = logging.getLogger(__name__)
logger.setLevel(LOGGER_LEVEL)

class DrivetrainPathFollower(Command):
    """
    This command will call the path which will go forward. The pickle file should have been created on the PC and placed into the autonomous folder
    to be uploaded with the robot code.
    """
    def __init__(self, robot, path, reverse, pid_kludge=False):
        super().__init__()
        self.requires(robot.driveTrain)

        # Create references to the robot and the path to follow
        self.robot = robot
        self.path = path
        self.reverse = reverse
        self.pidKludge = pid_kludge

        # Control variables
        self.finished = True

        # 4th value in MP's is sample period.  Assume the left and right sides are the same.  The
        # divide by 2 value is used to set the Talon control frames and notifier to twice the rate
        # of the trajectory duration.
        self._streamRate = int(self.path['left'][0][3] / 2)

    def isFinished(self):
        """
        This returns whether or not this command is complete.
        """
        return self.finished

    def initialize(self):
        """
        Create the left and right path follower controller objects and start the process of the
        following the path.
        """
        self.finished = False
        self.robot.driveTrain.initiaizeDrivetrainMotionProfileControllers(self._streamRate)
        if self.pidKludge:
            self.robot.driveTrain.pidKludge()
        self.pathFollower = DrivetrainMPController(self.robot.driveTrain.leftTalon,
                                                   self.path['left'],
                                                   self.robot.driveTrain.rightTalon,
                                                   self.path['right'],
                                                   self.reverse,
                                                   self.robot.driveTrain.MP_SLOT0_SELECT,
                                                   self.robot.driveTrain.MP_SLOT1_SELECT)
        self.pathFollower.start()

    def execute(self):
        """
        If the path followers has finished following the path, mark this command as complete.
        Otherwise, call the control method to update and act upon the controllers state machine.
        """
        if self.pathFollower.isFinished():
            self.finished = True
        else:
            self.pathFollower.control(self.robot.timer.get(), self.robot.smartDashboard)

    def end(self):
        '''
        Exit the DrivetrainMotionProfileControllers
        '''
        self.robot.driveTrain.cleanUpDrivetrainMotionProfileControllers()


import pathlib
import re
import sqlite3
import typing

dirname = pathlib.Path(__file__).resolve().parent

class Database:
    create_table_pattern = re.compile(r"CREATE TABLE (\w+) \((.+)\)")
    tyc_pattern = re.compile(r"^TYC (\d{1,4})-(\d{1,5})-(\d)$")
    tyc_identifier = 0
    right_ascension = 1
    declination = 2
    right_ascension_dot = 3
    declination_dot = 4
    magnitude = 5
    iau_name = 6
    iau_designation = 7
    iau_bayer_designation = 8
    iau_system_position = 9

    def __init__(self, path: pathlib.Path = dirname / "tycho2_m6.db"):
        self.path = str(path)

    def find(
        self,
        minimum_magnitude: float | None,
        maximum_magnitude: float,
    ) -> list[tuple[typing.Any]]:
        with sqlite3.connect(self.path) as connection:
            cursor = connection.cursor()
            cursor.execute(
                " ".join(
                    (
                        "SELECT * FROM catalogue",
                        f"WHERE magnitude < {maximum_magnitude}",
                        *(
                            ()
                            if minimum_magnitude is None
                            else (f"AND magnitude >= {minimum_magnitude}",)
                        ),
                    )
                )
            )
            return cursor.fetchall()


import h5py
import numpy as np
import torch

class EEGDataset:
    _INIT_SIZE = 1000
    _EEG_IND_NAME = '_ind'

    def __init__(self, path, open_type='r'):
        self._path = path
        self._open_type = open_type
        self._f = h5py.File(self._path, self._open_type)

    def initialize(self, items):
        for name, params in items.items():
            dataset = self._f.create_dataset(
                name=name,
                shape=(self._INIT_SIZE,) + params['sample_shape'],
                chunks=params['chunks'],
                maxshape=(None,) + params['sample_shape'],
                dtype=params['type'],
                compression=params['compression'],
            )
            dataset.attrs['len'] = 0

        dataset = self._f.create_dataset(
            name=self._EEG_IND_NAME,
            shape=(self._INIT_SIZE, 2),
            maxshape=(None, 2),
            dtype=np.int64,
        )
        dataset.attrs['len'] = 0

    def append(self, data):
        def append(dataset, sample):
            sample_len = v.shape[0] if len(v.shape) == len(dataset.shape) else 1
            self._resize_maybe(dataset, sample_len)
            dataset[dataset.attrs['len']:(dataset.attrs['len'] + sample_len)] = sample
            dataset.attrs['len'] += sample_len

        eeg_len_before = self._f['eeg'].attrs['len']
        for k, v in data.items():
            if not isinstance(v, np.ndarray):
                v = np.array(v)
            append(self._f[k], v)

        eeg_len_after = self._f['eeg'].attrs['len']
        append(self._f[self._EEG_IND_NAME], np.array([eeg_len_before, eeg_len_after]))

    @property
    def file(self):
        return self._f

    def reset(self):
        self._f = None
        return self

    def open(self):
        if self._f is not None:
            return
        self._f = h5py.File(self._path, self._open_type)

    def _resize_maybe(self, dataset, add_size):
        dataset_length = dataset.attrs['len']
        while dataset_length + add_size > dataset.shape[0]:
            dataset.resize((dataset.shape[0] * 2,) + dataset.shape[1:])

class SubsequenceDataset(torch.utils.data.Dataset):
    def __init__(self, eeg_dataset, subsequence_length):
        self._dataset = eeg_dataset
        self._dataset.open()

        self._ranges = []
        ind_data = self._dataset.file[EEGDataset._EEG_IND_NAME]
        for i, r in enumerate(ind_data[:ind_data.attrs['len']]):
            cnt = (r[1] - r[0]) // subsequence_length
            self._ranges.append(
                np.vstack([
                    np.arange(cnt) * subsequence_length + r[0],
                    (np.arange(cnt) + 1) * subsequence_length + r[0],
                    np.ones(cnt, dtype=np.int64) * i,
                ]).T.copy()
            )
        self._ranges = np.concatenate(self._ranges, axis=0)

    def __getitem__(self, idx):
        self._dataset.open()
        first, last, ind = self._ranges[idx]
        return {
            'eeg': self._dataset.file['eeg'][first:last].T,
        }

    def __len__(self):
        return self._ranges.shape[0]

    def reset(self):
        self._dataset.reset()
        return self

class EpochedDataset(torch.utils.data.Dataset):
    def __len__(self):
        return self._sub_indices.shape[0]

    def reset(self):
        self._dataset.reset()
        return self

    def lmso(self, folds, shuffle=False):
        thinkers = self.thinkers.copy()

        if shuffle:
            np.random.shuffle(thinkers)

        group_size = (len(thinkers) + folds - 1) // folds
        groups = []
        for i in range(folds):
            low = i * group_size
            high = min(len(thinkers), (i + 1) * group_size)
            groups.append(thinkers[low:high])

        for val_idx in range(len(groups)):
            test_idx = (val_idx + 1) % len(groups)

            train_thinkers = []
            for i in range(len(groups)):
                if i == val_idx or i == test_idx:
                    continue
                train_thinkers.extend(groups[i])

            train = self._subset(self._get_indices_for_thinkers(train_thinkers))
            val = self._subset(self._get_indices_for_thinkers(groups[val_idx]))
            test = self._subset(self._get_indices_for_thinkers(groups[test_idx]))

            yield train, val, test

    def loso(self):
        yield from self.lmso(folds=len(np.unique(self.thinkers)))

    @property
    def thinkers(self):
        return np.unique(self._thinkers[self._sub_indices])

    @property
    def person_id(self):
        thinkers = self.thinkers
        assert len(thinkers) == 1
        return int(thinkers[0])

    def _get_indices_for_thinkers(self, thinkers):
        s = set(thinkers)

        res = []
        for i in self._sub_indices:
            if self._thinkers[i] in s:
                res.append(i)

        res = np.array(res)
        return res

class EpochedClassificationDataset(EpochedDataset):
    def __init__(self, eeg_dataset, sub_indices=None):
        self._dataset = eeg_dataset
        self._dataset.open()
        self._thinkers = np.zeros(self._dataset.file['label'].attrs['len'], dtype=int)

        for i in range(self._dataset.file['thinker'].attrs['len']):
            ind = self._dataset.file['_ind'][i]
            self._thinkers[ind[0]:ind[1]] = self._dataset.file['thinker'][i]

        if sub_indices is None:
            self._sub_indices = np.arange(self._thinkers.shape[0], dtype=int)
        else:
            self._sub_indices = sub_indices

    def __getitem__(self, idx):
        self._dataset.open()
        X = self._dataset.file['eeg'][self._sub_indices[idx]].T
        y = self._dataset.file['label'][self._sub_indices[idx]]
        return X, y

    def _subset(self, ind):
        return EpochedClassificationDataset(self._dataset, ind)


from __future__ import print_function

import argparse
import os
import re
import shutil
import ssl
import subprocess
import sys
try:
  from urllib.request import urlretrieve
except ImportError:
  from urllib import urlretrieve
import uuid
import zipfile

class CommonConfig:
  def __init__(self, root_dir, pd_path):
    self.root_dir = root_dir
    self.pd_path = pd_path

    self.flext_dir = os.path.join(self.root_dir, "flext")
    self.py_dir = os.path.join(self.root_dir, "py")
    self.build_dir = os.path.join(self.root_dir, "build")
    self.flext_prefix = os.path.join(self.build_dir, "flext")
    self.py_package_dir = os.path.join(self.build_dir, "py")
    self.python_exe = sys.executable
    self.python_version = sys.version

class MacConfig:
  def __init__(self, common_cfg):
    self.common_cfg = common_cfg

    self.platform = "darwin"
    self.platform_alt = "mac"
    self.compiler = "gcc"

    self.pd_path = self.common_cfg.pd_path or self.find_pd()

    # python executable is at {conda_root}/bin/python
    self.conda_root = os.path.dirname(os.path.dirname(self.common_cfg.python_exe))

    self.flext_build_env = {
      #"BUILDMODE": "debug",
      #"TARGETMODE": "debug",
      "FLEXT_PD_APP": self.pd_path,
      "FLEXT_INSTALL_PATH": self.common_cfg.py_package_dir,
      "FLEXTPREFIX": self.common_cfg.flext_prefix
    }

    self.py_build_env = {
      **self.flext_build_env,
      "PY_CONDA_ROOT": self.conda_root
    }

    self.build_cmd = ("bash", os.path.join(self.common_cfg.flext_dir, "build.sh"))

    self.platform_output_name = "pd-darwin"

  def find_pd(self):
    apps_dir = "/Applications"

    if not os.path.exists(apps_dir):
      return None

    pd_re = re.compile(r"^Pd-(\d+(?:[-.]\d+)*)\.app$")
    ver_delim_re = re.compile(r"[-.]")

    pd_apps = ((app, pd_re.match(app)) for app in os.listdir(apps_dir))
    pd_apps = ((app, map(int, ver_delim_re.split(m.group(1)))) for app, m in pd_apps if m != None)
    pd_apps = list(pd_apps)

    if len(pd_apps) == 0:
      return None

    max_app = max(pd_apps, key = lambda x: x[0])[0]
    return os.path.join(apps_dir, max_app)

class LinuxConfig:
  def __init__(self, common_cfg):
    self.common_cfg = common_cfg

    self.platform = "linux"
    self.platform_alt = "lnx"
    self.compiler = "gcc"

    self.pd_path = self.common_cfg.pd_path or self.find_pd()

    # python executable is at {conda_root}/bin/python
    self.conda_root = os.path.dirname(os.path.dirname(self.common_cfg.python_exe))

    self.flext_build_env = {
      # "BUILDMODE": "debug",
      # "TARGETMODE": "debug",
      "PDPATH": self.pd_path,
      "PDINC": os.path.expanduser("~/usr/include"),
      "PDLIB": os.path.expanduser("~/usr/lib"),
      "FLEXT_INSTALL_PATH": self.common_cfg.py_package_dir,
      "FLEXTPREFIX": self.common_cfg.flext_prefix
    }

    self.py_build_env = {
      **self.flext_build_env,
      "PY_CONDA_ROOT": self.conda_root
    }

    self.build_cmd = ("bash", os.path.join(self.common_cfg.flext_dir, "build.sh"))

    self.platform_output_name = "pd-linux"

  def find_pd(self):
    pd_exe = subprocess.check_output(["which", "pd"]).rstrip().decode("utf-8")
    return os.path.dirname(os.path.dirname(pd_exe))

class WinConfig:
  def __init__(self, common_cfg):
    self.common_cfg = common_cfg

    self.platform = "win"
    self.platform_alt = "win"
    self.compiler = "mingw"

    self.pd_path = self.common_cfg.pd_path or self.find_pd()

    # python executable is at {conda_root}/python.exe
    self.conda_root = os.path.dirname(self.common_cfg.python_exe)

    self.flext_build_env = {
      "FLEXT_INSTALL_PATH": self.common_cfg.py_package_dir,
      "FLEXTPREFIX": self.common_cfg.flext_prefix
    }

    self.py_build_env = {
      **self.flext_build_env
    }

    self.build_cmd = (os.path.join(self.common_cfg.flext_dir, "build.bat"),)

    self.platform_output_name = "pd-msvc"

  def find_pd(self):
    pd_dir = os.path.join(os.environ["ProgramFiles"], "Pd")

    if not os.path.exists(pd_dir):
      return None

    return pd_dir

class SetupError(Exception):
  def __init__(self, message=""):
    super().__init__(message)

class Setup:
  def __init__(self, common_cfg, platform_cfg_ctor, ):
    self.common_cfg = common_cfg
    self.platform_cfg = platform_cfg_ctor(self.common_cfg)

    self.flext_platform_config_path = os.path.join(
      self.common_cfg.flext_dir,
      "buildsys",
      "config-{}-pd-{}.txt".format(self.platform_cfg.platform_alt, self.platform_cfg.compiler)
    )

    self.flext_config_path = os.path.join(self.common_cfg.flext_dir, "config.txt")

    self.py_config_path = os.path.join(self.common_cfg.py_dir, "config.txt")

    self.flext_platform_output_path = os.path.join(self.common_cfg.flext_dir, self.platform_cfg.platform_output_name)

    self.py_platform_output_path = os.path.join(self.common_cfg.py_dir, self.platform_cfg.platform_output_name)

  def __getattr__(self, name):
    if hasattr(self.platform_cfg, name):
      return getattr(self.platform_cfg, name)
    elif hasattr(self.common_cfg, name):
      return getattr(self.common_cfg, name)
    else:
      raise AttributeError(name)

  def validate(self):
    if not self.pd_path:
      raise SetupError("no Pd found. please specify a path with --pd")

    if not os.path.exists(self.pd_path):
      raise SetupError("Pd path {} does not exist")

    if not os.path.isdir(os.path.join(self.conda_root, "conda-meta")):
      raise SetupError("a Conda environment does not seem to be active. did you forget to run `conda activate`?")

  def flext_build(self, ok_codes=(0,), args=()):
    all_args = ("pd", self.platform_cfg.compiler) + args
    return call_with_env(
      self.platform_cfg.build_cmd + all_args,
      self.platform_cfg.flext_build_env,
      ok_codes
    )

  def py_build(self, ok_codes=(0,), args=()):
    all_args = ("pd", self.platform_cfg.compiler) + args
    return call_with_env(
      self.platform_cfg.build_cmd + all_args,
      self.platform_cfg.py_build_env,
      ok_codes
    )

supported_platform_cfgs = {
  "darwin": MacConfig,
  "linux": LinuxConfig,
  "win32": WinConfig
}

flext_url = "https://github.com/SopiMlab/flext/archive/master.zip"
py_url = "https://github.com/SopiMlab/py/archive/master.zip"

def flatten_dir(path):
    items = os.listdir(path)

    if len(items) != 1:
        return

    item = items[0]
    item_path = os.path.join(path, item)

    if not os.path.isdir(item_path):
        return

    subitems = os.listdir(item_path)

    item_path2 = item_path + "_" + str(uuid.uuid4())
    os.rename(item_path, item_path2)
    item_path = item_path2

    for subitem in subitems:
        os.rename(os.path.join(item_path, subitem), os.path.join(path, subitem))

    os.rmdir(item_path)

def download(url, path, ctx):
    try:
        urlretrieve(url, path, context = ctx)
    except TypeError:
        # no context support in python 3
        urlretrieve(url, path)

def download_and_unzip(root_dir, name, url, path, ctx):
    if os.path.exists(path):
        print(name, "already downloaded")
        return

    print("downloading", name)
    the_zip = os.path.join(root_dir, "{}.zip".format(name))
    download(url, the_zip, ctx)
    print("unzipping", name)
    os.mkdir(path)
    with zipfile.ZipFile(the_zip, "r") as zf:
        zf.extractall(path)
    flatten_dir(path)
    os.remove(the_zip)

def call_with_env(args, env, ok_codes=(0,)):
    env1 = os.environ.copy()
    for k in env:
        env1[k] = env[k]

    print("call_with_env")
    print("  args:", args)
    print("  env:", env)

    p = subprocess.Popen(args, env=env1)
    code = p.wait()

    if code not in ok_codes:
        raise Exception("command exited with unexpected code {}".format(code))

    if code != 0:
        print("command exited with nonzero code {}, but this is fine".format(code))

arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--pd", help="path to Pd")
arg_parser.add_argument("--info", help="don't build, just display environment info", action="store_true")
args = arg_parser.parse_args()

if sys.platform not in supported_platform_cfgs:
  print("platform {} is not supported".format(sys.platform))
  sys.exit(1)

setup = Setup(
  common_cfg = CommonConfig(
    root_dir = os.path.dirname(os.path.realpath(__file__)),
    pd_path = args.pd
  ),
  platform_cfg_ctor = supported_platform_cfgs[sys.platform]
)

print("platform config:", type(setup.platform_cfg).__name__)
print("Python version:", setup.python_version)
print("Python executable:", setup.python_exe)

try:
  setup.validate()
except SetupError as e:
  print(e)
  sys.exit(1)

print("Pd path:", setup.pd_path)
#variant = "Purr Data" if is_purr_data else "vanilla"
#print("Pd variant:", variant)
print("Conda root:", setup.conda_root)
print()

if "--info" in sys.argv:
    sys.exit(0)

ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
ssl_ctx.verify_mode = ssl.CERT_REQUIRED
ssl_ctx.check_hostname = True
ssl_ctx.load_default_certs()

if sys.platform == "darwin":
    import certifi
    ssl_ctx.load_verify_locations(
        cafile = certifi.where(),
        capath = None,
        cadata = None
    )

os.chdir(setup.root_dir)

download_and_unzip(setup.root_dir, "flext", flext_url, setup.flext_dir, ssl_ctx)
download_and_unzip(setup.root_dir, "py", py_url, setup.py_dir, ssl_ctx)

print()

if os.path.exists(setup.build_dir):
    print("cleaning build directory")
    shutil.rmtree(setup.build_dir)

os.mkdir(setup.build_dir)

os.chdir(setup.flext_dir)

print("flext_prefix:", setup.flext_prefix)

if os.path.exists(setup.flext_prefix):
    print("directory", setup.flext_prefix, "exists, so assuming flext was already installed")
else:
    if not os.path.exists(setup.flext_platform_config_path):
        print("creating flext system config")
        setup.flext_build((0, 2))

    if not os.path.exists(setup.flext_config_path):
        print("creating flext package config")
        setup.flext_build((0, 2))

    if not os.path.exists(setup.flext_platform_output_path):
        print("building flext")
        setup.flext_build()

    print("installing flext")
    setup.flext_build(args=("install",))

os.chdir(setup.py_dir)

if os.path.exists(setup.py_platform_output_path):
    print("directory", setup.py_platform_output_path, "exists, so assuming py was already built")
else:
    if not os.path.exists(setup.py_config_path):
        print("creating py config")
        setup.py_build((0, 2))

    print("building py")
    setup.py_build()

print("packaging files")
os.mkdir(setup.py_package_dir)
setup.py_build(args=("install",))

print()
print("done:", setup.py_package_dir)

print("flext_platform_output_path:", setup.flext_platform_output_path)
print("py_platform_output_path:", setup.py_platform_output_path)
print("py_package_dir:",setup.py_package_dir)


# This creates some configuration files for the experiment, because doing it
# manually would be tedious. Also we can try different parameters later.

import json
import os

hidden_layer_nums = [3, 5, 8, 10, 13, 15, 17, 20, 22]
algos = ["dnnre", "alpa"]
model_templ = {
    "data_path": "data/",
    "nwtype": "ff",
    "hidden_layer_units": [],
    "hidden_layer_activations": [],
    "use_class_weights": True,
    "batch_size": 1,
    "dropout": 0.2,
    "val_split": 0.25,
    "learning_rate": 0.001,
    "epochs": 0,
    "output_path": "model/"
}
ext_templ = {
    "data_path": "data/",
    "trained_model_path": "model/",
    "algorithm": "",
    "rules_dir": "rules/"
}

# generate model configs
for idx, h in enumerate(hidden_layer_nums):
    config_name = f"model_{h}h.json"
    # flipped parabola with center h // 2 and 0 at 0 and h
    # flattened to a maximum of 100 hidden units
    # increase size of hidden layers until center and then decrease
    hidden_layer_units = [
        min((-i * (i + 1 - h)) // 2 + 64, 80)
        for i in range(0, h)
    ]
    # hidden_layer_units = [64] * h
    # always use relu
    hidden_layer_activations = ["relu" for _ in range(h)]
    print(f"Creating config with {hidden_layer_units}")
    config_contents = model_templ.copy()
    config_contents["hidden_layer_units"] = hidden_layer_units
    config_contents["hidden_layer_activations"] = hidden_layer_activations
    # longer training for deeper networks
    config_contents["epochs"] = max(h * 25, 200)
    config_contents["learning_rate"] = 1.0 / h * 0.001
    config_contents["output_path"] = os.path.join(
        config_contents["output_path"], config_name[:-5])
    with open(config_name, "w+") as file:
        json.dump({"model": config_contents}, file, indent=4)

for idx, h in enumerate(hidden_layer_nums):
    model_name = f"model_{h}h"
    config_contents = ext_templ.copy()
    model_path = os.path.join(
        config_contents["trained_model_path"],
        model_name
    )
    config_contents["trained_model_path"] = model_path
    for algo in algos:
        config_name = f"ex_{algo}_{h}h.json"
        rules_path = os.path.join(
            "rules",
            f"{model_name}_{algo}"
        )
        print(f"writing config {config_name}")
        config_contents["algorithm"] = algo
        config_contents["rules_dir"] = rules_path
        with open(config_name, "w+") as file:
            json.dump({"extraction": config_contents}, file, indent=4)


from flask import Blueprint,request,redirect,render_template,url_for
from models import Cliente
from forms import ClienteForm
from app import db

appCliente = Blueprint('appcliente', __name__, template_folder="templates")
APP_BASE = 'cliente'

@appCliente.route('/')
@appCliente.route('/index')
def bienvenida():
    return render_template('bienvenida.html')

@appCliente.route(f'/{APP_BASE}')
@appCliente.route(f'/{APP_BASE}/index{APP_BASE}')
def inicial():
    clientes = Cliente.query.all()
    return render_template(f'index{APP_BASE}.html',clientes=clientes)

@appCliente.route(f"/{APP_BASE}/editar/<int:id>",methods=["GET","POST"])
def editar(id):
    cliente = Cliente.query.get_or_404(id)
    clienteForm = ClienteForm(obj=cliente)
    if request.method == "POST":
        if clienteForm.validate_on_submit():
            clienteForm.populate_obj(cliente)
            db.session.commit()
            return redirect(url_for(f'app{APP_BASE}.inicial'))
    return render_template(f'editar{APP_BASE}.html',forma=clienteForm)

@appCliente.route(f"/{APP_BASE}/eliminar/<int:id>")
def eliminar(id):
    cliente = Cliente.query.get_or_404(id)
    db.session.delete(cliente)
    db.session.commit()
    return redirect(url_for(f'app{APP_BASE}.inicial'))


import urllib.request
import json
import unidecode
import math

def elements_ordenation(total_elements):
    TOTAL_ELEMENTS = total_elements

    custo_inicial = TOTAL_ELEMENTS**2 #Definido um custo inicial, ou seja, a maior quantidade de solicitações possível
    for o in range (1,11): #Vai ser no máximo 25 destinos (4*25)
        for d in range (1,26): #Vai ser no máximo 10 origens (10*10)
            if (o*d<=100): #Limite de 100 elementos
                custo = math.ceil(TOTAL_ELEMENTS/o)*math.ceil(TOTAL_ELEMENTS/d) #Ceil arredonda para cima. Multiplicação de solicitações verticais e horizontais
                if (custo_inicial>custo): #busca o menor custo
                    custo_inicial = custo
                    N_ROWS_PER_ITERATION = o
                    N_COLUMNS_PER_ITERATION = d
                else:
                    pass
            else:
                pass

    TOTAL_ROWS_ITERATION    = int(TOTAL_ELEMENTS/N_ROWS_PER_ITERATION)
    TOTAL_COLUMNS_ITERATION = int(TOTAL_ELEMENTS/N_COLUMNS_PER_ITERATION)

    RESTO_ROWS    = TOTAL_ELEMENTS % N_ROWS_PER_ITERATION
    RESTO_COLUMNS = TOTAL_ELEMENTS % N_COLUMNS_PER_ITERATION

    origem = []
    destino = []
    quantidade_blocos = -1
    elementos = {}
    for r in range(TOTAL_ROWS_ITERATION + (1 if RESTO_ROWS != 0 else 0)):
        for c in range(TOTAL_COLUMNS_ITERATION + (1 if RESTO_COLUMNS != 0 else 0)):
            quantidade_blocos += 1
            origem = []
            destino = []
            for i in range(N_ROWS_PER_ITERATION):
                if r > TOTAL_ROWS_ITERATION-1:
                    if i+r*N_ROWS_PER_ITERATION >= TOTAL_ELEMENTS:
                        break
                origem.append(i+r*N_ROWS_PER_ITERATION)
            for j in range(N_COLUMNS_PER_ITERATION):
                if c > TOTAL_COLUMNS_ITERATION-1:
                    if j+c*N_COLUMNS_PER_ITERATION >= TOTAL_ELEMENTS:
                        break
                destino.append(j+c*N_COLUMNS_PER_ITERATION)
                elementos.update({quantidade_blocos:{'origem':origem, 'destino':destino}})
    return(elementos)

def get_distances_json(origins, destinations, api_key):

    ## build the url for the API call
    ELEVATION_BASE_URL = 'https://maps.googleapis.com/maps/api/distancematrix/json'

    # REMOVE ACCENTS AND REPLACE SPACES FOR + IN ORIGINS
    clean_origins = []
    for i in range(len(origins)):
        clean_origins.append(unidecode.unidecode(origins[i]).replace(" ", "+"))

    # COCATENATING ALL ORIGINS
    ORIGINS = 'origins='
    for clean_origin in clean_origins:
        ORIGINS += clean_origin + "|"
    ORIGINS = ORIGINS[:-1]

    # REMOVE ACCENTS AND REPLACE SPACES FOR + IN DESTINATIONS
    clean_destinations = []
    for i in range(len(destinations)):
        clean_destinations.append(unidecode.unidecode(destinations[i]).replace(" ", "+"))

    # COCATENATING ALL DESTINATIONS
    DESTNATIONS = 'destinations='
    for clean_destination in clean_destinations:
        DESTNATIONS += clean_destination + "|"
    DESTNATIONS = DESTNATIONS[:-1]

    URL_PARAMS = ORIGINS + '&' + DESTNATIONS + '&mode=driving&language=pt-BR&key=%s' % (api_key)

    url = ELEVATION_BASE_URL + "?" + URL_PARAMS

    print(url)

    with urllib.request.urlopen(url) as f:
        response = json.loads(f.read().decode())

    status = response['status']

    distance = []
    duration = []
    for row in response['rows']:
        for element in row['elements']:
            try:
                distance.append(element['distance']['value'])
                duration.append(element['duration']['value'])
            except KeyError or IndexError:
                distance.append("ERRO")
                duration.append("ERRO")
    return (distance, duration, status)

def convtime(t):
    horas = int(t // 3600)
    horas_rest = t % 3600
    minutos = int(horas_rest // 60)
    segundos_rest = int(round(horas_rest % 60, 0))
    print("Tempo de Processamento // " + str(horas) + ":" + str(minutos) + ":" + str(segundos_rest))
    return


# line = linecache.getline('C:/Users/THINK/Desktop/order_header.sql')
# for i in line
# f = open('C:/Users/THINK/Desktop/order_header.sql','r')
# for i in f.readlines():
#     print(i)
import datetime
import re

# with open('C:/Users/THINK/Desktop/order_header.sql','r',encoding="utf-8") as f1:
#     for i in f1:
#         print(i)

def test():
     file_name = 'C:/Users/THINK/Desktop/order_header.sql'
     # file = open(file_name, 'r',encoding='utf-8')
     acnum = [];
     time_res = [];
     lnum = 0

     def trans_time(time):
         t1 = datetime.datetime.strptime(time, '%y/%m/%d %H:%M:%S')
         return t1

     with open(file_name, 'r', encoding="utf-8") as file:

         for (num, line) in enumerate(file):
             print(num)
             print(line)
             if (re.search(r'^(.*)BEGINNING SIM PROCEDURE(.*)$', line)):
                 m = re.search(r'^(.*)BEGINNING SIM PROCEDURE(.*)$', line)
                 print
                 'Step %d:' % (lnum), m.group(0);
                 lnum += 1
                 acnum.append(trans_time(line[0:17]))
             elif (re.search(r'^(.*)CP_W(.*)$', line)):
                 m = re.search(r'^(.*)CP_W(.*)$', line)
                 print
                 'Step %d:' % (lnum), m.group(0);
                 lnum += 1
                 acnum.append(trans_time(line[0:17]))

         file.close()
         if (re.search(r"^(.*)backout(.*)$", file_name)):
             time_res.append((acnum[2] - acnum[0]).seconds / 60)
             time_res.append((acnum[4] - acnum[3]).seconds / 60)
             time_res.append((acnum[6] - acnum[5]).seconds / 60)
             time_res.append(((acnum[8] - acnum[7]) + (acnum[10] - acnum[9]) + (acnum[13] - acnum[11])).seconds / 60)
             print
             "\n3). sim --proc update --action apply to \"CP_WARNING\" %s mins" % (time_res[0])
             print
             "4). sim --proc update --action resume to  \"VERIFY_SOFTWARE\" %s mins" % (time_res[1])
             print
             "5). sim --proc update --action resume to  \"COMMIT\" %s mins" % (time_res[2])
             print
             "8). Backout from RXX to RXX  %s mins" % (time_res[3])
         elif (re.search(r"^(.*)rollback(.*)$", file_name)):
             time_res.append((acnum[2] - acnum[0]).seconds / 60)
             time_res.append((acnum[4] - acnum[3]).seconds / 60)
             time_res.append((acnum[6] - acnum[5]).seconds / 60)
             time_res.append((acnum[8] - acnum[7]).seconds / 60)
             time_res.append(((acnum[10] - acnum[9]) + (acnum[12] - acnum[11]) + (acnum[15] - acnum[13])).seconds / 60)
             print
             "\n3). sim --proc update --action apply to \"CP_WARNING\" %s mins" % (time_res[0])
             print
             "4). sim --proc update --action resume to  \"VERIFY_SOFTWARE\" %s mins" % (time_res[1])
             print
             "5). sim --proc update --action resume to  \"COMMIT\" %s mins" % (time_res[2])
             print
             "6). sim --proc update --action commit to end of Patch %s mins" % (time_res[3])
             print
             "8). Rollback from RXX to RXX %s mins" % (time_res[4])
     # if __name__ == '__main__':
     #     pass

print('运行前')
test()
print('运行后')