# -*- coding: utf-8 -*-
"""
Created on Mon Oct 28 21:11:38 2019

@author: Paulo
"""

from PIL import Image
from nightmares.images_defines import GDOT_SAVE_FOLDER
from nightmares.utils import mkdirs
from os.path import join, isdir
from os import listdir as ls
import tqdm

OUTPUT_FOLDER = "C:/Users/Paulo/Desktop/Python shit/GarfieldNightmares/outputpng"

def convert_to(source=GDOT_SAVE_FOLDER, dest=OUTPUT_FOLDER, ext="png"):
    """ Convert .gif images from GDOT to .png using PIL.
        Arguments:
            source: path to the folder where the /year/month directories are.
            dest: destination/save path.
            ext: extension to save. See PIL.Image for supported extensions.
    """
    for year_folder in tqdm.tqdm(ls(source)):
        mkdirs(dest,[year_folder])
        year_save_folder = join(dest, year_folder)
        year_folder = join(source, year_folder)
        if not isdir(year_folder): continue
        for month_folder in ls(year_folder):
            mkdirs(year_save_folder,[month_folder])
            month_save_folder = join(year_save_folder, month_folder)
            month_folder = join(year_folder, month_folder)
            if not isdir(month_folder): continue
            images = ls(month_folder)
            for image in images:
                image_filepath = join(month_folder, image)
                with Image.open(image_filepath) as img:
                    image_save_path = join(month_save_folder, image.split(".")[0] + "." + ext)
                    img.save(image_save_path)

if __name__ == "__main__":
    convert_to()


from json import dumps

from flask import Response
from flask import Blueprint, render_template

from .db import query_db

from .variables import genres as GENRES

scatter_routes = Blueprint('scatter_routes', __name__,
                           template_folder='templates')

@scatter_routes.route("/scatter/gross-time-genre/year/<year>")
@scatter_routes.route("/scatter/gross-time-genre/<genre>")
@scatter_routes.route("/scatter/gross-time-genre/")
def gross_time_genre(year="", genre=""):
    if genre:
        return render_template("scatter_gross_time_genre_one.html", genre=genre, genres=GENRES)
    else:
        return render_template("scatter_gross_time_genre_all.html", genres=GENRES, year=year)

@scatter_routes.route("/scatter/year-genre/")
def year_genre(film_index=""):
    return render_template("scatter_year_genre.html")

@scatter_routes.route("/api/scatter/gross-time-genre/year/<year>")
@scatter_routes.route("/api/scatter/gross-time-genre/<genre>")
@scatter_routes.route("/api/scatter/gross-time-genre/")
def APIgross_time_genre(genre=None, year=None):
    if year:
        query = """
            SELECT "Release Date", genre, "Worldwide Gross", "movie_title", "director_name", "Production Budget"
            FROM movies
            WHERE "Release Date" LIKE "{}-%"
            """
        query = query.format(year)
    else:
        query = """
            SELECT "Release Date", genre, "Worldwide Gross", "movie_title", "director_name", "Production Budget"
            FROM movies
            WHERE "Worldwide Gross" > 40000000 {}
        """
        if genre:
            if genre.lower() == "thriller":
                genre = "Thriller/Suspense"
            elif genre.lower() == "concert":
                genre = "Concert/Performance"
            query = query.format("AND \"genre\"=\"{}\"".format(genre.title()))
        else:
            query = query.format("")

    result = query_db(query)
    return Response(dumps(result), mimetype="application/json")

@scatter_routes.route("/api/scatter/year-genre/")
def APIyear_genre():
    query = """
        SELECT strftime("%Y", "Release Date") AS YEAR, genre, count(genre)
        FROM movies
        WHERE "Worldwide Gross" > 0
        GROUP BY YEAR, genre
    """
    result = query_db(query)
    processed_result = []
    max_freq = 0
    year_min_max = [2017, 0]
    genres = []

    for item in result:
        if item[2] > max_freq:
            max_freq = item[2]

        if int(item[0]) < year_min_max[0]:
            year_min_max[0] = int(item[0])
        if int(item[0]) > year_min_max[1]:
            year_min_max[1] = int(item[0])

        if item[1] not in genres:
            genres.append(item[1])

        processed_result.append([item[0], item[1], item[2]])

    return Response(dumps({"data": processed_result, "year_min_max": year_min_max, "max": max_freq, "genres": genres}), mimetype="application/json")


from losses import Loss
from layers import Layer
from tqdm import trange

class Network:
    def __init__(self):
        self.layers = []
        self.loss = None
        self.loss_prime = None
        self.err_log = []
        self.err_log_val = []

    # add layer to network
    def add(self, layer: Layer):
        self.layers.append(layer)

    # set loss to use
    def use(self, loss: Loss):
        self.loss = loss

    # predict output for given input
    def predict(self, input_data):
        # sample dimension first
        samples = len(input_data)
        result = []

        # run network over all samples
        for i in range(samples):
            # forward propagation
            output = input_data[i]
            for layer in self.layers:
                output = layer.forward_propagation(output)
            result.append(output)

        return result

    # train the network

    def fit(self, x_train, y_train, epochs=5, learning_rate=0.1):
        """
        Fit function does the training.
        Training data is passed 1-by-1 through the network layers during forward propagation.
        Loss (error) is calculated for each input and back propagation is performed via partial
        derivatives on each layer.
        """
        # sample dimension first
        samples = len(x_train)
        # samples_val = len(x_val)
        t = trange(epochs, desc="Error | Epoch", leave=True)

        # training loop
        for i in t:
            err = 0
            for j in range(samples):
                # forward propagation
                output = x_train[j]
                for layer in self.layers:
                    output = layer.forward_propagation(output)

                # compute loss (for display purpose only)
                err += self.loss(y_train[j], output)

                # backward propagation
                error = self.loss(y_train[j], output, derivative=True)
                for layer in reversed(self.layers):
                    error = layer.backward_propagation(error, learning_rate)

            # err_val = 0
            # for j in range(samples_val):
            #     # forward propagation
            #     output = x_val[j]
            #     for layer in self.layers:
            #         output = layer.forward_propagation(output)

            #     # compute loss (for display purpose only)
            #     err_val += self.loss(y_val[j], output)

            # calculate average error on all samples
            err /= samples
            # err_val /= samples_val

            self.err_log.append(err)
            # self.err_log_val.append(err_val)
            t.set_description(
                f"Epoch: {i} | Train loss: {err:.4}",
                refresh=True,
            )


# coding=utf-8
import requests
import json
# url = 'http://www.qq.com'
proxies = {'http': '118.31.220.3:8080'}
url = 'http://www.bisige8.com/forum.php'
params = {'x': '27835'}
headers = {'User-Agent': "(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
response = requests.get(url, params=params, proxies=proxies, headers=headers)

print(response.status_code)


import struct
from io import BytesIO

structure = {
    'source_port': ('>H', 2),
    'destination_port': ('>H', 2),
    'sequence_number': ('>I', 4),
    'acknowledgment_number': ('>I', 4),
    'data_offset': ('>H', 2),  # read two bytes and fill flags structure ad well
    'window_size': ('>H', 2),
    'checksum': ('>H', 2),
    'urgent_pointer': ('>H', 2)
}

class TCPDecoder:
    def __init__(self) -> None:
        self._stream = BytesIO()
        self.res = {}

    def _unpack(self, fmt, num):
        return struct.unpack(fmt, self._stream.read(num))[0]

    def get_source_port(self):
        fmt, num = structure['source_port']
        return self._unpack(fmt, num)

    def get_destination_port(self):
        fmt, num = structure['destination_port']
        return self._unpack(fmt, num)

    def get_sequence_number(self):
        fmt, num = structure['sequence_number']
        return self._unpack(fmt, num)

    def get_acknowledgment_number(self):
        fmt, num = structure['acknowledgment_number']
        return self._unpack(fmt, num)

    @staticmethod
    def get_decoded_flags(flags: str):
        return {
            'NS': flags[0],
            'CWR': flags[1],
            'ECE': flags[2],
            'URG': flags[3],
            'ACK': flags[4],
            'PSH': flags[5],
            'RST': flags[6],
            'SYN': flags[7],
            'FIN': flags[8]
        }

    def get_data_offset(self):
        # header length consists of 4 bits
        fmt, num = structure['data_offset']
        _unpacked = self._unpack(fmt, num)

        data = format(_unpacked, 'b')
        self.res['data_offset'] = int(data[:4], 2)
        self.res['reserved'] = data[4:7]

        flags = data[7:]

        self.res['flags'] = self.get_decoded_flags(flags)

    def get_window_size(self):
        fmt, num = structure['window_size']
        return self._unpack(fmt, num)

    def get_checksum(self):
        fmt, num = structure['checksum']
        return hex(self._unpack(fmt, num))

    def get_urgent_pointer(self):
        fmt, num = structure['urgent_pointer']
        return hex(self._unpack(fmt, num))

    def get_options(self):
        # todo: implementation
        # The length of this field is determined by the data offset field.
        # Depending on Option-Kind value, the next two fields may be set.
        options = {
            'option_kind': None,  # required
            'option_length': None,
            'option_data': None
        }
        return options

    def decode(self, row_data: bytes):
        self._stream.write(row_data)
        self._stream.seek(0)

        self.res = dict(
            source_port=self.get_source_port(),
            destination_port=self.get_destination_port(),
            sequence_number=self.get_sequence_number(),
            acknowledgment_number=self.get_acknowledgment_number(),
        )
        self.get_data_offset()
        self.res['window_size'] = self.get_window_size()
        self.res['checksum'] = self.get_checksum()
        self.res['urgent_pointer'] = self.get_urgent_pointer()
        self.res['options'] = self.get_options()
        return self.res

tcp_decoder = TCPDecoder()
with open('data.raw', 'rb') as f:
    content = f.read()

print(tcp_decoder.decode(content))


import boto3
from pprint import pprint

aws_mag_console = boto3.session.Session(profile_name=None)
ec2_client = aws_mag_console.client(service_name="ec2", region_name="us-east-1")

response = ec2_client.describe_instances()['Reservations']

for each_item in response:
    for each in each_item['Instances']:
        print ("==============================================")
        print ("The image id is: {}\nThe Instanceid is: {}\nThe instance type is: {}\nThe launch time is: {}\n".format(each['ImageId'],each['InstanceId'],each['InstanceType'],each['LaunchTime']))


import sys
from collections import deque

maximum = 100000
li = [0 for i in range(maximum+1)] # 이동거리를 저장하기 위한 배열
n, k = map(int,sys.stdin.readline().split())
queue = deque()
def bfs():
    queue.append(n) # 시작점을 queue 에 저장한다.
    while  queue:
        x = queue.popleft() # 이동거리를 x로 두고
        if x == k: # x가 도착점이랑 같아지면 출력
            print(li[x])
            break
        for i in (x - 1, x + 1, x * 2): # -1,+1,x2 하나씩 돌려보면서 queue에 넣는다.
            if 0 <= i <= maximum and not li[i]:
                li[i] = li[x] + 1 # 이동 거리를 하나씩 늘려서 저장한다.
                queue.append(i) # queue에 얼마나 이동했는지 그 거리를 저장한다.

bfs()


import collections
n = int(input())
dic = []
for i in range(n):
    dic.append(input())
c = collections.Counter(dic)
maxCount = max(c.items(), key=lambda x: x[1])[1]
maxWord = max(c.items(), key=lambda x: x[1])[0]
ans = []
for k, v in c.items():
    if v == maxCount:
        ans.append(k)
ans.sort()
for a in ans:
    print(a)


from typing import Sequence
from typing import Tuple

import tensorflow as tf

def tf_split_on_label(
    dataset: tf.data.Dataset, labels_in: Sequence[int]
) -> Tuple[tf.data.Dataset, tf.data.Dataset]:
    """
    Allows to split a tensoflow dataset in in-dataset and
    out-dataset according to labels_in
    Args:
        dataset: a tensoflow dataset
        labels_in: array of 'normal' labels
    Returns:
        a tuple of split datasets (dataset_in, dataset_out),
    """

    if not isinstance(dataset, tf.data.Dataset):
        raise ValueError("Invalid dataset type")

    return (
        dataset.filter(
            lambda x, y: tf.reduce_max(tf.gather(y, tf.constant(labels_in))) == 1
        ),
        dataset.filter(
            lambda x, y: tf.reduce_max(tf.gather(y, tf.constant(labels_in))) == 0
        ),
    )


import json
import math
from pandas.io.json import json_normalize
from src.data.fetch_trend_data_utils import display_max_cols, save_dictionary_to_csv

display_max_cols(10)
jfile = "/home/randilu/fyp_impact analysis module/impact_analysis_module/data/external/events/Kelani_Valley_Plantaitions_PLC_v2.json"
with open(jfile, 'r') as f:
    jdata = json.load(f)

events = jdata['events']
events_df = json_normalize(events)
print(events_df)
events_df.columns = events_df.columns.map(lambda x: x.split(".")[-1])

for i, row in events_df.iterrows():
    list = events_df['keywords'][i]
    events_df['keywords'][i] = list[0]

# events_df.drop(columns='keywords', inplace=True)
events_df.rename(columns={'content': 'event'}, inplace=True)
events_df.rename(columns={'keywords': 'keyword_1'}, inplace=True)
events_df.drop_duplicates(subset='keyword_1', keep="last", inplace=True)
events_df.reset_index(drop=True, inplace=True)
print(events_df)

events = events_df[['event', 'keyword_1']]
# events.drop_duplicates(subset='keyword_1', keep="last", inplace=True)
# events.reset_index(drop=True, inplace=True)
events.to_csv(
    '/home/randilu/fyp_impact analysis module/impact_analysis_module/src/data/dictionaries/event_dictionary.csv',
    sep=',', encoding='utf-8', index=False)
print(events)

df_of_kw_sent = events_df[['keyword_1', 'sentiment']]
print(df_of_kw_sent)
df_of_kw_sent['sentiment'] = df_of_kw_sent['sentiment'].apply(lambda x: 1 if x > 0 else -1 if x == 0 else 0)
print(df_of_kw_sent)
df_of_kw_sent = df_of_kw_sent[df_of_kw_sent.sentiment != 0.0]
df_of_kw_sent.reset_index(drop=True, inplace=True)
kw_sent_list = []

for row in df_of_kw_sent.iterrows():
    index, data = row
    kw_sent_list.append(data.tolist())

print(kw_sent_list)


"""This module contains the raw datasets.
The datasets are loaded in memory to be reusable by other functions.
"""
from datetime import datetime

import polars as pl

from src.data.data_extract import (
    get_bs_data,
    get_company_data,
    get_departement_geographical_data,
    get_naf_nomenclature_data,
    get_user_data,
)

# Load all needed data
BSDD_DATA = get_bs_data(
    "get_bsdd_data.sql",
)
BSDA_DATA = get_bs_data(
    "get_bsda_data.sql",
)
BSFF_DATA = get_bs_data(
    "get_bsff_data.sql",
)
BSDASRI_DATA = get_bs_data(
    "get_bsdasri_data.sql",
)

ALL_BORDEREAUX_DATA = pl.concat(
    [BSDD_DATA, BSDA_DATA, BSFF_DATA, BSDASRI_DATA], how="diagonal"
)

COMPANY_DATA = get_company_data()
USER_DATA = get_user_data()

DEPARTEMENTS_GEOGRAPHICAL_DATA = get_departement_geographical_data()
NAF_NOMENCLATURE_DATA = get_naf_nomenclature_data()

DATA_UPDATE_DATE = datetime.now()


import json
import shutil
from datetime import timedelta
from pathlib import Path
from typing import Union, List
from unittest import mock

import pytest
from django.conf import settings
from django.utils import timezone

from app_root.players.models import (
    PlayerShipOffer,
    PlayerFactory,
    PlayerFactoryProductOrder,
    PlayerJob,
)
from app_root.players.utils_import import InitdataHelper
from app_root.servers.models import RunVersion, SQLDefinition, EndPoint, TSProduct
from app_root.servers.utils_import import SQLDefinitionHelper
from app_root.strategies.commands import ShopPurchaseItem
from app_root.strategies.dumps import ts_dump, ts_dump_factory
from app_root.strategies.managers import (
    jobs_find,
    trains_find_match_with_job,
    trains_max_capacity,
    jobs_find_union_priority,
    daily_offer_get_slots,
    factory_order_product,
    factory_collect_product,
)
from app_root.strategies.utils import Strategy
from app_root.users.models import User
from app_root.utils import get_curr_server_str_datetime_s
from core.tests.factory import AbstractFakeResp
from core.utils import convert_datetime, hash10

from core.utils import create_datetime as dt

def prepare(initdata_filepath: Union[List[Path], Path]):
    user = User.objects.create_user(
        username="test", android_id="test", game_access_token="1", player_id="1"
    )
    version = RunVersion.objects.create(user_id=user.id, level_id=1)

    db_helper = SQLDefinitionHelper(version=version)
    instance = SQLDefinition.objects.create(
        version="206.013",
        checksum="077e2eff27bdd5cb079319c6d40f0916d9671b20",
        # url='https://cdn.trainstation2.com/client-resources/client-data-206.013.sqlite',
        # download_path=settings.DJANGO_PATH / 'fixtures' / '206.013.sqlite'
        url="https://cdn.trainstation2.com/client-resources/client-data-207.003.sqlite",
        download_path=settings.DJANGO_PATH / "fixtures" / "207.003.sqlite",
    )
    db_helper.read_sqlite(instance=instance)

    init_helper = InitdataHelper(version=version)
    if not isinstance(initdata_filepath, list):
        initdata_filepath = [initdata_filepath]

    for filepath in initdata_filepath:
        init_helper.parse_data(data=filepath.read_text(encoding="UTF-8"))

    EndPoint.objects.create(
        name=EndPoint.ENDPOINT_COMMAND_PROCESSING,
        name_hash=hash10(EndPoint.ENDPOINT_COMMAND_PROCESSING),
        url="a",
    )

    return version

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filenames",
    [
        [
            "init_data/gaolious_2022.12.30-1.json",
            "init_data/gaolious_2022.12.30-2.json",
        ]
    ],
)
def test_ship_offer(multidb, init_filenames):
    ###########################################################################
    # prepare
    paths = [settings.DJANGO_PATH / "fixtures" / path for path in init_filenames]
    version = prepare(initdata_filepath=paths)

    assert version
    assert version.guild_id
    assert PlayerShipOffer.objects.count() == 1

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filenames",
    [
        [
            "strategies/contract_check/2_2.json",
            "strategies/contract_check/2_3.json",
        ]
    ],
)
def test_strategy(multidb, init_filenames):
    ###########################################################################
    # prepare
    class FakeResp(AbstractFakeResp):
        text = """{"Success":true,"RequestId":"0a645537-2b6d-4c99-81ef-d968877ca303","Time":"2023-01-23T12:43:06Z","Data":{"CollectionId":1,"Commands":[]}}"""

    paths = [settings.DJANGO_PATH / "fixtures" / path for path in init_filenames]
    version = prepare(initdata_filepath=paths)

    txt = paths[-1].read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now
        ts_dump(version=version)

        with mock.patch("app_root.mixins.CrawlingHelper.post") as post:
            post.side_effect = FakeResp
            strategy = Strategy(user_id=version.user_id)
            strategy.run()

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filename, event_count, union_count, story_count, side_count",
    [
        ("strategies/6/6_0.json", 0, 0, 3, 0),
        ("init_data/gaolious1_2022.12.29.json", 0, 0, 4, 0),
        ("init_data/gaolious_2022.12.30-1.json", 3, 4, 6, 5),
        ("init_data/gaolious_2023.01.09_contract.json", 0, 4, 4, 5),
        ("init_data/gaolious_2023.01.09_gifts.json", 0, 3, 4, 5),
        ("init_data/gaolious_2023.01.11_jobs.json", 3, 3, 4, 5),
        ("init_data/gaolious_2023.01.14_fulltest.json", 3, 4, 4, 5),
        ("init_data/gaolious_2023.01.14_idle_destinations.json", 3, 4, 4, 5),
        ("init_data/gaolious_2023.01.14_results.json", 3, 4, 4, 5),
        ("init_data/gaolious_2023.01.11_jobs.json", 3, 3, 4, 5),
    ],
)
def test_prepare_material(
    multidb, init_filename, event_count, union_count, story_count, side_count
):
    ###########################################################################
    # prepare
    initdata_filepath = settings.DJANGO_PATH / "fixtures" / init_filename

    version = prepare(initdata_filepath=initdata_filepath)

    jobs = list(jobs_find(version, event_jobs=True))
    assert len(jobs) == event_count, "event only"
    for job in jobs:
        trains = list(trains_find_match_with_job(version=version, job=job))
        assert trains

    jobs = list(jobs_find(version, union_jobs=True))
    assert len(jobs) == union_count, "union only"
    for job in jobs:
        trains = list(trains_find_match_with_job(version=version, job=job))
        assert trains

    jobs = list(jobs_find(version, story_jobs=True))
    assert len(jobs) == story_count, "story only"
    for job in jobs:
        trains = list(trains_find_match_with_job(version=version, job=job))
        assert trains

    jobs = list(jobs_find(version, side_jobs=True))
    assert len(jobs) == side_count, "side only"
    for job in jobs:
        trains = list(trains_find_match_with_job(version=version, job=job))
        assert trains

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filename, event_count, union_count, story_count, side_count",
    [
        ("strategies/6/6_0.json", 0, 0, 3, 0),
        ("init_data/gaolious1_2022.12.29.json", 0, 0, 4, 0),
        ("init_data/gaolious_2022.12.30-1.json", 3, 4, 6, 5),
        ("init_data/gaolious_2023.01.09_contract.json", 0, 4, 4, 5),
        ("init_data/gaolious_2023.01.09_gifts.json", 0, 3, 4, 5),
        ("init_data/gaolious_2023.01.11_jobs.json", 3, 3, 4, 5),
        ("init_data/gaolious_2023.01.14_fulltest.json", 3, 4, 4, 5),
        ("init_data/gaolious_2023.01.14_idle_destinations.json", 3, 4, 4, 5),
        ("init_data/gaolious_2023.01.14_results.json", 3, 4, 4, 5),
    ],
)
def test_find_job_materials(
    multidb, init_filename, event_count, union_count, story_count, side_count
):
    ###########################################################################
    # prepare

    initdata_filepath = settings.DJANGO_PATH / "fixtures" / init_filename

    version = prepare(initdata_filepath=initdata_filepath)

    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now

        # for job in jobs_find(version=version):
        #     materials = jobs_find_sources(version=version, job=job)
        #
        #     assert materials

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filename, event_count, union_count, story_count, side_count",
    [
        ("strategies/6/6_0.json", 0, 0, 3, 0),
        ("init_data/gaolious1_2022.12.29.json", 0, 0, 4, 0),
        ("init_data/gaolious_2022.12.30-1.json", 3, 4, 6, 5),
        ("init_data/gaolious_2023.01.09_contract.json", 0, 4, 4, 5),
        ("init_data/gaolious_2023.01.09_gifts.json", 0, 3, 4, 5),
        ("init_data/gaolious_2023.01.11_jobs.json", 3, 3, 4, 5),
        ("init_data/gaolious_2023.01.14_fulltest.json", 3, 4, 4, 5),
        ("init_data/gaolious_2023.01.14_idle_destinations.json", 3, 4, 4, 5),
        ("init_data/gaolious_2023.01.14_results.json", 3, 4, 4, 5),
    ],
)
def test_dump(
    multidb, init_filename, event_count, union_count, story_count, side_count
):
    ###########################################################################
    # prepare
    initdata_filepath = settings.DJANGO_PATH / "fixtures" / init_filename

    version = prepare(initdata_filepath=initdata_filepath)

    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now

        ts_dump(version=version)

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filename",
    [
        "strategies/6/6_0.json",
        "init_data/gaolious1_2022.12.29.json",
        "init_data/gaolious_2022.12.30-1.json",
        "init_data/gaolious_2023.01.09_contract.json",
        "init_data/gaolious_2023.01.09_gifts.json",
        "init_data/gaolious_2023.01.11_jobs.json",
        "init_data/gaolious_2023.01.14_fulltest.json",
        "init_data/gaolious_2023.01.14_idle_destinations.json",
        "init_data/gaolious_2023.01.14_results.json",
    ],
)
def test_available_max_capacity(multidb, init_filename):
    ###########################################################################
    # prepare
    initdata_filepath = settings.DJANGO_PATH / "fixtures" / init_filename

    version = prepare(initdata_filepath=initdata_filepath)

    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now

        ret = trains_max_capacity(version=version, region=1)
        assert len(ret) >= 1

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filename",
    [
        # 'strategies/6/6_0.json',
        "init_data/gaolious1_2022.12.29.json",
        # 'init_data/gaolious_2022.12.30-1.json',
        # 'init_data/gaolious_2023.01.09_contract.json',
        # 'init_data/gaolious_2023.01.09_gifts.json',
        # 'init_data/gaolious_2023.01.11_jobs.json',
        # 'init_data/gaolious_2023.01.14_fulltest.json',
        # 'init_data/gaolious_2023.01.14_idle_destinations.json',
        # 'init_data/gaolious_2023.01.14_results.json',
        # 'init_data/gaolious_2023.02.05.json',
        # 'init_data/gaolious_2023.02.07.json',
    ],
)
def test_shop_purchase_item(multidb, init_filename, fixture_send_commands):
    ###########################################################################
    # prepare
    initdata_filepath = settings.DJANGO_PATH / "fixtures" / init_filename

    version = prepare(initdata_filepath=initdata_filepath)

    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now
        cnt1, cnt2 = 0, 0
        daily_offer_items = daily_offer_get_slots(
            version=version,
            available_video=True,
            availble_gem=False,
            available_gold=False,
        )
        for offer_item in daily_offer_items:
            cmd = ShopPurchaseItem(version=version, offer_item=offer_item)
            cmd.post_processing({})
            cnt1 += 1

        daily_offer_items = daily_offer_get_slots(
            version=version,
            available_video=True,
            availble_gem=False,
            available_gold=False,
        )
        for offer_item in daily_offer_items:
            cmd = ShopPurchaseItem(version=version, offer_item=offer_item)
            cmd.post_processing({})
            cnt2 += 1

        assert cnt1 >= 0
        assert cnt2 == 0

@pytest.mark.django_db
@pytest.mark.parametrize(
    "init_filename",
    [
        # 'strategies/6/6_0.json',
        # 'init_data/gaolious1_2022.12.29.json',
        # 'init_data/gaolious_2022.12.30-1.json',
        # 'init_data/gaolious_2023.01.09_contract.json',
        # 'init_data/gaolious_2023.01.09_gifts.json',
        # 'init_data/gaolious_2023.01.11_jobs.json',
        # 'init_data/gaolious_2023.01.14_fulltest.json',
        # 'init_data/gaolious_2023.01.14_idle_destinations.json',
        # 'init_data/gaolious_2023.01.14_results.json',
        # 'init_data/gaolious_2023.02.05.json',
        "init_data/gaolious_2023.02.07.json",
    ],
)
def test_job_prioirty(multidb, init_filename):
    ###########################################################################
    # prepare
    initdata_filepath = settings.DJANGO_PATH / "fixtures" / init_filename

    version = prepare(initdata_filepath=initdata_filepath)

    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now
        jobs_find_union_priority(version=version)

def convert_text(txt: str):
    data = json.loads(txt, strict=False)
    t = timezone.now().isoformat(sep="T", timespec="seconds").replace("+00:00", "Z")
    data.update({"Time": t})

    return json.dumps(data, separators=(",", ":"))

@pytest.fixture(scope="function")
def fixture_endpoint():
    with mock.patch("app_root.strategies.utils.EndpointHelper.get") as patch:
        patch.return_value = convert_text(
            (
                settings.DJANGO_PATH
                / "fixtures"
                / "endpoints"
                / "gaolious1_2022.12.29.json"
            ).read_text("utf-8")
        )
        yield patch

@pytest.fixture(scope="function")
def fixture_login():
    with mock.patch("app_root.strategies.utils.LoginHelper.post") as patch:
        patch.return_value = convert_text(
            (
                settings.DJANGO_PATH
                / "fixtures"
                / "login"
                / "gaolious1_2022.12.29_with_device_id.json"
            ).read_text("utf-8")
        )
        yield patch

@pytest.fixture(scope="function")
def fixture_definition():
    sqllite_filepath = settings.DJANGO_PATH / "fixtures" / "207.004.sqlite"

    def FakeDownload(instance, *args, **kwargs):
        download_filename = Path(instance.download_path)
        shutil.copy(sqllite_filepath, download_filename)
        return 1_000_000

    with mock.patch("app_root.strategies.utils.SQLDefinitionHelper.get") as patch:
        patch.return_value = convert_text(
            (
                settings.DJANGO_PATH
                / "fixtures"
                / "definition"
                / "gaolious1_2022.12.29.json"
            ).read_text("utf-8")
        )

        with mock.patch(
            "app_root.strategies.utils.SQLDefinitionHelper.download_data"
        ) as dn:
            dn.side_effect = FakeDownload
            yield dn

@pytest.fixture(scope="function")
def fixture_init_20230207():
    filename = "gaolious_2023.02.07.json"
    with mock.patch("app_root.strategies.utils.InitdataHelper.get") as patch:
        patch.side_effect = [
            convert_text(
                (settings.DJANGO_PATH / "fixtures" / "init_data" / filename).read_text(
                    "utf-8"
                )
            ),
            "{}",
        ]
        yield patch

@pytest.fixture(scope="function")
def fixture_init_20230210():
    filename = "gaolious_2023.02.10.json"
    with mock.patch("app_root.strategies.utils.InitdataHelper.get") as patch:
        patch.side_effect = [
            convert_text(
                (settings.DJANGO_PATH / "fixtures" / "init_data" / filename).read_text(
                    "utf-8"
                )
            ),
            "{}",
        ]
        yield patch

@pytest.fixture(scope="function")
def fixture_leader_board():
    with mock.patch("app_root.strategies.utils.LeaderboardHelper.run") as patch:
        yield patch

@pytest.fixture(scope="function")
def fixture_start_game():
    with mock.patch("app_root.strategies.utils.StartGame.run") as patch:
        yield patch

class FakeRunCommand(object):
    commands: List
    version: RunVersion

    def __init__(self, version, commands):
        self.commands = commands
        self.version = version

    def run(self):
        for cmd in self.commands:
            print(f"* FakeCmd : {cmd.COMMAND} {cmd.get_parameters()}")
            cmd.post_processing({})

@pytest.fixture(scope="function")
def fixture_send_commands():
    with mock.patch("app_root.strategies.commands.RunCommand") as patch:
        patch.side_effect = FakeRunCommand
        yield patch

@pytest.fixture(scope="function")
def fixture_sleep():
    with mock.patch("app_root.strategies.commands.sleep") as patch:
        yield patch

@pytest.fixture(scope="function")
def fixture_use_cache():
    with mock.patch("app_root.strategies.utils.USE_CACHE", True) as patch:
        yield patch

@pytest.mark.django_db
@pytest.mark.parametrize("user_name, run_version_id", [("gaolious", 13)])
def test_materials_find_redundancy(
    multidb,
    user_name,
    run_version_id,
    fixture_send_commands,
    fixture_sleep,
    fixture_use_cache,
):
    ###########################################################################
    # prepare
    user = User.objects.create_user(
        username=user_name, android_id="test", game_access_token="1", player_id="1"
    )
    version = RunVersion.objects.create(id=run_version_id, user_id=user.id, level_id=1)
    initdata_filepath = version.get_account_path() / "startgame_post.txt"
    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now
        s = Strategy(user.id)
        s.run()

@pytest.mark.django_db
@pytest.mark.parametrize(
    "user_name, run_version_id",
    [
        ("gaolious", 111),
        # ('gaolious', 34),  # 가능 상태
    ],
)
def test_prepare_contract(
    multidb,
    user_name,
    run_version_id,
    fixture_send_commands,
    fixture_sleep,
    fixture_use_cache,
):
    ###########################################################################
    # prepare
    user = User.objects.create_user(
        username=user_name, android_id="test", game_access_token="1", player_id="1"
    )
    version = RunVersion.objects.create(id=run_version_id, user_id=user.id, level_id=1)
    initdata_filepath = version.get_account_path() / "startgame_post.txt"
    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])
    version.now = now
    version.save()

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now
        s = Strategy(user.id)
        s.run()

@pytest.mark.django_db
@pytest.mark.parametrize(
    "user_name, run_version_id",
    [
        ("gaolious1", 2),  # 약 1분정도 남은 상태.
        # ('gaolious', 34),  # 가능 상태
    ],
)
def test_factory_order_product(
    multidb,
    user_name,
    run_version_id,
    fixture_send_commands,
    fixture_sleep,
    fixture_use_cache,
):
    ###########################################################################
    # prepare
    user = User.objects.create_user(
        username=user_name, android_id="test", game_access_token="1", player_id="1"
    )
    version = RunVersion.objects.create(id=run_version_id, user_id=user.id, level_id=1)
    initdata_filepath = version.get_account_path() / "startgame_post.txt"
    txt = initdata_filepath.read_text(encoding="utf-8")
    json_data = json.loads(txt, strict=False)
    now = convert_datetime(json_data["Time"])
    version.now = now
    version.save()

    with mock.patch("django.utils.timezone.now") as p:
        p.return_value = now
        s = Strategy(user.id)
        s.run()
        prod = TSProduct.objects.filter(article_id=224).first()
        order_list = list(
            PlayerFactoryProductOrder.objects.filter(
                player_factory__factory_id=prod.factory_id
            )
            .order_by("index")
            .all()
        )
        before = ts_dump_factory(version=version, factory_id=prod.factory_id)
        print("[INIT]")
        print("\n".join(before))

        for order in order_list:
            order.refresh_from_db()
            factory_collect_product(version=version, order=order)
            after1 = ts_dump_factory(
                version=version, factory_id=order.player_factory.factory_id
            )
            print("[After Collect]")
            print("\n".join(after1))

            factory_order_product(version=version, product=prod)
            after2 = ts_dump_factory(
                version=version, factory_id=order.player_factory.factory_id
            )
            print("[After add]")
            print("\n".join(after2))

@pytest.mark.django_db
def test_create_player_job():
    json_data = [
        {
            "JobLocationId": 200013,
            "CurrentArticleAmount": 2300,
            "Sequence": 0,
            "JobLevel": 1,
            "RequiredArticle": {"Amount": 2300, "Id": 200006},
            "ExpiresAt": "2023-06-02T12:00:00Z",
            "Duration": 3600,
            "CompletableFrom": "2023-03-25T15:48:48Z",
            "JobType": 49,
            "Reward": {
                "Items": [
                    {"Amount": 850, "Id": 8, "Value": 100000},
                    {"Id": 8, "Value": 100003, "Amount": 500},
                ]
            },
            "FinishedAt": "2023-03-25T14:48:48Z",
            "Id": "e75bbb80-5934-4c84-9d21-beb0bcc35171",
            "Requirements": [
                {"Value": 0, "Type": "relative_region"},
                {"Type": "content_category", "Value": 3},
                {"Value": 1, "Type": "era"},
            ],
        }
    ]

    a, b = PlayerJob.create_instance(data=json_data, version_id=1)

    CompletableFrom = dt(2023, 3, 26, 0, 48, 48)
    FinishedAt = dt(2023, 3, 25, 23, 48, 48)

    assert a
    assert a[0]
    job: PlayerJob = a[0]

    now = FinishedAt - timedelta(minutes=1)
    assert not job.is_completed(now)
    assert not job.is_collectable(now)

    now = CompletableFrom - timedelta(minutes=1)
    assert job.is_completed(now)
    assert not job.is_collectable(now)

    now = CompletableFrom + timedelta(minutes=1)
    assert job.is_completed(now)
    assert job.is_collectable(now)


"""Util that can interact with Zapier NLA.

Full docs here: https://nla.zapier.com/start/

Note: this wrapper currently only implemented the `api_key` auth method for testing
and server-side production use cases (using the developer's connected accounts on
Zapier.com)

For use-cases where LangChain + Zapier NLA is powering a user-facing application, and
LangChain needs access to the end-user's connected accounts on Zapier.com, you'll need
to use oauth. Review the full docs above and reach out to nla@zapier.com for
developer support.
"""
import json
from typing import Any, Dict, List, Optional

import aiohttp
import requests
from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
from langchain_core.utils import get_from_dict_or_env
from requests import Request, Session

class ZapierNLAWrapper(BaseModel):
    """Wrapper for Zapier NLA.

    Full docs here: https://nla.zapier.com/start/

    This wrapper supports both API Key and OAuth Credential auth methods. API Key
    is the fastest way to get started using this wrapper.

    Call this wrapper with either `zapier_nla_api_key` or
    `zapier_nla_oauth_access_token` arguments, or set the `ZAPIER_NLA_API_KEY`
    environment variable. If both arguments are set, the Access Token will take
    precedence.

    For use-cases where LangChain + Zapier NLA is powering a user-facing application,
    and LangChain needs access to the end-user's connected accounts on Zapier.com,
    you'll need to use OAuth. Review the full docs above to learn how to create
    your own provider and generate credentials.
    """

    zapier_nla_api_key: str
    zapier_nla_oauth_access_token: str
    zapier_nla_api_base: str = "https://nla.zapier.com/api/v1/"

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid

    def _format_headers(self) -> Dict[str, str]:
        """Format headers for requests."""
        headers = {
            "Accept": "application/json",
            "Content-Type": "application/json",
        }

        if self.zapier_nla_oauth_access_token:
            headers.update(
                {"Authorization": f"Bearer {self.zapier_nla_oauth_access_token}"}
            )
        else:
            headers.update({"X-API-Key": self.zapier_nla_api_key})

        return headers

    def _get_session(self) -> Session:
        session = requests.Session()
        session.headers.update(self._format_headers())
        return session

    async def _arequest(self, method: str, url: str, **kwargs: Any) -> Dict[str, Any]:
        """Make an async request."""
        async with aiohttp.ClientSession(headers=self._format_headers()) as session:
            async with session.request(method, url, **kwargs) as response:
                response.raise_for_status()
                return await response.json()

    def _create_action_payload(  # type: ignore[no-untyped-def]
        self, instructions: str, params: Optional[Dict] = None, preview_only=False
    ) -> Dict:
        """Create a payload for an action."""
        data = params if params else {}
        data.update(
            {
                "instructions": instructions,
            }
        )
        if preview_only:
            data.update({"preview_only": True})
        return data

    def _create_action_url(self, action_id: str) -> str:
        """Create a url for an action."""
        return self.zapier_nla_api_base + f"exposed/{action_id}/execute/"

    def _create_action_request(  # type: ignore[no-untyped-def]
        self,
        action_id: str,
        instructions: str,
        params: Optional[Dict] = None,
        preview_only=False,
    ) -> Request:
        data = self._create_action_payload(instructions, params, preview_only)
        return Request(
            "POST",
            self._create_action_url(action_id),
            json=data,
        )

    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key exists in environment."""

        zapier_nla_api_key_default = None

        # If there is a oauth_access_key passed in the values
        # we don't need a nla_api_key it can be blank
        if "zapier_nla_oauth_access_token" in values:
            zapier_nla_api_key_default = ""
        else:
            values["zapier_nla_oauth_access_token"] = ""

        # we require at least one API Key
        zapier_nla_api_key = get_from_dict_or_env(
            values,
            "zapier_nla_api_key",
            "ZAPIER_NLA_API_KEY",
            zapier_nla_api_key_default,
        )

        values["zapier_nla_api_key"] = zapier_nla_api_key

        return values

    async def alist(self) -> List[Dict]:
        """Returns a list of all exposed (enabled) actions associated with
        current user (associated with the set api_key). Change your exposed
        actions here: https://nla.zapier.com/demo/start/

        The return list can be empty if no actions exposed. Else will contain
        a list of action objects:

        [{
            "id": str,
            "description": str,
            "params": Dict[str, str]
        }]

        `params` will always contain an `instructions` key, the only required
        param. All others optional and if provided will override any AI guesses
        (see "understanding the AI guessing flow" here:
        https://nla.zapier.com/api/v1/docs)
        """
        response = await self._arequest("GET", self.zapier_nla_api_base + "exposed/")
        return response["results"]

    def list(self) -> List[Dict]:
        """Returns a list of all exposed (enabled) actions associated with
        current user (associated with the set api_key). Change your exposed
        actions here: https://nla.zapier.com/demo/start/

        The return list can be empty if no actions exposed. Else will contain
        a list of action objects:

        [{
            "id": str,
            "description": str,
            "params": Dict[str, str]
        }]

        `params` will always contain an `instructions` key, the only required
        param. All others optional and if provided will override any AI guesses
        (see "understanding the AI guessing flow" here:
        https://nla.zapier.com/docs/using-the-api#ai-guessing)
        """
        session = self._get_session()
        try:
            response = session.get(self.zapier_nla_api_base + "exposed/")
            response.raise_for_status()
        except requests.HTTPError as http_err:
            if response.status_code == 401:
                if self.zapier_nla_oauth_access_token:
                    raise requests.HTTPError(
                        f"An unauthorized response occurred. Check that your "
                        f"access token is correct and doesn't need to be "
                        f"refreshed. Err: {http_err}",
                        response=response,
                    )
                raise requests.HTTPError(
                    f"An unauthorized response occurred. Check that your api "
                    f"key is correct. Err: {http_err}",
                    response=response,
                )
            raise http_err
        return response.json()["results"]

    def run(
        self, action_id: str, instructions: str, params: Optional[Dict] = None
    ) -> Dict:
        """Executes an action that is identified by action_id, must be exposed
        (enabled) by the current user (associated with the set api_key). Change
        your exposed actions here: https://nla.zapier.com/demo/start/

        The return JSON is guaranteed to be less than ~500 words (350
        tokens) making it safe to inject into the prompt of another LLM
        call.
        """
        session = self._get_session()
        request = self._create_action_request(action_id, instructions, params)
        response = session.send(session.prepare_request(request))
        response.raise_for_status()
        return response.json()["result"]

    async def arun(
        self, action_id: str, instructions: str, params: Optional[Dict] = None
    ) -> Dict:
        """Executes an action that is identified by action_id, must be exposed
        (enabled) by the current user (associated with the set api_key). Change
        your exposed actions here: https://nla.zapier.com/demo/start/

        The return JSON is guaranteed to be less than ~500 words (350
        tokens) making it safe to inject into the prompt of another LLM
        call.
        """
        response = await self._arequest(
            "POST",
            self._create_action_url(action_id),
            json=self._create_action_payload(instructions, params),
        )
        return response["result"]

    def preview(
        self, action_id: str, instructions: str, params: Optional[Dict] = None
    ) -> Dict:
        """Same as run, but instead of actually executing the action, will
        instead return a preview of params that have been guessed by the AI in
        case you need to explicitly review before executing."""
        session = self._get_session()
        params = params if params else {}
        params.update({"preview_only": True})
        request = self._create_action_request(action_id, instructions, params, True)
        response = session.send(session.prepare_request(request))
        response.raise_for_status()
        return response.json()["input_params"]

    async def apreview(
        self, action_id: str, instructions: str, params: Optional[Dict] = None
    ) -> Dict:
        """Same as run, but instead of actually executing the action, will
        instead return a preview of params that have been guessed by the AI in
        case you need to explicitly review before executing."""
        response = await self._arequest(
            "POST",
            self._create_action_url(action_id),
            json=self._create_action_payload(instructions, params, preview_only=True),
        )
        return response["result"]

    def run_as_str(self, *args, **kwargs) -> str:  # type: ignore[no-untyped-def]
        """Same as run, but returns a stringified version of the JSON for
        insertting back into an LLM."""
        data = self.run(*args, **kwargs)
        return json.dumps(data)

    async def arun_as_str(self, *args, **kwargs) -> str:  # type: ignore[no-untyped-def]
        """Same as run, but returns a stringified version of the JSON for
        insertting back into an LLM."""
        data = await self.arun(*args, **kwargs)
        return json.dumps(data)

    def preview_as_str(self, *args, **kwargs) -> str:  # type: ignore[no-untyped-def]
        """Same as preview, but returns a stringified version of the JSON for
        insertting back into an LLM."""
        data = self.preview(*args, **kwargs)
        return json.dumps(data)

    async def apreview_as_str(  # type: ignore[no-untyped-def]
        self, *args, **kwargs
    ) -> str:
        """Same as preview, but returns a stringified version of the JSON for
        insertting back into an LLM."""
        data = await self.apreview(*args, **kwargs)
        return json.dumps(data)

    def list_as_str(self) -> str:  # type: ignore[no-untyped-def]
        """Same as list, but returns a stringified version of the JSON for
        insertting back into an LLM."""
        actions = self.list()
        return json.dumps(actions)

    async def alist_as_str(self) -> str:  # type: ignore[no-untyped-def]
        """Same as list, but returns a stringified version of the JSON for
        insertting back into an LLM."""
        actions = await self.alist()
        return json.dumps(actions)


import sys
from collections import deque
#sys.stdin = open('input.txt','r')

N = int(input())
lattice = [list(map(int,input().split())) for _ in range(N)]

c = (N//2,N//2)
s = lattice[c[0]][c[1]]
queue = deque()
queue.append(c)

L = [(0,1),(1,0),(0,-1),(-1,0)]
chk_matrix = [[0]*N for _ in range(N)]
chk_matrix[c[0]][c[1]] = 1

while queue:
    now = queue.popleft()
    if now[0] == 0 or now[0] == N-1 or now[1] == 0 or now[1] == N-1:
        break
    for x,y in L:
        if chk_matrix[now[0]+x][now[1]+y] == 0:
            s += lattice[now[0]+x][now[1]+y]
            chk_matrix[now[0]+x][now[1]+y] = 1
            queue.append((now[0]+x,now[1]+y))

print(s)


# -*- coding: utf-8 -*-

"""
中国海关总署————>进出口商品国别(地区)总值表
"""
# import asyncio
# from twisted.internet import asyncioreactor
# asyncioreactor.install(asyncio.get_event_loop())

import datetime
import re
import scrapy

from shannon_spider.util.notify import send_dingtalk
from shannon_spider.shannon_base_spider import ShannonBaseSpider, ShannonCrawledItem
from shannon_spider.orm.trading_table import TradingTableItem

class TradingTableSpider(ShannonBaseSpider):
    """
    进出口商品国别(地区)总值表
    """
    name = 'trading_table'
    result_orm_model = TradingTableItem
    allowed_domains = ['www.customs.gov.cn']
    base_host = 'http://www.customs.gov.cn'
    start_url = 'http://www.customs.gov.cn/customs/302249/302274/302277/index.html'
    # start_urls = [start_url]
    custom_settings = {
        "DOWNLOADER_MIDDLEWARES": {
            "shannon_spider.middlewares.PuppeteerMiddleware": 1
        },
        "CONCURRENT_REQUESTS": 1
    }

    def __init__(self, **kwargs):
        """
        初始化 CompanyHolderSpider 时，先初始化它的基类
        """
        ShannonBaseSpider.__init__(self, **kwargs)

    def start_requests(self):

        yield scrapy.Request(self.start_url, meta={"is_renderer": True}, callback=self.parse)

    def parse(self, response):
        """解析数据并构成下一页url"""
        # 校验
        if response.status != 200:
            self.logger.error('request: {} bad response status:{}'.format(response.request.url, response.status))
            return False

        # 获取所有详情url
        total_urls = []
        for date in range(2019, int(datetime.datetime.now().year) + 1):
            nodes = response.xpath('//*[@class="tbtj{}_tab"]//tr[4]/td[2]/a'.format(date))
            urls = [node.xpath('./@href').extract_first() for node in nodes if node.xpath('./@href').extract_first()]
            total_urls.extend(urls)
        print("----------------------" + str(len(total_urls)))
        for url in total_urls:
            url = self.base_host+url if 'www' not in url else url
            yield scrapy.Request(url=url, callback=self.parse_data, meta={"change_cookie": True})

    def parse_data(self, response):
        """解析数据, 获取详情页信息"""
        # 校验
        if response.status != 200:
            self.logger.error('request: {} bad response status:{}'.format(response.request.url, response.status))
            return False

        # 过滤不需要的数据
        save_index = None
        nodes = response.xpath('//*[@class="easysite-news-text"]/table/tbody/tr')
        title = response.xpath('/html/head/title/text()').extract_first()
        for index, node in enumerate(nodes):
            if node.xpath('./td[1]//text()').extract_first() == '亚洲':
                save_index = index+1
                break

        if not save_index:
            return False

        new_list = []
        for index, node in enumerate(nodes):
            if index+1 < save_index:
                continue
            country = node.xpath('./td[1]//text()').extract_first()
            if not country or '注' in country:
                continue
            item = dict()
            item_2 = dict()
            date = re.search(r'.*?(\w+)年(\w+)月.*', title)
            item['YEAR_MONTH'] = '{}年{}月'.format(date.group(1), date.group(2))
            item['COUNTRY'] = node.xpath('./td[1]//text()').extract_first()
            item['IMPORT_EXPORT'] = node.xpath('./td[2]//text()').extract_first()
            item['EXPORT'] = node.xpath('./td[4]//text()').extract_first()
            item['IMPORT'] = node.xpath('./td[6]//text()').extract_first()
            item['TRADE_BALANCE'] = None
            item['TYPE'] = '当期值'
            item['UNIT'] = '万' if '人民币' in title else '千'
            item['CURRENCY'] = '人民币' if '人民币' in title else '美元'

            item_2['YEAR_MONTH'] = '{}年{}月'.format(date.group(1), date.group(2))
            item_2['COUNTRY'] = node.xpath('./td[1]//text()').extract_first()
            item_2['IMPORT_EXPORT'] = node.xpath('./td[3]//text()').extract_first()
            item_2['EXPORT'] = node.xpath('./td[5]//text()').extract_first()
            item_2['IMPORT'] = node.xpath('./td[7]//text()').extract_first()
            item_2['TRADE_BALANCE'] = None
            item_2['TYPE'] = '累计值'
            item_2['UNIT'] = '万' if '人民币' in title else '千'
            item_2['CURRENCY'] = '人民币' if '人民币' in title else '美元'
            new_list.append(item)
            new_list.append(item_2)

        # 实例存储
        items = ShannonCrawledItem(self, response)
        for item in new_list:
            item = self.filter_data(item)
            try:
                if item['COUNTRY'] == '\u3000':
                    continue
                if len(item['COUNTRY']) > 15:
                    continue
                if not item['IMPORT_EXPORT'].isdigit():
                    continue
                items['YEAR_MONTH'] = item['YEAR_MONTH']
                items['COUNTRY'] = item['COUNTRY']
                items['IMPORT_EXPORT'] = item['IMPORT_EXPORT']
                items['EXPORT'] = item['EXPORT']
                items['IMPORT'] = item['IMPORT']
                items['TRADE_BALANCE'] = self.count(item['EXPORT'], item['IMPORT'])
                items['TYPE'] = item['TYPE']
                items['UNIT'] = item['UNIT']
                items['CURRENCY'] = item['CURRENCY']
                yield items
            except Exception as e:
                self.logger.error('城市:{}, url:{}, error:{}'.format(item['COUNTRY'], response.url, e))

    def count(self, c_export, c_import):
        """计算差值"""
        if c_export is None and c_import is None:
            return None
        c_export = c_export.replace('\u3000', '')
        c_import = c_import.replace('\u3000', '')
        try:
            diff_val = int(float(c_export)) - int(float(c_import))
        except Exception as e:
            self.logger.error(e)
            diff_val = None
        return diff_val

    @staticmethod
    def filter_data(item):
        """过滤数据"""
        for k, v in item.items():
            if k in ['IMPORT_EXPORT', 'EXPORT', 'IMPORT']:
                if not v:
                    v = '0'
                if '\\' in v:
                    v = v.split('\\')[0]
                if ',' in v:
                    v = v.replace(',', '')
                if '-' == v:
                    v = '0'
                if '\xa0' in v:
                    v = v.replace('\xa0', '')
                if ' ' in v:
                    v = v.replace(' ', '')
                item[k] = v
            else:
                item[k] = v
        return item


import scrapy
import csv
import json
import re
from html.parser import HTMLParser

def remove_non_ascii(text):
    return ''.join([i if ord(i) < 128 else '' for i in text])

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.strict = False
        self.convert_charrefs= True
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return '|'.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

class HCVID175_www_storyltd_com_Spider(scrapy.Spider):
    name = "HCVID175.www.storyltd.com"
    products_seen = []
    jsonData = []

    def closed(self, reason):
        toWrite = {'data' : HCVID175_www_storyltd_com_Spider.jsonData}
        with open('./json/HCVID175.www.storyltd.com.json', 'w+') as outfile:
            json.dump(toWrite, outfile, ensure_ascii=False)

    def start_requests(self):
        url = "https://www.storyltd.com/"
        yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        x = response.css('div#top_navigation_navigationListHolder ul#top_navigation_top_menus_list > li')
        for columns in x:
            head_category = columns.css('a::text').extract_first()
            if head_category != 'Artists' and head_category != 'Auctions':
                list_items = columns.css('div.block1:first-child ul li')
                for item in list_items:
                    page = item.css('a::attr(href)').extract_first()
                    sub_category = item.css('a::text').extract_first()
                    if page is not None:
                        print(page)
                        request = scrapy.Request(url=page, callback = self.parsePage)
                        request.meta['category'] = str(head_category) + '|' + str(sub_category)
                        yield request

    def parsePage(self, response):
        scripts = response.css('script[type=text\/javascript]').extract()
        for script in scripts:
            if 'var urlBase' in script and 'var pageIndex' in script:
                link = 'https://www.storyltd.com' + re.search('var urlBase = \'(.*?)\';', script).group(1) + '&pageSize=60&pageIndex=1'
                request = scrapy.Request(url=link, callback=self.parseOthers)
                request.meta['category'] = response.meta['category']
                request.meta['pagenum'] = 1
                yield request

    def parseOthers(self, response):
        products = response.css('body > li')
        if not products:
            return
        else:
            for product in products:
                product_link = product.css('a::attr(href)').extract_first()
                product_image = product.css('a img::attr(data-original)').extract_first()
                request = scrapy.Request(url=product_link, callback=self.parseProduct)
                request.meta['product_link'] = product_link
                request.meta['product_image'] = product_image
                request.meta['product_category'] = response.meta['category']
                yield request
            nextpage = response.meta['pagenum'] + 1
            product_link = response.request.url[:response.request.url.index('&pageIndex')] + '&pageIndex=' + str(nextpage)
            request = scrapy.Request(url=product_link, callback=self.parseOthers)
            request.meta['category'] = response.meta['category']
            request.meta['pagenum'] = nextpage
            yield request

    def parseProduct(self, response):
        productLink = response.meta['product_link']
        productCategory = response.meta['product_category']
        productImage = response.meta['product_image']
        productMerchant = 'NA'
        productPrice = response.css('li#ContentPlaceHolder1_primary_price::text').extract_first().replace('Rs','').strip()

        productName = response.css('div.author-details h1::text').extract_first()
        if productName:
            productName += ' by '
            productName += response.css('div.author-details h2::text').extract_first()
        else:
            productName = response.css('div.author-details h2::text').extract_first()

        productDesc = response.css('div#ContentPlaceHolder1_itemdetails').extract_first()
        if not productDesc:
            productDesc = 'NA'
        else:
            productDesc = strip_tags(productDesc)

        productSku = ''
        texts = response.css('div#ContentPlaceHolder1_itemdetails::text').extract()
        for text in texts:
            if 'StoryLTD Ref No:' in text:
                productSku = text.replace('StoryLTD Ref No:','').strip()
        if not productSku:
            productSku = 'SKU-NA'

        productCategory = productCategory.replace('"','``')
        productDesc = re.sub('\n' ,'|',productDesc)
        productDesc = re.sub('\t','',productDesc)
        productDesc = re.sub('\r','',productDesc)
        productDesc = re.sub('[ ]+',' ',productDesc)
        productDesc = re.sub('(\| )+','|',productDesc)
        productDesc = re.sub('\|+','|',productDesc)
        productDesc = productDesc.strip('|')
        productDesc = productDesc.replace('"','``')
        productName = productName.replace('"','``')

        productDesc = remove_non_ascii(productDesc)
        productCategory = remove_non_ascii(productCategory)
        productName = remove_non_ascii(productName)

        with open('HCVID175.www.storyltd.com.csv', 'a', newline='') as csvfile:
            fieldnames = ['Title', 'Category', 'URL', 'IMG_SRC', 'Merchant', 'Price', 'SKU', 'Description']
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writerow({'Title': '"'+str(productName)+'"', 'IMG_SRC': '"'+str(productImage)+'"', 'Description': '"'+str(productDesc)+'"','Price': '"'+str(productPrice)+'"', 'URL': '"'+str(productLink)+'"', 'Category': '"'+str(productCategory)+'"', 'SKU': '"'+str(productSku)+'"', 'Merchant': '"'+str(productMerchant)+'"'})

        HCVID175_www_storyltd_com_Spider.jsonData.append({'Title': productName, 'Description': productDesc, 'IMG_SRC': productImage, 'Price': productPrice, 'SKU': productSku, 'Category': productCategory, 'URL': productLink, 'Merchant': productMerchant})


import time
import warnings
import numpy as np
import numba as nb
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from sklearn.metrics import mean_squared_error as mse_func, mean_absolute_error as mae_func
from citylearnutils.utils import seed_everything

N_DAYS = 8
N_TARGETS = 10
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

DATA_PATH = "./data"
FORECAST_PATH = f"{DATA_PATH}/external/forecasts/demand"
MODEL_PATH = f"{DATA_PATH}/citylearn_challenge_2022_phase_1/models"

DEMAND_MODEl_PARAMS = {
    "training": {
        "batch_size": 256,
        "n_epochs": 25,
        "grad_clip": 0.1,
        "seed": 0
    },
    "scheduler": {
        "max_lr": 1.54e-05,
        "pct_start": 0.2,
        "div_factor": 1,
        "final_div_factor": 10000
    },
    "optimizer": {
        "weight_decay": 1e-07
    },
    "model": {
        "n_features": 282,
        "n_targets": 10,
        "n_hidden": 926,
        "dropout": 0.1,
        "emb_dim": 2
    }
}

@nb.njit()
def _uniform_update(x_new, value, count):
    return value + (x_new - value) / (count + 1)

@nb.njit()
def _average_update(x_new, value, baseline, count, decays):
    blend = value + (baseline - value) * (1 - decays) ** count
    return blend + decays * (x_new - blend)

class ExponentialSmoother():

    def __init__(self, daily_baseline, weekly_baseline):
        self.decays = np.array([0.125, 0.25, 0.5], dtype=np.float32)
        n_decays = len(self.decays)

        self.daily_x = np.repeat(daily_baseline, n_decays).reshape(24, -1)  # average
        self.daily_b = np.zeros((24, 1), np.float32)  # baseline
        self.daily_n = np.zeros(24, np.int32)  # count

        self.weekly_x = np.repeat(weekly_baseline, n_decays).reshape(168, -1)  # average
        self.weekly_b = np.zeros((168, 1), np.float32)  # baseline
        self.weekly_n = np.zeros(168, np.int32)  # count

        self.time_index = 0

    def fit_transform(self, x, n_lags=5, n_leads=N_TARGETS):
        # update daily state
        i = self.time_index % 24
        self.daily_b[i] = _uniform_update(x, self.daily_b[i], self.daily_n[i])
        self.daily_x[i] = _average_update(x, self.daily_x[i], self.daily_b[i], self.daily_n[i], self.decays)
        self.daily_n[i] += 1

        # update weekly state
        j = self.time_index % 168
        self.weekly_b[j] = _uniform_update(x, self.weekly_b[j], self.weekly_n[j])
        self.weekly_x[j] = _average_update(x, self.weekly_x[j], self.weekly_b[j], self.weekly_n[j], self.decays)
        self.weekly_n[j] += 1

        # extract current values
        i = np.arange(self.time_index - n_lags, self.time_index + n_leads) % 24
        j = np.arange(self.time_index - n_lags, self.time_index + n_leads) % 168
        y = np.append(self.daily_x[i], self.weekly_x[j])

        # update time
        self.time_index += 1

        return y

class DemandDataset(Dataset):

    def __init__(self, time_of_day, day_of_week, features, targets, device):
        super(DemandDataset, self).__init__()
        self.time_of_day = torch.tensor(time_of_day, dtype=torch.long, device=device)
        self.day_of_week = torch.tensor(day_of_week, dtype=torch.long, device=device)
        self.features = torch.tensor(features, dtype=torch.float32, device=device)
        self.targets = torch.tensor(targets, dtype=torch.float32, device=device)
        self.device = device

    def __len__(self):
        return len(self.targets)

    def __getitem__(self, index):
        time_of_day = self.time_of_day[index]
        day_of_week = self.day_of_week[index]
        features = self.features[index]
        targets = self.targets[index]
        return time_of_day, day_of_week, features, targets

class SkipMLP(nn.Module):

    def __init__(self, n_in, n_hidden, dropout=0.1):
        super(SkipMLP, self).__init__()
        self.linear1 = nn.Linear(n_in, n_hidden)
        self.linear2 = nn.Linear(n_hidden, n_in)
        self.activation = nn.ReLU()
        self.dropout = nn.Dropout(dropout)

    def forward(self, x0):
        x = self.activation(self.linear1(x0))
        x = self.linear2(self.dropout(x))
        return x + x0

class MLP(nn.Module):

    def __init__(self, n_in, n_hidden, n_out, dropout=0.1):
        super(MLP, self).__init__()
        self.linear1 = nn.Linear(n_in, n_hidden)
        self.linear2 = nn.Linear(n_hidden, n_out)
        self.activation = nn.ReLU()
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        x = self.activation(self.linear1(x))
        x = self.linear2(self.dropout(x))
        return x

class DemandModel(nn.Module):

    def __init__(self, n_features, n_targets, n_hidden, dropout, emb_dim):
        super(DemandModel, self).__init__()
        self.tod_embedding = nn.Embedding(24, emb_dim)
        self.dow_embedding = nn.Embedding(24, emb_dim)
        self.mlp1 = MLP(n_features + 2 * emb_dim, n_hidden, n_hidden, dropout)
        self.mlp2 = SkipMLP(n_hidden, n_hidden, dropout)
        self.mlp3 = MLP(n_hidden, n_hidden, n_targets, dropout)

    def forward(self, time_of_day, day_of_week, features):
        tod = self.tod_embedding(time_of_day)
        dow = self.dow_embedding(day_of_week)
        x = self.mlp1(torch.cat([tod, dow, features], 1))
        x = self.mlp2(x)
        x = self.mlp3(x)
        return x.clamp_min(0)

if __name__ == "__main__":

    # preprocess data
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=pd.errors.PerformanceWarning)

        # load data
        data = pd.read_csv(f"{DATA_PATH}/external/observations.csv")

        # create baseline forecast
        data["time_of_day"] = np.tile(np.arange(24), 365 * 5)
        data["day_of_week"] = np.concatenate([np.tile(np.repeat(np.arange(7), 24), 53)[:8760]] * 5)

        daily_baseline = data.groupby(["time_of_day"]).non_shiftable_load.mean().values
        weekly_baseline = data.groupby(["day_of_week", "time_of_day"]).non_shiftable_load.mean().values

        # lags
        lags = np.arange(N_DAYS * 24)
        lag_cols = [f"lag{i}" for i in lags]

        for building_num, df in data.groupby("building_num"):

            df_weekly_baseline = df.groupby(["day_of_week", "time_of_day"]).non_shiftable_load.mean().values
            df_weekly_baseline = (5 * weekly_baseline - df_weekly_baseline) / 4

            for i in lags:
                x = np.append(df.non_shiftable_load, np.tile(df_weekly_baseline, 5))
                data.loc[df.index, f"lag{i}"] = np.roll(x, i)[:len(df)]

        # exponential smoothing
        es_cols = [f"es{i}" for i in range(6 * 5 + 6 * N_TARGETS)]

        for building_num, df in data.groupby("building_num"):
            df_daily_baseline = df.groupby(["time_of_day"]).non_shiftable_load.mean().values
            df_daily_baseline = (5 * daily_baseline - df_daily_baseline) / 4

            df_weekly_baseline = df.groupby(["day_of_week", "time_of_day"]).non_shiftable_load.mean().values
            df_weekly_baseline = (5 * weekly_baseline - df_weekly_baseline) / 4

            es = ExponentialSmoother(df_daily_baseline, df_weekly_baseline)
            features = [es.fit_transform(x) for x in df.non_shiftable_load.values]
            data.loc[df.index, es_cols] = np.vstack(features)

        # targets
        leads = 1 + np.arange(N_TARGETS)
        target_cols = [f"target{i}" for i in leads]

        for building_num, df in data.groupby("building_num"):
            for i in leads:
                x = np.roll(df.non_shiftable_load, -i)
                x = np.append(x[:-N_TARGETS], weekly_baseline[:N_TARGETS])
                data.loc[df.index, f"target{i}"] = x

        # try to defragment dataframe
        feature_cols = lag_cols + es_cols
        data = data.dropna().reset_index(drop=True).copy()
        assert len(data) == 5 * 8760

    # train model and create out-of-fold predictions
    for seed in range(5):
        for drop_building_num in [1, 2, 3, 4, 5, None]:

            eval_start_time = time.time()
            seed_everything(seed)

            # create loaders
            if drop_building_num is None:

                trn_loader = DataLoader(
                    DemandDataset(
                        time_of_day=data.time_of_day.values,
                        day_of_week=data.day_of_week.values,
                        features=data[feature_cols].values,
                        targets=data[target_cols].values,
                        device=DEVICE
                    ),
                    batch_size=DEMAND_MODEl_PARAMS["training"]["batch_size"],
                    shuffle=True
                )

                val_loader = DataLoader(
                    DemandDataset(
                        time_of_day=data.time_of_day.values,
                        day_of_week=data.day_of_week.values,
                        features=data[feature_cols].values,
                        targets=data[target_cols].values,
                        device=DEVICE),
                    batch_size=DEMAND_MODEl_PARAMS["training"]["batch_size"],
                    shuffle=False
                )

            else:

                trn_loader = DataLoader(
                    DemandDataset(
                        time_of_day=data.loc[data.building_num != drop_building_num, "time_of_day"].values,
                        day_of_week=data.loc[data.building_num != drop_building_num, "day_of_week"].values,
                        features=data.loc[data.building_num != drop_building_num, feature_cols].values,
                        targets=data.loc[data.building_num != drop_building_num, target_cols].values,
                        device=DEVICE
                    ),
                    batch_size=DEMAND_MODEl_PARAMS["training"]["batch_size"],
                    shuffle=True
                )

                val_loader = DataLoader(
                    DemandDataset(
                        time_of_day=data.loc[data.building_num == drop_building_num, "time_of_day"].values,
                        day_of_week=data.loc[data.building_num == drop_building_num, "day_of_week"].values,
                        features=data.loc[data.building_num == drop_building_num, feature_cols].values,
                        targets=data.loc[data.building_num == drop_building_num, target_cols].values,
                        device=DEVICE),
                    batch_size=DEMAND_MODEl_PARAMS["training"]["batch_size"],
                    shuffle=False
                )

            # initialize model, optimizer, scheduler
            model = DemandModel(**DEMAND_MODEl_PARAMS["model"])
            model.to(DEVICE)

            optimizer = torch.optim.AdamW(model.parameters(), **DEMAND_MODEl_PARAMS["optimizer"])

            scheduler = torch.optim.lr_scheduler.OneCycleLR(
                optimizer,
                epochs=DEMAND_MODEl_PARAMS["training"]["n_epochs"],
                **DEMAND_MODEl_PARAMS["scheduler"],
                steps_per_epoch=len(trn_loader),
            )

            criterion = nn.MSELoss()

            metrics = []
            for epoch in range(DEMAND_MODEl_PARAMS["training"]["n_epochs"]):

                # training
                model.train()
                start_time = time.time()
                trn_targets, trn_preds = [], []
                for time_of_day, day_of_week, features, targets in trn_loader:

                    optimizer.zero_grad()
                    preds = model(time_of_day, day_of_week, features)

                    trn_targets.append(targets.detach().cpu().numpy())
                    trn_preds.append(preds.detach().cpu().numpy())

                    loss = criterion(preds, targets.to(DEVICE))
                    loss.backward(loss)

                    torch.nn.utils.clip_grad_norm_(model.parameters(), DEMAND_MODEl_PARAMS["training"]["grad_clip"])

                    optimizer.step()
                    scheduler.step()

                # Validation
                model.eval()
                val_targets, val_preds = [], []
                with torch.no_grad():
                    for time_of_day, day_of_week, features, targets in val_loader:
                        preds = model(time_of_day, day_of_week, features)
                        val_targets.append(targets.detach().cpu().numpy())
                        val_preds.append(preds.detach().cpu().numpy())

                # combine results
                trn_targets = np.concatenate(trn_targets)
                val_targets = np.concatenate(val_targets)

                trn_preds = np.concatenate(trn_preds)
                val_preds = np.concatenate(val_preds)

                # calculate metrics
                trn_rmse = np.sqrt(mse_func(trn_targets, trn_preds))
                val_rmse = np.sqrt(mse_func(val_targets, val_preds))

                trn_mae = mae_func(trn_targets, trn_preds)
                val_mae = mae_func(val_targets, val_preds)

                metrics.append(dict(trn_rmse=trn_rmse, trn_mae=trn_mae,
                                    val_rmse=val_rmse, val_mae=val_mae))

                print(f"drop {drop_building_num} - "
                      f"{epoch:02.0f} ({time.time() - start_time:0.1f}s): "
                      f"trn_rmse {trn_rmse:0.5f}  trn_mae {trn_mae:0.5f}  "
                      f"val_rmse {val_rmse:0.5f}  val_mae {val_mae:0.5f}")

            # save
            if drop_building_num is None:
                torch.save(model.state_dict(), f"{MODEL_PATH}/demand_model.{seed}.pt")
            else:
                np.save(f"{FORECAST_PATH}/pred_{drop_building_num}.{seed}", val_preds)
            print()

    # average predictions
    for building_num in range(5):
        preds = 0
        for seed in range(5):
            preds += np.load(f"{FORECAST_PATH}/pred_{building_num+1}.{seed}.npy")/5
        np.save(f"{FORECAST_PATH}/pred_{building_num+1}", preds)


import sys
import re
import json

mode = None # whether structure or content validation
pause_between_problems = False # extra option for verbose structure validation
output_bracketless = False # optional output to accompany verbose content validation
prompt_update_ngrams = False # option to accept all remaining "problem" ngrams and add to json

max_ngram_len = 2 # anything above 3 seems like overkill, but starting with 2 for now
flagged_ngrams_txt_fn = 'validation_files/flagged_ngrams.txt'
accepted_ngrams_json_fn = 'validation_files/accepted_ngrams.json'
bracketless_content_txt_fn = 'validation_files/bracketless_content.txt'

"""
structure validation functions

together ensure proper bracket pairing
"""
abs = '\[\]{}<>\(\)〈〉' # all_brackets_string
empty_bracket_pair_regex = '(\[\])|({})|(<>)|(\(\))|(〈〉)'
full_deletable_bracket_pair_regex = '(\[[^%s]*?\])|({[^%s]*?})|(<[^%s]*?>)|(\([^%s]*?\))' % tuple([abs]*4)
full_keepable_bracket_pair_regex = '〈([^%s]*?)〉' % abs

def view_problem_brackets(problem_brackets, raw_input_text):

	lefts = ['[', '{', '<', '(', '〈']
	rights = [']', '}', '>', ')', '〉']
	l_rs = zip(lefts,rights)
	corr_memb = {}
	for l, r in l_rs: corr_memb[l] = r; corr_memb[r] = l

	for bracket in problem_brackets:

		if bracket in ['[','(',']',')']: # chars need to be escaped
			regex = '\%s[^\%s]*?\%s' % (bracket, corr_memb[bracket], bracket)
		else: # chars don't need to be escaped
			regex = '%s[^%s]*?%s' % (bracket, corr_memb[bracket], bracket)

		print("problem bracket: ", bracket)
		print("find with this regex: ", regex)
		result = re.search(regex, raw_input_text) # find first instance
		try:
			print("context hint:\n", result.group())
		except: print('not a faulty bracket pair, look for bracket nesting...')
		print("\n****\n")
		if pause_between_problems: input()

# def check_clean_surroundings(content):
# 	print("# of non-line-initial [{< : ", len(re.findall('[^\n][\[{<]',content)))
# 	print("# of non-line-final ]}> : ", len(re.findall('[\]}>][^\n]',content)))
# 	print()

def validate_structure(raw_input_text, verbose=False):

	# if verbose: check_clean_surroundings(raw_input_text)

	brackets_only = re.sub('[^\[\]{}<>\(\)〈〉]', '', raw_input_text) # remove all chars besides brackets

	# remove valid empty bracket pairs, iteratively in case nested
	# leave behind only unsearched bracket chars
	while bool( re.search(empty_bracket_pair_regex, brackets_only) ) == True:
		brackets_only = re.sub(empty_bracket_pair_regex, '', brackets_only)

	if brackets_only == '':
		if verbose: print("no problem brackets, structure validated successfully\n")
		return 1
	else:
		if verbose:
			print("problematic remaining brackets:", brackets_only, '\n')
			view_problem_brackets(brackets_only, raw_input_text)
		return 0

"""
content validation function (only the one, not yet modularized)
"""

def get_ngrams_with_freqs(text, n):

	ngram_freqs = {} # dict contains multiple lists of ngrams (of lens 1...n)

	for n in range(1,n+1):

		# get list of ngrams for a given n
		ngram_list = [ text[i:i+n] for i in range(len(text)-n+1) ]

		# dedupe and sort
		unique_ngram_list = list(set(ngram_list))
		unique_ngram_list.sort()

		# count frequencies too
		ngram_freqs[n] = {}
		for u_ng in unique_ngram_list:
			ngram_freqs[n][u_ng] = text.count(u_ng)

	return ngram_freqs

def validate_content(raw_input_text, verbose=False):

	if verbose:
		print('comparing content of input text against known ngrams ...')

	content = raw_input_text

	# remove deletable brackets and their contents, iteratively in case nested
	# leave behind only text content
	# and check twice in case of lots of nesting

	for i in range(2):

		while bool( re.search(full_deletable_bracket_pair_regex, content) ) == True:
			content = re.sub(full_deletable_bracket_pair_regex, '', content)

		# keep content of keepable brackets
		while bool( re.search(full_keepable_bracket_pair_regex, content) ) == True:
			content = re.sub(full_keepable_bracket_pair_regex, '\\1', content)

	# clean up extra whitespace
	regex_replacements = [
		[' *(\n+) *', '\\1'], # extraneous space adjacent to newline
		['\n{2,}', '\n\n'], # extraneous newline (x2 max)
		[' {2,}', ' '], # duplicate space
		('\A\s*', ''),	# file-initial whitespace
		('\s*\Z', ''),	# file-final whitespace
	]
	for r_r in regex_replacements:
		content = re.sub(r_r[0], r_r[1], content)

	# useful for inspecting what actual content is being measured
	if output_bracketless == True:
		with open(bracketless_content_txt_fn, 'w') as f_out:
			f_out.write(content)

	# initialize dict for flagging unrecognized ngrams
	curr_flagged_ngrams_with_freqs = {}
	for i in range(1,max_ngram_len+1):
		curr_flagged_ngrams_with_freqs[i] = {}

	try:
		# load
		with open(accepted_ngrams_json_fn,'r') as f_in:
			accepted_ngrams = json.load(f_in)["accepted_ngrams"]
	except FileNotFoundError:
		# initialize
		accepted_ngrams = {}
		for i in range(1,max_ngram_len+1):
			accepted_ngrams[str(i)] = [] # strings for keys for sake of json dict

	# get all unique ngrams of lens 1...max_ngram_len
	text_ngram_freqs = get_ngrams_with_freqs(content, max_ngram_len) # dict with sorted 1-grams and 2-grams and freqs

	# compare against accepted_ngrams and flag unrecognized ones
	for i in range(1,max_ngram_len+1):
		for ng in text_ngram_freqs[i].keys():
			if ng not in accepted_ngrams[str(i)]: # dict from json files has strings for keys
				curr_flagged_ngrams_with_freqs[i][ng] = text_ngram_freqs[i][ng]

			# used to also treat certain chars differently and look for > 128-char sequences

			# tricky_chars = ['_', '-', '\t']
			# # also add: more as found
			# for c in tricky_chars:
			# 	if c in content:
			# 		if verbose: print("watch out for %s '%s': %d" % (repr(c), c, content.count(c)))

			# chunks = ( ( content.replace('\n',' ') ).replace('_',' ') ).split(' ')
			# chunks_gt_128 = []
			# for chunk in chunks:
			# 	if len(chunk) > 128: chunks_gt_128.append(chunk)
			# if verbose:
			# 	if chunks_gt_128 != []: print("# chunks > 128:", len(chunks_gt_128))
			# 	for chunk in chunks_gt_128: print(chunk)

	# create parallel dict with each sub-dict sorted for inspection in txt output
	curr_flagged_ngrams_sorted_by_freqs = {}
	for i in range(1,max_ngram_len+1):
		curr_flagged_ngrams_sorted_by_freqs[i] = dict(sorted(curr_flagged_ngrams_with_freqs[i].items(), key=lambda item: item[1], reverse=True))

	# create parallel dict without freqs for appending to json
	curr_flagged_ngrams = {}
	for i in range(1,max_ngram_len+1):
		curr_flagged_ngrams[i] = list(curr_flagged_ngrams_with_freqs[i].keys())

	# later can also aggregate collected freq info corpus-wide for more sensitive flagging

	# count flagged ngrams
	num_flagged_ngrams = 0
	for k in curr_flagged_ngrams.keys():
		num_flagged_ngrams += len(curr_flagged_ngrams[k])

	if num_flagged_ngrams != 0:

		if verbose: print('content did not validate, new ngrams flagged (see file \"%s\")' % flagged_ngrams_txt_fn)
		with open(flagged_ngrams_txt_fn,'w') as f_out:
			for i in range(1,max_ngram_len+1):
				if i not in curr_flagged_ngrams_sorted_by_freqs.keys(): continue
				f_out.write( '\n'.join(
					[ str(curr_flagged_ngrams_sorted_by_freqs[i][k]) + '\t' + repr(k) for k in curr_flagged_ngrams_sorted_by_freqs[i].keys()]
				) )
				f_out.write('\n')

		if prompt_update_ngrams:
			choice = input('remember all new ngrams (add to file \"%s\")? (Y/[n]) ' % accepted_ngrams_json_fn)
			if choice == 'Y':

				# add new ngrams to dict
				for i in range(1,max_ngram_len+1):
					if i not in curr_flagged_ngrams.keys(): continue
					for ngram in curr_flagged_ngrams[i]:
						if str(i) not in accepted_ngrams.keys(): continue
						if ngram not in accepted_ngrams[str(i)]:
							accepted_ngrams[str(i)].append(ngram)
						accepted_ngrams[str(i)].sort()

				# overwrite json
				with open(accepted_ngrams_json_fn,'w') as f_out:
					json_object = {}
					json_object["accepted_ngrams"] = accepted_ngrams # updated
					json.dump(json_object, f_out, indent=4, ensure_ascii=False)

				# clean up appearance a little more: keep ngram lists on one line each
				with open(accepted_ngrams_json_fn,'r') as f_in:
					json_string = f_in.read()
					json_string = json_string.replace(",\n            ", ", ")
				with open(accepted_ngrams_json_fn,'w') as f_out:
					f_out.write(json_string)

		return 0

	elif num_flagged_ngrams == 0:
		if verbose: print("content validated successfully")
		return 1

if __name__ == '__main__':

	if len(sys.argv) == 1:
		print("argument needed")
		exit()

	fn = sys.argv[1]
	if fn[-4:] != ".txt":
		print(".txt needed as first argument")
		exit()

	if not ("-s" in sys.argv or "-c" in sys.argv):
		print("which mode? ( -s / -c )")
		exit()
	elif "-s" in sys.argv:
		mode = 'structure'
	elif "-c" in sys.argv:
		mode = 'content'

	if "--pause" in sys.argv:
		pause_between_problems = True

	if "--output_bracketless" in sys.argv:
		output_bracketless = True

	if ("--prompt_update_ngrams" in sys.argv
		or "-u" in sys.argv
		):
		prompt_update_ngrams = True

	with open(fn,'r') as f_in:
		raw_input_text = f_in.read()

	if mode == 'structure':
		validate_structure(raw_input_text, verbose=True)
	elif mode == 'content':
		validate_content(raw_input_text, verbose=True)


import random
import urllib
from hello.domains import myRandom, memberlist
from quiz00 import Quiz00
import random

from bs4 import BeautifulSoup
from urllib.request import urlopen
import  pandas as pd

class Quiz20:

    def quiz20list(self) -> str: return None

    def quiz21tuple(self) -> str: return None

    def quiz22dict(self) -> str: return None

    def quiz23listcom(self) -> str:
        print('---legacy---')
        a=[]
        for i in range(5):
            a.append(i)
        print(a)

        print('---comprehension---')
        a2=[i for i in range(5)]
        print(a2)
        return None

    def quiz24bugs_zip(self) -> {}:
        url = 'https://music.bugs.co.kr/chart/track/realtime/total'
        html_doc = urlopen(url)
        soup = BeautifulSoup(html_doc, 'lxml')  # html.parser vs lxml

        # self.print_music_list(soup) # 아티스트와 타이틀을 분리해서 출력하기
        # self.find_rank(soup) # 랭킹보기
        # a = [i for i in cls_names]
        # print(soup.prettify())
        ls1 = self.find_bugs_music(soup, 'title')
        ls2 = self.find_bugs_music(soup, 'artist')
        a = [i if i == 0 or i == 0 else i for i in range(1)]
        b = [i for i in []]
        c = [(i, j) for i, j in enumerate([])]
        d = {i: j for i, j in zip(ls1, ls2)}
        dict = {i:j for i, j in zip(ls1,ls2)}
        l = [ i + j for i, j in zip(ls1,ls2)]
        l2 = list(zip(ls1,ls2))
        #d1 = dict(zip(ls1,ls2))
        # self.dict1(ls1, ls2)
        # self.dict2(ls1, ls2)
        # self.dict3(ls1, ls2)
        dict = {}
        for i, j in zip(ls1, ls2):
            dict[i] = j
            print(dict)
        return dict

    def print_bugs_list(self, soup) -> None:
        artists = soup.find_all('p', {'class': 'artist'})
        # print(type(artists)) # <class 'bs4.element.ResultSet'>
        artists = [i.get_text() for i in artists]
        # print(type(artists))
        print(''.join(i for i in artists))
        titles = soup.find_all('p', {'class': 'title'})
        titles = [i.get_text() for i in titles]
        print(''.join(i for i in titles))

    def find_bugs_rank(self, soup):
        for i, j in enumerate(['artist', 'title']):
            for i, j in enumerate(self.find_bugs_music(soup, j)):
                print(f'{i}위 : {j}')
                print('#' * 100)

    @staticmethod
    def find_bugs_music(soup, cls_nm) -> []:
        ls = soup.find_all('p', {'class': cls_nm})
        return [i.get_text() for i in ls]

    @staticmethod
    def dict3(ls1,ls2):
        dict = {}
        for i, j in zip(ls1, ls2):
            dict[i] = j
            print(dict)
        return dict

    @staticmethod
    def dict2(ls1,ls2):
        dict = {}
        for i, j in enumerate(ls1):
            dict[j] = ls2[i]
        print(dict)

    @staticmethod
    def dict1(ls1,ls2) -> None:
        dict = {}
        for i in range(len(ls1)):
            dict[ls1[i]] = ls2[i]
        print(dict)

    @staticmethod
    def quiz25dictcom()-> str:
        a = Quiz00
        #1명인데 5명 추출 0~100사이 랜덤
        b = set([a.quiz06memberChoice() for i in range(5)])
        print(b)
        while len(b) !=5:
            b.add(a.quiz06memberChoice())

        scores = [myRandom(0,101)for i in range(5)]
        students = list(b)
        aa={i : j for i , j in zip (students,scores)}

        print(aa)

    def quiz26map(self) -> str: return None

    def quiz27melon_zip(self) -> {}:

        headers = {'User-Agent': 'Mozilla/5.0'}
        url = 'https://www.melon.com/chart/index.htm?dayTime=2022030816'
        req = urllib.request.Request(url, headers=headers)
        soup = BeautifulSoup(urlopen(req).read(), 'lxml')
        ls1 = self.find_melon_music(soup,'ellipsis rank01')
        ls2 = self.find_melon_artist(soup, 'checkEllipsis')
        dict = {}
        for i, j in zip(ls1, ls2):
            dict[i] = j
        print(dict)
        return dict

        # artists = soup.find_all('span',{'class' : 'checkEllipsis'})
        # artists = [i.get_text() for i in artists]
        # print('\n'.join(i for i in artists))
        #title = soup.find_all('div', {'class': 'ellipsis rank01'})
        #title = [i.get_text() for i in title]
        #print(''.join(i for i in title))
        # print(soup)

    def print_melon_list(self, soup) -> None:
        for i, j in enumerate(['ellipsis rank01', 'checkEllipsis']):
            print(''.join(i for i in [i for i in self.find_melon(soup, j)]))
            print('*' * 100)

    def find_melon_rank(self, soup) -> None:
        for i, j in enumerate(['ellipsis rank01', 'checkEllipsis']):
            for i, j in enumerate(self.find_melon(soup, j)):
                print(f'{i}위 : {j}')
            print('*' * 100)

    @staticmethod
    def find_melon_music(soup, music) -> []:
        ls = soup.find_all('div', {'class':  music})
        return [i.get_text() for i in ls]

    @staticmethod
    def find_melon_artist(soup, artist) -> []:
        ls = soup.find_all('span', {'class': artist})
        return [i.get_text() for i in ls]

    def quiz28dataframe(self) -> None:
        dict = self.quiz24bugs_zip()
        df = pd.DataFrame.from_dict(dict, orient='index')
        soup = BeautifulSoup()
        print(df)
        df.to_csv('./save/bugs.csv', sep=',', na_rep='NaN')

        dict1 = self.quiz27melon_zip()
        ds = pd.DataFrame.from_dict(dict1, orient= 'index')
        soup = BeautifulSoup()
        print(ds)
        ds.to_csv('./save/melon.csv', sep=',', na_rep='NaN')
    '''
    다음 결과 출력
        a   b   c
     1  1   3   5
     2  2   4   6

    '''
    def quiz29_pandas_(self) -> object:
        d2 = {"1": [1, 3, 5], "2": [2, 4, 6]}
        d3 = {"1": [1, 3, 5]}
        d4 = {"2": [2, 4, 6]}
        '''
        *방법1
        df2 = pd.DataFrame.from_dict(d2, orient='index', columns=['a', 'b', 'c'])
        print(df2)
        ---------------------------------------
        *방법2
        d = {'a': [1, 2], 'b': [3, 4], 'c': [5, 6]}
        df1 = pd.DataFrame(d, index=[1, 2])
        print(df1)
        '''
        evens = []
        odds = []
        [evens.append(i) if i % 2 == 0 else odds.append(i) for i in range(1,7)]
        #print(j)
        #print(h)

        jh = [odds,evens]
        #print(jh)
        ind = ['1','2']
        c_zip = {i:j for i, j in zip(ind, jh)}
        #print(c_zip)
        c = [chr(i) for i in range(97, 100)]
        df3 = pd.DataFrame.from_dict(c_zip, orient='index', columns=c)
        print(df3)
        '''
           0  1  2
        1  1  3  5
        2  2  4  6
        '''
        df4 = pd.DataFrame.from_dict(c_zip, orient='index')
        #print(df4)

        #frame = pd.DataFrame(d1, index=[1,2])
        #print(frame)

        return None