from pathlib import Path
from typing import Dict, Type, Iterator, List, Tuple

import pyarrow as pa
from pyarrow import csv as pcsv
from pyarrow import parquet as pq
from sqlalchemy import MetaData as AlchemyMetadata, Table as AlchemyTable
from sqlalchemy import Integer, SmallInteger, Float, String, CHAR, Text, Boolean, Date, DateTime
from sqlalchemy.sql.type_api import TypeEngine

from src.schemas import all_metadata
from src import EXTRACT_PATH_PREFIX, TRANSFORM_PATH_PREFIX

PARQUET_PREFIX = TRANSFORM_PATH_PREFIX.joinpath("parquet")
CSV_PREFIX = TRANSFORM_PATH_PREFIX.joinpath("csv")
# How many bytes in each CSV chunk to bring into memory.
# Larger sizes result in better compression and slightly faster time,
# but don't want to risk OOM issues on small build boxes.
BUFFER_SIZE_BYTES = 500000000

sql_type_lookup: Dict[Type[TypeEngine], str] = {
    Integer: 'int32',
    SmallInteger: 'int16',
    Float: 'float64',
    String: 'str',
    CHAR: 'str',
    Text: 'str',
    Boolean: 'bool',
    # Some Parquet targets can't handle Parquet dates, so we need to parse and pass timestamps
    Date: 'timestamp[ms]',
    DateTime: 'timestamp[ms]'
}

def get_fields(table: AlchemyTable) -> List[Tuple[str, str]]:
    cols = [(c.name, c.type) for c in table.columns.values() if c.autoincrement is not True]
    return [(name, sql_type_lookup[type(dtype)]) for name, dtype in cols]

def write_files(metadata: AlchemyMetadata) -> None:
    """
    Creates a Parquet file for each table in the schema.
    """
    tables: Iterator[AlchemyTable] = metadata.tables.values()
    for table in tables:
        name = table.name
        print(name)

        def get_path(prefix: Path, suffix: str):
            parent_dir = prefix.joinpath(metadata.schema)
            parent_dir.mkdir(exist_ok=True, parents=True)
            return parent_dir.joinpath(name).with_suffix(suffix)

        extract_file = get_path(EXTRACT_PATH_PREFIX, ".csv.zst")
        parquet_file = get_path(PARQUET_PREFIX, ".parquet")

        arrow_schema = pa.schema(get_fields(table))
        column_names = [name for name, dtype in get_fields(table)]

        read_options = pcsv.ReadOptions(column_names=column_names, block_size=BUFFER_SIZE_BYTES)
        parse_options = pcsv.ParseOptions(newlines_in_values=True)
        convert_options = pcsv.ConvertOptions(column_types=arrow_schema, timestamp_parsers=["%Y%m%d", "%Y-%m-%d"],
                                              true_values=["1", "T"], false_values=["0", "F"], strings_can_be_null=True)

        parquet_writer = pq.ParquetWriter(parquet_file, schema=arrow_schema, compression='zstd',
                                          version="2.0", use_dictionary=True)
        stream_reader = pcsv.open_csv(extract_file, read_options=read_options, parse_options=parse_options,
                                      convert_options=convert_options)
        for batch in stream_reader:
            table = pa.Table.from_batches([batch])
            parquet_writer.write_table(table)
        parquet_writer.close()

if __name__ == "__main__":
    for m in all_metadata:
        write_files(m)


#!/usr/bin/python3
'''
Gatherin data form REST API dummy JSON file
'''
import requests
import sys

if __name__ == "__main__":

    employee_id = sys.argv[1]
    employee = requests.get('https://jsonplaceholder.typicode.com/users/{}'
                            .format(employee_id)).json()
    num_task = requests.get('https://jsonplaceholder.typicode.com/todos',
                            params={'userId': employee_id}).json()

    employee_name = employee.get('name')
    tasks = []
    total_number_tasks = 0
    number_done_task = 0
    for ind in num_task:
        total_number_tasks += 1
        if ind.get('completed'):
            tasks.append(ind.get('title'))
            number_done_task += 1

    print('Employee {} is done with tasks({}/{}):'.format(
        employee_name, number_done_task, total_number_tasks))

    for phrase in tasks:
        print('\t {}'.format(phrase))


from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time
from dotenv import load_dotenv
import os

load_dotenv()
path = os.getenv('PROFILE_PATH')
personalization_id = os.getenv('PERSONALIZATION_ID')
guest_id = os.getenv('GUEST_ID')
twid = os.getenv('TWID')
auth_token = os.getenv('AUTH_TOKEN')
ct0 = os.getenv('CT0')

try:

    # 手動でログインして取得したクッキー
    cookies = [
        {'name': 'personalization_id', 'value': personalization_id},
        {'name': 'guest_id', 'value': guest_id},
        {'name': 'twid', 'value': twid},
        {'name': 'auth_token', 'value': auth_token},
        {'name': 'ct0', 'value': ct0},
    ]

    # ドライバーの設定
    driver = webdriver.Chrome(service=Service('./chromedriver'))
    driver.get('https://twitter.com/')

    # 手動で取得したクッキーをセット
    for cookie in cookies:
        driver.add_cookie(cookie)

    # クッキーをセットした後にページに再度アクセス
    url = f"https://twitter.com/{path}"
    driver.get(url)

    time.sleep(3)

    # 返信タブに移動
    replay = driver.find_elements(By.CSS_SELECTOR, '.css-1dbjc4n.r-16y2uox.r-6b64d0.r-cpa5s6')
    if len(replay) > 1:
        fourth_element = replay[1]
        fourth_element.click()
    else:
        print("Element not found or less than two elements")

    # 削除実行（リツイートは不可）
    # リツイートをスキップするためのカウンタ
    skip_count = 0
    for i in range(500):
        print(str(i)+"start")
        time.sleep(1)
        elements = driver.find_elements(By.CSS_SELECTOR, '.css-18t94o4.css-1dbjc4n.r-1777fci.r-bt1l66.r-1ny4l3l.r-bztko3.r-lrvibr')
        if len(elements) > 0:
            print(elements[skip_count*6].get_attribute("outerHTML"))
            elements[skip_count*6].click()
            time.sleep(2)
            # span_element = elements[skip_count].find_elements(By.CSS_SELECTOR, 'div[role="menuitem"] span.css-901oao.css-16my406.r-1tl8opc.r-bcqeeo.r-qvutc0')
            span_elements = driver.find_elements(By.CSS_SELECTOR, 'div[role="menuitem"] span.css-901oao.css-16my406.r-1tl8opc.r-bcqeeo.r-qvutc0')
            print(len(span_elements))
            if len(span_elements) > 0:
                span_text = span_elements[0].text
                print(span_text)
            if span_text == '削除':
                time.sleep(1)
                delete = driver.find_elements(By.CSS_SELECTOR, '.css-1dbjc4n.r-1loqt21.r-18u37iz.r-1ny4l3l.r-ymttw5.r-1f1sjgu.r-o7ynqc.r-6416eg.r-13qz1uu')
                print('hi')
                delete[0].click()
                time.sleep(1)
                submit = driver.find_elements(By.CSS_SELECTOR, '.css-901oao.r-1awozwy.r-jwli3a.r-6koalj.r-18u37iz.r-16y2uox.r-1tl8opc.r-a023e6.r-b88u0q.r-1777fci.r-rjixqe.r-bcqeeo.r-q4m81j.r-qvutc0')
                submit[0].click()
                time.sleep(1)
            else:
                # ページ全体に対してエスケープキーを送信
                driver.find_element(By.TAG_NAME, 'body').click()
                skip_count += 1
                continue
            # if span_text != "削除":
            #     time.sleep(1)
            #     # ページ全体に対してエスケープキーを送信
            #     driver.find_element(By.TAG_NAME, 'body').click()
            #     skip_count += 1
            #     continue

            # time.sleep(1)
            # delete = driver.find_elements(By.CSS_SELECTOR, '.css-1dbjc4n.r-1loqt21.r-18u37iz.r-1ny4l3l.r-ymttw5.r-1f1sjgu.r-o7ynqc.r-6416eg.r-13qz1uu')
            # print('hi')
            # delete[0].click()
            # time.sleep(1)
            # submit = driver.find_elements(By.CSS_SELECTOR, '.css-901oao.r-1awozwy.r-jwli3a.r-6koalj.r-18u37iz.r-16y2uox.r-1tl8opc.r-a023e6.r-b88u0q.r-1777fci.r-rjixqe.r-bcqeeo.r-q4m81j.r-qvutc0')
            # submit[0].click()
            # time.sleep(1)
        else:
            print("Element not found")

    input("Press any key to close the browser...")

except Exception as e:
    print(f"An error occurred: {e}")


from django.db import models
from django.contrib.auth.models import User
from .department import Department

class Feature(models.Model):
    title = models.CharField(max_length=2000)
    description = models.TextField(blank=True,
                                   null=True)
    final_price = models.DecimalField(max_digits=15,
                                      decimal_places=2,
                                      blank=True,
                                      null=True)
    price = models.DecimalField(max_digits=15,
                                decimal_places=2,
                                blank=True,
                                null=True)
    discount = models.DecimalField(max_digits=15,
                                   decimal_places=2,
                                   blank=True,
                                   null=True)
    rate = models.FloatField(blank=True,
                             null=True)
    insert_date = models.DateTimeField(auto_now_add=True)
    update_date = models.DateTimeField(auto_now=True,
                                       null=True)
    user = models.ForeignKey(User,
                             on_delete=models.CASCADE)
    department = models.ForeignKey(Department,
                                   on_delete=models.CASCADE,
                                   related_name='department_id',
                                   db_column='department')

    def __str__(self):
        return self.title


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 28 16:29:52 2022

@author: ghiggi
"""
import re
import pathlib
import pandas as pd
import xarray as xr
import numpy as np
from warnings import warn
from zipfile import ZipFile
from xarray.core import dtypes

def xr_regularize_time_dimension(ds: xr.Dataset,
                                 t_res: str = "2min30s",
                                 fill_value = dtypes.NA):
    """Fill missing timesteps with default values in a xarray Dataset.

    Parameters
    ----------
    ds : xr.Dataset
        Input xarray Dataset
    range_freq : str, optional
        Interval between each timestep, by default "2min30s"
    fill_value : optional
        Fill value for missing timesteps

    Returns
    -------
    xr.Dataset
        Dataset with a contiguous time range and no missing timesteps
    """
    start = ds.time.to_numpy()[0]
    end = ds.time.to_numpy()[-1]
    start_date = pd.to_datetime(start).date()
    end_date = pd.to_datetime(pd.to_datetime(end).date()) + pd.Timedelta(hours=23, minutes=57, seconds=30)
    full_range = pd.date_range(start=start_date,
                               end=end_date,
                               freq=t_res).to_numpy()
    ds_reindexed = ds.reindex({"time": full_range}, fill_value=fill_value)
    return ds_reindexed

def xr_drop_duplicated_timesteps(ds: xr.Dataset):
    """Drop duplicate timesteps in the input Dataset by only keeping the
    data measurements taken with the highest number of available radars.

    Parameters
    ----------
    ds : xr.Dataset
        Input xarray Dataset

    Returns
    -------
    xr.Dataset
        Processed dataset with no duplicate timesteps
    """
    idx_keep = np.arange(len(ds.time))
    to_remove = []
    _, idx, count = np.unique(ds.time, return_counts=True, return_index=True)
    index_duplicates = [list(range(idx[i], idx[i]+count[i])) for i, _ in enumerate(idx) if count[i] > 1]

    for dup in index_duplicates:
        radar_names = [len(re.sub(r'[^a-zA-Z]', '', str(ds.radar_names[i].values))) for i in dup]
        to_remove.extend([dup[i] for i in range(len(radar_names)) if i != radar_names.index(max(radar_names))])

    idx_keep = [i for i in idx_keep if i not in to_remove]
    ds = ds.isel(time=idx_keep)
    return ds


import time
from xgboost_ray import RayDMatrix, RayParams, train, predict, RayFileType, RayShardingMode
import pandas as pd
import ray
import numpy as np
import shutil
from pathlib import Path
import os
import glob
import math
from sklearn import preprocessing
from category_encoders import TargetEncoder

edge_feature_data_path = '/workspace/data/graph/output/processed_data.csv'
edge_feature_ignore_cols = ['merchant_name','user', 'card', 'split']
save_train_path = '/workspace/data/graph/processed_data_split6/train'
save_valid_path = '/workspace/data/graph/processed_data_split6/valid'
save_test_path = '/workspace/data/graph/processed_data_split6/test'

df = pd.read_csv(edge_feature_data_path)
df.drop(columns=edge_feature_ignore_cols, inplace=True)

train_df = df[df["year"] < 2018]
valid_df = df[df["year"] == 2018]
test_df = df[df["year"] > 2018]
print(train_df.shape)
print(valid_df.shape)
print(test_df.shape)

def create_path(path):
    path = Path(path)

    if path.exists() and path.is_dir():
        shutil.rmtree(path)

    os.makedirs(path)

    return None

def save_divide_df(df, save_format, save_data_path, num_partitions):
    create_path(save_data_path)

    if save_format == 'csv':
        df_splits = np.array_split(df, num_partitions)
        for i, data in enumerate(df_splits):
            data.to_csv(f"{save_data_path}/partition_{i}.csv", index=False)
    else:
        print("other data format not supported")
    print("done!")

save_divide_df(train_df, 'csv', save_train_path, 100)
save_divide_df(valid_df, 'csv', save_valid_path, 100)
save_divide_df(test_df, 'csv', save_test_path, 100)

ray.init(address="auto")
cpus_per_actor = 15
num_actors = 20
ray_params = RayParams(num_actors=num_actors, cpus_per_actor=cpus_per_actor, elastic_training=True, max_failed_actors=1, max_actor_restarts=2)

train_path = list(sorted(glob.glob(f'{save_train_path}/*.csv')))
valid_path = list(sorted(glob.glob(f'{save_valid_path}/*.csv')))
test_path = list(sorted(glob.glob(f'{save_test_path}/*.csv')))

dtrain = RayDMatrix(train_path,
                    label="is_fraud?",
                    filetype=RayFileType.CSV)

dvalid = RayDMatrix(valid_path,
                    label="is_fraud?",
                    filetype=RayFileType.CSV)

dtest = RayDMatrix(test_path,
                    label="is_fraud?",
                    filetype=RayFileType.CSV)

model_params = {
          "learning_rate":0.1,
          "eval_metric":"aucpr",
          "objective":"binary:logistic"
         }
num_boost_round = 1000
eval_every = 100

tic = time.time()
model = train(model_params,
                dtrain,
                evals=[(dtrain,'train'), (dvalid,'valid'), (dtest, 'test')],
                num_boost_round=num_boost_round,
                #maximize=True,
                verbose_eval=eval_every,
                ray_params=ray_params)
print("Training of baseline model with feature size 22 takes {:.0f} seconds".format(time.time() - tic))


# coding=utf-8
from bs4 import BeautifulSoup as bs
from urllib.error import HTTPError
from urllib.request import urlopen
from sqlalchemy import create_engine
import json
import pandas as pd
from server_config import *

print('Starting to crawl......')

# 打开网页，获取源码
def open_page(url):
    try:
        netword=urlopen(url)
    except HTTPError as hp:
        print(hp)
    else:
    # 采用BeautifulSoup来解析，且指定解析器
        html=bs(netword,'lxml')
        return html

# 获取网页数据
def get_page(url):
    # 电影名称，上映天数，电影总票房，票房占比，排片场次，排片占比，场均人次，上座率
    movieName, boxInfo = [], []
    html=open_page(url)
    # print(html)
    p=html.find('p')
    text=p.get_text()
    # 将数据转换为python能够处理的格式
    jsonObj=json.loads(text)
    # 获取字典里面特定的键对应的键值
    data=jsonObj.get('data')
    # 想要的数据就在字典的键"list"对应的值
    lists=data.get('list')
    for list in lists:
        # 获取字典里面特定的键对应的键值,并存储到列表中去
        movieName.append(list.get('movieName'))
        boxInfo.append(list.get('boxInfo'))
    return movieName, boxInfo

a = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
movieName, boxInfos, year, month, day = [], [], [], [], []

for i in range(2015, 2019):
    for j in range(1, 13):
        if i == 2016 & j == 2:
            n = 29
        else:
            n = a[j]
        for k in range(1, n):
            tmpMonth = str(j)
            tmpDay = str(k)
            tmpYear = str(i)
            if j < 10:
                tmpMonth = '0' + tmpMonth
            if k < 10:
                tmpDay = '0' + tmpDay
            url = 'https://box.maoyan.com/promovie/api/box/second.json?beginDate=' + tmpYear + tmpMonth + tmpDay
            # print(url)
            tmpMovieName, tmpboxInfos = get_page(url)
            # print(tmpMovieName)
            for x in tmpMovieName:
                movieName.append(x)
                year.append(tmpYear)
                month.append(tmpMonth)
                day.append(tmpDay)
            for x in tmpboxInfos:
                boxInfos.append(float(x))
print('Summing up result......')
d=pd.DataFrame({'name': movieName, 'boxing': boxInfos, 'year': year, 'month': month, 'day': day})#数据有三列，列名分别为one,two,three
# d.to_csv("test.csv", index = False, sep=',')
print('Started to update database......')
engine = create_engine('mysql+pymysql://'+SQL_USERNAME+':'+SQL_PASSWORD+'@'+SQL_ADDRESS+':'+SQL_PORT+'/'+SQL_SCHEMA)
d.to_sql("boxing_day",engine,if_exists='replace', index=False)
print('Daily boxing data crawled Okay!')


import ast
import jinja2

from collections.abc import Mapping
from datetime import date, datetime, timedelta
from enum import Enum
from typing import Dict
from uuid import UUID

from clipped.utils.serialization import (
    date_serialize,
    datetime_serialize,
    timedelta_serialize,
    uuid_serialize,
)

from polyaxon._flow import ParamSpec
from polyaxon._polyaxonfile.specs.libs.engine import get_engine
from polyaxon._polyaxonfile.specs.sections import Sections
from polyaxon.exceptions import PolyaxonSchemaError

try:
    import numpy as np
except (ImportError, ModuleNotFoundError):
    np = None

class PolyaxonfileParser:
    """Parses the Polyaxonfile."""

    engine = get_engine()

    @staticmethod
    def _get_section_data(section_data):
        if hasattr(section_data, "to_dict"):
            return section_data.to_dict()
        return section_data

    @classmethod
    def _get_section(cls, config, section):
        if not hasattr(config, section):
            return None
        section_data = getattr(config, section)
        if isinstance(section_data, list):
            return [cls._get_section_data(d) for d in section_data]

        return cls._get_section_data(section_data)

    @staticmethod
    def get_parsed_params(param_spec: Dict[str, ParamSpec]) -> Dict:
        parsed_params = {"params": {}}
        as_arg = []
        for param in param_spec or {}:
            parsed_params[param] = param_spec[param].get_display_value()
            # Add params level
            if param_spec[param].is_requested:
                as_arg.append(param_spec[param].as_arg())
                parsed_params["params"][param] = param_spec[param].to_parsed_param()

        parsed_params["params"]["as_args"] = [a for a in as_arg if a]
        return parsed_params

    @classmethod
    def parse_operation(
        cls, config, param_spec: Dict[str, ParamSpec]
    ):  # pylint:disable=too-many-branches
        parsed_params = cls.get_parsed_params(param_spec)

        parsed_data = {Sections.VERSION: config.version, Sections.KIND: config.kind}

        if config.name:
            parsed_data[Sections.NAME] = config.name
        if config.description:
            parsed_data[Sections.DESCRIPTION] = config.description
        if config.tags:
            parsed_data[Sections.TAGS] = config.tags
        contexts = getattr(config, Sections.CONTEXTS)
        if contexts:
            parsed_data[Sections.CONTEXTS] = [
                cls.parse_io(io.to_dict(), parsed_params) for io in contexts
            ]
        inputs = getattr(config, Sections.INPUTS)
        if inputs:
            parsed_data[Sections.INPUTS] = [
                cls.parse_io(io.to_dict(), parsed_params) for io in inputs
            ]
        outputs = getattr(config, Sections.OUTPUTS)
        if outputs:
            parsed_data[Sections.OUTPUTS] = [
                cls.parse_io(io.to_dict(), parsed_params) for io in outputs
            ]

        # Check workflow
        for section in Sections.PARSING_SECTIONS:
            config_section = cls._get_section(config, section)
            if config_section:
                parsed_data[section] = cls.parse_expression(
                    config_section, parsed_params
                )

        parsed_data = cls._parse_conditions(parsed_data)

        config_section = cls._get_section(config, Sections.RUN)
        if config_section:
            parsed_data[Sections.RUN] = config_section
        config_section = cls._get_section(config, Sections.BUILD)
        if config_section:
            parsed_data[Sections.BUILD] = config_section
        config_section = cls._get_section(config, Sections.HOOKS)
        if config_section:
            parsed_data[Sections.HOOKS] = config_section

        return parsed_data

    @classmethod
    def _parse_conditions(cls, parsed_data: Dict) -> Dict:
        if Sections.CONDITIONS in parsed_data and not isinstance(
            parsed_data[Sections.CONDITIONS], str
        ):
            parsed_data[Sections.CONDITIONS] = str(parsed_data[Sections.CONDITIONS])
        if Sections.SKIP_ON_UPSTREAM_SKIP in parsed_data and not isinstance(
            parsed_data[Sections.SKIP_ON_UPSTREAM_SKIP], str
        ):
            parsed_data[Sections.SKIP_ON_UPSTREAM_SKIP] = str(
                parsed_data[Sections.SKIP_ON_UPSTREAM_SKIP]
            )
        if Sections.IS_APPROVED in parsed_data and not isinstance(
            parsed_data[Sections.IS_APPROVED], str
        ):
            parsed_data[Sections.IS_APPROVED] = str(parsed_data[Sections.IS_APPROVED])
        return parsed_data

    @classmethod
    def _parse_hook(cls, config, param_spec: Dict[str, ParamSpec]):
        hook = cls.parse_section(config, param_spec=param_spec, parse_params=True)
        return cls._parse_conditions(hook)

    @classmethod
    def parse_hooks(cls, config, param_spec: Dict[str, ParamSpec]):
        hooks = config.hooks or []
        return [
            cls._parse_hook(hook.to_dict(), param_spec=param_spec) for hook in hooks
        ]

    @classmethod
    def parse_runtime(cls, parsed_data, param_spec: Dict[str, ParamSpec]):
        config_section = cls.parse_section(
            parsed_data.get(Sections.RUN), param_spec=param_spec, parse_params=True
        )
        if config_section:
            parsed_data[Sections.RUN] = config_section
        return parsed_data

    @classmethod
    def parse_distributed_runtime(
        cls, parsed_data, param_spec: Dict[str, Dict[str, ParamSpec]]
    ):
        run_section = parsed_data.get(Sections.RUN)
        config_section = {}
        for k in run_section:
            if k in param_spec:
                config_section[k] = cls.parse_section(
                    run_section[k], param_spec=param_spec[k], parse_params=True
                )
            else:
                config_section[k] = run_section[k]
        if config_section:
            parsed_data[Sections.RUN] = config_section
        return parsed_data

    @classmethod
    def parse_section(
        cls, config_section, param_spec: Dict[str, ParamSpec], parse_params: bool = True
    ):
        if parse_params:
            param_spec = cls.get_parsed_params(param_spec)
        if config_section:
            return cls.parse_expression(config_section, param_spec)
        return config_section

    @classmethod
    def parse_io(
        cls,
        expression: Dict,
        params: Dict[str, ParamSpec],
        check_operators: bool = False,
    ):
        # Arg format should not be converted
        arg_format = expression.pop("argFormat", None)
        value = cls.parse_expression(expression, params, check_operators)
        if arg_format:
            value["argFormat"] = arg_format
        return value

    @classmethod
    def parse_expression(  # pylint:disable=too-many-branches
        cls, expression, params: Dict, check_operators: bool = False
    ):
        try:
            return cls._parse_expression(expression, params, check_operators)
        except jinja2.exceptions.TemplateError as e:
            raise PolyaxonSchemaError(
                "Encountered a problem parsing the template, "
                "please make sure your variables are resolvable. "
                "Error: {}".format(repr(e))
            )

    @classmethod
    def _parse_expression(  # pylint:disable=too-many-branches
        cls, expression, params: Dict, check_operators: bool = False
    ):
        if isinstance(expression, (int, float, complex, type(None))):
            return expression
        if isinstance(expression, Enum):
            return expression.value
        if isinstance(expression, datetime):
            return datetime_serialize(expression)
        if isinstance(expression, date):
            return date_serialize(expression)
        if isinstance(expression, timedelta):
            return timedelta_serialize(expression)
        if isinstance(expression, UUID):
            return uuid_serialize(expression)
        if np and isinstance(expression, np.integer):
            return int(expression)
        if np and isinstance(expression, np.floating):
            return float(expression)
        if isinstance(expression, Mapping):
            if len(expression) == 1:
                old_key, value = list(expression.items())[0]
                # always parse the keys, they must be base object or evaluate to base objects
                key = cls._parse_expression(old_key, params)
                if check_operators and cls.is_operator(key):
                    return cls._parse_operator({key: value}, params)
                else:
                    return {key: cls.parse_expression(value, params, check_operators)}

            new_expression = {}
            for k, v in expression.items():
                new_expression.update(
                    cls._parse_expression({k: v}, params, check_operators)
                )
            return new_expression

        if isinstance(expression, list):
            return list(
                cls.parse_expression(v, params, check_operators) for v in expression
            )
        if isinstance(expression, tuple):
            return tuple(
                cls.parse_expression(v, params, check_operators) for v in expression
            )
        if isinstance(expression, str):
            return cls._evaluate_expression(expression, params, check_operators)

    @classmethod
    def _evaluate_expression(cls, expression, params, check_operators):
        try:
            result = cls.engine.from_string(expression).render(**params)
        except (ValueError, TypeError) as e:
            raise PolyaxonSchemaError(
                "Encountered a problem parsing the template, "
                "please make sure your variables are resolvable. "
                "Error: {}".format(repr(e))
            )
        if result == expression:
            try:
                return ast.literal_eval(result)
            except (ValueError, SyntaxError):
                pass
            return result
        return cls.parse_expression(result, params, check_operators)

    @classmethod
    def _parse_operator(cls, expression, params):
        k, v = list(expression.items())[0]
        op = Sections.OPERATORS[k].from_dict(v)
        return op.parse(parser=cls, params=params)

    @staticmethod
    def is_operator(key):
        return key in Sections.OPERATORS


import logging
from typing import Any, Iterable

from scrapy import Request, Spider
from scrapy.http import Response, JsonRequest

from util import get_stripped, get_all_stripped, get_list

logger = logging.getLogger('aa')

PAGE_SIZE = 10
BASE_URL = 'https://www.auswaertiges-amt.de'
LIST_URL = (f'{BASE_URL}/ajax/json-filterlist/de/search/-/1798?search=%22%22'
            f'&documenttype=1796%23AAArticle%20OR%20AAPress%20OR%20AAInterview%20OR%20AAFaq'
            f'&startfield=date&endfield=date&limit={PAGE_SIZE}')

class AASpider(Spider):
    name = 'aa'
    start_urls = [LIST_URL]

    def start_requests(self) -> Iterable[Request]:
        for url in self.start_urls:
            yield JsonRequest(url=url, callback=self.parse_list, dont_filter=True, meta={'offset': 0})

    def parse_list(self, response: Response):
        body = response.json()
        num_total = body.get('itemsTotal')
        next_offset = response.meta.get('offset', 0) + PAGE_SIZE

        for article in body.get('items', []):
            logger.debug(f'Found article link: {article["link"]}')
            yield response.follow(BASE_URL + article['link'], self.parse_article)

        if next_offset < num_total:
            next_link = f'{LIST_URL}&offset={next_offset}'
            logger.info(f'Found next page: {next_link}')
            yield JsonRequest(url=next_link, callback=self.parse_list, meta={'offset': next_offset})

    def parse_article(self, response: Response, **kwargs: Any):
        logger.debug(f'Parsing article page: {response.request.url}')
        content = response.css('#main')
        head = response.css('#main header')

        date = None
        desc = None
        date_type = get_stripped(head, 'span.heading__meta::text')
        if date_type is not None:
            parts = date_type.split(' - ')
            if len(parts) >= 2:
                date = parts[0]
                desc = parts[1]

        yield {
            'date': date,
            'descriptor': desc,
            'tags': get_list(content, 'section.u-section ul.link-list__list a.link-list__link::attr("title")'),
            'title': get_all_stripped(head, 'h1.heading__title *::text'),
            'teaser': get_stripped(head, 'p.heading__intro *::text'),
            'text': get_all_stripped(content, 'div.c-rte--default *::text'),
            'link': response.request.url
        }


import math

import torch
import torch.nn as nn
import torch.nn.functional as F

from dril.a2c_ppo_acktr.utils import AddBias, init
import matplotlib.pyplot as plt
"""
Modify standard PyTorch distributions so they are compatible with this code.
"""

#
# Standardize distribution interfaces
#

# Categorical
FixedCategorical = torch.distributions.Categorical

old_sample = FixedCategorical.sample
FixedCategorical.sample = lambda self: old_sample(self).unsqueeze(-1)

log_prob_cat = FixedCategorical.log_prob
FixedCategorical.log_probs = lambda self, actions: log_prob_cat(
    self, actions.squeeze(-1)).view(actions.size(0), -1).sum(-1).unsqueeze(-1)

FixedCategorical.mode = lambda self: self.probs.argmax(dim=-1, keepdim=True)

# Normal
FixedNormal = torch.distributions.Normal

log_prob_normal = FixedNormal.log_prob
FixedNormal.log_probs = lambda self, actions: log_prob_normal(
    self, actions).sum(
        -1, keepdim=True)

normal_entropy = FixedNormal.entropy
FixedNormal.entropy = lambda self: normal_entropy(self).sum(-1)

FixedNormal.mode = lambda self: self.mean

###################################################################
# Beta
FixedBeta = torch.distributions.Beta

log_prob_beta = FixedBeta.log_prob
FixedBeta.log_probs = lambda self, actions: log_prob_beta(
    self, actions).sum(
        -1, keepdim=True)

beta_entropy = FixedBeta.entropy
FixedBeta.entropy = lambda self: beta_entropy(self).sum(-1)

FixedBeta.mode = lambda self: self.mean
###################################################################

# Bernoulli
FixedBernoulli = torch.distributions.Bernoulli

log_prob_bernoulli = FixedBernoulli.log_prob
FixedBernoulli.log_probs = lambda self, actions: log_prob_bernoulli(
    self, actions).view(actions.size(0), -1).sum(-1).unsqueeze(-1)

bernoulli_entropy = FixedBernoulli.entropy
FixedBernoulli.entropy = lambda self: bernoulli_entropy(self).sum(-1)
FixedBernoulli.mode = lambda self: torch.gt(self.probs, 0.5).float()

class Categorical(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(Categorical, self).__init__()

        init_ = lambda m: init(
            m,
            nn.init.orthogonal_,
            lambda x: nn.init.constant_(x, 0),
            gain=0.01)

        self.linear = init_(nn.Linear(num_inputs, num_outputs))

    def forward(self, x):
        x = self.linear(x)
        return FixedCategorical(logits=x)

    def get_logits(self, x):
        x = self.linear(x)
        return FixedCategorical(logits=x).logits

class DiagGaussian(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(DiagGaussian, self).__init__()

        init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init.
                               constant_(x, 0))

        self.fc_mean = init_(nn.Linear(num_inputs, num_outputs))
        self.logstd = AddBias(torch.zeros(num_outputs))

    def forward(self, x):
        action_mean = self.fc_mean(x)

        #  An ugly hack for my KFAC implementation.
        zeros = torch.zeros(action_mean.size())
        if x.is_cuda:
            zeros = zeros.cuda()

        action_logstd = self.logstd(zeros)
        return FixedNormal(action_mean, action_logstd.exp())

    def get_mean(self, x):
        return self.fc_mean(x)

###########################################################################
class DiagBeta(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(DiagBeta, self).__init__()

        init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init.
                               constant_(x, 0))

        #self.fc_mean = init_(nn.Linear(num_inputs, num_outputs))
        #self.logstd = AddBias(torch.zeros(num_outputs))

        self.alpha_head = init_(nn.Linear(num_inputs, num_outputs))
        self.beta_head  = init_(nn.Linear(num_inputs, num_outputs))
        self.sp = nn.Softplus()

    def forward(self, x):
        alpha = self.sp(self.alpha_head(x)) + 1
        beta = self.sp(self.beta_head(x)) + 1

        return FixedBeta(alpha, beta) #, alpha, beta

    def get_mean(self, x):
        alpha = self.sp(self.alpha_head(x)) + 1
        beta = self.sp(self.beta_head(x)) + 1

        return alpha / (alpha + beta)

###########################################################################

class Bernoulli(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(Bernoulli, self).__init__()

        init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init.
                               constant_(x, 0))

        self.linear = init_(nn.Linear(num_inputs, num_outputs))

    def forward(self, x):
        x = self.linear(x)
        return FixedBernoulli(logits=x)

    def get_logits(self, x):
        return self.linear(x)

# CHECKS DRIL BETA DISTRIBUTION

def test_1():
    input_th = torch.ones([8])
    input_th = input_th.unsqueeze(0)
    beta_dist = DiagBeta(8,1)
    dist_curve, alpha, beta = beta_dist(input_th)

    n = 1000
    sp_hist = []
    for i in range(n):
        s = dist_curve.sample()
        sp_hist.append(s.item())

    print(f'mean = {dist_curve.mean}')

    plt.hist(sp_hist, bins=100)
    plt.show()

def test_2():
    input_th = torch.ones([8])
    input_th = input_th.unsqueeze(0)
    beta_dist = DiagBeta(8, 2)
    dist_curve, alpha, beta = beta_dist(input_th)

    output = beta_dist.get_mean(input_th)
    print(alpha)
    print(beta)
    print(output)

if __name__ == '__main__':
    #test_1() #deprecated
    test_2()


import pygame
from MazeGame.GUI.GuiState.State import State
from MazeGame.GUI.GuiDesign.RankingScreen import RankingScreen
from MazeGame.Ranking.RankingManager import RankingManager
from Engine.Structs.GameSettings import GameMode

class RankingState(State):
    def __init__(self,setstatus,view):
       super().__init__(setstatus,view,)
       self.ranking = RankingManager()

       self.__game_mode = GameMode.EASY
       self.view.results = self.ranking.get_results_by_mode(GameMode.EASY)

    def set_game_mode(self, mode:GameMode):
      self.__game_mode = mode

    def render(self):
      #  self.view = RankingScreen()
      self.view.screen_design()
      return self.view.render()

    def update(self, event):
      if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:
        mouse_pos = pygame.mouse.get_pos()
        if self.view.buttons['Menu inicial'].clicked(mouse_pos):
            self.setstatus.state('home')

        if self.view.buttons['Fácil'].clicked(mouse_pos):
          self.set_game_mode(GameMode.EASY)
        if self.view.buttons['Intermediário'].clicked(mouse_pos):
          self.set_game_mode(GameMode.MEDIUM)
        if self.view.buttons['Difícil'].clicked(mouse_pos):
          self.set_game_mode(GameMode.HARD)
        self.view.results = self.ranking.get_results_by_mode(self.__game_mode)

Programação Orientada a Objetos 2/versao_final/MazeGame/GUI/GuiState/RankingState.py


import numpy as np
from PIL import Image
import os

def bfs(mask, i, j, vis, val):
    q = [(i, j)]
    vis[i][j] = False
    for i, j in q:
        for x, y in ((i - 1, j), (i + 1, j), (i, j - 1), (i, j + 1)):
            if 0 <= x < mask.shape[0] and 0 <= y < mask.shape[1] and mask[x][y] != [0, 0, 0] and vis[x][y] == True:
                if mask[x][y] != mask[i][j]:
                    mask[i][j] = 0
                else:
                    vis[x][y] = False
                    q.append((x, y))

def reduction(mask):
    print(mask.shape)
    vis = np.ones((mask.shape[0], mask.shape[1]), dtype=bool)
    for i in range(mask.shape[0]):
        for j in range(mask.shape[1]):
            if mask[i][j] != [0, 0, 0] and vis[i][j] == True:
                bfs(mask, i, j, vis, mask[i][j])

def process(input, output):
    print(len(os.listdir(input)))
    for f in os.listdir(input):
        print(os.path.join(input, f))
        mask = np.array(Image.open(os.path.join(input, f)))
        reduction(mask)

        output_img = np.ones((mask.shape[0], mask.shape[1]), dtype=np.uint8)
        for i in range(mask.shape[0]):
            for j in range(mask.shape[1]):
                output_img[i][j] = 0 if mask[i][j] == [0, 0, 0] else 255

        print(os.path.join(output, f))
        Image.fromarray(output_img).save(os.path.join(output, f))

if __name__ == '__main__':
    input = 'D:\\zl\\GraduationThesis\\data\\new_data\\label1'
    output = 'D:\\zl\\GraduationThesis\\data\\new_data\\label2'
    process(input, output)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Created on Dec 31, 2013

@author: zhong
'''
import random
import pickle
import os
import json

""" Basic blog using webpy 0.3 """
import web
import models

### Url mappings

urls = (
    '/favicon.ico','favicon',
    '/', 'Login',
    '/admin', 'Admin',
    '/restore', 'Restore',
    '/game/(\w+)', 'Game',
    '/process/(\w+)', 'Process',
    '/message/(\w+)', 'Message',
    '/update', 'Update',
)

class favicon:
    def GET(self):
        f = open("static/favicon.ico", 'rb')
        return f.read()

### Templates
render = web.template.render('templates', base='base', globals = {'models':models})

class Login:
    def GET(self):
        form = web.form.Form(
            web.form.Textbox('username', description = u"用户名"),
            web.form.Textbox('nickname', description = u"昵称"),
            web.form.Button('Login', description = u"登录"),
        )
        return render.login(form)

    def POST(self):
        global s
        if s is None:
            return "Game has not started yet."
        else:
            for p in s.context.players:
                if p.username == web.input()['username'].strip():
                    nickname = web.input()['nickname'].strip()
                    if nickname != '':
                        p.nickname = nickname
                    raise web.seeother('/game/%s' % web.input()['username'])
                    break
            else:
                return "User not found."

class Admin:
    def GET(self):
        form = web.form.Form(
            web.form.Textbox('players', description = u"全部玩家用户名，逗号分隔"),
            web.form.Textbox('killer_num', description = u"杀手人数"),
            web.form.Textbox('police_num', description = u"警察人数"),
            web.form.Textbox('civilian_num', description = u"平民人数"),
            web.form.Radio('civilian_all_die_winner', ['killer', 'police'], description = u"平民全部牺牲情况下的获胜方"),
            web.form.Button('submit', description = u"创建游戏"),
        )
        return render.admin(form)

    def POST(self):
        global s
        usernames = [n.strip() for n in web.input()['players'].split(',')]
        killer_num = int(web.input()['killer_num'])
        police_num = int(web.input()['police_num'])
        civilian_num = int(web.input()['civilian_num'])
        assert len(usernames) == killer_num + police_num + civilian_num
        identities = [models.KILLER] * killer_num + [models.POLICE] * police_num + [models.CIVILIAN] * civilian_num
        random.shuffle(identities)

        context = models.Context([models.Player(username, identity) for username, identity in zip(usernames, identities)])
        context.civilian_all_die_winner = models.POLICE if web.input()['civilian_all_die_winner'] == 'police' else models.KILLER

        s = models.GameStart(context).pre()

        raise web.seeother('/')

class Restore:
    def GET(self):
        form = web.form.Form(
            web.form.Dropdown('status', args = [filename for filename in os.listdir('status') if 'pickle' in filename]),
            web.form.Button('Restore', description = u"恢复状态"),
        )
        return render.restore(form)

    def POST(self):
        global s
        filename = web.input()['status']
        s = pickle.load(open('status/%s'%filename)).pre()
        raise web.seeother('/')

class Update:
    def POST(self):
        global s
        web.header('Content-Type', 'application/json')

        if s is None:
            messages = None
            new_message_count = 0
            players = None
            form = get_form(None, None)
            new_sid = 0
        else:
            username = web.input()['username']
            message_count = int(web.input()['message_count'])
            sid = int(web.input()['sid'])

            player = s.context.get_player(username)
            if player is None:
                messages = None
                new_message_count = 0
                players = None
                form = get_form(s, None)
                new_sid = 0
            else:
                messages = get_messages(s, player, message_count)
                new_message_count = len(player.message.data)

                new_sid = id(s)
                if new_sid == sid:
                    players = form = None
                else:
                    players = get_players(s, player)
                    form = get_form(s, player)

        return json.dumps(
            ((messages, new_message_count),
             players, form, new_sid)
        )

formrender = web.template.render('templates')

def get_messages(s, player, message_count):
    if message_count < len(player.message.data):
        return str(formrender.messages(s, player, message_count))
    else:
        return None

def get_players(s, player):
    return str(formrender.players(s, player))

def get_form(s, player):

    if s is None:
        return '
游戏尚未开始
 Admin   Restore'
    elif player is None:
        return '
该用户不在游戏中
 Admin   Restore'
    elif isinstance(s, models.KillStatus):
        if player in s.actors:
            return str(formrender.kill(s))
        else:
            return str(formrender.wait(s, secret = True))
    elif isinstance(s, models.InvestigateStatus):
        if player in s.actors:
            return str(formrender.investigate(s))
        else:
            return str(formrender.wait(s, secret = True))
    elif isinstance(s, models.TalkStatus):
        if player in s.actors:
            return str(formrender.talk(s))
        else:
            return str(formrender.wait(s, secret = False))
    elif isinstance(s, models.VoteStatus):
        if player in s.actors:
            return str(formrender.vote(s))
        else:
            return str(formrender.wait(s, secret = False))
    elif isinstance(s, models.LastWordsStatus):
        if player in s.actors:
            return str(formrender.lastwords(s))
        else:
            return str(formrender.wait(s, secret = False))
    elif isinstance(s, models.GameOver):
        return '
游戏结束
 Admin   Restore'
    else:
        return None

class Process:
    def POST(self, username):
        global s
        actor = s.context.get_player(username)
        if str(id(s)) == web.input()['sid']:
            kwargs = {'actor':actor}
            if 'target' in web.input():
                target_text = web.input()['target']
                if target_text == 'None':
                    kwargs['target'] = None
                else:
                    kwargs['target'] = s.context.get_player(target_text[5:]) # perfix is user:

            if 'words' in web.input():
                kwargs['words'] = web.input()['words']

            if web.input()['type'] == 'submit':
                s = s.act(**kwargs)
            elif web.input()['type'] == 'cancel':
                s = s.cancel(actor = actor)
        else:
            actor.message.add("warn", u"您刚刚的操作已过期，请重试")
        return json.dumps(""); # Create a success result

class Game:
    def GET(self, username):
        global s
        return render.game(username, s)

s = None

if __name__ == '__main__':
    app = web.application(urls, globals())
    app.run()


from __future__ import annotations

import logging
import sys

import numpy as np
import pandas as pd
from ak_api.security.appsec import Appsec
from ak_api.security.botmanager import BotManager
from ak_api.security.networklist import NetworkList
from rich import print_json
from utils import _logging as lg
from utils import dataframe

class AppsecWrapper(Appsec):
    def __init__(self, account_switch_key: str | None = None,
                 section: str | None = None,
                 cookies: str | None = None,
                 logger: logging.Logger = None):

        super().__init__(account_switch_key=account_switch_key, section=section, cookies=cookies)
        self.logger = logger

    def get_config_detail(self, config_id: int):
        return super().get_config_detail(config_id)

    def get_config_version_detail(self, config_id: int, version: int, exclude: list | None = None):
        return super().get_config_version_detail(config_id, version, exclude)

    def get_config_version_metadata_xml(self, config_name: str, version: int):
        return super().get_config_version_metadata_xml(config_name, version)

    def list_waf_configs(self):
        return super().list_waf_configs()

    def get_policy(self, config_id: int, version: int):
        return super().get_policy(config_id, version)

    def list_custom_rules(self, config_id: int):
        return super().list_custom_rules(config_id)

    def bypass_network_list(self, config_id: int, version: int, policy_id: str):
        return super().bypass_network_list(config_id, version, policy_id)

class NetworkListWrapper(NetworkList):
    def __init__(self, account_switch_key: str | None = None,
                 section: str | None = None,
                 logger: logging.Logger = None):
        super().__init__()
        self.account_switch_key = account_switch_key
        self.logger = logger

    def get_all_network_list(self):
        return super().get_all_network_list()

    def get_network_list(self, ids):
        self.logger.debug(ids)
        if isinstance(ids, str):
            status, result = super().get_network_list(ids)
            if status == 200:
                try:
                    return sorted(result['list'])
                except:
                    self.logger.error(f'{ids:<40} has no IPs')
                    return ['None']
        elif isinstance(ids, list):
            all_ips = []
            for id in ids:
                status, result = super().get_network_list(id)
                if status == 200:
                    try:
                        all_ips.extend(result['list'])
                    except:
                        self.logger.error(f'{id:<40} has no IPs')
                        return ['None']
            return sorted(all_ips)

class BotManagerWrapper(BotManager):
    def __init__(self, account_switch_key: str | None = None,
                 section: str | None = None,
                 logger: logging.Logger = None):
        super().__init__()
        self.account_switch_key = account_switch_key
        self.logger = logger

    def get_all_akamai_bot_catagories(self):
        status, response = super().get_all_akamai_bot_catagories()
        if status == 200:
            try:
                return response['categories']
            except:
                return 0

    def get_akamai_bot_catagory(self, id: str):
        status, response = super().get_akamai_bot_catagory(id)
        if status == 200:
            try:
                return response
            except:
                return 0

    def get_all_custom_bot_catagories(self, config_id: str, version: int):
        status, response = super().get_all_custom_bot_catagories(config_id, version)
        if status == 200:
            try:
                return response['categories']
            except:
                return 0

    def get_custom_bot_catagory(self, config_id: str, version: int, category_id: str):
        status, response = super().get_custom_bot_catagory(config_id, version, category_id)

        if status == 200:
            try:
                return response
            except:
                return 0

    def get_custom_bot_catagory_action(self, config_id: str, version: int, policy_id: str, category_id: str):
        status, response = super().get_custom_bot_catagory_action(config_id, version, policy_id, category_id)

        if status == 200:
            try:
                return response
            except:
                return 0

    def get_custom_bot_catagory_sequence(self, config_id: str, version: int):
        status, response = super().get_custom_bot_catagory_sequence(config_id, version)

        if status == 200:
            try:
                return response['sequence']
            except:
                return 0

    def get_custom_defined_bot(self, config_id: str, version: int, bot_id: str):
        status, response = super().get_custom_defined_bot(config_id, version, bot_id)

        if status == 200:
            try:
                return response
            except:
                return 0

    def process_custom_bot(self, data, network):
        feature = 'customDefinedBots'

        if isinstance(data, list):
            try:
                columns = data[0].keys()
            except:
                self.logger.critical(f'{feature:<40} no data')
                return pd.DataFrame()

            self.logger.debug(f'{feature:<40} {len(columns):<5} {columns}')
            df = pd.json_normalize(data)
            if 'description' in df.columns:
                del df['description']
            if 'notes' in df.columns:
                del df['notes']
            df['conditions_count'] = df['conditions'].apply(lambda x: len(x) if isinstance(x, list) else 0)

            # df = df[df['botName'] == 'Allow Botify'].copy()

            # extract conditions column
            all_keys = dataframe.extract_keys(df['conditions'].sum())
            columns_to_explode = list(all_keys)
            self.logger.debug(columns_to_explode)

            for key in all_keys:
                df[key] = df['conditions'].apply(lambda x: [d.get(key) for d in x])
            self.logger.debug(f'\n{df[columns_to_explode]}')

            exploded_data = exploded_data = dataframe.explode_cell(df, 'conditions', columns_to_explode)
            exploded_df = pd.DataFrame(exploded_data)
            col_1 = ['categoryId', 'botId', 'botName', 'conditions_no']
            col_2 = ['type', 'name', 'positiveMatch', 'value', 'checkIps', 'valueCase', 'nameWildcard']
            col_2 = [value for value in col_2 if value in columns_to_explode]

            try:
                columns = col_1 + col_2 + ['IPs']
                exploded_df['conditions_no'] = (exploded_df.groupby(['categoryId', 'botId', 'botName']).cumcount() + 1)

                exploded_df['IPs'] = exploded_df.apply(lambda row: network.get_network_list(row['value'][0])
                                                   if row['type'] == 'networkListCondition' else '', axis=1)
                exploded_df['IPs'] = exploded_df.apply(lambda row: dataframe.split_elements_newline(row['IPs'])
                                                        if row['type'] == 'networkListCondition' else '', axis=1)
                exploded_df['name'] = exploded_df.apply(lambda row: dataframe.split_elements_newline_withcomma(row['name'])
                                                       if isinstance(row['name'], list) else row['name'], axis=1)
                exploded_df['value'] = exploded_df.apply(lambda row: dataframe.split_elements_newline_withcomma(row['value'])
                                                        if row['value'] else '', axis=1)
            except:
                columns = col_1 + col_2

        return exploded_df[columns]

    def process_custom_deny_list(self, data):
        feature = 'customDenyList'

        if isinstance(data, list):
            try:
                columns = data[0].keys()
            except:
                self.logger.critical(f'{feature:<40} no data')
                return pd.DataFrame()

            self.logger.debug(f'{feature:<40} {len(columns):<5} {columns}')
            df = pd.json_normalize(data)
            df['rule_name'] = df['name']
            del df['name']
            if 'description' in df.columns:
                df['rule_description'] = df['description']
            df['parameters_count'] = df['parameters'].apply(lambda x: len(x) if isinstance(x, list) else 0)
            # extract parameters column
            all_keys = dataframe.extract_keys(df['parameters'].sum())
            parameters_columns = list(all_keys)

            for key in all_keys:
                df[key] = df['parameters'].apply(lambda x: [d.get(key) for d in x])

            exploded_data = exploded_data = dataframe.explode_cell(df, 'parameters', parameters_columns)
            exploded_df = pd.DataFrame(exploded_data)
            original_columns = ['id', 'rule_name']
            if 'description' in df.columns:
                original_columns.append('rule_description')

            exploded_df['parameters_no'] = (exploded_df.groupby(['id', 'rule_name']).cumcount() + 1)
            return exploded_df[original_columns + ['parameters_no'] + ['name', 'displayName', 'value']]

    def process_custom_rules(self, data):
        feature = 'customRules'

        if isinstance(data, list):
            try:
                columns = data[0].keys()
            except:
                self.logger.critical(f'{feature:<40} no data')
                return pd.DataFrame()
            self.logger.debug(f'{feature:<40} {len(columns):<5} {columns}')
            df = pd.json_normalize(data)
            original_keys = df.columns.tolist()
            if 'conditions' not in original_keys:
                return df
            else:
                original_keys.remove('conditions')
                df['conditions_count'] = df['conditions'].apply(lambda x: len(x) if isinstance(x, list) else 0)

                # extract conditions column
                all_keys = dataframe.extract_keys(df['conditions'].dropna().sum())
                columns_to_explode = list(all_keys)

                if all_keys is None:
                    return pd.DataFrame()
                else:
                    self.logger.debug(columns_to_explode)
                    for key in all_keys:
                        df[key] = df['conditions'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])

                    col_1 = original_keys
                    col_2 = ['type', 'name', 'value', 'valueWildcard', 'valueCase', 'positiveMatch', 'nameWildcard']
                    col_2 = [value for value in col_2 if value in columns_to_explode]
                    columns = col_1 + ['conditions_no'] + col_2
                    exploded_data = dataframe.explode_cell(df, 'conditions', columns_to_explode)
                    exploded_df = pd.DataFrame(exploded_data)
                    exploded_df['conditions_no'] = (exploded_df.groupby(['id']).cumcount() + 1)
        return exploded_df[columns]

    def process_rate_policies(self, data, network):
        feature = 'ratePolicies'

        if isinstance(data, list):
            try:
                columns = data[0].keys()
            except:
                self.logger.critical(f'{feature:<40} no data')
                return []

            self.logger.debug(f'{feature:<40} {len(columns):<5} {columns}')
            df = pd.json_normalize(data)

            if 'additionalMatchOptions' not in df.columns.tolist():
                return df
            else:
                original_keys = df.columns.tolist()
                self.logger.debug(original_keys)
                original_keys.remove('additionalMatchOptions')
                original_keys.remove('type')

                df['policy_type'] = df['type']
                df['additionalMatchOptions_count'] = df['additionalMatchOptions'].apply(lambda x: len(x) if isinstance(x, list) else 0)
                all_keys = dataframe.extract_keys(df['additionalMatchOptions'].dropna().sum())
                columns_to_explode = list(all_keys)
                self.logger.debug(columns_to_explode)

                for key in all_keys:
                    df[key] = df['additionalMatchOptions'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])

                exploded_data = dataframe.explode_cell(df, 'additionalMatchOptions', columns_to_explode)
                exploded_df = pd.DataFrame(exploded_data)
                col_1 = original_keys + ['policy_type']
                columns = col_1 + ['additionalMatchOptions_no'] + columns_to_explode + ['IPs']
                exploded_df['IPs'] = exploded_df.apply(lambda row: network.get_network_list(row['values'])
                                                    if row['type'] == 'NetworkListCondition' else '', axis=1)
                exploded_df['IPs'] = exploded_df.apply(lambda row: dataframe.split_elements_newline(row['IPs'])
                                                    if row['type'] == 'NetworkListCondition' else '', axis=1)

                exploded_df['additionalMatchOptions_no'] = (exploded_df.groupby(['id']).cumcount() + 1)

                return exploded_df[columns]

    def process_matchTargets(self, data, network):
        feature = 'matchTargets'

        if isinstance(data, list):
            try:
                columns = data[0].keys()
            except:
                self.logger.critical(f'{feature:<40} no data')
                return pd.DataFrame()

            self.logger.debug(f'{feature:<40} {len(columns):<5} {columns}')
            df = pd.json_normalize(data)
            df['policyId'] = df['securityPolicy.policyId']
            del df['securityPolicy.policyId']

            original_keys = df.columns.tolist()
            self.logger.debug(original_keys)

            if 'bypassNetworkLists' not in original_keys:
                return df
            else:
                df['matchTarget_type'] = df['type']
                df['matchTarget_id'] = df['id']
                original_keys.remove('type')
                original_keys.remove('id')
                original_keys.remove('bypassNetworkLists')
                df['bypassNetworkLists_count'] = df['bypassNetworkLists'].apply(lambda x: len(x) if isinstance(x, list) else 0)

                # extract conditions column
                all_keys = dataframe.extract_keys(df['bypassNetworkLists'].dropna().sum())
                columns_to_explode = list(all_keys)
                self.logger.debug(all_keys)

                for key in all_keys:
                    df[key] = df['bypassNetworkLists'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])

                self.logger.debug(f'\n{df[columns_to_explode]}')

                exploded_data = dataframe.explode_cell(df, 'bypassNetworkLists', columns_to_explode)
                exploded_df = pd.DataFrame(exploded_data)

                exploded_df['IPs'] = exploded_df.apply(lambda row: network.get_network_list(row['id'])
                                                            if row['listType'] == 'NL' else '', axis=1)
                exploded_df['IPs'] = exploded_df.apply(lambda row: dataframe.split_elements_newline(row['IPs'])
                                                            if row['listType'] == 'NL' else '', axis=1)
                self.logger.debug(f'\n{exploded_df}')
                exploded_df = exploded_df.rename(columns={'id': 'bypassNetworkListsId'})

                original_keys.remove('policyId')
                original_keys.remove('sequence')
                col_1 = ['policyId', 'sequence'] + ['matchTarget_type', 'matchTarget_id']
                col_2 = ['bypassNetworkListsId', 'listType', 'name', 'type'] + ['IPs']
                columns = col_1 + ['NetworkListsId_no'] + col_2 + original_keys
                exploded_df['NetworkListsId_no'] = (exploded_df.groupby(['policyId']).cumcount() + 1)

        return exploded_df[columns]

    def process_reputation_profiles(self, data, network):
        feature = 'reputationProfiles'

        df = pd.json_normalize(data)
        df = df.sort_values(by=['context', 'threshold'])
        if 'condition.atomicConditions' not in df.columns.tolist():
            return df
        else:
            df['atomicConditions'] = df['condition.atomicConditions']
            df['context_id'] = df['id']
            del df['condition.atomicConditions']
            del df['id']
            original_keys = df.columns.tolist()

            df['atomicConditions_count'] = df['atomicConditions'].apply(lambda x: len(x) if isinstance(x, list) else 0)
            all_keys = dataframe.extract_keys(df['atomicConditions'].dropna().sum())
            if all_keys:
                for key in all_keys:
                    df[key] = df['atomicConditions'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])
                rules_columns = list(all_keys)
                exploded_data = dataframe.explode_cell(df, 'atomicConditions', rules_columns)

                condition_columns = ['className', 'index', 'positiveMatch', 'value', 'valueCase', 'valueWildcard', 'checkIps', 'IPs']
                condition_columns = [value for value in condition_columns if value in all_keys]
                condition_df = pd.DataFrame(exploded_data)

                condition_df['condition.version'] = condition_df['condition.version'].astype(str)

                condition_df['condition.version'] = condition_df['condition.version'].replace({'nan': ' '})

                condition_df['IPs'] = condition_df.apply(lambda row: network.get_network_list(row['value'])
                                                            if row['className'] == 'NetworkListCondition' else '', axis=1)
                condition_df['IPs'] = condition_df.apply(lambda row: dataframe.split_elements_newline(row['IPs'])
                                                            if row['className'] == 'NetworkListCondition' else '', axis=1)

        return condition_df[original_keys + condition_columns]

    def process_response_actions(self, data, network):
        feature = 'responseActions'

        df = pd.json_normalize(data)
        original_keys = df.columns.tolist()
        self.logger.debug(original_keys)
        if 'conditionalActions' not in original_keys:
            return df, pd.DataFrame(), pd.DataFrame()
        else:
            original_keys.remove('conditionalActions')

            df['conditionalActions_count'] = df['conditionalActions'].apply(lambda x: len(x) if isinstance(x, list) else 0)

            # extract conditions column
            all_keys = dataframe.extract_keys(df['conditionalActions'].dropna().sum())
            columns_to_explode = list(all_keys)

            if all_keys is None:
                return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
            else:
                for key in all_keys:
                    df[key] = df['conditionalActions'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])

                self.logger.debug(f'\n{df[columns_to_explode]}')

                exploded_data = dataframe.explode_cell(df, 'conditionalActions', columns_to_explode)
                exploded_df = pd.DataFrame(exploded_data)
                conditionalActions_count = exploded_df['conditionalActions_count'].sum()

                if conditionalActions_count == 0:
                    return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
                else:
                    col_1 = original_keys
                    col_2 = ['conditionalActions_count', 'conditionalActions'] + ['actionId', 'actionName', 'defaultAction', 'conditionalActionRules', 'description']
                    if 'description' not in all_keys:
                        col_2.remove('description')
                        self.logger.debug(col_2)
                    columns = col_1 + col_2 + ['conditionalActionRules_count']

                    exploded_df['conditionalActionRules_count'] = exploded_df['conditionalActionRules'].apply(lambda x: len(x) if isinstance(x, list) else 0)
                    all_keys = dataframe.extract_keys(exploded_df['conditionalActionRules'].dropna().sum())
                    columns_to_explode = list(all_keys)
                    self.logger.debug(columns_to_explode)
                    for key in all_keys:
                        exploded_df[key] = exploded_df['conditionalActionRules'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])

                    exploded_data = dataframe.explode_cell(exploded_df, 'conditionalActionRules', columns_to_explode)
                    new_df = pd.DataFrame(exploded_data)

                    all_keys = dataframe.extract_keys(new_df['conditions'].dropna().sum())
                    columns_to_explode = list(all_keys)

                    for key in all_keys:
                        new_df[key] = new_df['conditions'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])

                    exploded_data = dataframe.explode_cell(new_df, 'conditions', columns_to_explode)
                    conditions_df = pd.DataFrame(exploded_data)

                    col_1 = ['challengeActions', 'customDenyActions', 'serveAlternateActions', 'challengeInjectionRules.injectJavaScript', 'challengeInterceptionRules.interceptAllRequests']
                    col_2 = ['actionId', 'defaultAction', 'actionName', 'percentageOfTraffic', 'action']
                    col_3 = ['checkIps', 'positiveMatch', 'type', 'value', 'host', 'valueCase', 'nameWildcard', 'valueWildcard']
                    col_3 = [value for value in col_3 if value in columns_to_explode]
                    cols = col_1 + col_2 + col_3 + ['IPs']
                    conditions_df['IPs'] = conditions_df.apply(lambda row: network.get_network_list(row['value'])
                                                            if row['type'] == 'networkListCondition' else '', axis=1)
                    conditions_df['IPs'] = conditions_df.apply(lambda row: dataframe.split_elements_newline(row['IPs'])
                                                            if row['type'] == 'networkListCondition' else '', axis=1)
                    self.logger.debug(conditions_df[cols])
                    return exploded_df[columns], new_df, conditions_df[cols]

    def process_rulesets(self, data):
        feature = 'rulesets'
        # extract rules column
        df = pd.json_normalize(data)
        df['ruleset_id'] = df['id']
        del df['id']
        original_keys = df.columns.tolist()
        original_keys.remove('rules')
        original_keys.remove('attackGroups')
        df['rules_count'] = df['rules'].apply(lambda x: len(x) if isinstance(x, list) else 0)
        all_keys = dataframe.extract_keys(df['rules'].dropna().sum())
        if all_keys:
            for key in all_keys:
                df[key] = df['rules'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])
            rules_columns = list(all_keys)
            exploded_data = dataframe.explode_cell(df, 'rules', rules_columns)
            rules_df = pd.DataFrame(exploded_data)
            original_keys.remove('ruleset_id')
            rules_columns = ['attackGroups', 'id', 'inspectRequestBody', 'inspectResponseBody', 'outdated', 'ruleVersion', 'score', 'tag', 'title']

            rules_df = rules_df[['ruleset_id'] + original_keys + rules_columns]
            rules_df = rules_df.sort_values(by='attackGroups')
        # extract attackGroups column
        df = pd.json_normalize(data)
        df['ruleset_id'] = df['id']
        original_keys = df.columns.tolist()
        original_keys.remove('attackGroups')
        original_keys.remove('id')
        del df['id']

        '''
        attack_group_df = pd.json_normalize(df['attackGroups'].explode().reset_index(drop=True))
        attack_group_df['order'] = attack_group_df.index + 1
        '''

        df['attackGroups_count'] = df['attackGroups'].apply(lambda x: len(x) if isinstance(x, list) else 0)
        all_keys = dataframe.extract_keys(df['attackGroups'].dropna().sum())
        # col = ['attackGroups', 'attackGroups_count']
        # self.logger.info(df[col])

        if all_keys:
            for key in all_keys:
                df[key] = df['attackGroups'].apply(lambda x: [d.get(key) for d in x] if isinstance(x, list) else [])
            group_columns = list(all_keys)
            exploded_data = dataframe.explode_cell(df, 'attackGroups', group_columns)
            attack_group_df = pd.DataFrame(exploded_data)
            group_columns = ['group', 'groupName', 'threshold']
            original_keys.remove('ruleset_id')
            attack_group_df = attack_group_df[['ruleset_id'] + original_keys + group_columns]
        return rules_df, attack_group_df

if __name__ == '__main__':
    pass


from django.contrib.auth.models import User
from django.shortcuts import get_object_or_404, redirect, render
from django.http  import HttpResponseRedirect,Http404
from . forms import UpdateUserForm, UpdateUserProfileForm, UserRegisterForm,HoodForm,BusinessForm,PostForm
from .models import NeighbourHood,Business,Post,Profile
from django.contrib.auth.decorators import login_required
from django.urls import reverse
from django.forms.models import model_to_dict
from django.contrib import messages

def index(request):
    return render(request,'all-neighbour/home.html')

def register(request):
    if request.user.is_authenticated:
    #redirect user to the profile page
        return redirect('home')
    if request.method=="POST":
        form = UserRegisterForm(request.POST)
        if form.is_valid():
            form.save()
            username = form.cleaned_data.get('username')
            return redirect('login')

    else:
        form = UserRegisterForm()
    return render(request,"registration/register.html",{'form':form})

@login_required(login_url='login')
def hoods(request):
    current_user=request.user
    hoods = NeighbourHood.objects.all()
    return render(request,"all-neighbour/hoods.html",{'hoods':hoods,'current_user':current_user})

@login_required(login_url='login')
def profile(request, username):
    current_user=request.user

    if request.method == 'POST':
        user_form = UpdateUserForm(request.POST, instance=request.user)
        profile_form = UpdateUserProfileForm(request.POST, request.FILES, instance=request.user.profile)
        if user_form.is_valid() and profile_form.is_valid():
            user_form.save()
            profile_form.save()
            return HttpResponseRedirect(request.path_info)
    else:
        user_form = UpdateUserForm(instance=request.user)
        profile_form = UpdateUserProfileForm(instance=request.user.profile)

    return render(request, 'all-neighbour/profile.html', {'user_form':user_form,'profile_form':profile_form})

@login_required(login_url='login')
def new_hood(request):
    if request.method == 'POST':
        form = HoodForm(request.POST, request.FILES)
        if form.is_valid():
            hood = form.save(commit=False)
            hood.admin = request.user
            hood.save()
            return HttpResponseRedirect(reverse("hoods"))
    else:
        form = HoodForm()
    return render(request, 'all-neighbour/newhood.html', {'form': form})

@login_required(login_url='login')
def user_hood(request,id):
    current_user = request.user
    hood = NeighbourHood.objects.get(id=id)
    members = Profile.objects.filter(neighbourhood=hood)
    businesses = Business.objects.filter(neighbourhood=hood)
    posts = Post.objects.filter(neighbourhood=hood)
    request.user.profile.neighbourhood = hood
    request.user.profile.save()

    return render(request, 'all-neighbour/user_hood.html', {'hood': hood,'businesses':businesses,
                                                            'posts':posts,'current_user':current_user,
                                                            'members':members})

@login_required(login_url='login')
def leave_hood(request,id):
    hood = NeighbourHood.objects.get(id=id)
    request.user.profile.neighbourhood = None
    request.user.profile.save()
    return redirect('hoods')

@login_required(login_url='login')
def new_business(request,id):
    hood = NeighbourHood.objects.get(id=id)
    title = 'ADD BUSINESS'
    if request.method=='POST':
        bus_form = BusinessForm(request.POST,request.FILES)
        if bus_form.is_valid():
            business = bus_form.save(commit=False)
            business.neighbourhood = hood
            business.owner = request.user.profile
            business.save()
            return redirect('my_hood', id)
    else:
        bus_form = BusinessForm()
    return render(request,'all-neighbour/business.html',{'bus_form':bus_form,'title':title})

@login_required(login_url='login')
def new_post(request,id):
    hood = NeighbourHood.objects.get(id=id)
    title = 'ADD POST'
    if request.method=='POST':
        post_form = PostForm(request.POST)
        if post_form.is_valid():
            post = post_form.save(commit=False)
            post.neighbourhood = hood
            post.owner = request.user
            post.save()
            return redirect('my_hood', id)
    else:
        post_form = PostForm()
    return render(request,'all-neighbour/post.html',{'post_form':post_form,'title':title})

@login_required(login_url='login')
def update_business(request,id,bus_id):
    instance= Business.objects.get(id=bus_id)
    title = 'UPDATE BUSINESS'
    if request.method=='POST':
        bus_form = BusinessForm(request.POST,request.FILES,instance=instance)
        if bus_form.is_valid():
            bus_form.save()
        return redirect('my_hood', id)
    else:
        # bus_form = BusinessForm(initial=model_to_dict(instance))
        bus_form = BusinessForm(instance=instance)
    return render(request,'all-neighbour/business.html',{'bus_form':bus_form,'title':title})

@login_required(login_url='login')
def update_post(request,id,post_id):
    title = 'UPDATE POST'
    instance= Post.objects.get(id=post_id)
    if request.method=='POST':
        post_form = PostForm(request.POST,instance=instance)
        if post_form.is_valid():
           post_form.save()
        return redirect('my_hood', id)
    else:
        post_form = PostForm(instance=instance)
    return render(request,'all-neighbour/post.html',{'post_form':post_form,'title':title})

@login_required(login_url='login')
def update_hood(request,id):
    title = 'UPDATE HOOD'
    instance= NeighbourHood.objects.get(id=id)
    if request.method=='POST':
        form = HoodForm(request.POST,request.FILES,instance=instance)
        if form.is_valid():
           form.save()
        messages.success(request, ('Hood Updated Successfullly'))
        return redirect('hoods')
    else:
        form = HoodForm(instance=instance)
    return render(request,'all-neighbour/newhood.html',{'form':form,'title':title})

@login_required(login_url='login')
def delete_business(request,id,bus_id):
    business= Business.objects.get(id=bus_id)
    business.delete_business()
    messages.info(request, ('Business Deleted'))
    return redirect('my_hood', id)

@login_required(login_url='login')
def delete_post(request,id,post_id):
    post= Post.objects.get(id=post_id)
    post.delete_post()
    messages.info(request, ('Post Deleted'))
    return redirect('my_hood', id)

@login_required(login_url='login')
def search_hood(request):
    current_user= request.user
    if request.method == 'GET':
        name = request.GET.get("name")
        hoods = NeighbourHood.objects.filter(name__icontains=name).all()

    return render(request, 'all-neighbour/search.html', {'hoods': hoods,'current_user':current_user})

def user_profile(request, username):
    current_user=request.user
    user_poster = get_object_or_404(User, username=username)

    if request.user == user_poster:
        return redirect('profile', username=request.user.username)

    return render(request, 'all-neighbour/member.html', {'user_poster': user_poster,
                                                     'current_user':current_user})


import collections
import re
from src.Validator.EmailValidator import EmailValidator

class GroupByDomainEmailProcessor:
    DOMAIN_REGEX = "^[^@]+@([^@]+\\.[a-zA-Z0-9]{1,4})$"
    __grouped_by_domains = {}

    def __init__(self, email_validator: EmailValidator):
        self.__email_validator = email_validator

    def process(self, email):
        if self.__email_validator.is_valid(email):
            domain = re.search(self.DOMAIN_REGEX, email).group(1)
            if domain not in self.__grouped_by_domains:
                self.__grouped_by_domains[domain] = [email]
            else:
                self.__grouped_by_domains[domain].append(email)

    def get_result(self):
        result = collections.OrderedDict(sorted(self.__grouped_by_domains.items()))
        for i in result:
            result[i] = list(dict.fromkeys(result[i]))
            result[i].sort()
        return result


import logging
import sys

class Formatter(logging.Formatter):

    grey = "\x1b[38;20m"
    yellow = "\x1b[33;20m"
    red = "\x1b[31;20m"
    bold_red = "\x1b[31;1m"
    reset = "\x1b[0m"
    msg = "%(message)s"

    FORMATS = {
        logging.DEBUG: grey + msg + reset,
        logging.INFO: grey + msg + reset,
        logging.WARNING: yellow + msg + reset,
        logging.ERROR: red + msg + reset,
        logging.CRITICAL: bold_red + msg + reset
    }

    def format(self, record):
        log_fmt = self.FORMATS.get(record.levelno)
        formatter = logging.Formatter(log_fmt)
        return formatter.format(record)

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
handler.setFormatter(Formatter())
# Add handler to stdout. Strangely just using basicConfig doesn't work
# May want to change the level to something else
# using an .ini config, or to put FileHandlers.
logger.addHandler(handler)


import pytest

from vyper import compiler
from vyper.exceptions import StructureException

fail_list = [
    """
@external
def foo() -> int128:
    x: int128 = 45
    return x.codesize
    """
]

@pytest.mark.parametrize("bad_code", fail_list)
def test_block_fail(bad_code):
    with pytest.raises(StructureException):
        compiler.compile_code(bad_code)

valid_list = [
    """
@external
def foo() -> uint256:
    x: address = 0x1234567890123456789012345678901234567890
    return x.codesize
    """,
    """
@external
def foo() -> uint256:
    return self.codesize
    """,
    """
struct Foo:
    t: address
foo: Foo

@external
def bar() -> uint256:
    return self.foo.t.codesize
    """,
]

@pytest.mark.parametrize("good_code", valid_list)
def test_block_success(good_code):
    assert compiler.compile_code(good_code) is not None


# coding=utf-8
"""Tests that publish rpm plugin repositories."""
import unittest
from random import choice
from urllib.parse import urljoin

from requests.exceptions import HTTPError

from pulp_smash import api, config
from pulp_smash.pulp3.constants import REPO_PATH
from pulp_smash.pulp3.utils import (
    gen_repo,
    get_content,
    get_content_summary,
    get_versions,
    publish,
    sync,
)

from pulp_rpm.tests.functional.utils import (
    gen_rpm_publisher,
    gen_rpm_remote,
)
from pulp_rpm.tests.functional.constants import (
    RPM_ALT_LAYOUT_FIXTURE_URL,
    RPM_FIXTURE_SUMMARY,
    RPM_LONG_UPDATEINFO_FIXTURE_URL,
    RPM_PACKAGE_CONTENT_NAME,
    RPM_PUBLISHER_PATH,
    RPM_REFERENCES_UPDATEINFO_URL,
    RPM_REMOTE_PATH,
    RPM_RICH_WEAK_FIXTURE_URL,
    RPM_SHA512_FIXTURE_URL,
    SRPM_UNSIGNED_FIXTURE_URL,
)
from pulp_rpm.tests.functional.utils import set_up_module as setUpModule  # noqa:F401

class PublishAnyRepoVersionTestCase(unittest.TestCase):
    """Test whether a particular repository version can be published.

    This test targets the following issues:

    * `Pulp #3324 <https://pulp.plan.io/issues/3324>`_
    * `Pulp Smash #897 <https://github.com/PulpQE/pulp-smash/issues/897>`_
    """

    @classmethod
    def setUpClass(cls):
        """Create class-wide variables."""
        cls.cfg = config.get_config()
        cls.client = api.Client(cls.cfg, api.json_handler)

    def test_all(self):
        """Test whether a particular repository version can be published.

        1. Create a repository with at least 2 repository versions.
        2. Create a publication by supplying the latest ``repository_version``.
        3. Assert that the publication ``repository_version`` attribute points
           to the latest repository version.
        4. Create a publication by supplying the non-latest
           ``repository_version``.
        5. Assert that the publication ``repository_version`` attribute points
           to the supplied repository version.
        6. Assert that an exception is raised when providing two different
           repository versions to be published at same time.
        """
        body = gen_rpm_remote()
        remote = self.client.post(RPM_REMOTE_PATH, body)
        self.addCleanup(self.client.delete, remote['_href'])

        repo = self.client.post(REPO_PATH, gen_repo())
        self.addCleanup(self.client.delete, repo['_href'])

        sync(self.cfg, remote, repo)

        publisher = self.client.post(RPM_PUBLISHER_PATH, gen_rpm_publisher())
        self.addCleanup(self.client.delete, publisher['_href'])

        # Step 1
        repo = self.client.get(repo['_href'])
        for rpm_content in get_content(repo)[RPM_PACKAGE_CONTENT_NAME]:
            self.client.post(
                repo['_versions_href'],
                {'add_content_units': [rpm_content['_href']]}
            )
        version_hrefs = tuple(ver['_href'] for ver in get_versions(repo))
        non_latest = choice(version_hrefs[:-1])

        # Step 2
        publication = publish(self.cfg, publisher, repo)

        # Step 3
        self.assertEqual(publication['repository_version'], version_hrefs[-1])

        # Step 4
        publication = publish(self.cfg, publisher, repo, non_latest)

        # Step 5
        self.assertEqual(publication['repository_version'], non_latest)

        # Step 6
        with self.assertRaises(HTTPError):
            body = {
                'repository': repo['_href'],
                'repository_version': non_latest
            }
            self.client.post(urljoin(publisher['_href'], 'publish/'), body)

class SyncPublishReferencesUpdateTestCase(unittest.TestCase):
    """Sync/publish a repo that ``updateinfo.xml`` contains references."""

    @classmethod
    def setUpClass(cls):
        """Create class-wide variables."""
        cls.cfg = config.get_config()
        cls.client = api.Client(cls.cfg, api.json_handler)

    def test_all(self):
        """Sync/publish a repo that ``updateinfo.xml`` contains references.

        This test targets the following issue:

        `Pulp #3998 <https://pulp.plan.io/issues/3998>`_.
        """
        repo = self.client.post(REPO_PATH, gen_repo())
        self.addCleanup(self.client.delete, repo['_href'])

        body = gen_rpm_remote(url=RPM_REFERENCES_UPDATEINFO_URL)
        remote = self.client.post(RPM_REMOTE_PATH, body)
        self.addCleanup(self.client.delete, remote['_href'])

        sync(self.cfg, remote, repo)
        repo = self.client.get(repo['_href'])

        self.assertIsNotNone(repo['_latest_version_href'])

        content_summary = get_content_summary(repo)
        self.assertDictEqual(
            content_summary,
            RPM_FIXTURE_SUMMARY,
            content_summary
        )

        publisher = self.client.post(RPM_PUBLISHER_PATH, gen_rpm_publisher())
        self.addCleanup(self.client.delete, publisher['_href'])

        publication = publish(self.cfg, publisher, repo)
        self.addCleanup(self.client.delete, publication['_href'])

class SyncPublishTestCase(unittest.TestCase):
    """Test sync and publish for different RPM repositories.

    This test targets the following issue:

    `Pulp #4108 <https://pulp.plan.io/issues/4108>`_.
    `Pulp #4134 <https://pulp.plan.io/issues/4134>`_.
    """

    @classmethod
    def setUpClass(cls):
        """Create class-wide variables."""
        cls.cfg = config.get_config()
        cls.client = api.Client(cls.cfg, api.page_handler)

    def test_rpm_rich_weak(self):
        """Sync and publish an RPM repository. See :meth: `do_test`."""
        self.do_test(RPM_RICH_WEAK_FIXTURE_URL)

    def test_rpm_long_updateinfo(self):
        """Sync and publish an RPM repository. See :meth: `do_test`."""
        self.do_test(RPM_LONG_UPDATEINFO_FIXTURE_URL)

    def test_rpm_alt_layout(self):
        """Sync and publish an RPM repository. See :meth: `do_test`."""
        self.do_test(RPM_ALT_LAYOUT_FIXTURE_URL)

    def test_rpm_sha512(self):
        """Sync and publish an RPM repository. See :meth: `do_test`."""
        self.do_test(RPM_SHA512_FIXTURE_URL)

    def test_srpm(self):
        """Sync and publish a SRPM repository. See :meth: `do_test`."""
        self.do_test(SRPM_UNSIGNED_FIXTURE_URL)

    def do_test(self, url):
        """Sync and publish an RPM repository given a feed URL."""
        repo = self.client.post(REPO_PATH, gen_repo())
        self.addCleanup(self.client.delete, repo['_href'])

        remote = self.client.post(RPM_REMOTE_PATH, gen_rpm_remote(url=url))
        self.addCleanup(self.client.delete, remote['_href'])

        sync(self.cfg, remote, repo)
        repo = self.client.get(repo['_href'])

        self.assertIsNotNone(repo['_latest_version_href'])

        publisher = self.client.post(RPM_PUBLISHER_PATH, gen_rpm_publisher())
        self.addCleanup(self.client.delete, publisher['_href'])

        publication = publish(self.cfg, publisher, repo)
        self.addCleanup(self.client.delete, publication['_href'])


import numpy as np
from scipy.linalg  import lu, lu_factor, lu_solve   # Biblioteca com funções para resolução LU
from time import process_time

# Imprime as saídas
PRINT = False

# Imprime a Matriz formatada
def printMatrix(A, b):
    y = 0
    for i in A:
        print("|", end='')
        for j in i:
            # Mostra os valores apena com 2 casas decimais
            v = "{:.2f}".format(j)
            print(f'{v:^8}', end="")
        # Mostra os valores apena com 2 casas decimais
        v = "{:.2f}".format(b[y])
        print("|", f'{v:^7}', end="|\n")
        y += 1

# Registra o tempo de inicio da execução
t1_start = process_time()

# Array usado na validação
A = np.array([ [11, -6,  -2,  0,  0,  0,  0],
            [-1,  3,   0, -2,  0,  0,  0],
            [-4,  0,   9, -2, -3,  0,  0],
            [ 0, -6,  -1, 10,  0, -3,  0],
            [ 0,  0,  -5,  0, 19,-10,  0],
            [ 0,  0,   0, -6, -3, 13, -4],
            [ 0,  0,   0,  0,  0, -5,  8]], float)

b = np.array([60, 0, 0, 0, 0, 0, 0], float )

# Imprime as matrizes
if PRINT:
    printMatrix(A,b)

# Função que decompõem a Matriz; P Matriz permutada, L triangular Inferior e U triangular superior
# P, L, U = lu(A)

# LU contém as matrizes Triangular U e L; piv contém a matriz de permutação
LU, piv = lu_factor(A)

# Resolve a matriz usando a fatoração LU
R = lu_solve((LU, piv), b)

# Imprime a resposta
print(R)

# Testa a solução com base na resposta encontrada em R
#print(np.allclose(np.dot(A, R), b))
#print(np.allclose(A @ R - b, np.zeros((len(A),))))

# Registra o tempo de final da execução
t1_stop = process_time()
print("Tempo decorrido do processamento [em segundos]:",
                                         t1_stop-t1_start)


from django import forms
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse

# python manage.py migrate --> allows tables to be created
#tasks = []

class NewTaskForm(forms.Form):
    task = forms.CharField(label="New Task")
    #priority = forms.IntegerField(label="Priority", min_value=1, max_value=10)

# Create your views here.
def index(request):
    if "tasks" not in request.session:  # is there any "tasks" inside of the session
        request.session["tasks"] = []   # if not, give a tasks

    return render(request, "tasks/index.html", {
        "tasks": request.session["tasks"]
    })

def add(request):
    if request.method == "POST":
        form = NewTaskForm(request.POST) # user's submitted data
        if form.is_valid():
            task = form.cleaned_data["task"] #all data user submitted
            request.session["tasks"] += [task] #adds the task comes from user post to the session's tasks
            return HttpResponseRedirect(reverse("tasks:index")) # reverse -> finds url of tasks:index
        else:
            return render(request, "tasks/add.html", {
                "form": form
            })
    return render(request, "tasks/add.html", {
        "form": NewTaskForm()
    })


from util.server_util import install_heroku_cli
import subprocess
import json

import  sqlalchemy as db

def get_database_url(task_name):
    try:
        with open("../appname.json", "r") as f:
            loaded_dict = json.loads(f.read())
            if task_name not in loaded_dict:
                raise RuntimeError("Uh oh. This task name does not exist!")
            app_name = loaded_dict[task_name]
            print(f"Reading database for {app_name}")
        heroku_executable_path = install_heroku_cli()
        result = subprocess.check_output([f'{heroku_executable_path}', 'config:get', 'DATABASE_URL',  '-a',  f'{app_name}'])
        return result.decode('ascii').strip()
    except:
        raise RuntimeError('"appname.json" in ../app does not exist! Please run the task first via launch_hits.py!')

def query_raw_data(task_name):
    engine = db.create_engine(get_database_url(task_name), echo=False)
    connection = engine.connect()
    metadata = db.MetaData()
    tasks = db.Table('task', metadata, autoload=True, autoload_with=engine)
    query = db.select([tasks])
    ResultProxy = connection.execute(query)
    ResultSet = ResultProxy.fetchall()
    return ResultSet

def load_data(task_name):
    results = []
    raw_data = query_raw_data(task_name)
    for i in raw_data:
        d = json.loads(i[2])

        for key in ['messages', 'text', 'compare_bot_a', 'compare_bot_b', 'compare_bot_preference', 'pipeline']:
            try:
                del d[key]
            except:
                pass

        results.append(d)
    return results


import datetime

from django.http import HttpResponse
from django.shortcuts import render

# Create your views here.
from django.views import View

from .models import Evento, Pais
from .forms import pais_form, evento_form

from django.views.decorators.http import require_http_methods

@require_http_methods(["GET"])
def home_view(request):
    eventos = Evento.objects.all()
    return render(request, 'home.html', {'eventos': eventos})

@require_http_methods(["GET", "POST"])
def pais_view(request):
    if request.method == 'GET':
        form = pais_form()
    else:
        # create a form instance and populate it with data from the request:
        form = pais_form(request.POST)
        if form.is_valid():
            # process the data in form.cleaned_data as required
            obj = Pais(**form.cleaned_data)
            obj.save()
            # redirect to a new URL:
            return HttpResponse('PAIS AGREGADO CON EXITO/')

    return render(request, 'pais.html', {'form':form})

@require_http_methods(["GET", "POST"])
def evento_view(request):
    if request.method == 'GET':
        form = evento_form()
    else:
        # create a form instance and populate it with data from the request:
        form = evento_form(request.POST)
        if form.is_valid():
            obj = Evento(**form.cleaned_data)
            obj.save()
            return HttpResponse('EVENTO AGREGADO CON EXITO/')

    return render(request, 'evento.html', {'form': form})


from django.conf.urls import patterns, include, url
from django.contrib import admin
from records.views import index, set_language, CreateRecordView, UpdateRecordView,\
         DeleteRecordView, UpdateInitialBalanceView, CreateInitialBalanceView,\
         CreateUnscheduledDebtView, CreateUnscheduledCreditView, UpdateUnscheduledDebtView,\
         UpdateUnscheduledCreditView, CreateRecurrentMonthView, EditRecurrentMonthView
from profiles.views import set_timezone, UpdateProfileView
from django.core.urlresolvers import reverse_lazy

urlpatterns = patterns('',
    url(r'^admin/', include(admin.site.urls)),
    # User general
    url(r'^logout/$', 'django.contrib.auth.views.logout',
         name="logout"),
    url(r'^i18n/$', set_language, name='set_language'),
    url(r'^tz/$', set_language, name='set_language'),
    url(r'^profile/update/(?P<pk>\d+)/$', UpdateProfileView.as_view(), name='update_profile'),

    # Records
    url(r'^record/create/(?P<type>\d+)/$', CreateRecordView.as_view(), name='create_record'),
    url(r'^record/update/(?P<pk>\d+)/$', UpdateRecordView.as_view(), name='update_record'),
    url(r'^record/delete/(?P<pk>\d+)/$', DeleteRecordView.as_view(), name='delete_record'),
    url(r'^record/recurrent/create/(?P<parent_pk>\d+)/(?P<month>\d+)/(?P<year>\d+)/$', CreateRecurrentMonthView.as_view(), name='create_recurrent_month'),
    url(r'^record/recurrent/edit/(?P<pk>\d+)/$', EditRecurrentMonthView.as_view(), name='edit_recurrent_month'),

    # System accounts
    url(r'^balance/create/$', CreateInitialBalanceView.as_view(), name='create_initial_balance'),
    url(r'^balance/update/(?P<pk>\d+)/$', UpdateInitialBalanceView.as_view(), name='update_initial_balance'),
    url(r'^u_debt/create/$', CreateUnscheduledDebtView.as_view(), name='create_unscheduled_debt'),
    url(r'^u_debt/update/(?P<pk>\d+)/$', UpdateUnscheduledDebtView.as_view(), name='update_unscheduled_debt'),
    url(r'^u_credit/create/$', CreateUnscheduledCreditView.as_view(), name='create_unscheduled_credit'),
    url(r'^u_credit/update/(?P<pk>\d+)/$', UpdateUnscheduledCreditView.as_view(), name='update_unscheduled_credit'),

    #Dashboard
    url(r'^$', index, name='dashboard'),
)


#Check
import bpy
import os
bpy.ops.object.select_all(action='SELECT')
bpy.ops.object.delete()
val = input("Enter the name of the python file(Make sure it is in .obj format): ")
bpy.ops.import_scene.obj(filepath=val + ".obj")
for obj in bpy.context.scene.objects:
    if(obj.type == 'MESH'):
        mesh = obj.data
        objectToSelect = bpy.data.objects[mesh.name]
        objectToSelect.select_set(True)
        bpy.context.view_layer.objects.active = objectToSelect
        bpy.context.object.data.remesh_voxel_size = 0.01
        bpy.ops.object.voxel_remesh()
        bpy.ops.object.editmode_toggle()
        bpy.ops.mesh.select_all(action='SELECT')
        bpy.ops.mesh.mark_sharp(clear=True)
        bpy.ops.object.editmode_toggle()
        bpy.ops.object.shade_smooth()
        bpy.context.object.data.use_auto_smooth = False
        val = input("Enter the name of the output file: ")
        bpy.ops.export_scene.obj(filepath = val + ".obj")