import streamlit as st
import pandas as pd

st.set_page_config(layout="wide")

col_1, col_2 = st.columns(2)

with col_1:
    st.image("profile_img/Mittens.png")

with col_2:
    st.title("Rommel Rudolf A. Dela Merced")
    content = """
        Python Developer
    """
    st.write(content)

    bio = """
        A Graduate of B.S. in Computer Science in Arellano University Andres Bonifacio Campus\n
        Software Engineer at Tsukiden Global Solutions Inc.\n
        Python, Javascript, Rust\n
    """
    st.info(bio)

st.subheader('Projects: ')

col3, empty_col, col4 = st.columns([1.5, 0.5, 1.5])

# Create dataframe from csv file
projects_df = pd.read_csv('projects/projects.csv', sep=',')

st.image('images/1.png', caption='TO-DO List WebApp')
st.markdown("***")
st.image('images/2.png', caption='TO-DO List GUI')
st.markdown("***")
st.image('images/3.png', caption='Projects Showcase WebApp')
st.markdown("***")
st.image('images/4.png', caption='Weather Forecast WebApp')
st.markdown("***")
st.image('images/5.png', caption='GUI Calculator')
st.markdown("***")

# Add to column 1, 1-10 project title column
with col3:
    for i, row in projects_df[:10].iterrows():
        st.header(row['title'])
        st.write(row['description'])
        st.write(f'[Source Code]({row["url"]})')
        st.markdown("***")

# Add to column 2, 11-20 project title column
with col4:
    for i, row in projects_df[10:].iterrows():
        st.header(row['title'])
        st.write(row['description'])
        st.write(f'[Source Code]({row["url"]})')
        st.markdown("***")

st.markdown("***")


import requests
import bs4

# req to get a response
res = requests.get("https://en.wikipedia.org/wiki/Python_(programming_language)")

# grab the source code
soup = bs4.BeautifulSoup(res.text, "lxml")

# link = requests.get('https://upload.wikimedia.org/wikipedia/commons/thumb/c/c3/Python-logo-notext.svg/121px-Python-logo-notext.svg.png')
# print(link.content)

pythonImgTag = soup.select('.image img')[0] # grab the image
pythonImg = requests.get('https:' + pythonImgTag['src']) # req to the image url
print(pythonImg.content) # binary text of the image

# save image into filesystem
f = open('./img/python.png', 'wb') # write binary
f.write(pythonImg.content)
f.close()


from vantage6.client import Client
import os
import numpy as np
import json
from io import BytesIO
import datetime
import psycopg2
from utils import init_global_params, average, define_model, get_results
import time
import matplotlib.pyplot as plt
dir_path = os.path.dirname(os.path.realpath(__file__))
import pickle
from tqdm import tqdm

print("Attempt login to Vantage6 API")
# client = Client("https://vantage6-server.researchlumc.nl", 443, "/api")
# client.authenticate("sgarst", "password")
# client.setup_encryption(None)

client = Client("http://localhost", 5000, "/api")
client.authenticate("researcher", "password")
client.setup_encryption(None)
ids = [org['id'] for org in client.collaboration.get(1)['organizations']]

#ID mapping:
# 2 - Leiden
# 3 - Rotterdam
# 4 - Maastricht

#ids = [org['id'] for org in client.collaboration.get(1)['organizations']]
ids = [2,3]

## Parameter settings ##

n_runs = 1 # amount of runs
n_rounds = 2 # communication rounds between centers
lr = 0.000005 # learning rate
model = "M4" # model selection (see analysis plan)
n_bins = 10
write_file = True
use_dm = False
n_clients = len(ids)
seed_offset = 0

#all_cols =  ["id", "metabo_age", "brain_age","date_metabolomics", "date_mri","birth_year", "sex", "bmi", "dm", "education_category_3" ]
#all_cols = [None]
image_name = "sgarst/association-analysis:1.5.3"
## init data structures ##

betas = np.zeros((n_runs, n_rounds, n_clients))
mses = np.zeros_like(betas)
maes = np.zeros_like(betas)
data_cols, extra_cols = define_model(model, use_dm = use_dm)

for run in range(n_runs):

    param_seed = run + seed_offset
    tt_split_seed = run + seed_offset
    # federated iterative process
    global_coefs, global_intercepts = init_global_params(data_cols, extra_cols, param_seed = param_seed)

    for round in tqdm(range(n_rounds)):
        #print(global_coefs, global_intercepts)
        #print("posting fit_round task to ids " + str(ids))
        task = client.post_task(
            input_ = {
                'method' : 'fit_round',
                'kwargs' : {
                    'coefs' : global_coefs,
                    "intercepts" :  global_intercepts,
                    "data_cols" : data_cols,
                    "extra_cols" : extra_cols,
                    #"all_cols": all_cols,
                    "lr" : lr,
                    "seed": tt_split_seed,
                    }
                },
            name = "Analysis fit regressor, round" + str(round),
            image = image_name,
            organization_ids=ids,
            collaboration_id=1
        )
        finished = False
        #local_coefs = np.empty((n_rounds, 3), dtype=object)
        #local_intercepts = np.empty((n_rounds, 3), dtype=object)
        dataset_sizes = np.empty(n_clients, dtype = object)
        local_coefs = np.empty((len(global_coefs), n_clients))
        local_intercepts = np.empty((len(global_intercepts),n_clients))
        local_cols = np.empty((n_clients), dtype = object)

        results = get_results(client, task, print_log = True)

        if psycopg2.Error in results:
            print("query error: ", results)
            break
        else:
            for i in range(n_clients):
                local_coefs[:,i], local_intercepts[:, i] = results[i]['param']
                dataset_sizes[i] = results[i]['size']
                mses[run, round, i] = results[i]['mse']
                maes[run, round, i] = results[i]["mae"]
                local_cols[i] = results[i]["data_cols"]
                #print(local_cols[i])
        #print(local_coefs, local_intercepts)
        global_coefs = average(local_coefs, dataset_sizes)
        global_intercepts = average(local_intercepts, dataset_sizes)
        #print(global_coefs, global_intercepts)
        #betas[run, round, 0] = np.copy(global_coefs)
#print(losses)

    print('finished model fitting, calculating standard error..')
    avg_task = client.post_task(
        input_= {
            "method" : "get_avg",
            "kwargs" : {     }
        },
        name= "get average",
        image = image_name,
        organization_ids= ids,
        collaboration_id=1
    )
    avg_results = get_results(client, avg_task, print_log=False)

    means = np.array([result['mean'] for result in avg_results])
    sizes = np.array([result['size'] for result in avg_results])

    global_mean = np.sum([(mean * size) for mean, size in zip(means, sizes)]) / np.sum(sizes)

    #print(means, global_mean)

    se_task = client.post_task(
        input_ = {
            "method" : "calc_se",
            "kwargs" : {
                "global_mean" : global_mean,
                "global_coefs" : global_coefs,
                "global_inter" : global_intercepts,
                "data_cols" : data_cols,
                #"all_cols" : all_cols,
                "extra_cols" : extra_cols,
            }
        },
            name ="se calculation",
            image = image_name,
            organization_ids=ids,
            collaboration_id=1
    )

    se_results = get_results(client, se_task, print_log = False)
    tops = [np.sum(result['top']) for result in se_results]
    bots = [np.sum(result['bot']) for result in se_results]
    sizes = [result['size'] for result in se_results]

    top_sum = np.sum(tops)
    bot_sum = np.sum(bots)
    full_size = np.sum(sizes)

    se = np.sqrt((1/(full_size - 2)) * (float(top_sum) / float(bot_sum)))

    #print(f'standard error: {se}')

branges = [result['resplot']['ranges'].tolist() for result in results]

print("finished! writing to file")
print(mses)
# write output to json file
final_results = {
    "lr" : lr,
    "nruns" : n_runs,
    "nrounds" : n_rounds,
    "model" : model,
    "betas" : global_coefs.tolist(),
    "cols" : local_cols[0],
    "intercept" : global_intercepts.tolist(),
    "mse" : mses.tolist(),
    "mae" : maes.tolist(),
    "bin_ranges" : branges,
    "standard_error" : se
}

if write_file:
    date = datetime.datetime.now()
    file_str = f'analysis_model_{model}_centers_{ids}_{date.strftime("%d")}_{date.strftime("%m")}_{date.strftime("%Y")}'
    jo = json.dumps(final_results)

    with open(file_str + ".json", "w", encoding="utf8") as f:
        #json.dump(jo, f)
        f.write(jo)

    for c in range(len(ids)):

        fig1 = results[c]['resplot']['fig']
        ax1 = results[c]['resplot']['ax']
        bp1 = results[c]['resplot']['bp']
#print(len(bp1['boxes']))
        n_boxplots = len(bp1['boxes'])
        ax1.set_xticks(np.arange(1, len(branges) + 1), [str(brange) for brange in branges])

        with open("resplot_" + file_str + "center_" + str(c) +  ".pickle", "wb") as f:
            pickle.dump(ax1, f)


""" Here one focuse more on feature engineering with realistic data set

I will be using data from a Kaggle data can be found:

https://www.kaggle.com/harlfoxem/housesalesprediction

"""
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

pd.set_option('display.max_columns',50)

house = pd.read_csv('kc_house_data.csv')
# first let us see if we have any missing data in each columns
# Looking for nulls
house.isnull().any()
house.isnull().sum()
# let us see the features name
house.columns

house.describe().transpose()

#%%
with sns.plotting_context("notebook",font_scale=2.5):
    g = sns.pairplot(house[['sqft_lot','sqft_above','price','sqft_living','bedrooms']],
                 hue='bedrooms', palette='tab20',size=1.4)
g.set(xticklabels=[]);

plt.subplots(figsize=(17,14))
sns.heatmap(house.corr(),annot=True,linewidths=0.5,linecolor="Black",fmt="1.1f")
plt.title("Data Correlation",fontsize=50)
plt.show()

#%%
# in order to have a better idea let us plot the price and see its distribution
plt.figure(figsize=(10,5))
sns.distplot(house['price'])

# as we can see the are some extream points (expensive houses which are not that many)
# so it is better to skip them when we build the model

sns.pairplot(house[['sqft_above','price','sqft_living','bedrooms']],hue='bedrooms', palette='husl');

sns.countplot(x= house['bedrooms'])
# we can see it is extended u tp 33 but it is a small bar and not visible
sns.pairplot(house, hue='price')
# now let us see the correlation of our target with other value
house.corr()['price'].sort_values()
# we see "sqft_living" has a strong correlation with price

sns.scatterplot(x= house['sqft_living'], y = house['price'])
sns.scatterplot(x=house['grade'], y = house['price'])
sns.scatterplot(x=house['bedrooms'], y = house['price'])

# in our data we have "lattitude" and "longitude" which give the house position in
# king country, USA
plt.figure(figsize=(5,8))
sns.scatterplot(x= house['long'],y =house['lat'], hue = house['price'])

# because of those extream points we are not getting a good
# color distribution. so now let us drop those extreame points

house2 = house.sort_values('price',ascending=False)
len(house)*(0.01)  #216
non_top_1_perc = house2[216:]

#non_top_1_perc = house.sort_values('price',ascending=False).iloc[216:]

# now let us plot it again, to get more color distribution

plt.figure(figsize=(12,8))
sns.scatterplot(x='long',y='lat',
                data=non_top_1_perc,hue='price',
                palette='RdYlGn',edgecolor=None,alpha=0.2)

#%%% 138
"""Feature engineering """
# now we can do some feature engineerin from data
# let us drop some featuers which is not very informatic in given data set
# we look at data again
house.head(5)
house = house.drop('id', axis=1)

house['date']
# next if we look at the date it shows some sort of the string "" dtype: object"".
# So we need to convert it by doing the following, we can use "to_datetime" function from Pandas

house['date'] = pd.to_datetime(house['date'])
# now the type of date is ""dtype: datetime64[ns]""

# this is called feature engineering, because these features  are hidden inside of
# string date. now we try to exteact or engineering more information off the original data

house['year'] = house['date'].apply(lambda date:date.year)
house['month'] = house['date'].apply(lambda date:date.month)

# we can do some explatory visualization to see the impact of month on house selling

sns.boxplot(x='year',y='price',data=house)

house.groupby('month').mean()['price'].plot()

house = house.drop('date', axis = 1)

# now if we look at the zipcode column we see we have 70 uniqu values
# which is basically too much to catagorize the data
house['zipcode'].value_counts()

# So I go ahead and for this particular example I drop this colomn

house = house.drop('zipcode',axis=1)
# could make sense due to scaling, higher should correlate to more value

# other thing er can d is looking at continoues or categorical data
house['yr_renovated'].value_counts()
house['sqft_basement'].value_counts()

#%%139
"""Scaling and Train Test Split"""

X = house.drop('price',axis=1).values
y = house['price'].values

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=101)

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
# we can save time by fitting ans transfering at the same time
X_train= scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
X_train.shape

""" Creating a Model"""
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.optimizers import Adam

model = Sequential()
# typcally what we do is we try to base the number of neurons
# in our layer from the size of actual feature data
model.add(Dense(19,activation='relu'))
model.add(Dense(19,activation='relu'))
model.add(Dense(19,activation='relu'))
model.add(Dense(19,activation='relu'))
model.add(Dense(1))

model.compile(optimizer='adam',loss='mse')

"""Training the Model"""
model.fit(x=X_train,y=y_train,
          validation_data=(X_test,y_test),
          batch_size=128,epochs=400)

#140
losses = pd.DataFrame(model.history.history)
# this data frame has two columns, one is loss and the other is called
# val_loss ---> this is loss on that test set
# that validation data and now I can directly compare the loss on training
# and loss on test data in order to see if i am overfitting to the training
# data on my model. Simply we can plot it
losses.plot()

#%%% we can do some evaluationon our test data 140
"""Evaluation on Test Data"""
from sklearn.metrics import mean_squared_error,mean_absolute_error,explained_variance_score

predictions = model.predict(X_test)

mean_absolute_error(y_test,predictions)
house['price'].mean()

explained_variance_score(y_test,predictions)

# Our predictions
plt.scatter(y_test,predictions)
# Perfect predictions
plt.plot(y_test,y_test,'r')


import json
from unittest.mock import AsyncMock

from httpx import AsyncClient

from saleor_app.deps import SALEOR_DOMAIN_HEADER
from saleor_app.schemas.handlers import SaleorEventType
from saleor_app.schemas.manifest import Manifest

async def test_manifest(saleor_app):
    base_url = "http://test_app.saleor.local"

    async with AsyncClient(app=saleor_app, base_url=base_url) as ac:
        response = await ac.get("configuration/manifest")

    manifest = saleor_app.manifest.dict(by_alias=True)
    manifest["appUrl"] = f"{base_url}/configuration"
    manifest["tokenTargetUrl"] = f"{base_url}/configuration/install"
    manifest["configurationUrl"] = None
    manifest["extensions"][0]["url"] = "/extension"

    manifest = json.loads(json.dumps(Manifest(**manifest).dict(by_alias=True)))

    assert response.status_code == 200
    assert response.json() == manifest

async def test_install(saleor_app_with_webhooks, get_webhook_details, monkeypatch):
    install_app_mock = AsyncMock()
    monkeypatch.setattr("saleor_app.endpoints.install_app", install_app_mock)
    base_url = "http://test_app.saleor.local"

    saleor_app_with_webhooks.validate_domain = AsyncMock(return_value=True)

    async with AsyncClient(app=saleor_app_with_webhooks, base_url=base_url) as ac:
        response = await ac.post(
            "configuration/install",
            json={"auth_token": "saleor-app-token"},
            headers={SALEOR_DOMAIN_HEADER: "example.com"},
        )

    assert response.status_code == 200

    install_app_mock.assert_awaited_once_with(
        saleor_domain="example.com",
        auth_token="saleor-app-token",
        manifest=saleor_app_with_webhooks.manifest,
        events={
            "awssqs://username:password@localstack:4566/account_id/order_created": [
                (SaleorEventType.ORDER_CREATED, None),
            ],
            "awssqs://username:password@localstack:4566/account_id/order_updated": [
                (SaleorEventType.ORDER_UPDATED, None),
            ],
            "http://test_app.saleor.local/webhook": [
                (SaleorEventType.PRODUCT_CREATED, None),
                (SaleorEventType.PRODUCT_UPDATED, None),
                (SaleorEventType.PRODUCT_DELETED, None),
            ],
        },
        use_insecure_saleor_http=False,
    )


import os.path
from selenium import webdriver

CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))

def compare_likes(urls, log=False):
    options = webdriver.ChromeOptions()
    options.add_argument('headless')
    options.add_argument('window-size=800x500')
    options.add_argument("disable-gpu")

    driver_path = os.path.join(CURRENT_DIR, './chromedriver')
    driver = webdriver.Chrome(driver_path, chrome_options=options)

    likes = []
    for url in urls:
        driver.get(url)
        driver.implicitly_wait(10)
        temp = driver.find_element_by_id('PagesLikesCountDOMID').text
        if log:
            print('[*] ', f'"{url}"', temp)
        likes.append(temp)

    return ' VS '.join(likes)

if __name__ == '__main__':
    LINKS = [
        'https://www.facebook.com/CIVAR18',
        'https://www.facebook.com/UNIverse-CONquest-316272942366152'
    ]
    print(compare_likes(LINKS, log=True))


import gzip
import json
from enum import Enum
from typing import Optional, Dict, Union, Iterator, Set, List

from problog.logic import Clause as ProblogClause, Term as ProblogTerm

from pylo.language.lp import Clause as PyloClause, Literal as PyloLiteral
from pylo.language.lp import global_context as pylo_global_context

import pandas as pd

from kbc_pul.amie.amie_rule_string_to_problog_clause_conversion import DatalogAMIERuleConvertor
from kbc_pul.prolog_utils.problog_to_pylo_conversion import convert_clause

from kbc_pul.confidence_naming import ConfidenceEnum

class AmieOutputKeyEnum(Enum):
    RULE = "Rule"
    HEAD_COVERAGE = 'Head Coverage'  # rule support / head relation size in observed KB
    STD_CONF = 'Std Confidence'
    PCA_CONF = 'PCA Confidence'
    N_POSITIVE_EXAMPLES = 'Positive Examples'  # rule support, i.e. nb of predictions supported by the rule
    BODY_SIZE = 'Body size'
    PCA_BODY_SIZE = 'PCA Body size'
    FUNCTIONAL_VARIABLE = 'Functional variable'

columns_header_without_amie = ["Rule", 'Nb supported predictions', 'Body size'] + [
    conf.get_name()
    for conf in ConfidenceEnum
]

class RuleWrapper:
    def __init__(self, rule: PyloClause, o_amie_dict: Optional[Dict] = None):
        """

        :param rule:
        :param o_amie_dict:
        """
        self.rule: PyloClause = rule

        self.amie_dict: Optional[Dict] = o_amie_dict

        # BODY support
        # # predictions
        self.o_n_predictions: Optional[int] = None  # a.k.a. the body support
        # RULE support
        # # predictions in the observed KB
        self.o_n_supported_predictions: Optional[int] = None  # a.k.a the 'Positive predictions' from AMIE

        self.o_std_confidence: Optional[float] = None
        self.o_pca_confidence_subject_to_object: Optional[float] = None
        self.o_pca_confidence_object_to_subject: Optional[float] = None

        self.o_c_weighted_std_conf: Optional[float] = None

        self.o_relative_pu_confidence_unbiased: Optional[float] = None

        self.o_relative_pu_confidence_pca_subject_to_object: Optional[float] = None
        self.o_relative_pu_confidence_pca_object_to_subject: Optional[float] = None

        # self.o_absolute_pu_confidence: Optional[float] = None

        self.o_true_confidence: Optional[float] = None
        self.o_true_pca_confidence_subject_to_object: Optional[float] = None
        self.o_true_pca_confidence_object_to_subject: Optional[float] = None

    def get_value(self, conf_name: ConfidenceEnum):

        if conf_name is ConfidenceEnum.CWA_CONF:
            return self.o_std_confidence
        elif conf_name is ConfidenceEnum.ICW_CONF:
            return self.o_c_weighted_std_conf

        elif conf_name is ConfidenceEnum.PCA_CONF_S_TO_O:
            return self.o_pca_confidence_subject_to_object
        elif conf_name is ConfidenceEnum.PCA_CONF_O_TO_S:
            return self.o_pca_confidence_object_to_subject

        elif conf_name is ConfidenceEnum.IPW_CONF:
            return self.o_relative_pu_confidence_unbiased
        elif conf_name is ConfidenceEnum.IPW_PCA_CONF_S_TO_O:
            return self.o_relative_pu_confidence_pca_subject_to_object
        elif conf_name is ConfidenceEnum.IPW_PCA_CONF_O_TO_S:
            return self.o_relative_pu_confidence_pca_object_to_subject

        elif conf_name is ConfidenceEnum.TRUE_CONF:
            return self.o_true_confidence
        elif conf_name is ConfidenceEnum.TRUE_CONF_BIAS_YS_ZERO_S_TO_O:
            return self.o_true_pca_confidence_subject_to_object
        elif conf_name is ConfidenceEnum.TRUE_CONF_BIAS_YS_ZERO_O_TO_S:
            return self.o_true_pca_confidence_object_to_subject

        else:
            raise Exception(f"Could not find RuleWrapper instance variable corresponding to {conf_name}")

    @staticmethod
    def create_rule_wrapper(
            rule: PyloClause,
            amie_dict: Optional[Dict],
            o_n_predictions: Optional[int],
            o_n_supported_predictions: Optional[int],
            o_std_confidence: Optional[float],

            o_pca_confidence_subject_to_object: Optional[float],
            o_pca_confidence_object_to_subject: Optional[float],

            o_c_weighted_std_conf: Optional[float],

            o_relative_pu_confidence_unbiased: Optional[float],

            o_relative_pu_confidence_pca_subject_to_object: Optional[float],
            o_relative_pu_confidence_pca_object_to_subject: Optional[float],

            o_true_confidence: Optional[float],
            o_true_pca_confidence_subject_to_object: Optional[float],
            o_true_pca_confidence_object_to_subject: Optional[float],

    ) -> 'RuleWrapper':
        new_rule_wrapper: RuleWrapper = RuleWrapper(rule=rule, o_amie_dict=amie_dict)
        new_rule_wrapper.o_n_predictions = o_n_predictions
        new_rule_wrapper.o_n_supported_predictions = o_n_supported_predictions
        new_rule_wrapper.o_std_confidence = o_std_confidence
        new_rule_wrapper.o_pca_confidence_subject_to_object = o_pca_confidence_subject_to_object
        new_rule_wrapper.o_pca_confidence_object_to_subject = o_pca_confidence_object_to_subject

        new_rule_wrapper.o_c_weighted_std_conf = o_c_weighted_std_conf

        new_rule_wrapper.o_relative_pu_confidence_unbiased = o_relative_pu_confidence_unbiased
        new_rule_wrapper.o_relative_pu_confidence_pca_subject_to_object = o_relative_pu_confidence_pca_subject_to_object
        new_rule_wrapper.o_relative_pu_confidence_pca_object_to_subject = o_relative_pu_confidence_pca_object_to_subject

        new_rule_wrapper.o_true_confidence = o_true_confidence
        new_rule_wrapper.o_true_pca_confidence_subject_to_object = o_true_pca_confidence_subject_to_object
        new_rule_wrapper.o_true_pca_confidence_object_to_subject = o_true_pca_confidence_object_to_subject

        return new_rule_wrapper

    def set_inverse_c_weighted_std_confidence(self, label_frequency: float) -> None:
        if label_frequency == 0.0:
            raise Exception("Label frequency cannot be zero")
        self.o_c_weighted_std_conf = 1 / label_frequency * self.o_std_confidence

    def instantiate_from_amie_dict(self, o_amie_dict: Optional[Dict] = None) -> None:

        if o_amie_dict is None:
            amie_dict_to_use: Optional[Dict] = self.amie_dict
        else:
            amie_dict_to_use = o_amie_dict

        if amie_dict_to_use is None:
            raise Exception(f"No AMIE dict available for rule wrapper {str(self)}")
        else:
            if self.o_n_predictions is not None:
                print("overwriting n_predictions")
            self.o_n_predictions = amie_dict_to_use[AmieOutputKeyEnum.BODY_SIZE.value]

            if self.o_n_supported_predictions is not None:
                print("overwriting o_n_supported_predictions")
            self.o_n_supported_predictions = amie_dict_to_use[AmieOutputKeyEnum.N_POSITIVE_EXAMPLES.value]

            if self.o_std_confidence is not None:
                print("overwriting o_std_confidence")
            self.o_std_confidence = amie_dict_to_use[AmieOutputKeyEnum.STD_CONF.value]

            if amie_dict_to_use[AmieOutputKeyEnum.FUNCTIONAL_VARIABLE.value] == '?a':
                if self.o_pca_confidence_subject_to_object is not None:
                    print("overwriting o_pca_confidence_subject_to_object")
                self.o_pca_confidence_subject_to_object = amie_dict_to_use[AmieOutputKeyEnum.PCA_CONF.value]
                print("ONLY value for o_pca_confidence_subject_to_object")
                print("NO value for o_pca_confidence_object_to_subject")
            elif amie_dict_to_use[AmieOutputKeyEnum.FUNCTIONAL_VARIABLE.value] == '?b':
                if self.o_pca_confidence_object_to_subject is not None:
                    print("overwriting o_pca_confidence_object_to_subject")
                self.o_pca_confidence_object_to_subject = amie_dict_to_use[AmieOutputKeyEnum.PCA_CONF.value]
                print("ONLY value for o_pca_confidence_object_to_subject")
                print("NO value for o_pca_confidence_subject_to_object")
            else:
                raise Exception(f"Unrecognized functional AMIE variabele: "
                                f"{amie_dict_to_use[AmieOutputKeyEnum.FUNCTIONAL_VARIABLE.value]}")

    def to_json_file(self, filename: str, gzipped: bool = True):

        dict_to_convert: Dict = dict()
        rule_str = str(self.rule)
        dict_to_convert['rule'] = rule_str

        if self.amie_dict is not None:
            dict_to_convert['amie_dict'] = self.amie_dict

        if self.o_n_predictions is not None:
            dict_to_convert['o_n_predictions'] = self.o_n_predictions

        if self.o_n_supported_predictions is not None:
            dict_to_convert['o_n_supported_predictions'] = self.o_n_supported_predictions

        if self.o_std_confidence is not None:
            dict_to_convert['o_std_confidence'] = self.o_std_confidence

        if self.o_pca_confidence_subject_to_object is not None:
            dict_to_convert['o_pca_confidence_subject_to_object'] = self.o_pca_confidence_subject_to_object

        if self.o_pca_confidence_object_to_subject is not None:
            dict_to_convert['o_pca_confidence_object_to_subject'] = self.o_pca_confidence_object_to_subject

        if self.o_c_weighted_std_conf is not None:
            dict_to_convert['o_c_weighted_std_conf'] = self.o_c_weighted_std_conf

        if self.o_relative_pu_confidence_unbiased is not None:
            dict_to_convert['o_relative_pu_confidence_unbiased'] = self.o_relative_pu_confidence_unbiased

        if self.o_relative_pu_confidence_pca_subject_to_object is not None:
            dict_to_convert[
                'o_relative_pu_confidence_pca_subject_to_object'
            ] = self.o_relative_pu_confidence_pca_subject_to_object

        if self.o_relative_pu_confidence_pca_object_to_subject is not None:
            dict_to_convert[
                'o_relative_pu_confidence_pca_object_to_subject'
            ] = self.o_relative_pu_confidence_pca_object_to_subject

        # if self.o_absolute_pu_confidence is not None:
        #     dict_to_convert['o_absolute_pu_confidence'] = self.o_absolute_pu_confidence

        if self.o_true_confidence is not None:
            dict_to_convert['o_true_confidence'] = self.o_true_confidence

        if self.o_true_pca_confidence_subject_to_object is not None:
            dict_to_convert['o_true_pca_confidence_subject_to_object'] = self.o_true_pca_confidence_subject_to_object

        if self.o_true_pca_confidence_object_to_subject is not None:
            dict_to_convert['o_true_pca_confidence_object_to_subject'] = self.o_true_pca_confidence_object_to_subject

        pretty_frozen = json.dumps(dict_to_convert, indent=2, sort_keys=True)
        if gzipped:
            open_func = gzip.open
        else:
            open_func = open

        with open_func(filename, 'wt') as output_file:
            output_file.write(pretty_frozen)

    @staticmethod
    def read_json(filename: str, gzipped: bool = True) -> 'RuleWrapper':

        if gzipped:
            open_func = gzip.open
        else:
            open_func = open

        with open_func(filename, 'rt') as input_file:
            dict_to_convert: Dict = json.load(input_file)
            rule_str: str = dict_to_convert["rule"]

            problog_rule: ProblogClause = ProblogTerm.from_string(rule_str)
            pylo_rule: Union[PyloLiteral, PyloClause] = convert_clause(
                clause=problog_rule,
                context=pylo_global_context
            )

            rule_wrapper = RuleWrapper.create_rule_wrapper(
                rule=pylo_rule,
                amie_dict=dict_to_convert.get('amie_dict', None),

                o_n_predictions=dict_to_convert.get('o_n_predictions', None),
                o_n_supported_predictions=dict_to_convert.get('o_n_supported_predictions', None),

                o_std_confidence=dict_to_convert.get('o_std_confidence', None),

                o_pca_confidence_subject_to_object=dict_to_convert.get(
                    'o_pca_confidence_subject_to_object', None),
                o_pca_confidence_object_to_subject=dict_to_convert.get(
                    'o_pca_confidence_object_to_subject', None),

                o_c_weighted_std_conf=dict_to_convert.get('o_c_weighted_std_conf', None),

                o_relative_pu_confidence_unbiased=dict_to_convert.get(
                    'o_relative_pu_confidence_unbiased', None),

                o_relative_pu_confidence_pca_subject_to_object=dict_to_convert.get(
                    'o_relative_pu_confidence_pca_subject_to_object', None),
                o_relative_pu_confidence_pca_object_to_subject=dict_to_convert.get(
                    'o_relative_pu_confidence_pca_object_to_subject', None),

                # o_absolute_pu_confidence = dict_to_convert.get('o_absolute_pu_confidence', None)

                o_true_confidence=dict_to_convert.get('o_true_confidence', None),
                o_true_pca_confidence_subject_to_object=dict_to_convert.get(
                    'o_true_pca_confidence_subject_to_object', None),
                o_true_pca_confidence_object_to_subject=dict_to_convert.get(
                    'o_true_pca_confidence_object_to_subject', None),
            )

            return rule_wrapper

    def clone_with_metrics_unset(self) -> 'RuleWrapper':
        return RuleWrapper(rule=self.rule)

    def clone(self) -> 'RuleWrapper':
        return RuleWrapper.create_rule_wrapper(
            rule=self.rule,
            amie_dict=self.amie_dict,

            o_n_predictions=self.o_n_predictions,
            o_n_supported_predictions=self.o_n_supported_predictions,

            o_std_confidence=self.o_std_confidence,

            o_pca_confidence_subject_to_object=self.o_pca_confidence_subject_to_object,
            o_pca_confidence_object_to_subject=self.o_pca_confidence_object_to_subject,

            o_c_weighted_std_conf=self.o_c_weighted_std_conf,

            o_relative_pu_confidence_unbiased=self.o_relative_pu_confidence_unbiased,

            o_relative_pu_confidence_pca_subject_to_object=self.o_relative_pu_confidence_pca_subject_to_object,
            o_relative_pu_confidence_pca_object_to_subject=self.o_relative_pu_confidence_pca_object_to_subject,

            o_true_confidence=self.o_true_confidence,
            o_true_pca_confidence_subject_to_object=self.o_true_pca_confidence_subject_to_object,
            o_true_pca_confidence_object_to_subject=self.o_true_pca_confidence_object_to_subject,

        )

    @property
    def o_rule_support(self) -> Optional[int]:
        """
        The support of the rule =
            the number of predictions in the observed KB.

        :return:
        """
        return self.o_n_supported_predictions

    @property
    def o_body_support(self) -> Optional[int]:
        """
        The support of the rule BODY =
            the number of predictions of the rule.

        :return:
        """
        return self.o_n_predictions

    def __str__(self):
        if self.amie_dict is not None:
            return str(self.amie_dict)
        else:
            return str(self.rule)

    def __repr__(self):
        return self.__str__()

    def get_columns_header(self) -> List[str]:
        header = self.get_columns_header_without_amie()
        if self.amie_dict is not None:
            header = header + [key.value + " (AMIE)" for key in AmieOutputKeyEnum]
        return header

    @staticmethod
    def get_columns_header_without_amie() -> List[str]:
        return columns_header_without_amie

    def to_row(self, include_amie_metrics: bool = True) -> List[Union[str, float]]:
        row = [
                  str(self.rule),
                  self.o_n_supported_predictions,
                  self.o_n_predictions
              ] + [self.get_value(conf) for conf in ConfidenceEnum]

        if include_amie_metrics and self.amie_dict is not None:
            row.extend([self.amie_dict[key.value] for key in AmieOutputKeyEnum])
        return row

    def get_amie_rule_string_repr(self) -> str:
        """
        :return: the string representation of the rule as generated by AMIE
        """
        if self.amie_dict is None:
            raise Exception()
        else:
            return self.amie_dict[AmieOutputKeyEnum.RULE.value]

    def get_amie_head_coverage(self) -> float:
        """
                                    support of the rule = # observed predictions
        Head coverage of a rule =  ------------------------------------------------------------
                                    number of literals with the head relation in the observed KB

        hc(r) =  \frac{ supp(r) }{ #(x,y): h(x,y) }

        The support of a rule (= the rule's observed predictions) is an integer number.
        Head coverage gives a relative version by dividing it by
         the number of literals with the head's relation in the observed KB.

        :return: the head coverage according AMIE
        """
        if self.amie_dict is None:
            raise Exception()
        else:
            return self.amie_dict[AmieOutputKeyEnum.HEAD_COVERAGE.value]

    def get_amie_rule_support(self) -> int:
        """
        Rule support = nb of predictions done by the rule

        :return:
        """
        pass

    @staticmethod
    def create_rule_wrapper_from(amie_output_rule_series: pd.Series) -> 'RuleWrapper':
        amie_rule_dict: Dict = amie_output_rule_series.to_dict()
        rule_string: str = amie_rule_dict[AmieOutputKeyEnum.RULE.value]

        problog_rule: ProblogClause = DatalogAMIERuleConvertor.convert_amie_datalog_rule_string_to_problog_clause(
            rule_string)

        pylo_rule: Union[PyloLiteral, PyloClause] = convert_clause(
            clause=problog_rule,
            context=pylo_global_context
        )
        return RuleWrapper(rule=pylo_rule, o_amie_dict=amie_rule_dict)

    def set_std_confidence(self, calculated_value: float) -> None:
        if self.amie_dict is not None:
            amie_std_conf_value: float = self.amie_dict[AmieOutputKeyEnum.STD_CONF.value]
            if abs(amie_std_conf_value - calculated_value) >= 0.01:
                print(f"Calculated STD conf differs from AMIE: {calculated_value:0.3f} vs {amie_std_conf_value:0.3f}")
        self.o_std_confidence = calculated_value

    def set_pca_confidence(self, calculated_value: float) -> None:
        if self.amie_dict is not None:
            amie_pca_conf_value: float = self.amie_dict[AmieOutputKeyEnum.PCA_CONF.value]
            if abs(amie_pca_conf_value - calculated_value) >= 0.01:
                print(f"Calculated PCA conf differs from AMIE: {calculated_value:0.3f} vs {amie_pca_conf_value:0.3f}")
        self.o_pca_confidence_subject_to_object = calculated_value

def get_pylo_rule_from_string(rule_str: str) -> PyloClause:
    problog_rule: ProblogClause = ProblogTerm.from_string(rule_str)
    pylo_rule: Union[PyloLiteral, PyloClause] = convert_clause(
        clause=problog_rule,
        context=pylo_global_context
    )
    return pylo_rule

def is_pylo_rule_recursive(pylo_rule: PyloClause) -> bool:
    head_functor: str = pylo_rule.get_head().get_predicate().get_name()

    body_functors: Set[str] = {
        body_literal.get_predicate().get_name()
        for body_literal in pylo_rule.get_body().get_literals()
    }
    return head_functor in body_functors

def filter_rules_predicting(
        rule_wrapper_sequence: Iterator[RuleWrapper],
        head_functor_set: Optional[Set[str]] = None
) -> List[RuleWrapper]:
    # filtered_rules: List[RuleWrapper] = []
    # rule_wrapper: RuleWrapper
    # for rule_wrapper in rule_wrapper_sequence:
    #     if rule_wrapper.rule.get_head().predicate.name in head_functor_set:
    #         filtered_rules.append(rule_wrapper)

    if head_functor_set is None:
        return [rule_wrapper for rule_wrapper in rule_wrapper_sequence]
    else:
        return [rule_wrapper for rule_wrapper in rule_wrapper_sequence
                if rule_wrapper.rule.get_head().predicate.name in head_functor_set]

def contains_rule_predicting_relation(
        rule_wrapper_sequence: Iterator[RuleWrapper],
        target_relation: str
) -> bool:
    return len(
        filter_rules_predicting(
            rule_wrapper_sequence=rule_wrapper_sequence,
            head_functor_set={target_relation}
        )
    ) > 0

def create_amie_dataframe_from_rule_wrappers(rule_wrapper_collection: List[RuleWrapper]) -> pd.DataFrame:
    columns = [AmieOutputKeyEnum.RULE.value] + [key for key in rule_wrapper_collection[0].amie_dict.keys()
                                                if key != AmieOutputKeyEnum.RULE.value]
    data: List[List] = []
    for rule in rule_wrapper_collection:
        row = [str(rule.rule)] + [rule.amie_dict[key] for key in columns if key != AmieOutputKeyEnum.RULE.value]
        data.append(row)
    df = pd.DataFrame(data=data, columns=columns)
    return df

def create_extended_dataframe_from_rule_wrappers(rule_wrapper_collection: List[RuleWrapper]) -> pd.DataFrame:
    columns_header = rule_wrapper_collection[0].get_columns_header()
    row_data = []
    rule_wrapper: RuleWrapper
    for rule_wrapper in rule_wrapper_collection:
        row = rule_wrapper.to_row()
        row_data.append(row)
    df = pd.DataFrame(data=row_data, columns=columns_header)
    return df

def create_dataframe_without_amie_from_rule_wrappers(rule_wrapper_collection: List[RuleWrapper]) -> pd.DataFrame:
    columns_header = RuleWrapper.get_columns_header_without_amie()
    row_data = []
    rule_wrapper: RuleWrapper
    for rule_wrapper in rule_wrapper_collection:
        row = rule_wrapper.to_row(include_amie_metrics=False)
        row_data.append(row)
    df = pd.DataFrame(data=row_data, columns=columns_header)
    return df


import os
import time
import argparse
import random
#import logging
import warnings
import json
import numpy as np
import pickle as pkl

from .utils import read_args
from .runner import *
from .options import add_default_arguments, add_data_arguments, add_logging_arguments, \
    add_model_arguments, add_training_arguments, \
    check_arguments

logger = logging.getLogger('nli')

def parse_args():
    """
    Parse arguments.
    """
    parser = argparse.ArgumentParser()
    add_default_arguments(parser)
    add_data_arguments(parser)
    add_logging_arguments(parser)
    add_model_arguments(parser)
    add_training_arguments(parser)
    args = parser.parse_args()
    args.remove_cheat = eval(args.remove_cheat)
    check_arguments(args)
    return args

def make_args_compatible(args):
    # Be compatible with previously trained models when we added new options
    if args.superficial == True:
        args.superficial == 'hypothesis'
    if not hasattr(args, 'remove'):
        args.remove = False
    if not hasattr(args, 'remove_cheat'):
        args.remove_cheat = 'False'
    return args

def get_runner(args, model_args, task, output_dir=None):
    core_runner = {
            'cbow': CBOWNLIRunner,
            'bert': BERTNLIRunner,
            'da': DANLIRunner,
            'esim': ESIMNLIRunner,
            }
    if not output_dir:
        output_dir = args.output_dir
    model_args = make_args_compatible(model_args)
    if model_args.superficial == 'hypothesis':
        runner = HypothesisNLIRunner(task, output_dir, args.exp_id)
    elif model_args.superficial == 'handcrafted':
        runner = HandcraftedNLIRunner(task, output_dir, args.exp_id)
    elif model_args.additive:
        prev_runners = []
        prev_args = []
        for i, path in enumerate(model_args.additive):
            _prev_args = read_args(path)
            # Change to inference model
            _prev_args.init_from = path
            _prev_args.dropout = 0.0
            _prev_runner = get_runner(args, _prev_args, task, output_dir='/tmp')
            prev_runners.append(_prev_runner)
            prev_args.append(_prev_args)
        runner = get_additive_runner(core_runner[model_args.model_type], remove=model_args.remove)(task, output_dir, prev_runners, prev_args, args.exp_id)
    else:
        runner = core_runner[model_args.model_type](task, output_dir, args.exp_id)
    return runner

def main(args):
    np.random.seed(args.seed)
    random.seed(args.seed)

    if args.mode == 'test':
        model_args = read_args(args.init_from)
    else:
        model_args = args

    task = tasks[args.task_name]

    runner = get_runner(args, model_args, task)

    pool_type = os.environ.get('MXNET_GPU_MEM_POOL_TYPE', '')
    if pool_type.lower() == 'round':
        logger.info(
            'Setting MXNET_GPU_MEM_POOL_TYPE="Round" may lead to higher memory '
            'usage and faster speed. If you encounter OOM errors, please unset '
            'this environment variable.')

    try:
        runner.run(args)
        runner.dump_report()
    except KeyboardInterrupt as e:
        print(e)
        logger.info('Terminated. Dumping report.')
        runner.dump_report()

if __name__ == '__main__':
    args = parse_args()
    main(args)


# 라이브러리 로드
import os
import numpy as np
import pandas as pd
import xgboost as xgb
import optuna
import pickle
from optuna import Trial, visualization
from optuna.samplers import TPESampler
from sklearn.metrics import r2_score, mean_squared_error
from scaling import scaled_into_trainset
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.externals import joblib

# 데이터 전처리

##

os.chdir('/Users/choeunsol/Python/PAINS 방중 프로젝트/최종 분석 데이터셋/인풋 데이터')

minor_data = pd.read_csv('MLB 마이너레벨 분석용 데이터셋.csv')
minor_data = minor_data.drop('Unnamed: 0', axis = 1)

## 40인 로스터 여부 원핫인코딩

minor_data['MAJOR_40'] = minor_data.loc[:, 'MAJOR_40'].fillna(0)
minor_with_nona = minor_data.dropna(axis = 0)

## 중복치 처리

minor_with_nona = minor_with_nona.drop_duplicates(['NAME', 'median.trade.value'], keep = False)

## FV 처리
minor_with_nona.groupby('FV').mean().loc[:, 'median.trade.value']

minor_with_nona = minor_with_nona.reset_index().drop('index', axis = 1)
minor_with_nona = minor_with_nona.replace(['35+', '40+', '45+'], [37.5, 42.5, 47.5])
minor_with_nona.groupby('FV').mean().loc[:, 'median.trade.value']

## 범주형 변수 처리

minor_xgboost = minor_with_nona
eta_dict = {2022.0 : 1, 2023.0 : 2, 2024.0 : 3, 2025.0 : 4, 2026.0 : 5, 2027.0 : 6, 2028.0 : 7}
pos_dict = {"1B" : 1, "2B" : 2, "3B" : 3, "C" : 4, "DH" : 5, "OF" : 6, "RP" : 7, "SP" : 8, "SS" : 9, "UTIL" : 10}
eta_real = minor_xgboost['ETA'].apply(lambda x : eta_dict[x])
pos_real = minor_xgboost['POS'].apply(lambda x : pos_dict[x])
minor_xgboost['ETA'] = eta_real
minor_xgboost['POS'] = pos_real

xgdata = minor_xgboost.drop(['NAME', 'MAJOR_40'], axis = 1)

minor_Y = xgdata.loc[:, 'median.trade.value']
minor_X_col = list(xgdata.columns.difference(['median.trade.value']))
minor_X = xgdata[minor_X_col]

minor_X['FV'] = minor_X.loc[:, 'FV'].astype('float64')

## train-test 분리

train_minor_x, test_minor_x, train_minor_y, test_minor_y = train_test_split(minor_X, minor_Y, test_size = 0.2, random_state = 0)
train_minor_x, val_minor_x, train_minor_y, val_minor_y = train_test_split(train_minor_x, train_minor_y, test_size = 1/3, random_state=0)

minor_name = minor_with_nona.iloc[test_minor_y.index].loc[:, 'NAME']

## 정규화

scaletr_minor_x = train_minor_x.loc[:, ['FV']]
scaleval_minor_x = val_minor_x.loc[:, ['FV']]
scaletest_minor_x = test_minor_x.loc[:, ['FV']]

tv_milb_scaler = StandardScaler()
tv_milb_scaler.fit(train_minor_y.values.reshape(-1,1))

minor_y_scaler = scaled_into_trainset(StandardScaler())
train_minor_y, test_minor_y, val_minor_y = minor_y_scaler.y_scaled(train_minor_y, test_minor_y, val_minor_y)

# 파라미터 튜닝

def xgb_params_tuning(trial: Trial) -> float:
    params_xgb = {
        "n_estimators" : trial.suggest_int('n_estimators', 50, 200),
        "max_depth": trial.suggest_int('max_depth', 5, 15),
        'random_state': 0,
        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 10.0),
        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 10.0),
        'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.0, 1.0),
        'subsample': trial.suggest_uniform('subsample', 0.0, 1.0),
        'learning_rate':trial.suggest_uniform('learning_rate', 0.01, 0.5)
    }

    model = xgb.XGBRegressor(**params_xgb)
    model.fit(train_minor_x, train_minor_y, eval_set = [(train_minor_x, train_minor_y), (val_minor_x, val_minor_y)], verbose= False)

    xgb_pred_m = model.predict(val_minor_x)
    rmse = np.sqrt(mean_squared_error(val_minor_y, xgb_pred_m))

    return rmse

sampler = TPESampler(seed = 0)

tuned = optuna.create_study(
    study_name = 'xgboost minor level params',
    direction = 'minimize',
    sampler = sampler
)

tuned.optimize(xgb_params_tuning, n_trials = 400)

# train

import xgboost as xgb

xgb_regressor_minor = xgb.XGBRegressor(**tuned.best_params)
xgb_regressor_minor.fit(train_minor_x, train_minor_y)

pred = xgb_regressor_minor.predict(test_minor_x)
rmse = np.sqrt(mean_squared_error(test_minor_y, pred))
print(rmse) # rmse

r2 = r2_score(test_minor_y, pred)
print(r2) # R^2

## 모델 저장

xgb_minor = pickle.dumps(xgb_regressor_minor)
joblib.dump(xgb_minor, 'xgb_regressor.pkl')

## 예측치 저장
os.chdir('/Users/choeunsol/Python/PAINS 방중 프로젝트/최종 분석 데이터셋/모델 예측치')

pred_origin, real_minor_y = minor_y_scaler.scale_to_origin(pred, test_minor_y)

pred_minor = pred_origin.reshape(-1)
value_minor = real_minor_y.reshape(-1)

df = pd.DataFrame({"name" : minor_name, "predict_value" : np.round(pred_minor,1), "real_value" : np.round(value_minor, 1)})
df.to_csv('xgBoost model_minor level_예측치.csv', index = False)

# KBO 데이터 적합

os.chdir('/Users/choeunsol/Python/PAINS 방중 프로젝트/최종 분석 데이터셋/인풋 데이터')

minor_kbo = pd.read_excel('kbo prospect.xlsx')

## 전처리 (군복무 여부 추가)
minor_kbo_name, minor_kbo_team = minor_kbo['name'].values, minor_kbo['team'].values

no_mil = minor_kbo[lambda x : (x.군 == '미필') | (x.군 == '군복무중')]['ETA'] + 1
minor_kbo['ETA'] = no_mil
minor_kbo

## 정규화

minor_kbo_foran = minor_kbo.loc[:, ['age', 'pos', 'score', 'ETA']]
minor_kbo_pos = minor_kbo_foran['pos'].apply(lambda x : pos_dict[x])
minor_kbo_foran['pos'] = minor_kbo_pos

minor_kbo_foran.rename(columns = {'age' : 'AGE', 'pos' : 'POS', 'score' : 'FV'}, inplace = True)
minor_kbo_foran = minor_kbo_foran[['AGE', 'ETA', 'FV', 'POS']]

# predict

minor_trade_value_norm = xgb_regressor_minor.predict(minor_kbo_foran)
minor_trade_value_tf = tv_milb_scaler.inverse_transform(minor_trade_value_norm.reshape(-1,1))
minor_trade_value = minor_trade_value_tf.reshape(-1)

tradevalue_df = pd.DataFrame({'name' : minor_kbo_name, 'team' : minor_kbo_team, 'trade_value' : np.round(minor_trade_value,2)})

os.chdir('/Users/choeunsol/Python/PAINS 방중 프로젝트/최종 분석 데이터셋/모델 예측치')
tradevalue_df.to_excel('kbo minor level trade value.xlsx')


import os
from setuptools import setup, find_packages

ROOT = os.path.abspath(os.path.dirname(__file__))

base_config = {'name': 'null_reality',
               'version': '0.2a1',
               'description': 'Model and field utils to make Django respect null=False validation in text-based fields.',
               'long_description': open(os.path.join(ROOT, 'README.md')).read(),
               'author': 'J Leadbetter',
               'author_email': 'codemonkey@jleadbetter.com',
               'url': 'http://bitbucket.org/kamni/django-null-reality',
               'license': 'Affero GPL v3',
               'packages': find_packages(),
               'include_package_data': True,
               'zip_safe': False,
               'install_requires': ['Django>=1.6'],
               'classifiers': ['Development Status :: 3 - Alpha',
                               'Environment :: Web Environment',
                               'Framework :: Django',
                               'Intended Audience :: Developers',
                               'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
                               'Programming Language :: Python :: 2 :: Only']
              }

setup(**base_config)


#!/usr/bin/env python

import argparse
import numpy as np
from scipy import signal
from matplotlib import pylab as plt

def setup_args():
    parser = argparse.ArgumentParser(description='Extract data from binary file obtained with extract_iq.')
    parser.add_argument('filepath', type=str, help='File to open.')
    parser.add_argument('--header_length', type=int, default=512, help='Length of the text header (default is 512)')
    args   = parser.parse_args()
    return args

def decode_header (header):
    eos = 0
    for b in header:
        if b == 0:
            break
        eos = eos +1
    return header[:eos].decode ()

def decode_int16 (data,n=1):
    return np.fromfile (data,np.dtype ('int16'), count = n)

def decode_uint32 (data,n=1):
    return np.fromfile (data,np.dtype ('uint32'), count = n)

def decode_uint64 (data,n=1):
    return np.fromfile (data,np.dtype ('uint64'), count = n)

def decode_float64 (data,n=1):
    return np.fromfile (data,np.dtype ('float64'), count = n)

def decode_fe_freq (data):
    return np.fromfile (data,np.dtype ('uint64'), count = 2)

def main (args):
    print (args.filepath)

    f = open (args.filepath,'rb')

    # This assumes you already know the length of the header
    header = decode_header (f.read (args.header_length))
    print ('Header: ',header)

    FE = decode_fe_freq (f)
    print ('Front-end frequencies: {}'.format (FE))
    number_of_channels = decode_uint32 (f,2)
    print ('Number of channels: {}'.format (number_of_channels))
    block_size = decode_uint32 (f)
    print ('Block size: {}'.format (block_size))
    while (True):
        # This assumes you know what to expect
        start_time_iq = decode_uint64 (f)
        if len (start_time_iq) == 0:
            break
        print (start_time_iq)
        sample_rate = decode_float64 (f)
        print (sample_rate)
        scaling = decode_int16 (f,n=1)  # Scaling
        print (scaling)
        real = decode_float64 (f,n=block_size) # Real
        imag = decode_float64 (f,n=block_size) # Real
        print (real[0])
        print (imag[0])

        scaling_lin = np.power (10,scaling/2000)
        print (10*np.log10(np.mean (np.power (real*scaling_lin,2.0) + np.power (imag*scaling_lin,2.0))))

    # Plot the last block
    real_scaled = real*scaling_lin
    imag_scaled = imag*scaling_lin

    # f, Pxx_den = signal.periodogram(real_scaled+np.complex(0,1)*imag_scaled, nfft=2048)
    f, Pxx_den = signal.welch(real_scaled+np.complex(0,1)*imag_scaled,
                              fs = sample_rate,
                              #scaling='density',
                              scaling='spectrum',
                              nperseg=4096)

    plt.ion ()
    plt.plot (f, 10*np.log10 (Pxx_den))
    # plt.ylim ([1e-11, 1e-5])
    plt.grid (True)
    plt.tight_layout ()
    input ('Press any key.')

if __name__ == '__main__':
    main (setup_args ())


from django.conf import settings
from django.urls import include, path
from rest_framework.routers import DefaultRouter, SimpleRouter

if settings.DEBUG:
    router = DefaultRouter()
else:
    router = SimpleRouter()

app_name = 'api'

urlpatterns = [
    # Merkel apps
    path(
        '',
        include(('src.workflow.api.urls', 'src.workflow'),
                namespace='workflow')
    ),
]

urlpatterns += router.urls


import prometheus_api_client as pac
import prometheus_api_client.utils as pac_u
import pyspark as ps
import findspark
findspark.init("C:\Spark")

spark = ps.sql.SparkSession.builder \
    .master("local") \
    .appName("Dimas Spark") \
    .config("spark.some.config.option", "some-value") \
    .getOrCreate()

def dataframe_creation():
    # точка входа в программирование Spark с помощью Dataset и DataFrame API.
    # master - задает главный URL-адрес Spark для подключения, например «local» для локального запуска
    # appName - устанавливает имя приложения, которое будет отображаться в веб-интерфейсе Spark
    # config - устанавливает параметр конфигурации
    # getOrCreate() - получает существующий SparkSession или, если его нет, создает новый на основе параметров, установленных в этом построителе

    # конект к прометеусу
    prom = pac.PrometheusConnect(url="http://localhost:9090/", disable_ssl=True)

    # получение названия всех метрик прометеуса
    all_metrics = prom.all_metrics()
    metrics = list()

    # установка рейнджа времени для данных
    # в данном случае от "минус 1 день" до "сейчас"
    start_time = pac_u.parse_datetime("1d")
    end_time = pac_u.parse_datetime("now")

    dataframe_list = dict()

    # цикл, позволяющий выбрать только те метрики, которые нам нужны,
    # а именно в которых есть название application и process.
    # такими словами помечены собираемые метрики с приложения.

    """
    for i in all_metrics:
        if i.split('application')[0] == '' or i.split('process')[0] == '':
            metric_data = prom.get_metric_range_data(
                i,
                start_time=start_time,
                end_time=end_time
            )
            if metric_data:
                # берем только значения метрики
                values = metric_data[0]['values']
                # создаем датафрейм со значениями метрики и сохраняем его в словаре,
                # где ключ - название метрики
                print(values)
                dataframe_list[i] = (spark.createDataFrame(values, ['time', 'value'])
    """
    dataframe = None
    for i in all_metrics:
        if i.split('application')[0] == '': #or i.split('process')[0] == '':
            metric_data = prom.get_metric_range_data(
                i,
                start_time=start_time,
                end_time=end_time
            )
            if metric_data:
                # берем только значения метрики
                values = metric_data[0]['values']
                if dataframe is None:
                    dataframe = spark.createDataFrame(values, ['time', i])
                else:
                    df = spark.createDataFrame(values, ['time', i])
                    dataframe = dataframe.join(df, 'time', 'right')

    # dataframe.show()
    dataframe.write.csv('metrics.csv')
    return dataframe

dataframe_creation()


import random
import datetime
import re
import os
import time

import discord
import requests
from discord.ext import commands

from bot.utils.extensions import KatCog, write_resource
from bot.utils import constants

class Fun(KatCog):
    def __init__(self, bot):
        super().__init__(bot)

        self.cocks = 0
        self.gif_cache = {}
        # TODO: Not sure if this is still used.
        self.megu_cache_time = 3600

        # day of the week reponse flags
        self.dayresponse = [
            "***:boom: It's Megumonday! :boom:***\n今日、メグー月曜日だよ！",
        ]

        self.bot.run_day_check = True

    def _get_and_cache_gifs(self, search_query):
        """Download gif cache and store results for 1 hour."""
        self.log.info(f"Caching gifs for {search_query} for 1 hour...")
        r = requests.get(
            "https://api.tenor.com/v1/search?q={}&key={}&limit=20&anon_id={}".format(
                search_query,
                constants.Fun.api_key,
                constants.Fun.anon_key,
            )
        )
        if r.status_code == 200:
            gif_links = []
            for gif in r.json()["results"]:
                # All of the gif links.
                gif_links.append(gif["media"][0]["gif"]["url"])

            self.gif_cache[search_query] = (time.time(), gif_links)

    async def _get_gif(self, search_query):
        """Returns a random gif from the gif cache.
        If query is not already in the cache, we download gif collection and then cache it.
        """
        if (
            search_query not in self.gif_cache
            or self.gif_cache[search_query][0] + 3600 < time.time()
        ):
            self.log.debug(f"Cache expired for {search_query}.")
            self._get_and_cache_gifs(search_query)

        # return a random gif from cached gif links.
        raw = self.gif_cache[search_query][1]
        return raw[random.randrange(0, len(raw))]

    def _generate_generic_embed(self, ctx, gif, action, user: discord.User = None):
        responses = {
            "everyone": f"performs {action} to everyone!",
            "user": "performs " + action + " to {user.display_name} O.o",
            "self": f"performs {action} to me!? :flushed:",
        }
        return self._generate_specific_embed(ctx, gif, responses, user, constants.Color.soft_red)

    def _generate_specific_embed(
        self, ctx, gif, responses, user: discord.User = None, color=discord.Color.red()
    ):
        """
        Generates an embed for emotes.
            responses should look like:
                {
                    "everyone" : "kisses everyone! <3",
                    "user" : "kisses {user.display_name} Aww!",
                    "self" : "kisses Ka- Wait a minute! O.o"
                }
        """
        if user is None:
            embed = discord.Embed(
                title=f"{ctx.author.display_name} " + responses["everyone"]
            )
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} " + responses['user'].format(user=user)
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title=responses["self"])

        embed.set_image(url=gif)
        embed.colour = color
        return embed

    @commands.Cog.listener()
    async def on_kat_hour_event(self):
        if not self.bot.run_day_check:
            return

        try:
            guild = discord.utils.get(self.bot.guilds, id=constants.HomeGuild.ids[0])
            channel = discord.utils.get(guild.channels, id=constants.HomeGuild.channels[0])
        except AttributeError:
            # we mustn't be able to see Reign guild
            self.bot.run_day_check = False
            return

        self.log.info("Daily check started.")
        date = datetime.datetime.today()

        if date.weekday():  # is monday
            if "megu_done" not in os.listdir("bot/resources/days/"):
                if channel != None:
                    await channel.send(
                        self.dayresponse[0], file=discord.File("bot/resources/days/0.png")
                    )

                    write_resource("days/megu_done", "1")
                    self.log.info("It's megumonday!")
        else:
            try:
                os.remove("bot/resources/days/megu_done")
            except FileNotFoundError:
                pass

    @commands.Cog.listener()
    async def on_message(self, ctx):
        if "gorl" in ctx.content.lower() and ctx.guild.id in constants.HomeGuild.ids:
            emoji = discord.utils.get(
                self.bot.get_guild(311612862554439692).emojis, name="gorl"
            )
            await ctx.add_reaction(emoji)

    @commands.command()
    async def time(self, ctx):
        embed = discord.Embed(title="The time is (GMT)")
        embed.description = str(datetime.datetime.now())
        embed.set_footer(text="WIP. Timezones coming soon.")
        await ctx.send(embed=embed)

    @commands.command()
    async def stuff(self, ctx):
        f = discord.File("bot/resources/stuff.png", filename="stuff.png")
        embed = discord.Embed()
        embed.set_author(name="I'll stuff you all in the crust!")
        embed.set_image(url="attachment://stuff.png")
        await ctx.send(file=f, embed=embed)

    @commands.command()
    async def emote(self, ctx, action: str, user: discord.User = None):
        """Do a custom emote ($emote <action> <mention>)"""
        gif = await self._get_gif("anime%20{}".format(action))
        await ctx.channel.send(
            embed=self._generate_generic_embed(ctx, gif, action, user)
        )

    @commands.command()
    async def spank(self, ctx, user: discord.User = None):
        """Spank ($spank <mention>)"""
        gif = await self._get_gif("anime%20spank")

        responses = {
            "everyone": "spanks everyone!",
            "user": "spanks {user.display_name} kinky!",
            "self": "I-I'm sorry! ;-;",
        }

        embed = self._generate_specific_embed(ctx, gif, responses, user=user)
        await ctx.channel.send(embed=embed)

    @commands.command()
    async def nom(self, ctx, user: discord.User = None):
        """Nom ($nom <mention>)"""
        gif = await self._get_gif("anime%20nom")

        responses = {
            "everyone": "noms everyone!",
            "user": "takes a nibble of {user.display_name}, tasty!",
            "self": "D-do I taste good?",
        }

        embed = self._generate_specific_embed(ctx, gif, responses, user=user)
        await ctx.channel.send(embed=embed)

    @commands.command(aliases=["cum"])
    async def lewd(self, ctx, user: discord.User = None):
        """Lewd someone ($lewd <mention>)"""
        gif = await self._get_gif("anime%20lewd")

        responses = {
            "everyone": "does lewd things to everyone!",
            "user": "lewdifies {user.display_name} How lewd!",
            "self": "Hentai!",
        }

        embed = self._generate_specific_embed(ctx, gif, responses, user=user)
        await ctx.channel.send(embed=embed)

    @commands.command(aliases=["giveflower"])
    async def flower(self, ctx, user: discord.User = None):
        """Give someone a flower ($flower <mention>)"""
        gif = await self._get_gif("anime%20flower")

        responses = {
            "everyone": "gives everyone a flower!",
            "user": "hands {user.display_name} a flower!",
            "self": "A flower for me? O.O",
        }

        embed = self._generate_specific_embed(ctx, gif, responses, user=user)
        await ctx.channel.send(embed=embed)

    @commands.command(aliases=["murder"])
    async def kill(self, ctx, user: discord.User = None):
        """Kill someone ($kill <mention>)"""
        gif = await self._get_gif("anime%20kill")

        responses = {
            "everyone": "kills everyone!",
            "user": "kills {user.display_name}! F",
            "self": "[insert death noises here]",
        }

        embed = self._generate_specific_embed(ctx, gif, responses, user=user)
        await ctx.channel.send(embed=embed)

    @commands.command(aliases=["snuggle"])
    async def cuddle(self, ctx, user: discord.User = None):
        """Snuggle ($snuggle <mention>)"""
        gif = await self._get_gif("anime%20snuggle")

        if user is None:
            embed = discord.Embed(title=f"{ctx.author.display_name} cuddles everyone!")
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} cuddles {user.display_name}:3"
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title="Nya~ (*´ω｀)")
        embed.set_image(url=gif)
        embed.colour = discord.Color.red()
        await ctx.channel.send(embed=embed)

    @commands.command()
    async def pat(self, ctx, user: discord.User = None):
        """Pat a user's head ($pat <mention>)"""
        gif = await self._get_gif("anime%20pat")

        if user is None:
            embed = discord.Embed(title=f"{ctx.author.display_name} pats everyone")
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} pats {user.display_name}'s head :3"
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title="Nya~ (*´ω｀)")
        embed.set_image(url=gif)
        embed.colour = discord.Color.red()
        await ctx.channel.send(embed=embed)

    @commands.command()
    async def lick(self, ctx, user: discord.User = None):
        """Lick a user ($lick <mention>)"""
        gif = await self._get_gif("anime%20lick")

        if user is None:
            embed = discord.Embed(title=f"{ctx.author.display_name} licks everyone")
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} licks {user.display_name} O.o"
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title="B-Baka!")
        embed.set_image(url=gif)
        embed.colour = discord.Color.red()
        await ctx.channel.send(embed=embed)

    @commands.command()
    async def hug(self, ctx, user: discord.User = None):
        """Hug a user ($hug <mention>)"""
        gif = await self._get_gif("anime%20hug")

        if user is None:
            embed = discord.Embed(title=f"{ctx.author.display_name} hugs everyone")
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} hugs {user.display_name} :3"
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title="W-What are you doing! :flushed:")
        embed.set_image(url=gif)
        embed.colour = discord.Color.red()
        await ctx.channel.send(embed=embed)

    @commands.command()
    async def slap(self, ctx, user: discord.User = None):
        """Slap a user ($slap <mention>)"""
        gif = await self._get_gif("anime%20slap")

        if user is None:
            embed = discord.Embed(title=f"{ctx.author.display_name} slaps everyone!")
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} slaps {user.display_name}!"
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title="いたい! That hurt!")
        embed.set_image(url=gif)
        embed.colour = discord.Color.red()
        await ctx.channel.send(embed=embed)

    @commands.command()
    async def kiss(self, ctx, user: discord.User = None):
        """Kiss a user ($kiss <mention>)"""
        gif = await self._get_gif("anime%20kiss")

        if user is None:
            embed = discord.Embed(title=f"{ctx.author.display_name} kisses everyone")
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} kisses {user.display_name} <3"
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title="I-It's not like I like you or anything!")
        embed.set_image(url=gif)
        embed.colour = discord.Color.red()
        await ctx.channel.send(embed=embed)

    @commands.command(aliases=["handhold"])
    async def holdhand(self, ctx, user: discord.User = None):
        """Hold a users hand ($holdhand/$handhold <mention>)"""
        gif = await self._get_gif("hold%20hand")

        if user is None:
            embed = discord.Embed(
                title=f"{ctx.author.display_name} holds everyones hand"
            )
        else:
            if user is not None and user.id != self.bot.id:
                user = ctx.guild.get_member(user.id)
                embed = discord.Embed(
                    title=f"{ctx.author.display_name} holds hands with {user.display_name}"
                )
            elif user.id == self.bot.id:
                embed = discord.Embed(title="I-It's not like I like you or anything!")
        embed.set_image(url=gif)
        embed.colour = discord.Color.red()
        await ctx.channel.send(embed=embed)

    @commands.command()
    async def random(self, ctx, *args):
        """Let Kat choose from a list of inputs ($random 1 2 3 4 ...)"""
        if args is None:

            raise IndexError("Missing list of items to choose from.")
        await ctx.send("**Hmm... I pick `{}`!**".format(random.choice(args)))

    @random.error
    async def random_error(self, ctx, error):
        if isinstance(error, IndexError):
            await ctx.send(self.get_response("common.error.missing_args", args="list"))

    # Hit or miss
    @commands.command(aliases=["huh"])
    async def hitormiss(self, ctx):
        """I guess they never miss, huh?"""
        await ctx.send(
            "**I guess they never miss, huh?**"
            "\nhttps://cdn.discordapp.com/attachments/569186352583933953/570671772320661544/nico1.mp4"
        )

    @commands.command(aliases=["owo"])
    async def owoifier(self, ctx):
        """Turn any message into cancer"""
        last_message = await ctx.channel.history(limit=2).flatten()
        last_message = last_message[1].content.lower()
        last_message = re.sub(r"[RrIl]", "w", last_message)
        last_message = last_message.replace("ove", "uv")
        last_message = last_message.replace("thi", "wi")

        await ctx.send(last_message)

        # Cock counter for omegle

    @commands.command(aliases=["cc"])
    async def cockcounter(self, ctx, arg=None):
        """Keep track of cocks seen during omegle ($cockcounter)"""
        guild = self.sql.ensure_exists("KatGuild", guild_id=ctx.guild.id)
        cock_highscore = guild.ensure_setting(constants.GuildSettings.fun_counter, 0)

        if arg is None:
            self.cocks += 1
            if self.cocks > cock_highscore:
                guild.set_setting(constants.GuildSettings.fun_counter, cock_highscore + 1)
                cock_highscore += 1
                gif = await self._get_gif("celebrate")

                # If new highscore achived, send embed.
                embedHighscore = discord.Embed
                embedHighscore = discord.Embed()
                embedHighscore.title = "NEW COCK HIGHSCORE"
                embedHighscore.description = " "
                embedHighscore.color = constants.Color.invisible
                embedHighscore.set_image(url=gif)
                await ctx.send(embed=embedHighscore)

        elif arg == "start":
            self.cocks = 0
        elif arg == "reset":
            self.cocks = 0
            guild.set_setting(constants.GuildSettings.fun_counter, 0)
            cock_highscore = 0

        embed = discord.Embed()
        embed.title = "Official Cock Counter"
        embed.description = (
            "Current Amount of cocks: {}\n"
            "Current Highscore: {}".format(self.cocks, cock_highscore)
        )
        embed.color = 15843965
        await ctx.send(embed=embed)

def setup(bot):
    bot.add_cog(Fun(bot))


import requests

import lib.settings
from lib.errors import AutoSploitAPIConnectionError
from lib.settings import (
    HOST_FILE,
    API_URLS,
    write_to_file
)

class CensysAPIHook(object):

    """
    Censys API hook
    """

    def __init__(self, identity=None, token=None, query=None, proxy=None, agent=None, save_mode=None, **kwargs):
        self.id = identity
        self.token = token
        self.query = query
        self.proxy = proxy
        self.user_agent = agent
        self.host_file = HOST_FILE
        self.save_mode = save_mode

    def search(self):
        """
        connect to the Censys API and pull all IP addresses from the provided query
        """
        discovered_censys_hosts = set()
        try:
            lib.settings.start_animation("searching Censys with given query '{}'".format(self.query))
            req = requests.post(
                API_URLS["censys"], auth=(self.id, self.token),
                json={"query": self.query}, headers=self.user_agent,
                proxies=self.proxy
            )
            json_data = req.json()
            for item in json_data["results"]:
                discovered_censys_hosts.add(str(item["ip"]))
            write_to_file(discovered_censys_hosts, self.host_file, mode=self.save_mode)
            return True
        except Exception as e:
            raise AutoSploitAPIConnectionError(str(e))


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^search_query/',views.search_query_view,name='search_query'),
    url(r'^watch/',views.watch_view,name='watch'),
    url(r'^channel/',views.channel_view,name='channel'),
]


from host import *
import argparse
from time import sleep

parser = argparse.ArgumentParser()

parser.add_argument('--start',
                    dest='start',
                    action="store_true",
                    default=False)

parser.add_argument('--stop',
                    dest="stop",
                    action="store_true",
                    default=True)

parser.add_argument('-P',
                    dest="P",
                    default=4)

parser.add_argument('--cpu',
                    dest="cpu",
                    default=None)

args = parser.parse_args()
if args.start:
    args.stop = False
if args.stop:
    args.start = False

hosts = []
hlist = HostList()

for h in host_ips:
    hst = Host(h)
    hosts.append(hst)
    hlist.append(hst)

def iperf_server():
    cmd = "iperf -s"
    if args.cpu:
        cmd = "taskset -c %s %s" % (args.cpu, cmd)
    return cmd

def iperf_client(dst,time='3600'):
    cmd = "iperf -c %s -P %s -t %s" % (dst.get_10g_ip(), args.P, time)
    if args.cpu:
        cmd = "taskset -c %s %s" % (args.cpu, cmd)
    return "(%s &)" % cmd

def start_mesh():
    for h in hosts:
        h.cmd_async(iperf_server())
        h.configure_rps()

    sleep(2)
    for i,hi in enumerate(hosts):
        hi.delay = True
        for j,hj in enumerate(hosts):
            if i == j:
                continue
            hi.cmd_async(iperf_client(hj))
        hi.delayed_async_cmds_execute()
    return

def stop_mesh():
    hlist.cmd("killall -9 iperf")

if args.start:
    start_mesh()
if args.stop:
    stop_mesh()


import re
import os
import html
from datetime import datetime
from Rfc_Errata.utils import strip_pagebreaks

order = {"Verified": 1, "Held": 2, "Reported": 3, "Rejected": 4}

def inlineSectionKey(data):
    return "{0}||{2}||{1}".format(data["section2"], data["errata_id"], order[data["status_tag"]])

def inlineKey(data):
    return "{0}||{1}||{2}||{4}||{3}".format(data[0]["section2"], data[1].start, data[1].end,
                                            data[0]["errata_id"], order[data[0]["status_tag"]])

def sortById(data):
    return "{0:6}".format(data["errata_id"])

def increment(tag):
    if tag[0].isdigit():
        return str(int(tag)+1)
    if not tag[-1].isdigit():
        tag = tag[:-1] + chr(ord(tag[-1])+1)
        return tag
    i = len(tag)-1
    while tag[i].isdigit():
        i -= 1
    return tag[:i] + str(int(tag[i:])+1)

startTime = datetime(1970, 1, 1)

class apply_errata(object):
    def __init__(self, toBeApplied, options, state):
        self.header_re = re.compile(u'$('
                                    u'\d+(\.\d+)*\.?|'         # 1.1.1
                                    u'\w(\.\d+)*\.?|'          # A.1.1
                                    u'Appendix \w(\.\d+)*\.?)'  # Appendix A.1.1.1
                                    )

        # Section "1.0"
        self.header_re1 = re.compile("\d+(\.\d+)*\.? |Appendix \w(\.\d+)*[\.:]? |\w(\.\d+)*[\.:]? ")
        self.header_re2 = re.compile("(\s*)\d+(\.\d+)*\.? |(\s*)Appendix \w(\.\d+)*[\.:]? " +
                                     "|(\s*)\w(\.\d+)*\.? ")
        self.header_re = self.header_re2
        self.InlineCount = 0
        self.SectionCount = 0
        self.EndnoteCount = 0
        self.toApply = toBeApplied
        self.title = toBeApplied[0]["doc-id"]
        self.options = options
        self.state = state

        # Strings used in the process

        self.inlineFormat = "{2}"
        self.inlineExpandWrapper = "
{1}
"
        self.buttonFormat = " <button id=\"btn_{0}\" target=\"expand_{0}\" onclick='hideFunction(\"expand_{0}\")'>Expand</button>\n"

    def apply(self, force, templates):
        self.RfcName = self.toApply[0]["doc-id"]
        fileName = os.path.join(self.state["text"],  self.RfcName + ".txt")
        htmlFile = os.path.join(self.state["html"],  self.RfcName + ".html")

        self.templates = templates

        if not force and os.path.exists(htmlFile):
            statIfno = os.stat(htmlFile)

            update = False
            for item in self.toApply:
                updateTime = item["update_date"]
                if not updateTime:
                    updateTime = item["submit_date"]
                    if not updateTime:
                        continue
                    else:
                        itemUpdate = datetime.strptime(updateTime, "%Y-%m-%d")
                else:
                    itemUpdate = datetime.strptime(updateTime, "%Y-%m-%d %H:%M:%S")

                itemUpdate = (itemUpdate - startTime).total_seconds()
                if itemUpdate > statIfno.st_mtime:
                    update = True
                    break

            if not update:
                return

        #  Open the source file and read in.

        self.buildHeader()

        self.loadDocument(fileName)

        self.sectionDocument()

        self.createInlineNotes()

        self.createSectionNotes()

        self.unsectionDocument()
        self.createFootnotes()
        self.emitHtml()

    def loadDocument(self, fileName):
        try:
            with open(fileName, "r", encoding='utf-8') as f:
                text = f.read()
        except UnicodeDecodeError:
            with open(fileName, "r", encoding='iso-8859-1') as f:
                text = f.read()

        # Do the the first set of simple changes
        # 1.  Strip page headers/footer and page-feed
        # 2.  ?????

        # Import the method from id2xml

        a, shortTitle = strip_pagebreaks(text)

        self.source = [x.txt for x in a]
        self.source = "\n".join(self.source).splitlines(True)

        # Figure out what the indent level is

        indent = 72
        pattern = re.compile("^( *)")
        for line in self.source:
            if len(line) == 1:
                continue
            match = pattern.match(line)
            if match:
                i = len(match.group(0))
                if i < indent:
                    indent = i
                    if indent == 0:
                        break

        if indent > 0:
            newSource = []
            for line in self.source:
                if len(line) > 1:
                    newSource.append(line[indent:])
                else:
                    newSource.append(line)

            self.source = newSource

    def addToKnowns(self, section):
        section = section.lower()
        if section not in self.knownSections:
            self.knownSections.append(section)

    def isSectionStart(self, line, i):
        if line.strip().lower() in self.knownSections:
            if line.strip().lower() == "table of contents":
                return line.strip().lower(), False
            return line.strip().lower(), True

        header = self.header_re.match(line)
        if not header:
            return None, True

        potentialSection = header.group(0).strip().lower()

        lpad = len(line) - len(line.lstrip(' '))

        if potentialSection[-1] == '.' or potentialSection[-1] == ':':
            potentialSection = potentialSection[:-1]

        if len(potentialSection) > 2 and potentialSection[-2:] == '.0':
            potentialSection = potentialSection[:-2]

        if "..." in line or ". ." in line:
            self.tocSections.append(potentialSection)
            return None, False

        if potentialSection not in self.knownSections:
            if potentialSection not in self.tocSections:
                return None, True

        if potentialSection == '1':
            self.addToKnowns('a')

        frags = potentialSection.split('.')
        if len(frags) > 1 and self.sectionIndent is None:
            if frags[-1] == '1':
                self.sectionIndent = lpad
            else:
                self.sectionIndent = 0
        if len(frags) > 1:
            if self.sectionIndent and lpad != self.sectionIndent * (len(frags)-1):
                return None, True
            elif self.sectionIndent is None:
                return None, True
        elif lpad > 0:
            return None, True

        if potentialSection in self.knownSections:
            self.knownSections.remove(potentialSection)

        # Remove items from the known set
        # For X.Y
        # Remove X.Y
        # Remove X.Y-1(.*)
        # Remove X-1.(.*)

        # Add items to the known set
        # For X.Y add
        # X+1 for Y=0 or absent
        # X.Y+1
        # X.Y.1to-
        # if X is "Appendix Z"
        # Appendix Z+1
        # Z.1

        if frags[-1] == '':
            frags = frags[:-1]
        if frags[-1] == '0':
            frags = frags[:-1]

        # --> X+1
        self.addToKnowns(increment(frags[0]))

        frags.append("1")
        self.addToKnowns('.'.join(frags))
        if len(frags) > 2:
            frags2 = frags[:-2]
            frags2.append(increment(frags[-2]))
            self.addToKnowns('.'.join(frags2))

        match = re.match("appendix (\w)", potentialSection)
        if match:
            frags = [match.group(1), '1']
            self.addToKnowns('.'.join(frags))
        return potentialSection, True

    def sectionDocument(self):
        # doc = SectionDocument(self.source)
        # doc.getSections()
        # self.allSections = doc.sectionList
        # self.sectionDict = doc.sectionDict

        self.sectionIndent = None
        self.centeredTitle = None

        self.knownSections = ["status of this memo", "copyright notice", "abstract",
                              "table of contents",
                              "1", "appendix a", "appendix i", "appendix 1",
                              "acknowledgements", "acknowledgments", "authors' addresses",
                              "author's address",
                              "full copyright statement", "security considerations"]
        self.currentSection = "Header"
        self.startLine = 0
        self.allSections = []
        self.sectionDict = {}
        self.tocSections = []

        i = 0
        while i < len(self.source):
            if self.source[i].strip() == "":
                i += 1
                continue

            isStart, skipBlock = self.isSectionStart(self.source[i], i)
            if isStart:
                self.currentSection = self.currentSection.lower().rstrip()
                self.currentSection = html.unescape(self.currentSection)
                self.allSections.append(self.currentSection)
                self.sectionDict[self.currentSection] = self.source[self.startLine:i]
                self.currentSection = isStart
                self.startLine = i

            if not skipBlock:
                i += 1
            else:
                while i < len(self.source) and self.source[i].strip() != "":
                    i += 1

        self.currentSection = self.currentSection.lower().rstrip()
        self.currentSection = html.unescape(self.currentSection)
        self.allSections.append(self.currentSection)
        self.sectionDict[self.currentSection] = self.source[self.startLine:i]

    def unsectionDocument(self):
        sections = []
        for header in self.allSections:
            sections.extend(self.sectionDict[header])
        self.source = sections

    def buildPattern(self, string):
        #  \u200C   ---- 3312 - RFC 5892  --- zero width non-joiner
        #  \u2019   ---- 5178 - RFC 6218  --- right single quotation mark
        #  \u2026   ---- 1117 - RFC 4601  --- ... - ellipsis
        #  \u2014   ---- 4483 - RFC 5952  --- emdash

        # string = string.replace("\u2018", "'").replace("\u201B", "'"). \
        #          replace('\u2019', "'").replace('\u201C', "'"). \
        #          replace("\u201C", '"').replace('\u201D', '"')

        string = html.escape(string)

        for ch in ['\\', '`', '*', '+', '?', '(', ')', '=', '[', ']', '.', '{', '}', '~', '$', '^', '|']:
            if ch in string:
                string = string.replace(ch, "\\"+ch)

        string = string.strip()
        string = re.sub(r"\s+", r"\\s+", string)
        return re.compile(string)

    def buildPattern2(self, item):
        oldTextIn = item["orig_text"]
        if "..." in oldTextIn:
            m = re.match("\s*\[?\.\.\.\]?", oldTextIn)
            if m:
                oldTextIn = oldTextIn[m.end():]
            m = re.search("\[?\.\.\.\]?\s*$", oldTextIn)
            if m:
                oldTextIn = oldTextIn[:m.start()]

        lines, a = strip_pagebreaks(oldTextIn)
        if len(lines) == 0:
            return None

        newLines = []

        for line in lines:
            line = line.txt
            if len(line) > 0 and line[0] == '|':
                line = line[1:]
            match = re.match("[\s\^v]+$", line)
            if match:
                line = None
            else:
                newLines.append(line)

        oldText = "\n".join(newLines)

        newText = item["correct_text"]
        if newText:
            lines, a = strip_pagebreaks(newText)
            newLines = []
            for line in lines:
                line = line.txt
                if len(line) > 0 and line[0] == '|':
                    line = line[1:]
                match = re.match("[\s\^v]+$", line)
                if match:
                    line = None
                else:
                    newLines.append(line)

            newText = "\n".join(newLines)

        if oldText == oldTextIn:
            return None

        if newText and "..." in newText:
            m = re.match("\s*\[?\.\.\.\]?", newText)
            if m:
                newText = newText[m.end():]
            m = re.search("\[?\.\.\.\]?\s*$", newText)
            if m:
                newText = newText[:m.start()]

        item["orig_text2"] = oldText
        item["correct_text2"] = newText
        return self.buildPattern(oldText)

    def buildInlines(self, section, itemList, searchText):

        # print("section = {0}   Number of inserts = {1}".format(section, len(itemList)))
        # print("original:\n{0}".format(searchText))

        if len(itemList) == 0:
            return

        editedText = searchText
        locations = []

        for item in itemList:
            match = re.search(item[2], editedText)
            # print("pattern = {0}".format(item[2]))
            if match:
                if "orig_text2" in item[0]:
                    newText = item[0]["orig_text2"] if item[0]["status_tag"] == "Rejected" else item[0]["correct_text2"]
                else:
                    newText = item[0]["orig_text"] if item[0]["status_tag"] == "Rejected" else item[0]["correct_text"]
                if not newText:
                    newText = ""
                newText = html.escape(newText)
                editedText = editedText[:match.start()] +  \
                    self.inlineFormat.format(item[0]["status_tag"], item[0]["errata_id"], newText) + \
                    editedText[match.end():]
                locations.append([item, item[1].start(), item[1].end(), False])

        sectionLines = editedText.splitlines(True)

        for item in itemList:
            for loc in locations:
                if item == loc[0]:
                    item[3] = True
                    endTag = re.compile("id__locate={0}".format(item[0]["errata_id"]))
                    startTag = re.compile("inline-{0}".format(item[0]["errata_id"]))
                    self.header = self.header.replace("#eid{0}".format(item[0]["errata_id"]),
                                                      "#btn_{0}".format(item[0]["errata_id"]))
                    for line in range(len(sectionLines)):
                        m = startTag.search(sectionLines[line])
                        if m:
                            sectionLines[line] = sectionLines[line][:-1] + self.buttonFormat.format(item[0]["errata_id"])
                        m = endTag.search(sectionLines[line])
                        if m:
                            sectionLines.insert(line+1, self.inlineExpandWrapper.format(item[0]["errata_id"],
                                                                                self.substitute(self.templates.inlineNote, item[0])))
                            break

        button = re.compile("Expand</button")
        for item in itemList:
            if item[3]:
                continue
            for locate in locations:
                if locate[1] > item[1].end() or locate[2] < item[1].start():
                    continue
                expandTag = re.compile("id='expand_{0}".format(locate[0][0]["errata_id"]))
                self.header = self.header.replace("#eid{0}".format(item[0]["errata_id"]),
                                                  "#btn_{0}".format(locate[0][0]["errata_id"]))
                for line in range(len(sectionLines)):
                    m = expandTag.search(sectionLines[line])
                    if m:
                        if sectionLines[line][-6:] != "
":
                            print("Internal Error")
                            break
                        sectionLines[line] = sectionLines[line][:-6] + \
                            self.substitute(self.templates.inlineNote, item[0]) + "
"
                        break
                    m = button.search(sectionLines[line])
                    if m:
                        sectionLines[line] = sectionLines[line][:m.start()] + \
                                                "Expand Multiple</button" + \
                                                sectionLines[line][m.end():]

        # print("**result:\n{0}**".format("".join(sectionLines)))
        self.sectionDict[section] = sectionLines

    def createInlineNotes(self):

        combinedLines = {}
        combinedLine = ""
        for section in self.sectionDict:
            combinedLines[section] = html.escape("".join(self.sectionDict[section]))
            self.sectionDict[section] = combinedLines[section].splitlines(True)

        if self.options.search:
            for item in self.toApply:
                section = item["section2"]
                if section in self.allSections:
                    continue
                if section[:9] == "appendix ":
                    test = section[9:]
                    if test in self.allSections:
                        item["section2"] = test
                        continue
                if len(section) > 0 and section[0].isalpha():
                    test = "appendix " + section
                    if test in self.allSections:
                        item["section2"] = test
                        continue

                if not item["orig_text"]:
                    continue

                pattern = self.buildPattern(item["orig_text"])
                pattern2 = self.buildPattern2(item)
                for section in self.sectionDict:
                    match = pattern.search(combinedLines[section])
                    if match:
                        item["section2"] = section
                        break

                    if pattern2:
                        match = pattern2.search(combinedLines[section])
                        if match:
                            item["section2"] = section
                            break

        ids = sorted(self.toApply, key=inlineSectionKey)

        putInline = []
        lastSection = ids[0]["section2"]

        for id in ids:
            item = id

            # nothing to be matched
            if not item["orig_text"]:
                continue

            section = item["section2"]
            if section not in self.allSections:
                continue

            if section != lastSection:
                self.buildInlines(lastSection, putInline, combinedLine)
                lastSection = section
                putInline = []

            combinedLine = combinedLines[section]
            pattern = self.buildPattern(item["orig_text"])

            match = pattern.search(combinedLine)
            if match:
                # print("*** FOUND PATTERN - section {0}".format(section))
                # print(pattern)
                self.toApply.remove(item)
                self.InlineCount += 1
                putInline.append([item, match, pattern, False])
            else:
                pattern2 = self.buildPattern2(item)
                if pattern2 is None:
                    continue
                match = pattern2.search(combinedLine)
                if match:
                    self.toApply.remove(item)
                    self.InlineCount += 1
                    putInline.append([item, match, pattern2, False])
                else:
                    # print("*** FAILED PATTERN - section {0}".format(section))
                    # print(pattern)
                    # print(item["orig_text"])
                    pass

        self.buildInlines(lastSection, putInline, combinedLine)

    def createSectionNotes(self):
        # M00BUG title continutations

        ids = sorted([item["errata_id"] for item in self.toApply], reverse=True)
        byId = dict((item["errata_id"], item) for item in self.toApply)

        for id in ids:
            item = byId[id]
            if not item["section2"] in self.allSections:
                continue

            if False:
                x = item["orig_text"].splitlines()
                print("{0}: {1}\n    {2}\n".format(item["errata_id"], x[0], x[1]))

            text = self.substitute(self.templates.sectionNote, item)

            section = self.sectionDict[item["section2"]]
            section = section[:2] + [text] + section[2:]
            self.sectionDict[item["section2"]] = section
            self.toApply.remove(item)

            self.SectionCount += 1

    def createFootnotes(self):
        errataFooter = ""

        ids = sorted(self.toApply, key=sortById)

        for item in ids:
            if False:
                x = item["orig_text"].splitlines()
                if len(x) >= 2:
                    print("{0}: {1}\n    {2}\n".format(item["errata_id"], x[0], x[1]))

            errataFooter += self.substitute(self.templates.endNote, item)
            self.EndnoteCount += 1

        self.errataFooter = errataFooter

    def buildHeader(self):
        anchorTemplate = " EID {0}, "

        ids = sorted(self.toApply, key=sortById)

        errataHeader = {}
        for item in ids:
            code = item["status_tag"]
            item["errata_id"] = str(item["errata_id"])

            if code not in errataHeader:
                errataHeader[code] = {"status_tag": code, "errata_list": ""}

            errataHeader[code]["errata_list"] += anchorTemplate.format(item["errata_id"])

        header = ""
        for keys in ["Verified", "Held", "Reported", "Rejected"]:
            if keys in errataHeader:
                errataHeader[keys]["errata_list"] = errataHeader[keys]["errata_list"][:-2]
                header += self.templates.headNote.substitute(errataHeader[keys])
        self.header = header

    def substitute(self, template, fields):
        fields2 = {}
        for k in fields:
            if isinstance(fields[k], str):
                lines = fields[k].splitlines()
                lines = [html.escape(line) for line in lines]
                fields2[k] = '\n'.join(lines)
                if k == "notes":
                    fields2["notes"] = fields2["notes"].replace("\n", "
")
            else:
                fields2[k] = fields[k]
        return template.substitute(fields2)

    def emitHtml(self):

        fields = {}
        fields["HEADER"] = self.header
        fields["PATH"] = self.state["ossPath"]
        fields["BODY"] = "".join(self.source)
        fields["FOOTER"] = self.errataFooter
        fields["TITLE"] = self.title

        with open(os.path.join(self.state["html"], self.RfcName + ".html"), "w", encoding='utf-8') as f:
            f.write(self.templates.html.substitute(fields))