from datetime import datetime
from pytz import FixedOffset

from vilya.libs.consts import LANGUAGES
from vilya.libs.store import cache, ONE_DAY
from vilya.models.user import get_author_by_email
from .consts import conf
from .utils import getcommitrange, getkeyssortedbyvaluekey
from .collector import DataCollector

REPO_EXTENSION_KEY = "repo:%s:stats:extension1"

class GitDataCollector(DataCollector):

    def __init__(self, gyt_repo):
        super(GitDataCollector, self).__init__()
        self.gyt_repo = gyt_repo

    @classmethod
    def _get_author_email(cls, author_email):
        author, email = author_email.split('<', 1)
        author = author.rstrip()
        email = email.rstrip('>')
        return (author, email)

    def commitrange(self, head, end=None):
        return getcommitrange(head, end)

    def fill_shortstat(self, commitrange):
        try:
            to_ref, from_ref = commitrange
            block_of_lines = self.gyt_repo.repo.log(to_ref,
                                                    from_ref=from_ref,
                                                    shortstat=True,
                                                    no_merges=True,
                                                    reverse=True
                                                    )
        except:
            block_of_lines = []
        return block_of_lines

    @cache(REPO_EXTENSION_KEY % '{proj_name}', expire=ONE_DAY)
    def compute_file_size_and_extensions(self, proj_name):
        # extensions and size of files
        # this should only run once than only compute delta
        extensions = {}
        total_size = 0
        total_files = 0
        source_files = 0
        source_lines = 0
        lines = self.gyt_repo.repo.ls_tree('HEAD', recursive=True, size=True)
        for line in lines:
            if line[0] == '160000' and line[3] == '-':
                # skip submodules
                continue
            sha1 = line[2]
            size = int(line[3])
            fullpath = line[4]

            total_size += size
            total_files += 1

            filename = fullpath.split('/')[-1]  # strip directories
            if filename.find('.') == -1 or filename.rfind('.') == 0:
                ext = ''
            else:
                ext = filename[(filename.rfind('.') + 1):]
            if len(ext) > conf['max_ext_length']:
                ext = ''
            name = LANGUAGES.get(ext, None)

            if name not in extensions:
                if ext in LANGUAGES.keys():
                    name = LANGUAGES[ext]
                    extensions[name] = {'files': 0, 'lines': 0}
                else:
                    continue

            extensions[name]['files'] += 1
            source_files += 1
            try:
                # should be text files
                count = self.getLinesInBlob(sha1)
                extensions[name]['lines'] += count
                source_lines += count
            except:
                pass
        return extensions, total_files, total_size, source_files, source_lines

    def fill_rev_list_commitrange(self, commitrange):
        to_ref, from_ref = commitrange
        commits = self.gyt_repo.repo.rev_list(to_ref, from_ref)
        for commit in commits:
            author = commit.author.name
            # email = email_normalizer(commit.author.name,
            #                          commit.author.email)
            email = commit.author.email
            stamp = commit.committer.time
            date = datetime.fromtimestamp(
                commit.committer.time,
                FixedOffset(commit.committer.offset)
            )

            author = get_author_by_email(email, author)
            if author in conf['merge_authors']:
                author = conf['merge_authors'][author]

            # First and last commit stamp
            # (may be in any order because of cherry-picking and patches)
            if stamp > self.last_commit_stamp:
                self.last_commit_stamp = stamp
            if self.first_commit_stamp == 0 or stamp < self.first_commit_stamp:
                self.first_commit_stamp = stamp

            # yearly/weekly activity
            yyw = date.strftime('%Y-%W')
            self.year_week_act[yyw] += 1
            if self.year_week_act_peak < self.year_week_act[yyw]:
                self.year_week_act_peak = self.year_week_act[yyw]

            # author stats
            if author not in self.authors:
                self.authors[author] = {
                    'lines_added': 0,
                    'lines_removed': 0,
                    'commits': 0,
                }
            # commits, note again that commits may be in any date order
            # because of cherry-picking and patches
            if 'last_commit_stamp' not in self.authors[author]:
                self.authors[author]['last_commit_stamp'] = stamp
            if stamp > self.authors[author]['last_commit_stamp']:
                self.authors[author]['last_commit_stamp'] = stamp
            if 'first_commit_stamp' not in self.authors[author]:
                self.authors[author]['first_commit_stamp'] = stamp
            if stamp < self.authors[author]['first_commit_stamp']:
                self.authors[author]['first_commit_stamp'] = stamp
            if 'email' not in self.authors[author]:
                self.authors[author]['email'] = email

    def fill_short_stats_commitrange(self, commitrange):
        to_ref, from_ref = commitrange
        if to_ref == 'HEAD' and from_ref is None:
            total_lines = 0
        else:
            total_lines = self.total_lines
        for commit in self.fill_shortstat(commitrange):
            files = commit['files']
            inserted = commit['additions']
            deleted = commit['deletions']
            total_lines += inserted
            total_lines -= deleted
            self.total_lines_added += inserted
            self.total_lines_removed += deleted
            stamp = commit['committer_time']
            author = commit['author_name']
            email = commit['author_email']
            author = get_author_by_email(email, author)
            if author in conf['merge_authors']:
                author = conf['merge_authors'][author]
            self.changes_by_date[stamp] = {
                'files': files,
                'ins': inserted,
                'del': deleted,
                'lines': total_lines
            }
            self.process_line_user(author, stamp, inserted, deleted)
        self.total_lines = total_lines

    def collect(self, dir, proj_name, head, n_author):
        DataCollector.collect(self, dir)
        self.loadCache(proj_name, '/stats/' + proj_name, n_author)
        last_sha = self.cache and self.cache.get('last_sha', '')
        if last_sha:
            commitrange = self.commitrange('HEAD', last_sha)
        else:
            commitrange = self.commitrange('HEAD')

        self.total_authors += n_author
        self.fill_rev_list_commitrange(commitrange)

        # TODO Optimize this, it's the worst bottleneck
        # outputs "<stamp> <files>" for each revision
        to_ref, from_ref = commitrange
        revlines = self.gyt_repo.repo.rev_list(to_ref, from_ref=from_ref)
        for commit in revlines:
            timest = commit.author.time
            rev = commit.tree.hex
            linecount = self.getFilesInCommit(rev)
            self.files_by_stamp[int(timest)] = int(linecount)
        self.total_commits += len(revlines)

        extensions, total_files, total_size, source_files, source_lines \
            = self.compute_file_size_and_extensions(proj_name)

        self.extensions = extensions
        self.total_files = total_files
        self.total_size = total_size
        self.source_files = source_files
        self.source_lines = source_lines

        self.fill_short_stats_commitrange(commitrange)

        self.refine()
        # here update new data after head sha

        # here need to save to cache up to head sha
        self.cache['last_sha'] = head
        self.saveCache(proj_name, '/stats/' + proj_name)

    def process_line_user(self, author, stamp, inserted, deleted):
        if author not in self.authors:
            self.authors[author] = {
                'lines_added': 0,
                'lines_removed': 0,
                'commits': 0,
            }
        self.authors[author]['commits'] += 1
        self.authors[author]['lines_added'] += inserted
        self.authors[author]['lines_removed'] += deleted
        if stamp not in self.changes_by_date_by_author:
            self.changes_by_date_by_author[stamp] = {}
        if author not in self.changes_by_date_by_author[stamp]:
            self.changes_by_date_by_author[stamp][author] = {}
        linesadd = self.authors[author]['lines_added']
        commits_n = self.authors[author]['commits']
        self.changes_by_date_by_author[stamp][author]['lines_added'] = linesadd
        self.changes_by_date_by_author[stamp][author]['commits'] = commits_n

    def refine(self):
        # authors
        # name -> {place_by_commits, commits_frac, date_first, date_last,
        # timedelta}
        self.authors_by_commits = getkeyssortedbyvaluekey(
            self.authors, 'commits')
        self.authors_by_commits.reverse()  # most first
        for i, name in enumerate(self.authors_by_commits):
            self.authors[name]['place_by_commits'] = i + 1

        for name in self.authors.keys():
            a = self.authors[name]
            a['commits_frac'] = (
                100 * float(a['commits'])) / self.getTotalCommits()
            date_first = datetime.fromtimestamp(a['first_commit_stamp'])
            date_last = datetime.fromtimestamp(a['last_commit_stamp'])
            delta = date_last - date_first
            a['date_first'] = date_first.strftime('%Y-%m-%d')
            a['date_last'] = date_last.strftime('%Y-%m-%d')
            a['timedelta'] = delta
            if 'lines_added' not in a:
                a['lines_added'] = 0
            if 'lines_removed' not in a:
                a['lines_removed'] = 0

    def getActiveDays(self):
        return self.active_days

    def getActivityByDayOfWeek(self):
        return self.d_of_week_act

    def getActivityByHourOfDay(self):
        return self.h_of_day_act

    def getAuthorInfo(self, author):
        return self.authors[author]

    def getAuthors(self, limit=None):
        res = getkeyssortedbyvaluekey(self.authors, 'commits')
        res.reverse()
        return res[:limit]

    def getCommitDeltaDays(self):
        return (self.last_commit_stamp / 86400 - self.first_commit_stamp / 86400) + 1  # noqa

    def getFilesInCommit(self, rev):
        try:
            res = self.cache['files_in_tree'][rev]
        except:
            res = len(self.gyt_repo.repo.ls_tree(rev,
                                                 recursive=True,
                                                 name_only=True))
            if 'files_in_tree' not in self.cache:
                self.cache['files_in_tree'] = {}
            self.cache['files_in_tree'][rev] = res

        return res

    def getFirstCommitDate(self):
        return datetime.fromtimestamp(self.first_commit_stamp)

    def getLastCommitDate(self):
        return datetime.fromtimestamp(self.last_commit_stamp)

    def getLinesInBlob(self, sha1):
        try:
            res = self.cache['lines_in_blob'][sha1]
        except:
            res = len(self.gyt_repo.repo.cat_file(sha1).split('\n'))
            if 'lines_in_blob' not in self.cache:
                self.cache['lines_in_blob'] = {}
            self.cache['lines_in_blob'][sha1] = res
        return res

    def getTotalAuthors(self):
        return self.total_authors

    def getTotalCommits(self):
        return self.total_commits

    def getTotalFiles(self):
        return self.total_files

    def getTotalLOC(self):
        return self.total_lines

    def getTotalSize(self):
        return self.total_size


import streamlit as st
import pandas as pd
import pyarrow.parquet as pq

from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

header         = st.container()
dataset        = st.container()
features       = st.container()
model_training = st.container()

st.markdown(
    """
    <style>
    .main {
    background-color: #F5F5F5;
    }
    </style>
    """,
    unsafe_allow_html=True
)

@st.cache_data
def get_data(filename):
    taxi_data = pq.read_table(filename)
    taxi_data = taxi_data.to_pandas()
    return taxi_data

with header:
    st.title('Welcome to my awesome data science project!')

with dataset:
    st.header('NYC taxi dataset')
    st.text("I found this dataset at: https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page")

    taxi_data = get_data('Data/taxi_data.parquet')
    taxi_data = taxi_data.head(100000)
    st.write(taxi_data.head())

    st.subheader('Pick-up location ID distribution on the NYC dataset')
    pulocation_dist = pd.DataFrame(taxi_data['PULocationID'].value_counts()).head(50)
    st.bar_chart(pulocation_dist)

with features:
    st.header('The features I created')

    st.markdown('* **first feature:** I created this feature because of this... I calculated it using this log...')
    st.markdown('* **second feature:** I created this feature because of this... I calculated it using this log...')

with model_training:
    st.header('Time to train the model!')
    st.text("Here you get to choose the hyperparameters of the model and see how the performance changes.")

    sel_col, disp_col = st.columns(2)

    max_depth = sel_col.slider('Whats should the max_depth of the model be?', min_value=10, max_value=100, value=20, step=10)

    n_estimators = sel_col.selectbox('How many trees should there be?', options=[100,200,300,'No limit'], index=0)

    input_feature = sel_col.selectbox('Which feature should be used as the input feature', taxi_data.keys())

    if n_estimators == 'No limit':
        regr = RandomForestRegressor(max_depth=max_depth)
    else:
        regr = RandomForestRegressor(max_depth=max_depth, n_estimators=n_estimators)

    x = taxi_data[[input_feature]].values
    y = taxi_data[['trip_distance']].values

    regr.fit(x,y)
    prediction = regr.predict(y)

    disp_col.subheader('Mean absolute error of the model is: ')
    disp_col.write(mean_absolute_error(y, prediction))

    disp_col.subheader('Mean squared error of the model is:')
    disp_col.write(mean_squared_error(y, prediction))

    disp_col.subheader('R squared score of the model is:')
    disp_col.write(r2_score(y, prediction))


#!/usr/bin/env python

from nornir import InitNornir
from nornir.core.filter import F
from nornir.plugins.tasks.networking import netmiko_send_command
from nornir.plugins.functions.text import print_result

DEFAULT_GATEWAY = ("10.220.88.1")

def exercise_3():
    """Get ip arp and format results
         takeaways: multigroup filters, netmiko_send_command, filtering parsed results
    """
    ios_filt = F(groups__contains="ios")
    eos_filt = F(groups__contains="eos")
    nr = InitNornir(config_file="config.yaml")
    nr = nr.filter(ios_filt | eos_filt)
    cmd = "show ip arp"
    result = nr.run(
        task=netmiko_send_command,
        command_string=cmd
    )

    parsed_res = []

    # get just the DG line
    for host, dd in result.items():
        output = dd[0].result
        for line in output.splitlines():
            if DEFAULT_GATEWAY in line:
                parsed_res.append({host:line}) # this is better done with a tuple, see exercise4

    # print table-like result
    # this is better done with a tuple, see exercise4
    for res in parsed_res:
        for k,v in res.items():
            print(f"Host: {k} || Gateway: {v}")

def main():
    exercise_3()

if __name__=="__main__":
    main()


import os
import json

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from skimage.io import imread
from skimage.transform import resize

from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, accuracy_score

# set paths
DIR_DATA_LOCAL = 'data/COVID-19_Radiography_Dataset/'  # set to local path
DIR_DATA_COLAB = 'data/'  # path in Google Drive

LST_GROUP = ['covid', 'normal', 'viral', 'opac']
LST_FOLDERS = ['COVID', 'Normal', 'Viral Pneumonia', 'Lung_Opacity']
FNAME_MAPPER = dict(zip(LST_GROUP, LST_FOLDERS))
LABEL_MAPPER = dict(zip(LST_GROUP, range(4)))

DIRS = dict(zip(LST_GROUP, [os.path.join(DIR_DATA_LOCAL, folder) for folder in LST_FOLDERS]))
DIR_IMAGES = dict(zip(LST_GROUP, [os.path.join(dir, 'images') for dir in DIRS.values()]))
DIR_MASKS = dict(zip(LST_GROUP, [os.path.join(dir, 'masks') for dir in DIRS.values()]))
DIR_OUTPUT = "projet_radio_covid/results/"

NUM_ALL_IMG = [3616, 10192, 1345, 6012]

# naming / paths
def get_fname(group, idx):
    return (f"{FNAME_MAPPER[group]}-{idx}.png")

def get_dir_pred(clf_name):
    dir_pred = os.path.join(DIR_OUTPUT, 'pred_' + clf_name)
    if not os.path.exists(dir_pred):
        os.mkdir(dir_pred)
        print(dir_pred, "folder created")
    return dir_pred

def get_path_metric(clf_name):
    return os.path.join(DIR_OUTPUT, 'metrics_' + clf_name + '.json')

def get_str_param(dict_params):
    """
    Transform a dictionary of parameters into a single string.
    :param dict_params: dict: parameters of a function
    :return: str: string of the parameters' name and value
    """
    str_param = ''
    if dict_params is None:
        return "default"
    else:
        for key, value in dict_params.items():
            if type(value) in [list, tuple]:  # convert list or tuple to str, using "_" to connect the elements
                str_param += '_' + key + '_' + "_".join([str(v) for v in value]) if len(value) > 1 else value
            else:
                str_param += ('_' + key + '_' + str(value))
        str_param = str_param.replace('True', 't')  # to make the string shorter: True -> t, False -> f
        str_param = str_param.replace('False', 'f')
        str_param = str_param.replace('.', 'd')  # replace dot of decimal by "d"
        str_param = str_param.replace('e-', 'e')  # remove '-' in scientific notation e.g. 5.3e-3
        str_param = str_param.replace(' ', '')
        return str_param[1:].lower()

# results saving
def save_pred(y_test, y_pred, clf_name, preprocess_params, clf_params):
    """
    save the prediction results to google drive
    """
    dir_pred = get_dir_pred(clf_name)
    if not os.path.exists(os.path.join(dir_pred, get_str_param(preprocess_params))):
        os.mkdir(os.path.join(dir_pred, get_str_param(preprocess_params)))

    pd.DataFrame({'y_test': y_test, 'y_pred': y_pred}).to_csv(
        os.path.join(dir_pred, get_str_param(preprocess_params), get_str_param(clf_params) + '.txt'),
        sep='\t', header=True, index=False)
    return None

def get_metrics(y_test, y_pred, clf_params, preprocess_params):
    """
    Get the metrics of interests and return them in a dictionary.
    :param y_test: True values of the labels of the test set
    :param y_pred: predicted labels of the test set
    :param clf_params: dict: parameters of the classifier
    :param preprocess_params: dict: parameters of the preprocessing
    :return: evaluation metrics as a dictionary
    """
    # Calculate the evaluation metrics
    precision = precision_score(y_test, y_pred, average=None)
    recall = recall_score(y_test, y_pred, average=None)
    f1 = f1_score(y_test, y_pred, average=None)
    accuracy = accuracy_score(y_test, y_pred)
    confusion_matrix_array = confusion_matrix(y_test, y_pred)

    # Store the metrics in a dictionary
    results = {
        'preprocess_params': get_str_param(preprocess_params),
        "clf_params": get_str_param(clf_params),
        "precision": precision.tolist(),
        "recall": recall.tolist(),
        "f1": f1.tolist(),
        "accuracy": accuracy.tolist(),
        "confusion_matrix": confusion_matrix_array.tolist()
    }

    return results

def save_metrics(y_test, y_pred, clf_name, preprocess_params, clf_params=None):
    """
    get and save the metrics in google drive
    """
    path_metric = get_path_metric(clf_name)
    new_result = get_metrics(y_pred, y_test, clf_params, preprocess_params)

    if not os.path.exists(path_metric):
        print("new file", path_metric, "created")
        with open(path_metric, 'w') as fo:
            json.dump({}, fo)

    try:
        with open(path_metric, 'r') as fi:
            all_results = json.load(fi)
    except:
        all_results = {}

    if all_results is None:
        all_results = {}

    all_results.update({len(all_results): new_result})

    with open(path_metric, 'w') as fo:
        json.dump(all_results, fo)

# results loading as format of dataframe
def load_summary_metrics(lst_clf_names, lst_str_clf_params, str_preprocess_param):
    """
    load the results of the metrics of previous training and present as a dataframe
    :param lst_clf_names: list of strs
    :param lst_str_clf_params: list of strings of parameters for each classifier
    :param str_preprocess_param: list of strings of preprocessing parameters
    :return: a dataframe with clf_name as columns and metrics as rows
    """
    recall_covid = []
    recall_sick = []
    precision_normal = []
    precision_non_covid = []
    mean_recall = []
    mean_precision = []
    mean_f1 = []
    accuracy = []

    assert len(lst_str_clf_params) == len(lst_clf_names)
    for i in range(len(lst_clf_names)):
        clf_name = lst_clf_names[i]
        str_clf_param = lst_str_clf_params[i]
        try:
            with open(get_path_metric(clf_name), 'r') as fi:
                all_metrics = json.load(fi)
        except:
            print(clf_name, "results are missing")

        metric = [value for key, value in all_metrics.items() if
                  (value['preprocess_params'] == str_preprocess_param) and (
                          value['clf_params'] == str_clf_param)]
        if len(metric) == 0:
            print(clf_name, "results are missing for given parameters")
            break
        else:
            # keep only the lastest result in case the same experiment was repeated.
            metric = metric[-1]

        recall_covid.append(metric['recall'][0])
        recall_sick.append((metric['recall'][0] + sum(metric['recall'][2:])) / 3)
        precision_non_covid.append(np.mean(metric['precision'][1:]))
        precision_normal.append(metric['precision'][1])
        mean_recall.append(np.mean(metric['recall']))
        mean_precision.append(np.mean(metric['precision']))
        mean_f1.append(np.mean(metric['f1']))
        accuracy.append(metric['accuracy'])

    df_summary_metric = pd.DataFrame({
        "recall_covid": recall_covid,
        'recall_sick': recall_sick,
        "precision_non_covid": precision_non_covid,
        "precision_normal": precision_normal,
        "mean_recall": mean_recall,
        "mean_precision": mean_precision,
        'mean_f1': mean_f1,
        "accuracy": accuracy
    }).T.round(2)
    df_summary_metric.columns = lst_clf_names
    return df_summary_metric


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 20 10:47:48 2021
@author: User
"""

# Note: There is currently no binary wheel for Fiona and Rasterio that are
# available on Pypi. So if using a pip installed version of Python, you need
# to install, in the right order, the following packages using wheels from
# Christopher Gohlke’s website.

# (1) https://www.lfd.uci.edu/~gohlke/pythonlibs/#gdal
# (2) https://www.lfd.uci.edu/~gohlke/pythonlibs/#fiona
# (3) https://www.lfd.uci.edu/~gohlke/pythonlibs/#shapely
# (4) https://www.lfd.uci.edu/~gohlke/pythonlibs/#rasterio

# You will also need a numpy version >= 1.20.0 for contextily to work properly.
# Numpy can be installed directly from PyPi. There is no need to use a wheel
# from Christopher Gohlke’s website.

import os.path as osp
import fiona
import geopandas as gpd
from shapely.geometry import Point

lat_ddeg = 46.40819
lon_ddeg = -70.3709

# https://www.donneesquebec.ca/recherche/dataset/decoupages-administratifs
dirname = "C:/Users/User/rsesq-bulletin"
gdbfile = "SDA_ 2018-05-25 .gdb.zip"

list_layers = fiona.listlayers(osp.join(dirname, gdbfile))

munic_s = gpd.read_file(
    osp.join(dirname, gdbfile), driver='FileGDB', layer='munic_s')

def get_region_mrc_municipality_at(lat_ddeg, lon_ddeg, munic_geometry):
    loc_point = Point(lon_ddeg, lat_ddeg)

    contains = munic_s[munic_s['geometry'].contains(loc_point)]
    region = contains.iloc[0]['MUS_NM_REG']
    mrc = contains.iloc[0]['MUS_NM_MRC']
    municipality = contains.iloc[0]['MUS_NM_MUN']

    return region, mrc, municipality

# %%
from sardes.database.accessors import DatabaseAccessorSardesLite
database = "D:/Desktop/rsesq_prod_28-06-2021.db"
accessor = DatabaseAccessorSardesLite(database)
accessor.connect()

obs_wells = accessor.get_observation_wells_data()
for index, obswell_data in obs_wells.iterrows():
    region, mrc, municipality = get_region_mrc_municipality_at(
        obswell_data['latitude'],
        obswell_data['longitude'],
        munic_s['geometry'])

    if obswell_data['municipality'] != municipality:
        print('| {} | {} | {} | {} | {} |'.format(
            obswell_data['obs_well_id'],
            obswell_data['municipality'],
            municipality,
            obswell_data['latitude'],
            obswell_data['longitude']
            ))

accessor.close_connection()


import unittest

import requests
import responses

class TestCase(unittest.TestCase):

  @responses.activate
  def testExample(self):
    responses.add(**{
      'method'         : responses.GET,
      'url'            : 'http://example.com/api/123',
      'body'           : '{"error": "reason"}',
      'status'         : 404,
      'content_type'   : 'application/json',
      'adding_headers' : {'X-Foo': 'Bar'}
    })

    response = requests.get('http://example.com/api/123')

    self.assertEqual({'error': 'reason'}, response.json())
    self.assertEqual(404, response.status_code)


import requests
import random
import lxml.etree as le
import xlwt

proxies_pool=[
    {'http':'202.109.157.61:9000'},
    {'http':'222.74.73.202:42055'},
    {'http':'202.109.157.60:9000'},
    {'http':'27.42.168.46:55481'},
    {'http':'182.139.111.149:9000'}
]
proxies=random.choice(proxies_pool)

headers={
    'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36 Edg/108.0.1462.76'
}

#url='https://m.ke.com/wh/loupan/p_whesscblhhv/'

url='https://m.ke.com/wh/loupan/'
resps=requests.get(url=url,headers=headers,proxies=proxies)
print(resps.status_code)
resps.encoding=resps.apparent_encoding
cont=resps.text

tr=le.HTML(cont)
page_link=tr.xpath('//div[@class="shell-resblock-card"]/a/@href')
#print(page_link)

title_data=[]
advantage_data=[]
price_data=[]
size_s_data=[]
type_s_data=[]
structure_data=[]
address_data=[]

for x in range(len(page_link)):
    link=page_link[x]
    new_link='https://m.ke.com'+link
    #print(new_link)
    #print(len(page_link))

    resp=requests.get(url=new_link,headers=headers,proxies=proxies)
    #print(resp.status_code)
    resp.encoding=resp.apparent_encoding
    con=resp.text

    tr=le.HTML(con)
    title=tr.xpath('//div[@class="title-wrapper resblock-name-line"]/h1/text()')
    advantage=tr.xpath('//div[@class="tag-wrapper"]/span/text()')
    price=tr.xpath('//*[@class="price-value item"]/text()')
    size_s=tr.xpath('//div[@class="address-value item"]/text()')[0]
    type_s=tr.xpath('//div[@class="address-value item"]/text()')[1]
    structure=tr.xpath('//div[@class="address-value item"]/text()')[2]
    address=tr.xpath('//div[@class="address-open-date"]/a/text()')

    title_data.append(title)
    advantage_data.append(advantage)
    price_data.append(price)
    size_s_data.append(size_s)
    type_s_data.append(type_s)
    structure_data.append(structure)
    address_data.append(address)

#print(advantage_data)

book=xlwt.Workbook(encoding='utf-8',style_compression=0)
sheet=book.add_sheet('武汉新房信息表',cell_overwrite_ok=True)
col=['名称','优势','均价','面积','户型','结构','地址']

#写入每一列
for i in range(7):
    sheet.write(0,i,col[i])

#写入每一行
for n in range(len(title_data)):
    sheet.write(n+1,0,title_data[n])
    sheet.write(n+1,1,advantage_data[n])
    sheet.write(n+1,2,price_data[n])
    sheet.write(n+1,3,size_s_data[n])
    sheet.write(n+1,4,type_s_data[n])
    sheet.write(n+1,5,structure_data[n])
    sheet.write(n+1,6,address_data[n])

book.save('E:\素材\素材\武汉新房信息表.xls')

print('save end')

自己独立完成的项目/爬取贝壳网.py


from lxml import etree
import unicodedata
from pathlib import Path
from progressbar import progressbar
import json

config = json.load(open("../conf/config.json"))

download_dir = Path("../build/download")
parsed_dir = Path("../build/parsed")
normalize_char_map = {'ё':'е', 'Ё':'Е'}
vowels = 'аэыуояеёюи'

def normalize_string(s):
   norms = unicodedata.normalize('NFC', s)
   noacc =        [c
                  for c in norms
                  if unicodedata.category(c) != 'Mn' #'Mark, Nonspacing' = accents
                  and (
                      not unicodedata.category(c).startswith('P') #Punctuation
                      or c == '-'
                  )]
   normalized = ''.join(normalize_char_map[c] if c in normalize_char_map else c
                  for c in noacc).lower()
   stress=None
   if vowel_count(normalized) > 1:
       for p, c in enumerate(norms):
           if unicodedata.category(c) == 'Mn':  # 'Mark, Nonspacing' = accents
               stress = p
               break

   return tuple([normalized, stress, ''.join(noacc)])

def add_norm(forms, html, xpath, optional_prefix=None, prefix_norm=None):
    matches = html.xpath(xpath)
    for match in matches:
        match_text = match.xpath("string(.)")

        #Cell may contain multiple forms, e.g. свой -> свое́й, свое́ю (https://en.wiktionary.org/wiki/%D1%81%D0%B2%D0%BE%D0%B9)
        for match_form in match_text.split(", "):

            # Add comparative with and without comparative suffix, e.g. попроще and проще
            if optional_prefix and match_form.startswith(optional_prefix):
                suffix = match_form[len(optional_prefix):]
                forms.add(normalize_string(prefix_norm + suffix))
                forms.add(normalize_string(suffix))
            else:
                forms.add(normalize_string(match_form))

def vowel_count(txt):
    count = 0
    txt = txt.lower()
    for vowel in vowels:
        count = count + txt.count(vowel)
    return count

def parse_file(f, src_lang, destdir):

    pageJson=json.load(open(str(f)))
    of=pageJson['html']
    title = pageJson['title']
    html = etree.fromstring(of)

    for target_lang, langpair in config["langpairs"][src_lang].items():
        lang_name = langpair["lang_span_name"]
        langs = html.xpath("//h2/span[text()='%s' and contains(@class,'mw-headline')]" % lang_name)
        if not langs: #does not work for Serbo-Croatian
            continue
        forms = set()

        span_selector = "//*[preceding-sibling::h2[1]/span[text()='%s']]" % lang_name
        td_selector_template = "%s//table[contains(@class,'inflection-table')]/%s/tr/td//span[@lang='%s']"
        for tbody_selector in ['tbody', '.']:
            td_selector = td_selector_template % (span_selector, tbody_selector, target_lang)
            add_norm(forms, html, td_selector)
        add_norm(forms, html, "%s//strong[contains(@class,'headword') and @lang='%s']" % (span_selector, target_lang))

        # Parse comparative. NB тёплый has two variants
        comp_select = "//b[@lang='%s' and preceding-sibling::*[name()='i' and text()='comparative']]" % target_lang
        add_norm(forms, html, comp_select, "(по)", "по")

        dir = Path(destdir, target_lang)
        dir.mkdir(parents=True, exist_ok=True)
        file = Path(dir, Path(f).with_suffix('.dat').name)
        s = ''.join(["%s\t%s\t%s\t%s\n" % (form[0], title, form[1] if form[1] else 0, form[2]) for form in forms])
        file.write_text(s, encoding='utf8')

        marker.write_bytes(b'')

for src_lang, targets in config["langpairs"].items():
    lang_dir = Path(download_dir, src_lang)

    destdir = Path(parsed_dir, src_lang)
    marker_dir = Path(destdir, "_done")
    marker_dir.mkdir(parents=True, exist_ok=True)

    print("Source language: [%s]" % src_lang)
    print("Listing files...")
    files = sorted(lang_dir.glob("*.json"))

    new_pages = 0

    print("Parsing files...")
    for f in progressbar(files):

        marker = Path(marker_dir, Path(f).name)
        if marker.is_file():
            continue

        new_pages = new_pages + 1

        parse_file(f, src_lang, destdir)

    print("Parsed %d new pages out of %d total pages." % (new_pages, len(files)))


"""Add member table

Revision ID: ee2d279cae47
Revises: e0e586eec464
Create Date: 2020-09-30 11:19:13.281166

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'ee2d279cae47'
down_revision = 'e0e586eec464'
branch_labels = None
depends_on = None

def upgrade():
    op.create_table(
        'member',
        sa.Column('id', sa.Integer(), primary_key=True, index=True),
        sa.Column('group_id', sa.Integer(), sa.ForeignKey("group.id", ondelete="CASCADE"), index=True, nullable=False),
        sa.Column('user_id', sa.Integer(), sa.ForeignKey("user.id", ondelete="CASCADE"), index=True, nullable=False),
        sa.Column('role', sa.Integer(), default=0, nullable=False),
        sa.Column('is_active', sa.Boolean(), default=False, nullable=False),
        sa.Column('created_at', sa.DateTime(), default=sa.sql.func.now(), nullable=False),
        sa.Column('updated_at', sa.DateTime(), default=sa.sql.func.now(), onupdate=sa.sql.func.now(), nullable=False),
    )

def downgrade():
    op.drop_table('member')


import numpy as np
import pytest

import pyswallow as ps
import pyswallow.handlers.boundary_handler as psbh

class TestMOSwallow:

    @pytest.fixture
    def swallow(self):
        bounds = {
            'x0': [-50.0, 50.0],
            'x1': [-50.0, 50.0]
        }

        swallow = ps.MOSwallow(bounds, n_obj=2)
        return swallow

    @pytest.fixture
    def opp_swallow(self):
        bounds = {
            'x0': [-50.0, 50.0],
            'x1': [-50.0, 50.0]
        }

        opp_swallow = ps.MOSwallow(bounds, n_obj=2)
        return opp_swallow

    def test_move(self, swallow):
        swallow.position = np.array([0.0, 0.0])
        swallow.velocity = np.array([10.0, 10.0])

        bh = psbh.StandardBH()
        swallow.move(bh)

        assert np.array_equal(swallow.position, swallow.velocity)

    def test_dominate(self, swallow, opp_swallow):
        opp_swallow.fitness = [50.0, 50.0]
        swallow.fitness = [5.0, 5.0]

        ret_bool = swallow.dominate(opp_swallow)

        assert ret_bool

    def test_self_dominate(self, swallow):
        swallow.fitness = [5.0, 5.0]
        swallow.pbest_fitness = [50.0, 50.0]

        ret_bool = swallow.self_dominate()

        assert ret_bool


from __future__ import print_function
import os
import sys
import argparse

# our tools are in "libexec"
sys.path.append(os.path.join(sys.path[0], "libexec"))

import kdz

class KDZFileTools(kdz.KDZFile):
	"""
	LGE KDZ File creation tool
	"""

	indir = "kdzextracted"

#kdz.KDZFile._dz_header

	def loadParams(self):
		"""
		Loads the .kdz.params file for output, saves the values
		"""

		params = dict()
		file = open(os.path.join(self.indir, ".kdz.params"), "rt")
		line = file.readline()
		while len(line) > 0:
			line.lstrip()
			line = line.partition("#")[0]
			if len(line) == 0:
				line = file.readline()
				continue
			parts = line.partition("=")
			if len(parts[1]) == 0:
				print("[!] Bad line in {:s}'s parameter file".format(self.indir), file=sys.stderr)
			var = parts[0].rstrip()
			try:
				val = int(parts[2].strip())
			except ValueError:
				val = parts[2].strip()
			params[var] = val
			line = file.readline()

		file.close()

		for k in 'version', 'dataStart', 'payload0', 'payload0head':
			if k not in params:
				print("Parameter value \"{:s}\" is missing, unable to continue".format(k))
				sys.exit(1)

		if params['version'] != 2:
			print("[!] File format version not 2, cannot continue", file=sys.stderr)
			sys.exit(1)

		self.dataStart = params['dataStart']

		i = 0
		self.headers = []
		headers = {}
		self.payload = []
		self.files = {}
		name = "payload"+str(i)
		while name in params:
			self.payload.append(params[name])
			headers[params[name+"head"]] = params[name]
			i += 1
			name = "payload"+str(i)
		for i in range(i):
			self.headers.append(headers[i])

	def cmdCreateFile(self):
		"""
		Create the specified KDZ file
		"""

		out = open(self.kdzfile, "wb")
		current = self.dataStart
		out.seek(current, os.SEEK_SET)

		for name in self.payload:
			print("[+] Writing {:s} to output file {:s}".format(name, self.kdzfile))
			inf = open(os.path.join(self.indir, name), "rb")
			buf = " "
			while len(buf) > 0:
				buf = inf.read(4096)
				out.write(buf)
			self.files[name] = [current, out.tell() - current]
			inf.close()
			current = out.tell()

		self.files[self.headers[-1]].append(0)

		print("\n[+] Writing headers to {:s}".format(self.kdzfile))

		out.seek(0, os.SEEK_SET)
		out.write(self._dz_header)
		for name in self.headers:
			# last record
			if len(self.files[name]) > 2:
				out.write(b'\x03')
			head = {
				'name':		name.encode("utf8"),
				'length':	self.files[name][1],
				'offset': 	self.files[name][0],
			}
			buf = self.packdict(head)
			out.write(buf)
		out.close()
		print("[+] Done!")

	def cmdList(self):
		pass

	def parseArgs(self):
		# Parse arguments
		parser = argparse.ArgumentParser(description='LG KDZ File creator by Elliott Mitchell')
		parser.add_argument('-f', '--file', help='KDZ File to read', action='store', required=True, dest='kdzfile')
		group = parser.add_mutually_exclusive_group(required=True)
		group.add_argument('-l', '--list', help='list partitions', action='store_true', dest='listOnly')
		group.add_argument('-m', '--make', help='extract all partitions', action='store_true', dest='createFile')
		parser.add_argument('-d', '--dir', help='input directory', action='store', dest='indir')

		return parser.parse_args()

	def main(self):
		args = self.parseArgs()
		self.kdzfile = args.kdzfile

		if args.indir:
			self.indir = args.indir

		self.loadParams()

		if args.listOnly:
			self.cmdList()

		elif args.createFile:
			self.cmdCreateFile()

if __name__ == "__main__":
	kdztools = KDZFileTools()
	kdztools.main()


# logger.py - Set up a4h logging (sending trace output to stdout)
# 1. In main script...
#     start_log(logging.INFO)  # Call this once
# 2. In specific module...
#     import logging
#     ...
#     log = logging.getLogger('a4h')
#     ...
#     log.info('Listening on port:'+str(port))
#     ...
#     log.debug('SomeVar='+str(SomeVar))

import logging

def start_log(level, full_meta=False):
    """Create a unified a4h logger"""

    # Create a single logger to be used by all
    new_log = logging.getLogger('a4h')
    new_log.setLevel(level)

    # Create a console handler
    ch = logging.StreamHandler()
    ch.setLevel(level)
    ch.setFormatter(logging.Formatter('%(message)s [%(filename)s:%(lineno)d]'))
    new_log.addHandler(ch)

    if full_meta:
        # Add more context info to end of each log line
        ch.setFormatter(logging.Formatter(
            '"%(message)s"'
            + ',%(processName)s(%(process)d)'
            + ',%(threadName)s(%(thread)d)'
            + ',%(module)s:%(pathname)s:%(funcName)s:%(lineno)d'
            + ',%(asctime)s'
            + ',%(levelname)s')
        )


"""Pendulum Swing-up Environment with full observation."""
import numpy as np
from mbse.models.reward_model import RewardModel
from mbse.models.dynamics_model import DynamicsModel
from gym.envs.classic_control.pendulum import angle_normalize
import jax.numpy as jnp
import jax
from functools import partial
from typing import Union, Optional, Any
from mbse.utils.type_aliases import ModelProperties

from pyur5.models.ens_model import EnsembleModel
import math

class Ur5PendulumDynamicsModel(DynamicsModel):
    def __init__(self, task_typ='new', use_cos=True, ctrl_cost_weight=0.001, sparse=False, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.use_cos = use_cos
        self.task_typ = task_typ
        self.model = EnsembleModel(use_cos=use_cos, task_typ=task_typ)
        self.max_action = self.model.action_max
        self.min_action = self.model.action_min
        self.n_action = self.model.n_action
        self.n_obs = self.model.n_obs

        # TODO: Maybe think of punishing v_ee to be zero as well
        self.target_state = self.model.target_state #jnp.array([math.pi, 0]) # target state to be theta = pi/2, theta_dot = 0
        self.cost_weights = self.model.cost_weights[:-self.n_action] #jnp.array([1, 0.1]) # weight for theta and theta_dot

        # self.reward_model = Ur5PendulumReward(self.action_space, self.target_state, self.cost_weights, sparse=sparse)

        self.pred_diff = False
        self.obs_dim = self.model.x_dim #observation
        self.obs_space = self.model.obs_space

        print("model: ", self.model)
    def set_bounds(self, max_action, min_action=None):
        self.max_action = max_action
        if min_action is None:
            min_action = - max_action
        self.min_action = min_action
        self._init_fn()

    @partial(jax.jit, static_argnums=0)
    def predict(self,
                obs,
                action,
                rng=None,
                parameters=None,
                model_props: ModelProperties = ModelProperties(),
                sampling_idx: Optional[Union[jnp.ndarray, int]] = None,
                rescaled: bool = False,
                ):
        '''
            internal function to predict next state
        '''
        u = jnp.clip(action, self.min_action, self.max_action) # action is assumed to be in the range of [-1, 1]
        next_obs, reward, terminate, output_dict = self.model.step(obs, u)

        return next_obs

    def reset(self):
        '''
            return the initial observation
        '''
        if self.task_typ == 'new':
            if self.use_cos:
                obs = jnp.array([1.0, 0.0, 0.0] + [0.0, 0.0] * self.n_obs)
            else:
                obs = jnp.array([0.0, 0.0] + [0.0, 0.0] * self.n_obs)
        elif self.task_typ == 'sim':
            obs = jnp.array([1.0, 0.0, 0.0])

        return obs

    def evaluate(self,
                 obs,
                 action,
                 rng=None,
                 parameters=None,
                 sampling_idx=None,
                 model_props: ModelProperties = ModelProperties(),
                 rescaled: bool = False,
                 ):
        '''
            function to evaluate next state and reward
        '''
        # action here is assumed to be in the range of [-1, 1]
        # next_obs = self.predict(obs, action, rng, rescaled=False)
        # reward = self.model._reward_fn() #self.reward_model.predict(obs, action, next_obs)
        # return next_obs, reward
        next_obs, reward, _, _ = self.model.step(obs, action)
        return next_obs, reward

    # @staticmethod
    # @jax.jit
    # def _get_obs(state):
    #     theta, theta_dot, p_ee, v_ee = state[..., 0], state[..., 1], state[..., 2], state[..., 3]
    #     return jnp.asarray([theta, theta_dot, p_ee, v_ee], dtype=jnp.float32)

    # @partial(jax.jit, static_argnums=0)
    # def rescale_action(self, action):
    #     """Rescales the action affinely from  [:attr:`min_action`, :attr:`max_action`] to the action space of the base environment, :attr:`env`.

    #     Args:
    #         action: The action to rescale

    #     Returns:
    #         The rescaled action
    #     """
    # #     action = jnp.clip(action, self.min_action, self.max_action)
    #     low = self.model.action_space.low
    #     high = self.model.action_space.high
    #     action = low + (high - low) * (
    #             (action - self.min_action) / (self.max_action - self.min_action)
    #     )
    #     action = jnp.clip(action, low, high)
    #     return action

# class Ur5PendulumReward(RewardModel):
#     """Get Pendulum Reward."""

#     def __init__(self, action_space, target_sate, cost_weights, ctrl_cost_weight=0.001, sparse=False, *args, **kwargs):
#         super().__init__(*args, **kwargs)
#         self.ctrl_cost_weight = ctrl_cost_weight
#         self.sparse = sparse
#         self.min_action = min_action
#         self.max_action = max_action
#         self.action_space = action_space
#         self.target_state = target_state
#         self.cost_weights = cost_weights
#         self._init_fn()

#         print("target state: {}".format(self.target_state))
#         print("cost weights: {}".format(self.cost_weights))
#         print("reward action space: {}".format(self.action_space))

#     def _init_fn(self):
#         # rescale actions from [-1, 1] to [min_action, max_action]
#         self.rescale_action = jax.jit(lambda action: self._rescale_action(action=action,
#                                                                           min_action=self.min_action,
#                                                                           max_action=self.max_action,
#                                                                           low=self.action_space.low,
#                                                                           high=self.action_space.high,
#                                                                           ))
#         self.input_cost = jax.jit(lambda u: self._input_cost(u=u, ctrl_cost_weight=self.ctrl_cost_weight))

#         def predict(obs, action, next_obs=None, rng=None):
#             # return self.model._reward_fn(obs, action)
#             return self._predict(
#                 state_reward_fn=self.state_reward,
#                 input_cost_fn=self.input_cost,
#                 action_transform_fn=self.rescale_action,
#                 obs=obs,
#                 action=action,
#                 target_state=self.target_state,
#                 cost_weights=self.cost_weights,
#                 next_obs=next_obs,
#                 rng=rng
#             )

#         self.predict = jax.jit(predict)

#     def set_bounds(self, max_action, min_action=None):
#         self.max_action = max_action
#         if min_action is None:
#             min_action = - max_action
#         self.min_action = min_action
#         self._init_fn()

#     @staticmethod
#     @jax.jit
#     def state_non_sparse_reward(theta, theta_dot, p_ee, v_ee, target_state, cost_weights):
#         """Get sparse reward."""
#         theta = angle_normalize(theta)
#         dtheta = theta - target_state[0]
#         dtheta = angle_normalize(dtheta)
#         dtheta_dot = theta_dot - target_state[1]
#         cost = cost_weights[0] * (dtheta)**2 + cost_weights[1] * (dtheta_dot)**2
#         return -cost

#     @staticmethod
#     def _input_cost(u, ctrl_cost_weight):
#         # compute the |u|^2 * w
#         return ctrl_cost_weight * (jnp.sum(jnp.square(u), axis=-1))

#     @staticmethod
#     @jax.jit
#     def state_reward(state, target_state, cost_weights):
#         # TODO: This is outdated. NEED MODIFICATION!!
#         """Compute reward associated with state dynamics."""
#         # print("state shape: {}".format(state.shape))
#         # if state.shape[0] == 5:
#         if state.shape == (5,):
#             cos, sin, theta_dot, p_ee, v_ee = state[..., 0], state[..., 1], state[..., 2], state[..., 3], state[..., 4]
#             theta = jnp.arctan2(sin, cos)
#         else:
#             theta, theta_dot, p_ee, v_ee = state[..., 0], state[..., 1], state[..., 2], state[..., 3]

#         theta = angle_normalize(theta)
#         dtheta = theta - target_state[0]
#         dtheta = angle_normalize(dtheta) # normalize the angle to [-pi, pi]
#         dtheta_dot = theta_dot - target_state[1]
#         print("dtheta: {}".format(dtheta))
#         print("dtheta_dot: {}".format(dtheta_dot))
#         cost = cost_weights[0] * (dtheta)**2 + cost_weights[1] * (dtheta_dot)**2

#         # the reward is to make sure that we are
#         return -cost

#     @staticmethod
#     def _predict(state_reward_fn, input_cost_fn, action_transform_fn, obs, action, target_state, cost_weights, next_obs=None, rng=None, action_rng=[-1, 1]):
#         # action = action_transform_fn(action) # transform the action to the normal range
#         action = jnp.clip(action, action_rng[0], action_rng[1]) # clip to [-1, 1]
#         return state_reward_fn(state=obs, target_state=target_state, cost_weights=cost_weights) - input_cost_fn(action)

#     def evaluate(self,
#                  parameters,
#                  obs,
#                  action,
#                  rng,
#                  sampling_idx=None,
#                  model_props: ModelProperties = ModelProperties()):
#         print("Ur5DynamicsReward evaluate called")
#         next_state = self.predict(obs=obs, action=action, rng=rng)
#         print("next_state is : ", next_state)
#         reward = jnp.zeros(next_state.shape[0])
#         return next_state, reward

#     @staticmethod
#     def _rescale_action(action, min_action, max_action, low, high):
#         """
#         Args:
#             action: The action to rescale

#         Returns:
#             The rescaled action
#         """
#         # if min_action is not None and max_action is not None:
#         #     action = jnp.clip(action, min_action, max_action)
#         #     action = low + (high - low) * (
#         #             (action - min_action) / (max_action - min_action)
#         #     )
#         #     action = jnp.clip(action, low, high)
#         return action


from jinja2 import Environment, FileSystemLoader, select_autoescape
import yaml
import os
import sys
import argparse

'''
TODO:
  1. No try-catch blocks have been used to account for empty/uninitialized params in attributes.yaml

  2. Add basic unit/bdd tests (manually checked currently) & necessary code refactor to facilitate this:

     a. (positive) Verify creation of CSV for --generate_type=chart
     b. (positive) Verify creation of CSV using category name for --generate_type=experiment
     c. (positive) Verify creation of business logic, job & experiment CRs for --generate_type=experiment
     d. (negative) Verify validation/err-handle upon empty name, category attributes
'''

'''
NOTES:
  1. Category attribute is expected to match with chart names(though not mandatory), as per convention
     followed in litmuschaos/chaos-charts
'''

# generate_csv creates the chartserviceversion manifest
def generate_csv(csv_parent_path, csv_name, csv_config, litmus_env):
    csv_filename = csv_parent_path + '/' + csv_name + '.' + 'chartserviceversion.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/chartserviceversion.tmpl')
    output_from_parsed_template = template.render(csv_config)
    with open(csv_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_chart creates the experiment-custom-resource manifest
def generate_chart(chart_parent_path, chart_config, litmus_env):
    chart_filename = chart_parent_path + '/' + 'experiment.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_custom_resource.tmpl')
    output_from_parsed_template = template.render(chart_config)
    with open(chart_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_rbac creates the rbac for the experiment
def generate_rbac(chart_parent_path, chart_config, litmus_env):
    rbac_filename = chart_parent_path + '/' + 'rbac.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_rbac.tmpl')
    output_from_parsed_template = template.render(chart_config)
    with open(rbac_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_engine creates the chaos engine for the experiment
def generate_engine(chart_parent_path, chart_config, litmus_env):
    engine_filename = chart_parent_path + '/' + 'engine.yaml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_engine.tmpl')
    output_from_parsed_template = template.render(chart_config)
    with open(engine_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_job creates the experiment job manifest
def generate_job(job_parent_path, job_name, job_config, litmus_env):
    job_filename = job_parent_path + '/' + job_name + '_' + 'k8s_job.yml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_k8s_job.tmpl')
    output_from_parsed_template = template.render(job_config)
    with open(job_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_ansible_logic creates the ansible_logic manifest
def generate_ansible_logic(ansible_logic_parent_path, ansible_logic_name, ansible_logic_config, litmus_env):
    ansible_logic_filename = ansible_logic_parent_path + '/' + ansible_logic_name + '_' + 'ansible_logic.yml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_ansible_logic.tmpl')
    output_from_parsed_template = template.render(ansible_logic_config)
    with open(ansible_logic_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_chaos_prerequisites creates the chaos_prerequisites manifest
def generate_chaos_prerequisites(chaos_prerequisites_parent_path, chaos_prerequisites_name, chaos_prerequisites_config, litmus_env):
    chaos_prerequisites_filename = chaos_prerequisites_parent_path + '/' + chaos_prerequisites_name + '_' + 'ansible_prerequisites.yml'

    # Load Jinja2 template
    template = litmus_env.get_template('./templates/experiment_ansible_prerequisites.tmpl')
    output_from_parsed_template = template.render(chaos_prerequisites_config)
    with open(chaos_prerequisites_filename, "w+") as f:
        f.write(output_from_parsed_template)

# generate_package creates the package manifest
def generate_package(package_parent_path, package_name):
    package_filename = package_parent_path + '/' + package_name + '.' + 'package.yaml'
    print(package_filename)
    with open(package_filename, "w+") as f:
        f.write('packageName: ' + package_name + '\n' + 'experiments:')

def main():
    # Required Arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-a", "--attributes_file", required=True,
                        help="metadata to generate chartserviceversion yaml")
    parser.add_argument("-t", "--generate_type", required=True,
                        help="scaffold a new chart or experiment into existing chart")
    # Optional Arguments
    parser.add_argument("-c", "--chart_name", required=False,
                        help="existing chart name to which experiment belongs, defaults to 'category' in attributes file")

    args = parser.parse_args()

    entity_metadata_source = args.attributes_file
    entity_type = args.generate_type
    entity_parent = args.chart_name

    # Load data from YAML file into a dictionary
    config = yaml.load(open(entity_metadata_source))
    entity_name = config['name']

    # Store the litmus root from bootstrap folder
    litmus_root = os.path.abspath(os.path.join("..", os.pardir))
    #env = Environment(loader = FileSystemLoader('./'), trim_blocks=True, lstrip_blocks=True, autoescape=True)
    env = Environment(loader = FileSystemLoader('./'), trim_blocks=True, lstrip_blocks=True, autoescape=select_autoescape(['yaml']))

    # if generate_type is chart, only create the chart(top)-level CSV & package manifests
    if entity_type == 'chart':
        chart_dir = litmus_root + '/experiments/' + entity_name
        if os.path.isdir(chart_dir) != True:
            os.makedirs(chart_dir)
        generate_csv(chart_dir, entity_name, config, env)
        generate_package(chart_dir, entity_name)

    # if generate_type is experiment, create the litmusbook arefacts (job, playbook, cr)
    elif entity_type == 'experiment':
        # if chart_name is not explicitly provided, use "category" from attributes.yaml as chart
        if entity_parent is None:
            experiment_category = config['category']
            chart_dir = litmus_root + '/experiments/' + experiment_category
        else:
            chart_dir = litmus_root + '/experiments/' + entity_parent
        # if a folder with specified/derived chart name is not present, create it
        if os.path.isdir(chart_dir) != True:
            os.makedirs(chart_dir)
            # generate csv for the freshly created chart folder
            generate_csv(chart_dir, experiment_category, config, env)

            # generate package for the freshly created chart folder
            generate_package(chart_dir, experiment_category)

        # create experiment folder inside the chart folder
        experiment_dir = chart_dir + '/' + entity_name
        if os.path.isdir(experiment_dir) != True:
            os.makedirs(experiment_dir)

        # generate experiment csv
        generate_csv(experiment_dir, entity_name, config, env)

        # generate experiment-custom-resource
        generate_chart(experiment_dir, config, env)

        # generate experiment specific rbac
        generate_rbac(experiment_dir, config, env)

        # generate experiment specific chaos engine
        generate_engine(experiment_dir, config, env)

        # generate experiment job
        generate_job(experiment_dir, entity_name, config, env)

        # generate chaos-ansible-logic
        generate_ansible_logic(experiment_dir, entity_name, config, env)

        # generate chaos-prerequisites
        generate_chaos_prerequisites(experiment_dir, entity_name, config, env)

if __name__=="__main__":
    main()


from __future__ import annotations

import pytest

from subtitles_translator.available_languages import AvailableLanguages
from subtitles_translator.translator import Subtitles, Translator

@pytest.fixture
def raw_subtitles() -> list[str]:
    raw_str = """
1
00:00:00,498 --> 00:00:02,827
Hello World!
"""
    raw_subtitles = raw_str.split("\n")

    return raw_subtitles

@pytest.fixture
def subtitles(raw_subtitles) -> Subtitles:
    subtitles = Subtitles(raw_subtitles)

    return subtitles

@pytest.fixture
def translator() -> Translator:
    translator = Translator(AvailableLanguages("en"), AvailableLanguages("fr"))

    return translator

def test_translate():
    translator = Translator(source_language=AvailableLanguages("en"), target_language=AvailableLanguages("fr"))
    source_test = "Hello World!"
    translated_test = translator.translate(source_test)
    assert translated_test == "Bonjour le monde!"

def test_translate_subtitles(translator, subtitles):
    translator.translate_subtitles(subtitles)
    assert subtitles.full_text_lines[3] == "Bonjour le monde!"


from django.contrib.auth.decorators import login_required, user_passes_test
from django.shortcuts import render, redirect
from django.http import JsonResponse
from django.db.models import Case, When, Sum, F, Min, Value, CharField, BooleanField, Q
import json, random
from index.defs import get_array, execute_query
from inventory.models import Inventory
from index.models import Player
from recon.models import Animal, Hunt
from map.models import Hexa
from inventory.views import make_grid

craft_box_range = [21, 26]

@login_required(redirect_field_name=None)
def index(request):

    player, hexa, animals, situation_instance = entry(request)

    state = ['', 'attacking', 'running', 'dead']

    animals_with_state = []
    for animal in animals:
        print(animal)
        animals_with_state.append(animal + (state[animal[3]],) )

    grid = make_grid(request, grid_max=34)
    slices = [6, 11, 16, 21, 24, 27, 29, 31, 33]

    if not hexa[5]:
        pass
    else:
        pass

    return render(request, 'recon.html', {'animals': animals_with_state,
                                          'grid': grid,
                                          'slices': slices,
                                          'craft_box_start': craft_box_range[0],
                                          'animals_start': craft_box_range[1]+1,
                                          })

def recon(request):
    if request.is_ajax():
        if request.method == 'POST':
            player, hexa, situation_list, situation = entry(request)
            animals, status = move(player, hexa, situation)
            return JsonResponse([animals, status], safe=False)

def leave_button(request):
    if request.is_ajax():
        if request.method == 'POST':
            animals, status, remaining = leave(request)
            return JsonResponse([animals, status], safe=False)

def leave(request):
    player, hexa, situation_list, situation = entry(request)
    attacking = situation.filter(behaviour=1)
    remaining = False
    for attacker in attacking:
        if random.randint(0, 100) < 75:
            attacker.delete()
        else:
            remaining = True

    animals, status = move(player, hexa, situation, leaving=True)
    return animals, status, remaining

def shot(request):
    if request.is_ajax():
        if request.method == 'POST':
            animal_number = json.loads(request.body.decode('utf-8'))
            hunt = Hunt.objects.filter(id=animal_number, player_id=request.user.active_player)

            animal = hunt.select_related('animal')\
                .values_list('hit_chance', 'hits', 'animal__shots', 'animal__aggression', 'behaviour',
                             'animal__meat', 'animal__leather', 'animal__feather', 'animal__scales', 'animal__venom')[0]

            Hunt.objects.filter(player_id=request.user.active_player, behaviour=2).exclude(id=animal_number).delete()

            status = 'miss'

            player, hexa, situation_list, situation = entry(request)

            if animal[0] >= random.randint(0, 100):
                status = 'hit'
                if int(animal[1])+1 >= animal[2]:
                    hunt.update(behaviour=3)
                    status = 'dead'
                    pos = craft_box_range[1]+1+(hunt[0].slot*2)
                    second = False
                    for i in range(1, 6):
                        if animal[i+4]:
                            if second: pos += 1
                            Inventory(player=player[0], pos=pos, consume_items_id=i, amount=animal[i+4]).save()
                            second = True
                else:
                    run_chance = random.randint(0, 14)*10+random.randint(0, 10)
                    if animal[4] != 1 and run_chance > animal[3]:
                        hunt.update(behaviour=2, hits=F('hits') + 1)
                    else:
                        hunt.update(behaviour=1, hits=F('hits') + 1)
            else:
                if animal[4]:
                    hunt.exclude(behaviour=1 or 3).delete()

            animals = on_place(player, hexa, situation)

            return JsonResponse([animals, status], safe=False)

def drag_and_drop(request):
    if request.is_ajax():
        if request.method == 'POST':
            arr = json.loads(request.body.decode('utf-8'))
            execute_query(
                query="""UPDATE inventory_inventory SET pos = %s
                         WHERE player_id = %s AND id = %s""",
                params=[arr["entNumber"], request.user.active_player, arr["dragNumber"]])

            animals = []

            if arr['initNumber'] > craft_box_range[1]:
                animals = loot(request)

            return JsonResponse(animals, safe=False)

def loot(request):
    player, hexa, situation_list, situation = entry(request)
    animals = on_place(player, hexa, situation)
    inventory = Inventory.objects.filter(player=request.user.active_player, pos__gt=craft_box_range[1]).values_list('pos')

    status = 'looting'

    for dead in situation.filter(behaviour=3):
        if dead.slot+craft_box_range[1] not in inventory:
            status += ' | '+str(dead.slot)

    return [animals, status]

def entry(request):
    player = Player.objects.filter(pk=request.user.active_player)
    hexa = player.select_related('pos') \
        .values_list('pos__grass', 'pos__wood', 'pos__hills', 'pos__mountains', 'pos__water', 'pos__town')[0]

    print('grass %d, wood %d, hills %d, mountains %d, water %d' % (hexa[0], hexa[1], hexa[2], hexa[3], hexa[4]))

    if not hexa[5]:
        situation_instance = Hunt.objects.filter(player=request.user.active_player).select_related('animal')
        situation = list(situation_instance.values_list('id', 'animal__name', 'hit_chance', 'behaviour', 'slot'))
    else:
        situation = None
        situation_instance = None

    return player, hexa, situation, situation_instance

def move(player, hexa, situation, leaving=False):
    animals = []

    attacking = situation.filter(behaviour=1).select_related('animal')\
        .values_list('id', 'animal__name', 'animal__size', 'hits', 'slot')

    for attacker in attacking:
        hit_chance = attacker[2] - random.randint(0, 30 - 12 * attacker[3])
        animals.append([attacker[0], attacker[1], hit_chance, 1, attacker[4]])

    if situation:
        situation.exclude(behaviour=1).delete()

    Inventory.objects.filter(player=player[0], pos__gt=craft_box_range[1]).delete()

    status = ''

    if not leaving:
        for s in range(4):
            if situation.filter(slot=s):
                continue
            animals, status = spawn_animal(animals, hexa, player, s)

    return animals, status

def on_place(player, hexa, situation):
    animals = []

    no_idles = situation.exclude(behaviour=0).select_related('animal')\
        .values_list('id', 'animal__name', 'animal__size', 'hits', 'behaviour', 'slot')

    for no_idle in no_idles:
        hit_chance = no_idle[2] - random.randint(0, 30) + 12 * no_idle[3]
        animals.append([no_idle[0], no_idle[1], hit_chance, no_idle[4], no_idle[5]])

    if situation:
        situation.filter(behaviour=0).delete()

    startled = random.randint(1, 3)
    print('startled %d' % startled)

    for s in range(4):
        if situation.filter(slot=s):
            continue
        if startled > 0:
            startled -= 1
            continue
        animals = spawn_animal(animals, hexa, player, s)
    return animals

def spawn_animal(animals, hexa, player, s):
    aim = random.randint(0, 660)
    biome = 0
    status = ''
    for i in range(5):
        biome += hexa[i]
        if aim < biome:
            shot = random.randint(0, 100)
            animal_instance = Animal.objects.filter(biome=i, presence__gte=shot).order_by('presence')
            animal = animal_instance.values_list('id', 'name', 'aggression', 'shots', 'size').first()
            if animal:
                if animal[0] < 12:
                    attack_chance = random.randint(0, 9) * 10 + random.randint(0, 10)
                    behaviour = 1 if attack_chance <= animal[2] else 0
                    hit_chance = animal[4] - random.randint(0, 18 if behaviour == 1 else 30)
                else:
                    behaviour = 3
                    hit_chance = 0

                    pos = craft_box_range[1] + 1 + s * 2
                    Inventory(player=player[0], pos=pos, consume_items_id=7, amount=random.randint(1, 3)).save()
                    status = 'items'

                hunt = Hunt(player=player[0], animal=animal_instance[0], behaviour=behaviour,
                            hit_chance=hit_chance, slot=s)
                hunt.save()
                animals.append([hunt.pk, animal[1], hit_chance, behaviour, s])
            break
    return animals, status


from random import choice

from emoji import emojize
from telegram import ReplyKeyboardMarkup, KeyboardButton
from utils import *

import settings

def get_user_emoji(user_data):
    if 'emoji' in user_data:
        return user_data['emoji']
    else:
        user_data['emoji'] = emojize(choice(settings.USER_EMOJI), use_aliases=True)
        return user_data['emoji']

def get_keyboard():
    contact_button = KeyboardButton('Отправить контакты', request_contact=True)
    location_button = KeyboardButton('Отправить геолокацию', request_location=True)
    my_keyboard = ReplyKeyboardMarkup([
                                        ['Хатю котика', 'Сменить аватар'],
                                        [contact_button, location_button]
                                       ], resize_keyboard=True
                                      )
    return my_keyboard


import numpy as np
import matplotlib.pyplot as plt
import sys,os
import pickle
import csv

# PREAMBLE
def validate_input():
	if len(sys.argv) != 4:
    		print("Usage: python {} ANALYZED_DATA_FILE EPSILON LABEL".format(sys.argv[0]))
    		sys.exit(2)
	return sys.argv[1], float(sys.argv[2]), sys.argv[3]

def binning_analysis(O,k):
	# determine number of blocks
        noBlocks = len(O)//k
        N = k*noBlocks

        # Compute block averages
        O = np.array(O)
        Osplit = np.split(O[0:N],noBlocks)
        bas = np.mean(Osplit,axis=1)

        # Calculate block average and variance
        blockAvg = np.mean(bas)
        blockVar = np.var(bas)
        # Calculate block error
        blockError = np.sqrt(blockVar/noBlocks)

      	# Calculate average and variance of measurements
        Oavg = np.mean(O)
        Ovar = np.var(O)

      	# Calculate estimated autocorrelation time and effective statistics
        if Ovar > 0:
               	tINTo = k*blockVar/(2*Ovar)
        else:
                tINTo = float('nan')
        if tINTo > 0:
                Neff = N/(2*tINTo)
        else:
                Neff = float("nan")
        return Oavg, tINTo, Neff, blockError

def perform_binning_analysis(O):
    N = len(O)

    # Initialize variables for tracking data
    Neff = 0.0
    blockError = 0.0
    tINTo = 0.0

    for i in range(N//10):
        k = i+1
        Oavg, tINToNew, NeffNew, blockErrorNew = binning_analysis(O,k)
        if tINToNew > tINTo:
            Neff = NeffNew
            blockError = blockErrorNew
            tINTo = tINToNew
    data = [Oavg,tINTo,Neff,blockError]
    return data

# Load and organize data
datafile, epsilon, label = validate_input()
data = pickle.load(open(datafile,"r"))

def plot_ac(data,epsilon,label):
	Ts = [i[0] for i in data]

	a = np.array([i[5][0] for i in data])/8 # [angstrom]
	c = np.array([i[7][0] for i in data])/4 # [angstrom]
	a_err = np.array([i[5][3] for i in data])/8 # [angstrom]
	c_err = np.array([i[7][3] for i in data])/4 # [angstrom]

	m_a,b_a = np.polyfit(Ts,a,1)
	m_c,b_c = np.polyfit(Ts,c,1)
	x = np.linspace(Ts[0],Ts[-1],100)

	# Print
	print("a:")
	print("Slope: {}".format(m_a))
	print("Intercept: {}".format(b_a))
	print("c:")
	print("Slope: {}".format(m_c))
	print("Intercept: {}".format(b_c))

	plt.errorbar(Ts,a,yerr=a_err,fmt='-o',label="a")
	plt.errorbar(Ts,c,yerr=c_err,fmt='-o',label="c")
	plt.plot(x,m_a*x+b_a,label="a fit")
	plt.plot(x,m_c*x+b_c,label="c fit")
# Plot
plt.figure()
plt.title("Lattice Constants")
plot_ac(data,epsilon,label)
plt.legend()
plt.show()


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from typing import List, Tuple
from kfold import KFold
from concept import mobility, mobility_diff, frontier, frontier_diff, steady, corner, corner_steady, steady_diff, corner_diff, score_f1, score
from read_wth import read_wthor_files
import pandas as pd
from player_color import BLACK, WHITE, EMPTY
from device_detect import detectDevice
from load_net import loadNetOutput
from load_data import into_input_format_2, load_tensor
import time
import random
from rewrite_print import print, print_acc
import warnings
warnings.filterwarnings("ignore")  # 忽略UserWarning兼容性警告

batchSize = 128

class LinearModel(nn.Module):  # 定义线性模型g,用于c-con
    def __init__(self, input_size):
        super(LinearModel, self).__init__()
        self.linear = nn.Linear(input_size, 1)

    def forward(self, x: torch.Tensor):
        x = x.view(x.size(0), -1)
        y = self.linear(x)
        return y.squeeze()

def batch_normalize(x, gamma=0.5, beta=0.5, eps=1e-5):
    # N, D = x.shape
    mean = np.mean(x)
    var = np.var(x)
    x_hat = (x - mean) / np.sqrt(var + eps)
    out = gamma * x_hat + beta
    return out

def loss_f1(pred: torch.Tensor, target: torch.Tensor, w, lamb, device: torch.device) -> torch.Tensor:
    mse_loss = torch.mean((pred - target)**2)
    # l2_loss = torch.norm(w)/pred.size(0)
    l1_loss = torch.sum(torch.abs(w))/pred.size(0)
    return mse_loss + lamb * l1_loss  # 返回的是平均loss

# 定义训练函数
def train(model: LinearModel, X_train: torch.Tensor, y_train: torch.Tensor, lamb: float, device: torch.device, learning_rate, num_epochs=500):
    optimizer = torch.optim.SGD(
        model.parameters(), lr=learning_rate, momentum=0.9)
    schduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
        optimizer, mode='min', patience=30, factor=0.7)

    X_train = load_tensor(X_train.clone().detach(), batchSize)
    y_train = load_tensor(y_train.reshape(-1), batchSize)

    for epoch in range(num_epochs):
        total_loss = 0
        datasize = 0
        for i in range(len(X_train)):

            inputs = X_train[i]
            targets = y_train[i]
            datasize += targets.size(0)
            optimizer.zero_grad()
            outputs = model(inputs)

            loss = loss_f1(outputs, targets, model.linear.weight, lamb, device)
            total_loss += loss.item()*targets.size(0)
            loss.backward()
            optimizer.step()
            schduler.step(loss)

        if num_epochs > 500 and (epoch+1) % 500 == 0:
            print('LR: {}, Epoch [{}/{}], Loss: {:.4f}'.format(
                learning_rate, epoch + 1, num_epochs, total_loss/datasize))

# 正则化参数通过交叉验证确定

def decide_lambda_lr(z_d: torch.Tensor, c_z: torch.Tensor, n_splits: int, device: torch.device, random_seed: int = 42):  # z_d为probe的输入，c_z为标签数据
    kf = KFold(n_splits, shuffle=True, random_state=random_seed)
    lamb_list = [0.01, 0.1, 1, 10]  # 网格化搜索之后发现lamb的影响很小，为节省时间干脆都定为10
    lr_list = [1e-06, 1e-07, 1e-08, 1e-09]
    best_loss_lamb = float('inf')
    best_lamb = None
    best_lamb_lr = None

    for lamb in lamb_list:  # 决定了best_lamb及其下的best_loss和best_lr
        best_lr = None
        best_loss = float('inf')
        for lr in lr_list:  # 决定了当前lamb下的best_loss和best_lr
            total_loss = 0
            datasize = 0
            for k, index in enumerate(kf.split(z_d)):
                train_index, val_index = index
                X_train, X_val = z_d[train_index], z_d[val_index]
                y_train, y_val = c_z[train_index], c_z[val_index]
                # 定义模型并训练
                input_size = z_d.shape[1]*z_d.shape[2]*z_d.shape[3]
                linear_model = LinearModel(input_size)
                linear_model = linear_model.to(device)
                # print(f'-------------The {k+1}/{n_splits} th training------------')
                train(linear_model, X_train.reshape(-1, input_size),
                      y_train.reshape(-1, 1), lamb, device, learning_rate=lr, num_epochs=500)

                # 验证
                inputs = load_tensor(X_val)
                targets = torch.tensor(y_val, device=device)
                batch_output = linear_model(inputs[0])
                for i in range(1, len(inputs)):
                    batch_output = torch.cat(
                        [batch_output, linear_model(inputs[i])], dim=0)

                loss = loss_f1(batch_output, targets,
                               linear_model.linear.weight, lamb, device)
                total_loss += loss.item()*targets.size(0)
                datasize += targets.size(0)

            avg_loss = total_loss / datasize

            if avg_loss < best_loss:
                best_loss = avg_loss
                best_lr = lr

        print(f"lambda: {lamb}, best lr: {best_lr}, best loss: {best_loss}")

        if best_loss < best_loss_lamb:
            best_loss_lamb = best_loss
            best_lamb = lamb
            best_lamb_lr = best_lr

    print(
        f"best lambda: {best_lamb}, best lr: {best_lamb_lr}, best loss: {best_loss_lamb}")
    return best_lamb, best_lamb_lr

def concept_probe(concept_name: str, train_val_data: pd.DataFrame, train_zd: torch.Tensor, device: torch.device, num_epochs: int = 1000):
    y_train = []
    for i in range(len(train_val_data)):
        board = train_val_data.iloc[i, :]
        c_z = concept_name(board, BLACK)
        y_train.append(c_z)

    y_train = torch.tensor(y_train, device=device)

    best_lamb, best_lr = decide_lambda_lr(train_zd, y_train, 3, device)
    input_size = train_zd.shape[1]*train_zd.shape[2]*train_zd.shape[3]

    linear_model = LinearModel(input_size)
    linear_model = linear_model.to(device)
    train(linear_model, train_zd.reshape(-1, input_size),
          y_train.reshape(-1, 1), best_lamb, device, best_lr, num_epochs)
    return linear_model

def r2_score_0(y_test, X_test):
    SStot = np.sum((y_test-np.mean(y_test))**2)
    SSres = np.sum((y_test-X_test)**2)
    r2 = 1-SSres/SStot
    return r2

def r2_score(y_test, X_test):
    r = np.corrcoef(X_test, y_test)[0, 1]
    return r**2

def test_accuracy(concept_name: str, model: LinearModel, test_data: pd.DataFrame, test_zd: torch.Tensor):
    y_test = []
    for i in range(len(test_data)):
        board = test_data.iloc[i, :]
        c_z = concept_name(board, BLACK)
        y_test.append(c_z)

    test_zd = load_tensor(test_zd)
    X_test = model(test_zd[0])
    for i in range(1, len(test_zd)):
        X_test = torch.cat([X_test, model(test_zd[i])], dim=0)

    r_squared = r2_score(np.array(y_test), np.array(X_test.tolist()))
    r_squared_0 = r2_score_0(np.array(y_test), np.array(X_test.tolist()))
    print(f"r^2: {r_squared}, r^2_0: {r_squared_0}")
    print("---------------------------------------------------------------\n")
    return r_squared, r_squared_0

# 这个部分只算一次，之后从结果中拿取即可
def zdlist(device: torch.device, epoch_list: List[int], trainset, testset):
    all_trainzd_list = []
    all_testzd_list = []
    for training_step in epoch_list:
        train_z_d_list = loadNetOutput(
            training_step, into_input_format_2(trainset), device)
        test_z_d_list = loadNetOutput(
            training_step, into_input_format_2(testset), device)
        all_trainzd_list.append(train_z_d_list)
        all_testzd_list.append(test_z_d_list)
    return all_trainzd_list, all_testzd_list  # list的长度就等于epoch_list的长度

def compute_results(device: torch.device, training_step, concept, trainset, testset, train_z_d_list, test_z_d_list, layer_num=15):
    accuracy_layer = []
    accuracy_layer_0 = []

    for layer in range(layer_num):
        print(
            f"---------NN training epoch: {training_step}, Layer num: {layer+1}/15---------")
        linear_model = concept_probe(
            concept, trainset, train_z_d_list[layer], device)
        accuracy, accuracy_0 = test_accuracy(
            concept, linear_model, testset, test_z_d_list[layer])

        accuracy_layer.append(accuracy)  # 记录当前training_step下遍历各layers的accuracy
        accuracy_layer_0.append(accuracy_0)

    return accuracy_layer, accuracy_layer_0

if __name__ == '__main__':

    globalDevice = detectDevice()
    BLACK = -1

    file_list = range(2000, 2022)  # 代表数据年份，可选：range(1977,2024)
    epoch_list = range(1, 58)  # 代表神经网络训练代数，可选：range(1,58)
    concept_list = [frontier, steady, corner, mobility, frontier_diff,
                    score_f1, score, steady_diff,  corner_diff, mobility_diff]

    paths = ['./gamedata/WTH_' + str(i)+'.wtb' for i in file_list]
    trainset, testset = read_wthor_files(paths)

    all_trainzd_list, all_testzd_list = zdlist(
        globalDevice, epoch_list, trainset, testset)

    for concept in concept_list:
        print(
            f'Current time: {time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())}')
        print(
            f'Use datasets in years: {file_list[0]}~{file_list[-1]}. Size of trainset: {len(trainset)}')
        print(f'Concept: {str(concept.__name__)}\n')

        accuracy_step, accuracy_step_0 = [], []
        for training_step in epoch_list:
            train_z_d_list = all_trainzd_list[training_step]
            test_z_d_list = all_testzd_list[training_step]
            accuracy_layer, accuracy_layer_0 = compute_results(
                globalDevice, training_step, concept, trainset, testset, train_z_d_list, test_z_d_list)
            accuracy_step.append(accuracy_layer)
            accuracy_step_0.append(accuracy_layer_0)

        print_acc(f"concept: {str(concept.__name__)}")
        print_acc(f"accuracy_step: {accuracy_step}")
        print_acc(f"accuracy_step_0: {accuracy_step_0}")
        print_acc(
            '---------------------------------------------------------------\n')


'''
公開情報からパーシャルチェックリストを作成する

update_ctestwin_pck.py
2021.Oct.25 7M4MON

2020年現在、日本のアマチュア無線局数はおよそ40万局ですが、
このうち、JARLの国内4大コンテストに 2015-2019の5年間で書類を提出した局は、
およそ6000局（7M4MON 調べ）です。
コールサインのアルファベット・数字の組み合わせは膨大な数がありますが、
コンテストに参加する局は、ほとんど上記の6000局のコールサインです。
そのため、過去の交信履歴等からある程度の予測と省力化が可能です。
コンテストロギングソフトでは、コールサインの一部を入力すると
交信相手のコールサイン候補が表示されるパーシャルチェックという機能があります。
パーシャルチェックではコールサインだけではなくコンテストナンバーの候補も表示されます。
上記の国内4大コンテストのコンテストナンバーは RS(T) ＋ 運用地の都府県・北海道地域番号または
市郡区番号＋空中線電力を表すアルファベット１文字です。
総務省の無線免許状検索ではコールサインで検索すると常置場所の市区町村名まで分かりますので、
コンテスト書類提出局のコールサインとつき合わせて、パーシャルチェックのデータベースを作成しました。

* プログラムの流れ
1.コンテストの結果ページを読み込んで、コールサインを抜き出す
2.既存の ctestwin.pck を読み込んで、コールサインを抜き出す
3. 1と2を結合し、重複を削除し、ソートする
4.総務省の免許状検索データベースから常置場所を取得する
5.常置場所の市町村名から市郡区番号を取得する
6. 3と4と5を結合して、ctestwin.pckを作成

以前は新規参加局を追記していくスタイルでしたが、時間経過とともに
廃局されたり常置場所を変更されたりする局が増えてきますので、
全てのコールサインを再検索して、最新の常置場所に置き換えるとともに
免許状検索に登録されていない局はリストから削除する方針に変更しました。
'''

contest_result_url = "https://www.jarl.org/Japanese/1_Tanoshimo/1-1_Contest/6m/2021/entry.html"
read_pck_filename = "./ctestwin_20211024.pck"       # 前回作成したパーシャルチェックファイル名
write_pck_filename = "./ctestwin_20211122.pck"      # 今回作成するパーシャルチェックファイル名
new_callsign_filename = "./2021_6m_new.txt"      # 今回のコンテストの新規参加局
read_acag_filename = './pref_acag.csv.txt'          # 市町村郡名とコードの対照表（.csvだとExcelで開いたときに先頭の 0 が消えるのを嫌って .txtにしてある)
wait_sec = 3                                        # 過負荷をかけないようにするための、1件あたりのウェイト時間。3のとき4878件で5時間半程度。
pass_idx = 0                                        # 中断したときに飛ばすインデックス番号

import requests, re, json, time, csv, datetime

'''コールサイン抜き出し用正規表現'''
cs_re = '[J|7][A-S][0-9][A-Z]+? '           # Jまたは7で始まり、2文字目がAからSのアルファベット、3文字目が数字、4文字目からアルファベットが続き、末尾にデリミタの半角スペース
#cs_re = ' (J[A-S][0-9]|[7-8])[A-Z]+? '     #サクラエディタではOKだがpythonだとだめ
#cs_re = ' (J[A-S][0-9][A-Z]+?|[7-8][A-Z]+?) '

'''コンテストの結果ページからコールサインを抜き出す（末尾に半角スペースあり）'''
def read_callsign_resultpage(url):
    res = requests.get(url)
    callsign_list = re.findall(cs_re,res.text)
    return callsign_list

'''過去の ctestwin.pck からコールサインを抜き出す（末尾に半角スペースあり）'''
def read_callsign_pckfile(pck_filename):
    pckfile = open(pck_filename, 'r')
    pckstr = pckfile.read()
    pckfile.close()
    callsign_list_pck = re.findall(cs_re,pckstr)
    return callsign_list_pck

'''コンテストの参加局で過去のデータベースに登録のない局を抜き出す'''
def pickup_new_cs(test_cs, pck_cs):
    new_cs = []
    for cs in test_cs:
        if (cs in pck_cs) == False :
            new_cs.append(cs)
    return new_cs

'''リストをダンプして保存'''
def dump_list(the_list, filename):
    f = open(filename, 'w')
    for x in the_list:
        f.write(str(x) + "\n")
    f.close()

''' コールサインを総務省の無線局等情報検索Web-APIで検索 '''
''' 応答結果から無線局の設置場所取得して市町村名を返す '''
''' get_qth.py から移植'''
# res_txt = {"musen":[{"listInfo":{"name":"＊＊＊＊＊（7M4MON）","radioStationPurpose":"アマチュア業務用","tdfkCd":"東京都中央区","no":"1","licenseDate":"2018-04-02"}}],"musenInformation":{"totalCount":"1","lastUpdateDate":"2020-08-02"}}
def get_city(callsign):
    response = requests.get('https://www.tele.soumu.go.jp/musen/list?ST=1&OF=2&OW=AT&DA=0&DC=1&SC=1&MA=' + callsign)
    city = ""
    if response.status_code == 200:
        res_txt =response.text    # レスポンスのHTMLを文字列で取得
        if 'tdfkCd' in res_txt:   # 存在（廃局）チェック
            d = json.loads(res_txt)                     # Json形式の文字列をdictにパース
            city = d['musen'][0]['listInfo']['tdfkCd']  # tdfkCd キーの中身を取得
            print(city)
        else :
            pass    #コールサインがデータベースにない
    else:
        pass        #サーバー応答なし
    return city

'''市町村名から市郡コードを返す'''
'''get_cgnum.py から移植'''
cgnum_list = [] # 町村郡名とコードの対照表を保持しておくリスト
# 北海道のダブリ郡の例外処理。コードが後の郡の町名だったらコードを直す。
hokkaido_dupe_code = ['01006','01014','01022','01042','01045','01050','01073']  # 先にあるコードのリスト
abuta_iburi = ['豊浦町','洞爺湖町']
uryu_kamikawa = ['幌加内町']    #例外中の例外
kamikawa_kamikawa = ['鷹栖町','東神楽町','当麻町','比布町','愛別町','上川町','東川町','美瑛町','和寒町','剣淵町','下川町']
sorachi_kamikawa = ['上富良野町','中富良野町','南富良野町']
teshio_souya = ['豊富町','幌延町']
nakagawa_tokachi = ['幕別町','池田町','豊頃町','本別町']
yuufutsu_kamikawa = ['占冠村']
hokkaido_dupe_towns = [abuta_iburi, uryu_kamikawa,kamikawa_kamikawa,sorachi_kamikawa,teshio_souya,nakagawa_tokachi,yuufutsu_kamikawa]
hokkaido_dupe_county = ['北海道虻田郡(胆振)','北海道雨竜郡(上川)','北海道上川郡(上川)','北海道空知郡(上川)','北海道天塩郡(宗谷)','北海道中川郡(十勝)','北海道勇払郡(上川)']

def get_cgnum(city_name):
    # 町村名は不要なので〇〇郡までを抜き出す。
    if '郡' in city_name:   # 愛知県蒲郡市, 福岡県小郡市, 奈良県大和郡山市, 福島県郡山市も含まれてしまうが削っても被らないので問題なし
        full_city_name = city_name  #バックアップ
        city_name = full_city_name[0:city_name.find('郡') + 1]  # 前方から検索
        town_name = full_city_name[city_name.find('郡') + 1:]
        print (town_name)

    cgnum_str = ""
    list_hit = False

    # 郡市区名を探す
    for cg_item in cgnum_list:
        if city_name in cg_item[1]:
            list_hit = True
            break

    if list_hit == True:    # （表記ゆれや合併などで名前が変わってなければ）必ずあるはず
        cgnum_str = cg_item[0]
        country_name = cg_item[1]
        '''北海道のダブリ郡を処理する'''
        if cgnum_str in hokkaido_dupe_code :
            print('dupe code:' + country_name)
            dupe_index = hokkaido_dupe_code.index(cgnum_str)
            if town_name in hokkaido_dupe_towns[dupe_index] :   #例外の町がある
                country_name = hokkaido_dupe_county[dupe_index]
                print('北海道の例外')
                if dupe_index == 1 : # 例外中の例外 北海道雨竜郡(上川)幌加内町
                    cgnum_str = '01081'
                    print ('中の例外 北海道雨竜郡(上川)幌加内町')
                else:
                    cgnum_str = str(int(cgnum_str) + 1).zfill(5) # 幌加内町以外は＋１すれば良い
    else:   #not in list
        pass

    print(cgnum_str + '\n')
    return cgnum_str

'''処理開始時に市郡コードを cgnum_list[] にロードしておく'''
def load_cgnum_list():
    with open(read_acag_filename,'r',encoding="SHIFT-JIS") as f_in:
        reader = csv.reader(f_in)
        for row in reader:
            cgnum_list.append(row)

'''メインの処理'''
if __name__ == '__main__':
    start_date = datetime.datetime.now()
    cs_list = read_callsign_resultpage(contest_result_url)
    cs_list.sort()
    print("対象局数: " + str(len(cs_list)))
    cs_list_pck = read_callsign_pckfile(read_pck_filename)
    print("既登録局数: " + str(len(cs_list_pck)))
    cs_new = pickup_new_cs(cs_list, cs_list_pck)
    print("新規局数: " + str(len(cs_new)))
    dump_list(cs_new, new_callsign_filename)
    cs_list.extend(cs_list_pck)
    dup = [x for x in set(cs_list) if cs_list.count(x) > 1]
    print("重複局数: " + str(len(dup)))
    uniq_cs_list = list(set(cs_list))       #重複を削除
    uniq_cs_list.sort()
    print("実行件数: " + str(len(uniq_cs_list)))
    fw = open(write_pck_filename, 'a')
    load_cgnum_list()
    i = 0
    qrt = 0
    no_cgnum = 0
    for cs in uniq_cs_list:
        if pass_idx > i:
            i += 1
            pass
        else :
            callsign = cs.strip()   #改行やスペースの削除
            print(str(i) + " " + callsign)
            city_name = get_city(callsign)
            if city_name != "" :
                cgnum_str = get_cgnum(city_name)
                print(callsign + ' ' + cgnum_str + ' '+ city_name , file=fw)
                if cgnum_str == "" :
                    no_cgnum += 1   #表記ゆれや合併などで市町村名が変わっている
            else:
                qrt += 1            #廃局などでコールサインが総務省のデータベースにない
            time.sleep(wait_sec)    #総務省のデータベースに過負荷をかけないようにちょっと待つ
            i += 1
    print('IDX:' + str(i) + ' QRT:' + str(qrt) + ' NIL:'+ str(no_cgnum) , file=fw)    #最後に結果を追記
    fw.close()
    stop_date = datetime.datetime.now()
    print('処理時間: '+ str(stop_date - start_date ))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import random
from PyQt5.QtWidgets import QWidget, QApplication
from PyQt5.QtGui import QPainter
from PyQt5.QtCore import Qt

class MainWindow(QWidget):
    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        self.setGeometry(300, 300, 250, 150)
        self.setWindowTitle('Drawing points')
        self.show()

    # 绘图是在paintEvent()方法中完成。
    # QPainter 对象放在begin()方法和end()方法之间，它执行部件上的低层次的绘画和其他绘图设备。
    # 实际的绘画我们委托给drawText()方法。
    def paintEvent(self, event):
        painter = QPainter()
        painter.begin(self)
        self.drawPoints(event, painter)
        painter.end()

    def drawPoints(self, event, painter):
        painter.setPen(Qt.red)      # 设置画笔颜色为红色
        size = self.size()          # 获取当前窗口大小

        for i in range(1000):
            x = random.randint(1, size.width()-1)
            y = random.randint(1, size.height()-1)
            painter.drawPoint(x, y)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    win = MainWindow()
    sys.exit(app.exec_())