import pandas as pd
from bs4 import BeautifulSoup
import numpy as np
import requests

pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)

################################ Attributions #########################################

# https://towardsdatascience.com/value-investing-dashboard-with-python-beautiful-soup-and-dash-python-43002f6a97ca
# https://www.codeproject.com/Articles/1227268/Accessing-Financial-Reports-in-the-EDGAR-Database
# "Python and Web Data Extraction: Introduction" by Alvin Zuyin Zheng
# "Scraping EDGAR with Python" by Rasha Ashraf, Journal of Education for Business
# https://stackoverflow.com/questions/47736600/how-to-get-a-value-from-a-text-document-that-has-an-unstructured-table
# https://stackoverflow.com/questions/2010481/how-do-you-get-all-the-rows-from-a-particular-table-using-beautifulsoup
# https://www.youtube.com/watch?v=gfpmKkxhb9M
# https://www.youtube.com/watch?v=XQgXKtPSzUI
# "Pandas for Everyone: Python Data Analysis" by Daniel Y. Chen
# https://www.crummy.com/software/BeautifulSoup/bs4/doc/
# "Python for Data Analysis" by Wes McKinney
# "Pandas Cookbook" by Theodore Petrou
# "Python Data Science Handbook" by Jake VanderPlas
# "Think Python" by Allen B. Downey
# https://realpython.com/
# Kevin Markham's Data School: https://www.youtube.com/watch?v=zXif_9RVadI

############################### Attributions ##########################################

# link to FCCY's 12/31/2017 10-K in html format.
# https://www.sec.gov/ix?doc=/Archives/edgar/data/1141807/000114180718000005/fccy-20171231x10k.htm

# link to all EDGAR filings
# https://www.sec.gov/Archives/edgar/full-index/
# select a year, then quarter, then master.idx which you will open with a text editor.
# you can see there are various filings (10-K, 10-Q, ). Iterating through each of these fillings can also be automated.

# Open FCCY's 12/31/2017 10-K
u = requests.get('https://www.sec.gov/Archives/edgar/data/1141807/000114180718000005/0001141807-18-000005.txt')

soup = BeautifulSoup(u.text, 'html.parser')

# Below includes spans and tables.
results= soup.find_all(['span', 'table'])

len(results)

# Find where the auditor's report(s) is. The financial statements will be after the auditor's report and before the beginning of the notes to the financial statements. There is substantial variation in the titles contained in various 10-K's. One approach is to condition on the various combinations of titles and look for those directly, but I opted to use the structure of the 10-K to narrow down the tables to only include the financial statements, and then to identify each financial statement based on the account names typically found within that financial statement.

# Condition where the audit report is.
position_audit= []
for i, j in enumerate(results):
        if "report of independent registered public accounting firm" in j.text.lower():
            position_audit.append(i)

position_audit

lower_bound= max(position_audit)

# Find start of the notes to the financial statements
position_notes= []
notes_text= []
for i, j in enumerate(results):
        if "notes to consolidated financial statements" in j.text.lower():
            position_notes.append(i)
            notes_text.append(j)

# The structure of the beginning of the notes has the company name, then the word 'notes', then the time period. I exploit that below. The list with the various month-ends is so that this generalizes to non-fiscal year end entities.
months= ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december']
for i in position_notes:
    for j in months:
        if j in results[i+1].text.lower():
            print(i)
            upper_bound=i

# The approximate boundaries to the financial statements are 9265 and 11478. I modify the results set to include the content between the upper and lower bounds above. These are the only parts of the 10-K we care about.
financial_statements= results[lower_bound:upper_bound]

# Read the file out as text so I can then read it back in and use pd.read_html() to pull the tables. Need to convert the list objects to strings prior to writing out. There is probably a better way to do this, but this works for now.
with open('financial_statements.txt', 'w') as fn:
    for item in financial_statements:
        fn.write(str(item))

content = open('financial_statements.txt', 'r').read()

tables2= pd.read_html(content)
len(tables2)

# It should seem strange that there are 9 tables in this 10-K. The financial statements typically consist of the balance sheet, income statement, statement of cash flows, and statement of changes in owners' equity. Depending on the types of transactions firms engage in, there may also be a comprehensive income statement. That's five total financial statements, but FCCY's 10-K contains 9 tables in the financial statement section. Let's explore that below.

tables2[0]
# The first table is the balance sheet.
tables2[1]
# The second table is the income statement.
tables2[2]
# The third table is the comprehensive income statement.
tables2[3]
# The fourth element is actually a note at the bottom of the comprehensive income statement. This is not a financial statement.
tables2[4]
# Same as above.
tables2[5]
# Same as above.
tables2[6]
#  The seventh table is the statement of changes in owners' equity.

tables2[7]
# The eight table is part of the statement of cash flows. FCCY's statement of cash flows is split across two different pages and has been split across two different tables.
tables2[8]
# The ninth element is the portion of the statement of cash flows that is on the split page.

# The order of the financial statements will differ across companies, so below we identify each financial statement based on a cumulative count of some account titles that typically show up on that financial statement. This will also eliminate the notes that were tagged as tables (the fourth, fifth, and sixth elements shown above).

df_temp= pd.DataFrame(tables2[0]).copy()

# Balance Sheet #########################
marker=0
keys= []
values= []
for e,j in enumerate(tables2):
    df_temp= pd.DataFrame(tables2[e]).copy()
    try:
        # Condition on first column being a pandas object which contains the account titles.
        if df_temp[0].dtypes=='O':
        # CIK 1007019 has extra spaces between the account titles so .sum() below returns no hits. To avoid this, first remove all of the spaces between the words in the account titles. Also, ignore case since capitalization conventions will vary across firms.
            df_temp[0]= df_temp[0].str.replace(' ', '')
            t=df_temp[0].str.match("assets|totalassets|totalliabilities|liabilities|liabilitiesandshareholders'equity|shareholder'sequity|stockholders'equity|liabilitiesandstockholders'equity|totalstockholders'equity|totalliabilities,redeemableconvertiblepreferredstock,andstockholders'equity|retainedearnings|retainedearnings(deficit)|accumulated othercomprehensiveincome|accumulatedothercomprehensiveincome(loss)|additionalpaidincapital|additionalpaid-incapital|stockholders'equity(deficiency)|noncontrollinginterests", case=False).sum()
            keys.append(e)
            values.append(t)
        else:
            continue
    except Exception:
        continue

dict= dict(zip(keys, values))
marker= max(dict, key=dict.get)
del dict

tables2[marker]

# No exceptions noted. p/f/r.

# Income Statement
marker=0
keys= []
values= []
for e,j in enumerate(tables2):
    df_temp= pd.DataFrame(tables2[e]).copy()
    try:
        # Condition on first column being a pandas object which contains the account titles.
        if df_temp[0].dtypes=='O':
        # CIK 1007019 has extra spaces between the account titles so .sum() below returns no hits. To avoid this, first remove all of the spaces between the words in the account titles. Also, ignore case since capitalization conventions will vary across firms.
            df_temp[0]= df_temp[0].str.replace(' ', '')
            t=df_temp[0].str.match("netincome|incometaxes|otherincome|netincomepercommonshare|basic|diluted|weightedaveragesharesoutstanding", case=False).sum()
            keys.append(e)
            values.append(t)
        else:
            continue
    except Exception:
        continue

dict= dict(zip(keys, values))
marker= max(dict, key=dict.get)
del dict

marker
tables2[marker]

# No exceptions noted.

# Owners Equity Statement #######
marker=0
keys= []
values= []
for e,j in enumerate(tables2):
    df_temp= pd.DataFrame(tables2[e]).copy()
    try:
        # Condition on first column being a pandas object which contains the account titles.
        if df_temp[0].dtypes=='O':
        # CIK 1007019 has extra spaces between the account titles so .sum() below returns no hits. To avoid this, first remove all of the spaces between the words in the account titles. Also, ignore case since capitalization conventions will vary across firms.
            df_temp[0]= df_temp[0].str.replace(' ', '')
            t=df_temp[0].str.match("share-basedcompensation|dividends|stockoptions", case=False).sum()
            keys.append(e)
            values.append(t)
        else:
            continue
    except Exception:
        continue

dict= dict(zip(keys, values))
marker= max(dict, key=dict.get)
del dict

marker
tables2[marker]

# OCI Statement ################
marker=0
keys= []
values= []
for e,j in enumerate(tables2):
    df_temp= pd.DataFrame(tables2[e]).copy()
    try:
        # Condition on first column being a pandas object which contains the account titles.
        if df_temp[0].dtypes=='O':
        # CIK 1007019 has extra spaces between the account titles so .sum() below returns no hits. To avoid this, first remove all of the spaces between the words in the account titles. Also, ignore case since capitalization conventions will vary across firms.
            df_temp[0]= df_temp[0].str.replace(' ', '')
            t=df_temp[0].str.match("othercomprehensiveincome(loss)|comprehensiveincome|totalothercomprehensiveincome|totalothercomprehensiveloss", case=False).sum()
            keys.append(e)
            values.append(t)
        else:
            continue
    except Exception:
        continue

dict= dict(zip(keys, values))
marker= max(dict, key=dict.get)
del dict

marker
tables2[marker]

# No exceptions noted.

# SOCF ###############
marker=0
keys= []
values= []
for e,j in enumerate(tables2):
    df_temp= pd.DataFrame(tables2[e]).copy()
    try:
        # Condition on first column being a pandas object which contains the account titles.
        if df_temp[0].dtypes=='O':
        # CIK 1007019 has extra spaces between the account titles so .sum() below returns no hits. To avoid this, first remove all of the spaces between the words in the account titles. Also, ignore case since capitalization conventions will vary across firms.
            df_temp[0]= df_temp[0].str.replace(' ', '')
            t=df_temp[0].str.match("operating|investing|financing", case=False).sum()
            keys.append(e)
            values.append(t)
        else:
            continue
    except Exception:
        continue

dict= dict(zip(keys, values))
dict
marker= max(dict, key=dict.get)
del dict

marker
tables2[marker]

# Since FCCY splits the statement of cash flows into two tables, only the first table meets the logical conditions of the code via the maximum. I'm not sure 1) how frequently tables are split across pages or 2) how the code could be modified in a systematic way to capture both parts of tables for entities with split financial statements. I may look into this in the future if it appears to cause systematic problems.

tables2_ref= tables2[0].copy()

# Drop null rows.
tables2[0].dropna(how='all', inplace=True)

# The next step is to turn this into a useable dataframe. You can see above that there are multiple columns in the table. The structure contains three columns for 2017, a null column, and then three columns for 2016. Negative numbers begin with "(" and the closing ")" is in the third column for each year. The columns that we want are the ones that contain the fewest null values.

col_count=tables2[0].notnull().sum()
col_count

# Column 0 has the account titles, columns 1 and 2 contain the 2017 numbers, and columns 5 and 6 contain the 2016 numbers. Note that columns 1 and 2 and 5 and 6 typically contain duplicates. Columns 1 and 5 also sometimes contain a dollar sign "$". We need to remove the duplicates and only keep one column for each year. But we also need to be aware that the dollar signs are not nulls, so the count of non-null values in the columns with and without dollar signs is the same. To deal with this, we reset the $'s equal to NaN's and then re-count the number of missing values.

tables2[0].replace({'$': np.nan}, inplace=True)

col_count=tables2[0].notnull().sum()
col_count

# Replace column 0, which is the account titles, to 0. This column has even more non-missing values than the account balances so I don't want the maximum to condition on this. I'm looking for the second highest non-missing columns since these are the account balances.
col_count[0]= 0
col_count

# Boolean series for which columns have the most non-missing values.
col_index =col_count== col_count.max()
col_index

# Can see above that the 3rd and 6th columns are the ones with the account balances. But later I also want column 0 with the account names, so below I manually change the 0th index back to True.
col_index[0]=True

bsheet= tables2[0].copy()

bsheet= bsheet.loc[:, col_index]

# Identify and drop header rows on the balance sheet without corresponding numbers (e.g., ASSETS, LIABILITIES AND SHAREHOLDERS' EQUITY, etc.)
dropper= (bsheet[bsheet.columns[-1]].notnull()) & (bsheet[bsheet.columns[-2]].notnull())
dropper
bsheet= bsheet[dropper]

bsheet= bsheet.transpose()
bsheet.reset_index(inplace= True, drop=True)

# Make the 0th row the column names of the DataFrame.
bsheet.columns= bsheet.loc[0]

bsheet.drop(0, inplace=True)
bsheet

# Pull in the CIK from the url into the dataframe.
url= 'https://www.sec.gov/Archives/edgar/data/1141807/000114180718000005/0001141807-18-000005.txt'
start= 'https://www.sec.gov/Archives/edgar/data/'
# remove above then keep everything until / and that will be the cik.
url2= url.replace(start,'')
url2
sep= '/'
# split on seperator one time, and keep the fist element.

cik= url2.split(sep)[0]
cik

bsheet['CIK']= int(cik)

# Rename the null column to Year.
bsheet.rename(columns={bsheet.columns[0]:'Year'}, inplace=True)

# Set the multi-index.
bsheet.set_index(['CIK', 'Year'], inplace=True)

# Next, convert the strings into integers. Per visual inspection, need to deal with three issues 1) leading ( for negative numbers, 2) pulling commas out of each string and 3) — for values of 0.
bsheet=bsheet.apply(lambda x: x.str.replace('(', '-'))
bsheet= bsheet.apply(lambda x: x.str.replace(',', ''))
bsheet

# can see above that variables with 0 values have a dash-type character.
bsheet['Other Real Estate Owned'].value_counts()
bsheet= bsheet.apply(lambda x: x.str.replace('—', '0'))
bsheet

# Convert strings to integers
bsheet= bsheet.apply(lambda x: pd.to_numeric(x))
bsheet
bsheet.info()


from functools import cached_property
from typing import Literal
from pydantic import BaseModel, computed_field
from datetime import datetime
import geopy.distance
from collections import defaultdict

class Position(BaseModel):
    lat: float
    lon: float
    uncertainty: float
    timestamp: datetime

    # easting: float
    # northing: float

class Trip(BaseModel):
    trip_id: str
    load: Literal["Stone", "Equipment", "Soil", "4"]
    quantity: float
    positions: list[Position]
    dump_latlon: tuple[float, float]
    load_latlon: tuple[float, float]

    @computed_field
    @cached_property
    def latlons(self) -> list[tuple[float, float]]:
        # List of latlons
        return [(pos.lat, pos.lon) for pos in self.positions]

    @computed_field
    @cached_property
    def length(self) -> float:
        # Distacnce in kilometers
        return geopy.distance.geodesic(*self.latlons).km

    @computed_field
    @cached_property
    def duration(self) -> float:
        # Duration in minutes
        return (self.end_date - self.start_date).total_seconds() / 60.0

    @property
    def start_date(self) -> datetime:
        return self.positions[0].timestamp

    @property
    def end_date(self) -> datetime:
        return self.positions[-1].timestamp

class Machine(BaseModel):
    machine_type: Literal["Truck", "Dumper", "Tippbil"]
    machine_id: int | str  # If looking at 2023 data needs to be string
    machine_name: str | None
    trips: list[Trip]

    @computed_field
    @cached_property
    def trips_dict(
        self,
    ) -> dict[Literal["Stone", "Equipment", "Soil", "4"], list[Trip]]:
        # Returns a dictionary of trips with key as the load type used
        partitions = defaultdict(lambda: [])
        set(partitions[trip.load].append(trip) for trip in self.trips)
        return partitions

    @computed_field
    @cached_property
    def total_length(self) -> float:
        # The combined length of all trips (in kilometers)
        return sum(trip.length for trip in self.trips)

    @computed_field
    @cached_property
    def total_duration(self) -> float:
        # The combined duration (in minutes) of each trip
        return sum(trip.duration for trip in self.trips)

    @computed_field
    @cached_property
    def total_quantity(self) -> float:
        # The combined quantity of each trip
        return sum(trip.quantity for trip in self.trips)

    @computed_field
    @cached_property
    def all_positions(self) -> list[Position]:
        all_pos = [trip.positions for trip in self.trips]
        all_pos = [item for sublist in all_pos for item in sublist]
        return all_pos

    @computed_field
    @cached_property
    def all_loads(self) -> list[Position]:
        temp_load: list[Position] = []
        for trip in self.trips:
            temp_load.append(
                Position(
                    lat=trip.load_latlon[0],
                    lon=trip.load_latlon[1],
                    uncertainty=0.0,
                    timestamp=trip.positions[0].timestamp,
                )
            )
        return temp_load

    @computed_field
    @cached_property
    def all_dumps(self) -> list[Position]:
        temp_dump: list[Position] = []
        for trip in self.trips:
            for position in trip.positions:
                if trip.dump_latlon == (position.lat, position.lon):
                    temp_dump.append(
                        Position(
                            lat=trip.dump_latlon[0],
                            lon=trip.dump_latlon[1],
                            uncertainty=0.0,
                            timestamp=position.timestamp,
                        )
                    )
        return temp_dump

    @computed_field
    @cached_property
    def list_of_idle_times(self) -> list[list[Position]]:
        list_of_idle_times: list[list[Position]] = []
        temp_idle_list: list[Position] = []
        # We start processing. Are going to iterate over all positions, from first to last
        for i in range(1, len(self.all_positions[1:])):
            current_pos = self.all_positions[i]
            prev_pos = self.all_positions[i - 1]
            current_time = current_pos.timestamp

            # Seconds passed since last timestamp
            seconds_gone = (current_pos.timestamp - prev_pos.timestamp).total_seconds()

            if seconds_gone > 0:
                # Meters driven since last timestamp
                meters_driven = geopy.distance.geodesic(
                    (current_pos.lat, current_pos.lon), (prev_pos.lat, prev_pos.lon)
                ).m

                # Speed during last timestamp
                speed_kmh = (meters_driven / seconds_gone) * 3.6

                if speed_kmh < 5:  # Speed limit - can be changed as user want
                    temp_idle_list.append(
                        Position(
                            lat=current_pos.lat,
                            lon=current_pos.lon,
                            uncertainty=0,
                            timestamp=current_time,
                        )
                    )
                    if i == len(self.all_positions[1:]) - 1:  # i.e. last iteration
                        list_of_idle_times.append(temp_idle_list)
                else:
                    if len(temp_idle_list) > 0:
                        list_of_idle_times.append(temp_idle_list)
                        temp_idle_list = []  # Re-initialize

        return list_of_idle_times

    @computed_field
    @cached_property
    def total_idle_seconds(self) -> float:
        return sum(
            [
                (item[-1].timestamp - item[0].timestamp).total_seconds()
                for item in self.list_of_idle_times
            ]
        )

    @property
    def soil_trips(self):
        return self.trips_dict["Soil"]

    @property
    def equipment_trips(self) -> list[Trip]:
        return self.trips_dict["Equipment"]

    @property
    def stone_trips(self) -> list[Trip]:
        return self.trips_dict["Stone"]

    @property
    def four_trips(self) -> list[Trip]:
        return self.trips_dict["4"]


# This Python 3 environment comes with many helpful analytics libraries installed

# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python

# For example, here's several helpful packages to load in

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.

# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

# import os

# for dirname, _, filenames in os.walk('/kaggle/input'):

#     for filename in filenames:

#         print(os.path.join(dirname, filename))

# Any results you write to the current directory are saved as output.
import matplotlib.pyplot as plt

import seaborn as sns
data = pd.read_csv("../input/covid19-global-forecasting-week-2/train.csv")

test = pd.read_csv("../input/covid19-global-forecasting-week-2/test.csv")

submit = pd.read_csv("../input/covid19-global-forecasting-week-2/submission.csv")
data.head()
data.shape
data.Country_Region.nunique()
data.Country_Region.value_counts()
data.Country_Region[data.Country_Region=="India"].value_counts()
print("Start date:",min(data.Date))

print("End date:",max(data.Date))
min(data.Date).split("-")
data.head()
# data.groupby(data.Country_Region)
# a = data.groupby(['Country_Region', 'ConfirmedCases'])
data.groupby(['Country_Region']).agg({'ConfirmedCases':['sum']})
confirmed_total = data.groupby(['Date']).agg({'ConfirmedCases':['sum']})

fatalities_total = data.groupby(['Date']).agg({'Fatalities':['sum']})
total = confirmed_total.join(fatalities_total)
fig, (axis1, axis2) = plt.subplots(1,2, figsize=(17,8))

axis1.set_xlabel("Date",size=20)

axis1.set_ylabel("Number of cases",size=20)

axis1.set_title("Global CONFIRMED cases",size=20)

axis2.set_xlabel("Date",size=20)

axis2.set_ylabel("Number of cases",size=20)

axis2.set_title("Global FATALITIES cases",size=20)

confirmed_total.plot(ax=axis1, color='orange')

fatalities_total.plot(ax=axis2, color='blue')
confirmed_total_india = data[data.Country_Region == 'India'].groupby(['Date']).agg({'ConfirmedCases':['sum']})

fatalities_total_india = data[data.Country_Region == 'India'].groupby(['Date']).agg({'Fatalities':['sum']})

total_india = confirmed_total_india.join(fatalities_total_india)
total_india
fig, (axis1, axis2) = plt.subplots(1,2, figsize=(17,8))

axis1.set_xlabel("Date",size=20)

axis1.set_ylabel("Number of cases",size=20)

axis1.set_title("Global CONFIRMED cases(INDIA)",size=20)

axis2.set_xlabel("Date",size=20)

axis2.set_ylabel("Number of cases",size=20)

axis2.set_title("Global FATALITIES cases(INDIA)",size=20)

confirmed_total_india.plot(ax=axis1, color='orange')

fatalities_total_india.plot(ax=axis2, color='blue')
confirmed_total_china = data[data.Country_Region == 'China'].groupby(['Date']).agg({'ConfirmedCases':['sum']})

fatalities_total_china = data[data.Country_Region == 'China'].groupby(['Date']).agg({'Fatalities':['sum']})

total_china = confirmed_total_china.join(fatalities_total_china)
fig, (axis1, axis2) = plt.subplots(1,2, figsize=(17,8))

axis1.set_xlabel("Date",size=20)

axis1.set_ylabel("Number of cases",size=20)

axis1.set_title("Global CONFIRMED cases(CHINA)",size=20)

axis2.set_xlabel("Date",size=20)

axis2.set_ylabel("Number of cases",size=20)

axis2.set_title("Global FATALITIES cases(CHINA)",size=20)

confirmed_total_china.plot(ax=axis1, color='orange')

fatalities_total_china.plot(ax=axis2, color='blue')
confirmed_total_italy = data[data.Country_Region == 'Italy'].groupby(['Date']).agg({'ConfirmedCases':['sum']})

fatalities_total_italy = data[data.Country_Region == 'Italy'].groupby(['Date']).agg({'Fatalities':['sum']})

total_italy = confirmed_total_italy.join(fatalities_total_italy)
fig, (axis1, axis2) = plt.subplots(1,2, figsize=(17,8))

axis1.set_xlabel("Date",size=20)

axis1.set_ylabel("Number of cases",size=20)

axis1.set_title("Global CONFIRMED cases(ITALY)",size=20)

axis2.set_xlabel("Date",size=20)

axis2.set_ylabel("Number of cases",size=20)

axis2.set_title("Global FATALITIES cases(ITALY)",size=20)

confirmed_total_italy.plot(ax=axis1, color='blue',linewidth=6)

fatalities_total_italy.plot(ax=axis2, color='red',linewidth=6)


from rembg import remove
from PIL import Image, ImageFilter

input_path = "input.png"
output_path = "output.png"

input_image = Image.open(input_path)
transparent_image = remove(input_image)

blurred_image = input_image.filter(ImageFilter.GaussianBlur(radius=5))

blurred_image.paste(transparent_image, (0, 0), transparent_image)
blurred_image.save(output_path, quality=100)


from nn_wtf.data_sets import DataSets
from nn_wtf.neural_network_graph import NeuralNetworkGraph
from nn_wtf.tests.util import create_train_data_set, train_data_input, \
    create_vector, allow_fail, train_neural_network

import unittest

__author__ = 'Lene Preuss <lene.preuss@gmail.com>'
# pylint: disable=missing-docstring

class PredictorTest(unittest.TestCase):

    @allow_fail(max_times_fail=1)
    def test_all_prediction_functions_at_once_to_save_computing_time(self):
        """Training takes time, if I run tests separately I have to train for each test."""

        graph = train_neural_network(create_train_data_set())

        self.assertEqual(0, graph.get_predictor().predict(train_data_input(0)))
        self.assertEqual(1, graph.get_predictor().predict(train_data_input(1)))

        probabilities_for_0 = graph.get_predictor().prediction_probabilities(train_data_input(0))
        self.assertGreater(probabilities_for_0[0], probabilities_for_0[1])

        probabilities_for_1 = graph.get_predictor().prediction_probabilities(train_data_input(1))
        self.assertGreater(probabilities_for_1[1], probabilities_for_1[0])

        self._check_multiple_values_get_predicted(graph, [0, 1])
        self._check_multiple_values_get_predicted(graph, [1, 0])
        self._check_multiple_values_get_predicted(graph, [0, 1, 0])

    def _check_multiple_values_get_predicted(self, graph, train_data):
        predictions = graph.get_predictor().predict_multiple(
            generate_train_data(train_data), len(train_data)
        )
        self.assertEqual(train_data, predictions)

def repeat_list_items(data, num_repeats=2):
    from itertools import repeat
    return [x for item in data for x in repeat(item, num_repeats)]

def generate_train_data(values):
    return create_vector(repeat_list_items(values, 2))


import json
import logging
import networkx as nx
import networkx.algorithms as nxa
import pdb
import requests

from herc_common.utils import empty_if_keyerror, WIKIDATA_BASE

logger = logging.getLogger(__name__)

WIKIDATA_PROPS_EXPAND = ['P31', 'P279', 'P301', 'P361', 'P366',
                         'P527', 'P910', 'P921', 'P2578', 'P2579']
WIKIDATA_EXTERNAL_IDS = {
    'P349': 'https://id.ndl.go.jp/auth/ndlsh/',
    'P352': 'https://www.uniprot.org/uniprot/',
    'P486': 'https://meshb.nlm.nih.gov/record/ui?ui=',
    'P646': 'https://freebase.toolforge.org/',
    'P672': 'http://id.nlm.nih.gov/mesh/',
    'P3382': 'https://www.genedb.org/#/gene/',
    'P3827': 'https://www.jstor.org/topic/',
    'P3916': 'http://vocabularies.unesco.org/thesaurus/',
    'P5748': 'http://uri.gbv.de/terminology/bk/',
    'P6366': 'https://academic.microsoft.com/v2/detail'
}

def get_centrality_algorithm_results(g, algorithm, stop_uris, top_n):
    """ Return top n nodes from a graph based on the given centrality algorithm.

    Parameters
    ----------
    g: :obj:`networkx.graph`
    algorithm: callable
    stop_uris: list of str
    top_n: int

    Returns
    -------
    list of (str, float)
        List of tuples where the first element is the label of the node and the second
        one is the score obtained for the given algorithm.
    """
    metrics = algorithm(g)
    metrics = {key: val for key, val in metrics.items()
                if key not in stop_uris}
    best_qids = sorted(metrics, key=metrics.get, reverse=True)[:top_n]
    return [(g.nodes[qid], metrics[qid]) for qid in best_qids]

def get_largest_connected_subgraph(g):
    S = [g.subgraph(c).copy() for c in nxa.components.connected_components(g)]
    return max(S, key=len)

def _build_uri(entity_id):
    return f"http://www.wikidata.org/entity/{entity_id}"

@empty_if_keyerror
def _get_aliases(entity_info, lang='en'):
    return [alias['value']
            for alias in entity_info['aliases'][lang]]

@empty_if_keyerror
def _get_desc(entity_info, lang='en'):
    return entity_info['descriptions'][lang]['value']

def _get_uris(entity_info, term_id):
    res = [f"{WIKIDATA_BASE}/{term_id}"]
    for claim_key, claim_values in entity_info['claims'].items():
        if claim_key not in WIKIDATA_EXTERNAL_IDS.keys():
            continue

        for value in claim_values:
            snaktype = value['mainsnak']['snaktype']
            if snaktype in ['novalue', 'somevalue']:
                continue

            base_url = WIKIDATA_EXTERNAL_IDS[claim_key]
            fragment = value['mainsnak']['datavalue']['value']
            external_url = base_url + fragment
            res.append(external_url)
    return res

@empty_if_keyerror
def _get_labels(entity_info, lang='en'):
    return entity_info['labels'][lang]['value']

class WikidataGraphBuilder():
    """ Build a Wikidata graph from a given set of seed concepts.

    This class can be used to build a graph with Wikidata

    Parameters
    ----------
    max_hops: int (default=2)
        Maximum depth of the graph with respect to the seed nodes used
        to build it.

    additional_props: list of str (default=None)
        List of properties to be expanded for each node in the graph. They
        will be added to the default list of properties of the graph builder.
    """

    def __init__(self, max_hops=2, additional_props=None, languages=['en', 'es']):
        self.entities_cache = {}
        self.max_hops = max_hops
        self.languages = languages
        self.props_to_expand = WIKIDATA_PROPS_EXPAND
        if additional_props:
            self.props_to_expand += additional_props

    def build_graph(self, terms):
        """Build the graph for the given terms."""
        logger.info("Started building graph.")
        G = nx.Graph()
        for term in terms:
            logger.debug("Seed term: %s", term[0])
            term_uri = term[1]
            if term_uri is not None:
                term_id = term_uri.split('/')[-1]
                self._add_wd_node_info(G, term_id, None, 0)
        logger.info("Finished building graph.")
        return G

    def _add_wd_node_info(self, graph, term_id, prev_node, curr_hop):
        logger.debug("Visiting entity '%s' - Curr hop: %d", term_id, curr_hop)
        if curr_hop > self.max_hops or term_id in ['Q4167410', 'Q4167836', 'Q37517',
            'Q16334298', 'Q59541917']:
            return

        if term_id not in self.entities_cache:
            entity_info = self._fetch_entity_info_of(term_id)
            self.entities_cache[term_id] = entity_info
        else:
            entity_info = self.entities_cache[term_id]

        if 'claims' not in entity_info:
            return

        if term_id not in graph.nodes:
            graph.add_node(term_id)
            #graph.nodes[term_id]['alias'] = _get_aliases(entity_info)
            graph.nodes[term_id]['qid'] = term_id
            graph.nodes[term_id]['descs'] = {lang: _get_desc(entity_info, lang)
                for lang in self.languages}
            graph.nodes[term_id]['labels'] = {lang: _get_labels(entity_info, lang)
                for lang in self.languages}
            graph.nodes[term_id]['label'] = _get_labels(entity_info)
            graph.nodes[term_id]['uris'] = _get_uris(entity_info, term_id)
            graph.nodes[term_id]['n'] = curr_hop

        if prev_node is not None and not graph.has_edge(prev_node, term_id):
            graph.add_edge(prev_node, term_id)

        for claim_key, claim_values in entity_info['claims'].items():
            if claim_key not in self.props_to_expand:
                continue

            for value in claim_values:
                snaktype = value['mainsnak']['snaktype']
                if snaktype in ['novalue', 'somevalue']:
                    continue

                new_node_id = value['mainsnak']['datavalue']['value']['id']
                self._add_wd_node_info(graph, new_node_id, term_id, curr_hop + 1)

    def _fetch_entity_info_of(self, term_id):
        endpoint = f"{WIKIDATA_BASE}/api.php?action=wbgetentities&ids={term_id}&languages=en|es&format=json"
        res = requests.get(endpoint)
        if res.status_code != 200:
            logger.warning("There was an error calling endpoint for term %s: %s",
                            term_id, res.content)
            raise Error()
        content = json.loads(res.text)
        return content['entities'][term_id]


import typing as t

from hako.bricks.interfaces import PRIMITIVE_ENTRIES

if t.TYPE_CHECKING:
    from .builder import CodeBuilder, SymbolAliasMapping, Symbol
    from ..bricks.shaping import ShapeNode

class SnippetGrocery:
    def __init__(self, node: "ShapeNode", CB: "CodeBuilder") -> None:
        self.CB = CB
        self.ST = ST = CB.ST
        self.Node = node

        boxtype = node.boxtype
        Primitives = boxtype.Primitives
        Heuristics = boxtype.Heuristics

        target = node.target
        self.HasTarget = target is not None
        self.TargetGrocery = SnippetGrocery(target, CB) if self.HasTarget else None

        self.DEFINED_ISNOTA2 = Primitives["ISNOTA2"] is not None
        self.DEFINED_ITER2 = Primitives["ITER2"] is not None
        self.DEFINED_NEW_FROM_ITER2 = Primitives["NEW_FROM_ITER2"] is not None
        self.DEFINED_GET_LENGTH = Primitives["GET_LENGTH"] is not None
        self.DEFINED_GET_LENGTH2 = Primitives["GET_LENGTH2"] is not None

        self.SYM_INDICES = ST.NewSymbol()
        self.SYM_LENGTH_REF = ST.NewSymbol()
        self.SYM_LENGTH_SELF = ST.NewSymbol()
        self.SYM_SPROOF = ST.NewSymbol()
        self.SYM_IPROOF = ST.NewSymbol()
        self.SYM_GLPROOF = ST.NewSymbol()
        self.SYM_NIPROOF = ST.NewSymbol()

        self._PrimitivesGet = Primitives.__getitem__
        self._HeuristicsGet = Heuristics.__getitem__

        mapping = boxtype.__prepare_constants__(node.mdata)
        for SYM, constant in mapping.items():
            mapping[SYM] = SYM2 = ST.NewDefinedSymbol()
            CB.BindConstant(SYM2, constant)
        self._symbol_alias_mapping: SymbolAliasMapping = mapping

        for key, value in Heuristics.items():
            setattr(self, f"H_{key}", value)

        self._proof_pool = []

    # Helper functions for generating snippets

    def Define_LENGTH_REF(self) -> None:
        ST, CB = self.ST, self.CB
        self.Define_GET_LENGTH2_PROOF()
        if not ST.SymbolDefined(self.SYM_LENGTH_REF):
            ST.Define(self.SYM_LENGTH_REF)

        if self.DEFINED_GET_LENGTH:
            CB.Push(f"{{{self.SYM_LENGTH_REF}}} = {self.CODE_GET_LENGTH()}")
        elif ST.SymbolDefined(self.SYM_INDICES):
            CB.Push(f"{{{self.SYM_LENGTH_REF}}} = len({{{self.SYM_INDICES}}})")

    def CODE_GET_LENGTH2_AUTO(self) -> str:
        if self.DEFINED_GET_LENGTH2:
            self.ST.Alias(self.SYM_GLPROOF, "PROOF")
            return self.CODE_GET_LENGTH2()
        else:
            return self.CODE_GET_LENGTH()

    def CODE_LENGTH_MISMATCHED_AUTO(self) -> str:
        if self.H_SHAPE_IMPLIES_LENGTH:
            return "False"

        self.ST.Alias(self.SYM_GLPROOF, "PROOF")
        return f"({self.CODE_GET_LENGTH2_AUTO()} != {{{self.SYM_LENGTH_REF}}})"

    def CODE_IS_NOT_EMPTY(self) -> str:
        return f"({{{self.SYM_LENGTH_REF}}})"

    def CODE_ITER_AUTO(self) -> str:
        if self.H_IS_NAIVE_ITERATOR:
            return "{VAL}"
        else:
            return self.CODE_ITER()

    def CODE_ITER2_AUTO(self) -> str:
        if self.H_IS_NAIVE_ITERATOR:
            return "{VAL}"

        if self.DEFINED_ITER2:
            self.ST.Alias(self.SYM_IPROOF, "PROOF")
            CODE_ITER = self.CODE_ITER2()
        else:
            CODE_ITER = self.CODE_ITER()

        return CODE_ITER

    def CODE_ITER2_ZIP_AUTO(self, VALS="VALS") -> str:
        if self.H_IS_NAIVE_ITERATOR:
            return f"zip(*{{{VALS}}})"

        if self.DEFINED_ITER2:
            self.ST.Alias(self.SYM_IPROOF, "PROOF")
            CODE_ITER = self.CODE_ITER2()
        else:
            CODE_ITER = self.CODE_ITER()

        return f"zip(*({CODE_ITER} for {{VAL}} in {{{VALS}}}))"

    def CODE_ISNOTA2_AUTO(self) -> str:
        if self.DEFINED_ISNOTA2:
            self.ST.Alias(self.SYM_SPROOF, "PROOF")
            return self.CODE_ISNOTA2()
        else:
            return self.CODE_ISNOTA()

    def CODE_NEW_FROM_ITER2_AUTO(self) -> str:
        if self.HasTarget:
            self = self.TargetGrocery

        if self.DEFINED_NEW_FROM_ITER2:
            self.ST.Alias(self.SYM_NIPROOF, "PROOF")
            return self.CODE_NEW_FROM_ITER2()
        else:
            return self.CODE_NEW_FROM_ITER()

    # Dummy functions for IntelliSense

    def CODE_ISNOTA(self) -> str:
        ...

    def CODE_ISNOTA2(self) -> str:
        ...

    def CODE_ISNOTA2_PROOF(self) -> str:
        ...

    def CODE_ITER(self) -> str:
        ...

    def CODE_ITER2(self) -> str:
        ...

    def CODE_ITER2_PROOF(self) -> str:
        ...

    def CODE_NEW(self) -> str:
        ...

    def CODE_NEW_FROM_ITER(self) -> str:
        ...

    def CODE_NEW_FROM_ITER2(self) -> str:
        ...

    def CODE_NEW_FROM_ITER2_PROOF(self) -> str:
        ...

    def CODE_GET_INDICES(self) -> str:
        ...

    def CODE_GET_DUMMY(self) -> str:
        ...

    def CODE_GET_ITEM(self) -> str:
        ...

    def CODE_PICK(self) -> str:
        ...

    def CODE_GET_LENGTH(self) -> str:
        ...

    def CODE_GET_LENGTH2(self) -> str:
        ...

    def CODE_GET_LENGTH2_PROOF(self) -> str:
        ...

    # The dirty part: function auto generation

    def _make_code_func(name: str):
        def _code_func(self: "SnippetGrocery") -> str:

            code = self._PrimitivesGet(name)

            if code is None:
                raise NotImplementedError(
                    f"Primitive {name!r} not implemented on BoxType {self.Node!r}"
                )

            self.ST.AliasMap(self._symbol_alias_mapping)

            if code.__class__ is str:
                return code

            return code(self)

        return _code_func

    for name in PRIMITIVE_ENTRIES:
        locals()[f"CODE_{name}"] = _make_code_func(name)

    del _make_code_func, name

    def _make_define_proof(
        name: str,
        default_cond: t.Callable[[t.Any], bool],
        sym_getter: t.Callable[["SnippetGrocery"], "Symbol"],
    ):
        def _define_proof(
            self: "SnippetGrocery",
            cond: bool = None,
            setup: str = None,
        ):
            if cond == False or cond is None and not default_cond(self):
                return

            SYM = sym_getter(self)

            ST = self.ST
            ST.DefineOrOverwrite(SYM)
            if setup is not None:
                self.CB.Push(setup)

            for name2, SYM2 in self._proof_pool:
                if (name, name2) in self.H__SAME_PROOF:
                    self.CB.Push(f"{{{SYM}}} = {{{SYM2}}}")
                    break
            else:
                code = getattr(self, f"CODE_{name}")()
                self.CB.Push(f"{{{SYM}}} = {code}")
            self._proof_pool.append((name, SYM))

        return _define_proof

    Define_ISNOTA2_PROOF = _make_define_proof(
        "ISNOTA2_PROOF",
        lambda self: self.DEFINED_ISNOTA2,
        lambda self: self.SYM_SPROOF,
    )

    Define_ITER2_PROOF = _make_define_proof(
        "ITER2_PROOF",
        lambda self: self.DEFINED_ITER2 and not self.H_IS_NAIVE_ITERATOR,
        lambda self: self.SYM_IPROOF,
    )

    Define_GET_LENGTH2_PROOF = _make_define_proof(
        "GET_LENGTH2_PROOF",
        lambda self: self.DEFINED_GET_LENGTH2,
        lambda self: self.SYM_GLPROOF,
    )

    Define_NEW_FROM_ITER2_PROOF = _make_define_proof(
        "NEW_FROM_ITER2_PROOF",
        lambda self: self.DEFINED_NEW_FROM_ITER2,
        lambda self: self.SYM_NIPROOF,
    )

    del _make_define_proof

    def ResetProofPool(self):
        self._proof_pool.clear()


import numpy as np
import statsmodels.api as sm
import scipy as sp

def breusch_pagan_test(y,x):
     results=sm.OLS(y,x).fit()
     resid=results.resid
     n=len(resid)
     sigma2 = sum(resid**2)/n
     f = resid**2/sigma2 - 1
     results2=sm.OLS(f,x).fit()
     fv=results2.fittedvalues
     bp=0.5 * sum(fv**2)
     df=results2.df_model
     p_value=1-sp.stats.chi.cdf(bp,df)
     return round(bp,6), df, round(p_value,7)


import os

from appium.webdriver.common.appiumby import AppiumBy
from appium.webdriver.webdriver import WebDriver
from selenium.common import TimeoutException
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait

from apptest_project.app_po.wework.utils.error_handle import black_wrapper
from apptest_project.app_po.wework.utils.utils import Utils
from utils.log_until import logger
# For W3C actions

class BasePage:
    def __init__(self, driver: WebDriver = None):
        self.driver = driver

    @black_wrapper
    def find(self, by, value):
        '''
        查找元素,返回元素
        :param by:
        :param value:
        :return:
        '''
        logger.info(f"{value}通过{by}方式寻找")
        return self.driver.find_element(by, value)

    @black_wrapper
    def finds(self, by, value):
        '''
        查找多个元素,返回元素对象的列表
        :param by:
        :param value:
        :return:
        '''
        logger.info(f"{value}通过{by}方式寻找")
        return self.driver.find_elements(by, value)

    def find_and_click(self, by, value):
        '''
        查找元素并点击
        :param by:
        :param value:
        :return:
        '''
        logger.info(f"{value}通过{by}方式寻找并点击")
        self.driver.find_element(by, value).click()

    def find_and_sendkeys(self, by, value, text):
        '''
        查找元素并输入内容
        :param by:
        :param value:
        :return:
        '''
        logger.info(f"{value}通过{by}方式寻找并输入{text}")
        self.driver.find_element(by, value).send_keys(text)

    def find_and_text(self, by, value):
        logger.info(f"{value}通过{by}方式寻找并输出为text")
        return self.driver.find_element(by, value).text

    def set_implicitly_wait(self, time=1):
        '''
        设置隐式等待
        :param time:
        :return:
        '''
        logger.info(f"设置隐式等待时间为{time}秒")
        self.driver.implicitly_wait(time)

    # 滑动找元素并点击
    @black_wrapper
    def swipe_find(self, by, value):
        logger.info(f"{value}通过{by}方式滑动找到并点击")
        window_size = self.driver.get_window_size()
        width = window_size["width"]
        height = window_size["height"]
        startx = width / 2
        starty = height * 0.8
        endx = startx
        endy = height * 0.2
        duration = 0
        i = 1
        self.set_implicitly_wait(2)
        while True:
            eles = self.driver.find_elements(by=by, value=value)
            if i > 10:
                print("超过10次啦")
                break
            if len(eles) == 0:
                # print(f"第{i}次滑动")
                self.driver.swipe(startx, starty, endx, endy, duration)
                i += 1
            else:
                eles[0].click()
                self.set_implicitly_wait(2)
                break

    @black_wrapper
    def get_tips(self, by, value):
        '''
        获取toast 的文本
        :return:
        '''
        logger.info(f"{value}通过{by}方式并输出为text")
        return self.driver.find_element(by, value).text

    def swipe_element(self, by, value):
        logger.info(f"{value}通过{by}方式滑动找到并输出为text")
        window_size = self.driver.get_window_size()
        width = window_size["width"]
        height = window_size["height"]
        startx = width / 2
        starty = height * 0.8
        endx = startx
        endy = height * 0.2
        duration = 0
        i = 1
        while True:
            eles = self.driver.find_elements(by=by, value=value)
            if i > 10:
                print("超过10次啦")
                break
            if len(eles) == 0:
                # print(f"第{i}次滑动")
                self.driver.swipe(startx, starty, endx, endy, duration)
                i += 1
            else:
                return eles[0].text

    def delete_circulate(self, by, value):
        i = 1
        while True:
            eles = self.driver.find_elements(by=by, value=value)
            if i > 5:
                print("重名的有5个人?滚")
                break
            if len(eles) == 0:
                print(f"第{i}次删除")
                # 点击搜索结果
                el4 = self.driver.find_element(by=AppiumBy.XPATH, value="//*[@resource-id='com.tencent.wework:id/gdx']")
                el4.click()
                # 点击更多选项
                el5 = self.driver.find_element(by=AppiumBy.XPATH, value="//*[@resource-id='com.tencent.wework:id/l6l']")
                el5.click()
                # 编辑成员
                el6 = self.driver.find_element(by=AppiumBy.XPATH, value="//*[@resource-id='com.tencent.wework:id/c5x']")
                el6.click()
                self.swipe_find(AppiumBy.XPATH, "//*[@resource-id='com.tencent.wework:id/gp1']")
                # 确认删除
                el7 = self.driver.find_element(by=AppiumBy.XPATH, value="//*[@resource-id='com.tencent.wework:id/cst']")
                el7.click()
                i += 1
            else:
                return eles[0]

    def wait_elements(self, locator, time=10):
        '''
        显式等待找元素后停止
        :param locator:
        :param time:
        :return:
        '''
        logger.info(f"显式等待找到{locator}后停止")
        WebDriverWait(self.driver, time).until(expected_conditions.visibility_of_element_located(locator))

    def wait_for_text(self, by, value):
        '''
        显式等待文本可见
        :param by:
        :param value:
        :return:
        '''
        try:
            logger.info(f"显式等待找到{by, value}文本可见停止")
            self.driver.find_element(by, value=value)
            return True
        except:
            return False

    def wait_for_text2(self, text, time=10):
        '''
        显式等待文本可见
        :param text:
        :param time:
        :return:
        '''
        try:
            logger.info(f"显式等待找到{text}文本可见停止")
            WebDriverWait(self.driver, time).until(
                expected_conditions.visibility_of_element_located((AppiumBy.XPATH, f"//*[@text='{text}']")),
                message=f"{text}没找到"
            )
            return True
        except TimeoutException as e:
            print(e)
            return False

    def wait_for_click(self, value, time=10):
        '''
        显式等待元素可点击
        :param value:
        :param time:
        :return:
        '''
        try:
            logger.info(f"显式等待找到{value}按钮可点击停止")
            WebDriverWait(self.driver, time).until(
                expected_conditions.element_to_be_clickable((AppiumBy.XPATH, value)),
            )
            return True
        except TimeoutException as e:
            print(e)
            return False

    def screenshot(self, root_path):
        '''
        截图
        :param root_path: 当前文件的路径
        :return: 截图保存的路径
        '''
        logger.info("完成截图操作")
        # 使用当前时间作为截图的文件名
        image_name = Utils.get_current_time() + ".png"
        # 拼接当前图片保存的路径
        image_dir_path = os.sep.join([root_path, "..", "image/"])
        # 如果图片保存路径不存在
        if not os.path.isdir(image_dir_path):
            # 创建这个路径
            os.mkdir(image_dir_path)
        # 拼接图片保存路径
        image_path = image_dir_path + image_name
        logger.info(f"截图路径为 {image_path}")
        # 截图
        self.driver.save_screenshot(image_path)
        return image_path

    def save_page_source(self, root_path):
        '''
        保存页面源码
        :param root_path: 当前文件的路径
        :return: 页面源码文件的路径
        '''
        logger.info("保存页面源码")
        # 使用当前时间作为页面源码的文件名
        page_source_name = Utils.get_current_time() + "_page_source.xml"
        # 拼接当前文件保存的路径
        page_source_dir_path = os.sep.join([root_path, "..", "page_source/"])
        # 如果文件保存路径不存在
        if not os.path.isdir(page_source_dir_path):
            # 创建这个路径
            os.mkdir(page_source_dir_path)
        # 拼接文件保存路径
        page_source_path = page_source_dir_path + page_source_name
        logger.info(f"page source 的保存路径为 {page_source_path}")
        # 写 page source 文件
        with open(page_source_path, "w", encoding="utf-8") as f:
            f.write(self.driver.page_source)
        return page_source_path


from django.core.exceptions import ValidationError
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django_countries.fields import CountryField
from registrasion import models as rego

class AttendeeProfile(rego.AttendeeProfileBase):

    @classmethod
    def name_field(cls):
        ''' This is used to pre-fill the attendee's name from the
        speaker profile. If it's None, that functionality is disabled. '''
        return "name"

    def invoice_recipient(self):

        lines = [
            self.name_per_invoice,
        ]

        if self.company:
            lines.append("C/- " + self.company)

        if self.address_line_1:
            lines.append(self.address_line_1)

        if self.address_line_2:
            lines.append(self.address_line_2)

        if self.address_suburb or self.address_postcode:
            lines.append("%s %s" % (
                self.address_suburb or "",
                self.address_postcode or "",
            ))

        if self.state:
            lines.append(self.state)

        if self.country:
            lines.append(self.country.name)

        return "\n".join(unicode(line) for line in lines)

    def clean(self):
        errors = []
        if self.country == "US" and not self.state:
            errors.append(
                ("state", "US-based attendees must list their state"),
            )

        if self.address_line_2 and not self.address_line_1:
            errors.append((
                "address_line_1",
                "Please fill in line 1 before filling line 2",
            ))

        if not self.agreement:
            errors.append((
                "agreement",
                "You must accept the agreement.",
            ))

        if errors:
            raise ValidationError(dict(errors))

    def save(self):
        if not self.name_per_invoice:
            self.name_per_invoice = self.name
        super(AttendeeProfile, self).save()

    # Things that appear on badge
    name = models.CharField(
        verbose_name="Your name (for your conference nametag)",
        max_length=64,
        help_text="Your name, as you'd like it to appear on your badge. ",
    )

    company = models.CharField(
        max_length=64,
        help_text="The name of your company, as you'd like it on your badge and receipt",
        blank=True,
    )

    name_per_invoice = models.CharField(
        verbose_name="Your legal name (for your receipt)",
        max_length=256,
        help_text="If your legal name is different to the name on your badge, "
                  "fill this in, and we'll put it on your receipt. Otherwise, "
                  "leave it blank.",
        blank=True,
        )

    address_line_1 = models.CharField(
        verbose_name="Address line 1",
        help_text="This address, if provided, will appear on your receipt.",
        max_length=1024,
        blank=True,
    )
    address_line_2 = models.CharField(
        verbose_name="Address line 2",
        max_length=1024,
        blank=True,
    )
    address_suburb = models.CharField(
        verbose_name="City/Town/Suburb",
        max_length=1024,
        blank=True,
    )
    address_postcode = models.CharField(
        verbose_name="Postal/Zip code",
        max_length=1024,
        blank=True,
    )
    country = CountryField(
        default="US",
    )
    state = models.CharField(
        max_length=256,
        verbose_name="State/Territory/Province",
        blank=True,
    )

    dietary_restrictions = models.CharField(
        verbose_name="Food allergies, intolerances, or dietary restrictions",
        max_length=256,
        blank=True,
    )
    accessibility_requirements = models.CharField(
        verbose_name="Accessibility-related requirements",
        max_length=256,
        blank=True,
    )
    gender = models.CharField(
        help_text="Gender data will only be used for demographic purposes.",
        max_length=64,
        blank=True,
    )

    newsletter = models.BooleanField(
        verbose_name="Subscribe to PyOhio newsletter",
        help_text="Select to be subscribed to the low-volume PyOhio "
                  "announcements newsletter",
        blank=True,
    )

    agreement = models.BooleanField(
        verbose_name="Agreement",
        help_text="I agree to act according to the  "
                  "PyOhio Code of Conduct. I also agree with the "
                  "PyOhio Terms and Conditions.",
        blank=False,
        default=False,
    )


#!/usr/bin/env python3

import MISMIP

from pylab import figure, subplot, hold, plot, xlabel, ylabel, title, axis, vlines, savefig, text
from sys import exit

import numpy as np
from optparse import OptionParser
import os.path

try:
    from netCDF4 import Dataset as NC
except:
    print("netCDF4 is not installed!")
    sys.exit(1)

def parse_filename(filename, opts):
    "Get MISMIP info from a file name."
    tokens = filename.split('_')
    if tokens[0] == "ex":
        tokens = tokens[1:]

    try:
        model = tokens[0]
        experiment = tokens[1]
        mode = int(tokens[2][1])
        step = int(tokens[3][1])

    except:
        if opts.experiment is None:
            print("Please specify the experiment name (e.g. '-e 1a').")
            exit(0)
        else:
            experiment = opts.experiment

        if opts.step is None:
            print("Please specify the step (e.g. '-s 1').")
            exit(0)
        else:
            step = opts.step

        if opts.model is None:
            print("Please specify the model name (e.g. '-m ABC1').")
            exit(0)
        else:
            model = opts.model

        try:
            nc = NC(filename)
            x = nc.variables['x'][:]
            N = (x.size - 1) / 2
            if N == 150:
                mode = 1
            elif N == 1500:
                mode = 2
            else:
                mode = 3
        except:
            mode = 3

    return model, experiment, mode, step

def process_options():
    "Process command-line options and arguments."
    parser = OptionParser()
    parser.usage = "%prog <input files> [options]"
    parser.description = "Plots the ice flux as a function of the distance from the divide."
    parser.add_option("-o", "--output", dest="output", type="string",
                      help="Output image file name (e.g. -o foo.png)")
    parser.add_option("-e", "--experiment", dest="experiment", type="string",
                      help="MISMIP experiment: 1a,1b,2a,2b,3a,3b (e.g. -e 1a)")
    parser.add_option("-s", "--step", dest="step", type="int",
                      help="MISMIP step: 1,2,3,... (e.g. -s 1)")
    parser.add_option("-m", "--model", dest="model", type="string",
                      help="MISMIP model (e.g. -M ABC1)")
    parser.add_option("-f", "--flux", dest="profile", action="store_false", default=True,
                      help="Plot ice flux only")
    parser.add_option("-p", "--profile", dest="flux", action="store_false", default=True,
                      help="Plot geometry profile only")

    opts, args = parser.parse_args()

    if len(args) == 0:
        print("ERROR: An input file is requied.")
        exit(0)

    if len(args) > 1 and opts.output:
        print("More than one input file given. Ignoring the -o option...\n")
        opts.output = None

    if opts.output and opts.profile and opts.flux:
        print("Please choose between flux (-f) and profile (-p) plots.")
        exit(0)

    return args, opts.output, opts

def read(filename, name):
    "Read a variable and extract the middle row."
    nc = NC(filename)

    try:
        var = nc.variables[name][:]
    except:
        print("ERROR: Variable '%s' not present in '%s'" % (name, filename))
        exit(1)

    N = len(var.shape)
    if N == 1:
        return var[:]               # a coordinate variable ('x')
    elif N == 2:
        return var[1]               # get the middle row
    elif N == 3:
        return var[-1, 1]            # get the middle row of the last record
    else:
        raise Exception("Can't read %s. (It's %d-dimensional.)" % (name, N))

def find_grounding_line(x, topg, thk, mask):
    "Find the modeled grounding line position."
    # "positive" parts of x, topg, thk, mask
    topg = topg[x > 0]
    thk = thk[x > 0]
    mask = mask[x > 0]
    x = x[x > 0]                        # this should go last

    def f(j):
        "See equation (7) in Pattyn et al, 'Role of transition zones in marine ice sheet dynamics', 2005."
        z_sl = 0
        return (z_sl - topg[j]) * MISMIP.rho_w() / (MISMIP.rho_i() * thk[j])

    for j in range(x.size):
        if mask[j] == 2 and mask[j + 1] == 3:  # j is grounded, j+1 floating
            nabla_f = (f(j + 1) - f(j)) / (x[j + 1] - x[j])

            # See equation (8) in Pattyn et al
            return (1.0 - f(j) + nabla_f * x[j]) / nabla_f

    raise Exception("Can't find the grounding line")

def plot_profile(in_file, out_file):
    print("Reading %s to plot geometry profile for model %s, experiment %s, grid mode %s, step %s" % (
        in_file, model, experiment, mode, step))

    if out_file is None:
        out_file = os.path.splitext(in_file)[0] + "-profile.pdf"

    mask = read(in_file, 'mask')
    usurf = read(in_file, 'usurf')
    topg = read(in_file, 'topg')
    thk = read(in_file, 'thk')
    x = read(in_file, 'x')

    # theoretical grounding line position
    xg = MISMIP.x_g(experiment, step)
    # modeled grounding line position
    xg_PISM = find_grounding_line(x, topg, thk, mask)

    # mask out ice-free areas
    usurf = np.ma.array(usurf, mask=mask == 4)

    # compute the lower surface elevation
    lsrf = topg.copy()
    lsrf[mask == 3] = -MISMIP.rho_i() / MISMIP.rho_w() * thk[mask == 3]
    lsrf = np.ma.array(lsrf, mask=mask == 4)

    # convert x to kilometers
    x /= 1e3

    figure(1)
    ax = subplot(111)
    hold(True)
    plot(x, np.zeros_like(x), ls='dotted', color='red')
    plot(x, topg, color='black')
    plot(x, usurf, 'o', color='blue', markersize=4)
    plot(x, lsrf,  'o', color='blue', markersize=4)
    xlabel('distance from the divide, km')
    ylabel('elevation, m')
    title("MISMIP experiment %s, step %d" % (experiment, step))
    text(0.6, 0.9, "$x_g$ (model) = %4.0f km" % (xg_PISM / 1e3), color='r',
         transform=ax.transAxes)
    text(0.6, 0.85, "$x_g$ (theory) = %4.0f km" % (xg / 1e3), color='black',
         transform=ax.transAxes)

    _, _, ymin, ymax = axis(xmin=0, xmax=x.max())
    vlines(xg / 1e3, ymin, ymax, linestyles='dashed', color='black')
    vlines(xg_PISM / 1e3, ymin, ymax, linestyles='dashed', color='red')

    print("Saving '%s'...\n" % out_file)
    savefig(out_file)

def plot_flux(in_file, out_file):
    print("Reading %s to plot ice flux for model %s, experiment %s, grid mode %s, step %s" % (
        in_file, model, experiment, mode, step))

    if out_file is None:
        out_file = os.path.splitext(in_file)[0] + "-flux.pdf"

    x = read(in_file, 'x')
    flux_mag = read(in_file, 'flux_mag')

    # plot positive xs only
    flux_mag = flux_mag[x >= 0]
    x = x[x >= 0]

    figure(2)
    hold(True)

    plot(x / 1e3, flux_mag, 'k.-', markersize=10, linewidth=2)
    plot(x / 1e3, x * MISMIP.a() * MISMIP.secpera(), 'r:', linewidth=1.5)

    title("MISMIP experiment %s, step %d" % (experiment, step))
    xlabel("x ($\mathrm{km}$)", size=14)
    ylabel(r"flux ($\mathrm{m}^2\,\mathrm{a}^{-1}$)", size=14)

    print("Saving '%s'...\n" % out_file)
    savefig(out_file, dpi=300, facecolor='w', edgecolor='w')

if __name__ == "__main__":
    args, out_file, opts = process_options()

    for in_file in args:
        model, experiment, mode, step = parse_filename(in_file, opts)

        if opts.profile:
            plot_profile(in_file, out_file)

        if opts.flux:
            plot_flux(in_file, out_file)


"""toursanak URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from django.conf.urls import *
from django.conf.urls import include, url
#import css js image ...
from django.conf import settings
from django.conf.urls.static import static
admin.autodiscover()
urlpatterns = [
    # url(r'^search-form/$',"toursanak.views.search",name="search"),
    url(r'^search/$', "toursanak.views.search", name="search"),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^$', "toursanak.views.index", name='index'),
    url(r'^(\d+)/(\d+)/booking/', "toursanak.views.booking", name='booking'),

    url(r'^contact/', "toursanak.views.contact", name='contact'),
    url(r'^cat/(?P<slug>[-\w\d]+)/$', 'toursanak.views.category', name='category'),
    url(r'^createContact/', "toursanak.views.createContact", name='createContact'),
    url(r'^(\d+)/(\d+)/createbooking/', "toursanak.views.createBooking", name='createBooking'),
    url(r'^(?P<slug>[-\w\d]+)/$', 'toursanak.views.single', name='single'),
    url(r'^scroll/(\d+)/$', "toursanak.views.PostScroll", name='scroll'),
    url(r'^scrollCategory/(\w+)/(\d+)/$', "toursanak.views.scrollCategory", name='scrollcategory'),

]

if settings.DEBUG:
	urlpatterns+=static(settings.STATIC_URL,document_root=settings.STATIC_ROOT)


import numpy as np
import cv2
import argparse # For Command Line Arguments
import imutils
from transform.transform import four_point_transform # perspective transorm
#from skimage.filters import threshold_local

ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required = True,
	help = "Path to the image to be scanned")
args = vars(ap.parse_args())

image = cv2.imread(args["image"])
ratio = image.shape[0]/500.0 # Ratio of the original image
orig = image.copy() # Copy of Original Image
image = imutils.resize(image, height = 500) # Resizing image

imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) # Converting image from RGB to Grayscale
imgray = cv2.GaussianBlur(imgray, (5, 5), 0) # Performing Gaussian Blur to reduce noises
edge = cv2.Canny(imgray, 75, 200) # Canny Edge Detection

print("STEP 1: Edge Detection")
cv2.imshow("Original Image with Resizing", image)
cv2.imshow("Canny Edged Image", edge)
cv2.waitKey(0)
cv2.destroyAllWindows()

# Finding and storing contours of the Edged Image
cnts = cv2.findContours(edge.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts) # grabbing contours tuples based on OpenCV version
cnts = sorted(cnts, key = cv2.contourArea, reverse = True)[:5] #Sorting for the largest contours area

# Document usually has 4 edges, so checking if our contour grabbing is right
for c in cnts:
	peri = cv2.arcLength(c, True)
	approx = cv2.approxPolyDP(c, 0.02 * peri, True)
	if len(approx) == 4:
		screenCnt = approx
		break

print("STEP 2: Find contours of paper")
cv2.drawContours(image, [screenCnt], -1, (0, 255, 0), 2)
cv2.imshow("Outline", image)
cv2.waitKey(0)
cv2.destroyAllWindows()

# Applying 4-Point Perspective Transformation on the original image
warped = four_point_transform(orig, screenCnt.reshape(4, 2) * ratio)

# converting warped image to grayscale
warped = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)
#T = threshold_local(warped, 11, offset = 10, method = "gaussian")
#warped = (warped > T).astype("uint8") * 255

print("STEP 3: Apply perspective transform")
cv2.imshow("Original", imutils.resize(orig, height = 650))
cv2.imshow("Scanned", imutils.resize(warped, height = 650))
cv2.waitKey(0)


# -*-coding:utf-8 -*-
u"""
:创建时间: 2022/4/13 0:11
:作者: 苍之幻灵
:我的主页: https://cpcgskill.com
:QQ: 2921251087
:爱发电: https://afdian.net/@Phantom_of_the_Cang
:aboutcg: https://www.aboutcg.org/teacher/54335
:bilibili: https://space.bilibili.com/351598127

"""
from __future__ import unicode_literals, print_function, division
import io
import ast
import contextlib
from pyeal.exc import *

def _repr(o):
    s = repr(o)
    if isinstance(s, bytes):
        return s.decode('utf-8')
    return s

class Unparser(object):
    def __init__(self, tree, f=None):
        if f is None:
            f = io.StringIO()
        self.f = f
        self.col_off = 0
        # 记录最后一个写入的字符串， 配合_write_code， _write_line方法可以避免不必要的空行
        self.end_char = ''

        self._dispatch(tree)

    def _write_code(self, *s):
        if len(s) > 0:
            for i in s:
                if isinstance(i, bytes):
                    self.f.write(i.decode('utf-8'))
                else:
                    self.f.write(i)
            else:
                if len(i) > 0:
                    self.end_char = i[-1]

    @contextlib.contextmanager
    def _write_block(self):
        self.col_off += 1
        yield
        self.col_off -= 1

    @contextlib.contextmanager
    def _write_line(self):
        if self.end_char == '\n':
            self._write_code('    ' * self.col_off)
        yield
        if self.end_char != '\n':
            self._write_code('\n')

    def _dispatch(self, node):
        if isinstance(node, ast.slice):
            self._dispatch_slice(node)
        elif isinstance(node, ast.stmt):
            self._dispatch_stmt(node)
        elif isinstance(node, ast.expr):
            self._dispatch_expr(node)
        elif isinstance(node, ast.unaryop):
            self._dispatch_unaryop(node)
        elif isinstance(node, ast.boolop):
            self._dispatch_boolop(node)
        elif isinstance(node, ast.operator):
            self._dispatch_operator(node)
        elif isinstance(node, ast.cmpop):
            self._dispatch_cmpop(node)

        elif isinstance(node, ast.mod):
            self._dispatch_mod(node)

        elif isinstance(node, ast.ExceptHandler):
            self._except_handler(node)
        elif isinstance(node, ast.comprehension):
            self._comprehension(node)

        else:
            raise SealException("无法识别的节点: ", node)

    def _dispatch_mod(self, node):
        """
        :type node: ast.mod
        """
        if False:
            pass
        elif isinstance(node, ast.Module):
            self._module(node)
        else:
            raise SealException("无法识别的节点: ", node)

    def _dispatch_slice(self, node):
        """
        :type node: ast.slice
        """
        if isinstance(node, ast.Index):
            self._dispatch(node.value)
        elif isinstance(node, ast.Slice):
            if node.lower is not None:
                self._dispatch(node.lower)
            self._write_code(':')
            if node.upper is not None:
                self._dispatch(node.upper)
            self._write_code(':')
            if node.step is not None:
                self._dispatch(node.step)
        elif isinstance(node, ast.ExtSlice):
            end_id = len(node.dims) - 1
            for id_, i in enumerate(node.dims):
                self._dispatch(i)
                if id_ != end_id:
                    self._write_code(', ')
        else:
            raise SealException("无法识别的节点: ", node)

    def _dispatch_expr(self, node):
        """
        :type node: ast.expr
        """
        if False:
            pass
        elif isinstance(node, ast.Yield):
            if node.value is None:
                self._write_code('yield')
            else:
                self._write_code('yield ')
                self._dispatch(node.value)

        elif isinstance(node, ast.Attribute):
            self._attribute(node)
        elif isinstance(node, ast.Call):
            self._call(node)
        elif isinstance(node, ast.Compare):
            self._compare(node)
        elif isinstance(node, ast.Subscript):
            self._subscript(node)
        elif isinstance(node, ast.IfExp):
            self._if_exp(node)

        elif isinstance(node, ast.UnaryOp):
            self._unary_op(node)
        elif isinstance(node, ast.BoolOp):
            self._bool_op(node)
        elif isinstance(node, ast.BinOp):
            self._bin_op(node)

        elif isinstance(node, ast.Set):
            self._set(node)
        elif isinstance(node, ast.Dict):
            self._dict(node)
        elif isinstance(node, ast.List):
            self._list(node)
        elif isinstance(node, ast.Tuple):
            self._tuple(node)

        elif isinstance(node, ast.DictComp):
            self._dict_comp(node)
        elif isinstance(node, ast.SetComp):
            self._set_comp(node)
        elif isinstance(node, ast.ListComp):
            self._list_comp(node)
        elif isinstance(node, ast.GeneratorExp):
            self._generator_exp(node)

        elif isinstance(node, ast.Str):
            self._str(node)
        elif isinstance(node, ast.Num):
            self._num(node)
        elif isinstance(node, ast.Name):
            self._name(node)
        elif isinstance(node, ast.Lambda):
            self._lambda(node)

        else:
            raise SealException("无法识别的节点: ", node)

    def _dispatch_stmt(self, node):
        """
        :type node: ast.stmt
        """
        if False:
            pass
        elif isinstance(node, ast.ClassDef):
            self._class_def(node)
        elif isinstance(node, ast.FunctionDef):
            self._function_def(node)
        elif isinstance(node, ast.For):
            self._for(node)
        elif isinstance(node, ast.If):
            self._if(node)
        elif isinstance(node, ast.Return):
            self._return(node)
        elif isinstance(node, ast.Pass):
            self._write_code('pass')
        elif isinstance(node, ast.Raise):
            if node.type is None:
                self._write_code('raise')
            else:
                self._write_code('raise ')
                self._dispatch(node.type)
        elif isinstance(node, ast.Assert):
            self._write_code('assert ')
            self._dispatch(node.test)
            if node.msg is not None:
                self._write_code(', ')
                self._dispatch(node.msg)
        elif isinstance(node, ast.Print):
            self._write_code('print ')
            for id_, i in enumerate(node.values):
                if id_ != 0:
                    self._write_code(', ')
                self._dispatch(i)
        elif isinstance(node, ast.Delete):
            self._write_code('del ')
            for id_, i in enumerate(node.targets):
                if id_ != 0:
                    self._write_code(', ')
                self._dispatch(i)

        elif isinstance(node, ast.Global):
            self._write_code('global ', ', '.join(node.names))
        elif isinstance(node, ast.Continue):
            self._write_code('continue')
        elif isinstance(node, ast.Break):
            self._write_code('break')

        elif isinstance(node, ast.AugAssign):
            self._dispatch(node.target)
            self._write_code(' ')
            self._dispatch(node.op)
            self._write_code('= ')
            self._dispatch(node.value)

        elif isinstance(node, ast.Expr):
            self._expr(node)
        elif isinstance(node, ast.ImportFrom):
            self._import_from(node)
        elif isinstance(node, ast.Import):
            self._import(node)
        elif isinstance(node, ast.Assign):
            self._assign(node)

        elif isinstance(node, ast.With):
            self._with(node)
        elif isinstance(node, ast.While):
            self._while(node)
        elif isinstance(node, ast.TryExcept):
            self._try_except(node)
        elif isinstance(node, ast.TryFinally):
            self._try_finally(node)

        else:
            raise SealException("无法识别的节点: ", node)

    def _dispatch_unaryop(self, node):
        typemap = {
            ast.Invert: '~',
            ast.Not: 'not',
            ast.UAdd: '+',
            ast.USub: '-',
        }
        v = typemap.get(type(node), None)
        if v is None:
            raise SealException("无法识别的节点: ", node)
        self._write_code(v)

    def _dispatch_boolop(self, node):
        typemap = {
            ast.And: 'and',
            ast.Or: 'or',
        }
        v = typemap.get(type(node), None)
        if v is None:
            raise SealException("无法识别的节点: ", node)
        self._write_code(v)

    def _dispatch_operator(self, node):
        typemap = {
            ast.Add: '+',
            ast.Sub: '-',
            ast.Mult: '*',
            ast.Div: '/',
            ast.FloorDiv: '//',
            ast.RShift: '>>',
            ast.LShift: '<<',
            ast.BitAnd: '&',
            ast.BitOr: '|',
            ast.BitXor: '^',
            ast.Mod: '%',
            ast.Pow: '**',
        }
        v = typemap.get(type(node), None)
        if v is None:
            raise SealException("无法识别的节点: ", node)
        self._write_code(v)

    def _dispatch_cmpop(self, node):
        """
        :type node: ast.cmpop
        """
        typemap = {
            ast.Is: 'is',
            ast.IsNot: 'is not',
            ast.In: 'in',
            ast.NotIn: 'not in',
            ast.Eq: '==',
            ast.NotEq: '!=',
            ast.Gt: '>',
            ast.GtE: '>=',
            ast.Lt: '<',
            ast.LtE: '<=',
        }
        v = typemap.get(type(node), None)
        if v is None:
            raise SealException("无法识别的节点: ", node)
        self._write_code(v)

    def _module(self, node):
        """
        :type node: ast.Module
        """
        for i in node.body:
            with self._write_line():
                self._dispatch(i)

    def _class_def(self, node):
        """
        :type node: ast.ClassDef
        """
        for i in node.decorator_list:
            with self._write_line():
                self._write_code('@')
                self._dispatch(i)
        # 类头部输出
        with self._write_line():
            self._write_code('class ', node.name, '(')
            end_id = len(node.bases) - 1
            for id_, i in enumerate(node.bases):
                self._dispatch(i)
                if id_ != end_id:
                    self._write_code(', ')
            self._write_code('):')

        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)

    def _function_def(self, node):
        """
        :type node: ast.FunctionDef
        """
        for i in node.decorator_list:
            with self._write_line():
                self._write_code('@')
                self._dispatch(i)
        # 函数头部输出
        with self._write_line():
            self._write_code('def ', node.name, '(')
            end_id = len(node.args.args) - 1
            id_ = 0
            if node.args.vararg is not None:
                end_id += 1
            if node.args.kwarg is not None:
                end_id += 1

            args = node.args.args[:len(node.args.args) - len(node.args.defaults)]
            def_args = node.args.args[len(node.args.args) - len(node.args.defaults):]
            for i in args:
                self._dispatch(i)
                if id_ != end_id:
                    self._write_code(', ')
                id_ += 1
            for i, v in zip(def_args, node.args.defaults):
                self._dispatch(i)
                self._write_code('=')
                self._dispatch(v)
                if id_ != end_id:
                    self._write_code(', ')
                id_ += 1
            if node.args.vararg is not None:
                self._write_code('*', node.args.vararg)
                if id_ != end_id:
                    self._write_code(', ')
                id_ += 1
            if node.args.kwarg is not None:
                self._write_code('**', node.args.kwarg)
                if id_ != end_id:
                    self._write_code(', ')
                id_ += 1
            self._write_code('):')

        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)

    def _for(self, node):
        """
        :type node: ast.For
        """

        # 函数头部输出
        self._write_code('for ')
        self._dispatch(node.target)
        self._write_code(' in ')
        self._dispatch(node.iter)
        self._write_code(':\n')

        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)
        if len(node.orelse) > 0:
            with self._write_line():
                self._write_code('else:')
            with self._write_block():
                for i in node.orelse:
                    with self._write_line():
                        self._dispatch(i)

    def _if(self, node, is_elif=False):
        """
        :type node: ast.If
        :type is_elif: bool
        """

        # 函数头部输出
        if is_elif:
            self._write_code('elif ')
        else:
            self._write_code('if ')
        self._dispatch(node.test)
        self._write_code(':\n')

        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)
        if len(node.orelse) == 1 and isinstance(node.orelse[0], ast.If):
            node = node.orelse[0]
            with self._write_line():
                self._if(node, True)
        else:
            if len(node.orelse) > 0:
                with self._write_line():
                    self._write_code('else:\n')
                with self._write_block():
                    for i in node.orelse:
                        with self._write_line():
                            self._dispatch(i)

    def _expr(self, node):
        """
        :type node: ast.Expr
        """
        self._dispatch(node.value)

    def _import_from(self, node):
        """
        :type node: ast.ImportFrom
        """

        co = 'from {}{} import '.format(''.join(('.' for i in range(node.level))),
                                        '' if node.module is None else node.module)
        imps = (i.name if i.asname is None else '{} as {}'.format(i.name, i.asname) for i in node.names)
        imps = ', '.join(imps)
        self._write_code(co, imps)

    def _import(self, node):
        """
        :type node: ast.Import
        """
        imps = (i.name if i.asname is None else '{} as {}'.format(i.name, i.asname) for i in node.names)
        imps = ', '.join(imps)
        self._write_code('import ', imps)

    def _compare(self, node):
        """
        :type node: ast.Compare
        """
        self._dispatch(node.left)
        for o, c in zip(node.ops, node.comparators):
            self._write_code(' ')
            self._dispatch(o)
            self._write_code(' ')
            self._dispatch(c)

    def _assign(self, node):
        """
        :type node: ast.Assign
        """
        self._dispatch(node.targets[0])
        self._write_code(' = ')
        self._dispatch(node.value)

    def _with(self, node):
        """
        :type node: ast.With
        """
        self._write_code('with ')
        self._dispatch(node.context_expr)
        if node.optional_vars is not None:
            self._write_code(' as ')
            self._dispatch(node.optional_vars)
        self._write_code(':\n')
        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)

    def _while(self, node):
        """
        :type node: ast.While
        """
        self._write_code('while ')
        self._dispatch(node.test)
        self._write_code(':\n')
        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)
        if len(node.orelse) > 0:
            with self._write_line():
                self._write_code('else:')
            with self._write_block():
                for i in node.orelse:
                    with self._write_line():
                        self._dispatch(i)

    def _try_except(self, node):
        """
        :type node: ast.TryExcept
        """
        self._write_code('try:\n')
        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)
        for i in node.handlers:
            with self._write_line():
                self._dispatch(i)
        if len(node.orelse) > 0:
            with self._write_line():
                self._write_code('else:')
            with self._write_block():
                for i in node.orelse:
                    with self._write_line():
                        self._dispatch(i)

    def _try_finally(self, node):
        """
        :type node: ast.TryFinally
        """
        if len(node.body) == 1 and isinstance(node.body[0], ast.TryExcept):
            with self._write_line():
                self._dispatch(node.body[0])
        else:
            with self._write_line():
                self._write_code('try:')
            with self._write_block():
                for i in node.body:
                    with self._write_line():
                        self._dispatch(i)
        with self._write_line():
            self._write_code('finally:')
        with self._write_block():
            for i in node.finalbody:
                with self._write_line():
                    self._dispatch(i)

    def _except_handler(self, node):
        """
        :type node: ast.ExceptHandler
        """
        if node.name is None:
            self._write_code('except:\n')
        elif node.type is None:
            self._write_code('except ')
            self._dispatch(node.name)
            self._write_code(':\n')
        else:
            self._write_code('except ')
            self._dispatch(node.name)
            self._write_code(' as ')
            self._dispatch(node.type)
            self._write_code(':\n')
        with self._write_block():
            for i in node.body:
                with self._write_line():
                    self._dispatch(i)

    def _return(self, node):
        """
        :type node: ast.Return
        """
        if node.value is None:
            self._write_code('return')
        else:
            self._write_code('return ')
            self._dispatch(node.value)

    def _call(self, node):
        """
        :type node: ast.Call
        """
        self._dispatch(node.func)
        self._write_code('(')
        id_ = 0
        end_id = len(node.args) + len(node.keywords) - 1
        if node.starargs is not None:
            end_id += 1
        if node.kwargs is not None:
            end_id += 1

        for i in node.args:
            self._dispatch(i)
            if id_ != end_id:
                self._write_code(', ')
                id_ += 1
        for i in node.keywords:
            self._write_code(i.arg, '=')
            self._dispatch(i.value)
            if id_ != end_id:
                self._write_code(', ')
                id_ += 1
        if node.starargs is not None:
            self._write_code('*')
            self._dispatch(node.starargs)
            if id_ != end_id:
                self._write_code(', ')
                id_ += 1
        if node.kwargs is not None:
            self._write_code('**')
            self._dispatch(node.kwargs)
            if id_ != end_id:
                self._write_code(', ')
                id_ += 1
        self._write_code(')')

    def _attribute(self, node):
        """
        :type node: ast.Attribute
        """
        self._dispatch(node.value)
        self._write_code('.', node.attr)

    def _subscript(self, node):
        """
        :type node: ast.Subscript
        """

        self._dispatch(node.value)
        self._write_code('[')
        self._dispatch(node.slice)
        self._write_code(']')

    def _if_exp(self, node):
        """
        :type node: ast.IfExp
        """
        self._dispatch(node.body)
        self._write_code(' if ')
        self._dispatch(node.test)
        self._write_code(' else ')
        self._dispatch(node.orelse)

    def _unary_op(self, node):
        """
        :type node: ast.UnaryOp
        """
        self._dispatch(node.op)
        self._write_code(' ')
        self._dispatch(node.operand)

    def _bool_op(self, node):
        """
        :type node: ast.BoolOp
        """
        for id_, i in enumerate(node.values):
            if id_ != 0:
                self._write_code(' ')
                self._dispatch(node.op)
                self._write_code(' ')
            self._dispatch(i)

    def _bin_op(self, node):
        """
        :type node: ast.BinOp
        """
        self._dispatch(node.left)
        self._write_code(' ')
        self._dispatch(node.op)
        self._write_code(' ')
        self._dispatch(node.right)

    def _set(self, node):
        """
        :type node: ast.Set
        """
        self._write_code('{')
        if len(node.elts) > 0:
            for i in node.elts:
                self._dispatch(i)
                self._write_code(', ')
        self._write_code('}')

    def _dict(self, node):
        """
        :type node: ast.Dict
        """
        self._write_code('{')
        if len(node.keys) > 0:
            for k, v in zip(node.keys, node.values):
                self._dispatch(k)
                self._write_code(': ')
                self._dispatch(v)
                self._write_code(', ')
        self._write_code('}')

    def _list(self, node):
        """
        :type node: ast.List
        """
        self._write_code('[')
        if len(node.elts) > 0:
            for i in node.elts:
                self._dispatch(i)
                self._write_code(', ')
        self._write_code(']')

    def _tuple(self, node):
        """
        :type node: ast.Tuple
        """
        self._write_code('(')
        if len(node.elts) > 0:
            for i in node.elts:
                self._dispatch(i)
                self._write_code(', ')
        self._write_code(')')

    def _dict_comp(self, node):
        """
        :type node: ast.DictComp
        """
        self._write_code('{')
        self._dispatch(node.key)
        self._write_code(': ')
        self._dispatch(node.value)
        for i in node.generators:
            self._write_code(' ')
            self._dispatch(i)
        self._write_code('}')

    def _set_comp(self, node):
        """
        :type node: ast.SetComp
        """
        self._write_code('{')
        self._dispatch(node.elt)
        for i in node.generators:
            self._write_code(' ')
            self._dispatch(i)
        self._write_code('}')

    def _list_comp(self, node):
        """
        :type node: ast.ListComp
        """
        self._write_code('[')
        self._dispatch(node.elt)
        for i in node.generators:
            self._write_code(' ')
            self._dispatch(i)
        self._write_code(']')

    def _generator_exp(self, node):
        """
        :type node: ast.GeneratorExp
        """
        self._write_code('(')
        self._dispatch(node.elt)
        for i in node.generators:
            self._write_code(' ')
            self._dispatch(i)
        self._write_code(')')

    def _comprehension(self, node):
        """
        :type node: ast.comprehension
        """
        self._write_code('for ')
        self._dispatch(node.target)
        self._write_code(' in ')
        self._dispatch(node.iter)
        for i in node.ifs:
            self._write_code(' if ')
            self._dispatch(i)

    def _str(self, node):
        """
        :type node: ast.Str
        """
        self._write_code(repr(node.s))
        # co = _repr(node.s)
        # if co[0] == 'b' or co[0] == 'u':
        #     co = co[1:]
        # if isinstance(co, bytes):
        #     self._write_code('b', co)
        # else:
        #     self._write_code('u', co)

    def _num(self, node):
        """
        :type node: ast.Num
        """
        self._write_code(repr(node.n))

    def _name(self, node):
        """
        :type node: ast.Name
        """
        self._write_code(node.id)

    def _lambda(self, node):
        """
        :type node: ast.Lambda
        """
        self._write_code('lambda ')
        end_id = len(node.args.args) - 1
        id_ = 0
        if node.args.vararg is not None:
            end_id += 1
        if node.args.kwarg is not None:
            end_id += 1

        args = node.args.args[:len(node.args.args) - len(node.args.defaults)]
        def_args = node.args.args[len(node.args.args) - len(node.args.defaults):]
        for i in args:
            self._dispatch(i)
            if id_ != end_id:
                self._write_code(', ')
            id_ += 1
        for i, v in zip(def_args, node.args.defaults):
            self._dispatch(i)
            self._write_code('=')
            self._dispatch(v)
            if id_ != end_id:
                self._write_code(', ')
            id_ += 1
        if node.args.vararg is not None:
            self._write_code('*', node.args.vararg)
            if id_ != end_id:
                self._write_code(', ')
            id_ += 1
        if node.args.kwarg is not None:
            self._write_code('**', node.args.kwarg)
            if id_ != end_id:
                self._write_code(', ')
            id_ += 1
        self._write_code(':')

        self._dispatch(node.body)

def parser(code):
    """
    :type code: bytes
    :rtype: str
    """
    return ast.parse(code)

def unparse(tree):
    f = io.StringIO()
    Unparser(tree, f)
    return f.getvalue()


# @Author   : ChaoQiezi
# @Time     : 2023/8/3  15:44
# @Email    : chaoqiezi.one@qq.com

"""
This script is used to manage the Data
"""

import os
import glob
import h5py
import numpy as np
import keras
from keras.utils import Sequence
from sklearn.model_selection import train_test_split
from skimage.util import view_as_windows
from utils.utils import read_img

class DataManager:

    def __init__(self, config):
        self.config = config
        self.img = type('', (), {})()  # create an empty class
        self.img.nodata = None
        self.img.rows = None
        self.img.cols = None
        self.img.transform = None
        self.img.projection = None
        self.time_steps = None

    def load_data(self, dataset_dir, wildcard='*.tif', save_path=None):
        """

        :param dataset_dir:
        :return:
        """

        dataset_paths = glob.iglob(os.path.join(dataset_dir, wildcard))
        datasets = []
        for dataset_path in dataset_paths:
            # get img and info
            dataset, img_info = read_img(dataset_path)

            # set nodata
            self.img.nodata = img_info[2]
            if self.img.nodata is np.nan:
                continue
            else:
                dataset[dataset == self.img.nodata] = np.nan

            # add
            datasets.append(dataset)
            print('>>> {} is loaded.'.format(os.path.basename(dataset_path)))
        if len(datasets) == 0:
            raise ValueError("There is no {} image in the {}".format(wildcard, dataset_dir))
        # get img info
        self.time_steps = len(datasets)
        self.img.rows = img_info[0]
        self.img.cols = img_info[1]
        self.img.transform = img_info[3]
        self.img.projection = img_info[4]

        datasets = np.array(datasets)

        if save_path:
            np.save(save_path, datasets)

        return datasets

    def read_img(self, img_path):
        return read_img(img_path)

    def nomalize(self, dataset, train_flag=False):

        if train_flag:
            for i in range(dataset.shape[-1]):  # dataset.shape = (None, rows, cols, bands)
                dataset[:, :, :, i] = (dataset[:, :, :, i] - np.nanmin(dataset[:, :, :, i])) \
                                      / (np.nanmax(dataset[:, :, :, i]) - np.nanmin(dataset[:, :, :, i]))
        else:
            dataset = (dataset - np.nanmin(dataset)) / (np.nanmax(dataset) - np.nanmin(dataset))

        return dataset

    def balance_data(self, features, labels):
        class_label = np.unique(labels)
        class_amount = {label: np.sum(labels == label) for label in class_label}
        min_amount = np.min(list(class_amount.values()))

        for label in class_label:
            if class_amount[label] > min_amount:
                mask = labels == label
                mask = np.random.choice(mask, min_amount, replace=False)
                labels = labels[mask]
                features = features[mask, :, :, :, :]

        return features, labels

    def extract_weight(self, labels):
        class_label = np.unique(labels)
        class_amount = {int(label): int(np.sum(labels == label)) for label in class_label}
        class_amount_sum = np.sum(list(class_amount.values()))

        class_weight = {int(label): float(label_amount / class_amount_sum * 10) for label, label_amount in
                        class_amount.items()}

        # # discard the count of nodata
        # class_weight.pop(self.config['geo_info'][2])

        return class_weight

    def split_data(self, features, labels, test_size=0.2, random_state=0):

        features_train, features_test, labels_train, labels_test = \
            train_test_split(features, labels, test_size=test_size, random_state=random_state)

        return features_train, features_test, labels_train, labels_test

class GenerateData(Sequence):

    def __init__(self, config: dict, is_train: bool, batch_size=512, shuffle=True):

        self.config = config
        self.is_train = is_train
        self.batch_size = batch_size
        self.channels = config['bands']
        self.n_class = config['n_class']
        self.shuffle = shuffle
        if self.is_train:
            self.n_samples = self.config['train_samples']
        else:
            self.n_samples = self.config['test_samples']
        self.on_epoch_end()  # on_epoch_end() means that the function will be executed after each epoch

    def __len__(self):
        return self.n_samples // self.batch_size  # get the number of batches per epoch
        # return 100

    def __getitem__(self, index):
        # generate indexes of the batch
        batch_indexes = self.epoch_indexes[(index * self.batch_size):((index + 1) * self.batch_size)]

        # generate data
        features, labels = self.__data_generation(batch_indexes)
        return features, labels

    def on_epoch_end(self):
        self.epoch_indexes = np.arange(self.n_samples)
        if self.shuffle:
            np.random.shuffle(self.epoch_indexes)

    def __data_generation(self, batch_indexes: list):
        # because reading a h5 file through  the indexes  requires that the indexes are in increasing order
        batch_indexes.sort()
        # open h5 file
        with h5py.File(self.config['features_labels_path'], 'r') as f:
            if self.is_train:
                features = f['features_train'][batch_indexes, :, :, :,
                           :]  # shape = (batch_size, time_steps, rows, cols, bands)
                labels = f['labels_train'][batch_indexes]  # shape = (batch_size, )
            else:
                features = f['features_test'][batch_indexes, :, :, :, :]
                labels = f['labels_test'][batch_indexes]
            # labels -= 1  # for the moment
        return features, keras.utils.to_categorical(labels, num_classes=self.n_class)  # one-hot encoding
        # return features, labels

class PredictGenerator(Sequence):

    def __init__(self, predict_data_path):
        self.predict_data_path = predict_data_path
        with h5py.File(self.predict_data_path, 'r') as f:
            self.rows = range(5, f['features'].shape[1] - 5)

    def __len__(self):
        # return len(self.rows)
        return 20

    def __getitem__(self, index):
        row = self.rows[index]
        return self.__data_generation(row)

    def __data_generation(self, row):
        with h5py.File(self.predict_data_path, 'r') as f:
            predict_data = f['features'][:]
            predict_data = view_as_windows((predict_data[:, row - 5:row + 6, :, :]), (30, 11, 11, 2)).squeeze()
        return predict_data

# class PredictGenerator2(Sequence):
#
#     def __init__(self, predict_data):
#         self.predict_data = predict_data
#         self.rows = range(5, predict_data.shape[1] - 5)
#
#     def __len__(self):
#         # return len(self.rows)
#         return 10
#
#     def __getitem__(self, index):
#         row = self.rows[index]
#         return view_as_windows((self.predict_data[:, row - 5:row + 6, :, :]), (30, 11, 11, 2)).squeeze()


#! /usr/bin/env python

from typing import List
from random import randint

def main() -> None:

    numbers: List[int] = [ randint(0,9) ] * 10000000

    for number in numbers:
        number = number ** 1000

if __name__ == '__main__':
    main()