import gspread
from oauth2client.service_account import ServiceAccountCredentials

from datetime import datetime

import pandas as pd
import numpy as np

import plotly
import plotly.express as px
import plotly.io as pio

import chart_studio
import chart_studio.plotly as py
import chart_studio.tools as tls

# Please note that this function is included in the demo.py program for the USG Overview; SEE Chart 8

date_after_var = input("Date to show the USAID Business Forecast After (MM/DD/YYYY): ")
username = input("Chart Studio Username?: ")
api_key = input("Chart Studio Password?: ")
creds_json = input("Google API Service Account JSON (use .json)?: ")
sheet_name = input("Google Sheet Name?: ")

def usaid_bf_wash_fxn(date_after_var,username,api_key,creds_json,sheet_name):

    chart_studio.tools.set_credentials_file(username=username, api_key=api_key)

    #Access Google Sheets
    scope = ["https://spreadsheets.google.com/feeds","https://www.googleapis.com/auth/spreadsheets","https://www.googleapis.com/auth/drive.file","https://www.googleapis.com/auth/drive"]
    creds = ServiceAccountCredentials.from_json_keyfile_name(creds_json,scope)
    client = gspread.authorize(creds)
    sheet = client.open(sheet_name).sheet1
    lyst1 = sheet.get_all_records()

    df = pd.DataFrame(lyst1)

    def split(value):
        a, b= value.split('-')
        return a

    lyst2 =[]
    for i in df["Total Estimated Cost/Amount Range"]:
        lyst2.append(split(i))

    df["Estimated Cost Minimum"] = lyst2

    df['Anticipated Solicitation Release Date']= pd.to_datetime(df['Anticipated Solicitation Release Date'])

    date1 = date_after_var

    df = df.loc[df["Anticipated Solicitation Release Date"] > datetime.strptime(date1, "%m/%d/%Y")]
    df = df.loc[df["Location"] == "Washington"]

    df["Award Description"] = df["Award Description"].str.wrap(45)
    df["Award Description"] = df["Award Description"].apply(lambda x: x.replace('\n', '
'))

    scatterplot = px.scatter(
        title = "USAID Business Forecast (For Washington Operating Unit)",
        data_frame=df,
        x = "Anticipated Solicitation Release Date",
        y = "Estimated Cost Minimum",
        color = "Sector",
        hover_name = "Award Title",
        hover_data=["Operating Unit","Total Estimated Cost/Amount Range","Anticipated Award Date","Award Length","Award Description"],
        opacity = 0.9,
        orientation="v",
        template='gridon',
        labels={
            "Anticipated Solicitation Release Date": "Anticipated Solicitation Release Date",
            "Total Estimated Cost/Amount Range":"Estimated Cost Range",
            "Award Description":"
Award Description",
            "Sector":"Sector",
            "Estimated Cost Minimum":"Estimated Cost Minimum",
            "Anticipated Award Date":"Anticipated Award Date",
            "Award Length":"Award Length",
            "Operating Unit":"Operating Unit"
        }
    )

    scatterplot.update_traces(marker={'size': 15})

    # py.plot(scatterplot, filename = 'USAID BF for Washington', auto_open = False)

    scatterplot.show()

usaid_bf_wash_fxn(date_after_var,username,api_key,creds_json,sheet_name)


from django.db import transaction

from food_ordering.orders.models import Order, OrderFood
from food_ordering.orders.pubsub import orders_pubsub_service

class OrderService:
    @staticmethod
    @transaction.atomic()
    def create_order(user, restaurant, orderfood_set):
        order = Order.objects.create(user=user, restaurant=restaurant)

        orderfood_objs = [OrderFood(order=order, **orderfood) for orderfood in orderfood_set]
        OrderFood.objects.bulk_create(orderfood_objs)

        # Publish order to pubsub channel
        OrderService.publish_order(order)

        return order

    @staticmethod
    def publish_order(order: Order):
        orders_pubsub_service.publish(str(order.uuid))
        order.status = Order.OrderStatus.WAITING
        order.save()

    @staticmethod
    def process_orders():
        orders_pubsub_service.process_subscribers()


import pathlib

if __name__ == "__main__" and __package__ is None:
    __package__ = pathlib.Path(__file__).parent.name

from .pipeline import Pipeline

pipeline = Pipeline()
pipeline.model_export(
    commands=(
        r"""
        python3 triton/export_model.py \
            --input-path triton/model.py \
            --input-type tf-keras \
            --output-path ${SHARED_DIR}/exported_model.savedmodel \
            --output-type ${EXPORT_FORMAT} \
            --ignore-unknown-parameters \
            \
            --checkpoint-dir ${CHECKPOINT_DIR}/checkpoint \
            --batch-size ${MAX_BATCH_SIZE} \
            --precision ${EXPORT_PRECISION} \
            \
            --dataloader triton/dataloader.py \
            --batch-size ${MAX_BATCH_SIZE} \
            --data-pattern "${DATASETS_DIR}/outbrain/valid/*.parquet"
        """,
    )
)
pipeline.model_conversion(
    commands=(
        r"""
        model-navigator convert \
            --model-name ${MODEL_NAME} \
            --model-path ${SHARED_DIR}/exported_model.savedmodel \
            --output-path ${SHARED_DIR}/converted_model \
            --target-formats ${FORMAT} \
            --target-precisions ${PRECISION} \
            --launch-mode local \
            --override-workspace \
            --verbose \
            \
            --onnx-opsets 13 \
            --max-batch-size ${MAX_BATCH_SIZE} \
            --max-workspace-size 8589934592 \
            --atol wide_deep_model=0.015 \
            --rtol wide_deep_model=12.0
        """,
    )
)

pipeline.model_deploy(
    commands=(
        r"""
        model-navigator triton-config-model \
            --model-repository ${MODEL_REPOSITORY_PATH} \
            --model-name ${MODEL_NAME} \
            --model-version 1 \
            --model-path ${SHARED_DIR}/converted_model \
            --model-format ${FORMAT} \
            --model-control-mode explicit \
            --load-model \
            --load-model-timeout-s 120 \
            --verbose \
            \
            --batching ${MODEL_BATCHING} \
            --backend-accelerator ${BACKEND_ACCELERATOR} \
            --tensorrt-precision ${PRECISION} \
            --tensorrt-capture-cuda-graph \
            --max-batch-size ${MAX_BATCH_SIZE} \
            --preferred-batch-sizes ${MAX_BATCH_SIZE} \
            --engine-count-per-device ${DEVICE_KIND}=${NUMBER_OF_MODEL_INSTANCES}
        """,
    )
)
pipeline.triton_performance_offline_tests(
    commands=(
        r"""
        python triton/run_performance_on_triton.py \
            --model-repository ${MODEL_REPOSITORY_PATH} \
            --model-name ${MODEL_NAME} \
            --input-data random \
            --batch-sizes ${MEASUREMENT_OFFLINE_BATCH_SIZES} \
            --concurrency ${MEASUREMENT_OFFLINE_CONCURRENCY} \
            --performance-tool ${PERFORMANCE_TOOL} \
            --measurement-request-count 100 \
            --evaluation-mode offline \
            --warmup \
            --result-path ${SHARED_DIR}/triton_performance_offline.csv
        """,
    ),
    result_path="${SHARED_DIR}/triton_performance_offline.csv",
)
pipeline.triton_performance_online_tests(
    commands=(
        r"""
        python triton/run_performance_on_triton.py \
            --model-repository ${MODEL_REPOSITORY_PATH} \
            --model-name ${MODEL_NAME} \
            --input-data random \
            --batch-sizes ${MEASUREMENT_ONLINE_BATCH_SIZES} \
            --concurrency ${MEASUREMENT_ONLINE_CONCURRENCY} \
            --performance-tool ${PERFORMANCE_TOOL} \
            --measurement-request-count 500 \
            --evaluation-mode online \
            --warmup \
            --result-path ${SHARED_DIR}/triton_performance_online.csv
        """,
    ),
    result_path="${SHARED_DIR}/triton_performance_online.csv",
)


import os
import argparse
import csv
import pandas as pd
import torch
from torch.utils.data import DataLoader
from torchvision.utils import save_image
from tqdm import tqdm
from src.models.model import BCANF
from src.datasets.dataset import VideoDataset
from src.util.tools import seed_everything, estimate_bpp, get_setting, get_order
from src.util.metrics import psnr as psnr_fn
from src.util.metrics import MS_SSIM
from src.util.stream import Stream

class Tester:

    def __init__(self, args, model):
        self.args = args
        self.model = model
        self.psnr_fn = psnr_fn
        self.ssim_fn = MS_SSIM(data_range=1., reduction="none").to(args.device)

    @torch.no_grad()
    def quality_fn(self, rec, target):
        rec = rec.clamp(0, 1)
        psnr = self.psnr_fn(rec, target).mean()
        ssim = self.ssim_fn(rec, target).mean()
        return psnr, ssim

    @staticmethod
    @torch.no_grad()
    def rate_fn(likelihood, input):
        rate_y = estimate_bpp(likelihood['z'], input=input).mean()
        rate_z = estimate_bpp(likelihood['h'], input=input).mean()
        rate = rate_y + rate_z
        return rate

    @torch.no_grad()
    def test(self, eval_mode="test"):
        dataset = VideoDataset(self.args.src, self.args.num_frames, self.args.intra_period, self.args.gop_size)
        test_loader = DataLoader(dataset, batch_size=1, num_workers=0, shuffle=False)
        order = get_order(dataset.pairs)

        _, img = next(test_loader.__iter__())
        shape = img.size()[-2:]
        num_frames = len(dataset)

        name = os.path.basename(self.args.src)
        os.makedirs(self.args.save_dir, exist_ok=True)

        if eval_mode == "compress":
            stream = Stream(os.path.join(self.args.save_dir, f"{name}.bin"), 'wb')
            stream.write_header(self.args.intra_period, self.args.gop_size, shape, num_frames)

        csv_report = open(os.path.join(self.args.save_dir, f"{name}.csv"), 'w', newline='')
        writer = csv.writer(csv_report, delimiter=',')
        writer.writerow(["frame_idx", "psnr", "ms-ssim", "rate", "coding mode"])

        frame_buffer = {}
        frame_bias = 1
        for i, (pair, x) in tqdm(enumerate(test_loader), total=len(test_loader)):
            img = x.to(self.args.device)
            pair = [int(p) for p in pair]

            inputs = {"xt": img}
            if len(pair) == 1:
                frame_idx = pair[0]
                mode = "i-frame"
                frame_type = -1
            elif len(pair) == 2:
                frame_idx = pair[1]
                inputs['x1'] = frame_buffer[pair[0]]
                mode = "b*-frame"
                frame_type = 2
            else:
                frame_idx = pair[1]
                inputs['x1'] = frame_buffer[pair[0]]
                inputs['x2'] = frame_buffer[pair[-1]]

                if order[pair[0]] > order[pair[-1]]:
                    inputs['x1'], inputs['x2'] = inputs['x2'], inputs['x1']

                mode = "b-frame"
                if abs(pair[-1] - pair[0]) > 2:
                    frame_type = 0
                else:
                    frame_type = 1

            output = self.model(inputs, mode, frame_type, shape, eval_mode)
            psnr, ssim = self.quality_fn(output["rec_frame"], img)

            if eval_mode == "test":
                rate = sum([self.rate_fn(l, img) for l in output["likelihood"]])
            elif eval_mode == "compress":
                bytes = sum([stream.writeStream(s['z']) + \
                             stream.writeStream(s['h']) for s in output["likelihood"]])
                rate = bytes * 8 / (shape[0] * shape[1])

            writer.writerow([frame_idx + frame_bias, float(psnr), float(ssim), float(rate), mode])
            csv_report.flush()

            frame_buffer[frame_idx] = output["rec_frame"]
            if i > 0 and i  % self.args.intra_period == 0:
                frame_buffer = {0: frame_buffer[self.args.intra_period]}
                frame_bias += self.args.intra_period

        if eval_mode == "compress":
            stream.close()

        df = pd.read_csv(csv_report.name)
        average = df.loc[:, df.columns != 'coding mode'].mean()
        writer.writerow(["average", float(average['psnr']), float(average['ms-ssim']), float(average['rate'])])
        csv_report.close()

    @torch.no_grad()
    def compress(self):
        self.test(eval_mode="compress")

    @torch.no_grad()
    def decompress(self):
        eval_mode = "decompress"
        stream = Stream(self.args.src, 'rb')
        self.args.intra_period, self.args.gop_size, shape, num_frames = stream.read_header()
        dataset = VideoDataset(self.args.src, num_frames, self.args.intra_period,
                               self.args.gop_size, no_img=True)
        test_loader = DataLoader(dataset, batch_size=1, num_workers=0, shuffle=False)
        order = get_order(dataset.pairs)

        name = "rec_" + os.path.basename(self.args.src)[:-4]
        self.args.save_dir = os.path.join(self.args.save_dir, name)
        os.makedirs(self.args.save_dir, exist_ok=True)

        frame_buffer = {}
        frame_bias = 1
        for i, pair in tqdm(enumerate(test_loader), total=len(test_loader)):
            pair = [int(p) for p in pair]

            inputs = {}
            if len(pair) == 1:
                frame_idx = pair[0]
                inputs['xt'] = stream.readStream()
                mode = "i-frame"
                frame_type = -1
            elif len(pair) == 2:
                frame_idx = pair[1]
                inputs['x1'] = frame_buffer[pair[0]]
                inputs['xt'] = {
                    "residual_stream": stream.readStream(),
                    "motion_stream": stream.readStream(),
                }
                mode = "b*-frame"
                frame_type = 2
            else:
                frame_idx = pair[1]
                inputs['x1'] = frame_buffer[pair[0]]
                inputs['x2'] = frame_buffer[pair[-1]]
                inputs['xt'] = {
                    "residual_stream": stream.readStream(),
                    "motion_stream": stream.readStream(),
                }

                if order[pair[0]] > order[pair[-1]]:
                    inputs['x1'], inputs['x2'] = inputs['x2'], inputs['x1']

                mode = "b-frame"
                if abs(pair[-1] - pair[0]) > 2:
                    frame_type = 0
                else:
                    frame_type = 1

            output = self.model(inputs, mode, frame_type, shape, eval_mode)
            filename = os.path.join(self.args.save_dir, f"frame_{frame_idx + frame_bias}.png")
            save_image(output["rec_frame"], filename)

            frame_buffer[frame_idx] = output["rec_frame"]
            if i > 0 and i  % self.args.intra_period == 0:
                frame_buffer = {0: frame_buffer[self.args.intra_period]}
                frame_bias += self.args.intra_period

        stream.close()

if __name__ == '__main__':
    seed_everything(888888)
    torch.backends.cudnn.deterministic = True

    parser = argparse.ArgumentParser(add_help=True)
    parser.add_argument("--config",           type=str, default="./cfgs/bcanf.yaml")
    parser.add_argument('--src',              type=str, required=True)
    parser.add_argument('--ckpt',             type=str, required=True)
    parser.add_argument('--save_dir',         type=str, required=True)
    parser.add_argument('--mode',             type=str, choices=["test", "compress", "decompress"], required=True)
    parser.add_argument('--intra_period',     type=int, default=32, help="intra period")
    parser.add_argument('--gop_size',         type=int, default=16, help="gop size")
    parser.add_argument('--device',           type=str, choices=["cuda", "cpu"], default="cuda")
    parser.add_argument('--num_workers',      type=int, default=8)
    parser.add_argument('--num_frames',       type=int, default=-1, help="number of frame to test")
    args = parser.parse_args()

    if args.mode in ["test", "compress"]:
        if args.num_frames < 0:
            parser.error("test/compress mode requires --num_frames")

    args = parser.parse_args()

    config = get_setting(args.config)
    model = BCANF(config).to(args.device)
    model.eval()
    model.load_state_dict(torch.load(args.ckpt, map_location=args.device), strict=True)

    tester = Tester(args, model)
    if args.mode == "test":
        tester.test()
    elif args.mode == "compress":
        tester.compress()
    elif args.mode == "decompress":
        tester.decompress()
    else:
        raise ValueError(f"invalid running mode: {args.mode}")


# -*- coding: utf-8 -*-
import re
import time
import pandas as pd
import scrapy
import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from elasticsearch import Elasticsearch

class ProductTradeSpider(scrapy.Spider):
    name = 'product_trade'
    allowed_domains = ['https://www.trademap.org/']
    start_urls = ['https://www.trademap.org/Product_SelCountry_TS.aspx?nvpm=1|788||||TOTAL|||2|1|1|2|2|1|1|1|1']

    def parse(self, response):
        title=response.xpath("//span[@id='ctl00_Label_Title']/text()").extract()
        reporter=re.findall(r'exported by ([\w\s]+)', title[0])[0]
        outputs=response.xpath("//a[contains(@href,'Sort$output')]//text()").extract()
        year=re.findall(r'\d\d\d\d',' '.join(outputs))
        driver = webdriver.Firefox()
        driver.get(response.url)
        driver.switch_to_active_element
        product_list = driver.find_elements_by_xpath("//table[@id='ctl00_PageContent_MyGridView1']//tr[position() > 2]")
        for co in product_list[:-1]:
            line=' '.join(co.text.split(' ')[1:])
            product=re.findall(r'[\D,\s.\'()_-]*',line)[0].strip()
            vals=line[len(product)+1:].split()
            for i in range(len(vals)):
                driver.switch_to_active_element
                item = {}
                item['partners'] = "All"
                item['reporters'] = reporter
                item['products'] = product
                item['trade_value'] = int((vals[i]+'000').replace(',',''))
                item['years'] = year[i]
                yield item
        links = list(set(response.xpath("//a[contains(@href,'Page$')]//text()").extract()))
        for link in links:
            next_page = driver.find_element_by_link_text(link)
            next_page.click()
            #wait=WebDriverWait(driver, 20)
            #wait.until(elementIdentified(By.id("ctl00_PageContent_MyGridView1")))
            driver.manage().timeouts().implicitlyWait(20, TimeUnit.SECONDS)
            driver.switch_to_active_element
            driver.manage().timeouts().implicitlyWait(20, TimeUnit.SECONDS)
            product_list = driver.find_elements_by_xpath("//table[@id='ctl00_PageContent_MyGridView1']//tr[position() > 2]")

            for co in product_list[:-1]:
                driver.manage().timeouts().implicitlyWait(20, TimeUnit.SECONDS)
                time.sleep(20)
                line=' '.join(co.text.split(' ')[1:])
                product=re.findall(r'[\D,\s.\'()_-]*',line)[0].strip()
                vals=line[len(product)+1:].split()
                for i in range(len(vals)):
                    driver.switch_to_active_element
                    item = {}
                    item['partners'] = "All"
                    item['reporters'] = reporter
                    item['products'] = product
                    item['trade_value'] = int((vals[i]+'000').replace(',',''))
                    item['years'] = year[i]
                    yield item
            time.sleep(2)


"""Service for Imgur images and videos."""
import json
import os
import re
from typing import Any
from urllib.parse import urlparse

import requests
from requests import Response

from ..models.content_type import ContentType
from ..models.media import Media
from .service import Service

class Imgur(Service):
    """Service for Imgur images and videos."""

    @classmethod
    def preprocess(cls, url: str, data: Any) -> str:
        """
        Override of `pyreddit.services.service.Service.preprocess` method.

        Gets the media hash from the url and creates the accepted provider media
        url.
        """
        media_hash: str = urlparse(url).path.rpartition("/")[2]
        r = re.compile(r"image|gallery").search(url)
        api: str = r.group() if r else "image"
        if "." in media_hash:
            media_hash = media_hash.rpartition(".")[0]
        return f"https://api.imgur.com/3/{api}/{media_hash}"

    @classmethod
    def get(cls, url: str) -> Response:
        """
        Override of `pyreddit.services.service.Service.get` method.

        Makes an API call with the client ID as authorization.
        """
        return requests.get(
            url,
            headers={
                "Authorization": f"Client-ID {os.getenv('IMGUR_CLIENT_ID')}"  # type: ignore
            },
        )

    @classmethod
    def postprocess(cls, response) -> Media:
        """
        Override of `pyreddit.services.service.Service.postprocess` method.

        Creates the right media object based on the size of provider's media.
        """
        data: Any = json.loads(response.content)["data"]
        media: Media
        if "images" in data:
            data = data["images"][0]
        if "image/jpeg" in data["type"] or "image/png" in data["type"]:
            media = Media(data["link"], ContentType.PHOTO, data["size"])
        elif "video" or "image/gif" in data["type"]:
            media = Media(data["mp4"], ContentType.VIDEO, data["mp4_size"])

        return media


import numpy as np
from tensorflow.keras.utils import to_categorical
from sklearn.model_selection import StratifiedKFold
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import BatchNormalization, InputLayer, Dense, TimeDistributed, Conv2D, MaxPool2D, Flatten, LSTM, Dropout

def decode(datum):
    a = np.zeros((datum.shape[0], 1))
    for j in range(datum.shape[0]):
        a[j] = np.argmax(datum[j])
    return a

def encode(datum):
    return to_categorical(datum)

np.random.seed(1)
K = 10
inner_activation_fun = 'relu'
outer_activation_fun = 'sigmoid'
optimizer_loss_fun = 'mse'
optimizer_algorithm = 'adam'
number_inner_layers = 3
number_inner_neurons = 256
number_epoch = 1
batch_length = 1
show_inter_results = 0
verbose = 0

print("Loading Data ...")
Data = np.loadtxt(r"C:\users\alan9\Desktop\Msc\Research\My Analysis\DroneRF-master\Python\RF_Data_short.csv", delimiter=",")

# print("Preparing Data ...")
# x = np.transpose(Data[0:10000, :])
# Label_1 = np.transpose(Data[10000:10001, :])
# Label_1 = Label_1.astype(int)
#
# Label_2 = np.transpose(Data[10001:10002, :])
# Label_2 = Label_2.astype(int)
#
# Label_3 = np.transpose(Data[10002:10003, :])
# Label_3 = Label_3.astype(int)
#
# outputType = [2, 4, 10]
# y = encode(Label_2)

print("Preparing Data ...")
x = np.transpose(Data[0:7, :])
Label_1 = np.transpose(Data[7:8, :])
Label_1 = Label_1.astype(int)

Label_2 = np.transpose(Data[8:9, :])
Label_2 = Label_2.astype(int)

Label_3 = np.transpose(Data[9:10, :])
Label_3 = Label_3.astype(int)

outputType = [2, 4, 10]
y = encode(Label_2)

cvscores = []
cnt = 0
kfold = StratifiedKFold(n_splits=K, shuffle=True, random_state=1)

print("Starting teaching process ...")
for train, test in kfold.split(x, decode(y)):

    print('Iteration index -- ' + str(cnt/len(train)*100) + ' %')
    cnt = cnt + 1

    x = x.reshape(x.shape[0], x.shape[1], 1)
    y = y.reshape(1, -1)[0]

    timesteps = x.shape[1]
    n_features = 1
    model = Sequential()
    model.add(LSTM(100, input_shape=(timesteps, n_features)))
    model.add(Dropout(0.5))
    model.add(Dense(100, activation='relu'))
    model.add(Dense(1, activation='softmax'))
    model.compile(loss=optimizer_loss_fun, optimizer=optimizer_algorithm, metrics=['accuracy'])

    model.fit(x[train], y[train], epochs=number_epoch, batch_size=batch_length, verbose=show_inter_results)
    scores = model.evaluate(x[test], y[test], verbose=show_inter_results)

    print(scores[1] * 100)
    cvscores.append(scores[1] * 100)
    y_pred = model.predict(x[test])
    np.savetxt("Results_2%s.csv" % cnt, np.column_stack((y[test], y_pred)), delimiter=",", fmt='%s')


# Import Packages
import os
import json
import requests
from dotenv import load_dotenv
from pathlib import Path

# Data
import numpy as np
import pandas as pd

# Visualizations
import streamlit as st
import plotly.express as px
import plotly.graph_objects as go
import plotly.express as px

#-----------APP CONFIG---------------#
APP_NAME = 'Churn Prediction App'
st.set_page_config(page_title=APP_NAME,
                   page_icon=':bar_chart:',
                   layout='wide') #responsive layout
st.title(APP_NAME)
st.markdown(
    '''
    This app predicts whether a given customer is a Churn Customer
    or a Staying Customer along with other actionable insights
    '''
)
cache_path = Path('./Data/cache/__df__.csv')

#-----------Hide Styles---------------#
unwanted_styles = """

"""
st.markdown(unwanted_styles, unsafe_allow_html=True)

#------------Mapbox API--------------#

# Read the Mapbox API key
load_dotenv()
map_box_api = os.getenv("mapbox")

# Set the mapbox access token
px.set_mapbox_access_token(map_box_api)

#------------Sessions--------------#

# APP State
if 'state' not in st.session_state:
    st.session_state['state'] = 'begin'

# Upload State
if 'upload' not in st.session_state:
    st.session_state['upload'] = False

#------------Helper Functions--------------#

# Save Data with predictions
def save_data(df):
    open(cache_path, 'w').write(df.to_csv(index=False))

# Load Data with predictions
def load_data():
    return pd.read_csv(cache_path,
                    encoding='UTF-8',
                    index_col=False)

#------------Plot Functions--------------#

# Customer Location Map
def churn_map(df):
    churn_map = px.scatter_mapbox(
            df,
            lat='lat',
            lon='long',
            color='prediction',
            size='monthly_charges',
            color_continuous_scale=px.colors.sequential.Rainbow,
            size_max=25,
            zoom=8)

    churn_map.update_traces(marker=dict(size=np.where(df['prediction'] == 'Churn Customer', 25, 8)),
                                    selector=dict(mode='markers'))

    st.plotly_chart(churn_map, use_container_width=True)

# Customer Insights
def customer_analytics(df):
    col1, col2, col3 = st.columns(3)
    contract_details = df['contract'].value_counts()
    internet_details = df['internet_service'].value_counts()
    payment_details = df['payment_method'].value_counts()

    with col1:
        labels = contract_details.index.tolist()
        values = contract_details.values.tolist()
        fig = go.Figure(data=[go.Pie(labels=labels, values=values, hole=.5)])
        fig.update_layout(title_text="Contract Type")
        st.plotly_chart(fig, use_container_width=True)
    with col2:
        labels = internet_details.index.tolist()
        values = internet_details.values.tolist()
        fig = go.Figure(data=[go.Pie(labels=labels, values=values, hole=.5)])
        fig.update_layout(title_text="Internet Service")
        st.plotly_chart(fig, use_container_width=True)
    with col3:
        labels = payment_details.index.tolist()
        values = payment_details.values.tolist()
        fig = go.Figure(data=[go.Pie(labels=labels, values=values, hole=.5)])
        fig.update_layout(title_text="Payment Method")
        st.plotly_chart(fig, use_container_width=True)

#----------SIDEBAR----------------#

# Sidebar Header
st.sidebar.header('Upload your EXCEL / CSV data')

# Loading Dataset Module
upload_data = st.sidebar.file_uploader("Upload your customers data here..", type=['csv', 'xlsx'])

if upload_data is not None:
    st.session_state['upload'] = True
else:
    st.session_state['upload'] = False

if st.session_state['upload']:

    # Custom Header
    st.subheader('Customer Data')

    # Get Uploaded Data as a DataFrame
    def get_data():
        if st.session_state['state'] == 'begin':
            if upload_data is not None:
                # Read the dataset
                ext = upload_data.name.rsplit('.', 1)[1]
                if ext == 'csv':
                    df = pd.read_csv(upload_data,
                                        encoding='UTF-8',
                                        index_col=False)
                elif ext == 'xlsx':
                    df = pd.read_excel(upload_data,
                                        sheet_name=0,
                                        index_col=False)
                else:
                    return None

                if (df is not None):
                    if not df.empty:
                        st.caption(f"Rows: {df.shape[0]} | Columns: {df.shape[1]}")
                        st.info('Your Dataset is Uploaded. Hit Predict!')
                        return df
                    else:
                        st.info('Your Dataset is empty, please check if you have data in the file before uploading!')
                        return None
                else:
                    st.info('Your Dataset has a problem, please check before uploading!')
                    return None
            else:
                st.info('Upload your CSV / Excel file from the sidebar section')
                return None

        else:
            # If prediction is already made then load the Dataset with prediction
            return load_data()

    # Load the data
    telco_df = get_data()

    #-----------BODY---------------#

    # Machine Learning Prediction through API
    if st.session_state['state'] == 'begin':
        if st.button("Predict"):
            # Checking if the data is available
            if upload_data is not None:
                # Chage the session state to predict
                st.session_state['state'] = 'predict'
                # Loading Bar Begin
                my_bar = st.progress(0)

                predictions = []
                for i in range(telco_df.shape[0]):
                    data = telco_df[['city',
                                'gender',
                                'senior_citizen',
                                'partner',
                                'dependents',
                                'phone_service',
                                'multiple_lines',
                                'internet_service',
                                'online_security',
                                'online_backup',
                                'device_protection',
                                'tech_support',
                                'streaming_tv',
                                'streaming_movies',
                                'contract',
                                'paperless_billing',
                                'payment_method',
                                'monthly_charges',
                                'total_charges',
                                'tenure',
                                'lat',
                                'long',
                                'zip']][i:i+1].to_json(orient='records')

                    # Request a prediction from the API for a customer
                    res = requests.post(
                        "http://ec2-3-25-148-140.ap-southeast-2.compute.amazonaws.com:8000/predict/", # AWS End Point
                        json=json.loads(data[1:-1])) # Pass Customer Data in JSON format
                    # Read the Response as a JSON Format
                    prediction = json.loads(res.text)
                    # Store each prediction in a list
                    predictions.append(prediction['prediction'][0])
                    # Loading Bar
                    my_bar.progress(i+1)

                # Convert predictions to a DataFrame
                prediction_df = pd.DataFrame({'prediction' : predictions})
                # Append it to loaded dataset
                telco_df = pd.concat([prediction_df, telco_df], axis=1)
                # Save the dataset
                save_data(telco_df)

    # If the data is already predicted
    if st.session_state['state'] == 'predict':

        # Select Churn
        churn = st.sidebar.multiselect(
            "Churn",
            options=telco_df['prediction'].unique(),
            default=telco_df['prediction'].unique()
        )

        # Select Gender
        gender = st.sidebar.multiselect(
            "Gender",
            options=telco_df['gender'].unique(),
            default=telco_df['gender'].unique()
        )

        # Select Contract
        contract = st.sidebar.multiselect(
            "Contract",
            options=telco_df['contract'].unique(),
            default=telco_df['contract'].unique()
        )

        # Select Internet Service
        internet_service = st.sidebar.multiselect(
            "Internet Service",
            options=telco_df['internet_service'].unique(),
            default=telco_df['internet_service'].unique()
        )

        # Select Payment Method
        payment_method = st.sidebar.multiselect(
            "Payment Method",
            options=telco_df['payment_method'].unique(),
            default=telco_df['payment_method'].unique()
        )

        # Select City
        city = st.sidebar.multiselect(
            "City",
            options=telco_df['city'].unique(),
            default=telco_df['city'].unique()
        )

        # Filtering Data
        telco_df_query = telco_df.query(
            '''
            gender == @gender & \
            prediction == @churn & \
            contract == @contract & \
            internet_service == @internet_service & \
            payment_method == @payment_method & \
            city == @city
            '''
        )

        # Query out based on the filters
        telco_df = telco_df_query.reset_index(drop=True)
        st.dataframe(telco_df)
        st.caption(f"Rows: {telco_df.shape[0]} | Columns: {telco_df.shape[1]}")

        # Customers Location MAP
        st.subheader('Churn Map')
        churn_map(telco_df)

        # Customer Analytics
        st.subheader('Customer Analytics')
        customer_analytics(telco_df)

else:
    # Check if the cache data is available and remove before restart
    try:
        if cache_path.is_file():
            os.remove(cache_path)
    except OSError as e:
        print(f"Error log: {e.filename} : {e.strerror}")

    # Reset the session
    st.session_state['state'] = 'begin'
    # Upload notification to begin
    st.info('Upload your CSV / Excel file from the sidebar section')


# -*- coding: utf-8 -*-
"""
Created on Sat Aug  3 23:26:12 2019

@author: pawan
"""

"""Air QualityUCI data"""

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

#importing and visualizing data
air_data = pd.read_excel('AirQualityUCI.xlsx')

air_data.head()
air_data.shape

#dropping ueless information from the data

air_data.dropna(axis=0, how='all')

features=air_data
features = features.drop('Date', axis=1)
features = features.drop('Time', axis=1)
features = features.drop('C6H6(GT)', axis=1)
features = features.drop('PT08.S4(NO2)', axis=1)
features

labels = air_data['C6H6(GT)']
#features=features.values

#training and testing

from sklearn.model_selection import train_test_split

X_train,X_test,Y_train,Y_test = train_test_split(features,labels,test_size=.2)

#applying differebt regression models

#1. Linear

from sklearn.linear_model import LinearRegression
regressor=LinearRegression()
regressor.fit(X_train,Y_train)

Y_pred=regressor.predict(X_test)

#2. Polynomial

from sklearn.preprocessing import PolynomialFeatures
Poly=PolynomialFeatures(degree=4)
X_train_poly=Poly.fit_transform(X_train)
X_test_poly=Poly.fit_transform(X_test)
regressor.fit(X_train_poly,Y_train)
Y_pred_poly=regressor.predict(X_test_poly)

#3. Support Vector Regression

     #feature scaling

from sklearn.preprocessing import StandardScaler
X_sc=StandardScaler()
Y_sc=StandardScaler()
Y_train=Y_train.values.reshape(-1,1) #<did this to resshape 1D to 2D>
X_train_sc=X_sc.fit_transform(X_train)
Y_train_sc=Y_sc.fit_transform(Y_train)
#Y_test=Y_test.values.reshape(-1,1)
X_test_sc=X_sc.fit_transform(X_test)
Y_test_sc=Y_sc.fit_transform(Y_test)

   #building the svr
from sklearn.svm import SVR
svmregressor=SVR(kernel='rbf',C=1000)
svmregressor.fit(X_train_sc,Y_train_sc)

Y_pred_svr=svmregressor.predict(X_test_sc)

Y_pred_svr=Y_sc.inverse_transform(Y_pred_svr)

#4. Decision tree regression

from sklearn.tree import DecisionTreeRegressor

DT=DecisionTreeRegressor()
DT.fit(X_train,Y_train)
  #feature importances in DTR
print(DT.feature_importances_)
featureimpDT=np.argsort(DT.feature_importances_)[::-1]
Y_pred_DT=DT.predict(X_test)

#5. Lasso Regression

from sklearn.linear_model import Lasso

classifier=Lasso(alpha=1)
classifier.fit(X_train,Y_train)

Y_pred_L=classifier.predict(X_test)

#6. ExtraTreeRegressor

from sklearn.ensemble import ExtraTreesRegressor
etr=ExtraTreesRegressor(n_estimators=300)
etr.fit(X_train,Y_train)

#Feature Importances on ETR
print(etr.feature_importances_)
indecis = np.argsort(etr.feature_importances_)[::-1]
plt.figure(num=None, figsize=(14, 10), dpi=80, facecolor='w')
plt.title("Feature importances")
plt.bar(range(X_train.shape[1]), etr.feature_importances_[indecis],
       color="r", align="center")
plt.xticks(range(X_train.shape[1]), indecis)
plt.show()

AirQualityUCI Benzene prediction project/AirQualityUCI prediction with LR,PR,DTR,Lasso,ETR.py


from abc import ABCMeta, abstractmethod
from typing import List

from tensorflow import Tensor
from tensorflow.keras.layers import MaxPooling2D, AveragePooling2D

from spivak.models.assembly.convolution_stacks import StridedBlockInterface, \
    ConvolutionStackInterface

POOLING_MAX = "max"
POOLING_AVERAGE = "average"

class BottomUpStackInterface(metaclass=ABCMeta):

    @abstractmethod
    def downsample_and_convolve(
            self, bottom_up: Tensor, num_filters_in: int, num_filters_out: int,
            layer_index: int) -> Tensor:
        pass

    @abstractmethod
    def convolve(
            self, bottom_up: Tensor, num_filters: int,
            layer_index: int) -> Tensor:
        pass

class BottomUpLayer:

    def __init__(self, tensor: Tensor, num_channels: int) -> None:
        self.tensor = tensor
        self.num_channels = num_channels

class PoolingBottomUpStack(BottomUpStackInterface):

    def __init__(
            self, pooling: str,
            convolution_stack: ConvolutionStackInterface) -> None:
        self._pooling = pooling
        self._convolution_stack = convolution_stack
        self._name = "bu"

    def convolve(
            self, bottom_up: Tensor, num_filters: int,
            layer_index: int) -> Tensor:
        return self._convolution_stack.convolve(
            bottom_up, num_filters, layer_index, self._name)

    def downsample_and_convolve(
            self, bottom_up: Tensor, num_filters_in: int, num_filters_out: int,
            layer_index: int) -> Tensor:
        if self._pooling == POOLING_MAX:
            pooling = MaxPooling2D(
                (2, 1), name=f"{self._name}_layer{layer_index}_max_pooling")
        elif self._pooling == POOLING_AVERAGE:
            pooling = AveragePooling2D(
                (2, 1), name=f"{self._name}_layer{layer_index}_average_pooling")
        else:
            raise ValueError(f"Unrecognized pooling: {self._pooling}")
        pooled = pooling(bottom_up)
        return self._convolution_stack.convolve(
            pooled, num_filters_out, layer_index, self._name)

class StridedBottomUpStack(BottomUpStackInterface):

    def __init__(
            self, strided_block: StridedBlockInterface,
            layer_num_blocks: List[int], strided_reduction: bool) -> None:
        self._strided_block = strided_block
        self._layer_num_blocks = layer_num_blocks
        self._strided_reduction = strided_reduction
        self._name = "bu"

    def convolve(
            self, bottom_up: Tensor, num_filters: int,
            layer_index: int) -> Tensor:
        for block_index in range(self._layer_num_blocks[layer_index]):
            bottom_up = self._strided_block.convolve(
                bottom_up, num_filters,
                name=f"{self._name}_layer{layer_index}_block{block_index}")
        return bottom_up

    def downsample_and_convolve(
            self, bottom_up: Tensor, num_filters_in: int, num_filters_out: int,
            layer_index: int) -> Tensor:
        num_blocks_in_layer = self._layer_num_blocks[layer_index]
        if self._strided_reduction or num_blocks_in_layer < 2:
            stride_filters = num_filters_out
        else:
            stride_filters = num_filters_in
        bottom_up = self._strided_block.strided_convolve(
            bottom_up, stride_filters,
            name=f"{self._name}_layer{layer_index}_block0")
        for block_index in range(1, num_blocks_in_layer):
            bottom_up = self._strided_block.convolve(
                bottom_up, num_filters_out,
                name=f"{self._name}_layer{layer_index}_block{block_index}")
        return bottom_up

def create_bottom_up_layers(
        input_mlp_out: Tensor, num_layers: int, base_num_filters: int,
        max_num_filters: int, bottom_up_stack: BottomUpStackInterface
) -> List[BottomUpLayer]:
    # VGG-16 applied dropout of 0.5 to their last two layers. U-net paper did
    # something similar, only applying dropout at the end of their bottom-up
    # layers.
    # https://arxiv.org/pdf/1409.1556.pdf
    # https://arxiv.org/pdf/1505.04597.pdf
    bottom_up_layers = []
    # Start with just a convolution stack and add that to the layers.
    num_filters_out = min(max_num_filters, base_num_filters)
    x = bottom_up_stack.convolve(input_mlp_out, num_filters_out, 0)
    bottom_up_layers.append(BottomUpLayer(x, num_filters_out))
    # Now, for each layer, add a stack that downsamples then convolves.
    for layer_index in range(1, num_layers):
        num_filters_in = num_filters_out
        num_filters_out = min(
            max_num_filters, 2 ** layer_index * base_num_filters)
        x = bottom_up_stack.downsample_and_convolve(
            x, num_filters_in, num_filters_out, layer_index)
        bottom_up_layers.append(BottomUpLayer(x, num_filters_out))
    return bottom_up_layers


from __future__ import division

import math
import numbers

import paddle
import paddle.nn.functional as F
import tensorlayerx as tlx
import sys
import collections

__all__ = ["resize","pad","hflip","vflip","crop","normalize","to_grayscale"]

def _assert_image_tensor(img, data_format):
    if not isinstance(
            img, paddle.Tensor
    ) or img.ndim < 3 or img.ndim > 4 or not data_format.lower() in ('chw',
                                                                     'hwc'):
        raise RuntimeError(
            'not support [type={}, ndim={}, data_format={}] paddle image'.
            format(type(img), img.ndim, data_format))

def _get_image_h_axis(data_format):
    if data_format.lower() == 'chw':
        return -2
    elif data_format.lower() == 'hwc':
        return -3

def _get_image_w_axis(data_format):
    if data_format.lower() == 'chw':
        return -1
    elif data_format.lower() == 'hwc':
        return -2

def _get_image_c_axis(data_format):
    if data_format.lower() == 'chw':
        return -3
    elif data_format.lower() == 'hwc':
        return -1

def _get_image_size(img, data_format):
    return img.shape[_get_image_w_axis(data_format)], img.shape[
        _get_image_h_axis(data_format)]

def _is_channel_first(data_format):
    return _get_image_c_axis(data_format) == -3

def resize(img, size, interpolation='bilinear', data_format='CHW'):
    """
    Resizes the image to given size
    Args:
        input (paddle.Tensor): Image to be resized.
        size (int|list|tuple): Target size of input data, with (height, width) shape.
        interpolation (int|str, optional): Interpolation method. when use paddle backend,
            support method are as following:
            - "nearest"
            - "bilinear"
            - "bicubic"
            - "trilinear"
            - "area"
            - "linear"
        data_format (str, optional): paddle.Tensor format
            - 'CHW'
            - 'HWC'
    Returns:
        paddle.Tensor: Resized image.
    """
    _assert_image_tensor(img, data_format)

    if not (isinstance(size, int) or
            (isinstance(size, (tuple, list)) and len(size) == 2)):
        raise TypeError('Got inappropriate size arg: {}'.format(size))

    if isinstance(size, int):
        w, h = _get_image_size(img, data_format)
        if (w <= h and w == size) or (h <= w and h == size):
            return img
        if w < h:
            ow = size
            oh = int(size * h / w)
        else:
            oh = size
            ow = int(size * w / h)
    else:
        oh, ow = size

    img = img.unsqueeze(0)
    # img = F.interpolate(img,
    #                     size=(oh, ow),
    #                     mode=interpolation.lower(),
    #                     data_format='N' + data_format.upper())
    img = tlx.ops.interpolate(img,
                        size=(oh, ow),
                        mode=interpolation.lower(),
                        data_format='N' + data_format.upper())

    return img.squeeze(0)

def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'):
    """
    Pads the given paddle.Tensor on all sides with specified padding mode and fill value.
    Args:
        img (paddle.Tensor): Image to be padded.
        padding (int|list|tuple): Padding on each border. If a single int is provided this
            is used to pad all borders. If tuple of length 2 is provided this is the padding
            on left/right and top/bottom respectively. If a tuple of length 4 is provided
            this is the padding for the left, top, right and bottom borders
            respectively.
        fill (float, optional): Pixel fill value for constant fill. If a tuple of
            length 3, it is used to fill R, G, B channels respectively.
            This value is only used when the padding_mode is constant. Default: 0.
        padding_mode: Type of padding. Should be: constant, edge, reflect or symmetric. Default: 'constant'.
            - constant: pads with a constant value, this value is specified with fill
            - edge: pads with the last value on the edge of the image
            - reflect: pads with reflection of image (without repeating the last value on the edge)
                       padding [1, 2, 3, 4] with 2 elements on both sides in reflect mode
                       will result in [3, 2, 1, 2, 3, 4, 3, 2]
            - symmetric: pads with reflection of image (repeating the last value on the edge)
                         padding [1, 2, 3, 4] with 2 elements on both sides in symmetric mode
                         will result in [2, 1, 1, 2, 3, 4, 4, 3]
    Returns:
        paddle.Tensor: Padded image.
    """
    _assert_image_tensor(img, data_format)

    if not isinstance(padding, (numbers.Number, list, tuple)):
        raise TypeError('Got inappropriate padding arg')
    if not isinstance(fill, (numbers.Number, str, list, tuple)):
        raise TypeError('Got inappropriate fill arg')
    if not isinstance(padding_mode, str):
        raise TypeError('Got inappropriate padding_mode arg')

    if isinstance(padding, (list, tuple)) and len(padding) not in [2, 4]:
        raise ValueError(
            "Padding must be an int or a 2, or 4 element tuple, not a " +
            "{} element tuple".format(len(padding)))

    assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \
        'Padding mode should be either constant, edge, reflect or symmetric'

    if isinstance(padding, int):
        pad_left = pad_right = pad_top = pad_bottom = padding
    elif len(padding) == 2:
        pad_left = pad_right = padding[0]
        pad_top = pad_bottom = padding[1]
    else:
        pad_left = padding[0]
        pad_top = padding[1]
        pad_right = padding[2]
        pad_bottom = padding[3]

    padding = [pad_left, pad_right, pad_top, pad_bottom]

    if padding_mode == 'edge':
        padding_mode = 'replicate'
    elif padding_mode == 'symmetric':
        raise ValueError('Do not support symmetric mode')

    img = img.unsqueeze(0)
    #  'constant', 'reflect', 'replicate', 'circular'
    img = F.pad(img,
                pad=padding,
                mode=padding_mode,
                value=float(fill),
                data_format='N' + data_format)

    return img.squeeze(0)

def hflip(img, data_format='CHW'):
    """Horizontally flips the given paddle.Tensor Image.
    Args:
        img (paddle.Tensor): Image to be flipped.
        data_format (str, optional): Data format of img, should be 'HWC' or
            'CHW'. Default: 'CHW'.
    Returns:
        paddle.Tensor:  Horizontall flipped image.
    """
    _assert_image_tensor(img, data_format)

    w_axis = _get_image_w_axis(data_format)

    return img.flip(axis=[w_axis])

def vflip(img, data_format='CHW'):
    """Vertically flips the given paddle tensor.
    Args:
        img (paddle.Tensor): Image to be flipped.
        data_format (str, optional): Data format of img, should be 'HWC' or
            'CHW'. Default: 'CHW'.
    Returns:
        paddle.Tensor:  Vertically flipped image.
    """
    _assert_image_tensor(img, data_format)

    h_axis = _get_image_h_axis(data_format)

    return img.flip(axis=[h_axis])

def crop(img, top, left, height, width, data_format='CHW'):
    """Crops the given paddle.Tensor Image.
    Args:
        img (paddle.Tensor): Image to be cropped. (0,0) denotes the top left
            corner of the image.
        top (int): Vertical component of the top left corner of the crop box.
        left (int): Horizontal component of the top left corner of the crop box.
        height (int): Height of the crop box.
        width (int): Width of the crop box.
        data_format (str, optional): Data format of img, should be 'HWC' or
            'CHW'. Default: 'CHW'.
    Returns:
        paddle.Tensor: Cropped image.
    """
    _assert_image_tensor(img, data_format)

    if _is_channel_first(data_format):
        return img[:, top:top + height, left:left + width]
    else:
        return img[top:top + height, left:left + width, :]

def normalize(img, mean, std, data_format='CHW'):
    """Normalizes a tensor image given mean and standard deviation.
    Args:
        img (paddle.Tensor): input data to be normalized.
        mean (list|tuple): Sequence of means for each channel.
        std (list|tuple): Sequence of standard deviations for each channel.
        data_format (str, optional): Data format of img, should be 'HWC' or
            'CHW'. Default: 'CHW'.
    Returns:
        Tensor: Normalized mage.
    """
    _assert_image_tensor(img, data_format)

    mean = paddle.to_tensor(mean, place=img.place)
    std = paddle.to_tensor(std, place=img.place)
    # mean = tlx.convert_to_tensor(mean)
    # std = tlx.convert_to_tensor(std)
    if _is_channel_first(data_format):
        mean = mean.reshape([-1, 1, 1])
        std = std.reshape([-1, 1, 1])

    return (img - mean) / std

def to_grayscale(img, num_output_channels=1, data_format='CHW'):
    """Converts image to grayscale version of image.
    Args:
        img (paddel.Tensor): Image to be converted to grayscale.
        num_output_channels (int, optionl[1, 3]):
            if num_output_channels = 1 : returned image is single channel
            if num_output_channels = 3 : returned image is 3 channel
        data_format (str, optional): Data format of img, should be 'HWC' or
            'CHW'. Default: 'CHW'.
    Returns:
        paddle.Tensor: Grayscale version of the image.
    """
    _assert_image_tensor(img, data_format)

    if num_output_channels not in (1, 3):
        raise ValueError('num_output_channels should be either 1 or 3')

    rgb_weights = paddle.to_tensor([0.2989, 0.5870, 0.1140],
                                   place=img.place).astype(img.dtype)
    # rgb_weights = tlx.convert_to_tensor([0.2989, 0.5870, 0.1140]).astype(img.dtype)

    if _is_channel_first(data_format):
        rgb_weights = rgb_weights.reshape((-1, 1, 1))

    _c_index = _get_image_c_axis(data_format)

    img = (img * rgb_weights).sum(axis=_c_index, keepdim=True)
    _shape = img.shape
    _shape[_c_index] = num_output_channels

    return img.expand(_shape)


import os
import time
import requests
from bs4 import BeautifulSoup

page_range = [129, 159]
limit_cache_num = 25

list_collect = []
save_idx = 0
while page_range[0] < page_range[1]:
	try:
		page = page_range[0]
		page_link = 'https://github.com/programthink?page=%d&tab=followers' % page
		print(page_link)
		r = requests.get(page_link)
		soup = BeautifulSoup(r.text, 'html.parser')
		results = soup.find_all("span", "link-gray pl-1")
		for idx, res in enumerate(results):
			if idx < save_idx:
				continue
			save_idx = idx
			user = res.string
			print('page: %s, user: %s' % (page,user))
			user_link = 'https://github.com/' + str(user)
			r = requests.get(user_link)
			so = BeautifulSoup(r.text, 'html.parser')
			links = so.find_all('a', "u-url")
			for l in links:
				link = links[0].string
				list_collect.append(str(link) + '#' + str(user_link))
				print(list_collect)
				print(len(list_collect))
				if len(list_collect) > limit_cache_num:
					raise Exception('overpage')
		page_range[0] += 1
		save_idx = 0
	except (Exception, KeyboardInterrupt):
		print('Ctrl-C interrupt, enter to continue ...')
		inp = input()
		if inp == 'quit':
			quit();
		elif inp == 'open':
			tot = len(list_collect)
			for idx, link in enumerate(list_collect):
				print("[%d / %d]" % (idx, tot))
				os.system('chromium ' + link)
				time.sleep(1)
			list_collect = []
		else:
			print('starting from ' + str(save_idx))
			continue

tot = len(list_collect)
for idx, link in enumerate(list_collect):
	print("[%d / %d]" % (idx, tot))
	os.system('chromium ' + link)
	time.sleep(1)


from datetime import datetime
from fastapi import APIRouter, WebSocket, WebSocketDisconnect

router = APIRouter(prefix="/ws")

@router.websocket("/session")
async def websocket_endpoint(websocket: WebSocket):
    await websocket.accept()
    start = datetime.now()
    while True:
        try:
            data = await websocket.receive_text()
            await websocket.send_text(f"Message text was: {data}")
        except WebSocketDisconnect as e:
            print(f"connection duration {datetime.now()-start}")
            return


from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait

class BaseAction:

    def __init__(self, driver):
        self.driver = driver

    def click(self, loc):
        self.find_element(loc).click()

    def input_text(self, loc, text):
        self.find_element(loc).send_keys(text)

    def find_element(self, loc, timeout=5.0, poll=1.0):
        by = loc[0]
        value = loc[1]  # "text,0"
        if by == By.XPATH:
            value = self.make_xpath_with_feature(value) # //*
            print(value)
        return WebDriverWait(self.driver, timeout, poll).until(lambda x: x.find_element(by, value))

    def find_elements(self, loc, timeout=5.0, poll=1.0):
        by = loc[0]
        value = loc[1]
        if by == By.XPATH:
            value = self.make_xpath_with_feature(value) # //*
        return WebDriverWait(self.driver, timeout, poll).until(lambda x: x.find_elements(by, value))

    def make_xpath_with_unit_feature(self, loc):
        """
        拼接xpath中间的部分
        :param loc:
        :return:
        """
        key_index = 0
        value_index = 1
        option_index = 2

        args = loc.split(",")
        feature = ""

        if len(args) == 2:
            feature = "contains(@" + args[key_index] + ",'" + args[value_index] + "')" + "and "
        elif len(args) == 3:
            if args[option_index] == "1":
                feature = "@" + args[key_index] + "='" + args[value_index] + "'" + "and "
            elif args[option_index] == "0":
                feature = "contains(@" + args[key_index] + ",'" + args[value_index] + "')" + "and "

        return feature

    def make_xpath_with_feature(self, loc):
        feature_start = "//*["
        feature_end = "]"
        feature = ""

        if isinstance(loc, str):
            # 如果是正常的xpath
            if loc.startswith("//"):
                return loc

            # loc str
            feature = self.make_xpath_with_unit_feature(loc)
        else:
            # loc 列表
            for i in loc:
                feature += self.make_xpath_with_unit_feature(i)

        feature = feature.rstrip("and ")

        loc = feature_start + feature + feature_end

        return loc

    def find_toast(self, message, is_screenshot=False, screenshot_name=None, timeout=3, poll=0.1):
        """
        # message: 预期要获取的toast的部分消息
        """
        message = "//*[contains(@text,'" + message + "')]"  # 使用包含的方式定位
        element = self.find_element((By.XPATH, message), timeout, poll)
        if is_screenshot:
            self.screenshot(screenshot_name)

        return element.text

    def is_toast_exist(self, message, is_screenshot=False, screenshot_name=None, timeout=3, poll=0.1):
        try:
            self.find_toast(message, is_screenshot, screenshot_name, timeout, poll)
            return True
        except Exception:
            return False

    def screenshot(self, file_name):
        self.driver.get_screenshot_as_file("./screen/" + file_name + ".png")


from collections import Counter

def solution(a):
    answer = 0
    if len(a) < 1:
        return 0
    count = Counter(a)
    for k, c in count.items():
        if c * 2 <= 0:
            continue

        index = 1
        num = 0

        while index < len(a):
            if (a[index-1] != k and a[index] != k) or a[index - 1] == a[index]:
                index +=1
                continue
            num += 2
            index += 2
        answer = max(answer, num)
    return answer

print(solution([5,2,3,3,5,3]))


import logging
import os

import pandas as pd

from demandprediction.prediction.model import BaseModel
from demandprediction.prediction.postprocessing.error_measures import log_grid_error
from demandprediction.prediction.postprocessing.postprocessing import calculate_grid_error
from demandprediction.prediction.preprocessing.preprocessing import DataPreprocessor
from demandprediction.utils.shared_filenames import get_predictions_filename

class ConvLSTMBaseModel(BaseModel):
    def __init__(self, model_name: str, config: dict):
        """
        Initializes the ConvLSTM model.
        Inputs are: Demand and Metainformation
        The Architecture does use a Hadamard layer
        :param config: Configuration for the prediction.
        """
        super().__init__(model_name, config['data_directory'])
        self.start_date = config['start_date']
        self.stop_date = config['stop_date']
        self.number_of_grid_rows = config['number_of_grid_rows']
        self.number_of_grid_columns = config['number_of_grid_columns']
        self.lags = config['lags']
        self.number_of_filters = config['number_of_filters']
        self.kernel_size = config['kernel_size']
        self.batch_size = config['batch_size']
        self.learning_rate = config['learning_rate']
        self.step_size = config['step_size']
        self.epochs = config['epochs']
        self.x_limits = (config['x_min'], config['x_max'])
        self.y_limits = (config['y_min'], config['y_max'])
        prediction_relative_path = get_predictions_filename(self.model_name, self.start_time)
        self.prediction_filepath = os.path.join(self.data_directory, prediction_relative_path)
        self.number_of_prediction_timesteps = 0

        self.logger = logging.getLogger(__name__)

    def prepare_data(self):
        super().prepare_data()
        data_preprocessor = DataPreprocessor(self.data_directory, self.x_limits, self.y_limits)
        self.ground_truth_filepath = data_preprocessor.generate_window_grid(
            stop_date=self.stop_date,
            number_of_rows=self.number_of_grid_rows,
            number_of_columns=self.number_of_grid_columns,
            step_size=self.step_size,
            feature_name='demand'
        )

        self.number_of_prediction_timesteps = len(
            DataPreprocessor.get_date_range_between_dates(
                start_date=self.start_date,
                end_date=self.stop_date,
                freq=f'{self.step_size}min',
                end_inclusive=False
            )
        )
        # set flag
        self.data_prepared = True

    def save_predictions(self):
        super().save_predictions()
        predictions = self.predictions.reshape(self.predictions.shape[0],
                                               self.number_of_grid_rows * self.number_of_grid_columns)
        predictions_df = pd.DataFrame(predictions)

        # load actual data for index
        # noinspection PyTypeChecker
        actual_data_df = pd.read_hdf(self.ground_truth_filepath, key='even_grid')  # type: pd.DataFrame
        predictions_df.index = actual_data_df.index[-len(predictions_df.index):]
        self.logger.debug(predictions_df.index[:10])
        self.logger.debug(predictions_df.index[-10:])

        predictions_df.to_hdf(self.prediction_filepath, key='predictions', complevel=6)
        self.logger.info(f"Predictions saved to file: {self.prediction_filepath}")

    def score_model(self):
        """
        Calculate the evaluation metrics (RMSE, MAPE, etc.).
        Save the metrics and model hyperparams to csv file.
        """
        super().score_model()

        # Calculate error
        model_id = "{}_{date:%Y-%m-%d_%H-%M-%S}.h5".format(
            self.model_name, date=self.start_time)

        errors = calculate_grid_error(
            filepath=self.ground_truth_filepath,
            prediction_filepath=self.prediction_filepath,
            number_of_rows=self.number_of_grid_rows,
            number_of_columns=self.number_of_grid_columns,
            ceiling=1,
            logger=self.logger
        )

        # Write the errors and all hyperparams to a csv file!
        df = pd.DataFrame({
            'model_id': [model_id],
            'model_name': [self.model_name],
            'avg_grid_rmse_above_ceiling': [errors[0]],
            'avg_grid_mae_above_ceiling': [errors[1]],
            'avg_grid_mapec_above_ceiling': [errors[2]],
            'global_rmse': [errors[3]],
            'global_mae': [errors[4]],
            'global_mape': [errors[5]],
            'number_of_rows': [self.number_of_grid_rows],
            'number_of_columns': [self.number_of_grid_columns],
            'lags': [self.lags],
            'nb_filters': [self.number_of_filters],
            'k_size': [self.kernel_size],
            'batch_size': [self.batch_size],
            'lr': [self.learning_rate],
            'step_size': [self.step_size],
        })

        filename = os.path.join(self.data_directory, "evaluation/metrics_and_hyperparams.csv")
        with open(filename, 'a') as f:
            df.to_csv(filename, mode='a', header=not f.tell())

        log_grid_error(self.logger, self.prediction_filepath, errors)

        self.logger.info("Model errors were calculated successfully")


import csv
import argparse
from datetime import datetime

parser = argparse.ArgumentParser()
parser.add_argument('-f', '--file')
args = parser.parse_args()

if args.file:
    filename = args.file
else:
    filename = input('Enter filename (including \'.csv\'): ')

dt = datetime.now().strftime('%Y-%m-%d %H.%M.%S')

f = csv.writer(open('expandedIdentifiers' + filename, 'w'))
f.writerow(['identifier'] + ['dc.identifier.other'])

with open(filename) as itemMetadataFile:
    itemMetadata = csv.DictReader(itemMetadataFile)
    for row in itemMetadata:
        identifier = row['identifier']
        zfill_identifier = str(identifier).zfill(5)
        prefix = 'jhu_coll-0002_'
        expandedId = prefix + zfill_identifier
        f.writerow([identifier] + [expandedId])


"""
Data Structures
===============

Defines various data structures classes:

-   :class:`colour.utilities.Structure`: An object similar to C/C++ structured
    type.
-   :class:`colour.utilities.Lookup`: A :class:`dict` sub-class acting as a
    lookup to retrieve keys by values.
-   :class:`CaseInsensitiveMapping`: A case insensitive
    :class:`dict`-like object allowing values retrieving from keys while
    ignoring the key case.
-   :class:`colour.utilities.LazyCaseInsensitiveMapping`: Another case
    insensitive mapping allowing lazy values retrieving from keys while
    ignoring the key case.
-   :class:`colour.utilities.Node`: A basic node object supporting creation of
    basic node trees.

References
----------
-   :cite:`Mansencalc` : Mansencal, T. (n.d.). Lookup.
    https://github.com/KelSolaar/Foundations/blob/develop/foundations/\
data_structures.py
-   :cite:`Rakotoarison2017` : Rakotoarison, H. (2017). Bunch. Retrieved
    December 4, 2021, from https://github.com/scikit-learn/scikit-learn/blob/\
0d378913b/sklearn/utils/__init__.py#L83
-   :cite:`Reitza` : Reitz, K. (n.d.). CaseInsensitiveDict.
    https://github.com/kennethreitz/requests/blob/v1.2.3/requests/\
structures.py#L37
"""

from __future__ import annotations

from collections.abc import MutableMapping

from colour.hints import (
    Any,
    Boolean,
    Dict,
    Generator,
    Integer,
    Iterable,
    List,
    Mapping,
    Optional,
    Union,
)
from colour.utilities.documentation import is_documentation_building

__author__ = "Colour Developers"
__copyright__ = "Copyright 2013 Colour Developers"
__license__ = "New BSD License - https://opensource.org/licenses/BSD-3-Clause"
__maintainer__ = "Colour Developers"
__email__ = "colour-developers@colour-science.org"
__status__ = "Production"

__all__ = [
    "attest",
    "Structure",
    "Lookup",
    "CaseInsensitiveMapping",
    "LazyCaseInsensitiveMapping",
    "Node",
]

def attest(condition: Boolean, message: str = ""):
    """
    Provide the `assert` statement functionality without being disabled by
    optimised Python execution.

    See :func:`colour.utilities.assert` for more information.

    Notes
    -----
    -   This definition is duplicated to avoid import circular dependency.
    """

    # Avoiding circular dependency.
    import colour.utilities

    colour.utilities.attest(condition, message)

class Structure(dict):
    """
    Define a :class:`dict`-like object allowing to access key values using dot
    syntax.

    Other Parameters
    ----------------
    args
        Arguments.
    kwargs
        Key / value pairs.

    Methods
    -------
    -   :meth:`~colour.utilities.Structure.__init__`
    -   :meth:`~colour.utilities.Structure.__setattr__`
    -   :meth:`~colour.utilities.Structure.__delattr__`
    -   :meth:`~colour.utilities.Structure.__dir__`
    -   :meth:`~colour.utilities.Structure.__getattr__`
    -   :meth:`~colour.utilities.Structure.__setstate__`

    References
    ----------
    :cite:`Rakotoarison2017`

    Examples
    --------
    >>> person = Structure(first_name='John', last_name='Doe', gender='male')
    >>> person.first_name
    'John'
    >>> sorted(person.keys())
    ['first_name', 'gender', 'last_name']
    >>> person['gender']
    'male'
    """

    def __init__(self, *args: Any, **kwargs: Any):
        super().__init__(*args, **kwargs)

    def __setattr__(self, name: str, value: Any):
        """
        Assign given value to the attribute with given name.

        Parameters
        ----------
        name
            Name of the attribute to assign the ``value`` to.
        value
            Value to assign to the attribute.
        """

        self[name] = value

    def __delattr__(self, name: str):
        """
        Delete the attribute with given name.

        Parameters
        ----------
        name
            Name of the attribute to delete.
        """

        del self[name]

    def __dir__(self) -> Iterable:
        """
        Return a list of valid attributes for the :class:`dict`-like object.

        Returns
        -------
        :class:`list`
            List of valid attributes for the :class:`dict`-like object.
        """

        return self.keys()

    def __getattr__(self, name: str) -> Any:
        """
        Return the value from the attribute with given name.

        Parameters
        ----------
        name
            Name of the attribute to get the value from.

        Returns
        -------
        :class:`object`

        Raises
        ------
        AttributeError
            If the attribute is not defined.
        """

        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)

    def __setstate__(self, state):
        """Set the object state when unpickling."""
        # See https://github.com/scikit-learn/scikit-learn/issues/6196 for more
        # information.

        pass

class Lookup(dict):
    """
    Extend :class:`dict` type to provide a lookup by value(s).

    Methods
    -------
    -   :meth:`~colour.utilities.Lookup.keys_from_value`
    -   :meth:`~colour.utilities.Lookup.first_key_from_value`

    References
    ----------
    :cite:`Mansencalc`

    Examples
    --------
    >>> person = Lookup(first_name='John', last_name='Doe', gender='male')
    >>> person.first_key_from_value('John')
    'first_name'
    >>> persons = Lookup(John='Doe', Jane='Doe', Luke='Skywalker')
    >>> sorted(persons.keys_from_value('Doe'))
    ['Jane', 'John']
    """

    def keys_from_value(self, value: Any) -> List:
        """
        Get the keys associated with given value.

        Parameters
        ----------
        value
            Value to find the associated keys.

        Returns
        -------
        :class:`list`
            Keys associated with given value.
        """

        keys = []
        for key, data in self.items():
            matching = data == value
            try:
                matching = all(matching)

            except TypeError:
                matching = all((matching,))

            if matching:
                keys.append(key)

        return keys

    def first_key_from_value(self, value: Any) -> Any:
        """
        Get the first key associated with given value.

        Parameters
        ----------
        value
            Value to find the associated first key.

        Returns
        -------
        :class:`object`
            First key associated with given value.
        """

        return self.keys_from_value(value)[0]

class CaseInsensitiveMapping(MutableMapping):
    """
    Implement a case-insensitive :class:`dict`-like object.

    Allows values retrieving from keys while ignoring the key case.
    The keys are expected to be str or :class:`str`-like objects supporting the
    :meth:`str.lower` method.

    Parameters
    ----------
    data
        Data to store into the case-insensitive :class:`dict`-like object at
        initialisation.

    Other Parameters
    ----------------
    kwargs
        Key / value pairs to store into the mapping at initialisation.

    Attributes
    ----------
    -   :attr:`~colour.utilities.CaseInsensitiveMapping.data`

    Methods
    -------
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__init__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__repr__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__setitem__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__getitem__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__delitem__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__contains__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__iter__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__len__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__eq__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.__ne__`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.copy`
    -   :meth:`~colour.utilities.CaseInsensitiveMapping.lower_items`

    References
    ----------
    :cite:`Reitza`

    Examples
    --------
    >>> methods = CaseInsensitiveMapping({'McCamy': 1, 'Hernandez': 2})
    >>> methods['mccamy']
    1
    """

    def __init__(
        self, data: Optional[Union[Generator, Mapping]] = None, **kwargs: Any
    ):
        self._data: Dict = dict()

        self.update({} if data is None else data, **kwargs)

    @property
    def data(self) -> Dict:
        """
        Getter property for the case-insensitive :class:`dict`-like object
        data.

        Returns
        -------
        :class:`dict`
            Data.
        """

        return self._data

    def __repr__(self) -> str:
        """
        Return an evaluable string representation of the case-insensitive
        :class:`dict`-like object.

        Returns
        -------
        :class:`str`
            Evaluable string representation.
        """

        if is_documentation_building():  # pragma: no cover
            representation = repr(
                dict(zip(self.keys(), ["..."] * len(self)))
            ).replace("'...'", "...")
            return f"{self.__class__.__name__}({representation})"
        else:
            return f"{self.__class__.__name__}({dict(self.items())})"

    def __setitem__(self, item: Union[str, Any], value: Any):
        """
        Set given item with given value in the case-insensitive
        :class:`dict`-like object.

        Parameters
        ----------
        item
            Item to set in the case-insensitive :class:`dict`-like object.
        value
            Value to store in the case-insensitive :class:`dict`-like object.

        Notes
        -----
        -   The item is stored as lower-case while the original name and its
            value are stored together as the value in a *tuple*::

            {"item.lower()": ("item", value)}
        """

        self._data[self._lower_key(item)] = (item, value)

    def __getitem__(self, item: Union[str, Any]) -> Any:
        """
        Return the value of given item from the case-insensitive
        :class:`dict`-like object.

        Parameters
        ----------
        item
            Item to retrieve the value of from the case-insensitive
            :class:`dict`-like object.

        Returns
        -------
        :class:`object`
            Item value.

        Notes
        -----
        -   The item value is retrieved by using its lower-case variant.
        """

        return self._data[self._lower_key(item)][1]

    def __delitem__(self, item: Union[str, Any]):
        """
        Delete given item from the case-insensitive :class:`dict`-like object.

        Parameters
        ----------
        item
            Item to delete from the case-insensitive :class:`dict`-like object.

        Notes
        -----
        -   The item is deleted by using its lower-case variant.
        """

        del self._data[self._lower_key(item)]

    def __contains__(self, item: Union[str, Any]) -> bool:
        """
        Return whether the case-insensitive :class:`dict`-like object contains
        given item.

        Parameters
        ----------
        item
            Item to find whether it is in the case-insensitive
            :class:`dict`-like object.

        Returns
        -------
        :class:`bool`
            Whether given item is in the case-insensitive :class:`dict`-like
            object.
        """

        return self._lower_key(item) in self._data

    def __iter__(self) -> Generator:
        """
        Iterate over the items of the case-insensitive :class:`dict`-like
        object.

        Yields
        ------
        Generator
            Item generator.

        Notes
        -----
        -   The iterated items are the original items.
        """

        return (item for item, value in self._data.values())

    def __len__(self) -> Integer:
        """
        Return the items count.

        Returns
        -------
        :class:`numpy.integer`
            Items count.
        """

        return len(self._data)

    def __eq__(self, other: Any) -> bool:
        """
        Return whether the case-insensitive :class:`dict`-like object is equal
        to given other object.

        Parameters
        ----------
        other
            Object to test whether it is equal to the case-insensitive
            :class:`dict`-like object

        Returns
        -------
        :class:`bool`
            Whether given object is equal to the case-insensitive
            :class:`dict`-like object.
        """

        if isinstance(other, Mapping):
            other_mapping = CaseInsensitiveMapping(other)
        else:
            raise ValueError(
                f"Impossible to test equality with "
                f'"{other.__class__.__name__}" class type!'
            )

        return dict(self.lower_items()) == dict(other_mapping.lower_items())

    def __ne__(self, other: Any) -> bool:
        """
        Return whether the case-insensitive :class:`dict`-like object is not
        equal to given other object.

        Parameters
        ----------
        other
            Object to test whether it is not equal to the case-insensitive
            :class:`dict`-like object

        Returns
        -------
        :class:`bool`
            Whether given object is not equal to the case-insensitive
            :class:`dict`-like object.
        """

        return not (self == other)

    @staticmethod
    def _lower_key(key: Union[str, Any]) -> Union[str, Any]:
        """
        Return the lower-case variant of given key, if the key cannot be
        lower-cased, it is passed unmodified.

        Parameters
        ----------
        key
            Key to return the lower-case variant.

        Returns
        -------
        :class:`str` or :class:`object`
            Key lower-case variant.
        """

        try:
            return key.lower()
        except AttributeError:
            return key

    def copy(self) -> CaseInsensitiveMapping:
        """
        Return a copy of the case-insensitive :class:`dict`-like object.

        Returns
        -------
        :class:`CaseInsensitiveMapping`
            Case-insensitive :class:`dict`-like object copy.

        Warnings
        --------
        -   The :class:`CaseInsensitiveMapping` class copy returned is a
            *copy* of the object not a *deepcopy*!
        """

        return CaseInsensitiveMapping(dict(self._data.values()))

    def lower_items(self) -> Generator:
        """
        Iterate over the lower-case items of the case-insensitive
        :class:`dict`-like object.

        Yields
        ------
        Generator
            Item generator.

        Notes
        -----
        -   The iterated items are the lower-case items.
        """

        return ((item, value[1]) for (item, value) in self._data.items())

class LazyCaseInsensitiveMapping(CaseInsensitiveMapping):
    """
    Implement a lazy case-insensitive :class:`dict`-like object inheriting
    from :class:`CaseInsensitiveMapping` class.

    Allows lay values retrieving from keys while ignoring the key case.
    The keys are expected to be str or :class:`str`-like objects supporting the
    :meth:`str.lower` method.

    The lazy retrieval is performed as follows: If the value is a callable,
    then it is evaluated and its return value is stored in place of the current
    value.

    Parameters
    ----------
    data
        Data to store into the lazy case-insensitive :class:`dict`-like object
        at initialisation.

    Other Parameters
    ----------------
    kwargs
        Key / value pairs to store into the mapping at initialisation.

    Methods
    -------
    -   :meth:`~colour.utilities.LazyCaseInsensitiveMapping.__getitem__`

    Examples
    --------
    >>> def callable_a():
    ...     print(2)
    ...     return 2
    >>> methods = LazyCaseInsensitiveMapping(
    ...     {'McCamy': 1, 'Hernandez': callable_a})
    >>> methods['mccamy']
    1
    >>> methods['hernandez']
    2
    2
    """

    def __getitem__(self, item: Union[str, Any]) -> Any:
        """
        Return the value of given item from the case-insensitive
        :class:`dict`-like object.

        Parameters
        ----------
        item
            Item to retrieve the value of from the case-insensitive
            :class:`dict`-like object.

        Returns
        -------
        :class:`object`
            Item value.

        Notes
        -----
        -   The item value is retrieved by using its lower-case variant.
        """

        import colour

        value = super().__getitem__(item)

        if callable(value) and hasattr(colour, "__disable_lazy_load__"):
            value = value()
            super().__setitem__(item, value)

        return value

class Node:
    """
    Represent a basic node supporting the creation of basic node trees.

    Parameters
    ----------
    name
        Node name.
    parent
        Parent of the node.
    children
        Children of the node.
    data
        The data belonging to this node.

    Attributes
    ----------
    -   :attr:`~colour.utilities.Node.name`
    -   :attr:`~colour.utilities.Node.parent`
    -   :attr:`~colour.utilities.Node.children`
    -   :attr:`~colour.utilities.Node.id`
    -   :attr:`~colour.utilities.Node.root`
    -   :attr:`~colour.utilities.Node.leaves`
    -   :attr:`~colour.utilities.Node.siblings`
    -   :attr:`~colour.utilities.Node.data`

    Methods
    -------
    -   :meth:`~colour.utilities.Node.__new__`
    -   :meth:`~colour.utilities.Node.__init__`
    -   :meth:`~colour.utilities.Node.__str__`
    -   :meth:`~colour.utilities.Node.__len__`
    -   :meth:`~colour.utilities.Node.is_root`
    -   :meth:`~colour.utilities.Node.is_inner`
    -   :meth:`~colour.utilities.Node.is_leaf`
    -   :meth:`~colour.utilities.Node.walk`
    -   :meth:`~colour.utilities.Node.render`

    Examples
    --------
    >>> node_a = Node('Node A')
    >>> node_b = Node('Node B', node_a)
    >>> node_c = Node('Node C', node_a)
    >>> node_d = Node('Node D', node_b)
    >>> node_e = Node('Node E', node_b)
    >>> node_f = Node('Node F', node_d)
    >>> node_g = Node('Node G', node_f)
    >>> node_h = Node('Node H', node_g)
    >>> [node.name for node in node_a.leaves]
    ['Node H', 'Node E', 'Node C']
    >>> print(node_h.root.name)
    Node A
    >>> len(node_a)
    7
    """

    _INSTANCE_ID: Integer = 1
    """
    Node id counter.

    _INSTANCE_ID
    """

    def __new__(cls, *args: Any, **kwargs: Any) -> Node:
        """
        Return a new instance of the :class:`colour.utilities.Node` class.

        Other Parameters
        ----------------
        args
            Arguments.
        kwargs
            Keywords arguments.
        """

        instance = super().__new__(cls)

        instance._id = Node._INSTANCE_ID  # type: ignore[attr-defined]
        Node._INSTANCE_ID += 1

        return instance

    def __init__(
        self,
        name: Optional[str] = None,
        parent: Optional[Node] = None,
        children: Optional[List[Node]] = None,
        data: Optional[Any] = None,
    ):
        self._name: str = f"{self.__class__.__name__}#{self.id}"
        self.name = self._name if name is None else name
        self._parent: Optional[Node] = None
        self.parent = parent
        self._children: List[Node] = []
        self.children = self._children if children is None else children
        self._data: Optional[Any] = data

    @property
    def name(self) -> str:
        """
        Getter and setter property for the name.

        Parameters
        ----------
        value
            Value to set the name with.

        Returns
        -------
        :class:`str`
            Node name.
        """

        return self._name

    @name.setter
    def name(self, value: str):
        """Setter for the **self.name** property."""

        attest(
            isinstance(value, str),
            f'"name" property: "{value}" type is not "str"!',
        )

        self._name = value

    @property
    def parent(self) -> Optional[Node]:
        """
        Getter and setter property for the node parent.

        Parameters
        ----------
        value
            Parent to set the node with.

        Returns
        -------
        :class:`Node` or :py:data:`None`
            Node parent.
        """

        return self._parent

    @parent.setter
    def parent(self, value: Optional[Node]):
        """Setter for the **self.parent** property."""

        if value is not None:
            attest(
                issubclass(value.__class__, Node),
                f'"parent" property: "{value}" is not a '
                f'"{Node.__class__.__name__}" subclass!',
            )

            value.children.append(self)

        self._parent = value

    @property
    def children(self) -> List[Node]:
        """
        Getter and setter property for the node children.

        Parameters
        ----------
        value
            Children to set the node with.

        Returns
        -------
        :class:`list`
            Node children.
        """

        return self._children

    @children.setter
    def children(self, value: List[Node]):
        """Setter for the **self.children** property."""

        attest(
            isinstance(value, list),
            f'"children" property: "{value}" type is not a "list" instance!',
        )

        for element in value:
            attest(
                issubclass(element.__class__, Node),
                f'"children" property: A "{element}" element is not a '
                f'"{Node.__class__.__name__}" subclass!',
            )

        for node in value:
            node.parent = self

        self._children = value

    @property
    def id(self) -> Integer:
        """
        Getter property for the node id.

        Returns
        -------
        :class:`numpy.integer`
            Node id.
        """

        return self._id  # type: ignore[attr-defined]

    @property
    def root(self) -> Node:
        """
        Getter property for the node tree.

        Returns
        -------
        :class:`Node`
            Node root.
        """

        if self.is_root():
            return self
        else:
            return list(self.walk(ascendants=True))[-1]

    @property
    def leaves(self) -> Generator:
        """
        Getter property for the node leaves.

        Yields
        ------
        Generator
            Node leaves.
        """

        if self.is_leaf():
            return (node for node in (self,))
        else:
            return (node for node in self.walk() if node.is_leaf())

    @property
    def siblings(self) -> Generator:
        """
        Getter property for the node siblings.

        Returns
        -------
        Generator
            Node siblings.
        """

        if self.parent is None:
            return (sibling for sibling in ())  # type: ignore[var-annotated]
        else:
            return (
                sibling
                for sibling in self.parent.children
                if sibling is not self
            )

    @property
    def data(self) -> Any:
        """
        Getter property for the node data.

        Returns
        -------
        :class:`object`
            Node data.
        """

        return self._data

    @data.setter
    def data(self, value: Any):
        """Setter for the **self.data** property."""

        self._data = value

    def __str__(self) -> str:
        """
        Return a formatted string representation of the node.

        Returns
        -------
        :class`str`
            Formatted string representation.
        """

        return f"{self.__class__.__name__}#{self.id}({self._data})"

    def __len__(self) -> Integer:
        """
        Return the number of children of the node.

        Returns
        -------
        :class:`numpy.integer`
            Number of children of the node.
        """

        return len(list(self.walk()))

    def is_root(self) -> Boolean:
        """
        Return whether the node is a root node.

        Returns
        -------
        :class:`bool`
            Whether the node is a root node.

        Examples
        --------
        >>> node_a = Node('Node A')
        >>> node_b = Node('Node B', node_a)
        >>> node_c = Node('Node C', node_b)
        >>> node_a.is_root()
        True
        >>> node_b.is_root()
        False
        """

        return self.parent is None

    def is_inner(self) -> Boolean:
        """
        Return whether the node is an inner node.

        Returns
        -------
        :class:`bool`
            Whether the node is an inner node.

        Examples
        --------
        >>> node_a = Node('Node A')
        >>> node_b = Node('Node B', node_a)
        >>> node_c = Node('Node C', node_b)
        >>> node_a.is_inner()
        False
        >>> node_b.is_inner()
        True
        """

        return all([not self.is_root(), not self.is_leaf()])

    def is_leaf(self) -> Boolean:
        """
        Return whether the node is a leaf node.

        Returns
        -------
        :class:`bool`
            Whether the node is a leaf node.

        Examples
        --------
        >>> node_a = Node('Node A')
        >>> node_b = Node('Node B', node_a)
        >>> node_c = Node('Node C', node_b)
        >>> node_a.is_leaf()
        False
        >>> node_c.is_leaf()
        True
        """

        return len(self._children) == 0

    def walk(self, ascendants: Boolean = False) -> Generator:
        """
        Return a generator used to walk into :class:`colour.utilities.Node`
        trees.

        Parameters
        ----------
        ascendants
            Whether to walk up the node tree.

        Yields
        ------
        Generator
            Node tree walker.

        Examples
        --------
        >>> node_a = Node('Node A')
        >>> node_b = Node('Node B', node_a)
        >>> node_c = Node('Node C', node_a)
        >>> node_d = Node('Node D', node_b)
        >>> node_e = Node('Node E', node_b)
        >>> node_f = Node('Node F', node_d)
        >>> node_g = Node('Node G', node_f)
        >>> node_h = Node('Node H', node_g)
        >>> for node in node_a.walk():
        ...     print(node.name)
        Node B
        Node D
        Node F
        Node G
        Node H
        Node E
        Node C
        """

        attribute = "children" if not ascendants else "parent"

        nodes = getattr(self, attribute)
        nodes = nodes if isinstance(nodes, list) else [nodes]

        for node in nodes:
            yield node

            if not getattr(node, attribute):
                continue

            yield from node.walk(ascendants=ascendants)

    def render(self, tab_level: Integer = 0):
        """
        Render the current node and its children as a string.

        Parameters
        ----------
        tab_level
            Initial indentation level

        Returns
        -------
        :class:`str`
            Rendered node tree.

        Examples
        --------
        >>> node_a = Node('Node A')
        >>> node_b = Node('Node B', node_a)
        >>> node_c = Node('Node C', node_a)
        >>> print(node_a.render())
        |----"Node A"
            |----"Node B"
            |----"Node C"
        <BLANKLINE>
        """

        output = ""

        for _i in range(tab_level):
            output += "    "

        tab_level += 1

        output += f'|----"{self.name}"\n'

        for child in self._children:
            output += child.render(tab_level)

        tab_level -= 1

        return output


import tensorflow as tf
from io import BytesIO
from PIL import Image
import os
import io
import PIL
import hashlib
import numpy as np
import matplotlib.pyplot as plt
import cv2

def int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def int64_list_feature( value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def bytes_list_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))

def float_list_feature( value):
    return tf.train.Feature(float_list=tf.train.FloatList(value=value))

def read_examples_list(path):
    """Read list of training or validation examples.

    The file is assumed to contain a single example per line where the first
    token in the line is an identifier that allows us to find the image and
    annotation xml for that example.

    For example, the line:
    xyz 3
    would allow us to find files xyz.jpg and xyz.xml (the 3 would be ignored).

    Args:
    path: absolute path to examples list file.

    Returns:
    list of example identifiers (strings).
    """
    with tf.io.gfile.GFile(path) as fid:
        lines = fid.readlines()
    return [line.strip().split(' ')[0] for line in lines]

def recursive_parse_xml_to_dict(xml):
    """
    Recursively parses XML contents to python dict.
    We assume that `object` tags are the only ones that can appear
    multiple times at the same level of a tree.
    Args:
    xml: xml tree obtained by parsing XML file contents using lxml.etree
    Returns:
    Python dictionary holding XML contents.
    """
    if not xml:
        return {xml.tag: xml.text}
    result = {}
    for child in xml:
        child_result = recursive_parse_xml_to_dict(child)
        if child.tag != 'object':
            result[child.tag] = child_result[child.tag]
        else:
            if child.tag not in result:
                result[child.tag] = []
            result[child.tag].append(child_result[child.tag])
    return {xml.tag: result}

def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

class TFConverter(object):
    def __init__(self, object_list, annotation_path, image_path):
        self.object_list = object_list  # 0 is background
        self.annotation_path = annotation_path
        self.image_path = image_path
        self.annotations = [f for f in os.listdir(annotation_path) if f.endswith(".npy")]
        self.images = [i for i in os.listdir(image_path) if (i.endswith("png") or i.endswith("jpg"))]

    def convert(self, gray_scale=False, output_path="."):
        writer = tf.io.TFRecordWriter(output_path + "/tf.record")
        for image in self.images:
            annotation = image.split(".")[0] + ".npy"
            if annotation not in self.annotations:
                print("No annotation file: Ignore "+image)
                continue
            else:
                annotation_file = os.path.join(self.annotation_path, annotation)
                image_file = os.path.join(self.image_path, image)
                masks = np.load(annotation_file)
                if masks.max()  < 1:
                    continue
                else:
                    tf_example = self._getTFExample(annotation_file, image_file, gray_scale)
                    writer.write(tf_example.SerializeToString())
        writer.close()

    def _getTFExample(self, annotation_file_path, image_file_path, gray_scale):
        if not gray_scale:
            fid = tf.io.gfile.GFile(image_file_path, 'rb')  # tensorflow 2.0
            encoded_jpg = fid.read()
            encoded_jpg_io = io.BytesIO(encoded_jpg)
            image = PIL.Image.open(encoded_jpg_io)
        else:
            gray_image = cv2.imread(image_file_path, cv2.IMREAD_GRAYSCALE)
            image = Image.fromarray(gray_image)
            encoded_jpg_io = BytesIO()
            image.save(encoded_jpg_io, 'JPEG')
            encoded_jpg = encoded_jpg_io.getvalue()
            encoded_jpg_io = io.BytesIO(encoded_jpg)
            image = PIL.Image.open(encoded_jpg_io)

        if image.format != 'JPEG':
            raise ValueError('Image format not JPEG')
        key = hashlib.sha256(encoded_jpg).hexdigest()

        width = image.width
        height = image.height
        masks = np.load(annotation_file_path)
        masks = np.swapaxes(masks, axis1=0, axis2=1)
        masks = np.swapaxes(masks, axis1=1, axis2=2)
        masks = (masks > 0.5) * 255
        if len(masks.shape) == 2:
            masks = np.expand_dims(masks, 2)
        obj_nm = masks.shape[-1]
        masks = (masks/255.0).astype(np.uint8)

        xmin = []
        ymin = []
        xmax = []
        ymax = []
        classes = []
        classes_text = []
        masks_PNG = []
        masks_list = []

        for i in range(obj_nm):
            mask = masks[:, :, i]
            if mask.max() < 1:
                continue
            masks_list.append(mask.tostring())
            bbox = self._getBBox(mask)
            xmin.append(float(bbox[0]) / width)
            ymin.append(float(bbox[2]) / height)
            xmax.append(float(bbox[1]) / width)
            ymax.append(float(bbox[3]) / height)
            id = int(mask.max())
            classes.append(id)
            classes_text.append(self.object_list[id-1])
            masks_PNG.append(self._image2PNGString(mask))

        example = tf.train.Example(features=tf.train.Features(feature={
            'image/height': int64_feature(height),
            'image/width': int64_feature(width),
            'image/filename': bytes_feature(image_file_path.encode('utf8')),  #
            'image/source_id': bytes_feature(image_file_path.encode('utf8')),  #
            'image/encoded': bytes_feature(encoded_jpg),
            'image/format': bytes_feature('jpeg'.encode('utf8')),
            'image/object/bbox/xmin': float_list_feature(xmin),
            'image/object/bbox/xmax': float_list_feature(xmax),
            'image/object/bbox/ymin': float_list_feature(ymin),
            'image/object/bbox/ymax': float_list_feature(ymax),
            'image/object/class/text': bytes_list_feature(classes_text),
            'image/object/class/label': int64_list_feature(classes),
            'image/object/mask': bytes_list_feature(masks_PNG)
            }))
        return example

    def _image2PNGString(self, image):
        image = Image.fromarray(image)
        byte_io = BytesIO()
        image.save(byte_io, 'PNG')
        return byte_io.getvalue()

    def _getBBox(self, mask):
        rows = np.any(mask, axis=1)
        cols = np.any(mask, axis=0)
        rmin, rmax = np.where(rows)[0][[0, -1]]
        cmin, cmax = np.where(cols)[0][[0, -1]]

        #xmin, xmax, ymin, ymax = rmin, rmax, cmin, cmax  # the bounding box is transposed
        ymin, ymax, xmin, xmax = rmin, rmax, cmin, cmax

        return xmin, xmax, ymin, ymax

if __name__ == "__main__":
    tfrecorder = TFConverter(['crater'], "./npy", "./moon_images")
    tfrecorder.convert(gray_scale=True)