text:
import re
import io
import demjson
import requests
import numpy as np
import pandas as pd
from fake_useragent import UserAgent

# TODO need add comments

url = {
    "eastmoney": "http://datainterface.eastmoney.com/EM_DataCenter/JS.aspx",
    "fred_econ": "https://fred.stlouisfed.org/graph/fredgraph.csv?",
    "OECD": "https://stats.oecd.org/sdmx-json/data/DP_LIVE/"
}
# https://fred.stlouisfed.org/release/tables?rid=205&eid=712378
def gdp_quarterly():
    """
    ABS: absolute value (per 100 million CNY)
    YoY: year on year growth
    Data source: http://data.eastmoney.com/cjsj/gdp.html
    """
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable7519513",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "20",
        "_": "1622020352668"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Absolute_Value",
        "YoY_Rate",
        "Primary_Industry_ABS",
        "Primary_Industry_YoY_Rate",
        "Secondary_Industry_ABS",
        "Secondary_Industry_YoY_Rate",
        "Tertiary_Industry_ABS",
        "Tertiary_Industry_YoY_Rate",
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df["Absolute_Value"] = df["Absolute_Value"].astype(float)
    df["Secondary_Industry_ABS"] = df["Secondary_Industry_ABS"].astype(float)
    df["Tertiary_Industry_ABS"] = df["Tertiary_Industry_ABS"].astype(float)
    df["Absolute_Value"] = df["Absolute_Value"].astype(float)
    df["YoY_Rate"] = df["YoY_Rate"].astype(float) / 100
    df["Secondary_Industry_YoY_Rate"] = df["Secondary_Industry_YoY_Rate"].astype(
        float) / 100
    df["Tertiary_Industry_YoY_Rate"] = df["Tertiary_Industry_YoY_Rate"].astype(
        float) / 100
    return df

def ppi_monthly():
    """
    ABS: absolute value (per 100 million CNY)
    YoY: year on year growth
    Accum: Accumulation
    Data source: http://data.eastmoney.com/cjsj/ppi.html
    """
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable9051497",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "22",
        "_": "1622047940401"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "Current_Month_YoY_Rate",
        "Current_Month_Accum"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df["Current_Month"] = df["Current_Month"].astype(float)
    df["Current_Month_YoY_Rate"] = df["Current_Month_YoY_Rate"].astype(
        float) / 100
    df["Current_Month_Accum"] = df["Current_Month_Accum"].astype(float)
    return df

def cpi_monthly():
    """
    Accum: Accumulation
    YoY: year on year growth
    MoM: month on month growth
    Data source: http://data.eastmoney.com/cjsj/cpi.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable2790750",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "19",
        "_": "1622020352668"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Notion_Monthly",
        "Notion_YoY_Rate",
        "Notion_MoM_Rate",
        "Notion_Accum",
        "Urban_Monthly",
        "Urban_YoY_Rate",
        "Urban_MoM_Rate",
        "Urban_Accum",
        "Rural_Monthly",
        "Rural_YoY_Rate",
        "Rural_MoM_Rate",
        "Rural_Accum",
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Notion_Monthly",
        "Notion_Accum",
        "Urban_Monthly",
        "Urban_Accum",
        "Rural_Monthly",
        "Rural_Accum"]] = df[["Notion_Monthly",
                              "Notion_Accum",
                              "Urban_Monthly",
                              "Urban_Accum",
                              "Rural_Monthly",
                              "Rural_Accum"]].astype(float)
    df[["Notion_YoY_Rate",
        "Notion_MoM_Rate",
        "Urban_YoY_Rate",
        "Urban_MoM_Rate",
        "Rural_YoY_Rate",
        "Rural_MoM_Rate"]] = df[["Notion_YoY_Rate",
                                 "Notion_MoM_Rate",
                                 "Urban_YoY_Rate",
                                 "Urban_MoM_Rate",
                                 "Rural_YoY_Rate",
                                 "Rural_MoM_Rate"]].astype(float) / 100
    return df

def pmi_monthly():
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/pmi.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable4515395",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "2",
        "ps": "200",
        "mkt": "21",
        "_": "162202151821"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Man_Industry_Index",
        "Man_Index_YoY_Rate",
        "Non-Man_Industry_Index",
        "Non-Man_Index_YoY_Rate",
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Man_Industry_Index", "Non-Man_Industry_Index"]] = \
        df[["Man_Industry_Index", "Non-Man_Industry_Index"]].astype(float)
    df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]] = \
        df[["Man_Index_YoY_Rate", "Non-Man_Index_YoY_Rate"]].astype(float) / 100
    return df

def fai_monthly():  # fix asset investment
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/gdzctz.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable607120",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "12",
        "_": "1622021790947"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "YoY_Rate",
        "MoM_Rate",
        "Current_Year_Accum"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Current_Month", "Current_Year_Accum"]] = \
        df[["Current_Month", "Current_Year_Accum"]].astype(float)
    df[["YoY_Rate", "MoM_Rate"]] = \
        df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100
    return df

def hi_old_monthly():  # house index old version (2008-2010)
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/house.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable1895714",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "10",
        "_": "1622022794457"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Housing_Prosperity_Index",
        "HPI_YoY_Rate",
        "Land_Development_Area_Index",
        "LDAI_YoY_Rate",
        "Sales_Price_Index",
        "SPI_YoY_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Housing_Prosperity_Index",
        "Land_Development_Area_Index",
        "Sales_Price_Index"]] = df[["Housing_Prosperity_Index",
                                    "Land_Development_Area_Index",
                                    "Sales_Price_Index"]].astype(float)
    df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]] = \
        df[["HPI_YoY_Rate", "LDAI_YoY_Rate", "SPI_YoY_Rate"]].astype(float) / 100
    return df

# mkt=1&stat=2&city1=%E5%B9%BF%E5%B7%9E&city2=%E4%B8%8A%E6%B5%B7

# newly built commercial housing &  second-hand commercial housing
def hi_new_monthly(city1: str, city2: str):
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/newhouse.html
    """
    tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getnewhousechartdata?"
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    request_params_nbch_MoM = {
        "mkt": "1",
        "stat": "2",
        "city1": "{}".format(city1),
        "city2": "{}".format(city2)
    }
    request_params_shch_MoM = {
        "mkt": "1",
        "stat": "3",
        "city1": "{}".format(city1),
        "city2": "{}".format(city2)
    }
    r_nbch_MoM = requests.get(
        tmp_url,
        params=request_params_nbch_MoM,
        headers=request_header)
    r_shch_MoM = requests.get(
        tmp_url,
        params=request_params_shch_MoM,
        headers=request_header)
    data_text_nbch_MoM = r_nbch_MoM.text
    data_text_shch_MoM = r_shch_MoM.text
    data_json_nbch_MoM = demjson.decode(data_text_nbch_MoM)
    data_json_shch_MoM = demjson.decode(data_text_shch_MoM)
    date_nbch = data_json_nbch_MoM['chart']['series']['value']
    data1_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][0]['value']
    data2_nbch_MoM = data_json_nbch_MoM['chart']['graphs']['graph'][1]['value']
    data1_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][0]['value']
    data2_shch_MoM = data_json_shch_MoM['chart']['graphs']['graph'][1]['value']
    df_MoM = pd.DataFrame({"Date": date_nbch,
                           "City1_nbch_MoM": data1_nbch_MoM,
                           "City1_shch_MoM": data1_shch_MoM,
                           "City2_nbch_MoM": data2_nbch_MoM,
                           "City2_shch_MoM": data2_shch_MoM})
    df_MoM["Date"] = pd.to_datetime(df_MoM["Date"], format="%m/%d/%Y")

    request_params_nbch_YoY = {
        "mkt": "2",
        "stat": "2",
        "city1": "{}".format(city1),
        "city2": "{}".format(city2)
    }
    request_params_shch_YoY = {
        "mkt": "2",
        "stat": "3",
        "city1": "{}".format(city1),
        "city2": "{}".format(city2)
    }
    r_nbch_YoY = requests.get(
        tmp_url,
        params=request_params_nbch_YoY,
        headers=request_header)
    r_shch_YoY = requests.get(
        tmp_url,
        params=request_params_shch_YoY,
        headers=request_header)
    data_text_nbch_YoY = r_nbch_YoY.text
    data_text_shch_YoY = r_shch_YoY.text
    data_json_nbch_YoY = demjson.decode(data_text_nbch_YoY)
    data_json_shch_YoY = demjson.decode(data_text_shch_YoY)
    date_nbch = data_json_nbch_YoY['chart']['series']['value']
    data1_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][0]['value']
    data2_nbch_YoY = data_json_nbch_YoY['chart']['graphs']['graph'][1]['value']
    data1_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][0]['value']
    data2_shch_YoY = data_json_shch_YoY['chart']['graphs']['graph'][1]['value']
    df_YoY = pd.DataFrame({"Date": date_nbch,
                           "City1_nbch_YoY": data1_nbch_YoY,
                           "City1_shch_YoY": data1_shch_YoY,
                           "City2_nbch_YoY": data2_nbch_YoY,
                           "City2_shch_YoY": data2_shch_YoY})
    df_YoY["Date"] = pd.to_datetime(df_YoY["Date"], format="%m/%d/%Y")
    df = df_YoY.merge(df_MoM, on="Date")
    return df

def ci_eei_monthly():  # Climate Index & Entrepreneur Expectation Index
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/qyjqzs.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable7709842",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "8",
        "_": "1622041485306"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Climate_Index",
        "CI_YoY_Rate",
        "CI_MoM_Rate",
        "Entrepreneur_Expectation_Index",
        "EEI_YoY_Rate",
        "EEI_MoM_Rate"
    ]
    df.replace('', np.nan, inplace=True)
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Climate_Index", "Entrepreneur_Expectation_Index"]] = \
        df[["Climate_Index", "Entrepreneur_Expectation_Index"]].astype(float)
    df[["CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]] = df[[
        "CI_YoY_Rate", "CI_MoM_Rate", "EEI_YoY_Rate", "EEI_MoM_Rate"]].astype(float) / 100
    return df

def ig_monthly():  # Industry Growth
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/gyzjz.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable4577327",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "0",
        "_": "1622042259898"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "IG_YoY_Rate",
        "IG_Accum_Rate",
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["IG_YoY_Rate", "IG_Accum_Rate"]] = \
        df[["IG_YoY_Rate", "IG_Accum_Rate"]].astype(float) / 100
    return df

def cgpi_monthly():  # Corporate Goods Price Index
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/qyspjg.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable7184534",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "9",
        "_": "1622042652353"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "General_Index",
        "General_Index_YoY_Rate",
        "Total_Index_MoM_Rate",
        "Agricultural_Product",
        "Agricultural_Product_YoY_Rate",
        "Agricultural_Product_MoM_Rate",
        "Mineral_Product",
        "Mineral_Product_YoY_Rate",
        "Mineral_Product_MoM_Rate",
        "Coal_Oil_Electricity",
        "Coal_Oil_Electricity_YoY_Rate",
        "Coal_Oil_Electricity_MoM_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["General_Index",
        "Agricultural_Product",
        "Mineral_Product",
        "Coal_Oil_Electricity"]] = df[["General_Index",
                                       "Agricultural_Product",
                                       "Mineral_Product",
                                       "Coal_Oil_Electricity"]].astype(float)
    df[["General_Index_YoY_Rate",
        "Total_Index_MoM_Rate",
        "Agricultural_Product_YoY_Rate",
        "Agricultural_Product_MoM_Rate",
        "Coal_Oil_Electricity_YoY_Rate",
        "Coal_Oil_Electricity_MoM_Rate"]] = df[["General_Index_YoY_Rate",
                                                "Total_Index_MoM_Rate",
                                                "Agricultural_Product_YoY_Rate",
                                                "Agricultural_Product_MoM_Rate",
                                                "Coal_Oil_Electricity_YoY_Rate",
                                                "Coal_Oil_Electricity_MoM_Rate"]].astype(float) / 100
    return df

def cci_csi_cei_monthly():  # Consumer Confidence Index & Consumer Satisfaction Index & Consumer Expectation Index
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/xfzxx.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable1243218",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "4",
        "_": "1622043704818"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "CCI",
        "CCI_YoY_Rate",
        "CCI_MoM_Rate",
        "CSI",
        "CSI_YoY_Rate",
        "CSI_MoM_Rate",
        "CEI",
        "CEI_YoY_Rate",
        "CEI_MoM_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["CCI", "CSI", "CEI"]] = \
        df[["CCI", "CSI", "CEI"]].astype(float)
    df[["CCI_YoY_Rate", "CCI_MoM_Rate",
        "CSI_YoY_Rate", "CSI_MoM_Rate",
        "CEI_YoY_Rate", "CEI_MoM_Rate"]] = \
        df[["CCI_YoY_Rate", "CCI_MoM_Rate",
            "CSI_YoY_Rate", "CSI_MoM_Rate",
            "CEI_YoY_Rate", "CEI_MoM_Rate"]].astype(float) / 100
    return df

def trscg_monthly():  # Total Retail Sales of Consumer Goods
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/xfp.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable3665821",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "5",
        "_": "1622044011316"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "TRSCG_YoY_Rate",
        "TRSCG_MoM_Rate",
        "TRSCG_Accum",
        "TRSCG_Accum_YoY_Rate"
    ]
    df.replace("", np.nan, inplace=True)
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Current_Month", "TRSCG_Accum"]] = \
        df[["Current_Month", "TRSCG_Accum"]].astype(float)
    df[["TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]] = df[[
        "TRSCG_YoY_Rate", "TRSCG_MoM_Rate", "TRSCG_Accum_YoY_Rate"]].astype(float) / 100
    return df

def ms_monthly():  # monetary Supply
    """
    Man: manufacturing
    Non-Man: Non-manufacturing
    Data Source: http://data.eastmoney.com/cjsj/hbgyl.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable3818891",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "11",
        "_": "1622044292103"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "M2",
        "M2_YoY_Rate",
        "M2_MoM_Rate",
        "M1",
        "M1_YoY_Rate",
        "M1_MoM_Rate",
        "M0",
        "M0_YoY_Rate",
        "M0_MoM_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["M0", "M1", "M2"]] = \
        df[["M0", "M1", "M2"]].astype(float)
    df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate",
        "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]] = \
        df[["M0_YoY_Rate", "M1_YoY_Rate", "M2_YoY_Rate",
            "M0_MoM_Rate", "M1_MoM_Rate", "M2_MoM_Rate"]].astype(float) / 100
    return df

def ie_monthly():  # Import & Export
    """
    Data Source: http://data.eastmoney.com/cjsj/hgjck.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable3818891",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "1",
        "_": "1622044292103"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month_Export",
        "Current_Month_Export_YoY_Rate",
        "Current_Month_Export_MoM_Rate",
        "Current_Month_Import",
        "Current_Month_Import_YoY_Rate",
        "Current_Month_Import_MoM_Rate",
        "Accumulation_Export",
        "Accumulation_Export_YoY_Rate",
        "Accumulation_Import",
        "Accumulation_Import_YoY_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df.replace("", np.nan, inplace=True)
    df[["Current_Month_Export", "Current_Month_Import",
        "Accumulation_Export", "Accumulation_Import"]] = \
        df[["Current_Month_Export", "Current_Month_Import",
            "Accumulation_Export", "Accumulation_Import"]].astype(float)
    df[["Current_Month_Export_YoY_Rate",
        "Current_Month_Export_MoM_Rate",
        "Current_Month_Import_YoY_Rate",
        "Current_Month_Import_MoM_Rate",
        "Accumulation_Export_YoY_Rate",
        "Accumulation_Import_YoY_Rate"]] = \
        df[["Current_Month_Export_YoY_Rate",
            "Current_Month_Export_MoM_Rate",
            "Current_Month_Import_YoY_Rate",
            "Current_Month_Import_MoM_Rate",
            "Accumulation_Export_YoY_Rate",
            "Accumulation_Import_YoY_Rate"]].astype(float)/100
    return df

def stock_monthly():  # Stock Trading Statistics
    """
    Data Source: http://data.eastmoney.com/cjsj/gpjytj.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "jQuery112308659690274138041_1622084599455",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "2",
        "_": "1622084599456"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("(") + 1:-1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "SH_Total_Stock_issue",
        "SZ_Total_Stock_Issue",
        "SH_Total_Market_Capitalization",
        "SZ_Total_Market_Capitalization",
        "SH_Turnover",
        "SZ_Turnover",
        "SH_Volume",
        "SZ_Volume",
        "SH_Highest",
        "SZ_Highest",
        "SH_lowest",
        "SZ_lowest"
    ]
    df.replace("", np.nan, inplace=True)
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[list(df.columns[1:])] = df[list(df.columns[1:])].astype(float)
    return df

def fgr_monthly():  # Forex and Gold Reserve
    """
    Data Source: http://data.eastmoney.com/cjsj/gpjytj.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "tatable6260802",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "16",
        "_": "1622044863548"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Forex",
        "Forex_YoY_Rate",
        "Forex_MoM_Rate",
        "Gold",
        "Gold_YoY_Rate",
        "Gold_MoM_Rate"
    ]
    df.replace("", np.nan, inplace=True)
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Forex", "Gold"]] = \
        df[["Forex", "Gold"]].astype(float)
    df[["Forex_YoY_Rate", "Gold_YoY_Rate",
        "Forex_MoM_Rate", "Gold_MoM_Rate"]] = \
        df[["Forex_YoY_Rate", "Gold_YoY_Rate",
           "Forex_MoM_Rate", "Gold_MoM_Rate"]].astype(float) / 100
    return df

def ctsf_monthly():  # Client Transaction Settlement Funds
    """
    http://data.eastmoney.com/cjsj/banktransfer.html
    """
    tmp_url = "http://data.eastmoney.com/dataapi/cjsj/getbanktransferdata?"
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    request_params = {
        "p": "1",
        "ps": "200"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("["):-11])
    df = pd.DataFrame(data_json)
    df.replace("", np.nan, inplace=True)
    df["StartDate"] = pd.to_datetime(df["StartDate"], format="%Y-%m-%d")
    df["EndDate"] = pd.to_datetime(df["EndDate"], format="%Y-%m-%d")
    df[list(df.columns)[2:]] = df[list(df.columns)[2:]].astype(float)
    return df

# TODO: SPECIAL CASE

def sao_monthly():  # Stock Account Overview
    """
    http://data.eastmoney.com/cjsj/gpkhsj.html
    """
    tmp_url = "http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get?"
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    request_params = {
        "callback": "datatable4006236",
        "type": "GPKHData",
        "js": "({data:[(x)],pages:(pc)})",
        "st": "SDATE",
        "sr": "-1",
        "token": "894050c76af8597a853f5b408b759f5d",
        "p": "1",
        "ps": "2000",
        "_": "1622079339035"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{") + 6: -14])
    df = pd.DataFrame(data_json[0])
    df.columns = [
        "Date",
        "New_Investor",
        "New_Investor_MoM_Rate",
        "New_Investor_YoY_Rate",
        "Active_Investor",
        "Active_Investor_A_Share",
        "Active_Investor_B_share",
        "SHIndex_Close",
        "SHIndex_Rate",
        "SHSZ_Market_Capitalization",
        "SHSZ_Average_Capitalization"
    ]
    df.replace("-", np.nan, inplace=True)
    df.Date = pd.to_datetime(df.Date, format="%Y年%m月")
    df[list(df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])] = df[list(
        df.columns[~df.columns.isin(["Date", "New_Investor_MoM_Rate", "New_Investor_YoY_Rate"])])].astype(float)
    df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]] = \
        df[["New_Investor_MoM_Rate", "New_Investor_YoY_Rate"]].astype(float) / 100
    return df

def fdi_monthly():  # Foreign Direct Investment
    """
    http://data.eastmoney.com/cjsj/fdi.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable1477466",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "15",
        "_": "1622044863548"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "YoY_Rate",
        "MoM_Rate",
        "Accumulation",
        "Accum_YoY_Rate"
    ]
    df.replace("", np.nan, inplace=True)
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Current_Month", "Accumulation"]] = \
        df[["Current_Month", "Accumulation"]].astype(float)
    df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] = \
        df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100
    return df

def gr_monthly():  # Government Revenue
    """
    http://data.eastmoney.com/cjsj/czsr.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable7840652",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "14",
        "_": "1622080618625"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "YoY_Rate",
        "MoM_Rate",
        "Accumulation",
        "Accum_YoY_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Current_Month", "Accumulation"]] = \
        df[["Current_Month", "Accumulation"]].astype(float)
    df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] = \
        df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100
    return df

def ti_monthly():  # Tax Income
    """
    http://data.eastmoney.com/cjsj/qgsssr.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable8280567",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "3",
        "_": "1622080669713"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "YoY_Rate",
        "MoM_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df = df.replace("", np.nan)
    df[["Current_Month"]] = \
        df[["Current_Month"]].astype(float)
    df[["YoY_Rate", "MoM_Rate"]] = \
        df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100
    return df

def nl_monthly():  # New Loan
    """
    http://data.eastmoney.com/cjsj/xzxd.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable2533707",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "7",
        "_": "1622080800162"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "YoY_Rate",
        "MoM_Rate",
        "Accumulation",
        "Accum_YoY_Rate"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Current_Month", "Accumulation"]] = \
        df[["Current_Month", "Accumulation"]].astype(float)
    df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]] =\
        df[["YoY_Rate", "MoM_Rate", "Accum_YoY_Rate"]].astype(float) / 100
    return df

def dfclc_monthly():  # Deposit of Foreign Currency and Local Currency
    """
    http://data.eastmoney.com/cjsj/wbck.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable2899877",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "18",
        "_": "1622081057370"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "YoY_Rate",
        "MoM_Rate",
        "Accumulation"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Current_Month", "Accumulation"]] = \
        df[["Current_Month", "Accumulation"]].astype(float)
    df[["YoY_Rate", "MoM_Rate"]] = \
        df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100
    return df

def fl_monthly():  # Forex Loan
    """
    http://data.eastmoney.com/cjsj/whxd.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable636844",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "17",
        "_": "1622081336038"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Date",
        "Current_Month",
        "YoY_Rate",
        "MoM_Rate",
        "Accumulation"
    ]
    df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    df[["Current_Month", "Accumulation"]] = \
        df[["Current_Month", "Accumulation"]].astype(float)
    df[["YoY_Rate", "MoM_Rate"]] = \
        df[["YoY_Rate", "MoM_Rate"]].astype(float) / 100
    return df

def drr_monthly():  # Deposit Reserve Ratio
    """
    http://data.eastmoney.com/cjsj/ckzbj.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable4285562",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "23",
        "_": "1622081448882"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Announcement Date",
        "Effective Date",
        "Large_Financial_institution_Before",
        "Large_Financial_institution_After",
        "Large_Financial_institution_Adj_Rate",
        "S&M_Financial_institution_Before",
        "S&M_Financial_institution_After",
        "S&M_Financial_institution_Adj_Rate",
        "Comment",
        "SHIndex_Rate",
        "SZIndex_Rate"
    ]
    df["Announcement Date"] = pd.to_datetime(
        df["Announcement Date"], format="%Y-%m-%d")
    df["Date"] = df["Announcement Date"]
    df["Effective Date"] = pd.to_datetime(
        df["Effective Date"], format="%Y-%m-%d")
    df[["SHIndex_Rate", "SZIndex_Rate"]] = df[["SHIndex_Rate", "SZIndex_Rate"]].astype(float)
    df[["Large_Financial_institution_Before",
        "Large_Financial_institution_After",
        "Large_Financial_institution_Adj_Rate",
        "S&M_Financial_institution_Before",
        "S&M_Financial_institution_After",
        "S&M_Financial_institution_Adj_Rate"]] = df[["Large_Financial_institution_Before",
                                                     "Large_Financial_institution_After",
                                                     "Large_Financial_institution_Adj_Rate",
                                                     "S&M_Financial_institution_Before",
                                                     "S&M_Financial_institution_After",
                                                     "S&M_Financial_institution_Adj_Rate"]].astype(float) / 100
    return df

def interest_monthly():  # Interest
    """
    http://data.eastmoney.com/cjsj/yhll.html
    """
    tmp_url = url["eastmoney"]
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    tmp_url = url["eastmoney"]
    request_params = {
        "cb": "datatable7591685",
        "type": "GJZB",
        "sty": "ZGZB",
        "js": "({data:[(x)],pages:(pc)})",
        "p": "1",
        "ps": "200",
        "mkt": "13",
        "_": "1622081956464"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -1])
    df = pd.DataFrame([item.split(",") for item in data_json["data"]])
    df.columns = [
        "Announcement Date",
        "Deposit_Benchmark_Interest_Rate_Before",
        "Deposit_Benchmark_Interest_Rate_After",
        "Deposit_Benchmark_Interest_Rate_Adj_Rate",
        "Loan_Benchmark_Interest_Rate_Before",
        "Loan_Benchmark_Interest_Rate_After",
        "Loan_Benchmark_Interest_Rate_Adj_Rate",
        "SHIndex_Rate",
        "SZIndex_Rate",
        "Effective Date"
    ]
    df["Announcement Date"] = pd.to_datetime(
        df["Announcement Date"], format="%Y-%m-%d")
    df["Date"] = df["Announcement Date"]
    df["Effective Date"] = pd.to_datetime(
        df["Effective Date"], format="%Y-%m-%d")
    df["Date"] = df["Announcement Date"]
    df = df.replace("", np.nan)
    df[["Deposit_Benchmark_Interest_Rate_Before",
        "Deposit_Benchmark_Interest_Rate_After",
        "Deposit_Benchmark_Interest_Rate_Adj_Rate",
        "Loan_Benchmark_Interest_Rate_Before",
        "Loan_Benchmark_Interest_Rate_After",
        "Loan_Benchmark_Interest_Rate_Adj_Rate"]] = df[["Deposit_Benchmark_Interest_Rate_Before",
                                                        "Deposit_Benchmark_Interest_Rate_After",
                                                        "Deposit_Benchmark_Interest_Rate_Adj_Rate",
                                                        "Loan_Benchmark_Interest_Rate_Before",
                                                        "Loan_Benchmark_Interest_Rate_After",
                                                        "Loan_Benchmark_Interest_Rate_Adj_Rate"]].astype(float) / 100
    return df

def gdc_daily():  # gasoline, Diesel and Crude Oil
    """
    http://data.eastmoney.com/cjsj/oil_default.html
    """
    tmp_url = "http://datacenter-web.eastmoney.com/api/data/get?"
    ua = UserAgent(verify_ssl=False)
    request_header = {"User-Agent": ua.random}
    request_params = {
        "callback": "jQuery112302601302322321093_1622082348721",
        "type": "RPTA_WEB_JY_HQ",
        "sty": "ALL",
        "st": "date",
        "sr": "-1",
        "token": "894050c76af8597a853f5b408b759f5d",
        "p": "1",
        "ps": "50000",
        "source": "WEB",
        "_": "1622082348722"
    }
    r = requests.get(tmp_url, params=request_params, headers=request_header)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"): -2])
    df = pd.DataFrame(data_json["result"]["data"])
    df.columns = ["Crude_Oil", "Date", "Gasoline", "Diesel"]
    df = df[["Date", "Gasoline", "Diesel", "Crude_Oil"]]
    df = pd.to_datetime(df["Date"], format="%Y-%m-%d")
    return df

def Leading_Indicators_OECD(startdate = "1950-01", enddate = "2021-05"):
    # CLI
    tmp_url = url["OECD"] + "CHN.CLI.AMPLITUD.LTRENDIDX.M/OECD"
    ua = UserAgent(verify_ssl=False)
    request_params = {
        "contentType": "csv",
        "detail": "code",
        "separator": "comma",
        "csv-lang": "en",
        "startPeriod": "{}".format(startdate),
        "endPeriod": "{}".format(enddate)
    }
    request_header = {"User-Agent": ua.random}
    r = requests.get(tmp_url, params = request_params, headers=request_header)
    data_text = r.content
    df_cli = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]]
    df_cli.columns = ["Date", "CN_OECD_CLI"]
    df_cli["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m")
    df_cli["CN_OECD_CLI"] = df_cli["CN_OECD_CLI"].astype(float)
    #BCI
    tmp_url = url["OECD"] + "CHN.BCI.AMPLITUD.LTRENDIDX.M/OECD"
    ua = UserAgent(verify_ssl=False)
    request_params = {
        "contentType": "csv",
        "detail": "code",
        "separator": "comma",
        "csv-lang": "en",
        "startPeriod": "{}".format(startdate),
        "endPeriod": "{}".format(enddate)
    }
    request_header = {"User-Agent": ua.random}
    r = requests.get(tmp_url, params = request_params, headers=request_header)
    data_text = r.content
    df_bci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]]
    df_bci.columns = ["Date", "CN_OECD_BCI"]
    df_bci["Date"] = pd.to_datetime(df_cli["Date"], format = "%Y-%m")
    df_bci["CN_OECD_BCI"] = df_bci["CN_OECD_BCI"].astype(float)
    # CCI
    tmp_url = url["OECD"] + "CHN.CCI.AMPLITUD.LTRENDIDX.M/OECD"
    ua = UserAgent(verify_ssl=False)
    request_params = {
        "contentType": "csv",
        "detail": "code",
        "separator": "comma",
        "csv-lang": "en",
        "startPeriod": "{}".format(startdate),
        "endPeriod": "{}".format(enddate)
    }
    request_header = {"User-Agent": ua.random}
    r = requests.get(tmp_url, params = request_params, headers=request_header)
    data_text = r.content
    df_cci = pd.read_csv(io.StringIO(data_text.decode('utf-8')))[["TIME", "Value"]]
    df_cci.columns = ["Date", "CN_OECD_CCI"]
    df_cci["Date"] = pd.to_datetime(df_cci["Date"], format = "%Y-%m")
    df_cci["CN_OECD_CCI"] = df_cci["CN_OECD_CCI"].astype(float)
    df = pd.merge_asof(df_cli, df_bci, on = "Date")
    df = pd.merge_asof(df, df_cci, on = "Date")

    return df

def CN_EPU_Monthly():
    df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/cnepu_1_july_2021_updated.xlsx")[:-2]
    df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S")
    return df

def CN_Hongkong_EPU():
    df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/hkepu_1_july_2021_updated.xlsx")[:-2]
    df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S")
    return df

def CN_Macau_EPU():
    df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/mcepu_1_july_2021_updated.xlsx")[:-2]
    df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S")
    return df

def CN_EPU_Daily():
    df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/cnepu_daily_7_july_2021_updated.xlsx")[:-2]
    df = df[["Date", "CNEPU_Daily"]]
    df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S")
    return df

def Policy_Specific_EPU():
    df = pd.read_excel("https://economicpolicyuncertaintyinchina.weebly.com/uploads/1/2/2/7/122762465/cnepu_policy-specific_1_july_2021_updated.xlsx")[:-2]
    df.Date = pd.to_datetime(df.Date, format="%Y-%m-%d %H:%M:%S")
    return df

"""
if __name__ == "__main__":
"""


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# *****************************************************************************/
# * Authors: Yan Zhu, Chin-Chia Michael Yeh, Zachary Zimmerman, Kaveh Kamgar, Eamonn Keogh, Tyler Woods, Joseph Tarango
# *****************************************************************************/
from __future__ import absolute_import, division, print_function, \
    unicode_literals  # , nested_scopes, generators, generator_stop, with_statement, annotations

import numpy, numpy.fft, matplotlib.pyplot

def utils_ErrorNotImplemented(strError):
    print(strError)
    return

def utils_error(dataObject):
    print(dataObject)

def utils_inf():
    return float(numpy.inf)

def utils_nan():
    return float(numpy.nan)

def utils_sqrt(series, window=3):
    return numpy.sqrt(series)

def utils_mean(series):
    return numpy.mean(series)

def utils_std(series):
    return numpy.std(series)

def utils_cumsum(series):
    return numpy.sum(series)

def utils_isnan(keyValue):
    return numpy.isnan(keyValue)

def utils_isinf(keyValue):
    return numpy.isinf(keyValue)

def utils_fft(series):
    """
    Computes the inverse fast foyer transform.
    Parameters
    Input
        series : array_like
    Output
        The transformed array.
    """
    return numpy.fft.fft(series)

def utils_ifft(series):
    """
    Computes the inverse fast foyer transform.
    Parameters
    Input
        series : array_like
    Output
        The transformed array.
    """
    return numpy.fft.ifft(series)

def utils_rolling_window(series, window):
    """
    Provides a rolling window on a numpy array given an array and window size.
    Input
        series : array_like
            The array to create a rolling window on.
        window : int
            The window size.
    Output
        Strided array for computation.
    """
    shape = series.shape[:-1] + (series.shape[-1] - window + 1, window)
    strides = series.strides + (series.strides[-1],)

    return numpy.lib.stride_tricks.as_strided(series, shape=shape, strides=strides)

def utils_movsum(series, window=3):
    """
    Computes the moving sum over an array given a window size.
    Parameters
    Input
        series : array_like
            The array to compute the moving average on.
        window : int
            The window size.
    Output
        The moving sum over the array.
    """
    return numpy.sum(utils_rolling_window(series, window), -1)

def utils_movavg(series, window=3):
    """
    Computes the moving average over an array given a window size.
    Parameters
    Input
        a : array_like
            The array to compute the moving average on.
        window : int
            The window size.
    Output
        The moving average over the array.
    """
    return numpy.mean(utils_rolling_window(series, window), -1)

def utils_movmean(series, window=3):
    return utils_movavg(series, window)

def utils_movstd(series, window=3):
    """
    Computes the moving std. over an array given a window size.
    Input
        a : array_like
            The array to compute the moving std. on.
        window : int
            The window size.
    Output
        The moving std. over the array.
    """
    return numpy.std(utils_rolling_window(series, window), -1)

def validNumericalFloat(inValue):
    """
    Determines if the value is a valid numerical object.
    Args:
        inValue: floating point value

    Returns: Value in floating point or Not-A-Number.

    """
    try:
        return numpy.longdouble(inValue)
    except ValueError:
        return numpy.nan

def util_calculateMean(data):
    """
    Calculates the mean in a multiplication method since division produces a infinity or NaN
    Args:
        data: Input data set. We use a dataframe.

    Returns: Calculated mean for a vector dataframe.

    """
    try:
        n = data.size
    except BaseException:
        n = len(data)
    n = numpy.longdouble(n)
    if isinstance(data, list):
        dataArray = numpy.asarray(data)
    else:
        dataArray = data
    weightsMask = numpy.ones_like(numpy.longdouble(dataArray))
    avgVal = numpy.average(dataArray, weights=weightsMask)
    return avgVal

def util_calculateMeanStd(data):
    """
    Calculates the standard deviation in a multiplication method since division produces a infinity or NaN
    Args:
        data: Input data set. We use a dataframe.

    Returns: Calculated standard deviation for a vector dataframe.

    """
    zero = numpy.longdouble(0.0)
    try:
        n = data.size
    except BaseException:
        n = len(data)
    n = numpy.longdouble(n)
    if n <= 1:
        return (zero, zero)
    if isinstance(data, list):
        dataArray = numpy.asarray(data)
    else:
        dataArray = data
    # Use multiplication version of mean since numpy bug causes infinity.
    mean = util_calculateMean(dataArray)
    sd = zero
    # Calculate standard deviation
    for el in dataArray:
        diff = numpy.longdouble(el) - numpy.longdouble(mean)
        sd += (diff) ** 2
    points = numpy.longdouble(n - 1)
    sd = numpy.longdouble(numpy.sqrt(numpy.longdouble(sd) / numpy.longdouble(points)))
    return (mean, sd)

def utils_figure(series=None, query=None, xLabel="Time in seconds", yLabel="Magnitude", title="Unknown Graph",
                 filename=None):
    if series is None or query is None:
        return

    matplotlib.pyplot.plot(series, color="red", linewidth=2.5, linestyle="-")
    matplotlib.pyplot.plot(query, color="blue", linewidth=2.5, linestyle="-")
    matplotlib.pyplot.fill_between(series, query, facecolor='yellow')

    # The data limits are not updated automatically.
    matplotlib.pyplot.gca().relim()

    # With tight True, graph flows smoothly.
    matplotlib.pyplot.gca().autoscale_view(tight=True, scalex=True, scaley=True)

    # Label Figure
    matplotlib.pyplot.xlabel(xLabel)
    matplotlib.pyplot.ylabel(yLabel)
    matplotlib.pyplot.title(title)
    matplotlib.pyplot.legend()
    matplotlib.pyplot.show()

    if filename is not None:
        matplotlib.pyplot.savefig(filename, dpi=300)  # Save figure
    return

def utils_figureList(series, xLabel="Time in seconds", yLabel="Magnitude", title="Unknown Graph", filename=None):
    if series is None or series is []:
        return

    for index, stream in enumerate(series):
        matplotlib.pyplot.plot(stream)

    matplotlib.pyplot.plot(subplots=True, legend=False, layout=(1, len(series)))

    # The data limits are not updated automatically.
    matplotlib.pyplot.gca().relim()

    # With tight True, graph flows smoothly.
    matplotlib.pyplot.gca().autoscale_view(tight=True, scalex=True, scaley=True)

    # Label Figure
    matplotlib.pyplot.xlabel(xLabel)
    matplotlib.pyplot.ylabel(yLabel)
    matplotlib.pyplot.title(title)
    matplotlib.pyplot.legend()
    matplotlib.pyplot.show()

    if filename is not None:
        matplotlib.pyplot.savefig(filename, dpi=300)  # Save figure
    return

def zNormalize(ts):
    """
    Returns a z-normalized version of a time series.

    Parameters
    ----------
    ts: Time series to be normalized
    """

    ts -= numpy.mean(ts)
    std = numpy.std(ts)

    if std == 0:
        raise ValueError("The Standard Deviation cannot be zero")
    else:
        ts /= std

    return ts

def zNormalizeEuclidean(tsA, tsB):
    """
    Returns the z-normalized Euclidean distance between two time series.

    Parameters
    ----------
    tsA: Time series #1
    tsB: Time series #2
    """

    if len(tsA) != len(tsB):
        raise ValueError("tsA and tsB must be the same length")

    return numpy.linalg.norm(zNormalize(tsA.astype("float64")) - zNormalize(tsB.astype("float64")))

def movmeanstd(ts, m=0):
    """
    Calculate the mean and standard deviation within a moving window passing across a time series.

    Parameters
    ----------
    ts: Time series to evaluate.
    m: Width of the moving window.
    """
    if m <= 1:
        raise ValueError("Query length must be longer than one")
    mInt = int(m)
    zero = 0
    ts = ts.astype(numpy.longdouble)
    # Add zero to the beginning of the cumulative sum of ts
    s = numpy.insert(numpy.cumsum(ts), zero, zero)

    # Add zero to the beginning of the cumulative sum of ts ** 2
    sSq = numpy.insert(numpy.cumsum(ts ** 2), zero, zero)
    segSum = s[mInt:] - s[:-mInt]
    segSumSq = sSq[mInt:] - sSq[:-mInt]

    mov_mean = segSum / m
    mov_stdP = (segSumSq / m) - ((segSum / m) ** 2)
    if not numpy.all(mov_stdP == 0):
        mov_std = numpy.sqrt(numpy.abs(mov_stdP))
    else:
        mov_std = mov_stdP

    return [mov_mean, mov_std]

def movstd(ts, m):
    """
    Calculate the standard deviation within a moving window passing across a time series.

    Parameters
    ----------
    ts: Time series to evaluate.
    m: Width of the moving window.
    """
    if m <= 1:
        raise ValueError("Query length must be longer than one")

    ts = ts.astype("float")
    # Add zero to the beginning of the cumulative sum of ts
    s = numpy.insert(numpy.cumsum(ts), 0, 0)
    # Add zero to the beginning of the cumulative sum of ts ** 2
    sSq = numpy.insert(numpy.cumsum(ts ** 2), 0, 0)
    segSum = s[m:] - s[:-m]
    segSumSq = sSq[m:] - sSq[:-m]

    return numpy.sqrt(segSumSq / m - (segSum / m) ** 2)

def slidingDotProduct(query, ts):
    """
    Calculate the dot product between a query and all subsequences of length(query) in the timeseries ts. Note that we use Numpy's rfft method instead of fft.

    Parameters
    ----------
    query: Specific time series query to evaluate.
    ts: Time series to calculate the query's sliding dot product against.
    """
    m = len(query)
    n = len(ts)

    # If length is odd, zero-pad time series
    ts_add = 0
    if n % 2 == 1:
        ts = numpy.insert(ts, 0, 0)
        ts_add = 1

    q_add = 0
    # If length is odd, zero-pad query
    if m % 2 == 1:
        query = numpy.insert(query, 0, 0)
        q_add = 1

    # This reverses the array
    query = query[::-1]

    query = numpy.pad(query, (0, n - m + ts_add - q_add), 'constant')

    # Determine trim length for dot product. Note that zero-padding of the query has no effect on array length, which is solely determined by the longest vector
    trim = m - 1 + ts_add

    dot_product = numpy.fft.irfft(numpy.fft.rfft(ts) * numpy.fft.rfft(query))

    # Note that we only care about the dot product results from index m-1 onwards, as the first few values aren't true dot products (due to the way the FFT works for dot products)
    return dot_product[trim:]

def DotProductStomp(ts, m, dot_first, dot_prev, order):
    """
    Updates the sliding dot product for a time series ts from the previous dot product dot_prev.

    Parameters
    ----------
    ts: Time series under analysis.
    m: Length of query within sliding dot product.
    dot_first: The dot product between ts and the beginning query (QT1,1 in Zhu et.al).
    dot_prev: The dot product between ts and the query starting at index-1.
    order: The location of the first point in the query.
    """

    curLen = len(ts) - m + 1
    dot = numpy.roll(dot_prev, 1)

    dot += ts[order + m - 1] * ts[m - 1:curLen + m] - ts[order - 1] * numpy.roll(ts[:curLen], 1)

    # Update the first value in the dot product array
    dot[0] = dot_first[order]

    return dot

def mass(query, ts):
    """
    Calculates Mueen's ultra-fast Algorithm for Similarity Search (MASS): a Euclidean distance similarity search algorithm. Note that we are returning the square of MASS.

    Parameters
    ----------
    :query: Time series snippet to evaluate. Note that the query does not have to be a subset of ts.
    :ts: Time series to compare against query.
    """
    numericalResolution = numpy.finfo(float).eps
    # query_normalized = zNormalize(numpy.copy(query))
    mQFloat = numpy.longdouble(len(query))
    m = int(len(query))
    # Original
    # q_mean = numpy.mean(query)
    # q_std = numpy.std(query)
    # Multiply version, more robust
    q_mean, q_std = util_calculateMeanStd(data=query)
    mean, std = movmeanstd(ts, m)
    if q_std == 0:
        q_std = numericalResolution
    std = numpy.array(std)
    std[std == 0.0] = numericalResolution
    dot = slidingDotProduct(query, ts)

    # Original
    # res = numpy.sqrt(2*m*(1-(dot-m*mean*q_mean)/(m*std*q_std)))
    # Modified
    demoniator = (mQFloat * std * q_std)
    numerator = (dot - mQFloat * mean * q_mean)
    res = 2 * mQFloat * (1 - numerator / demoniator)
    res = numpy.sqrt(numpy.abs(res)) * numpy.sign(
        res)  # @todo note we are taking the absolute value then preserving the sign from before the square root.
    return res

def massStomp(query, ts, dot_first, dot_prev, index, mean, std):
    """
    Calculates Mueen's ultra-fast Algorithm for Similarity Search (MASS) between a query and timeseries using the STOMP dot product speedup. Note that we are returning the square of MASS.

    Parameters
    ----------
    query: Time series snippet to evaluate. Note that, for STOMP, the query must be a subset of ts.
    ts: Time series to compare against query.
    dot_first: The dot product between ts and the beginning query (QT1,1 in Zhu et.al).
    dot_prev: The dot product between ts and the query starting at index-1.
    index: The location of the first point in the query.
    mean: Array containing the mean of every subsequence in ts.
    std: Array containing the standard deviation of every subsequence in ts.
    """
    m = len(query)
    dot = DotProductStomp(ts, m, dot_first, dot_prev, index)
    std = numpy.array(std)
    std[std == 0.0] = 0.0001

    # Return both the MASS calculation and the dot product
    res = 2 * m * (1 - (dot - m * mean[index] * mean) / (m * std[index] * std))

    return res, dot

def apply_av(mp, av=None):
    """
    Applies an annotation vector to a Matrix Profile.

    Parameters
    ----------
    mp: Tuple containing the Matrix Profile and Matrix Profile Index.
    av: Numpy array containing the annotation vector.
    """

    if av is None:
        av = [1.0]

    if len(mp[0]) != len(av):
        raise ValueError(
            "Annotation Vector must be the same length as the matrix profile")
    else:
        av_max = numpy.max(av)
        av_min = numpy.min(av)
        if av_max > 1 or av_min < 0:
            raise ValueError("Annotation Vector must be between 0 and 1")
        mp_corrected = mp[0] + (1 - numpy.array(av)) * numpy.max(mp[0])
        return (mp_corrected, mp[1])

def is_self_join(tsA, tsB):
    """
    Helper function to determine if a self join is occurring or not. When tsA
    is absolutely equal to tsB, a self join is occurring.

    Parameters
    ----------
    tsA: Primary time series.
    tsB: Subquery time series.
    """
    return tsB is None or numpy.array_equal(tsA, tsB)

ext: py

max_stars_repo_name: intel/RAAD

max_issues_repo_name: intel/RAAD

max_forks_repo_name: intel/RAAD

reason: import numpy


text:
import unittest
from fastestimator.architecture.tensorflow import LeNet
import numpy as np
import fastestimator as fe

class TestSaliencyNetGetMask(unittest.TestCase):
    def test_salency_net_get_masks(self):
        outputs= "saliency"
        batch = {"x": np.random.uniform(0, 1, size=[1, 28, 28, 1]).astype(np.float32)}

        model = fe.build(model_fn=LeNet, optimizer_fn="adam")
        saliency = fe.xai.SaliencyNet(model=model, model_inputs="x", model_outputs="y_pred", outputs=outputs)
        new_batch = saliency.get_masks(batch)

        with self.subTest("check outputs exist"):
            self.assertIn(outputs, new_batch)

        with self.subTest("check output size"):
            self.assertEqual(new_batch[outputs].numpy().shape, (1, 28, 28, 1))

class TestSaliencyGetSmoothedMasks(unittest.TestCase):
    def test_salency_net_get_smoothed_masks(self):
        outputs = "saliency"
        batch = {"x": np.random.uniform(0, 1, size=[1, 28, 28, 1]).astype(np.float32)}

        model = fe.build(model_fn=LeNet, optimizer_fn="adam")
        saliency = fe.xai.SaliencyNet(model=model, model_inputs="x", model_outputs="y_pred", outputs=outputs)
        new_batch = saliency.get_smoothed_masks(batch)

        with self.subTest("check outputs exist"):
            self.assertIn(outputs, new_batch)

        with self.subTest("check output size"):
            self.assertEqual(new_batch[outputs].numpy().shape, (1, 28, 28, 1))

class TestSaliencyGetIntegratedMasks(unittest.TestCase):
    def test_salency_net_get_integrated_masks(self):
        outputs = "saliency"
        batch = {"x": np.random.uniform(0, 1, size=[1, 28, 28, 1]).astype(np.float32)}

        model = fe.build(model_fn=LeNet, optimizer_fn="adam")
        saliency = fe.xai.SaliencyNet(model=model, model_inputs="x", model_outputs="y_pred", outputs=outputs)
        new_batch = saliency.get_integrated_masks(batch)

        with self.subTest("check outputs exist"):
            self.assertIn(outputs, new_batch)

        with self.subTest("check output size"):
            self.assertEqual(new_batch[outputs].numpy().shape, (1, 28, 28, 1))

max_stars_repo_name: hanskrupakar/fastestimator

max_issues_repo_name: hanskrupakar/fastestimator

max_forks_repo_name: hanskrupakar/fastestimator


text:
import random
import numpy as np
from easydict import EasyDict as edict

def get_default_augment_config():
    config = edict()
    config.do_aug = True

    config.scale_factor = 0.25
    config.rot_factor = 15
    config.center_factor = 0.10 # 15% relative to the patch size
    config.color_factor = 0.2
    config.do_flip_aug = True

    config.rot_aug_rate = 0.6  #possibility to rot aug
    config.flip_aug_rate = 0.5 #possibility to flip aug

    config.use_color_normalize = True
    config.mean = np.array([0.485 * 255, 0.456 * 255, 0.406 * 255])
    config.std = np.array([0.229 * 255, 0.224 * 255, 0.225 * 255])

    config.multiplier = [0.5, 1., 1.5, 2, 2.5]
    return config

def do_augmentation(aug_config):
    scale = np.clip(np.random.randn(), -0.5, 1.0) * aug_config.scale_factor + 1.0
    rot = np.clip(np.random.randn(), -1.0, 1.0) * aug_config.rot_factor \
        if random.random() <= aug_config.rot_aug_rate else 0
    center = np.abs(np.clip(np.random.randn(2), -1.0, 1.0)) * aug_config.center_factor

    do_flip = aug_config.do_flip_aug and random.random() <= aug_config.flip_aug_rate
    c_up = 1.0 + aug_config.color_factor
    c_low = 1.0 - aug_config.color_factor
    color_scale = [random.uniform(c_low, c_up), random.uniform(c_low, c_up), random.uniform(c_low, c_up)]

    return scale, rot, center, do_flip, color_scale

def get_multiplier(img_size, scale_search, patch_size):
    """Computes the sizes of image at different scales
    :param img: numpy array, the current image
    :returns : list of float. The computed scales
    """
    return [x * patch_size / float(img_size) for x in scale_search]


text:
from app import app

import numpy as np
import datetime

@app.context_processor
def utility_processor():
    def get_interval(value, log_interval_width):
        if value:
            log_value = np.log10(value)
            log_low, log_high = log_value - log_interval_width, log_value + log_interval_width
            low, high = 10 ** log_low, 10 ** log_high
            return int(low), int(high)
        else:
            return "", ""

    def lowercase(s):
        if s:
            return s.lower()
        else:
            return ""

    def parse_number(number, n_decimals=0):
        if number:
            if n_decimals > 0:
                return f"{round(number, n_decimals):,}"
            elif n_decimals == 0:
                return f"{round(number):,}"
            else:
                return ""
        else:
            return ""

    def parse_seconds(number):
        if number:
            return str(datetime.timedelta(seconds=666))
        else:
            return ""

    def parse_date(date):
        return date.year, str(date.month).zfill(2), str(date.day).zfill(2)

    def get_screenshot_path(store_url):
        assert isinstance(store_url, str)
        return f"{app.config.get('STORE_SCREENSHOT_URI')}/{store_url}.png"

    return dict(get_interval=get_interval, lowercase=lowercase, parse_number=parse_number,
                parse_seconds=parse_seconds, parse_date=parse_date,
                get_screenshot_path=get_screenshot_path)

max_stars_repo_name: charlesashby/marketvault-front-end

max_issues_repo_name: charlesashby/marketvault-front-end

max_forks_repo_name: charlesashby/marketvault-front-end


text:
# -*- encoding: utf-8 -*-
'''
.. versionadded:: v0.12.0
'''
import functools as ft
import itertools as it
import threading

from dropbot_chip_qc.ui.render import get_summary_dict, render_summary
from logging_helpers import _L, caller_name
import asyncio_helpers as aioh
import dropbot_chip_qc as qc
import dropbot_chip_qc.ui.plan
import dropbot_chip_qc.ui.render
import networkx as nx
import numpy as np
import pandas as pd
import path_helpers as ph
import si_prefix as si
import trollius as asyncio

from .mqtt_proxy import DropBotMqttProxy

# For colors, see: https://gist.github.com/cfobel/fd939073cf13a309d7a9
light_blue = '#88bde6'
light_green = '#90cd97'

class Executor(object):
    def __init__(self, channels_graph, channel_plan):
        self.base_channels_graph = channels_graph.copy()
        self.channels_graph = channels_graph.copy()
        self.base_channel_plan = list(channel_plan)
        self.completed_results = []
        self._thread = None
        self._task = None

    def is_alive(self):
        return self._thread is not None and self._thread.is_alive()

    def remove_channels(self, bad_channels):
        self.channels_graph.remove_nodes_from(bad_channels)

    def channel_plan(self):
        if self.completed_results:
            channel_plan = self.completed_results[-1]['channel_plan']
            completed_transfers = \
                self.completed_results[-1]['completed_transfers']
        else:
            channel_plan = self.base_channel_plan
            completed_transfers = []

        channel_plan_ = [c for c in channel_plan if c in self.channels_graph]
        if len(channel_plan_) < len(channel_plan):
            _L().debug('reroute around missing channels')
            channel_plan = list(qc.ui.plan\
                .create_channel_plan(self.channels_graph, channel_plan_,
                                     loop=False))
        return channel_plan, completed_transfers

    def start(self, aproxy, signals, bad_channels=None, min_duration=.15):
        '''
        # TODO

         - incorporate `execute()` coroutine
         - add
        '''
        if self.is_alive():
            raise RuntimeError('Executor is already running.')

        channel_plan, completed_transfers = self.channel_plan()

        @asyncio.coroutine
        def execute_test(*args, **kwargs):
            yield asyncio.From(set_capacitance_update_interval())
            try:
                result = yield asyncio\
                    .From(qc.ui.plan.transfer_windows(*args, **kwargs))
            except qc.ui.plan.TransferFailed as exception:
                # Save intermediate result.
                result = dict(channel_plan=exception.channel_plan,
                              completed_transfers=exception.completed_transfers)
                signals.signal('test-interrupt').send(caller_name(0), **result)
            self.completed_results.append(result)
            yield asyncio.From(aproxy.set_state_of_channels(pd.Series(), append=False))
            # result = dict(channel_plan=channel_plan_i,
                          # completed_transfers=completed_transfers_i)
            raise asyncio.Return(result)

        @asyncio.coroutine
        def set_capacitance_update_interval():
            state = yield asyncio.From(aproxy.state)
            max_update_interval = int(.5 * min_duration * 1e3)
            if state.capacitance_update_interval_ms > max_update_interval \
                    or state.capacitance_update_interval_ms == 0:
                yield asyncio\
                    .From(aproxy.update_state(capacitance_update_interval_ms=
                                              max_update_interval))

        looped_channel_plan = (channel_plan +
                               nx.shortest_path(self.channels_graph,
                                                channel_plan[-1],
                                                self.base_channel_plan[0])[1:])
        self._task = aioh.cancellable(execute_test)
        transfer_liquid = ft.partial(qc.ui.plan.transfer_liquid, aproxy,
                                     min_duration=min_duration)
        self._thread = threading.Thread(target=self._task,
                                        args=(signals, looped_channel_plan,
                                              completed_transfers,
                                              transfer_liquid),
                                        kwargs={'n': 3})

        self._thread.daemon = True
        self._thread.start()

    def pause(self):
        if self.is_alive():
            self._task.cancel()

    def reset(self):
        self.pause()
        del self.completed_results[:]
        self.channels_graph = self.base_channels_graph.copy()

class ExecutorController(object):
    def __init__(self, aproxy, ui, executor):
        self.ui = ui
        channel_electrodes = ui['channel_electrodes']
        channel_patches = ui['channel_patches']
        chip_info = ui['chip_info']
        chip_info_mm = ui['chip_info_mm']
        figure = ui['figure']
        signals = ui['signals']

        def calibrate_sheet_capacitance(target_force, *args):
            '''Calibrate sheet capacitance with liquid present

            **NOTE** Prior to running the following cell:

             - _at least_ one electrode **MUST** be **actuated**
             - all actuated electrodes **MUST** be completely covered with liquid

            It may be helpful to use the interactive figure UI to manipulate liquid until
            the above criteria are met.

            This function performs the following steps:

             1. Measure **total capacitance** across **all actuated electrodes**
             2. Compute sheet capacitance with liquid present ($\Omega_L$) based on nominal
                areas of actuated electrodes from `chip_file`
             3. Compute voltage to match 25 μN of force, where
                $F = 10^3 \cdot 0.5 \cdot \Omega_L \cdot V^2$
             4. Set DropBot voltage to match target of 25 μN force.
            '''
            proxy = DropBotMqttProxy.from_uri('dropbot', aproxy.__client__._host)
            name = 'liquid'
            states = proxy.state_of_channels
            channels = states[states > 0].index.tolist()
            electrodes_by_id = pd.Series(chip_info_mm['electrodes'],
                                         index=(e['id'] for e in
                                                chip_info_mm['electrodes']))
            actuated_area = (electrodes_by_id[channel_electrodes[channels]]
                             .map(lambda x: x['area'])).sum()
            capacitance = pd.Series(proxy.capacitance(0)
                                    for i in range(20)).median()
            sheet_capacitance = capacitance / actuated_area
            message = ('Measured %s sheet capacitance: %sF/%.1f mm^2 = %sF/mm^2'
                       % (name, si.si_format(capacitance), actuated_area,
                          si.si_format(sheet_capacitance)))
            print(message)
            voltage = np.sqrt(target_force / (1e3 * 0.5 * sheet_capacitance))
            return sheet_capacitance, voltage

        def pause(*args):
            executor.pause()

        def reset(*args):
            executor.reset()
            channel_patches.map(lambda x: x.set_facecolor(light_blue))
            for collection in list(figure._ax.collections):
                collection.remove()
            figure._ax.figure.canvas.draw()

        def save_results(output_directory, chip_uuid, *args):
            output_dir = ph.path(output_directory)
            channel_plan, completed_transfers = executor.channel_plan()
            proxy = DropBotMqttProxy.from_uri('dropbot', aproxy.__client__._host)
            summary_dict = \
                get_summary_dict(proxy, chip_info,
                                 sorted(set(executor.base_channel_plan)),
                                 channel_plan, completed_transfers,
                                 chip_uuid=chip_uuid)
            output_path = output_dir.joinpath('Chip test report - %s.html' %
                                              summary_dict['chip_uuid'])
            print('save to: `%s`' % output_path)
            render_summary(output_path, **summary_dict)

        def start(bad_channels, *args):
            executor.channels_graph = executor.base_channels_graph.copy()
            executor.remove_channels(bad_channels)
            executor.start(aproxy, signals)

        self.calibrate_sheet_capacitance = calibrate_sheet_capacitance
        self.pause = pause
        self.reset = reset
        self.save_results = save_results
        self.start = start

max_stars_repo_name: cfobel/dropbot-chip-qc

max_issues_repo_name: cfobel/dropbot-chip-qc

max_forks_repo_name: cfobel/dropbot-chip-qc

reason: import numpy,import networkx


text:
import numpy as np

from rectangle import Rectangle

class Component():
    ''' ... '''

    def __init__(self, kind):
        self.kind = kind

class Corner():
    ''' ... '''

    def __init__(self, pix):
        self.pix = pix

class Chip(Component):
    ''' ... '''

    def __init__(self, rectangle=None, image=None, spec=None):
        super().__init__("chip")
        self.rectangle = rectangle
        self.image = image
        self.signature = None
        self.symbol = None

        if spec is None:
            return
        spec.pop(0)
        while spec:
            tag = spec.pop(0)
            if tag == "tl":
                y = float(spec.pop(0))
                x = float(spec.pop(0))
                tl = Corner((y, x))
            elif tag == "tr":
                y = float(spec.pop(0))
                x = float(spec.pop(0))
                tr = Corner((y, x))
            elif tag == "bl":
                y = float(spec.pop(0))
                x = float(spec.pop(0))
                bl = Corner((y, x))
            elif tag == "br":
                y = float(spec.pop(0))
                x = float(spec.pop(0))
                br = Corner((y, x))
                self.rectangle = Rectangle(tl, tr, bl, br)
            elif tag == "img":
                self.image = spec.pop(0)
            elif tag == "sig":
                self.signature = spec.pop(0)
            elif tag == "symbol":
                self.symbol = spec.pop(0)
            else:
                raise Exception("Component 'chip' Syntax Error")

    def __str__(self):
        i = [self.kind]
        for c in ("tl", "tr", "bl", "br"):
            j = getattr(self.rectangle, c)
            i.append(c)
            i.append(str(j.pix[0]))
            i.append(str(j.pix[1]))
        i.append("img")
        i.append(self.image)
        if self.signature:
            i.append("sig")
            i.append(self.signature)
        if self.symbol:
            i.append("symbol")
            i.append(self.symbol)
        return " ".join(i)

    PIN_WIDTH = 15
    PIN_LENGTH = 50

    def top_side_pins(self, sheet):
        tl = sheet.pix2inch(self.rectangle.tl.pix)
        tr = sheet.pix2inch(self.rectangle.tr.pix)
        x = round(tl[1], 1)
        while x < tr[1]:
            x += .1
            pix = sheet.inch2pix((tl[0], x))
            while True:
                snippet = sheet.img[
                    pix[0],
                    pix[1] - self.PIN_WIDTH : pix[1] + self.PIN_WIDTH,
                ]
                if np.amax(snippet) > 0:
                    break
                pix = (pix[0] - 1, pix[1])
            snippet = sheet.img[
                pix[0] - self.PIN_LENGTH : pix[0],
                pix[1] - self.PIN_WIDTH : pix[1] + self.PIN_WIDTH,
            ]
            yield snippet

    def bottom_side_pins(self, sheet):
        tl = sheet.pix2inch(self.rectangle.tl.pix)
        tr = sheet.pix2inch(self.rectangle.tr.pix)
        bl = sheet.pix2inch(self.rectangle.bl.pix)
        br = sheet.pix2inch(self.rectangle.br.pix)
        x = round(tl[1], 1)
        while x < tr[1]:
            x += .1
            pix = sheet.inch2pix((bl[0], x))
            while True:
                snippet = sheet.img[
                    pix[0],
                    pix[1] - self.PIN_WIDTH : pix[1] + self.PIN_WIDTH,
                ]
                if np.amax(snippet) > 0:
                    break
                pix = (pix[0] + 1, pix[1])
            snippet = sheet.img[
                pix[0] : pix[0] + self.PIN_LENGTH,
                pix[1] - self.PIN_WIDTH : pix[1] + self.PIN_WIDTH,
            ]
            yield snippet

    def left_side_pins(self, sheet):
        tl = sheet.pix2inch(self.rectangle.tl.pix)
        bl = sheet.pix2inch(self.rectangle.bl.pix)
        y = round(tl[0], 1)
        while y > bl[0]:
            y -= .1
            pix = sheet.inch2pix((y, tl[1]))
            while True:
                snippet = sheet.img[
                    pix[0] - self.PIN_WIDTH : pix[0] + self.PIN_WIDTH,
                    pix[1],
                ]
                if np.amax(snippet) > 0:
                    break
                pix = (pix[0], pix[1] - 1)
            snippet = sheet.img[
                pix[0] - self.PIN_WIDTH : pix[0] + self.PIN_WIDTH,
                pix[1] - self.PIN_LENGTH : pix[1],
            ]
            yield snippet

    def right_side_pins(self, sheet):
        tl = sheet.pix2inch(self.rectangle.tl.pix)
        bl = sheet.pix2inch(self.rectangle.bl.pix)
        tr = sheet.pix2inch(self.rectangle.tr.pix)
        br = sheet.pix2inch(self.rectangle.br.pix)
        y = round(tl[0], 1)
        while y > bl[0]:
            y -= .1
            pix = sheet.inch2pix((y, tr[1]))
            while True:
                snippet = sheet.img[
                    pix[0] - self.PIN_WIDTH : pix[0] + self.PIN_WIDTH,
                    pix[1],
                ]
                if np.amax(snippet) > 0:
                    break
                pix = (pix[0], pix[1] + 1)
            snippet = sheet.img[
                pix[0] - self.PIN_WIDTH : pix[0] + self.PIN_WIDTH,
                pix[1] : pix[1] + self.PIN_LENGTH,
            ]
            yield snippet

class BOM():
    ''' ... '''
    def __init__(self):
        self.bomlist = []

    def __iter__(self):
        yield from self.bomlist

    def add_chip(self, rectangle, image):
        retval = Chip(rectangle, image)
        self.bomlist.append(retval)
        return retval

    def load_from_file(self, filename):
        for i in open(filename):
            j = i.split()
            if j[0] == "chip":
                self.bomlist.append(Chip(spec=j))
            else:
                raise Exception("Component Syntax Error")

    def write_to_file(self, filename):
        file = open(filename, "w")
        for i in self.bomlist:
            file.write(str(i) + "\n")