"""Test functions in generation.py"""
import logging
import sqlite3
import os
from pathlib import Path
from powergenome.GenX import (
    RESOURCE_TAGS,
    add_cap_res_network,
    add_misc_gen_values,
    check_resource_tags,
    create_policy_req,
    create_regional_cap_res,
    hydro_energy_to_power,
    max_cap_req,
    min_cap_req,
    network_line_loss,
    network_max_reinforcement,
    network_reinforcement_cost,
    reduce_time_domain,
    round_col_values,
    set_int_cols,
)
from powergenome.eia_opendata import add_user_fuel_prices
from powergenome.external_data import make_generator_variability

from powergenome.fuels import fuel_cost_table
from powergenome.nrelatb import db_col_values

CWD = Path.cwd()
# os.environ["RESOURCE_GROUPS"] = str(CWD / "data" / "resource_groups_base")
# os.environ["PUDL_DB"] = "sqlite:////" + str(
#     CWD / "tests" / "data" / "pudl_test_data.db"
# )
# os.environ["PG_DB"] = "sqlite:////" + str(
#     CWD / "tests" / "data" / "pg_misc_tables.sqlite3"
# )

import numpy as np
import pandas as pd
import sqlalchemy
import powergenome
import pytest
from powergenome.generators import (
    fill_missing_tech_descriptions,
    gentype_region_capacity_factor,
    group_technologies,
    label_retirement_year,
    label_small_hydro,
    load_demand_response_efs_profile,
    remove_leading_zero,
    unit_generator_heat_rates,
    load_860m,
    GeneratorClusters,
    energy_storage_mwh,
)
from powergenome.load_profiles import (
    add_load_growth,
    make_final_load_curves,
    make_load_curves,
)
from powergenome.params import DATA_PATHS, SETTINGS  # , SETTINGS
from powergenome.transmission import (
    agg_transmission_constraints,
    transmission_line_distance,
)
from powergenome.util import (
    build_scenario_settings,
    find_region_col,
    init_pudl_connection,
    check_settings,
    load_settings,
    map_agg_region_names,
    regions_to_keep,
    remove_fuel_scenario_name,
    reverse_dict_of_lists,
    write_results_file,
)

logger = logging.getLogger(powergenome.__name__)
logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
formatter = logging.Formatter(
    # More extensive test-like formatter...
    "%(asctime)s [%(levelname)8s] %(name)s:%(lineno)s %(message)s",
    # This is the datetime format string.
    "%Y-%m-%d %H:%M:%S",
)
handler.setFormatter(formatter)
logger.addHandler(handler)

# pudl_engine = sqlite3.connect(DATA_PATHS["test_data"] / "pudl_test_data.db")
# pg_engine = sqlalchemy.create_engine(
#     "sqlite:////" + str(DATA_PATHS["test_data"] / "pg_misc_tables.sqlite3")
# )
if os.name == "nt":
    # if user is using a windows system
    sql_prefix = "sqlite:///"
else:
    sql_prefix = "sqlite:////"
pudl_engine, pudl_out, pg_engine = init_pudl_connection(
    start_year=2018,
    end_year=2020,
    pudl_db=sql_prefix + str(DATA_PATHS["test_data"] / "pudl_test_data.db"),
    pg_db=sql_prefix + str(DATA_PATHS["test_data"] / "pg_misc_tables.sqlite3"),
)

@pytest.fixture(scope="module")
def generation_fuel_eia923_data():
    gen_fuel = pd.read_sql_query(
        "SELECT * FROM generation_fuel_eia923",
        pudl_engine,
        parse_dates=["report_date"],
    )
    return gen_fuel

@pytest.fixture(scope="module")
def generators_eia860_data():
    sql = """
        SELECT *
        FROM generators_eia860
        WHERE operational_status_code = 'OP'
    """
    gens_860 = pd.read_sql_query(
        sql, pudl_engine, parse_dates=["report_date", "planned_retirement_date"]
    )
    return gens_860

@pytest.fixture(scope="module")
def generators_entity_eia_data():
    gen_entity = pd.read_sql_query(
        "SELECT * FROM generators_entity_eia",
        pudl_engine,
        parse_dates=["operating_date"],
    )
    return gen_entity

@pytest.fixture(scope="module")
def plant_region_map_ipm_data():
    plant_region_map = pd.read_sql_query(
        "SELECT * FROM plant_region_map_epaipm", pudl_engine
    )
    return plant_region_map

@pytest.fixture(scope="module")
def test_settings():
    settings = load_settings(DATA_PATHS["test_data"] / "test_settings.yml")
    settings["RESOURCE_GROUPS"] = DATA_PATHS["test_data"] / "resource_groups_base"
    return settings

@pytest.fixture(scope="module")
def CA_AZ_settings():
    settings = load_settings(
        DATA_PATHS["powergenome"].parent / "example_systems" / "CA_AZ" / "settings"
    )
    settings["input_folder"] = Path(
        DATA_PATHS["powergenome"].parent
        / "example_systems"
        / "CA_AZ"
        / settings["input_folder"]
    )
    settings["RESOURCE_GROUPS"] = DATA_PATHS["test_data"] / "resource_groups_base"
    scenario_definitions = pd.read_csv(
        settings["input_folder"] / settings["scenario_definitions_fn"]
    )
    scenario_settings = build_scenario_settings(settings, scenario_definitions)

    return scenario_settings[2030]["p1"]

class MockPudlOut:
    """
    The methods in this class read pre-calculated tables from a sqlite db and return
    the expected values from pudl_out methods.
    """

    def hr_by_unit():
        "Heat rate by unit over multiple years"
        hr_by_unit = pd.read_sql_query(
            "SELECT * FROM hr_by_unit", pudl_engine, parse_dates=["report_date"]
        )
        return hr_by_unit

    def bga():
        "Boiler generator associations with unit_id_pudl values"
        bga = pd.read_sql_query(
            "SELECT * FROM boiler_generator_assn_eia860", pudl_engine
        )
        return bga

def test_load_single_settings():
    settings = load_settings(
        DATA_PATHS["powergenome"].parent
        / "example_systems"
        / "CA_AZ"
        / "combined_settings.yml"
    )

def test_load_multiple_settings():
    settings = load_settings(
        DATA_PATHS["powergenome"].parent / "example_systems" / "CA_AZ" / "settings"
    )

def test_group_technologies(generators_eia860_data, test_settings):
    df = generators_eia860_data.loc[
        generators_eia860_data.report_date.dt.year == 2020, :
    ]
    # df = df.query("report_date.dt.year==2017")
    df = df.drop_duplicates(subset=["plant_id_eia", "generator_id"])

    grouped_by_tech = group_technologies(
        df,
        test_settings.get("group_technologies"),
        test_settings.get("tech_groups", {}) or {},
        test_settings.get("regional_no_grouping", {}) or {},
    )
    techs = grouped_by_tech["technology_description"].unique()
    capacities = grouped_by_tech.groupby("technology_description")[
        test_settings["capacity_col"]
    ].sum()
    # expected_hydro_cap = 48.1
    hydro_cap = capacities["Conventional Hydroelectric"]
    expected_peaker_cap = 354.8
    # peaker_cap = capacities["Peaker"]

    assert len(df) == len(grouped_by_tech)
    assert df["capacity_mw"].sum() == grouped_by_tech["capacity_mw"].sum()
    assert "Peaker" in techs
    # assert np.allclose(hydro_cap, expected_hydro_cap)
    # assert np.allclose(peaker_cap, expected_peaker_cap)

def test_fill_missing_tech_descriptions(generators_eia860_data):
    filled = fill_missing_tech_descriptions(generators_eia860_data)

    assert len(generators_eia860_data) == len(
        filled.dropna(subset=["technology_description"])
    )

def test_label_small_hyro(
    generators_eia860_data, test_settings, plant_region_map_ipm_data
):
    region_agg_map = reverse_dict_of_lists(test_settings.get("region_aggregations", {}))
    model_region_map_df = map_agg_region_names(
        df=plant_region_map_ipm_data,
        region_agg_map=region_agg_map,
        original_col_name="region",
        new_col_name="model_region",
    )
    df = pd.merge(
        generators_eia860_data, model_region_map_df, on="plant_id_eia", how="left"
    )
    logger.info(df[["plant_id_eia", "technology_description", "model_region"]].head())

    # df["model_region"] = df["region"].map(reverse_dict_of_lists)

    df = label_small_hydro(df, test_settings, by=["plant_id_eia", "report_date"])
    print(df.query("plant_id_eia==34"))
    logger.info(df["technology_description"].unique())

    assert "Small Hydroelectric" in df["technology_description"].unique()
    assert np.allclose(
        df.loc[df.technology_description == "Small Hydroelectric", "capacity_mw"].sum(),
        140.5,
    )

# def test_label_retirement_year(
#     generators_eia860_data, generators_entity_eia_data, test_settings
# ):
#     gens = pd.merge(
#         generators_eia860_data,
#         generators_entity_eia_data,
#         on=["plant_id_eia", "generator_id"],
#         how="left",
#     )
#     df = label_retirement_year(gens, test_settings)
#     print(df)

#     assert df.loc[df["retirement_year"].isnull(), :].empty is True

def test_unit_generator_heat_rates(data_years=[2020]):
    hr_df = unit_generator_heat_rates(MockPudlOut, data_years)

    assert hr_df.empty is False
    assert "heat_rate_mmbtu_mwh" in hr_df.columns
    assert np.allclose(
        hr_df.query("plant_id_eia==117 & unit_id_pudl == 2")[
            "heat_rate_mmbtu_mwh"
        ].values,
        [7.635626],
    )

def test_load_860m(test_settings):
    eia_860m = load_860m(test_settings)
    test_settings["eia_860m_fn"] = None
    eia_860m = load_860m(test_settings)

def test_agg_transmission_constraints(test_settings):
    agg_transmission_constraints(pg_engine, test_settings)

def test_demand_curve(test_settings):
    make_load_curves(pg_engine, test_settings)

def test_alt_table_load_sources(CA_AZ_settings):
    # Test with a single non-default load table
    CA_AZ_settings["load_source_table_name"] = {
        # "EFS": "load_curves_nrel_efs",
        "FERC": "load_curves_ferc",
    }
    CA_AZ_settings["regional_load_source"] = "FERC"
    CA_AZ_settings["EFS_DATA"] = DATA_PATHS["test_data"] / "efs"
    make_final_load_curves(pg_engine, CA_AZ_settings)

def test_combined_load_sources(CA_AZ_settings):
    # Test with a combination of user and database load sources
    CA_AZ_settings["regional_load_fn"] = "test_regional_load_profiles.csv"
    CA_AZ_settings["load_source_table_name"] = {"EFS": "load_curves_nrel_efs"}
    CA_AZ_settings["regional_load_source"] = {
        "USER": ["CA_N", "CA_S"],
        "EFS": ["WECC_AZ"],
    }
    CA_AZ_settings["load_source_table_name"] = {
        "EFS": "load_curves_nrel_efs",
        "FERC": "load_curves_ferc",
    }
    CA_AZ_settings["EFS_DATA"] = DATA_PATHS["test_data"] / "efs"
    CA_AZ_settings["electrification"] = "reference"
    make_final_load_curves(pg_engine, CA_AZ_settings)

def test_check_settings(test_settings):
    check_settings(test_settings, pg_engine)

# def test_gentype_region_capacity_factor(plant_region_map_ipm_data, test_settings):
#     cf_techs = test_settings["capacity_factor_techs"]

#     plant_region_map_ipm_data = plant_region_map_ipm_data.rename(
#         columns={"region": "model_region"}
#     )
#     df = gentype_region_capacity_factor(
#         pudl_engine, plant_region_map_ipm_data, test_settings
#     )
#     print(df.technology.unique())
#     assert "Biomass" in df.technology.unique()
#     # CF can sometime be greater than 1, but shouldn't be significantly higher.
#     assert df.loc[df["technology"].isin(cf_techs), "capacity_factor"].max() < 2

def test_gen_integration(CA_AZ_settings, tmp_path):
    CA_AZ_settings["atb_modifiers"] = {
        "ngccccs": {
            "technology": "NaturalGas",
            "tech_detail": "CCCCSAvgCF",
            "Heat_Rate_MMBTU_per_MWh": 7.159,
        }
    }
    CA_AZ_settings["modified_atb_new_gen"]["NGCCS100"]["heat_rate"] = 7.5
    CA_AZ_settings["EFS_DATA"] = DATA_PATHS["test_data"] / "efs"
    CA_AZ_settings["electrification"] = "reference"
    gc = GeneratorClusters(
        pudl_engine, pudl_out, pg_engine, CA_AZ_settings, supplement_with_860m=True
    )
    all_gens = gc.create_all_generators()
    all_gens = add_misc_gen_values(all_gens, CA_AZ_settings)

    assert (
        all_gens.loc[all_gens["Resource"] == "CA_N_biomass_1", "Eff_Up"].values[0] == 1
    )
    assert np.allclose(
        all_gens.query("technology.str.contains('NaturalGas_CCCCS', case=False)")[
            "Heat_Rate_MMBTU_per_MWh"
        ].mean(),
        7.159,
    )
    assert np.allclose(
        all_gens.query("technology.str.contains('CCS100', case=False)")[
            "Heat_Rate_MMBTU_per_MWh"
        ].mean(),
        7.5,
    )
    # Capacity in existing clusters should be > 0
    assert all(gc.results["Existing_Cap_MW"] > 0)
    batteries = gc.results.query("technology == 'Batteries'")

    # Battery energy capacity should be larger than battery power capacity
    assert all(batteries["Existing_Cap_MWh"] > batteries["Existing_Cap_MW"])

    gen_variability = make_generator_variability(all_gens)
    assert (gen_variability >= 0).all().all()

    fuels = fuel_cost_table(
        fuel_costs=gc.fuel_prices,
        generators=gc.all_resources,
        settings=gc.settings,
    )
    fuels.index.name = "Time_Index"
    write_results_file(
        df=remove_fuel_scenario_name(fuels, gc.settings)
        .pipe(set_int_cols)
        .pipe(round_col_values),
        folder=tmp_path,
        file_name="Fuels_data.csv",
        include_index=True,
    )
    load = make_final_load_curves(pg_engine=pg_engine, settings=gc.settings)
    (
        reduced_resource_profile,
        reduced_load_profile,
        time_series_mapping,
        representative_point,
    ) = reduce_time_domain(gen_variability, load, gc.settings)
    if gc.settings["reduce_time_domain"]:
        assert len(representative_point) == gc.settings["time_domain_periods"]
        assert (
            time_series_mapping["Rep_Period"].nunique()
            == gc.settings["time_domain_periods"]
        )
        assert representative_point.isna().any().all() == False
        assert time_series_mapping.isna().any().all() == False
    assert len(reduced_load_profile) == len(reduced_resource_profile)

    gc.settings["distributed_gen_method"]["CA_N"] = "fraction_load"
    gc.settings["distributed_gen_values"][2030]["CA_N"] = 0.1
    gc.settings["regional_load_fn"] = "test_regional_load_profiles.csv"
    gc.settings["regional_load_includes_demand_response"] = False
    make_final_load_curves(pg_engine=pg_engine, settings=gc.settings)

    model_regions_gdf = gc.model_regions_gdf
    transmission = (
        agg_transmission_constraints(pg_engine=pg_engine, settings=gc.settings)
        .pipe(
            transmission_line_distance,
            ipm_shapefile=model_regions_gdf,
            settings=gc.settings,
            units="mile",
        )
        .pipe(network_line_loss, settings=gc.settings)
        .pipe(network_max_reinforcement, settings=gc.settings)
        .pipe(network_reinforcement_cost, settings=gc.settings)
        .pipe(set_int_cols)
        .pipe(round_col_values)
        .pipe(add_cap_res_network, settings=gc.settings)
    )

    if gc.settings.get("emission_policies_fn"):
        energy_share_req = create_policy_req(gc.settings, col_str_match="ESR")
        co2_cap = create_policy_req(gc.settings, col_str_match="CO_2")
    min_cap = min_cap_req(gc.settings)

    cap_res = create_regional_cap_res(gc.settings)

def test_existing_gen_profiles():
    ipm_regions = pd.read_sql_table("regions_entity_epaipm", pg_engine)
    regions = [r for r in ipm_regions.region_id_epaipm.to_list() if "CN_" not in r]

    s = """
    SELECT DISTINCT technology_description
    FROM generators_eia860
    """
    technologies = (
        pd.read_sql_query(s, pudl_engine).dropna()["technology_description"].to_list()
    )
    technologies.remove("Natural Gas with Compressed Air Storage")
    settings = dict(
        RESOURCE_GROUPS=DATA_PATHS["test_data"] / "resource_groups_base",
        target_usd_year=2019,
        model_year=2030,
        model_first_planning_year=2022,
        model_regions=regions,
        eia_data_years=[2020],
        capacity_col="capacity_mw",
        num_clusters={tech: 2 for tech in technologies},
        retirement_ages={tech: 200 for tech in technologies},
        atb_data_year=2022,
        atb_existing_year=2020,
        fuel_eia_aeo_year=2022,
        load_eia_aeo_year=2020,
        aeo_fuel_usd_year=2021,
        eia_series_region_names={
            "mountain": "MTN",
        },
        eia_series_fuel_names={
            "naturalgas": "NG",
        },
        eia_series_scenario_names={
            "reference": "REF2022",
        },
        aeo_fuel_scenarios={
            "coal": "reference",
            "naturalgas": "reference",
            "distillate": "reference",
            "uranium": "reference",
        },
        eia_atb_tech_map={
            "Battery": "Battery_*",
            "Batteries": "Battery_*",
            "Biomass": "Biopower_Dedicated",
            "Solar Thermal without Energy Storage": "CSP_Class1",
            "Conventional Steam Coal": "Coal_newAvgCF",
            "Coal Integrated Gasification Combined Cycle": "NaturalGas_CCAvgCF",
            "Natural Gas Fired Combined Cycle": "NaturalGas_CCAvgCF",  # [NaturalGas_CCAvgCF, NETL_NGCC]
            "Natural Gas Fired Combustion Turbine": "NaturalGas_CTAvgCF",
            "Peaker": "NaturalGas_CTAvgCF",
            "Natural Gas Internal Combustion Engine": "NaturalGas_CTAvgCF",
            "Landfill Gas": "NaturalGas_CTAvgCF",
            "Petroleum Liquids": "NaturalGas_CTAvgCF",
            "Municipal Solid Waste": "Biopower_Dedicated",
            "Other Waste Biomass": "Biopower_Dedicated",
            "Wood/Wood Waste Biomass": "Biopower_Dedicated",
            "Solar Photovoltaic": "UtilityPV_Class1",
            "Geothermal": "Geothermal_HydroFlash",  # assume installed capacity is dominated by flash
            "Conventional Hydroelectric": "Hydropower_NSD4",  # Large variability based on choice
            "Hydroelectric Pumped Storage": "Hydropower_NSD4",  # Large variability based on choice
            "Small Hydroelectric": "Hydropower_NSD3",  # Large variability based on choice
            "Onshore Wind Turbine": "LandbasedWind_Class4",  # All onshore wind is the same
            "Offshore Wind Turbine": "OffShoreWind_Class10",  # Mid-range of floating offshore wind
            "Nuclear": "Nuclear_Nuclear",
            "Natural Gas Steam Turbine": "Coal_newAvgCF",  # No gas steam turbines in ATB, using coal instead
            "Solar Thermal with Energy Storage": "CSP_Class1",
            "Solar Thermal without Energy Storage": "CSP_Class1",
            "Other Gases": "NaturalGas_CTAvgCF",
            "Other Natural Gas": "NaturalGas_CTAvgCF",
            "Petroleum Coke": "Coal_newAvgCF",
            "All Other": "NaturalGas_CTAvgCF",
            "Flywheels": "Battery_*",
            "Natural Gas with Compressed Air Storage": "NaturalGas_CTAvgCF",
        },
        startup_vom_costs_mw={
            "coal_small_sub": 2.81,
            "coal_large_sub": 2.69,
            "coal_supercritical": 2.98,
            "gas_cc": 1.03,
            "gas_large_ct": 0.77,
            "gas_aero_ct": 0.70,
            "gas_steam": 1.03,
            "nuclear": 5.4,
        },
        startup_vom_costs_usd_year=2011,
        startup_costs_type="startup_costs_per_cold_start_mw",
        startup_costs_per_cold_start_usd_year=2011,
        startup_costs_per_cold_start_mw={
            "coal_small_sub": 147,
            "coal_large_sub": 105,
            "coal_supercritical": 104,
            "gas_cc": 79,
            "gas_large_ct": 103,
            "gas_aero_ct": 32,
            "gas_steam": 75,
            "nuclear": 210,
        },
        existing_startup_costs_tech_map={
            "Conventional Steam Coal": "coal_large_sub",
            "Natural Gas Fired Combined Cycle": "gas_cc",
            "Natural Gas Fired Combustion Turbine": "gas_large_ct",
            "Natural Gas Steam Turbine": "gas_steam",
            "Nuclear": "nuclear",
        },
    )
    gc = GeneratorClusters(
        pudl_engine, pudl_out, pg_engine, settings, supplement_with_860m=False
    )
    existing_gen = gc.create_region_technology_clusters()
    gen_variability = make_generator_variability(existing_gen)
    assert (gen_variability >= 0).all().all()

def test_cap_req():
    settings = {
        "model_tag_names": ["MinCapTag_1", "MinCapTag_2", "MaxCapTag_1", "MaxCapTag_2"],
        "MinCapReq": {
            "MinCapTag_1": {"description": "Landbasedwind", "min_mw": 8000},
            "MinCapTag_2": {"description": "CA_S_solar", "min_mw": 10000},
        },
        "MaxCapReq": {
            "MaxCapTag_1": {"description": "Landbasedwind", "max_mw": 8000},
            "MaxCapTag_2": {"description": "CA_S_solar", "max_mw": 10000},
        },
        "generator_columns": [],
    }

    max_cap = max_cap_req(settings)
    min_cap = min_cap_req(settings)

    assert set(settings["generator_columns"]) == set(settings["model_tag_names"])
    assert min_cap.isna().any().all() == False
    assert max_cap.isna().any().all() == False

def test_check_resource_tags():
    # Check something that should fail
    cols = ["region", "technology"] + RESOURCE_TAGS
    data = [pd.Series(["a", "b"] + [1] * len(RESOURCE_TAGS), index=cols)]
    df = pd.DataFrame(data)

    with pytest.raises(Exception):
        check_resource_tags(df)

    # Check something that should pass
    cols = ["region", "technology"] + RESOURCE_TAGS
    data = [pd.Series(["a", "b", 1] + [0] * (len(RESOURCE_TAGS) - 1), index=cols)]
    df = pd.DataFrame(data)

    check_resource_tags(df)

    # Check something that should pass
    cols = ["region", "technology"] + RESOURCE_TAGS
    data = [pd.Series(["a", "b", 2] + [0] * (len(RESOURCE_TAGS) - 1), index=cols)]
    df = pd.DataFrame(data)

    check_resource_tags(df)

def test_storage_duration(caplog):
    settings = {
        "energy_storage_duration": {
            "hydroelectric pumped": 15.5,
            "batteries": {"A": 2, "B": 1, "D": 1},
        }
    }

    data = {
        "region": ["A", "A", "A", "B", "B", "B", "C"],
        "technology": ["Hydroelectric Pumped Storage", "Batteries", "Nuclear"] * 2
        + ["Batteries"],
        "Existing_Cap_MW": [1] * 7,
    }
    df = pd.DataFrame(data)

    caplog.set_level(logging.WARNING)
    df_mwh = energy_storage_mwh(
        df,
        settings["energy_storage_duration"],
        "technology",
        "Existing_Cap_MW",
        "Existing_Cap_MWh",
    )
    assert "The regions ['C'] are missing from technology batteries" in caplog.text
    assert "technology 'batteries' has the region 'D'" in caplog.text

    assert df.equals(df_mwh[df.columns])
    mwh = pd.Series([15.5, 2, 0, 15.5, 1, 0, 0])
    assert mwh.equals(df_mwh["Existing_Cap_MWh"])

def test_hydro_energy_to_power():
    settings = {
        "hydro_factor": 2,
        "regional_hydro_factor": {
            "A": 4,
            "B": 1,
        },
    }

    data = {
        "region": ["A", "A", "A", "B", "B", "B", "C"],
        "technology": ["Hydro", "NG", "Nuclear"] * 2 + ["Hydro"],
        "HYDRO": [1, 0, 0] * 2 + [1],
        "profile": [[0.5], np.nan, np.nan] * 2 + [[0.6]],
    }
    df = pd.DataFrame(data)

    df_hydro_ratio = hydro_energy_to_power(
        df, settings["hydro_factor"], settings["regional_hydro_factor"]
    )
    assert df.equals(df_hydro_ratio[df.columns])
    hydro_ratio = pd.Series([2, 0, 0, 1, 0, 0, 1.2])
    assert hydro_ratio.equals(df_hydro_ratio["Hydro_Energy_to_Power_Ratio"])

def test_efs_flex_demand():
    flex_demand = load_demand_response_efs_profile(
        resource="trans_light_duty",
        electrification_stock_fn="EFS_STOCK_AGG.parquet",
        model_year=2034,
        electrification_scenario="REFERENCE ELECTRIFICATION - MODERATE TECHNOLOGY ADVANCEMENT",
        model_regions=["WECC_AZ", "WEC_BANC", "WEC_CALN"],
        region_aggregations={"CA_N": ["WEC_BANC", "WEC_CALN"]},
        path_in=DATA_PATHS["test_data"] / "efs",
    )

    assert set(flex_demand.columns) == set(["WECC_AZ", "CA_N"])
    assert flex_demand.isnull().any().any() == False
    assert flex_demand.min().min() >= 0

    with pytest.raises(ValueError):
        load_demand_response_efs_profile(
            resource="trans_light_duty",
            electrification_stock_fn="EFS_STOCK_AGG.parquet",
            model_year=2034,
            electrification_scenario="REFERENCE ELECTRIFICATION - MODERATE TECHNOLOGY",
            model_regions=["WECC_AZ", "WEC_BANC", "WEC_CALN"],
            region_aggregations={"CA_N": ["WEC_BANC", "WEC_CALN"]},
            path_in=DATA_PATHS["test_data"] / "efs",
        )

def test_flex_resources(CA_AZ_settings):
    CA_AZ_settings["model_year"] = 2035
    CA_AZ_settings["model_first_planning_year"] = 2030
    CA_AZ_settings["electrification_stock_fn"] = "EFS_STOCK_AGG.parquet"
    CA_AZ_settings[
        "electrification_scenario"
    ] = "REFERENCE ELECTRIFICATION - MODERATE TECHNOLOGY ADVANCEMENT"
    CA_AZ_settings["flexible_demand_resources"] = {
        2035: {
            "trans_light_duty": {
                "fraction_shiftable": 0.8,
                "parameter_values": {"Max_DSM_delay": 5, "DR": 2},
            },
            "res_water_heat": {
                "fraction_shiftable": 0.1,
                "parameter_values": {"Max_DSM_delay": 2, "DR": 2},
            },
        }
    }
    CA_AZ_settings["atb_new_gen"] = []
    CA_AZ_settings["renewables_clusters"] = None
    CA_AZ_settings["additional_technologies_fn"] = None
    CA_AZ_settings["modified_atb_new_gen"] = None
    CA_AZ_settings["atb_modifiers"] = None
    CA_AZ_settings["efs_path"] = DATA_PATHS["test_data"] / "efs"

    gc = GeneratorClusters(
        pudl_engine, pudl_out, pg_engine, CA_AZ_settings, current_gens=False
    )
    flex_resources = gc.create_new_generators()

    assert len(flex_resources) == 6
    non_null_cols = ["profile", "Max_DSM_delay", "DR"]
    assert flex_resources[non_null_cols].isnull().any().any() == False

def test_usr_tx(tmp_path):
    settings = {
        "input_folder": tmp_path,
        "user_transmission_constraints_fn": "usr_tx.csv",
        "model_regions": ["A", "B", "C"],
        "tx_value_col": "nonfirm_ttc_mw",
    }

    usr_tx = pd.DataFrame(
        data={
            "region_from": ["A", "C", "C"],
            "region_to": ["B", "A", "B"],
            "nonfirm_ttc_mw": [100, 200, 300],
        }
    )
    usr_tx.to_csv(tmp_path / "usr_tx.csv", index=False)

    tx_constraints = agg_transmission_constraints(pg_engine, settings=settings)

    assert tx_constraints["Line_Max_Flow_MW"].to_list() == [100, 200, 300]

    usr_tx = pd.DataFrame(
        data={
            "region_from": ["A", "C", "C"],
            "region_to": ["B", "A", "B"],
            "firm_ttc_mw": [100, 200, 300],
        }
    )
    usr_tx.to_csv(tmp_path / "usr_tx.csv", index=False)

    settings["tx_value_col"] = "firm_ttc_mw"
    tx_constraints = agg_transmission_constraints(pg_engine, settings=settings)

    assert tx_constraints["Line_Max_Flow_MW"].to_list() == [100, 200, 300]

    usr_tx = pd.DataFrame(
        data={
            "region_from": ["A", "C", "C"],
            "region_to": ["B", "A", "B"],
            "nonfirm_ttc_mw": [100, 200, 300],
        }
    )
    usr_tx = pd.concat(
        [
            usr_tx,
            usr_tx.rename(
                columns={"region_from": "region_to", "region_to": "region_from"}
            ),
        ]
    )
    usr_tx.to_csv(tmp_path / "usr_tx.csv", index=False)

    settings["tx_value_col"] = "nonfirm_ttc_mw"
    tx_constraints = agg_transmission_constraints(pg_engine, settings=settings)

    assert tx_constraints["Line_Max_Flow_MW"].to_list() == [100, 200, 300]

    usr_tx = pd.DataFrame(
        data={
            "region_from": ["A", "C", "C", "A"],
            "region_to": ["B", "A", "B", "B"],
            "nonfirm_ttc_mw": [100, 200, 300, 50],
        }
    )
    usr_tx.to_csv(tmp_path / "usr_tx.csv", index=False)

    with pytest.raises(KeyError):
        agg_transmission_constraints(pg_engine, settings=settings)

def test_load_growth(CA_AZ_settings):
    data = {
        "year": [2019] * 3,
        "region": ["WECC_AZ"] * 3,
        "time_index": [1] * 3,
        "sector": ["commercial", "residential", "industrial"],
        "load_mw": [1, 1, 1],
    }

    load_curves = pd.DataFrame(data)

    load_curves = add_load_growth(load_curves, CA_AZ_settings)

    assert all(load_curves["load_mw"] > 1)

    data = {
        "year": [2017] * 3,
        "region": ["WECC_AZ"] * 3,
        "time_index": [1] * 3,
        "sector": ["commercial", "residential", "industrial"],
        "load_mw": [1, 1, 1],
    }

    load_curves_2 = pd.DataFrame(data)

    load_curves_2 = add_load_growth(load_curves_2, CA_AZ_settings)

    assert all(load_curves["load_mw"] < load_curves_2["load_mw"])

    data = {
        "year": [2019] * 3,
        "region": ["WECC_AZ"] * 3,
        "time_index": [1] * 3,
        "load_mw": [1, 1, 1],
    }

    load_curves_3 = pd.DataFrame(data)

    load_curves_3 = add_load_growth(load_curves_3, CA_AZ_settings)

    assert all(load_curves_3["load_mw"] > 1)
    assert load_curves_3.iloc[0, -1] == load_curves_3.iloc[1, -1]

    data = {
        "year": [2017] * 3,
        "region": ["WECC_AZ"] * 3,
        "time_index": [1] * 3,
        "load_mw": [1, 1, 1],
    }

    load_curves_4 = pd.DataFrame(data)

    load_curves_4 = add_load_growth(load_curves_4, CA_AZ_settings)

    assert load_curves_4.iloc[0, -1] == load_curves_4.iloc[1, -1]
    assert all(load_curves_3["load_mw"] < load_curves_4["load_mw"])

def test_db_col_values():
    values = db_col_values(pg_engine, "technology_costs_nrelatb", ["technology"])
    assert "NaturalGas" in values

def test_find_region_col():
    df = pd.DataFrame(columns=["A", "Region", "C"])
    region_col = find_region_col(df.columns)
    assert region_col == "Region"

    df = pd.DataFrame(columns=["A", "model_region", "C"])
    region_col = find_region_col(df.columns)
    assert region_col == "model_region"

    with pytest.raises(ValueError):
        df = pd.DataFrame(columns=["A", "model_region", "region"])
        region_col = find_region_col(df.columns)

    with pytest.raises(ValueError):
        df = pd.DataFrame(columns=["A", "B", "C"])
        region_col = find_region_col(df.columns)

def test_remove_leading_zeros():
    s = "01"
    assert remove_leading_zero(s) == "1"

    i = 1
    assert remove_leading_zero(i) == 1

    s = "GEN1"
    assert remove_leading_zero(s) == "GEN1"


import wx

class Example(wx.Frame):
    def __init__(self, parent, title):
        super(Example, self).__init__(parent, title=title)

        self.InitUI()
        self.Centre()
        self.Show()

    def InitUI(self):

        panel = wx.Panel(self)
        sizer = wx.GridBagSizer(0, 0)
        panel2 = wx.Panel(self)

        # pos=(0, 0)：放在1行1列，不设置span，默认span=(1,1)占一行一列的大小
        text = wx.StaticText(panel, label="Name:")
        sizer.Add(text, pos=(0, 0), flag=wx.ALL, border=5)

        sb = wx.StaticBox(panel, label="Optional Attributes")
        boxsizer = wx.StaticBoxSizer(sb, wx.VERTICAL)

        panel.SetSizerAndFit(sizer)

app = wx.App()
Example(None, 'Demo')
app.MainLoop()


from django.urls import path
from . import views

urlpatterns = [

     path('confirmado', views.CreateConfirmadoView.create, name='confirmado'),
     path('f_confirmados/', views.ListToFreelancerConfirmadoView.listToFreelancer,
          name='f_confirmados'),
     path('e_confirmados/', views.ListToEstabelecimentoConfirmadoView.listToEstabelecimento,
          name='e_confirmados'),
]


#-----------------------------------------------------
# Mimas: conference submission and review system
# (c) Allan Kelly 2016-2020 http://www.allankelly.net
# Licensed under MIT License, see LICENSE file
# -----------------------------------------------------

import unittest

from google.appengine.ext import ndb
from google.appengine.ext import testbed

from conference_lib import confquestion
from conference_lib import conference

class TestConferenceQuestions(unittest.TestCase):
    def setUp(self):
        self.testbed = testbed.Testbed()
        self.testbed.activate()
        self.testbed.init_datastore_v3_stub()
        self.testbed.init_memcache_stub()

    def tearDown(self):
        self.testbed.deactivate()

    def mk_conference(self):
        c = conference.Conference()
        c.name = "Allans conference_lib"
        c.shortname = "Allans"
        c.dates = "1-2 January 2017"
        c.creator_id = "Harry1"

        c.put()
        return c

    def test_conf_question_basics(self):
        c = self.mk_conference()
        q = confquestion.mk_question(c.key, "How are you?")
        self.assertEquals("How are you?", q.question_text())
        self.assertTrue(q.is_free_text())
        self.assertEquals(set(), q.answer_options())

        q.add_option("Good")
        self.assertFalse(q.is_free_text())
        self.assertSetEqual({"Good"}, q.answer_options())

        q.add_option("Bad")
        self.assertFalse(q.is_free_text())
        self.assertSetEqual({"Good", "Bad"}, q.answer_options())

    def test_retrieve_questions(self):
        c = self.mk_conference()
        questions = confquestion.retrieve_questions(c.key)
        self.assertEquals(0, len(questions))

        q = confquestion.mk_question(c.key, "How are you?")
        questions = confquestion.retrieve_questions(c.key)
        self.assertEquals(1, len(questions))
        self.assertEquals("How are you?", questions[0].question_text())

        q2 = confquestion.mk_question(c.key , "What day is it?")
        questions = confquestion.retrieve_questions(c.key)
        self.assertEquals(2, len(questions))

    def test_delete_question_answers(self):
        c = self.mk_conference()
        q = confquestion.mk_question(c.key, "How are you?")
        q.add_option("Good")
        q.add_option("Bad")

        self.assertEquals(2, len(q.answer_options()))
        q.delete_answers()
        self.assertEquals(0, len(q.answer_options()))

    def test_delete_question_and_answers(self):
        c = self.mk_conference()
        q = confquestion.mk_question(c.key, "How are you?")
        q.add_option("Good")
        q.add_option("Bad")

        self.assertEquals(1, len(confquestion.retrieve_questions(c.key)))
        confquestion.delete_question_and_answers(q)
        self.assertEquals(0, len(confquestion.retrieve_questions(c.key)))

    # see submission_lib/submission_ans.py for actual answers


import random
from typing import Union, Tuple

import cv2
import librosa
import numpy as np
from albumentations.core.transforms_interface import BasicTransform

from config import AUDIO_LEN, FREQUENCY_DISCREDITING

class AudioTransform(BasicTransform):
    """Transform for Audio task"""

    @property
    def targets(self):
        return {"data": self.apply, "image": self.apply}

    def update_params(self, params, **kwargs):
        if hasattr(self, "interpolation"):
            params["interpolation"] = self.interpolation
        if hasattr(self, "fill_value"):
            params["fill_value"] = self.fill_value
        return params

class PadTranc(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, max_ms: int = AUDIO_LEN):
        """
        This class results in the total length according to the max_ms parameter.
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param max_ms: output length of the audio signal.
        """
        super(PadTranc, self).__init__(always_apply, p)
        self.max_ms = max_ms

    def apply(self, data, **params):
        sig, sr = data
        sig_len = len(sig)
        max_len = sr // 1000 * self.max_ms

        if sig_len > max_len:
            pad_begin_len = random.randint(0, sig_len - max_len)
            return sig[pad_begin_len:pad_begin_len+max_len], sr

        elif sig_len < max_len:
            pad_begin_len = random.randint(0, max_len - sig_len)
            pad_end_len = max_len - sig_len - pad_begin_len
            pad_begin = np.random.uniform(-0.001, 0.001, pad_begin_len)
            pad_end = np.random.uniform(-0.001, 0.001, pad_end_len)
            sig = np.concatenate((pad_begin, sig, pad_end))
            return sig, sr
        else:
            return sig, sr

class Resample(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, new_sr: int = FREQUENCY_DISCREDITING):
        """
        Changing the sampling rate if it does not correspond to the desired one.
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param new_sr: new frequency of discrediting.
        """
        super(Resample, self).__init__(always_apply, p)
        self.new_sr = new_sr

    def apply(self, data, **params):
        sig, sr = data
        if sr == self.new_sr:
            return sig, sr
        resig = librosa.resample(y=sig, orig_sr=sr, target_sr=self.new_sr)
        return resig, self.new_sr

class PitchShift(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, n_steps: Union[int, None] = None):
        """
        Change the voice.
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param n_steps: a specific or random parameter for pitchShift
        """
        super(PitchShift, self).__init__(always_apply, p)
        self.n_steps = n_steps

    def apply(self, data, **params):
        if self.n_steps is None:
            self.n_steps = np.random.randint(-7, 7, dtype=int)
        wav, sr = data

        return librosa.effects.pitch_shift(wav, sr=sr, n_steps=self.n_steps), sr

class StretchAudio(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, rate: Union[float, None] = None):
        """
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param rate:
        """
        super(StretchAudio, self).__init__(always_apply, p)
        self.rate = rate

    def apply(self, data, **params):
        if self.rate is None:
            self.rate = np.random.uniform(0.8, 1.3)

        wav, sr = data
        wav = librosa.effects.time_stretch(wav, self.rate)
        return wav, sr

class FreqMask(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, F: int = 30, num_masks: int = 1,
                 replace_with_zero: bool = False):
        """
        Applying a mask to spectogram along the X axis.
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param F:
        :param num_masks: number of generated masks.
        :param replace_with_zero: change the values in the mask to zero.
        """
        super(FreqMask, self).__init__(always_apply, p)
        self.F = F
        self.num_masks = num_masks
        self.replace_with_zero = replace_with_zero

    def apply(self, image, **params):
        cloned = image.copy()
        num_mel_channels = cloned.shape[0]

        for i in range(0, self.num_masks):
            f = random.randrange(0, self.F)
            f_zero = random.randrange(0, num_mel_channels - f)

            if f_zero == f_zero + f:
                return cloned

            mask_end = random.randrange(f_zero, f_zero + f)
            if self.replace_with_zero:
                cloned[f_zero:mask_end, :] = 0
            else:
                cloned[f_zero:mask_end, :] = cloned.mean()

        return cloned

class TimeMask(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, T: int = 40, num_masks: int = 1,
                 replace_with_zero: bool = False):
        """
        Applying a mask to spectogram along the Y axis.
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param T:
        :param num_masks: number of generated masks.
        :param replace_with_zero: change the values in the mask to zero.
        """
        super(TimeMask, self).__init__(always_apply, p)
        self.T = T
        self.num_masks = num_masks
        self.replace_with_zero = replace_with_zero

    def apply(self, image, **params):
        cloned = image.copy()
        len_spectro = cloned.shape[1]

        for i in range(0, self.num_masks):
            t = random.randrange(0, self.T)
            t_zero = random.randrange(0, len_spectro - t)

            if t_zero == t_zero + t:
                return cloned

            mask_end = random.randrange(t_zero, t_zero + t)

            if self.replace_with_zero:
                cloned[:, t_zero:mask_end] = 0
            else:
                cloned[:, t_zero:mask_end] = cloned.mean()

        return cloned

class AddNoise(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5):
        """
        Adding noise ti the audio signal.
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        """
        super(AddNoise, self).__init__(always_apply, p)

    def apply(self, data, **params):
        wav, sr = data
        data_wn = wav + 0.005 * np.random.randn(len(wav))
        return data_wn, sr

class PolarityInversion(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5):
        """
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        """
        super(PolarityInversion, self).__init__(always_apply, p)

    def apply(self, data, **params):
        wav, sr = data
        return -wav, sr

class SpeedTuning(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, speed_rate: Union[Tuple, None] = None):
        """
        Audio speed change, if none, then the value changes within (0.7, 1.3)
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param speed_rate:
        """
        super(SpeedTuning, self).__init__(always_apply, p)
        self.speed_rate = speed_rate

    def apply(self, data, **params):

        if self.speed_rate is None:
            self.speed_rate = np.random.uniform(0.7, 1.3)

        wav, sr = data
        audio_speed_tune = cv2.resize(wav, (1, int(len(wav) * self.speed_rate))).squeeze()

        return audio_speed_tune, sr

class Gain(AudioTransform):
    def __init__(self, min_gain_in_db: int = -12, max_gain_in_db: int = 12, always_apply: bool = False, p: float = 0.5):
        """
        Change the volume.
        :param min_gain_in_db:
        :param max_gain_in_db:
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        """
        super(Gain, self).__init__(always_apply, p)
        assert min_gain_in_db <= max_gain_in_db
        self.min_gain_in_db = min_gain_in_db
        self.max_gain_in_db = max_gain_in_db

    def apply(self, data, **args):
        wav, sr = data
        amplitude_ratio = 10**(random.uniform(self.min_gain_in_db, self.max_gain_in_db)/20)
        return wav * amplitude_ratio, sr

class OverlayAudio(AudioTransform):
    def __init__(self, always_apply: bool = False, p: float = 0.5, intensity_overlay: Tuple = (3, 7)):
        """
        Adding noise to audio.
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        :param intensity_overlay: intensity of noise volume.
        """
        super(OverlayAudio, self).__init__(always_apply, p)
        self.intensity_overlay = intensity_overlay

    def apply(self, data, **params):

        wav_speech, sr_speech = data[0][0], data[0][1]
        wav_noise, sr_noise = data[1][0], data[1][1]

        max_speech = np.max(wav_speech)
        max_noise = np.max(wav_noise)

        min_speech = np.min(wav_speech)
        min_noise = np.min(wav_noise)

        dif_max = max_noise / max_speech
        dif_min = min_noise / min_speech

        coof = ((abs(dif_max) + abs(dif_min)) / 2) * np.random.randint(self.intensity_overlay[0],
                                                                       self.intensity_overlay[1])

        waw = (wav_speech + (wav_noise / coof)) / 2
        sr = (sr_speech + sr_noise) / 2

        return waw, int(sr)

class MakeQuiter(AudioTransform):
    def __init__(self, always_apply=False, p=0.5, quiter: Union[Tuple, None] = None):
        """
        :param always_apply: always use or not.
        :param p: the probability of use is from 0 to 1.
        """
        super(MakeQuiter, self).__init__(always_apply, p)
        self.quiter = quiter

    def apply(self, data, **params):

        if self.quiter is None:
            self.quiter = np.random.randint(2, 12)

        wav_speech, sr_speech = data
        wav_speech = wav_speech / self.quiter

        return wav_speech, sr_speech


#!/usr/bin/python
# -*- coding: utf-8 -*-
# utf-8 中文编码
import logging
from gevent import socket as _socket
import struct
import gevent
import math
from gevent.pool import Group
from handler.base import HandlerBase

# 悲剧，windows python 3.4 才支持 ipv6 的 inet_ntop
# https://bugs.python.org/issue7171
if not hasattr(_socket, 'inet_ntop'):
    import win_inet_pton

    _socket.inet_ntop = win_inet_pton.inet_ntop
if not hasattr(_socket, 'inet_pton'):
    import win_inet_pton

    _socket.inet_pton = win_inet_pton.inet_pton

class Socks5Handler(HandlerBase):
    u'''socks5 、socks4 双协议支持'''

    def __init__(self, sock,server):
        HandlerBase.__init__(self,sock,server)

    def handler(self):
        (ver,) = self.sock.unpack('B')
        self.ver = ver
        if ver == 0x04:
            self.socks4Handle()
        elif ver == 0x05:
            self.socks5Handle()
        else:
            logging.error(u'[Socks5Handler]异常的协议类型， ver = %s' % (ver))
            self.sock.close()
            return

    def socks4Handle(self):
        cd,port,dstip = self.sock.unpack('!BHI')

        if cd != 1:
            # 1 == 连接， 2 == 绑定
            self.sock.pack('!BBHI',0,91,port,dstip)
            self.sock.close()

        userid = self.sock.readline(end='\0')[:-1]

        hostname = _socket.inet_ntoa(struct.pack('!I', dstip))

        if hostname.startswith('0.0.0.'):
            # SOCKS4a 支持 ，chrome 不支持 socks4a 扩展。
            hostname = self.sock.readline(end='\0')[:-1]

        # 对外发起请求
        try:
            remote_sock = self.server.upstream.create_connection((hostname,port),)
        except:
            logging.exception(u'所有线路连接 tcp host:%s port:%s 失败。'%(hostname,port))
            # 按照socks4协议，这里返回和请求相同的ip和端口
            # https://zh.wikipedia.org/wiki/SOCKS
            self.sock.pack('!BBHI',0,91,port,dstip)
            self.sock.close()
            return

        # 按照socks4协议，这里返回和请求相同的ip和端口
        # https://zh.wikipedia.org/wiki/SOCKS
        self.sock.pack('!BBHI',0,90,port,dstip)

        self.forward(self.sock,remote_sock,5*60)

    def socks5Handle(self):
        # 鉴定
        (nmethods,) = self.sock.unpack('B')
        if nmethods > 0:
            (methods,) = self.sock.unpack('B' * nmethods)
            # TODO: 未检查客户端支持的鉴定方式
        self.sock.pack('BB', 0x05, 0x00)
        logging.debug(u'[Socks5Handler]client login')

        # 接收代理请求
        ver, cmd, rsv, atyp = self.sock.unpack('BBBB')

        if ver != 0x05 or cmd != 0x01:
            logging.warn(u'[Socks5Handler]收到未知类型的请求，关闭连接。 ver=%s ,cmd=%s' % (ver, cmd))
            self.sock.pack('BBBBIH', 0x05, 0x07, 0x00, 0x01, 0, 0)
            self.sock.close()
            return

        if atyp == 0x01:
            # ipv4
            host, port = self.sock.unpack('!IH')
            hostname = _socket.inet_ntoa(struct.pack('!I', host))
        elif atyp == 0x03:
            # 域名
            length = self.sock.unpack('B')[0]
            hostname, port = self.sock.unpack("!%dsH" % length)
        elif atyp == 0x04:
            # ipv6
            ipv61, ipv62, port = self.sock.unpack('!2QH')
            hostname = _socket.inet_ntop(_socket.AF_INET6, struct.pack('!2Q', ipv61, ipv62))
        else:
            logging.warn(u'[SClient]收到未知的目的地址类型，关闭连接。 atyp=%s ' % (atyp))
            self.sock.pack('!BBBBIH', 0x05, 0x07, 0x00, 0x01, 0, 0)
            self.sock.close()
            return
        logging.debug(u'[SClient] host:%s   prot:%s' % (hostname, port))

        # 对外发起请求
        try:
            remote_sock = self.server.upstream.create_connection((hostname,port),)
        except:
            logging.exception(u'所有线路连接 tcp host:%s port:%s 失败。'%(hostname,port))
            # TODO: 按照socks5协议，这里应该返回服务器绑定的地址及端口
            # http://blog.csdn.net/testcs_dn/article/details/7915505
            self.sock.pack('!BBBBIH', 0x05, 0x03, 0x00, 0x01, 0, 0)
            self.sock.close()
            return

        if hasattr(remote_sock,'display_name'):
            remote_sock_display_name = remote_sock.display_name
        else:
            remote_sock_display_name = str(remote_sock)

        logging.info(u'连接 tcp目标 host:%s port:%s 通过 %s 建立成功。'%(hostname,port,remote_sock_display_name))

        # TODO: 按照socks5协议，这里应该返回服务器绑定的地址及端口
        # http://blog.csdn.net/testcs_dn/article/details/7915505
        self.sock.pack('!BBBBIH', 0x05, 0x00, 0x00, 0x01, 0, 0)

        self.forward(self.sock,remote_sock,5*60)

    @staticmethod
    def create(sock,server):
        u'''创建handler
如果是本类可处理的协议返回本类实例，否则返回None
'''
        (ver,) = sock.unpack('B')
        if ver in (0x04, 0x05):
            logging.debug(u'[Socks5Handler]收到 socks%s 协议头。' % ver)
            return (Socks5Handler(sock,server), True)
        return (None, True)


import time
import torch
import numpy as np
from copy import deepcopy
from argparse import ArgumentParser
from torch.utils.data import DataLoader

from .incremental_learning import Inc_Learning_Appr
from datasets.exemplars_dataset import ExemplarsDataset

class Appr(Inc_Learning_Appr):
    """Class implementing the Bias Correction (BiC) approach described in
    http://openaccess.thecvf.com/content_CVPR_2019/papers/Wu_Large_Scale_Incremental_Learning_CVPR_2019_paper.pdf
    Original code available at https://github.com/wuyuebupt/LargeScaleIncrementalLearning
    """

    def __init__(self, model, device, nepochs=250, lr=0.1, lr_min=1e-5, lr_factor=3, lr_patience=5, clipgrad=10000,
                 momentum=0.9, wd=0.0002, multi_softmax=False, wu_nepochs=0, wu_lr_factor=1, fix_bn=False,
                 eval_on_train=False, logger=None, exemplars_dataset=None, val_exemplar_percentage=0.1,
                 num_bias_epochs=200, T=2, lamb=-1):
        # Sec. 6.1. CIFAR-100: 2,000 exemplars, ImageNet-1000: 20,000 exemplars, Celeb-10000: 50,000 exemplars
        # Sec. 6.2. weight decay for CIFAR-100 is 0.0002, for ImageNet-1000 and Celeb-10000 is 0.0001
        super(Appr, self).__init__(model, device, nepochs, lr, lr_min, lr_factor, lr_patience, clipgrad, momentum, wd,
                                   multi_softmax, wu_nepochs, wu_lr_factor, fix_bn, eval_on_train, logger,
                                   exemplars_dataset)
        self.val_percentage = val_exemplar_percentage
        self.bias_epochs = num_bias_epochs
        self.model_old = None
        self.T = T
        self.lamb = lamb
        self.bias_layers = []

        self.x_valid_exemplars = []
        self.y_valid_exemplars = []

        if self.exemplars_dataset.max_num_exemplars != 0:
            self.num_exemplars = self.exemplars_dataset.max_num_exemplars
        elif self.exemplars_dataset.max_num_exemplars_per_class != 0:
            self.num_exemplars_per_class = self.exemplars_dataset.max_num_exemplars_per_class

        have_exemplars = self.exemplars_dataset.max_num_exemplars + self.exemplars_dataset.max_num_exemplars_per_class
        assert (have_exemplars > 0), 'Error: BiC needs exemplars.'

    @staticmethod
    def exemplars_dataset_class():
        return ExemplarsDataset

    @staticmethod
    def extra_parser(args):
        """Returns a parser containing the approach specific parameters"""
        parser = ArgumentParser()
        # Sec. 3. "lambda is set to n / (n+m)" where n=num_old_classes and m=num_new_classes - so lambda is not a param
        # To use the original, set lamb=-1, otherwise, we allow to use specific lambda for the distillation loss
        parser.add_argument('--lamb', default=-1, type=float, required=False,
                            help='Forgetting-intransigence trade-off (default=%(default)s)')
        # Sec. 6.2. "The temperature scalar T in Eq. 1 is set to 2 by following [13,2]."
        parser.add_argument('--T', default=2, type=int, required=False,
                            help='Temperature scaling (default=%(default)s)')
        # Sec. 6.1. "the ratio of train/validation split on the exemplars is 9:1 for CIFAR-100 and ImageNet-1000"
        parser.add_argument('--val-exemplar-percentage', default=0.1, type=float, required=False,
                            help='Percentage of exemplars that will be used for validation (default=%(default)s)')
        # In the original code they define epochs_per_eval=100 and epoch_val_times=2, making a total of 200 bias epochs
        parser.add_argument('--num-bias-epochs', default=200, type=int, required=False,
                            help='Number of epochs for training bias (default=%(default)s)')
        return parser.parse_known_args(args)

    def bias_forward(self, outputs):
        """Utility function --- inspired by https://github.com/sairin1202/BIC"""
        bic_outputs = []
        for m in range(len(outputs)):
            bic_outputs.append(self.bias_layers[m](outputs[m]))
        return bic_outputs

    def train_loop(self, t, trn_loader, val_loader):
        """Contains the epochs loop
        Some parts could go into self.pre_train_process() or self.post_train_process(), but we leave it for readability
        """
        # add a bias layer for the new classes
        self.bias_layers.append(BiasLayer().to(self.device))

        # STAGE 0: EXEMPLAR MANAGEMENT -- select subset of validation to use in Stage 2 -- val_old, val_new (Fig.2)
        print('Stage 0: Select exemplars from validation')
        clock0 = time.time()

        # number of classes and proto samples per class
        num_cls = sum(self.model.task_cls)
        num_old_cls = sum(self.model.task_cls[:t])
        if self.exemplars_dataset.max_num_exemplars != 0:
            num_exemplars_per_class = int(np.floor(self.num_exemplars / num_cls))
            num_val_ex_cls = int(np.ceil(self.val_percentage * num_exemplars_per_class))
            num_trn_ex_cls = num_exemplars_per_class - num_val_ex_cls
            # Reset max_num_exemplars
            self.exemplars_dataset.max_num_exemplars = (num_trn_ex_cls * num_cls).item()
        elif self.exemplars_dataset.max_num_exemplars_per_class != 0:
            num_val_ex_cls = int(np.ceil(self.val_percentage * self.num_exemplars_per_class))
            num_trn_ex_cls = self.num_exemplars_per_class - num_val_ex_cls
            # Reset max_num_exemplars
            self.exemplars_dataset.max_num_exemplars_per_class = num_trn_ex_cls

        # Remove extra exemplars from previous classes -- val_old
        if t > 0:
            if self.exemplars_dataset.max_num_exemplars != 0:
                num_exemplars_per_class = int(np.floor(self.num_exemplars / num_old_cls))
                num_old_ex_cls = int(np.ceil(self.val_percentage * num_exemplars_per_class))
                for cls in range(num_old_cls):
                    assert (len(self.y_valid_exemplars[cls]) == num_old_ex_cls)
                    self.x_valid_exemplars[cls] = self.x_valid_exemplars[cls][:num_val_ex_cls]
                    self.y_valid_exemplars[cls] = self.y_valid_exemplars[cls][:num_val_ex_cls]

        # Add new exemplars for current classes -- val_new
        non_selected = []
        for curr_cls in range(num_old_cls, num_cls):
            self.x_valid_exemplars.append([])
            self.y_valid_exemplars.append([])
            # get all indices from current class
            cls_ind = np.where(np.asarray(trn_loader.dataset.labels) == curr_cls)[0]
            assert (len(cls_ind) > 0), "No samples to choose from for class {:d}".format(curr_cls)
            assert (num_val_ex_cls <= len(cls_ind)), "Not enough samples to store for class {:d}".format(curr_cls)
            # add samples to the exemplar list
            self.x_valid_exemplars[curr_cls] = [trn_loader.dataset.images[idx] for idx in cls_ind[:num_val_ex_cls]]
            self.y_valid_exemplars[curr_cls] = [trn_loader.dataset.labels[idx] for idx in cls_ind[:num_val_ex_cls]]
            non_selected.extend(cls_ind[num_val_ex_cls:])
        # remove selected samples from the validation data used during training
        trn_loader.dataset.images = [trn_loader.dataset.images[idx] for idx in non_selected]
        trn_loader.dataset.labels = [trn_loader.dataset.labels[idx] for idx in non_selected]
        clock1 = time.time()
        print(' > Selected {:d} validation exemplars, time={:5.1f}s'.format(
            sum([len(elem) for elem in self.y_valid_exemplars]), clock1 - clock0))

        # make copy to keep the type of dataset for Stage 2 -- not efficient
        bic_val_dataset = deepcopy(trn_loader.dataset)
        # add exemplars to train_loader -- train_new + train_old (Fig.2)
        if t > 0:
            trn_loader = torch.utils.data.DataLoader(trn_loader.dataset + self.exemplars_dataset,
                                                     batch_size=trn_loader.batch_size,
                                                     shuffle=True,
                                                     num_workers=trn_loader.num_workers,
                                                     pin_memory=trn_loader.pin_memory)

        # STAGE 1: DISTILLATION
        print('Stage 1: Training model with distillation')
        super().train_loop(t, trn_loader, val_loader)
        # From LwF: Restore best and save model for future tasks
        self.model_old = deepcopy(self.model)
        self.model_old.eval()
        self.model_old.freeze_all()

        # STAGE 2: BIAS CORRECTION
        if t > 0:
            print('Stage 2: Training bias correction layers')
            # Fill bic_val_loader with validation protoset
            if isinstance(bic_val_dataset.images, list):
                bic_val_dataset.images = sum(self.x_valid_exemplars, [])
            else:
                bic_val_dataset.images = np.vstack(self.x_valid_exemplars)
            bic_val_dataset.labels = sum(self.y_valid_exemplars, [])
            bic_val_loader = DataLoader(bic_val_dataset, batch_size=trn_loader.batch_size, shuffle=True,
                                        num_workers=trn_loader.num_workers, pin_memory=trn_loader.pin_memory)

            # bias optimization on validation
            self.model.eval()
            # Allow to learn the alpha and beta for the current task
            self.bias_layers[t].alpha.requires_grad = True
            self.bias_layers[t].beta.requires_grad = True

            # In their code is specified that momentum is always 0.9
            bic_optimizer = torch.optim.SGD(self.bias_layers[t].parameters(), lr=self.lr, momentum=0.9)
            # Loop epochs
            for e in range(self.bias_epochs):
                # Train bias correction layers
                clock0 = time.time()
                total_loss, total_acc = 0, 0
                for inputs, targets in bic_val_loader:
                    # Forward current model
                    with torch.no_grad():
                        outputs = self.model(inputs.to(self.device))
                        old_cls_outs = self.bias_forward(outputs[:t])
                    new_cls_outs = self.bias_layers[t](outputs[t])
                    pred_all_classes = torch.cat([torch.cat(old_cls_outs, dim=1), new_cls_outs], dim=1)
                    # Eqs. 4-5: outputs from previous tasks are not modified (any alpha or beta from those is fixed),
                    #           only alpha and beta from the new task is learned. No temperature scaling used.
                    loss = torch.nn.functional.cross_entropy(pred_all_classes, targets.to(self.device))
                    # However, in their code, they apply a 0.1 * L2-loss to the gamma variable (beta in the paper)
                    loss += 0.1 * ((self.bias_layers[t].beta[0] ** 2) / 2)
                    # Log
                    total_loss += loss.item() * len(targets)
                    total_acc += ((pred_all_classes.argmax(1) == targets.to(self.device)).float()).sum().item()
                    # Backward
                    bic_optimizer.zero_grad()
                    loss.backward()
                    bic_optimizer.step()
                clock1 = time.time()
                # reducing the amount of verbose
                if (e + 1) % (self.bias_epochs / 4) == 0:
                    print('| Epoch {:3d}, time={:5.1f}s | Train: loss={:.3f}, TAg acc={:5.1f}% |'.format(
                          e + 1, clock1 - clock0, total_loss / len(bic_val_loader.dataset.labels),
                          100 * total_acc / len(bic_val_loader.dataset.labels)))
            # Fix alpha and beta after learning them
            self.bias_layers[t].alpha.requires_grad = False
            self.bias_layers[t].beta.requires_grad = False

        # Print all alpha and beta values
        for task in range(t + 1):
            print('Stage 2: BiC training for Task {:d}: alpha={:.5f}, beta={:.5f}'.format(task,
                  self.bias_layers[task].alpha.item(), self.bias_layers[task].beta.item()))

        # STAGE 3: EXEMPLAR MANAGEMENT
        self.exemplars_dataset.collect_exemplars(self.model, trn_loader, val_loader.dataset.transform)

    def train_epoch(self, t, trn_loader):
        """Runs a single epoch"""
        self.model.train()
        if self.fix_bn and t > 0:
            self.model.freeze_bn()
        for images, targets in trn_loader:
            # Forward old model
            targets_old = None
            if t > 0:
                targets_old = self.model_old(images.to(self.device))
                targets_old = self.bias_forward(targets_old)  # apply bias correction
            # Forward current model
            outputs = self.model(images.to(self.device))
            outputs = self.bias_forward(outputs)  # apply bias correction
            loss = self.criterion(t, outputs, targets.to(self.device), targets_old)
            # Backward
            self.optimizer.zero_grad()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clipgrad)
            self.optimizer.step()

    def eval(self, t, val_loader):
        """Contains the evaluation code"""
        with torch.no_grad():
            total_loss, total_acc_taw, total_acc_tag, total_num = 0, 0, 0, 0
            self.model.eval()
            for images, targets in val_loader:
                # Forward old model
                targets_old = None
                if t > 0:
                    targets_old = self.model_old(images.to(self.device))
                    targets_old = self.bias_forward(targets_old)  # apply bias correction
                # Forward current model
                outputs = self.model(images.to(self.device))
                outputs = self.bias_forward(outputs)  # apply bias correction
                loss = self.criterion(t, outputs, targets.to(self.device), targets_old)
                hits_taw, hits_tag = self.calculate_metrics(outputs, targets)
                # Log
                total_loss += loss.item() * len(targets)
                total_acc_taw += hits_taw.sum().item()
                total_acc_tag += hits_tag.sum().item()
                total_num += len(targets)
        return total_loss / total_num, total_acc_taw / total_num, total_acc_tag / total_num

    def cross_entropy(self, outputs, targets, exp=1.0, size_average=True, eps=1e-5):
        """Calculates cross-entropy with temperature scaling"""
        out = torch.nn.functional.softmax(outputs, dim=1)
        tar = torch.nn.functional.softmax(targets, dim=1)
        if exp != 1:
            out = out.pow(exp)
            out = out / out.sum(1).view(-1, 1).expand_as(out)
            tar = tar.pow(exp)
            tar = tar / tar.sum(1).view(-1, 1).expand_as(tar)
        out = out + eps / out.size(1)
        out = out / out.sum(1).view(-1, 1).expand_as(out)
        ce = -(tar * out.log()).sum(1)
        if size_average:
            ce = ce.mean()
        return ce

    def criterion(self, t, outputs, targets, targets_old):
        """Returns the loss value"""

        # Knowledge distillation loss for all previous tasks
        loss_dist = 0
        if t > 0:
            loss_dist += self.cross_entropy(torch.cat(outputs[:t], dim=1),
                                            torch.cat(targets_old[:t], dim=1), exp=1.0 / self.T)
        # trade-off - the lambda from the paper if lamb=-1
        if self.lamb == -1:
            lamb = (self.model.task_cls[:t].sum().float() / self.model.task_cls.sum()).to(self.device)
            return (1.0 - lamb) * torch.nn.functional.cross_entropy(torch.cat(outputs, dim=1),
                                                                    targets) + lamb * loss_dist
        else:
            return torch.nn.functional.cross_entropy(torch.cat(outputs, dim=1), targets) + self.lamb * loss_dist

class BiasLayer(torch.nn.Module):
    """Bias layers with alpha and beta parameters"""

    def __init__(self):
        super(BiasLayer, self).__init__()
        # Initialize alpha and beta with requires_grad=False and only set to True during Stage 2
        self.alpha = torch.nn.Parameter(torch.ones(1, requires_grad=False, device="cuda"))
        self.beta = torch.nn.Parameter(torch.zeros(1, requires_grad=False, device="cuda"))

    def forward(self, x):
        return self.alpha * x + self.beta


import sys,os
from random import randint
plat = sys.platform
try:
    import PySimpleGUI as sg
except:
    if plat == 'win32':
        os.system('pip install pysimplegui')
        import PySimpleGUI as sg
    elif plat == 'linux':
        os.system('pip3 install pysimplegui')
        import PySimpleGUI as sg
aleatorio = randint(0,100)

class telapython:
  def __init__(self):
    sg.change_look_and_feel('DarkTeal7')
    layout = [
      [sg.Text('Chute um numero de 0 a 100',size=(12,0)),sg.Input(size=(10,10),key='numero')],
      [sg.Button('enviar')],
      [sg.Output(size=(30,10))]

    ]
    self.janela = sg.Window('Dados do usuario').layout(layout)

  def iniciar(self):
    while True:
      self.Button, self.values = self.janela.Read()
      num = int(self.values['numero'])
      if aleatorio == num:
          print('Parabens, voce acertou!!')
      else:
          if num > aleatorio:
              print()
              print('Chute um numero menor')
          else:
              print()
              print('Chute um valor maior')

tela = telapython()
tela.iniciar()


def give_working_day():
    # prints the nearest working day date
    from datetime import date
    from datetime import timedelta

    day = date.today()
    # day = date(2017, 9, 30)

    if day.weekday() == 5:
        working_day = day + timedelta(days=2)

    elif day.weekday() == 6:
        working_day = day + timedelta(days=1)

    else:
        working_day = day
    print('Working day for', day, 'is', working_day)
    return

give_working_day()
print('-----------------------------------------------------------')

#  Laboratory

def days_to_new_year_eve():
    # This functions is for counting days left to New Year's Eve
    from datetime import date

    days_to_left = date(2020, 12, 31) - date.today()
    print(days_to_left.days)

    return

days_to_new_year_eve()


"""
simple log parser like logstash
"""
import threading
import logging
import os
import re
import enum
from datetime import datetime
from queue import Queue, Empty
from time import sleep
from glob import glob
from weakref import WeakValueDictionary
from contextlib import contextmanager

from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine, Column, String, Integer, DateTime, Enum
from sqlalchemy.ext.declarative import declarative_base

from .outputs import *

__version__ = '1.0.0'
__author__ = 'thuhak.zhou@nio.com'
Base = declarative_base()

class LogStat(enum.Enum):
    ready = 0
    running = 1
    file_deleted = 2
    parse_error = 3
    no_update = 4
    read_file_error = 5

class Logfile(Base):
    __tablename__ = 'log'

    filename = Column(String(256), primary_key=True)
    inode = Column(Integer, nullable=False)
    pos = Column(Integer, nullable=False, default=0)
    last_update = Column(DateTime, nullable=True)
    status = Column(Enum(LogStat), default=LogStat.ready)

@contextmanager
def session_scope(DBSession):
    session = DBSession()
    try:
        yield session
        session.commit()
    except Exception as e:
        logging.error('db commit error, rolling back')
        session.rollback()
        raise e
    finally:
        session.close()

def is_thread_running(thread: threading.Thread) -> bool:
    return thread.is_alive() if thread is not None else False

class LogParser:
    """
    logfile parser
    """
    _instances = WeakValueDictionary()

    def __new__(cls, filename, *args, **kwargs):
        """flyweight"""
        if filename in cls._instances:
            instance = cls._instances[filename]
        else:
            instance = super().__new__(cls)
            cls._instances[filename] = instance
        return instance

    def __init__(self, filename: str, que: Queue, dbsession, encoding='utf-8', flush_interval=3, stop_time=30):
        """
        :param filename: path of logfile
        :param que: parsed result will be pushed to this queue
        :param encoding: encoding of logfile
        :param dbsession: database session
        :param flush_interval: every flush_interval seconds, latest running will be saved to database
        :param stop_time: stop parser when file not changed with stop_time seconds
        """
        self.filename = filename
        self.inode = os.stat(filename).st_ino
        self.dbsession = dbsession
        with session_scope(self.dbsession) as sess:
            data = sess.query(Logfile).filter_by(filename=filename).first()
            if data:
                if self.inode != data.inode:
                    data.inode = self.inode
                    data.pos = self.pos = 0
                    data.last_update = self.last_update = None
                else:
                    self.pos = data.pos
                    self.last_update = data.last_update
            else:
                self.pos = 0
                self.last_update = None
                data = Logfile(filename=filename, inode=self.inode)
                sess.add(data)
            self._data = data
        self.que = que
        self.flush_interval = flush_interval
        self.stop_time = stop_time
        self.encoding = encoding

        self.running = False
        self.status = LogStat.ready
        self.context = None
        self._working_thread = None
        self._flush_thread = None
        self._flag = threading.Event()
        self.start()

    def parser(self, line):
        """
        default parser, you can define you own parser, override the default one
        """
        return line

    def start(self):
        """start parser"""
        logging.info('starting logfile parser')
        self._working_thread = threading.Thread(target=self.run, daemon=True)
        self._working_thread.start()
        self._flush_thread = threading.Thread(target=self.flush, daemon=True)
        self._flush_thread.start()

    def stop(self):
        """stop parser"""
        logging.info('stopping logfile parser')
        self._flag.set()
        while True:
            if not is_thread_running(self._flush_thread) and not is_thread_running(self._working_thread):
                self.running = False
                break
            sleep(0.1)

    def _flush(self):
        """
        flush status to database
        """
        try:
            with session_scope(self.dbsession) as sess:
                self._data.pos = self.pos
                self._data.inode = self.inode
                self._data.last_update = datetime.now()
                self._data.status = self.status
                sess.merge(self._data)
        except Exception as e:
            logging.error('can not save parser status to database, reason {}'.format(str(e)))

    def flush(self):
        """
        run _flush job every self.flush_interval seconds
        """
        while True:
            if not is_thread_running(self._working_thread):
                self._flush()
                self.running = False
                break
            elif self._flag.is_set():
                sleep(0.1)
                continue
            else:
                sleep(self.flush_interval)
                self._flush()

    def run(self):
        """
        run parser
        """
        try:
            stream = open(self.filename, 'r', encoding=self.encoding)
            stream.seek(self.pos)
            self.running = True
        except Exception as e:
            logging.error('can not open file {}, reason: {}'.format(self.filename, str(e)))
            self.status = LogStat.read_file_error
            return
        while True:
            if self._flag.is_set():
                break
            line = stream.readline().strip()
            if line:
                try:
                    result = self.parser(line)
                except Exception as e:
                    logging.error('parser error for line {}, error is {}'.format(line, str(e)))
                    break
                else:
                    if result:
                        self.que.put(result)
                    self.status = LogStat.running
                    self.pos = stream.tell()
                    self.last_update = datetime.now()
            else:
                if not os.path.exists(self.filename):
                    logging.info('log file {} no longer exists'.format(self.filename))
                    self.status = LogStat.file_deleted
                    break
                else:
                    stat = os.stat(self.filename)
                    inode, mtime = stat.st_ino, stat.st_mtime
                    if inode != self.inode:
                        stream.close()
                        self.pos = 0
                        self.last_update = None
                        self.inode = inode
                        self.status = LogStat.ready
                        try:
                            stream = open(self.filename, 'r', encoding=self.encoding)
                        except Exception as e:
                            logging.error('can not open file {}, reason: {}'.format(self.filename, str(e)))
                            self.status = LogStat.read_file_error
                            raise e
                    elif self.last_update and \
                            self.last_update.timestamp() - mtime >= self.stop_time:
                        logging.info('{} has no changing over {} seconds,stop parsing'.format(self.filename,
                                                                                self.stop_time))
                        self.status = LogStat.no_update
                        break
                    else:
                        sleep(1)
        stream.close()

class LogSlice:
    """
    log parser manager
    """

    def __init__(self, path, output, dbpath='/var/lib/logslice', parser=LogParser, file_filter=None,
                 encoding='utf-8', flush_interval=3, rescan_interval=30, close_file_time=30,
                 cache_size=10):
        """
        :param path: search log files using globbing patterns
        :param output: callable object for output
        :param dbpath: path of database
        :param parser: parser class
        :param file_filter: regex filter
        :param encoding: file encoding
        :param flush_interval: every flush_interval seconds, latest running will be saved to database
        :param rescan_interval: every rescan_interval seconds, rescan logs
        :param close_file_time: close file which not changed within close_file_time seconds
        :param cache_size: a iterable object for output
        """
        if isinstance(path, str):
            self.path = [path]
        else:
            self.path = path
        self.output_callback = output
        self.parser = parser
        self.filter = None if file_filter is None else re.compile(file_filter)
        self.encoding = encoding
        self.flush_interval = flush_interval
        self.rescan_interval = rescan_interval
        self.close_file_time = close_file_time
        self.cache_size = cache_size

        if not os.path.exists(dbpath):
            os.makedirs(dbpath)
        dbpath = os.path.join(dbpath, 'logslice.db')
        engine = create_engine('sqlite:///{}'.format(dbpath), echo=False)
        self.dbsession = sessionmaker(bind=engine)
        Base.metadata.create_all(engine)

        self.que = Queue()
        self.jobs = []
        self._stop_input_flag = threading.Event()
        self._stop_output_flag = threading.Event()
        self.input_thread = None
        self.output_thread = None

    def scan_logs(self) -> set:
        """
        search logs in path
        :return: all log files
        """
        all_logs = set()
        for logpath in self.path:
            logs = set(glob(logpath, recursive=True))
            if self.filter:
                logs = {x for x in logs if self.filter.match(x)}
            all_logs |= logs
        return all_logs

    def start(self):
        self.input_thread = threading.Thread(target=self.scan, daemon=True)
        self.input_thread.start()
        self.output_thread = threading.Thread(target=self.output, daemon=True)
        self.output_thread.start()
        while True:
            try:
                if is_thread_running(self.input_thread) and is_thread_running(self.output_thread):
                    continue
            except KeyboardInterrupt:
                self.stop()
                break

    def stop(self):
        self._stop_input_flag.set()
        logging.info('stopping output')
        for job in self.jobs:
            job.stop()
        sleep(0.5)
        while True:
            self.clean_stopped_jobs()
            if not self.jobs:
                self._stop_output_flag.set()
                break
            else:
                logging.info('some job still working {}'.format([j.filename for j in self.jobs]))
                sleep(0.5)

    def clean_stopped_jobs(self):
        for job in self.jobs[:]:
            if not job.running:
                self.jobs.remove(job)

    def cache(self):
        for _ in range(self.cache_size):
            try:
                yield self.que.get(timeout=1)
            except Empty:
                break

    def output(self):
        while True:
            if self._stop_output_flag.is_set():
                logging.info('stop out put')
                break
            else:
                try:
                    self.output_callback(self.cache())
                except Exception as e:
                    logging.error('output error, reason: {}'.format(str(e)))

    def scan(self):
        while True:
            if self._stop_input_flag.is_set():
                break
            self._scan()
            sleep(self.rescan_interval)

    def _scan(self):
        self.clean_stopped_jobs()
        logs = self.scan_logs()
        no_update_logs = set()
        with session_scope(self.dbsession) as sess:
            check_logs = sess.query(Logfile).filter(Logfile.status == LogStat.no_update, Logfile.filename.in_(logs))
            for log in check_logs:
                try:
                    mtime = os.stat(log.filename).st_mtime
                    if mtime < log.last_update.timestamp():
                        no_update_logs.add(log.filename)
                except Exception as e:
                    logging.error('read file {} error, reason {}'.format(log.filename, str(e)))
                    continue
        jobs = {j.filename for j in self.jobs}
        logs = logs - no_update_logs - jobs
        for log in logs:
            self.jobs.append(self.parser(filename=log, que=self.que, dbsession=self.dbsession, encoding=self.encoding,
                                         flush_interval=self.flush_interval, stop_time=self.close_file_time))


# conda create -n chatvenv python=3.6.2
# pip install nltk
# numpy 1.14.0
# pip install tensorflow==1.4
# pip install tflearn==0.3.2
# pip install scipy==1.4.1
# pip install h5py==2.8.0rc1
# pip install windows-curses==1.0 (if you're on windows)

import nltk
from nltk import word_tokenize
from nltk.stem.lancaster import LancasterStemmer
import numpy as np
import tflearn
import tensorflow
import random
import json
import string
import pickle
from gtts import gTTS
import playsound
import pyttsx3
import os
from datetime import datetime
import time

def speak(text):
    engine = pyttsx3.init()
    engine.setProperty('rate', 125)
    volume = engine.getProperty('volume')
    engine.setProperty('volume',1.0)
    engine.say(text)
    engine.runAndWait()
    engine.stop()

def speak2(text):
    tts = gTTS(text=text, lang='ro')
    date_string = datetime.now().strftime("%d%m%Y%H%M%S")
    filename = "voice"+date_string+".mp3"
    #filename = "voice.mp3"
    tts.save(filename)
    playsound.playsound(filename)
    time.sleep(2)
    os.remove(filename)

stemmer = LancasterStemmer()

with open("intents.json") as d:
    data = json.load(d)

#print(data['intents']) # list of dictionaries
try:
    with open("data.pickle", "rb") as f:
        words, labels, training, output = pickle.load(f)
except:
    words = []
    labels = []
    docs_x = []
    docs_y = []
    ########################### testing word_tokenize ###########
    # asd = "how are you?"
    # w = word_tokenize(asd) #['how', 'are', 'you', '?']
    # print(w)
    ##############################################################

    for intent in data['intents']:
        for pattern in intent["patterns"]:
            wrds = nltk.word_tokenize(pattern)
            words.extend(wrds)
            docs_x.append(wrds)
            docs_y.append(intent["tag"])

        if intent["tag"] not in labels:
            labels.append(intent["tag"])

    #### stemmer the words (punctuation) ########
    words = [stemmer.stem(w.lower()) for w in words if w != "?"]
    #words = [char.lower() for char in words if char not in string.punctuation]

    #  and duplicates
    words = sorted(list(set(words)))
    # #################################################
    labels = sorted(labels)

    # ########### Training ################
    # # Bag of words
    training = []
    output = []

    out_empty = [0 for _ in range(len(labels))]

    for x, doc in enumerate(docs_x):
        bag = []
        # no punctuation from docs_x
        #wrds = [char.lower() for char in doc if char not in string.punctuation]
        wrds = [stemmer.stem(w) for w in doc]

        for w in words:
            if w in wrds:
                bag.append(1)
            else:
                bag.append(0)
        output_row = out_empty[:]
        output_row[labels.index(docs_y[x])] = 1

        training.append(bag)
        output.append(output_row)

    #print(training)
    training = np.array(training)
    output = np.array(output)

    with open("data.pickle", "wb") as f:
        pickle.dump((words, labels, training, output), f)

# Learning Model A.I
tensorflow.reset_default_graph() # doesnt matter
net = tflearn.input_data(shape=[None, len(training[0])])
net = tflearn.fully_connected(net, 8) # hidden layer 8 neurons
net = tflearn.fully_connected(net, 8) # hidden layer 8 neurons
net = tflearn.fully_connected(net, len(output[0]), activation="softmax")
net = tflearn.regression(net)

model = tflearn.DNN(net)
try:
    model.load("model.tflearn")
except:
    model.fit(training, output, n_epoch=1000, batch_size=8, show_metric=True)
    model.save("model.tflearn")

def bag_of_words(s, words):
    bag = [0 for _ in range(len(words))]
    s_words = nltk.word_tokenize(s)
    s_words = [stemmer.stem(word.lower()) for word in s_words]

    for se in s_words:
        for i,w in enumerate(words):
            if w == se:
                bag[i] = 1
    return np.array(bag)

def chat():
    print("Start talking with the bot")
    speak("Start talking with the bot")
    while True:
        inp = input("You: ")
        if inp.lower() == "quit":
            break

        results = model.predict([bag_of_words(inp,words)])[0]
        #print(results)
        results_index = np.argmax(results) # pick the biggest procent
        #print(results[results_index]) # 0.934
        tag = labels[results_index]
        #print(tag)
        if results[results_index] > 0.7:
            for tg in data["intents"]:
                if tg["tag"] == tag:
                    responses = tg["responses"]
            v = random.choice(responses)
            print(v)
            speak(v)
        else:
            print("I didn't understand. Please try again!")
            speak("I didn't understand. Please try again!")

chat()