import json
import re
import time
import requests
import telethon
import boto3
from boto3.dynamodb.conditions import Attr
from telethon import TelegramClient, events
with open('config.json', encoding='UTF-8') as json_file:
    config = json.load(json_file)
channel_list = config['channel_list']
api_id = config["api_id"]
api_hash = config["api_hash"]
client = TelegramClient('main', api_id, api_hash)
aws_access_key_id = config['aws_access_key_id']
aws_secret_access_key = config['aws_secret_access_key']
region_name = config['region_name']
table_name = config['table_name']
dynamodb = boto3.resource('dynamodb',
                          aws_access_key_id=aws_access_key_id,
                          aws_secret_access_key=aws_secret_access_key,
                          region_name=region_name)
table = dynamodb.Table(table_name)
def push_info_to_db(id_time, channel_id, title, pair_name, address, chain, tbl):
    tbl.put_item(
        Item={
            'id': str(id_time),
            'channel_id': channel_id,
            'channel_name': title,
            'pair_name': pair_name,
            'contract': address,
            'chain': chain
        }
    )
def check_and_delete(channel_id, chain):
    try:
        response = table.scan(FilterExpression=Attr('channel_id').eq(channel_id)&Attr('chain').eq(chain))
    except:
        return
    result = sorted(response['Items'], key=lambda x: x['id'])
    for i in range(len(result) - 100):
        try:
            table.delete_item(Key={'id': result[i]['id']})
        except:
            pass
        time.sleep(2)
def get_pair_name(address):
    a = requests.get("https://aeyp0uqbrf.execute-api.us-east-1.amazonaws.com/token-pair-name-from-address?id={}".format(address))
    return a.text
@client.on(events.NewMessage(chats=channel_list))
async def listen_for_contract(event: telethon.events.newmessage.NewMessage.Event):
    global table
    message = event.raw_text
    contract_links = re.findall(r'(https://www\.dextools\.io.*0x[A-Za-z0-9]{40})', message)
    print(contract_links)
    id_time = int(time.time())
    for link in contract_links:
        chain = link.split('/')[-3]
        contract_address = link.split('/')[-1]
        push_info_to_db(id_time=id_time,
                        channel_id=str(event.chat_id)[1:],
                        title=event.chat.title,
                        pair_name=get_pair_name(contract_address),
                        address=contract_address,
                        chain=chain,
                        tbl=table)
        check_and_delete(channel_id=str(event.chat_id)[1:], chain=chain)
    contract_links = re.findall(r'(https://dexscreener\.com.*0x[A-Za-z0-9]{40})', message)
    print(contract_links)
    id_time = int(time.time())
    for link in contract_links:
        chain = link.split('/')[-2]
        contract_address = link.split('/')[-1]
        push_info_to_db(id_time=id_time,
                        channel_id=str(event.chat_id)[1:],
                        title=event.chat.title,
                        pair_name=get_pair_name(contract_address),
                        address=contract_address,
                        chain=chain,
                        tbl=table)
        check_and_delete(channel_id=str(event.chat_id)[1:], chain=chain)
client.start()
client.run_until_disconnected()


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 12 01:00:24 2019

@author: acn00
"""

import sys
import qiskit
import logging
from qiskit import QuantumProgram
# Main sub
def main():
  # create a  program
  qp = QuantumProgram()
  # create 1 qubit
  quantum_r = qp.create_quantum_register("qr", 1)
  print("It works")
   # create 1 classical register
  classical_r = qp.create_classical_register("cr", 1)
    # create a circuit
  qp.create_circuit("Circuit", [quantum_r], [classical_r])
  # get the circuit by name
  circuit = qp.get_circuit('Circuit')
  # enable logging
  qp.enable_logs(logging.DEBUG);
  # Pauli X gate to qubit 1 in the Quantum Register "qr"
  circuit.x(quantum_r[0])
  # measure gate from qubit 0 to classical bit 0
  circuit.measure(quantum_r[0], classical_r[0])
  # backend simulator
  backend = 'local_qasm_simulator'
  # Group of circuits to execute
  circuits = ['Circuit']
  # Compile your program
  qobj = qp.compile(circuits, backend)
  # run in simulator
  result = qp.run(qobj, timeout=240)
  # Show result counts
  print (str(result.get_counts('Circuit')))

if __name__ ==  '__main__':
  main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# __coconut_hash__ = 0xde71c936

# Compiled with Coconut version 2.0.0-a_dev33 [How Not to Be Seen]

# Coconut Header: -------------------------------------------------------------

from __future__ import print_function, absolute_import, unicode_literals, division
import sys as _coconut_sys, os as _coconut_os
_coconut_file_dir = _coconut_os.path.dirname(_coconut_os.path.abspath(__file__))
_coconut_cached_module = _coconut_sys.modules.get(str("__coconut__"))
if _coconut_cached_module is not None and _coconut_os.path.dirname(_coconut_cached_module.__file__) != _coconut_file_dir:  # type: ignore
    del _coconut_sys.modules[str("__coconut__")]
_coconut_sys.path.insert(0, _coconut_file_dir)
_coconut_module_name = _coconut_os.path.splitext(_coconut_os.path.basename(_coconut_file_dir))[0]
if _coconut_module_name and _coconut_module_name[0].isalpha() and all(c.isalpha() or c.isdigit() for c in _coconut_module_name) and "__init__.py" in _coconut_os.listdir(_coconut_file_dir):
    _coconut_full_module_name = str(_coconut_module_name + ".__coconut__")
    import __coconut__ as _coconut__coconut__
    _coconut__coconut__.__name__ = _coconut_full_module_name
    for _coconut_v in vars(_coconut__coconut__).values():
        if getattr(_coconut_v, "__module__", None) == str("__coconut__"):
            try:
                _coconut_v.__module__ = _coconut_full_module_name
            except AttributeError:
                _coconut_v_type = type(_coconut_v)
                if getattr(_coconut_v_type, "__module__", None) == str("__coconut__"):
                    _coconut_v_type.__module__ = _coconut_full_module_name
    _coconut_sys.modules[_coconut_full_module_name] = _coconut__coconut__
from __coconut__ import *
from __coconut__ import _coconut_tail_call, _coconut_tco, _coconut_call_set_names, _coconut_handle_cls_kwargs, _coconut_handle_cls_stargs, _namedtuple_of, _coconut, _coconut_MatchError, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr
_coconut_sys.path.pop(0)

# Compiled Coconut: -----------------------------------------------------------

from argparse import ArgumentParser
from collections import namedtuple
if _coconut_sys.version_info < (3, 3):
    from collections import Iterable
else:
    from collections.abc import Iterable

import hace

parser = ArgumentParser()
parser.add_argument("--host", type=str, default="localhost", help="Host address")
parser.add_argument("-p", "--port", type=int, default="6006", help="Server Port")
parser.add_argument("-e", "--env", type=str, default="op2", help="ACE Environment ID, see GACE doc for what's available")
parser.add_argument("-n", "--num", type=int, default=1, help="Number of Pooled Envs")
parser.add_argument("--pdk", type=str, default="xh035-3V3", help="ACE backend, see GACE doc for what's available")

@_coconut_tco
def isiterable(obj):
    return _coconut_tail_call(isinstance, obj, Iterable)

def make_env(env_id,  #type: str
     backend,  #type: str
     num=1  #type: int
    ):
    env = (hace.make_env(env_id, backend) if num == 1 else hace.make_same_env_pool(num, env_id, backend))

    return env

def simulate_pool(envs, sizings  #type: dict[int, dict[str, float]]
    ):
    sizing = dict(((int(i)), (s)) for i, s in sizings.items())
    perf = hace.evaluate_circuit_pool(envs, sizing)

    return perf

def simulate_single(env, sizing  #type: dict[str, float]
    ):
    perf = hace.evaluate_circuit(env, sizing)

    return perf

def simulate(env, sizing):
    perf = (simulate_pool(env, sizing) if isiterable(env) else simulate_single(env, sizing))

    return perf

def performance(env):
    perf = ((hace.current_performance_pool if isiterable(env) else hace.current_performance))(env)

    return perf

def sizing(env):
    size = ((hace.current_sizing_pool if isiterable(env) else hace.current_sizing))(env)

    return size

def performance_parameters(env):
    pps = {"params": ((hace.performance_identifiers_pool if isiterable(env) else hace.performance_identifiers))(env)}

    return pps

def sizing_parameters(env):
    sps = {"params": ((hace.sizing_identifiers_pool if isiterable(env) else hace.sizing_identifiers))(env)}

    return sps

def initial_sizing(env):
    init = ((hace.initial_sizing_pool if isiterable(env) else hace.initial_sizing))(env)

    return init

def random_sizing(env):
    rng = ((hace.random_sizing_pool if isiterable(env) else hace.random_sizing))(env)

    return rng


from __future__ import annotations

from datetime import timedelta
from hashlib import md5
import logging
import voluptuous as vol

from homeassistant.config_entries import ConfigEntry, ConfigFlow, OptionsFlow
from homeassistant.const import (
    CONF_API_TOKEN,
    CONF_SCAN_INTERVAL,
    CONF_LATITUDE,
    CONF_LONGITUDE,
    CONF_RADIUS,
)
from homeassistant.core import callback
from homeassistant.data_entry_flow import FlowResult
from tier import TIER, UnauthorizedException

import homeassistant.helpers.config_validation as cv

from . import TIERUpdateCoordinator
from .const import (
    CONF_FILTER_NON_RENTABLE_VEHICLES,
    CONF_MINIMUM_BATTERY_LEVEL,
    DEFAULT_FILTER_NON_RENTABLE_VEHICLES,
    DEFAULT_MINIMUM_BATTERY_LEVEL,
    DEFAULT_RADIUS,
    DEFAULT_SCAN_INTERVAL,
    DOMAIN,
)

_LOGGER: logging.Logger = logging.getLogger(__package__)

class TIERConfigFlow(ConfigFlow, domain=DOMAIN):
    """The configuration flow for an TIER system."""

    async def async_step_user(self, user_input=None) -> FlowResult:
        """Ask the user for an API token, site id, and a name for the system."""
        errors = {}
        if user_input is not None:
            try:
                vehicles = await self.hass.async_add_executor_job(
                    lambda: get_vehicles(
                        api_token=user_input[CONF_API_TOKEN],
                        latitude=user_input[CONF_LATITUDE],
                        longitude=user_input[CONF_LONGITUDE],
                        radius=DEFAULT_RADIUS,
                    )
                )
                if vehicles:
                    # Make sure we're not configuring the same device
                    latitude_hash = md5(
                        str(user_input[CONF_LATITUDE]).encode("utf-8")
                    ).hexdigest()
                    longitude_hash = md5(
                        str(user_input[CONF_LONGITUDE]).encode("utf-8")
                    ).hexdigest()

                    await self.async_set_unique_id(
                        f"tier_{latitude_hash}_{longitude_hash}"
                    )
                    self._abort_if_unique_id_configured()

                    return self.async_create_entry(
                        title=f"TIER ({user_input[CONF_LATITUDE]}, {user_input[CONF_LONGITUDE]})",
                        data=user_input,
                    )
            except UnauthorizedException:
                errors[CONF_API_TOKEN] = "invalid_api_token"
            else:
                errors[CONF_API_TOKEN] = "server_error"

        config_schema = vol.Schema(
            {
                vol.Required(CONF_API_TOKEN): cv.string,
                vol.Inclusive(
                    CONF_LATITUDE, "coordinates", default=self.hass.config.latitude
                ): cv.latitude,
                vol.Inclusive(
                    CONF_LONGITUDE, "coordinates", default=self.hass.config.longitude
                ): cv.longitude,
            }
        )

        return self.async_show_form(
            step_id="user", data_schema=config_schema, errors=errors
        )

    @staticmethod
    @callback
    def async_get_options_flow(
        config_entry: ConfigEntry,
    ) -> TIEROptionsFlowHandler:
        return TIEROptionsFlowHandler(config_entry)

class TIEROptionsFlowHandler(OptionsFlow):
    """Config flow options handler for TIER."""

    def __init__(self, config_entry: ConfigEntry):
        """Initialize options flow."""
        self.config_entry = config_entry
        # Cast from MappingProxy to dict to allow update.
        self.options = dict(config_entry.options)

    async def async_step_init(self, user_input=None) -> FlowResult:
        """Manage the options."""
        if user_input is not None:
            self.options.update(user_input)
            coordinator: TIERUpdateCoordinator = self.hass.data[DOMAIN][
                self.config_entry.entry_id
            ]

            update_interval = timedelta(
                seconds=self.options.get(CONF_SCAN_INTERVAL, DEFAULT_SCAN_INTERVAL)
            )
            radius = self.options.get(CONF_RADIUS, DEFAULT_RADIUS)
            minimum_battery_level = self.options.get(
                CONF_MINIMUM_BATTERY_LEVEL, DEFAULT_MINIMUM_BATTERY_LEVEL
            )
            filter_non_rentable_vehicles = self.options.get(
                CONF_FILTER_NON_RENTABLE_VEHICLES, DEFAULT_FILTER_NON_RENTABLE_VEHICLES
            )

            _LOGGER.debug(
                "Updating coordinator, update_interval: %s, radius: %s, min battery level: %s, filter non-rentable: %s",
                update_interval,
                radius,
                minimum_battery_level,
                filter_non_rentable_vehicles,
            )

            coordinator.update_interval = update_interval

            self.hass.data[DOMAIN][self.config_entry.entry_id] = coordinator

            return self.async_create_entry(title="", data=self.options)

        options_schema = vol.Schema(
            {
                vol.Optional(
                    CONF_SCAN_INTERVAL,
                    default=self.config_entry.options.get(
                        CONF_SCAN_INTERVAL, DEFAULT_SCAN_INTERVAL
                    ),
                ): vol.All(vol.Coerce(int), vol.Range(min=1)),
                vol.Required(
                    CONF_RADIUS,
                    default=self.config_entry.options.get(CONF_RADIUS, DEFAULT_RADIUS),
                ): cv.positive_int,
                vol.Required(
                    CONF_MINIMUM_BATTERY_LEVEL,
                    default=self.config_entry.options.get(
                        CONF_MINIMUM_BATTERY_LEVEL,
                        DEFAULT_MINIMUM_BATTERY_LEVEL,
                    ),
                ): vol.All(vol.Coerce(int), vol.Range(min=0, max=100)),
                vol.Optional(
                    CONF_FILTER_NON_RENTABLE_VEHICLES,
                    default=self.config_entry.options.get(
                        CONF_FILTER_NON_RENTABLE_VEHICLES,
                        DEFAULT_FILTER_NON_RENTABLE_VEHICLES,
                    ),
                ): cv.boolean,
            }
        )

        return self.async_show_form(
            step_id="init",
            data_schema=options_schema,
        )

def get_vehicles(api_token: str, latitude: float, longitude: float, radius: int):
    return TIER(api_token=api_token).vehicles.in_radius(latitude, longitude, radius)


from watchfiles import watch, Change as FSE
from pathlib import Path
import asyncio
import sys

def main():
    root = Path('.').resolve()
    for changes in watch("./src"):
        for change in changes:
            fp = Path(change[1])
            if fp.suffix == '.md' and (change[0] == FSE.added or change[0] == FSE.modified):
                print(fp.relative_to(root))
                sys.stdout.flush()
try:
    main()
except KeyboardInterrupt:
    sys.exit(0)


from selenium import webdriver
import time

driver = webdriver.Chrome("C:\\Users\\Tolga\\AppData\\Local\\SeleniumBasic\\chromedriver.exe")
url = "http://github.com"
driver.get(url)
time.sleep(2)
driver.maximize_window() # Açılan tarayıcıyı tam ekran yapar
# driver.save_screenshot("github.com-homepage.png")
url = "http://github.com/tolgaeken"
driver.get(url)
print(driver.title)
if "tolgaeken" in driver.title:
    driver.save_screenshot("tolgae.png") # Ekran görüntüsü alır
time.sleep(2)
driver.back() # Bir önceki sayfaya gider
# driver.forward() # Bir sonraki sayfaya gider
time.sleep(2)
driver.close()


import sys

import tokenizers

tokenizer = tokenizers.ByteLevelBPETokenizer(
    "/data/models/icebert/icebert-first-rmh_different_vocab/icebert-base-36k/icebert-bpe-vocab.json",
    "/data/models/icebert/icebert-first-rmh_different_vocab/icebert-base-36k/icebert-bpe-merges.txt",
    add_prefix_space=True,
)

in_path = sys.argv[1]
out_path = sys.argv[2]

with open(in_path, "r", encoding="utf8") as text_in:
    with open(out_path, "w", encoding="utf8") as text_out:
        for line in text_in:
            fields = [" ".join(str(id) for id in tokenizer.encode(field).ids) for field in line.strip("\n").split("\t")]
            text_out.write("\t".join(fields))
            text_out.write("\n")


import sqlite3

def get_feedback(feedback, userid, quick):

    #DB開關
    conn = sqlite3.connect('response.db')
    c = conn.cursor()

    #建response表格
    c.execute("""CREATE TABLE IF NOT EXISTS responses (
            YN integer,
            description text,
            userid text,
            time datetime default current_timestamp,
            question_id integer,
            assessment_id integer default 0,
            building_id integer default 0);""")

    a = {i:None for i in range(65, 78)} if quick else {i:None for i in range(1, 65)}

    for i,value in feedback:
        a[i] = value

    for i, value in a.items():
        yn = 1 if value is None else 0
        c.execute('INSERT INTO responses (YN, description, userid, question_id) VALUES (?,?,?,?);', (yn, value, userid, i))

    conn.commit()
    conn.close()

    # 功能：把app中的暫時使用者回覆(feedback)，寫進資料庫
    # 輸入：1. feedback:使用者所回覆的待改進內容
    #      2. userid
    #      3. (Bool) quick：看看他是不是填Quick Check，是則T否則F
    # 輸出：寫進response.db


import dash
from dash import Dash, html, callback, Output, Input, State, dcc
import dash_bootstrap_components as dbc
dash.register_page(__name__, path='/model1')

# Creating FORM
name = html.Div(
    [
        dbc.Label("Manufacturer", html_for="example-email"),
        dbc.Input(id="name", type="text", placeholder="Input manufucturer of car"),
        dbc.FormText(
            "This is the value for the car manufacturer",
            color="secondary",
        ),
    ],
    className="mb-3",
)

km_driven = html.Div(
    [
        dbc.Label("KM Driven", html_for="example-email"),
        dbc.Input(id="km_driven", type="number", placeholder="Input kilometers driven of the car"),
        dbc.FormText(
            "This is the value for km_driven",
            color="secondary",
        ),
    ],
    className="mb-3",
)

fuel = html.Div(
    [
        dbc.Label("Fuel Type", html_for="example-email"),
        dbc.RadioItems(
          id="fuel",
          inline=True,
          options=[
            {'label': 'Diesel', 'value': 0},
            {'label': 'Petrol', 'value': 1},
          ],
        ),
    ],
    className="mb-3",
)

seller_type = html.Div(
    [
        dbc.Label("Seller Type", html_for="example-email"),
        dbc.RadioItems(
          id="seller_type",
          inline=True,
          options=[
            {'label': 'Dealer', 'value': "Dealer"},
            {'label': 'Individual', 'value': "Individual"},
            {'label': 'Trustmark', 'value': "Trustmark Dealer"}
          ],
        ),
    ],
    className="mb-3",
)

transmission = html.Div(
    [
        dbc.Label("Transmission Type", html_for="example-email"),
        dbc.RadioItems(
          id="transmission",
          inline=True,
          options=[
            {'label': 'Automatic', 'value': 0},
            {'label': 'Manual', 'value': 1}
          ],
        ),
    ],
    className="mb-3",
)

owner = html.Div(
    [
        dbc.Label("Owner Type", html_for="example-email"),
        dbc.RadioItems(
          id="owner",
          inline=True,
          options=[
            {'label': 'First owner', 'value': 1},
            {'label': 'Second owner', 'value': 2},
            {'label': 'Third owner', 'value': 3},
            {'label': 'Fourth or higher owner', 'value': 4},
            {'label': 'Test driven car', 'value': 5},
          ],
        ),
    ],
    className="mb-3",
)

mileage = html.Div(
    [
        dbc.Label("Mileage (kmpl)", html_for="example-email"),
        dbc.Input(id="mileage", type="number", placeholder="Input mileage of the car in kmpl"),
        dbc.FormText(
            "This is the value for mileage",
            color="secondary",
        ),
    ],
    className="mb-3",
)

engine = html.Div(
    [
        dbc.Label("Engine (CC)", html_for="example-email"),
        dbc.Input(id="engine", type="number", placeholder="Input engine of the car in CC"),
        dbc.FormText(
            "This is the value for engine",
            color="secondary",
        ),
    ],
    className="mb-3",
)

max_power = html.Div(
    [
        dbc.Label("Max Power (bhp)", html_for="example-email"),
        dbc.Input(id="max_power", type="number", placeholder="Input max power of the car in bhp"),
        dbc.FormText(
            "This is the value for max_power",
            color="secondary",
        ),
    ],
    className="mb-3",
)

seats = html.Div(
    [
        dbc.Label("Seats", html_for="example-email"),
        dbc.Input(id="seats", type="number", placeholder="Input seats of the car in bhp"),
        dbc.FormText(
            "This is the value for seats",
            color="secondary",
        ),
    ],
    className="mb-3",
)

submit = html.Div([
            dbc.Button(id="submit", children="calculate selling price using random forest regressor", color="primary", className="me-1"),
            dbc.Label("y is:  "),
            html.Output(id="y", children="")
], style={'marginTop':'10px'})

# submit_model = html.Div([
#             dbc.Button(id="submit_model", children="calculate selling price using model", color="primary", className="me-1"),
#             dbc.Label("y is: "),
#             html.Output(id="y_model", children="")
# ], style={'marginTop':'10px'})

form =  dbc.Form([
            name,
            km_driven,
            fuel,
            seller_type,
            transmission,
            owner,
            mileage,
            engine,
            max_power,
            seats,
            submit,
        ],
        className="mb-3")

# Explain Text
text = html.Div([
    html.H1("A1 - Car Price Prediction"),
    html.P("Please input custom data for your car and receive a predicted selling price."),
    html.P("The model uses a random forest regressor."),
    html.P(""),
])

# Dataset Example
from dash import Dash, dash_table
import pandas as pd
df = pd.read_csv(r'/root/code/datasets/Cars_abridged.csv')

table = dbc.Table.from_dataframe(df,
                        striped=True,
                        bordered=True,
                        hover=True,
                        responsive=True,
                        size='sm'
                            )

layout =  dbc.Container([
        text,
        form,
        html.H1("The Dataset I use to train the model (Only the first few rows displayed for faster loading)"),
        table
    ], fluid=True)

# Accessing user inputs and loading model
import pickle
import sklearn
import pandas
import joblib

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.metrics import r2_score

with open(r'/root/code/models/random_forest_regressor.pickle', 'rb') as f:
  model = pickle.load(f)

pipeline1 = joblib.load(r'/root/code/models/pipeline1.joblib')

columns = ['name', 'km_driven', 'fuel', 'seller_type', 'transmission', 'owner', 'mileage',
       'engine', 'max_power', 'seats']

# data = [['Maruti', 145500, 0, 'Individual', 1, 1, 23.4, 1248, 74, 5]]

@callback(
    Output(component_id="y", component_property="children"),
    State(component_id="name", component_property="value"),
    State(component_id="km_driven", component_property="value"),
    State(component_id="fuel", component_property="value"),
    State(component_id="seller_type", component_property="value"),
    State(component_id="transmission", component_property="value"),
    State(component_id="owner", component_property="value"),
    State(component_id="mileage", component_property="value"),
    State(component_id="engine", component_property="value"),
    State(component_id="max_power", component_property="value"),
    State(component_id="seats", component_property="value"),
    Input(component_id="submit", component_property='n_clicks'),
    prevent_initial_call=True
)
def calculate_y_hardcode(name, km_driven, fuel, seller_type, transmission, owner, mileage, engine, max_power, seats, submit):
    data = [[name, km_driven, fuel, seller_type, transmission, owner, mileage, engine, max_power, seats]]
    print(data)
    X_input = pd.DataFrame(columns=columns, data=data)
    X_input_transformed = pipeline1.transform(X_input)
    selling_price = model.predict(X_input_transformed)
    return selling_price

# @callback(
#     Output(component_id="y_model", component_property="children"),
#     State(component_id="x_1", component_property="value"),
#     State(component_id="x_2", component_property="value"),
#     Input(component_id="submit_model", component_property='n_clicks'),
#     prevent_initial_call=True
# )
# def calculate_y_model(x_1, x_2, submit):
#     from utils import load
#     import pandas as pd
#     import numpy as np
#     model = load('./models/myModel.pickle')
#     X = np.array([x_1,x_2]).reshape(-1,2)
#     X = pd.DataFrame(X, columns=['x1', 'x2'])
#     pred = model.predict(X)
#     return f" model said: {pred=} {model.coef_=}"


# -*- coding: utf-8 -*-
"""
Created on Sun Dec 31 01:52:02 2017

@author: HERVE
"""
import numpy
from math import sqrt
from numpy import concatenate
from matplotlib import pyplot
from pandas import read_csv
from pandas import DataFrame
from pandas import concat
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import mean_squared_error
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM

# fix random seed for reproducibility / Fixer les graines aléatoires pour la reproductibilité
seed = 7
numpy.random.seed(seed)
# convert series to supervised learning / convertir la série en apprentissage supervisé
def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
	n_vars = 1 if type(data) is list else data.shape[1]
	df = DataFrame(data)
	cols, names = list(), list()
	# input sequence (t-n, ... t-1)
	for i in range(n_in, 0, -1):
		cols.append(df.shift(i))
		names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
	# forecast sequence (t, t+1, ... t+n)
	for i in range(0, n_out):
		cols.append(df.shift(-i))
		if i == 0:
			names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
		else:
			names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
	# put it all together
	agg = concat(cols, axis=1)
	agg.columns = names
	# drop rows with NaN values
	if dropnan:
		agg.dropna(inplace=True)
	return agg

# load dataset / charger les données
dataset = read_csv('DENGUE_OK.csv', header=0, index_col=0)
values = dataset.values
# integer encode direction
#encoder = LabelEncoder()
#values[:,4] = encoder.fit_transform(values[:,4])
# ensure all data is float   /assurez-vous que toutes les données soient de type flottant
values = values.astype('float32')
# normalize features  / normaliser les fonctionnalités
scaler = MinMaxScaler(feature_range=(0, 1))
scaled = scaler.fit_transform(values)
# frame as supervised learning   /cadré comme apprentissage supervisé
reframed = series_to_supervised(scaled, 1, 1)
# drop columns we don't want to predict  / Enlever les colonnes que nous ne presdissons pas
reframed.drop(reframed.columns[[9,10,11,12,13,14,15]], axis=1, inplace=True)
print(reframed.head())

# split into train and test sets  / divisé en trains et ensembles de test
values = reframed.values
n_train_hours = 7300#365
train = values[:n_train_hours, :]
test = values[n_train_hours:, :]
# split into input and outputs  / divisé en entrée et en sortie
train_X, train_y = train[:, :-1], train[:, -1]
test_X, test_y = test[:, :-1], test[:, -1]
# reshape input to be 3D [samples, timesteps, features]  / remodeler l'entrée pour qu'elle soit en 3D [samples, timesteps, features]
train_X = train_X.reshape((train_X.shape[0], 1, train_X.shape[1]))
test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1]))
print(train_X.shape, train_y.shape, test_X.shape, test_y.shape)

# design network  /concevoir le reseau
model = Sequential()
model.add(LSTM(100, input_shape=(train_X.shape[1], train_X.shape[2])))
#model.add(Flatten())
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam', metrics=['accuracy'])
# fit network    /ajustement du reseau
history = model.fit(train_X, train_y, epochs=10, batch_size=100, validation_data=(test_X, test_y), verbose=2, shuffle=False)
# plot history  / historique
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='test')
pyplot.legend()
pyplot.show()

# make a prediction    / faire la prediction
yhat = model.predict(test_X)
test_X = test_X.reshape((test_X.shape[0], test_X.shape[2]))
# invert scaling for forecast    / inverser la mise à l'échelle pour la prévision
inv_yhat = concatenate((yhat, test_X[:, 1:]), axis=1)
inv_yhat = scaler.inverse_transform(inv_yhat)
inv_yhat = inv_yhat[:,0]
# invert scaling for actual   / inverser l'échelle pour réel
test_y = test_y.reshape((len(test_y), 1))
inv_y = concatenate((test_y, test_X[:, 1:]), axis=1)
inv_y = scaler.inverse_transform(inv_y)
inv_y = inv_y[:,0]
# calculate RMSE     / calculer le RMSE
rmse = sqrt(mean_squared_error(inv_y, inv_yhat))
print('Test RMSE: %.3f' %rmse)           #(100-scores[1]*100)


import numpy as np
import pandas as pd
from sklearn import datasets

data = datasets.load_iris()

names = data['target_names']

X = data['data']
X = np.array(X, dtype = int)
y = data['target']

data_ = pd.DataFrame(X)
data_.columns = data['feature_names']
data_['Output'] = Y

data['feature_names']

data_.isna().sum()

def entropy(col):

    counts = np.unique(col,return_counts=True)
    N = float(col.shape[0])

    ent = 0.0

    for ix in counts[1]:
        p  = ix/N
        ent += (-1.0*p*np.log2(p))

    return ent

def information_gain(x_data,key,val):
    left,right = divide_data(x_data,key,val)

    l = float(left.shape[0])/x_data.shape[0]
    r = float(right.shape[0])/x_data.shape[0]

    if left.shape[0] == 0 or right.shape[0] ==0:
        return -1000000 #Min Information Gain

    i_gain = entropy(x_data.Output) - (l*entropy(left.Output)+r*entropy(right.Output))
    return i_gain

def divide_data(x_data,key,val):
    #Work with Pandas Data Frames
    x_right = pd.DataFrame([],columns=x_data.columns)
    x_left = pd.DataFrame([],columns=x_data.columns)

    for ix in range(x_data.shape[0]):
        val_ = x_data[key].loc[ix]

        if val_ > val:
            x_right = x_right.append(x_data.loc[ix])
        else:
            x_left = x_left.append(x_data.loc[ix])

    return x_left,x_right

def find_count(X_train):
        count = []
        count.append(X_train[X_train['Output'] == 0].shape[0])
        count.append(X_train[X_train['Output'] == 1].shape[0])
        count.append(X_train[X_train['Output'] == 2].shape[0])
        return count

class DecisionTree:

    def __init__(self,depth=0,max_depth=5):
        self.left = None
        self.right = None
        self.key = None
        self.val = None
        self.count = None
        self.max_depth = max_depth
        self.depth = depth
        self.target = None

    def train(self,X_train,names):

        features = ['sepal length (cm)','sepal width (cm)','petal length (cm)','petal width (cm)']
        info_gains = []

        for ix in features:
            i_gain = information_gain(X_train,ix,X_train[ix].mean())
            info_gains.append(i_gain)

        self.key = features[np.argmax(info_gains)]
        self.val = X_train[self.key].mean()
        print("Level " , self.depth)
        self.count = find_count(X_train)
        cnt = 0
        for i in range(len(self.count)):
            if(self.count[i]):
                print("Count of " , names[i] , " = " , self.count[i])
                cnt += 1
        print("Current entropy = " , entropy(X_train.Output))
        if cnt != 1:
            print("Splitting on Tree Features ",self.key,"with information gain",np.argmax(info_gains))

        data_left,data_right = divide_data(X_train,self.key,self.val)
        data_left = data_left.reset_index(drop=True)
        data_right = data_right.reset_index(drop=True)

        if cnt == 1:
            if X_train.Output.mean() >= 1.5:
                self.target = names[2]
            elif X_train.Output.mean() <= 0.5:
                self.target = names[0]
            else:
                self.target = names[1]
            print("Reached leaf Node")
            print()
            print()
            return

        if(self.depth>=self.max_depth):
            if X_train.Output.mean() >= 1.5:
                self.target = names[2]
            elif X_train.Output.mean() <= 0.5:
                self.target = names[0]
            else:
                self.target = names[1]
            print("Max depth Reached")
            print()
            print()
            return

        print()
        print()

        self.left = DecisionTree(depth=self.depth+1,max_depth=self.max_depth)
        self.left.train(data_left, names)

        self.right = DecisionTree(depth=self.depth+1,max_depth=self.max_depth)
        self.right.train(data_right, names)

        if X_train.Output.mean() >= 1.5:
            self.target = names[2]
        elif X_train.Output.mean() <= 0.5:
            self.target = names[0]
        else:
            self.target = names[1]
        return

dt = DecisionTree()

dt.train(data_, names)


import os
import pickle
import torch
from collections import OrderedDict
import numpy as np
import time
import data_process
import transformer_data_process
import bert_cnn_data_process
from model import VSE
from transformer_model import transformer_VSE
from bert_cnn_model import BERT_CNN_VSE
from pprint import pprint
from param import args

if args.gpu.lower() == 'cpu':
    DEVICE = torch.device('cpu')
elif args.gpu in ['0' , '1' , '2' , '3']:
    DEVICE = torch.device(f"cuda:{args.gpu}" if torch.cuda.is_available() else "cpu")
else:
    raise ValueError('Invalid GPU ID')

class AverageMeter(object):
    """计算并且存储当前值和总计平均值(平滑)"""
    def __init__(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / (self.count + 1e-5)

    def __str__(self):
        if self.count == 0:
            return str(self.eval)
        return '%.4f (%.4f)' % (self.val, self.avg)

class LogCollector(object):
    """
        记录train和val的logging的对象
    """
    def __init__(self):
        self.meters = OrderedDict()
    def update(self, k, v, n=1):
        if k not in self.meters:
            self.meters[k] = AverageMeter()
        self.meters[k].update(v, n)
    def __str__(self):
        s = ''
        for i , (k , v) in enumerate(self.meters.items()):
            if i > 0:
                s += ' '
            s += f' {k} : {v}'
        return s
    def tensorboard_log(self , tb_logger , prefix='', step=None):
        for k , v in self.meters.items():
            tb_logger.log_value(prefix + k , v.val , step=step)

def im2cap(images , captions):
    image_nums = int(images.shape[0] / 5)
    ranks = np.zeros(image_nums)

    for index in range(image_nums):
        # 获取查询图片
        im = images[5 * index].reshape(1 , images.shape[1])
        # 计算相似度
        sim_score = np.dot(im , captions.T) # (1 , 5*image_nums)

        inds = np.argsort(sim_score[0])[::-1]
        rank = 1e20
        for i in range(5 * index , 5 * index + 5 , 1): # 在5个匹配的文本中选取相似度最大的
            tmp = np.where(inds == i)[0][0]
            if tmp < rank:
                rank = tmp
        ranks[index] = rank

    # 计算召回率指标
    r_im2cap_1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
    r_im2cap_5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
    r_im2cap_10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
    r_im2cap_medi = np.floor(np.median(ranks)) + 1
    r_im2cap_mean = np.floor(np.mean(ranks)) + 1

    return r_im2cap_1 , r_im2cap_5 , r_im2cap_10 , r_im2cap_medi , r_im2cap_mean

def cap2im(images , captions):
    image_nums = int(images.shape[0] / 5)
    ims = np.array([images[5 * i] for i in range(image_nums)]) # (image_nums , image_dim)
    ranks = np.zeros(images.shape[0])

    for index in range(image_nums):
        # 获取查询文本
        caps = captions[5*index : 5*index + 5]
        # 计算相似度
        sim_score = np.dot(caps , ims.T) # (5 , image_nums)
        inds = np.zeros(sim_score.shape)

        for i in range(len(inds)):
            inds[i] = np.argsort(sim_score[i])[::-1]
            ranks[5*index + i] = np.where(inds[i] == index)[0][0]

    # 计算召回率指标
    r_cap2im_1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
    r_cap2im_5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
    r_cap2im_10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
    r_cap2im_medi = np.floor(np.median(ranks)) + 1
    r_cap2im_mean = np.floor(np.mean(ranks)) + 1

    return r_cap2im_1 , r_cap2im_5 , r_cap2im_10 , r_cap2im_medi , r_cap2im_mean

def encode_data(model , data_loader , log_step=10 , logging=print):
    batch_time = AverageMeter()
    val_logger = LogCollector()
    model.val_model()

    timer = time.time()
    img_embs = None
    cap_embs = None
    isInit = False

    for i , (images , captions , lengths , ids) in enumerate(data_loader):
        model.logger = val_logger

        with torch.no_grad():
            img_emb , cap_emb = model.forward(images , captions , lengths)

            # 初始化
            if not isInit:
                img_embs = np.zeros((len(data_loader.dataset) , img_emb.shape[1]))
                cap_embs = np.zeros((len(data_loader.dataset) , cap_emb.shape[1]))
                isInit = True

            img_embs[ids] = img_emb.data.cpu().numpy().copy()
            cap_embs[ids] = cap_emb.data.cpu().numpy().copy()

            model.calc_loss(img_emb , cap_emb)
            # del images, captions # 有用嘛说实话 ？？？

        batch_time.update(time.time() - timer)

        if i % log_step == 0:
            logging(
                'Test: [{0}/{1}]\t'
                '{e_log}\t'
                'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                .format(
                    i , len(data_loader) ,
                    e_log = model.logger ,
                    batch_time = batch_time
                )
            )

            timer = time.time()

    return img_embs , cap_embs

def evalrank(model_path):
    checkpoint = torch.load(model_path)
    old_args = checkpoint['args']

    with open(os.path.join(old_args.vocab_path , 'flickr30k_vocab.pkl') , 'rb') as f:
        vocab = pickle.load(f)
    old_args.vocab_size = len(vocab)

    print("-----------------------------------------------")
    pprint(old_args)
    print("-----------------------------------------------")

    if old_args.model_class == 'CNN_and_GRU':
        # 构建模型
        model = VSE(
            old_args.embed_size,
            old_args.finetune,
            old_args.word_dim,
            old_args.num_layers,
            vocab,
            old_args.margin,
            old_args.max_violation,
            old_args.grad_clip,
            old_args.use_InfoNCE_loss,
            old_args.rnn_mean_pool,
            old_args.bidirection_rnn,
            old_args.use_word2vec,
            old_args.cnn_type,
            old_args.use_attention_for_text,
            old_args.num_heads
        )
    elif old_args.model_class == 'ViT_and_BERT':
        # 构建模型
        model = transformer_VSE(
            old_args.embed_size,
            old_args.finetune,
            old_args.margin,
            old_args.max_violation,
            old_args.grad_clip,
            old_args.use_InfoNCE_loss
        )
    elif old_args.model_class == 'CNN_and_BERT':
        # 构建模型
        model = BERT_CNN_VSE(
            old_args.embed_size,
            old_args.finetune,
            old_args.margin,
            old_args.max_violation,
            old_args.grad_clip,
            old_args.use_InfoNCE_loss,
            old_args.cnn_type
        )
    else:
        raise ValueError('Wrong model class !! It must be in (CNN_and_GRU , ViT_and_BERT , CNN_and_BERT)')

    model.load_state_dict(checkpoint['model'])

    print('Loading dataset')

    if old_args.model_class == 'CNN_and_GRU':
        data_loader = data_process.get_test_loader(
            vocab,
            old_args.batch_size,
            old_args.workers
        )
    elif old_args.model_class == 'ViT_and_BERT':
        data_loader = transformer_data_process.get_test_loader(
            old_args.batch_size,
            old_args.workers
        )
    elif old_args.model_class == 'CNN_and_BERT':
        data_loader = bert_cnn_data_process.get_test_loader(
            old_args.batch_size,
            old_args.workers
        )
    else:
        raise ValueError('Wrong model class !! It must be in (CNN_and_GRU , ViT_and_BERT , CNN_and_BERT)')

    print('Computing result....')
    img_embs , cap_embs = encode_data(model , data_loader)
    print(f'Images: {img_embs.shape[0] / 5} , Captions: {cap_embs.shape[0]}')

    r_im2cap = im2cap(img_embs , cap_embs)
    r_cap2im = cap2im(img_embs , cap_embs)

    ave_r_im2cap = (r_im2cap[0] + r_im2cap[1] + r_im2cap[2]) / 3
    ave_r_cap2im = (r_cap2im[0] + r_cap2im[1] + r_cap2im[2]) / 3
    R_sum = r_im2cap[0] + r_im2cap[1] + r_im2cap[2] + r_cap2im[0] + r_cap2im[1] + r_cap2im[2]

    print("----------------------------------------")
    print("R_sum : %.1f" % R_sum)
    print("Average im2cap Recall: %.1f" % ave_r_im2cap)
    print("Image to text: %.1f %.1f %.1f %.1f %.1f" % r_im2cap)
    print("Average cap2im Recall: %.1f" % ave_r_cap2im)
    print("Text to image: %.1f %.1f %.1f %.1f %.1f" % r_cap2im)


from typing import List
from pytest import fixture
from textwrap import dedent
import numpy as np

@fixture
def sample():
    return dedent("""\
        1163751742
        1381373672
        2136511328
        3694931569
        7463417111
        1319128137
        1359912421
        3125421639
        1293138521
        2311944581
    """).splitlines()

def read(filename: str) -> List[str]:
    with open(filename, 'r') as f:
        return f.read().splitlines()

def get_adjacent(idx, visited):
    x, y = idx
    if x != 0 and not visited[x-1, y]:
        yield x-1, y
    if x != visited.shape[0]-1 and not visited[x+1, y]:
        yield x+1, y
    if y != 0 and not visited[x, y-1]:
        yield x, y-1
    if y != visited.shape[1]-1 and not visited[x, y+1]:
        yield x, y+1

def dijkstra(weights: np.ndarray) -> np.ndarray:
    inf = max(weights.shape) ** 2 * 9
    costs = np.ones_like(weights, dtype=int) * inf
    costs[0, 0] = 0
    visited = np.zeros_like(weights, dtype=bool)
    costs_visited = costs.copy()
    i = 0
    while not visited[-1, -1]:  # we need only costs for right-bottom corner
        # TODO optimize: track nonvisited nodes with non-inf value
        curr_node = np.unravel_index(np.argmin(costs_visited, axis=None), costs.shape)
        visited[curr_node] = True
        costs_visited[curr_node] = inf
        for node in get_adjacent(curr_node, visited):
            if costs[node] > costs[curr_node] + weights[node]:
                costs[node] = costs[curr_node] + weights[node]
                costs_visited[node] = costs[node]

        i += 1
        if i % 1000 == 0:
            print(np.sum(visited) / (weights.shape[0] * weights.shape[1]))
    return costs

def main_part1(lines: List[str]) -> int:
    # You start in the top left position, your destination is the bottom right position, and you cannot move diagonally.
    levels = np.asarray([list(map(int, line)) for line in lines], dtype=int)
    costs = dijkstra(levels)
    return costs[-1, -1]

def test_part1(sample):
    assert main_part1(sample) == 40

def populate(weights: np.ndarray, k) -> np.ndarray:
    n, m = weights.shape
    levels_all = np.zeros((k*n, k*m), dtype=int)
    for i in range(k):
        part = weights + i
        part = part % 10 + part // 10
        levels_all[n * i: n * (i + 1), 0: m] = part
        for j in range(1, k):
            part = part + 1
            part = part % 10 + part // 10
            levels_all[n * i: n * (i + 1), m * j: m * (j + 1)] = part
    return levels_all

def main_part2(lines: List[str]) -> int:
    levels = np.asarray([list(map(int, line)) for line in lines], dtype=int)
    # construct bigger map:
    levels = populate(levels, k=5)
    costs = dijkstra(levels)
    return costs[-1, -1]

def test_populate():
    levels = np.asarray([[8, ]], dtype=int)
    levels = populate(levels, k=5)
    res = np.asarray([
        [8, 9, 1, 2, 3],
        [9, 1, 2, 3, 4],
        [1, 2, 3, 4, 5],
        [2, 3, 4, 5, 6],
        [3, 4, 5, 6, 7],
    ])
    assert np.all(levels == res)

def test_part2(sample):
    assert main_part2(sample) == 315

if __name__ == "__main__":
    input_data = read('input')
    print(main_part1(input_data))
    print(main_part2(input_data))


from mpi4py import MPI
import numpy as np
from scipy import sparse
from bitarray import util
import copy
from bitarray import bitarray
from random import random as rando
import pickle
import timeit
from itertools import islice,chain
comm=MPI.COMM_WORLD
size=comm.Get_size()
rank=comm.Get_rank()
file=open('PBGENA_parameters.txt','r+')
while True:
    line=file.readline()
    if not line:
        break
    argument=line.split(' ')
    if argument[0]=='graph':
        graph=argument[1][:-1]
    elif argument[0]=='N':
        N=int(argument[1][:-1])
    elif argument[0]=='alpha':
        alpha=float(argument[1][:-1])
    elif argument[0]=='b_a':
        b_a=float(argument[1][:-1])
    elif argument[0]=='b_t':
        b_t=float(argument[1][:-1])
    elif argument[0]=='l_t':
        l_t=int(argument[1][:-1])
    elif argument[0]=='l_a':
        l_a=int(argument[1][:-1])
    elif argument[0]=='f_t':
        f_t=int(argument[1][:-1])
    elif argument[0]=='f_a':
        f_a=int(argument[1][:-1])
    elif argument[0]=='fragments':
        f=int(argument[1][:-1])
    elif argument[0]=='nodes':
        nodes=int(argument[1][:-1])
    elif argument[0]=='edges':
        edges=int(argument[1][:-1])
    elif argument[0]=='attributes':
        attributes=int(argument[1][:-1])
del(argument,line)
file.close()
N_t=round((1-alpha)*N)
N_a=N-N_t
if rank==0:
    start_time=timeit.default_timer()
    Phi=np.random.randint(low=0,high=size,size=nodes)
else:
    Phi=np.empty(nodes,dtype='int')
comm.Bcast(Phi)
Phi_inverse=np.zeros(nodes,dtype=int)
j=0
for i in range(nodes):
    if Phi[i]==rank:
        Phi_inverse[j]=i
        j+=1
Phi_inverse=Phi_inverse[0:j]
if rank==0:
    time=timeit.default_timer()-start_time
edge_list=np.load('../../Datasets/'+graph+'/edge_list_preprocessed.npy')
in_edges=np.zeros((edges,2),dtype=int)
cross_edges=np.zeros((edges,2),dtype=int)
node_set=Phi_inverse.tolist()
node_set=set(node_set)
j=0
k=0
for i in edge_list:
    true_1=i[0] in node_set
    true_2=i[1] in node_set
    if true_1 and true_2:
        in_edges[j]=i
        j+=1
    elif true_1:
        cross_edges[k]=i
        k+=1
    elif true_2:
        cross_edges[k][0]=i[1]
        cross_edges[k][1]=i[0]
        k+=1
del(node_set,edge_list)
in_edges=in_edges[0:j,:]
cross_edges=cross_edges[0:k,:]
if N_a>0:
    attribute_matrix=sparse.load_npz('../../Datasets/'+graph+'/attribute_matrix.npz')
    attribute_list={i:attribute_matrix[i].nonzero()[1] for i in Phi_inverse}
    del(attribute_matrix)
if rank==0:
    start_time=timeit.default_timer()
    if N_t>0:
        Pi_t=np.random.randint(low=0,high=N_t,size=nodes)
    if N_a>0:
        Pi_a=np.random.randint(low=0,high=N_a,size=attributes)
else:
    if N_t>0:
        Pi_t=np.empty(nodes,dtype=int)
    if N_a>0:
        Pi_a=np.empty(attributes,dtype=int)
if N_a>0:
    comm.Bcast(Pi_a)
    S_a={i:util.zeros(N_a) for i in Phi_inverse}
    for i in Phi_inverse:
        for j in attribute_list[i]:
            S_a[i][Pi_a[j]]=1
    del(Pi_a,attribute_list)
    if b_a==0:
        E_a=S_a
    else:
        for i in range(l_a):
            E_a=copy.deepcopy(S_a)
            if b_a==1:
                for j in in_edges:
                    E_a[j[1]]=S_a[j[0]]|E_a[j[1]]
                    E_a[j[0]]=S_a[j[1]]|E_a[j[0]]
                batch_S_a=[dict() for _ in range(size)]
                for j in cross_edges:
                    batch_S_a[Phi[j[1]]][j[0]]=S_a[j[0]]
                S_a_temp=dict()
                if f==1:
                    batch_S_a=comm.alltoall(batch_S_a)
                    for j in range(size):
                        S_a_temp.update(batch_S_a[j])
                else:
                    batch_sizes_S_a=[len(batch_S_a[j]) for j in range(size)]
                    batch_S_a=[iter(batch_S_a[j].items()) for j in range(size)]
                    for j in range(f):
                        mini_batch_S_a=[dict(islice(batch_S_a[k],batch_sizes_S_a[k]//(f-j))) for k in range(size)]
                        batch_sizes_S_a=[batch_sizes_S_a[k]-len(mini_batch_S_a[k]) for k in range(size)]
                        mini_batch_S_a=comm.alltoall(mini_batch_S_a)
                        for k in range(size):
                            S_a_temp.update(mini_batch_S_a[k])
                    del(batch_sizes_S_a,mini_batch_S_a)
                for j in cross_edges:
                    E_a[j[0]]=S_a_temp[j[1]]|E_a[j[0]]
                del(S_a_temp,batch_S_a)
            else:
                D_a=dict()
                for j in Phi_inverse:
                    Q_a=bitarray(rando()<b_a for _ in range(N_a))
                    D_a[j]=S_a[j]&Q_a
                for j in in_edges:
                    E_a[j[1]]=D_a[j[0]]|E_a[j[1]]
                    E_a[j[0]]=D_a[j[1]]|E_a[j[0]]
                batch_D_a=[dict() for _ in range(size)]
                for j in cross_edges:
                    batch_D_a[Phi[j[1]]][j[0]]=D_a[j[0]]
                D_a=dict()
                if f==1:
                    batch_D_a=comm.alltoall(batch_D_a)
                    for j in range(size):
                        D_a.update(batch_D_a[j])
                else:
                    batch_sizes_a=[len(batch_D_a[j]) for j in range(size)]
                    batch_D_a=[iter(batch_D_a[j].items()) for j in range(size)]
                    for j in range(f):
                        mini_batch_D_a=[dict(islice(batch_D_a[k],batch_sizes_a[k]//(f-j))) for k in range(size)]
                        batch_sizes_a=[batch_sizes_a[k]-len(mini_batch_D_a[k]) for k in range(size)]
                        mini_batch_D_a=comm.alltoall(mini_batch_D_a)
                        for k in range(size):
                            D_a.update(mini_batch_D_a[k])
                    del(batch_sizes_a,mini_batch_D_a)
                for j in cross_edges:
                    E_a[j[0]]=D_a[j[1]]|E_a[j[0]]
                del(D_a,Q_a,batch_D_a)
            S_a=E_a
            b_a/=f_a
    del(S_a)
if N_t>0:
    comm.Bcast(Pi_t)
    S_t={i:util.zeros(N_t) for i in Phi_inverse}
    for i in Phi_inverse:
        S_t[i][Pi_t[i]]=1
    for i in in_edges:
        S_t[i[0]][Pi_t[i[1]]]=1
        S_t[i[1]][Pi_t[i[0]]]=1
    for i in cross_edges:
        S_t[i[0]][Pi_t[i[1]]]=1
    del(Pi_t)
    if b_t==0:
        E_t=S_t
    else:
        for i in range(l_t):
            E_t=copy.deepcopy(S_t)
            if b_t==1:
                for j in in_edges:
                    E_t[j[1]]=S_t[j[0]]|E_t[j[1]]
                    E_t[j[0]]=S_t[j[1]]|E_t[j[0]]
                batch_S_t=[dict() for _ in range(size)]
                for j in cross_edges:
                    batch_S_t[Phi[j[1]]][j[0]]=S_t[j[0]]
                S_t_temp=dict()
                if f==1:
                    batch_S_t=comm.alltoall(batch_S_t)
                    for j in range(size):
                        S_t_temp.update(batch_S_t[j])
                else:
                    batch_sizes_S_t=[len(batch_S_t[j]) for j in range(size)]
                    batch_S_t=[iter(batch_S_t[j].items()) for j in range(size)]
                    for j in range(f):
                        mini_batch_S_t=[dict(islice(batch_S_t[k],batch_sizes_S_t[k]//(f-j))) for k in range(size)]
                        batch_sizes_S_t=[batch_sizes_S_t[k]-len(mini_batch_S_t[k]) for k in range(size)]
                        mini_batch_S_t=comm.alltoall(mini_batch_S_t)
                        for k in range(size):
                            S_t_temp.update(mini_batch_S_t[k])
                    del(batch_sizes_S_t,mini_batch_S_t)
                for j in cross_edges:
                    E_t[j[0]]=S_t_temp[j[1]]|E_t[j[0]]
                del(S_t_temp,batch_S_t)
            else:
                D_t=dict()
                for j in Phi_inverse:
                    Q_t=bitarray(rando()<b_t for _ in range(N_t))
                    D_t[j]=S_t[j]&Q_t
                for j in in_edges:
                    E_t[j[1]]=D_t[j[0]]|E_t[j[1]]
                    E_t[j[0]]=D_t[j[1]]|E_t[j[0]]
                batch_D_t=[dict() for _ in range(size)]
                for j in cross_edges:
                    batch_D_t[Phi[j[1]]][j[0]]=D_t[j[0]]
                D_t=dict()
                if f==1:
                    batch_D_t=comm.alltoall(batch_D_t)
                    for j in range(size):
                        D_t.update(batch_D_t[j])
                else:
                    batch_sizes_t=[len(batch_D_t[j]) for j in range(size)]
                    batch_D_t=[iter(batch_D_t[j].items()) for j in range(size)]
                    for j in range(f):
                        mini_batch_D_t=[dict(islice(batch_D_t[k],batch_sizes_t[k]//(f-j))) for k in range(size)]
                        batch_sizes_t=[batch_sizes_t[j]-len(mini_batch_D_t[j]) for j in range(size)]
                        mini_batch_D_t=comm.alltoall(mini_batch_D_t)
                        for k in range(size):
                            D_t.update(mini_batch_D_t[k])
                    del(batch_sizes_t,mini_batch_D_t)
                for j in cross_edges:
                    E_t[j[0]]=D_t[j[1]]|E_t[j[0]]
                del(D_t,Q_t,batch_D_t)
            S_t=E_t
            b_t/=f_t
    del(S_t)
del(in_edges,cross_edges,Phi_inverse,Phi)
if f==1:
    if N_t>0:
        E_t=comm.gather(E_t)
    if N_a>0:
        E_a=comm.gather(E_a)
else:
    if N_t>0:
        E_t_iterator=iter(E_t.items())
        buffer=len(E_t)
        E_t=[[] for _ in range(f)]
        for i in range(f):
            mini_batch_E_t=dict(islice(E_t_iterator,buffer//(f-i)))
            buffer-=len(mini_batch_E_t)
            E_t[i]=comm.gather(mini_batch_E_t)
        del(E_t_iterator,mini_batch_E_t)
    if N_a>0:
        E_a_iterator=iter(E_a.items())
        buffer=len(E_a)
        E_a=[[] for _ in range(f)]
        for i in range(f):
            mini_batch_E_a=dict(islice(E_a_iterator,buffer//(f-i)))
            buffer-=len(mini_batch_E_a)
            E_a[i]=comm.gather(mini_batch_E_a)
        del(E_a_iterator,mini_batch_E_a)
if rank==0:
    if f>1:
        if N_t>0:
            E_t=list(chain.from_iterable(E_t))
        if N_a>0:
            E_a=list(chain.from_iterable(E_a))
    emb=[[] for _ in range(nodes)]
    if N_t>0:
        for i in E_t:
            for k,l in i.items():
                emb[k]=l
        del(E_t)
    if N_a>0 and N_t==0:
        for i in E_a:
            for k,l in i.items():
                emb[k]=l
        del(E_a)
    elif N_a>0:
        for i in E_a:
            for k,l in i.items():
                emb[k]=emb[k]+l
        del(E_a)
    time+=timeit.default_timer()-start_time
    print('\nEmbedding Time = %.2fs'%time)
    pickle.dump(emb,open('../../Embeddings/'+graph+'_PBGENA_emb.pkl','wb'))
    del(emb)
else:
    if N_t>0:
        del(E_t)
    if N_a>0:
        del(E_a)


import cv2
import numpy as np
from progress.bar import Bar
import csv
import math
from processing_options import argsHandler
import os
from utils import verbosePrint, getConnectedComponents, heightBox, resultPlotting, dataLoader
from matplotlib import pyplot as plt
import matplotlib
matplotlib.use('tkAgg')
# Default Values
MAX_PIXEL_VALUE = 255
MAX_CENTROID_TOLERANCE = 10 #px, converted from the 1mm by authors
#DER_LOW_BOUND = 2e-5
SP_THRESHOLD = 1 #px, converted from the 0,1mm by authors

# Drawing values
CENTROID_RADIUS = 10

# Fig save padding
FIG_PADDING = 200 #px

TARGET_SP = 22.7
SP_TOL = 0.25
DELTA = 1

def px_2_mm(px_val):
	mm = (0.1038*px_val) + 3.55
	return mm

def smokepoint(args):
	# Check folder for output data
	out_path = args.runName
	if not(os.path.isdir(out_path)):
		os.mkdir(out_path)

	out_frame_folder = os.path.join(out_path,'frames')
	if not(os.path.isdir(out_frame_folder)):
		os.mkdir(out_frame_folder)

	# Load the data (video or folder)
	error = float('inf')
	core_por = 2.5
	contour_por = 1.5

	while(error > SP_TOL):
		parsed = dataLoader(args.Input)
		media = cv2.VideoCapture(parsed,0)
		n_frames = int(media.get(cv2.CAP_PROP_FRAME_COUNT))
		width = int(media.get(cv2.CAP_PROP_FRAME_WIDTH))
		height = int(media.get(cv2.CAP_PROP_FRAME_HEIGHT))
		# Set threshold for core and countour
		core_threshold_value = round(MAX_PIXEL_VALUE*core_por/100)
		contour_threshold_value = round(MAX_PIXEL_VALUE*contour_por/100)

		save_cut = height//2 + 100

		h = []
		H = []
		first_frame_flag = True
		reference_centroid_x = -float('inf')
		reference_centroid_y = -float('inf')
		invalid_frame_counter = 0
		frame_counter = 0
		# Perform frame by frame processing
		while media.isOpened():
			ret, frame = media.read()
			if not ret:
				break
			frame_counter += 1

			# -> Check if cutting the frame is needed
			if(args.cut):
				cut = args.cut//2
				center_line = width//2
				frame = frame[:,center_line-cut:center_line+cut]

			# -> Convert to grayscale
			gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

			# -> Apply core and contour umbrals
			ret, core_thresh = cv2.threshold(gray_frame, core_threshold_value, MAX_PIXEL_VALUE, cv2.THRESH_BINARY)
			ret, contour_thresh = cv2.threshold(gray_frame, contour_threshold_value, MAX_PIXEL_VALUE, cv2.THRESH_BINARY)

			# -> From connected components, get stats
			core_components = getConnectedComponents(core_thresh,4)
			contour_components = getConnectedComponents(contour_thresh,4)

			# -> Get heights and centroid of the first frame
			contour_height = contour_components['h']
			# Consider that (0,0) is top left corner
			tip_height = core_components['y'] - contour_components['y']

			if(first_frame_flag):
				reference_centroid_x = contour_components['cX']
				reference_centroid_y = contour_components['cY']
				first_frame_flag = False
				# Manually add the values of h and H, before continuing
				h.append(contour_height)
				H.append(tip_height)
				continue

			# Draw reference centroid and frame centroid [TO-DO:ARGUMENT?]

			centroid_diff = abs(contour_components['cX']-reference_centroid_x)

			# Check if frame is not valid
			if(centroid_diff > MAX_CENTROID_TOLERANCE):
				invalid_frame_counter +=1
				continue
			# If valid, get height data
			h.append(contour_height)
			H.append(tip_height)

		# End of media reading

		media.release()

		# -> Fit a 10th order poly
		tenth_poly = np.polyfit(h, H, 10)

		# -> Derivative of the 10th poly
		der_poly = np.polyder(tenth_poly,2)

		# -> Get linear region
		linear_interval_flag = True
		linear_region_start = -float('inf')
		linear_region_end = float('inf')
		linear_region_points = {}

		for flame_pos, flame_height in enumerate(h):
			eval = abs(np.polyval(der_poly, flame_height))
			if(eval <= args.DerivativeThreshold):
				linear_region_points[flame_height] = np.polyval(tenth_poly, flame_height)
				if linear_interval_flag:
					linear_region_start = flame_pos
					linear_interval_flag = False
				linear_region_end = flame_pos

		# Check if linear region does in fact was found
		if(len(linear_region_points) <= 2):
			print("No enough points ({}) were found to set the linear region with derivative threshold of {}".format(len(linear_region_points),args.DerivativeThreshold))
			linear_poly = None
			sp_height = None
			sp_Height = None
		else:
		# Linear regresion for points of region
			linear_poly = np.polyfit(list(linear_region_points.keys()),list( linear_region_points.values()),1)

			# -> Get first value over the linear region
			# Save coordinates of the point
			sp_height = -float('inf')
			sp_Height = -float('inf')
			for flame_pos in range(linear_region_start, len(h)):
				flame_height = h[flame_pos]
				poly_H_val = np.polyval(tenth_poly, flame_height)
				poly_linear_val = np.polyval(linear_poly, flame_height)
				distance = poly_H_val - poly_linear_val
				if(distance > SP_THRESHOLD):
					sp_height = flame_height
					sp_Height = poly_H_val
					break

		if(sp_height):
			sp_mm = px_2_mm(sp_height)
			error = TARGET_SP - sp_mm

		print("Core % {} | SPH = {} mm | TARGET = {} | ERROR = {}".format(core_por, sp_mm, TARGET_SP, error))
		# Recalculate thresholds
		core_por += DELTA*error

if __name__ == '__main__':
	args = argsHandler()
	smokepoint(args)


import os
os.environ["TF_CPP_MIN_LOG_LEVEL"]='3'
from tensorflow.keras.layers import Bidirectional, LSTM, Dense, Input, TimeDistributed, Embedding
from tensorflow.keras.models import Model
from tensorflow.keras.utils import to_categorical
import pandas as pd
from itertools import groupby
from tensorflow import concat
import tensorflow as tf
import numpy as np
from CRF_TF2 import CRF
from nltk import pos_tag
from transformers import BertTokenizer,BertConfig,XLNetTokenizer,XLNetConfig,DebertaConfig,DebertaTokenizer
from transformers import TFBertModel,TFXLNetModel,TFDebertaModel
from transformers import logging
import string
from tensorflow.keras.initializers import RandomUniform
import itertools
from sklearn.preprocessing import StandardScaler
from math import pow,floor
from tensorflow.keras.callbacks import LearningRateScheduler,ReduceLROnPlateau

def file_read(input_url):
    data_read=pd.read_csv(input_url)
    words=list(data_read['word'])
    lalbels=list(data_read['annotation'])
    pos=list(data_read['pos_tag'])
    return words,lalbels,pos

#标签与id的相互转化
def idx_switch(sentence,label_tag,label_id):
    idx=sentence.copy()
    for i in range(len(sentence)):
        if sentence[i] in label_tag:
           idx[i]=label_id[label_tag.index(sentence[i])]
        else:
            idx[i]=len(label_tag)
    return idx

#分割句子，将list转化为以句子为单元的嵌套list
def sentence_segmention(words_sequences,label_sequences,pos_sequences):
    Index_fullstop = [i for i, x in enumerate(words_sequences) if x == '###']
    # print(Index_fullstop)

    words_sentences_nest=[]

    labels_sentences_nest=[]

    pos_sentences_nest=[]

    # char_words=[]
    char_sentences_nest=[]

    # 单独插入第一句话
    first_sentence_word = words_sequences[:Index_fullstop[0] + 1]
    words_sentences_nest.append(first_sentence_word)
    # print("第一句话word",len(words_sequences[:Index_fullstop[0]+1]),words_sequences[:Index_fullstop[0]+1])

    first_sentence_label = label_sequences[:Index_fullstop[0] + 1]
    labels_sentences_nest.append(first_sentence_label)
    # print("第一句话label", len(label_sequences[:Index_fullstop[0] + 1]),label_sequences[:Index_fullstop[0] + 1])

    first_sentence_pos = pos_sequences[:Index_fullstop[0] + 1]
    pos_sentences_nest.append(first_sentence_pos)
    # print("第一句话pos", len(pos_sequences[:Index_fullstop[0] + 1]),pos_sequences[:Index_fullstop[0] + 1])

    first_sentence_word_copy = first_sentence_word.copy()
    for i in range(len(first_sentence_word)):
        first_sentence_word_char = list(first_sentence_word[i])
        first_sentence_word_copy[i] = first_sentence_word_char

    char_sentences_nest.append(first_sentence_word_copy)
    # print("第一句话char", len(first_sentence_word_copy),first_sentence_word_copy)

    for j in range(len(Index_fullstop)):
        if j % 100 == 0:
            print(j)

        if j != len(Index_fullstop) - 1:
            # 单词（word）
            word_sentence = words_sequences[Index_fullstop[j] + 1:Index_fullstop[j + 1] + 1]
            words_sentences_nest.append(word_sentence)
            # print('word_sentence:',len(word_sentence),word_sentence)

            # 词性（part of speech）
            pos_sentence = pos_sequences[Index_fullstop[j] + 1:Index_fullstop[j + 1] + 1]
            pos_sentences_nest.append(pos_sentence)
            # print('pos_sentence:', len(pos_sentence), pos_sentence)

            # 标签（label）
            label_sentence = label_sequences[Index_fullstop[j] + 1:Index_fullstop[j + 1] + 1]
            labels_sentences_nest.append(label_sentence)
            # print('label_sentence:', len(label_sentence), label_sentence)

            # 字符（char）
            word_sentence_copy = word_sentence.copy()
            for k in range(len(word_sentence)):
                word_char = list(str(word_sentence[k]))
                word_sentence_copy[k] = word_char
            char_sentences_nest.append(word_sentence_copy)
            # print('char_sentence',len(word_sentence_copy),word_sentence_copy)

    return words_sentences_nest, labels_sentences_nest, pos_sentences_nest, char_sentences_nest

def create_inputs(words_sentences_nest,pos_sentences_nest,char_sentences_nest,labels_sentences_nest):

    maxlenth_sentence=256
    maxlenth_word=64

    input_idx=[]
    token_type_idx=[]
    attention_mask_idx=[]
    pos_idx=[]
    label_idx=[]
    char_idx=[]
    sentences_lenth=[]
    #调用BERT的分词类，用于token转换
    tokenizer = DebertaTokenizer.from_pretrained('kamalkraj/deberta-base')

    # 定义词性与其id的对应list
    pos_tag=["ADP","DET","NOUN","ADJ","PUNCT","VERB","AUX","PART","ADV","PRON","PROPN","NUM","CCONJ","X","SYM","SCONJ","INTJ"]
    pos_tag_id=list(range(len(pos_tag)))
    # print("pos_tag_id:",pos_tag_id)

    #定义标签与其id对应的list
    label_tag=['O','B-geo','I-geo']
    label_tag_id=list(range(len(label_tag)))
    # print(label_tag_id)

    #定义字符与其id的对应list
    char_tag=list(string.printable)
    char_tag_id=list(range(len(char_tag)))
    # print("char_tag_id:",char_tag_id)

    for i in range(len(words_sentences_nest)):
        if i%100==0:
            print(i)
        sentence_lenth=len(words_sentences_nest[i])
        sentences_lenth.append(sentence_lenth)
        # print("句子长度：",i,sentence_lenth)
        #单词索引转换，不使用BERT的分词器
        input_id=tokenizer.convert_tokens_to_ids(words_sentences_nest[i])
        input_id_padding=input_id+[0]*(maxlenth_sentence-len(input_id))
        input_idx.append(input_id_padding)
        # print("input_id:", len(input_id_padding), input_id_padding)

        #token类型，单个句子作为输入，全都是0，两个句子输入的话，一个句子为0，一个句子为1，这里是单个句子，全都设置为0
        token_type_id=[0] * len(input_id_padding)
        token_type_idx.append(token_type_id)
        # print("token_type:",len(token_type_id),token_type_id)

        #attention标记，1为计算attention，0为padding部分不计算
        attention_mask_id = [1] * len(input_id)+ ([0] * (maxlenth_sentence-len(input_id)))
        attention_mask_idx.append(attention_mask_id)
        # print("attention_mask:",len(attention_mask_id),attention_mask_id)

        #pos_idx
        pos_id=idx_switch(pos_sentences_nest[i],pos_tag,pos_tag_id)+([0] * (maxlenth_sentence-len(pos_sentences_nest[i])))
        pos_idx.append(pos_id)
        # print("pos_id:",len(pos_id),pos_id)

        # label_idx
        # print("labels_sentence_nest_i:", i, len(labels_sentences_nest[i]), labels_sentences_nest[i])
        # print("label转换：", i, len(idx_switch(labels_sentences_nest[i], label_tag, label_tag_id)),idx_switch(labels_sentences_nest[i], label_tag, label_tag_id))

        label_id = idx_switch(labels_sentences_nest[i], label_tag, label_tag_id) + ([0] * (maxlenth_sentence - len(labels_sentences_nest[i])))
        label_idx.append(label_id)
        # print("label_id:", len(label_id), label_id)

        #char_idx（记得在句子级别重新padding一次）
        char_sentences_id=char_sentences_nest[i].copy()
        for j in range(len(char_sentences_nest[i])):
            if j%100==0:
                print(j)
            char_id=idx_switch(char_sentences_nest[i][j],char_tag,char_tag_id)+([0] * (maxlenth_word-len(char_sentences_nest[i][j])))
            char_sentences_id[j]=char_id

        #padding
        char_id_padding=char_sentences_id+[[0] * maxlenth_word]*(maxlenth_sentence-len(char_sentences_id))

        # print("char_id:",len(char_id_padding),char_id_padding)
        char_idx.append(char_id_padding)

    # print("分割：----------------------------------------------------------------------------")
    # print("分割：----------------------------------------------------------------------------")
    # print("分割：----------------------------------------------------------------------------")
    # print("input_idx:",len(input_idx),input_idx)
    # print("token_type_idx:", len(token_type_idx), token_type_idx)
    # print("attention_mask_idx:", len(attention_mask_idx), attention_mask_idx)
    # print("pos_idx:", len(pos_idx), pos_idx)
    # print("char_idx:", len(char_idx), char_idx)
    # print("label_idx:", len(label_idx), label_idx)
    # print("每个句子的长度为：",sentences_lenth)
    # wp_sentence=pd.DataFrame({"句子长度":sentences_lenth})
    # wp_sentence.to_excel("句子长度.xlsx",index=False)
    return input_idx,token_type_idx,attention_mask_idx,pos_idx,char_idx,label_idx,sentences_lenth

def create_model():

    HIDDEN_SIZE = 64
    MAX_LEN = 256
    CLASS_NUMS = 3
    POS_SIZE=18
    word_maxlen=64
    char_maxlen=101

    # 采用绝对位置embedding时，relative_attention设置为False, position_bias_input设置为True
    configuration = DebertaConfig.from_pretrained('kamalkraj/deberta-base',output_attentions=True, output_hidden_states=True,use_cache=True, return_dict=True)
    encoding = TFDebertaModel.from_pretrained('kamalkraj/deberta-base',config=configuration)
    print(encoding.config)

    # hub调用BERT需要将数据类型设置为tf.int32
    input_ids = Input(shape=(MAX_LEN,), dtype='int32', name="input_ids")
    token_type_ids = Input(shape=(MAX_LEN,), dtype='int32', name="token_type_ids")
    attention_mask = Input(shape=(MAX_LEN,), dtype='int32', name="attention_mask")

    # BERT[2]为所有的隐藏层（Transformer的encoder部分）
    Bert = encoding(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
    # Bert = encoding(input_ids=input_ids, attention_mask=attention_mask)
    print(len(Bert))

    # 最后一个隐藏层序列，所有句子的单词组成
    Bert_Last_Hidden = Bert[0]
    print('最后一层:', Bert_Last_Hidden)

    # 池化层是将[CLS]标记对应的表示取出来，并做一定的变换，作为整个序列的表示并返回，以及原封不动地返回所有的标记表示，其中，激活函数默认是tanh。
    Hidden_layer = Bert[1]
    print('Hidden_layer:', Hidden_layer)
    print('Hidden_layer_lenth:', len(Hidden_layer))
    print("...................................................")
    # # 所有的隐藏层
    # Hidden_layer = Bert[2]
    # for i in range(len(Hidden_layer)):
    #     if i == 0:
    #         print("Embedding层:", Hidden_layer[i])
    #     else:
    #         print("第", i, "个隐藏层(Transformer的encoder部分):", Hidden_layer[i])
    print("...................................................")
    print("后四个隐藏层")
    Hidden_layer_12 = Hidden_layer[12]
    Hidden_layer_11 = Hidden_layer[11]
    Hidden_layer_10 = Hidden_layer[10]
    Hidden_layer_9 = Hidden_layer[9]

    print(Hidden_layer_12)
    print(Hidden_layer_11)
    print(Hidden_layer_10)
    print(Hidden_layer_9)

    # 拼接最后四个隐藏层
    Bert_Last_Four_Hidden = concat([Hidden_layer_9, Hidden_layer_10, Hidden_layer_11, Hidden_layer_12], axis=-1)
    print(Bert_Last_Four_Hidden)

    """""
    input_dim:字典长度，即输入数据最大下标+1,注意，如果字典从1开始编码，输入维度需要设置为字典长度加1
    output_dim:代表全连接嵌入的维度
    """
    #pos_Embedding
    input_pos = Input(shape=(None,), dtype='int32', name="input_pos")
    pos_embedding = Embedding(input_dim=POS_SIZE, output_dim=POS_SIZE, dtype='float32', name='POS_Embedding')(input_pos)

    #char_embedding
    input_char = Input(shape=(None, word_maxlen), dtype='float32')
    # input_dim词汇表数量，最大索引值+1
    char_embedding = TimeDistributed(Embedding(input_dim=char_maxlen, output_dim=64,embeddings_initializer=
                                     RandomUniform(minval=-0.5, maxval=0.5)))(input_char)

    char_lstm = TimeDistributed(Bidirectional(LSTM(25, return_sequences=False, return_state=False)),name='char_LSTM')(char_embedding)

    #将Bert_Embedding和POS_Embedding、char_Embedding做拼接
    #Concatenated_Embedding = concat([Bert_Last_Four_Hidden, char_lstm,pos_embedding], axis=-1, name='Concatenated_Embedding')
    Concatenated_Embedding = concat([Bert_Last_Four_Hidden, char_lstm, pos_embedding], axis=-1,name='Concatenated_Embedding')

    # Bi-LSTM层
    Bilstm_layer = Bidirectional(LSTM(HIDDEN_SIZE,return_sequences=True,dropout=0.50))(Concatenated_Embedding)

    #self-Attention层
    #Attention_Layer=Attention(name='self-attention')([Bilstm_layer,Bilstm_layer])

    Full_connection_layer = TimeDistributed(Dense(3))(Bilstm_layer)

    # CRF层
    crf = CRF(CLASS_NUMS, name='crf_layer')
    outputs = crf(Full_connection_layer)

    # 创建模型
    model = Model(inputs=[input_ids, token_type_ids, attention_mask, input_char,input_pos], outputs=outputs)

    model.summary()
    model.compile(loss=crf.loss, optimizer=tf.keras.optimizers.Adam(learning_rate=2.5e-5), metrics=crf.accuracy)
    return model
# def step_decay(epoch):
#     init_lrate=0.1
#     drop=0.5
#     epochs_drop=10
#     lrate=init_lrate*pow(drop,floor(1+epoch)/epochs_drop)
#     return lrate
#设置回调函数，当训练精度达到0.9999时，停止
class myCallback(tf.keras.callbacks.Callback):
    def on_epoch_end(self, epoch, logs=None):
        if (logs.get('loss')<0.1):
            self.model.stop_training=True

if __name__ == '__main__':

    callback=myCallback()
    #解决显存溢出问题
    # physical_devices = tf.config.experimental.list_physical_devices('GPU')
    #
    # if len(physical_devices) > 0:
    #     for k in range(len(physical_devices)):
    #         tf.config.experimental.set_memory_growth(physical_devices[k], True)
    #         tf.config.experimental
    #         print('memory growth:', tf.config.experimental.get_memory_growth(physical_devices[k]))
    # else:
    #     print("Not enough GPU hardware devices available")

    # Data_NUM=5000
    #input_url_train="G:\\Project\\Topic Phrases Extraction\\topic_phrase_data\\final result\\train_data\\paper_"+str(Data_NUM)+"+wiki_3000.csv"
    input_url_train = "D:\\SCI Paper\\Geographic entities Extraction\\Data\\Train Data\\subword\\Ontonote&wiki1500_subword.csv"
    train_data=file_read(input_url_train)
    words_sequences=train_data[0]
    labels_sequences=train_data[1]
    pos_sequences=train_data[2]
    #print(words_sequences)

    #按句子处理成嵌套list
    data_sequence_nest=sentence_segmention(words_sequences,labels_sequences,pos_sequences)
    words_sentences_nest=data_sequence_nest[0]
    labels_sentences_nest=data_sequence_nest[1]
    pos_sentences_nest=data_sequence_nest[2]
    char_sentences_nest=data_sequence_nest[3]

    #将训练数据转成id，并做单词、词性、字符的padding
    train_data_id=create_inputs(words_sentences_nest,pos_sentences_nest,char_sentences_nest,labels_sentences_nest)

    #BERT模型输入
    input_idx=train_data_id[0]
    token_type_idx=train_data_id[1]
    attention_mask_idx=train_data_id[2]

    #新增输入特征
    pos_idx=train_data_id[3]
    char_idx=train_data_id[4]

    #输出并转one-hot
    # print("lable_idx为",train_data_id[5])
    print("train_lenth:",len(train_data_id[5]),train_data_id[5])
    # test1 = train_data_id[5][0]
    # test2 = train_data_id[5][1][0:300]
    # test = [test1, test2]
    label_idx=to_categorical(train_data_id[5],3)

    # print("lable_idx为(one-hot)：",label_idx)
    #创建模型
    model=create_model()
    #设置学习率衰减
    # lrate=ReduceLROnPlateau(monitor='loss',patience=1,mode='auto',factor=0.1)
    early_stopping=tf.keras.callbacks.EarlyStopping(monitor='loss',min_delta=0,patience=3,mode='auto',baseline=None,restore_best_weights=False,verbose=1)
    #模型训练
    model.fit([np.array(input_idx), np.array(token_type_idx), np.array(attention_mask_idx),np.array(char_idx),np.array(pos_idx)], label_idx, epochs=100, verbose=1, batch_size=16,callbacks=[callback])

    print("模型训练完成。。。。。。。。。。。。。。。。。。。。。。。")

    #将测试数据处理成与训练数据相同的格式
    input_url_test="D:\\SCI Paper\\Geographic entities Extraction\\Data\\Train Data\\subword\\Test_Data_Literature_subword_fixed.xlsx"
    test_data_read=pd.read_excel(input_url_test)
    words_sequences_test = list(test_data_read['word'])#左闭右开，不包括135，0到134[0:135]
    labels_sequences_test = list(test_data_read['annotation'])
    pos_sequences_test = list(test_data_read['pos_tag'])

    #按句子处理成嵌套list
    data_sequence_nest_test=sentence_segmention(words_sequences_test,labels_sequences_test,pos_sequences_test)
    words_sentences_nest_test=data_sequence_nest_test[0]
    labels_sentences_nest_test=data_sequence_nest_test[1]
    pos_sentences_nest_test=data_sequence_nest_test[2]
    char_sentences_nest_test=data_sequence_nest_test[3]

    # 将训练数据转成id，并做单词、词性、字符的padding
    test_data_id = create_inputs(words_sentences_nest_test, pos_sentences_nest_test, char_sentences_nest_test, labels_sentences_nest_test)

    # BERT模型输入
    input_idx_test = test_data_id[0]
    token_type_idx_test = test_data_id[1]
    attention_mask_idx_test = test_data_id[2]

    #新增输入特征
    pos_idx_test = test_data_id[3]
    char_idx_test = test_data_id[4]

    #模型预测
    predict_lables_one_hot=model.predict([np.array(input_idx_test),np.array(token_type_idx_test),np.array(attention_mask_idx_test), np.array(char_idx_test),np.array(pos_idx_test)],verbose=1)

    #print(predict_lables_one_hot)

    #one-hot转list,并剔除padding过的0
    sentence_lenth_test=test_data_id[6]
    predict_result_idx=[]
    predict_result_labels=[]
    for i in range(len(predict_lables_one_hot)):
        predict_label_id=np.argmax(predict_lables_one_hot[i],axis=1)
        predict_id=list(predict_label_id[0:sentence_lenth_test[i]])
        predict_id_copy = predict_id.copy()
        for j in range(len(predict_id)):
            if predict_id[j]==0:
                predict_id_copy[j]="O"
            elif predict_id[j]==1:
                predict_id_copy[j] = "B-geo"
            else:
                predict_id_copy[j] = "I-geo"
        predict_result_labels.append(predict_id_copy)
        predict_result_idx.append(predict_id)

    # print("predict_result_idx:", predict_result_idx)
    # print("predict_result_labels:",predict_result_labels)
    print("预测完成。。。。。。。。。。。。。。。。。。。。。。。。。。")

    #保存预测结果
    #Output_url="G:\\Project\\Topic Phrases Extraction\\topic_phrase_data\\final result\\baseline result\\paper_"+str(Data_NUM)+"+wiki_3000_predict_result(epoch=30,lr=3e-5,absolutely).csv"
    Output_url = "D:\\SCI Paper\\Geographic entities Extraction\\Data\\Results\\Ontonote&wiki1500_subword_test_result(lr=2.5e-5_absolute3).csv"
    result_DataFrame = pd.DataFrame({"predict_results": list(itertools.chain.from_iterable(predict_result_labels))})
    result_DataFrame.to_csv(Output_url, index=False)