text:
import streamlit as st
import pandas as pd
import numpy as np
import altair as alt
import os
import time
from datetime import datetime
import pickle
import itertools
import scipy.optimize as sco
import plotly.express as px
from plt_setup import finastra_theme
from download_data import Data
from Markowitz_Portfolio import *
import sys

import metadata_parser

####### CACHED FUNCTIONS ######
@st.cache(show_spinner=False, suppress_st_warning=True)
def filter_company_data(df_company, esg_categories, start, end):
    #Filter E,S,G Categories
    comps = []
    for i in esg_categories:
        X = df_company[df_company[i] == True]
        comps.append(X)
    df_company = pd.concat(comps)
    # df_company = df_company[(df_company.DATE >= start) &
    #                         (df_company.DATE <= end)]
    df_company = df_company[df_company.DATE.between(start, end)]
    return df_company

@st.cache(show_spinner=False, suppress_st_warning=True,
          allow_output_mutation=True)
def load_data(start_data, end_data, flag):
    data = Data().read(start_data, end_data, flag)
    companies = data["data"].Organization.sort_values().unique().tolist()
    if flag == 'USA':
        companies.remove('netflix')
        companies.insert(0,"netflix")
    elif flag == 'UK':
        companies.remove('astrazeneca')
        companies.insert(0,"astrazeneca")
    elif flag == 'CANADA':
        companies.remove('enbridge')
        companies.insert(0, 'enbridge')
    #elif flag == 'AUSTRALIA':
    #    companies.remove('national australia bank ltd')
    #    companies.insert(0, 'national australia bank ltd')
    return data, companies

@st.cache(show_spinner=False,suppress_st_warning=True)
def filter_publisher(df_company,publisher):
    if publisher != 'all':
        df_company = df_company[df_company['SourceCommonName'] == publisher]
    return df_company

def get_melted_frame(data_dict, frame_names, keepcol=None, dropcol=None):
    if keepcol:
        reduced = {k: df[keepcol].rename(k) for k, df in data_dict.items()
                   if k in frame_names}
    else:
        reduced = {k: df.drop(columns=dropcol).mean(axis=1).rename(k)
                   for k, df in data_dict.items() if k in frame_names}
    df = (pd.concat(list(reduced.values()), axis=1).reset_index().melt("date")
            .sort_values("date").ffill())
    df.columns = ["DATE", "ESG", "Score"]
    return df.reset_index(drop=True)

def filter_on_date(df, start, end, date_col="DATE"):
    df = df[(df[date_col] >= pd.to_datetime(start)) &
            (df[date_col] <= pd.to_datetime(end))]
    return df

def get_clickable_name(url):
    try:
        T = metadata_parser.MetadataParser(url=url, search_head_only=True)
        title = T.metadata["og"]["title"].replace("|", " - ")
        return f"[{title}]({url})"
    except:
        try:
            title = []
            for i in range(len(list(url))-1,0,-1):
                if list(url)[i] == '/' and len(title)!=0:
                    break
                else:
                    if list(url)[i] == '/':
                        pass
                    else:
                        title.append(list(url)[i])

            title = ''.join(reversed(title))
            return f"[{title}]({url})"
        except:
            return f"[{url}]({url})"

def main(start_data, end_data):
    ###### CUSTOMIZE COLOR THEME ######
    alt.themes.register("finastra", finastra_theme)
    alt.themes.enable("finastra")
    violet, fuchsia = ["#694ED6", "#C137A2"]

    ###### SET UP PAGE ######
    icon_path = os.path.join("./img", "KBlogo.jpg")
    st.set_page_config(page_title="ESG AI", page_icon=icon_path,
                       layout='centered', initial_sidebar_state="collapsed")
    _, logo, _ = st.columns(3)
    logo.image(icon_path, width=230)
    style = ("text-align:center; padding: 0px; font-family: arial black;, "
             "font-size: 400%")
    title = f"
ESGAI

"
    st.write(title, unsafe_allow_html=True)

    ####### CREATE SIDEBAR CATEGORY FILTER######
    st.sidebar.title("Please Choose Options!")
    date_place = st.sidebar.empty()
    esg_categories = st.sidebar.multiselect("Select Articles' Theme among ESG",
                                            ["E", "S", "G"], ["E", "S", "G"])
    pub = st.sidebar.empty()
    num_neighbors = st.sidebar.slider("Select Number of Similar Company for Portfolio", 1, 20, value=8)

    ###### LayOut ######
    box1, box2 = st.columns([1, 1.5])

    ###### LOAD DATA ######
    ###default setting###
    market = 'USA'
    INFO = 'Please Choose your Stock'
    state = []
    with box1:
        st.write('', unsafe_allow_html=True)
        market_options = ["USA", "UK", "CANADA"]
        market_metric = st.radio("Please Select your Market", options=market_options)
    with box2:
        with st.spinner(text="Fetching Data..."):
            data, companies = load_data(start_data, end_data, market_metric)

            df_conn = data["conn"]
            df_data = data["data"]
            embeddings = data["embed"]

        company = st.selectbox(INFO, companies)

    ###### FILTER ######
    df_company = df_data[df_data.Organization == company]
    diff_col = f"{company.replace(' ', '_')}_diff"
    tone_df = get_melted_frame(data, ["overall_score"], keepcol=diff_col)
    ind_tone_df = get_melted_frame(data, ["overall_score"],
                                   dropcol="industry_tone")

    esg_keys = ["E_score", "S_score", "G_score"]
    e_df = get_melted_frame(data, esg_keys[0], keepcol=diff_col)
    ind_e_df = get_melted_frame(data, esg_keys[0], dropcol="industry_tone")
    s_df = get_melted_frame(data, esg_keys[1], keepcol=diff_col)
    ind_s_df = get_melted_frame(data, esg_keys[1], dropcol="industry_tone")
    g_df = get_melted_frame(data, esg_keys[2], keepcol=diff_col)
    ind_g_df = get_melted_frame(data, esg_keys[2], dropcol="industry_tone")

    ###### DATE WIDGET ######
    start = df_company.DATE.min()
    end = df_company.DATE.max()
    selected_dates = date_place.date_input("Select Date",
        value=[start, end], min_value=start, max_value=end, key=None)
    time.sleep(0.8)  #Allow user some time to select the two dates -- hacky :D
    start, end = selected_dates

    ###### FILTER DATA ######
    df_company = filter_company_data(df_company, esg_categories,
                                     start, end)
    df_company = df_company.drop_duplicates(keep='last', subset=['DATE', 'SourceCommonName'])
    tone_df = filter_on_date(tone_df, start, end)
    ind_tone_df = filter_on_date(ind_tone_df, start, end)
    date_filtered = filter_on_date(df_data, start, end)

    e_df = filter_on_date(e_df, start, end)
    ind_e_df = filter_on_date(ind_e_df, start, end)
    s_df = filter_on_date(s_df, start, end)
    ind_s_df = filter_on_date(ind_s_df, start, end)
    g_df = filter_on_date(g_df, start, end)
    ind_g_df = filter_on_date(ind_g_df, start, end)

    ###### PUBLISHER SELECT BOX ######
    publishers = df_company.SourceCommonName.sort_values().unique().tolist()
    publishers.insert(0, "all")
    publisher = pub.selectbox("Select Publisher", publishers)
    df_company = filter_publisher(df_company, publisher)

    ###### DISPLAY DATA ######
    URL_Expander = st.expander(f"{company.title()}'s Data: ", True)
    URL_Expander.write(f"### Chosen {company.title()}'s {len(df_company):,d} Article ESG Tone Table")
    display_cols = ["DATE", "SourceCommonName", "Tone", "Polarity",
                    "NegativeTone", "PositiveTone"]  #  "WordCount"
    URL_Expander.write(df_company[display_cols][::-1])

    ####
    URL_Expander.write(f"#### Sample Article Information")
    link_df = df_company[["DATE", "URL"]].drop_duplicates(keep='last', subset='DATE').tail(5).copy()
    # link_df["URL"] = link_df["URL"].apply(lambda R: f"[{R}]({R})")
    link_df["ARTICLE"] = link_df.URL.apply(get_clickable_name)
    link_df = link_df[["DATE", "ARTICLE"]][::-1].to_markdown(index=False)
    URL_Expander.markdown(link_df)
    ####

    ###### CHART: METRIC OVER TIME ######
    st.markdown("---")

    ###### NUMBER OF NEIGHBORS TO FIND #####
    neighbor_cols = [f"n{i}_rec" for i in range(num_neighbors)]
    company_df = df_conn[df_conn.company == company]
    try:
        neighbors = company_df[neighbor_cols].iloc[0]
    except:
        neighbors = []
        print("There is no similar company!")

    col1, col2 = st.columns((1, 4))
    metric_options = ["Tone", "NegativeTone", "PositiveTone", "Polarity",
                      "WordCount", "E Score", "S Score", "G Score"]
    line_metric = col1.radio("Please Select Evaluation Metric", options=metric_options)

    if line_metric == "E Score":
        # Get E Scores
        e_df["WHO"] = company.title()
        ind_e_df["WHO"] = "Industry Average"
        esg_plot_df = pd.concat([e_df, ind_e_df]
                                ).reset_index(drop=True)

        esg_plot_df.replace({"E_score": "Environment"}, inplace=True)

        metric_chart = alt.Chart(esg_plot_df, title=f"{line_metric} TimeSeries Graph", padding={"left": 30, "top": 1, "right": 10, "bottom": 1}
                                   ).mark_line().encode(
            x=alt.X("yearmonthdate(DATE):O", title=""), #title="DATE"
            y=alt.Y("Score:Q", title="E Score"),
            color=alt.Color("WHO", sort=None, legend=alt.Legend(
                title=None, orient="top")),
            strokeDash=alt.StrokeDash("WHO", sort=None, legend=alt.Legend(
                title=None, symbolType="stroke", symbolFillColor="gray",
                symbolStrokeWidth=4, orient="top")),
            tooltip=["DATE", alt.Tooltip("Score", format=".5f")]
            )
    elif line_metric == "S Score":
        # Get E Scores
        s_df["WHO"] = company.title()
        ind_s_df["WHO"] = "Industry Average"
        esg_plot_df = pd.concat([s_df, ind_s_df]
                                ).reset_index(drop=True)

        esg_plot_df.replace({"S_score": "Social"}, inplace=True)

        metric_chart = alt.Chart(esg_plot_df, title=f"{line_metric} TimeSeries Graph", padding={"left": 30, "top": 1, "right": 10, "bottom": 1}
                                   ).mark_line().encode(
            x=alt.X("yearmonthdate(DATE):O", title=""), #title="DATE"
            y=alt.Y("Score:Q", title="S Score"),
            color=alt.Color("WHO", sort=None, legend=alt.Legend(
                title=None, orient="top")),
            strokeDash=alt.StrokeDash("WHO", sort=None, legend=alt.Legend(
                title=None, symbolType="stroke", symbolFillColor="gray",
                symbolStrokeWidth=4, orient="top")),
            tooltip=["DATE", alt.Tooltip("Score", format=".5f")]
            )
    elif line_metric == "G Score":
        # Get E Scores
        g_df["WHO"] = company.title()
        ind_g_df["WHO"] = "Industry Average"
        esg_plot_df = pd.concat([g_df, ind_g_df]
                                ).reset_index(drop=True)

        esg_plot_df.replace({"G_score": "Governance"}, inplace=True)

        metric_chart = alt.Chart(esg_plot_df, title=f"{line_metric} TimeSeries Graph", padding={"left": 30, "top": 1, "right": 10, "bottom": 1}
                                   ).mark_line().encode(
            x=alt.X("yearmonthdate(DATE):O", title=""), #title="DATE"
            y=alt.Y("Score:Q", title="G Score"),
            color=alt.Color("WHO", sort=None, legend=alt.Legend(
                title=None, orient="top")),
            strokeDash=alt.StrokeDash("WHO", sort=None, legend=alt.Legend(
                title=None, symbolType="stroke", symbolFillColor="gray",
                symbolStrokeWidth=4, orient="top")),
            tooltip=["DATE", alt.Tooltip("Score", format=".5f")]
            )
    else:
        df1 = df_company.groupby("DATE")[line_metric].mean(
            ).reset_index()
        df2 = filter_on_date(df_data.groupby("DATE")[line_metric].mean(
            ).reset_index(), start, end)
        df1["WHO"] = company.title()
        df2["WHO"] = "Industry Average"
        plot_df = pd.concat([df1, df2]).reset_index(drop=True)
        metric_chart = alt.Chart(plot_df, title=f"{line_metric} TimeSeries Graph", padding={"left": 40, "top": 1, "right": 10, "bottom": 1}
                             ).mark_line().encode(
        x=alt.X("yearmonthdate(DATE):O", title=""),
        y=alt.Y(f"{line_metric}:Q", scale=alt.Scale(type="linear")),
        color=alt.Color("WHO", legend=None),
        strokeDash=alt.StrokeDash("WHO", sort=None,
            legend=alt.Legend(
                title=None, symbolType="stroke", symbolFillColor="gray",
                symbolStrokeWidth=4, orient="top",
                ),
            ),
        tooltip=["DATE", alt.Tooltip(line_metric, format=".3f")]
        )
    metric_chart = metric_chart.properties(
        height=340,
        width=200
    ).interactive()
    col2.altair_chart(metric_chart, use_container_width=True)

    empty, box = st.columns((1, 4))
    with empty:
        pass
    with box:
        with st.expander("Spread Out"):
            for i in range(1, len(metric_options)-3):
                df1 = df_company.groupby("DATE")[metric_options[i]].mean(
                ).reset_index()
                df2 = filter_on_date(df_data.groupby("DATE")[metric_options[i]].mean(
                    ).reset_index(), start, end)
                df1["WHO"] = company.title()
                df2["WHO"] = "Industry Average"
                plot_df = pd.concat([df1, df2]).reset_index(drop=True)
                metric_chart = alt.Chart(plot_df, title=f"{metric_options[i]} TimeSeries Graph", padding={"left": 40, "top": 1, "right": 10, "bottom": 1}
                                         ).mark_line().encode(
                    x=alt.X("yearmonthdate(DATE):O", title=""),
                    y=alt.Y(f"{metric_options[i]}:Q", scale=alt.Scale(type="linear")),
                    color=alt.Color("WHO", legend=None),
                    strokeDash=alt.StrokeDash("WHO", sort=None,
                        legend=alt.Legend(
                            title=None, symbolType="stroke", symbolFillColor="gray",
                            symbolStrokeWidth=4, orient="top",
                            ),
                        ),
                    tooltip=["DATE", alt.Tooltip(metric_options[i], format=".3f")]
                    )
                metric_chart = metric_chart.properties(
                    height=340,
                    width=200
                ).interactive()
                st.altair_chart(metric_chart, use_container_width=True)
            for i in range(3):
                if i == 0:
                    e_df["WHO"] = company.title()
                    ind_e_df["WHO"] = "Industry Average"
                    esg_plot_df = pd.concat([e_df, ind_e_df]
                                            ).reset_index(drop=True)
                    esg_plot_df.replace({"E_score": "Environment"}, inplace=True)
                elif i==1:
                    s_df["WHO"] = company.title()
                    ind_s_df["WHO"] = "Industry Average"
                    esg_plot_df = pd.concat([s_df, ind_s_df]
                                            ).reset_index(drop=True)
                    esg_plot_df.replace({"S_score": "Social"}, inplace=True)
                else:
                    g_df["WHO"] = company.title()
                    ind_g_df["WHO"] = "Industry Average"
                    esg_plot_df = pd.concat([g_df, ind_g_df]
                                            ).reset_index(drop=True)

                    esg_plot_df.replace({"G_score": "Governance"}, inplace=True)

                metric_chart = alt.Chart(esg_plot_df, title=f"{metric_options[len(metric_options)-3+i]} TimeSeries Graph", padding={"left": 30, "top": 1, "right": 10, "bottom": 1}
                                           ).mark_line().encode(
                    x=alt.X("yearmonthdate(DATE):O", title=""), #title="DATE"
                    y=alt.Y("Score:Q", title=metric_options[len(metric_options)-3+i]),
                    color=alt.Color("WHO", sort=None, legend=alt.Legend(
                        title=None, orient="top")),
                    strokeDash=alt.StrokeDash("WHO", sort=None, legend=alt.Legend(
                        title=None, symbolType="stroke", symbolFillColor="gray",
                        symbolStrokeWidth=4, orient="top")),
                    tooltip=["DATE", alt.Tooltip("Score", format=".5f")]
                    )
                metric_chart = metric_chart.properties(
                    height=340,
                    width=200
                ).interactive()
                st.altair_chart(metric_chart, use_container_width=True)

    st.markdown("---")
    st.write('', unsafe_allow_html=True)
    choose_graph = ["ESG Rader", "Tone Density", "Polarity Graph", "Company Distribution", "Similarity Company & Score"]
    if len(neighbors)==0:
        choose_graph = ["ESG Rader", "Tone Density", "Polarity Graph"]
    graph_metric = st.radio("Please Select your Graph", options=choose_graph)

    ###### CHART: ESG RADAR ######
    if graph_metric == 'ESG Rader':
        avg_esg = data["ESG"]
        avg_esg.rename(columns={"Unnamed: 0": "Type"}, inplace=True)
        avg_esg.replace({"T": "Overall", "E": "Environment",
                         "S": "Social", "G": "Governance"}, inplace=True)
        avg_esg["Industry Average"] = avg_esg.mean(axis=1)

        radar_df = avg_esg[["Type", company, "Industry Average"]].melt("Type",
            value_name="score", var_name="entity")

        radar = px.line_polar(radar_df, r="score", theta="Type",
            color="entity", line_close=True, hover_name="Type",
            hover_data={"Type": True, "entity": True, "score": ":.2f"},
            color_discrete_map={"Industry Average": fuchsia, company: violet})
        radar.update_layout(template=None,
                            polar={
                                   "radialaxis": {"showticklabels": False,
                                                  "ticks": ""},
                                   "angularaxis": {"showticklabels": False,
                                                   "ticks": ""},
                                   },
                            legend={"title": None, "yanchor": "middle",
                                    "orientation": "h"},
                            title={"text": "ESG Rader Chart",
                                   "x": 0.15, "y": 0.93,
                                   "xanchor": "center",
                                   "yanchor": "top",
                                   "font": {"family": "Futura", "size": 23}},
                            margin={"l": 5, "r": 5, "t": 0, "b": 0},
                            )#0.8875
        radar.update_layout(showlegend=False)
        st.plotly_chart(radar, use_container_width=True)

    elif graph_metric == 'Tone Density':
        ###### CHART: DOCUMENT TONE DISTRIBUTION #####
        # add overall average
        dist_chart = alt.Chart(df_company, title="All Publishers' ESG Tone Density Chart", padding={"left": 1, "top": 10, "right": 25, "bottom": 1}
                               ).transform_density(
                density='Tone',
                as_=["Tone", "density"]
            ).mark_area(opacity=0.5,color="purple").encode(
                    x=alt.X('Tone:Q', scale=alt.Scale(domain=(-10, 10)), title="Tone"),
                    y=alt.Y('density:Q', title="Density"),
                    tooltip=[alt.Tooltip("Tone", format=".3f"),
                             alt.Tooltip("density:Q", format=".4f")]
                ).properties(
                    height=325,
                ).configure_title(
                    dy=-10
                ).interactive()
        st.markdown("###
", unsafe_allow_html=True)
        st.altair_chart(dist_chart,use_container_width=True)

    ###### CHART: SCATTER OF ARTICLES OVER TIME #####
    elif graph_metric == 'Polarity Graph':
        scatter = alt.Chart(df_company, title= "ESG Polarity Chart", padding={"left": 10, "top": 10, "right": 1, "bottom": 1}).mark_circle().encode(
            x=alt.X("NegativeTone:Q", title="Negative Tone"),
            y=alt.Y("PositiveTone:Q", title="Positive Tone"),
            size="WordCount:Q",
            color=alt.Color("Polarity:Q", scale=alt.Scale()),
            tooltip=[alt.Tooltip("Polarity", format=".3f"),
                     alt.Tooltip("NegativeTone", format=".3f"),
                     alt.Tooltip("PositiveTone", format=".3f"),
                     alt.Tooltip("DATE"),
                     alt.Tooltip("WordCount", format=",d"),
                     alt.Tooltip("SourceCommonName", title="Site")]
            ).properties(
                height=440
            ).interactive()
        st.altair_chart(scatter, use_container_width=True)

    ###### CHART: 3D EMBEDDING WITH NEIGHBORS ######
    elif graph_metric == 'Company Distribution' and len(neighbors)!=0:
        color_f = lambda f: f"Company: {company.title()}" if f == company else (
            "Connected Company" if f in neighbors.values else "Other Company")
        embeddings["colorCode"] = embeddings.company.apply(color_f)
        point_colors = {company: violet, "Connected Company": fuchsia,
                        "Other Company": "lightgrey"}
        fig_3d = px.scatter_3d(embeddings, x="0", y="1", z="2",
                               color='colorCode',
                               color_discrete_map=point_colors,
                               opacity=0.4,
                               hover_name="company",
                               hover_data={c: False for c in embeddings.columns},
                               )
        fig_3d.update_layout(legend={"orientation": "h",
                                     "yanchor": "bottom",
                                     "title": None},
                             title={"text": "Similar Company Distribution Chart",
                                    "x": 0.5, "y": 0.9,
                                    "xanchor": "center",
                                    "yanchor": "top",
                                    "font": {"family": "Futura", "size": 23}},
                             scene={"xaxis": {"visible": False},
                                    "yaxis": {"visible": False},
                                    "zaxis": {"visible": False}},
                             margin={"l": 0, "r": 0, "t": 0, "b": 0},
                             )
        st.plotly_chart(fig_3d, use_container_width=True)

    ###### CHART: NEIGHBOR SIMILIARITY ######
    elif graph_metric == 'Similarity Company & Score' and len(neighbors)!=0:
        neighbor_conf = pd.DataFrame({
            "Neighbor": neighbors,
            "Confidence": company_df[[f"n{i}_conf" for i in
                                      range(num_neighbors)]].values[0]})
        conf_plot = alt.Chart(neighbor_conf, title=f"Top {num_neighbors} Company's Similarity Score", padding={"left": 1, "top": 10, "right": 1, "bottom": 1}
                              ).mark_bar().encode(
            x=alt.X("Confidence:Q", title="Confidence"),
            y=alt.Y("Neighbor:N", sort="-x", title="Similar Company"),
            tooltip=["Neighbor", alt.Tooltip("Confidence", format=".3f")],
            color=alt.Color("Confidence:Q", scale=alt.Scale(), legend=None)
        ).properties(
            height=25 * num_neighbors + 90
        ).configure_axis(grid=False)
        st.altair_chart(conf_plot, use_container_width=True)

    with st.expander("Spread Out"):
        ###### CHART: DOCUMENT TONE DISTRIBUTION #####
        # add overall average
        dist_chart = alt.Chart(df_company, title="All Publishers' ESG Tone Density Chart", padding={"left": 1, "top": 10, "right": 25, "bottom": 1}
                               ).transform_density(
                density='Tone',
                as_=["Tone", "density"]
            ).mark_area(opacity=0.5,color="purple").encode(
                    x=alt.X('Tone:Q', scale=alt.Scale(domain=(-10, 10)), title="Tone"),
                    y=alt.Y('density:Q', title="Density"),
                    tooltip=[alt.Tooltip("Tone", format=".3f"),
                             alt.Tooltip("density:Q", format=".4f")]
                ).properties(
                    height=325,
                ).configure_title(
                    dy=-10
                ).interactive()
        st.markdown("###
", unsafe_allow_html=True)
        st.altair_chart(dist_chart,use_container_width=True)

        scatter = alt.Chart(df_company, title= "ESG Polarity Chart", padding={"left": 10, "top": 10, "right": 1, "bottom": 1}).mark_circle().encode(
            x=alt.X("NegativeTone:Q", title="Negative Tone"),
            y=alt.Y("PositiveTone:Q", title="Positive Tone"),
            size="WordCount:Q",
            color=alt.Color("Polarity:Q", scale=alt.Scale()),
            tooltip=[alt.Tooltip("Polarity", format=".3f"),
                     alt.Tooltip("NegativeTone", format=".3f"),
                     alt.Tooltip("PositiveTone", format=".3f"),
                     alt.Tooltip("DATE"),
                     alt.Tooltip("WordCount", format=",d"),
                     alt.Tooltip("SourceCommonName", title="Site")]
            ).properties(
                height=440
            ).interactive()
        st.altair_chart(scatter, use_container_width=True)

        if len(neighbors)!=0:
            color_f = lambda f: f"Company: {company.title()}" if f == company else (
                "Connected Company" if f in neighbors.values else "Other Company")
            embeddings["colorCode"] = embeddings.company.apply(color_f)
            point_colors = {company: violet, "Connected Company": fuchsia,
                            "Other Company": "lightgrey"}
            fig_3d = px.scatter_3d(embeddings, x="0", y="1", z="2",
                                   color='colorCode',
                                   color_discrete_map=point_colors,
                                   opacity=0.4,
                                   hover_name="company",
                                   hover_data={c: False for c in embeddings.columns},
                                   )
            fig_3d.update_layout(legend={"orientation": "h",
                                         "yanchor": "bottom",
                                         "title": None},
                                 title={"text": "Similar Company Distribution Chart",
                                        "x": 0.5, "y": 0.9,
                                        "xanchor": "center",
                                        "yanchor": "top",
                                        "font": {"family": "Futura", "size": 23}},
                                 scene={"xaxis": {"visible": False},
                                        "yaxis": {"visible": False},
                                        "zaxis": {"visible": False}},
                                 margin={"l": 0, "r": 0, "t": 0, "b": 0},
                                 )
            st.plotly_chart(fig_3d, use_container_width=True)

            neighbor_conf = pd.DataFrame({
            "Neighbor": neighbors,
            "Confidence": company_df[[f"n{i}_conf" for i in
                                      range(num_neighbors)]].values[0]})
            conf_plot = alt.Chart(neighbor_conf, title=f"Top {num_neighbors} Company's Similarity Score", padding={"left": 1, "top": 10, "right": 1, "bottom": 1}
                                  ).mark_bar().encode(
                x=alt.X("Confidence:Q", title="Confidence"),
                y=alt.Y("Neighbor:N", sort="-x", title="Similar Company"),
                tooltip=["Neighbor", alt.Tooltip("Confidence", format=".3f")],
                color=alt.Color("Confidence:Q", scale=alt.Scale(), legend=None)
            ).properties(
                height=25 * num_neighbors + 90
            ).configure_axis(grid=False)
            st.altair_chart(conf_plot, use_container_width=True)

    if market_metric == 'USA':
        st.markdown("---")
        portfolio1 = data["Portfolio"][company]
        portfolio2 = data["Portfolio"][neighbors]
        table = pd.concat([portfolio1, portfolio2], axis=1)

        returns = table.pct_change()
        mean_returns = returns.mean()
        cov_matrix = returns.cov()
        num_portfolios = 25000
        risk_free_rate = 0.0178

        max_sharpe = max_sharpe_ratio(mean_returns, cov_matrix, risk_free_rate)
        sdp, rp = portfolio_annualised_performance(max_sharpe['x'], mean_returns, cov_matrix)
        max_sharpe_allocation = pd.DataFrame(max_sharpe.x,index=table.columns,columns=['allocation'])
        max_sharpe_allocation.allocation = [round(i*100,2)for i in max_sharpe_allocation.allocation]
        max_sharpe_allocation = pd.DataFrame({"company": max_sharpe_allocation.T.columns, "allocation": max_sharpe_allocation.T.loc['allocation']})

        min_vol = min_variance(mean_returns, cov_matrix)
        sdp_min, rp_min = portfolio_annualised_performance(min_vol['x'], mean_returns, cov_matrix)
        min_vol_allocation = pd.DataFrame(min_vol.x,index=table.columns,columns=['allocation'])
        min_vol_allocation.allocation = [round(i*100,2)for i in min_vol_allocation.allocation]
        min_vol_allocation = pd.DataFrame({"company": min_vol_allocation.T.columns, "allocation": min_vol_allocation.T.loc['allocation']})

        an_vol = np.std(returns) * np.sqrt(252)
        an_rt = mean_returns * 252

        choose_portfolio = ["Min Vol", "Max Sharpe"]
        portfolio_metric = st.radio("Please Select your Portfolio", options=choose_portfolio)

        if portfolio_metric == 'Max Sharpe':
            pie_chart = alt.Chart(max_sharpe_allocation, title="Maximum Sharpe Ratio Portfolio Allocation").mark_arc().encode(
                        theta=alt.Theta(field="allocation", type="quantitative"),
                        color=alt.Color(field="company", type="nominal"),
                        ).properties(height=350)
            st.altair_chart(pie_chart, use_container_width=True)
            st.write("Annualised Return:", round(rp,2))
            st.write("Annualised Volatility:", round(sdp,2))
        else:
            pie_chart = alt.Chart(min_vol_allocation, title="Minimum Volatility Portfolio Allocation").mark_arc().encode(
                        theta=alt.Theta(field="allocation", type="quantitative"),
                        color=alt.Color(field="company", type="nominal"),
                        ).properties(height=350)
            st.altair_chart(pie_chart, use_container_width=True)
            st.write("Annualised Return:", round(rp_min,2))
            st.write("Annualised Volatility:", round(sdp_min,2))

        with st.expander("Spread Out"):
            pie_chart = alt.Chart(max_sharpe_allocation, title="Maximum Sharpe Ratio Portfolio Allocation").mark_arc().encode(
                        theta=alt.Theta(field="allocation", type="quantitative"),
                        color=alt.Color(field="company", type="nominal"),
                        ).properties(height=350)
            st.altair_chart(pie_chart, use_container_width=True)
            st.write("Annualised Return:", round(rp,2))
            st.write("Annualised Volatility:", round(sdp,2))

if __name__ == "__main__":
    args = sys.argv
    if len(args) != 3:
        start_data = "2022-01-01"
        end_data = "2022-04-26"
    else:
        start_data = args[1]
        end_data = args[2]
    print(start_data, end_data)
    if f"{start_data}__to__{end_data}" not in os.listdir("./Data"):
        print(f"There isn't data for {os.listdir('./Data')}")
        raise NameError(f"Please pick from {os.listdir('./Data')}")
        sys.exit()
        st.stop()
    else:
        print("streamlit start!")
        main(start_data, end_data)
    alt.themes.enable("default")

max_stars_repo_name: Seong-JiHyeon/ESG-investment

max_issues_repo_name: Seong-JiHyeon/ESG-investment

max_forks_repo_name: Seong-JiHyeon/ESG-investment


text:
import numpy as np
import cv2

# Create image from matrix
photo = np.zeros((450, 450, 3), dtype='uint8')
# RGB = BGR
photo[:] = 255, 0, 0
# Create rectangle
cv2.rectangle(photo, (0, 0), (100, 100), (119, 201, 105), thickness=cv2.FILLED)
# Create line
cv2.line(photo, (0, photo.shape[0] // 2), (photo.shape[1], photo.shape[0] // 2), (119, 201, 105), thickness=3)
# Create circle
cv2.circle(photo, (photo.shape[1] // 2, photo.shape[0] // 2), 50, (119, 201, 105), thickness=2)
# Create text
cv2.putText(photo, 'Mezidia', (100, 150), cv2.FONT_HERSHEY_TRIPLEX, 1., (255, 0, 0), 4)
cv2.imshow('Photo', photo)
cv2.waitKey(0)

ext: py

max_stars_repo_name: mezgoodle/opencv-tutorial

max_issues_repo_name: mezgoodle/opencv-tutorial

max_forks_repo_name: mezgoodle/opencv-tutorial

reason: import numpy


text:
# -*- coding: utf-8 -*-

"""
>>> with_stat("foo", "bar")
Traceback (most recent call last):
      ...
NumbaError: ...: Only 'with python' and 'with nopython' is supported at this moment
"""

from __future__ import print_function, division, absolute_import

import numba
from numba.testing import testmod

@numba.autojit(warnstyle="simple")
def with_stat(fn, msg):
    with open(fn, 'w') as fp:
        fp.write(msg)

testmod()

max_stars_repo_name: liuzhenhai/numba

max_issues_repo_name: liuzhenhai/numba

max_forks_repo_name: liuzhenhai/numba

reason: import numba,from numba


text:
from __future__ import annotations

import logging
import os
import shutil
from pathlib import Path

import numpy as np
import pandas as pd
from pandas import DataFrame
from plumbum import cli
from tqdm import tqdm

from behavior import BASE_TARGET_COLS, BENCHDB_TO_TABLES, PLAN_NODE_NAMES
from behavior.plans import (
    PlanDiffIncompleteSubinvocationException,
    PlanDiffInvalidDataException,
    PlanDiffUnsupportedParallelException,
)

from . import BLOCKED_OUS, DIFFERENCING_SCHEMA, STANDARDIZE_COLUMNS

logger = logging.getLogger(__name__)

def load_csv(ou_index, csv_file):
    """
    Read the CSV file for a given OU.

    Parameters
    ----------
    ou_index : int
        Index into PLAN_NODE_NAMES that describes the particular OU this CSV file
        corresponds to.

    csv_file: Path
        Path to the CSV file that should be read.

    Returns
    -------
    ou_index : int
        Index of the PLAN_NODE_NAMES that the dataframe corresponds to.
    df_targets : pd.DataFrame
        Dataframe constructed from the CSV file for differencing.
    df_features : pd.DataFrame
        Dataframe constructed from the CSV file of non-differencing columns.

    Notes
    -----
        - Both `df_targets` and `df_features` contain `ou_index` and `data_id`. These features are used for joining the two dataframes.
        - `df_targets` follows the DIFFERENCING_SCHEMA layout
    """

    # Read the CSV file and add `ou_index` and `data_id`.
    df = pd.read_csv(csv_file, index_col=None)
    df["ou_index"] = ou_index
    df["data_id"] = df.index

    # Determine which columns to remap. A non-target input column that is suffixed by a
    # column in STANDARDIZE_COLUMNS is remapped to produce a common schema.
    remapper: dict[str, str] = {}
    for init_col in df.columns:
        if init_col not in STANDARDIZE_COLUMNS and init_col not in BASE_TARGET_COLS:
            for common_col in STANDARDIZE_COLUMNS:
                if init_col.endswith(common_col):
                    remapper[init_col] = common_col
                    break
    df.rename(columns=remapper, inplace=True)

    # Here we produce two different dataframes from the original CSV file.
    #
    # df[targets] produces a dataframe containing all the common schema columns and the
    # target columns. Plan differencing is performed on this dataframe.
    #
    # df[features] contains all the other columns in the input data that were not used
    # for differencing along with `ou_index` and `data_id`. It is worth noting that
    # `ou_index` and `data_id` can be used to reconstruct a datapoint across the
    # two dataframes.
    targets = DIFFERENCING_SCHEMA
    features = ["ou_index", "data_id"] + list(set(df.columns) - set(targets))

    # pylint: disable=E1136
    return ou_index, df[targets], df[features]

def diff_query_invocation(subinvocation, diffed_matrices):
    """
    Diffs a given query invocation by calling a CPython function.

    Parameters
    ----------
    subinvocation : pd.DataFrame
        Dataframe describing a single invocation of an unique query instance.

    diffed_matrices : list[np.pdarray]
        Output list to store diffed numpy matrices.
    """
    # diff_c.so is compiled from behavior/plans/diff_c.pyx. If an error is thrown
    # saying that diff_c can't be found, then please ensure your PYTHONPATH is
    # setup correctly.
    #
    # pylint: disable=E0401,C0415
    from diff_c import diff_query_tree

    # The 2D underlying subinvocation array is cast to a float64[][] for efficient Cython
    # indexing into numpy ndarrays.
    matrix = subinvocation.to_numpy(dtype=np.float64, copy=False)

    try:
        diff_query_tree(matrix)
    except PlanDiffInvalidDataException as e:
        print("Invalid Data detected for subinvocation", subinvocation, matrix)
        raise e
    except (PlanDiffUnsupportedParallelException, PlanDiffIncompleteSubinvocationException):
        # These are not fatal errors. In these cases, we just return None to indicate
        # that there is no data that needs to be merged.
        return None

    # Append the diffed numpy array to diffed_matrices for us to post-process at once.
    diffed_matrices.append(matrix)
    return None

def separate_subinvocation(start_times, end_times, root_start_times, root_end_times, subinvocations):
    """
    For a given query session template, this function identifies the OUs associated with a
    given invocation of the query template.

    Parameters
    ----------
    start_times : np.array[int64]
        Array of start times for all OUs belonging to the same query template.

    end_times : np.array[int64]
        Array of end times for all OUs belonging to the same query template.
        start_times[i] and end_times[i] correspond to the same OU.

    root_start_times : np.array[int64]
        Array of start times of all root plan nodes (plan_node_id = 0).

    root_end_times : np.array[int64]
        Array of end times of all root plan nodes (plan_node_id = 0).

    subinvocations : np.array[int64]
        Output array to indicate for an OU [i] which [y] in root_start_times the OU belongs to.
        If output[i] = y, then root_start_times[y] <= start_times[i] && end_times[i] <= root_end_times[y].
    """

    # For each OU start time, find all root plan nodes [y] that start earlier.
    start_matches = [np.argwhere(root_start_times <= start) for start in start_times]

    # For each OU end time, find all root plan nodes [y] that end afterwards.
    end_matches = [np.argwhere(root_end_times >= end) for end in end_times]

    # For each OU data point, find the intersection between root plan nodes that start earlier and end afterwards.
    intersects = [
        np.intersect1d(start_match, end_match) for (start_match, end_match) in zip(start_matches, end_matches)
    ]

    # If there is an intersection, then the OU must belong to that "invocation".
    subinvocations[:] = [-1 if len(intersect) == 0 else intersect[0] for intersect in intersects]

def process_query_invocation(subframe, diffed_matrices):
    """
    Function used to difference all data associated with a given query session template.

    Parameters
    ----------
    subframe : pd.DataFrame
        Dataframe contains the data that we want to difference. The dataframe must be
        data that is associated with a given query session template.

        In other words, <query_id, statement_timestamp, pid> is the same for all rows.

    diffed_matrices : list[np.pdarray]
        Output list to store diffed numpy matrices.
    """
    root_plans_times = subframe[subframe["plan_node_id"] == 0]
    if root_plans_times.shape[0] > 1:
        # This is the case where there are multiple root plan node IDs detected directly.
        # In this case, we actually need to run separate_subinvocation to separate them.
        #
        # In the case where OUs can't be mapped to a particular invocation ID (because the
        # root plan itself might be lost), then the subinvocation_id will remain -1.
        # diff_query_tree() will then drop the data point.
        subinvocation_ids = subframe["subinvocation_id"].values
        separate_subinvocation(
            subframe["start_time"].values,
            subframe["end_time"].values,
            root_plans_times["start_time"].values,
            root_plans_times["end_time"].values,
            subinvocation_ids,
        )
        subframe["subinvocation_id"] = subinvocation_ids

        # Now group by subinvocation_id and apply diff_query_invocation.
        subframe.groupby(by=["subinvocation_id"]).apply(diff_query_invocation, diffed_matrices=diffed_matrices)
        return None

    diff_query_invocation(subframe, diffed_matrices=diffed_matrices)
    return None

def diff_queries(unified, diffed_matrices):
    """
    Diff all queries in the input data.

    Parameters
    ----------
    unified : pd.DataFrame
        Dataframe contains all the data that needs to be diferenced. The dataframe must folow
        the DIFFERENCING_SCHEMA layout.

    diffed_matrices : list[np.pdarray]
        Output list to store diffed numpy matrices.
    """
    # Here we assume that <query_id, statement_timestamp, pid> identifies a unique query session template.
    # Grouping by <query_id, statement_timestamp, pid> will produce subframes of OUs for a given query session template.
    invocation_groups = unified.groupby(by=["query_id", "statement_timestamp", "pid"], sort=False)

    # We use apply() because we want to process the entire subframe as a single unit. transform() will not work
    # for this because transform() may separate the columns.
    invocation_groups.progress_apply(process_query_invocation, diffed_matrices=diffed_matrices)

def load_tscout_data(tscout_data_dir):
    """
    Load TScout data into dataframes.

    Parameters
    ----------
    tscout_data_dir : Path
        Data directory containing all the TScout raw data.

    Returns
    --------
    unified : pd.DataFrame
         Dataframe containing datapoints from all OUs arranged by DIFFERENCING_SCHEMA.

    features : dict[int, pd.DataFrame]
         Dictionary mapping a ou_index (index in PLAN_NODE_NAMES) to OU specific features.
    """

    result_paths = {
        i: tscout_data_dir / f"Exec{node_name}.csv"
        for i, node_name in enumerate(PLAN_NODE_NAMES)
        if node_name not in BLOCKED_OUS
    }
    result_paths = {
        key: value for (key, value) in result_paths.items() if value.exists() and os.stat(value).st_size > 0
    }
    ou_indexes, commons, features = zip(*[load_csv(key, value) for (key, value) in result_paths.items()])
    features = dict(zip(ou_indexes, features))
    unified = pd.concat(commons, axis=0, copy=False)
    return unified, features

def save_results(diff_data_dir, ou_to_features, unified):
    """
    Save the new dataframes to disk.

    Parameters
    ----------
    diff_data_dir : Path
        Directory to save the differenced data.

    ou_to_features : dict[ou_index, DataFrame]
        Map from index indicating OU to OU specific features.

    unified : DataFrame
        DataFrame of all differenced records.
    """

    unified.set_index(["ou_index", "data_id"], drop=True, inplace=True)
    _ = [df.set_index(["ou_index", "data_id"], drop=True, inplace=True) for (_, df) in ou_to_features.items()]
    for ou_index, features in tqdm(ou_to_features.items()):
        # Perform an inner join with the index (on=None) of `ou_index` and `data_id`.
        # No suffixes are specified since features and unified should not share columns.
        result = features.join(unified, on=None, how="inner")
        if result.shape[0] > 0:
            # If we find that there are matching output rows, write them out.
            # Don't write out the index columns.
            result.to_csv(f"{diff_data_dir}/{PLAN_NODE_NAMES[ou_index]}.csv", index=False)

def main(data_dir, output_dir, experiment) -> None:
    logger.info("Differencing experiment: %s", experiment)

    for mode in ["train", "eval"]:
        experiment_root: Path = data_dir / experiment / mode
        # The data folders must be prefixed by the name of the benchmark,
        # and suffixed by the concatenated parameters.
        # For example, `tpcc_scalefactor_0.1_terminals_1_rate_10000_time_60`.
        bench_names: list[str] = [
            d.name
            for d in experiment_root.iterdir()
            if d.is_dir() and d.name.startswith(tuple(BENCHDB_TO_TABLES.keys()))
        ]

        for bench_name in bench_names:
            logger.info("Mode: %s | Benchmark: %s", mode, bench_name)
            bench_root = experiment_root / bench_name
            tscout_data_dir = bench_root / "tscout"
            diff_data_dir: Path = output_dir / experiment / mode / bench_name
            if diff_data_dir.exists():
                shutil.rmtree(diff_data_dir)
            diff_data_dir.mkdir(parents=True, exist_ok=True)

            unified, features = load_tscout_data(tscout_data_dir)

            # Reset the index on unified and default initialize the subinvocation_id field.
            unified.reset_index(drop=True, inplace=True)
            unified["subinvocation_id"] = -1
            diffed_matrices: list[np.ndarray] = []
            diff_queries(unified, diffed_matrices)

            # Concatenate diffed_matrices back into a single dataframe.
            unified_np = np.concatenate(diffed_matrices, axis=0)
            unified = DataFrame(unified_np, copy=False, columns=unified.columns)
            unified.drop("subinvocation_id", axis=1, inplace=True)
            save_results(diff_data_dir, features, unified)

class DataDiffCLI(cli.Application):
    dir_datagen_data = cli.SwitchAttr(
        "--dir-datagen-data",
        Path,
        mandatory=True,
        help="Directory containing DataGenerator output data.",
    )
    dir_output = cli.SwitchAttr(
        "--dir-output",
        Path,
        mandatory=True,
        help="Directory to output differenced CSV files to.",
    )
    glob_pattern = cli.SwitchAttr(
        "--glob-pattern", mandatory=False, help="Glob pattern to use for selecting valid experiments."
    )

    def main(self):
        tqdm.pandas()
        train_folder = self.dir_datagen_data

        pattern = "*" if self.glob_pattern is None else self.glob_pattern
        experiments = sorted(path.name for path in train_folder.glob(pattern))
        assert len(experiments) > 0, "No training data found?"

        if self.glob_pattern is None:
            experiments = [experiments[-1]]

        for experiment in experiments:
            main(self.dir_datagen_data, self.dir_output, experiment)

if __name__ == "__main__":
    DataDiffCLI.run()


text:
"""
test_common_converge.py

Testing common_converge.py
"""

import os

import numpy as np
import pytest

import seekr2.modules.common_base as base
import seekr2.modules.common_converge as common_converge
import seekr2.converge as converge
import seekr2.toy.smoluchowski as smol
import seekr2.tests.test_analyze as test_analyze

def make_model_completed(smoluchowski_model, steps):
    """
    Using the Smoluchowski engine, create a model that has all its
    statistics filled out.
    """
    smoluchowski_model.openmm_settings = base.Openmm_settings()
    smoluchowski_model.openmm_settings.langevin_integrator \
        = base.Langevin_integrator_settings()
    D = 1.0
    b_surface_dir_path = os.path.join(
        smoluchowski_model.anchor_rootdir,
        smoluchowski_model.k_on_info.b_surface_directory)
    smoluchowski_model.k_on_info.bd_milestones[0].directory \
        = "bd_milestone_{}".format(
            smoluchowski_model.k_on_info.bd_milestones[0].index)
    if not os.path.exists(b_surface_dir_path):
        os.mkdir(b_surface_dir_path)
    k_on_src, transition_probs = test_analyze.make_bd_statistics_flat(
        smoluchowski_model, D)
    potential_energy_function = smol.FlatPotentialEnergyFunction()
    if smoluchowski_model.get_type() == "mmvt":
        mmvt_time, k_on = test_analyze.make_mmvt_calculation_based_on_output_files(
            smoluchowski_model, potential_energy_function, k_on_src, transition_probs, steps=steps)
    elif smoluchowski_model.get_type() == "elber":
        mmvt_time, k_on = test_analyze.make_elber_calculation_based_on_output_files(
            smoluchowski_model, potential_energy_function, k_on_src, transition_probs, steps=steps)
    bound_state_radius = smoluchowski_model.anchors[0].milestones[0].variables["radius"]
    standard_k_on = 4.0 * np.pi * bound_state_radius * D
    standard_time = test_analyze.make_smoluchowski_standard_for_k_off(
        smoluchowski_model, potential_energy_function)
    return smoluchowski_model, standard_time, standard_k_on

def test_analyze_bd_only(smoluchowski_mmvt_model):
    steps = 10000
    cutoff = 0.1
    minimum_anchor_transitions = 100
    image_directory = None
    k_on_state = 0
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    data_sample_list = converge.converge(
        smoluchowski_mmvt_model, k_on_state, image_directory=image_directory,
        verbose=True)
    common_converge.analyze_bd_only(smoluchowski_mmvt_model,
                                    data_sample_list[-1])

    return

def test_mmvt_max_steps(smoluchowski_mmvt_model):
    steps = 10000
    cutoff = 0.1
    minimum_anchor_transitions = 100
    image_directory = None
    k_on_state = 0
    dt = 0.002
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    max_step_list = common_converge.get_mmvt_max_steps(
        smoluchowski_mmvt_model, dt)
    return

def test_elber_max_steps(smoluchowski_elber_model):
    steps = 10000
    cutoff = 0.1
    minimum_anchor_transitions = 100
    image_directory = None
    k_on_state = 0
    dt = 0.002
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_elber_model, steps)
    max_step_list = common_converge.get_mmvt_max_steps(
        smoluchowski_mmvt_model, dt)
    return

def test_check_milestone_convergence(smoluchowski_mmvt_model):
    steps = 10000
    k_on_state = 0
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    k_on_conv, k_off_conv, N_ij_conv, R_i_conv, max_step_list, \
        timestep_in_ns, data_sample_list \
        = common_converge.check_milestone_convergence(
            smoluchowski_mmvt_model, k_on_state=k_on_state,
            pre_equilibrium_approx=False, verbose=True)
    return

def test_check_milestone_convergence_pre_equilibrium_approx(
        smoluchowski_mmvt_model):
    steps = 10000
    k_on_state = 0
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    k_on_conv, k_off_conv, N_ij_conv, R_i_conv, max_step_list, \
        timestep_in_ns, data_sample_list \
        = common_converge.check_milestone_convergence(
            smoluchowski_mmvt_model, k_on_state=k_on_state,
            pre_equilibrium_approx=True, verbose=True)
    return

def test_plot_scalar_conv(smoluchowski_mmvt_model):
    steps = 10000
    k_on_state = 0
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    k_on_conv, k_off_conv, N_ij_conv, R_i_conv, max_step_list, \
        timestep_in_ns, data_sample_list \
        = common_converge.check_milestone_convergence(
            smoluchowski_mmvt_model, k_on_state=k_on_state,
            pre_equilibrium_approx=False, verbose=True)

    k_off_fig, ax = common_converge.plot_scalar_conv(
        k_off_conv, max_step_list[0,:], title="$k_{off}$ Convergence",
        label="k_{off} (s^{-1})", timestep_in_ns=timestep_in_ns)
    return

def test_plot_dict_conv(smoluchowski_mmvt_model):
    steps = 10000
    k_on_state = 0
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    k_on_conv, k_off_conv, N_ij_conv, R_i_conv, max_step_list, \
        timestep_in_ns, data_sample_list \
        = common_converge.check_milestone_convergence(
            smoluchowski_mmvt_model, k_on_state=k_on_state,
            pre_equilibrium_approx=False, verbose=True)

    N_ij_fig_list, ax, N_ij_title_list, N_ij_name_list \
        = common_converge.plot_dict_conv(
        N_ij_conv, max_step_list[0,:], label_base="N",
        timestep_in_ns=timestep_in_ns)

    R_i_fig_list, ax, R_i_title_list, R_i_name_list \
        = common_converge.plot_dict_conv(
        R_i_conv, max_step_list[0,:], label_base="R",
        timestep_in_ns=timestep_in_ns)
    return

def test_calc_window_rmsd():
    conv_values = [1.0, 2.0, 3.0]
    rmsd, average = common_converge.calc_window_rmsd(conv_values)
    expected_rmsd = np.sqrt(2.0/3.0)
    expected_average = 2.0
    assert np.isclose(rmsd, expected_rmsd)
    assert np.isclose(average, expected_average)

    conv_values = [5.0, 9.0]
    rmsd, average = common_converge.calc_window_rmsd(conv_values)
    expected_rmsd = 2.0
    expected_average = 7.0
    assert np.isclose(rmsd, expected_rmsd)
    assert np.isclose(average, expected_average)

def test_calc_transition_steps(smoluchowski_mmvt_model):
    steps = 10000
    cutoff = 0.1
    minimum_anchor_transitions = 100
    image_directory = None
    k_on_state = 0
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    data_sample_list = converge.converge(
        smoluchowski_mmvt_model, k_on_state, image_directory=image_directory,
        verbose=True)
    transition_minima, transition_prob_results, transition_time_results \
        = common_converge.calc_transition_steps(
        smoluchowski_mmvt_model, data_sample_list[-1])
    return

def test_calc_RMSD_conv_amount(smoluchowski_mmvt_model):
    steps = 10000
    cutoff = 0.1
    minimum_anchor_transitions = 100
    image_directory = None
    k_on_state = 0
    smoluchowski_mmvt_model, standard_time, standard_k_on\
        = make_model_completed(smoluchowski_mmvt_model, steps)
    data_sample_list = converge.converge(
        smoluchowski_mmvt_model, k_on_state, image_directory=image_directory,
        verbose=True)
    convergence_results = common_converge.calc_RMSD_conv_amount(
        smoluchowski_mmvt_model, data_sample_list, window_size=30,
        number_of_convergence_windows=20)
    return


text:
import cupy

# TODO(okuta): Implement delete

# TODO(okuta): Implement insert

# TODO(okuta): Implement append

# TODO(okuta): Implement resize

# TODO(okuta): Implement trim_zeros

def unique(ar, return_index=False, return_inverse=False,
           return_counts=False, axis=None):
    """Find the unique elements of an array.

    Returns the sorted unique elements of an array. There are three optional
    outputs in addition to the unique elements:

    * the indices of the input array that give the unique values
    * the indices of the unique array that reconstruct the input array
    * the number of times each unique value comes up in the input array

    Args:
        ar(array_like): Input array. This will be flattened if it is not
            already 1-D.
        return_index(bool, optional): If True, also return the indices of `ar`
            (along the specified axis, if provided, or in the flattened array)
            that result in the unique array.
        return_inverse(bool, optional): If True, also return the indices of the
            unique array (for the specified axis, if provided) that can be used
            to reconstruct `ar`.
        return_counts(bool, optional): If True, also return the number of times
            each unique item appears in `ar`.
        axis(int or None, optional): Not supported yet.

    Returns:
        cupy.ndarray or tuple:
            If there are no optional outputs, it returns the
            :class:`cupy.ndarray` of the sorted unique values. Otherwise, it
            returns the tuple which contains the sorted unique values and
            followings.

            * The indices of the first occurrences of the unique values in the
              original array. Only provided if `return_index` is True.
            * The indices to reconstruct the original array from the
              unique array. Only provided if `return_inverse` is True.
            * The number of times each of the unique values comes up in the
              original array. Only provided if `return_counts` is True.

    .. seealso:: :func:`numpy.unique`
    """
    if axis is not None:
        raise NotImplementedError('axis option is not supported yet.')

    ar = cupy.asarray(ar).flatten()

    if return_index or return_inverse:
        perm = ar.argsort()
        aux = ar[perm]
    else:
        ar.sort()
        aux = ar
    mask = cupy.empty(aux.shape, dtype=cupy.bool_)
    mask[0] = True
    mask[1:] = aux[1:] != aux[:-1]

    ret = aux[mask]
    if not return_index and not return_inverse and not return_counts:
        return ret

    ret = ret,
    if return_index:
        ret += perm[mask],
    if return_inverse:
        imask = cupy.cumsum(mask) - 1
        inv_idx = cupy.empty(mask.shape, dtype=cupy.intp)
        inv_idx[perm] = imask
        ret += inv_idx,
    if return_counts:
        nonzero = cupy.nonzero(mask)[0]
        idx = cupy.empty((nonzero.size + 1,), nonzero.dtype)
        idx[:-1] = nonzero
        idx[-1] = mask.size
        ret += idx[1:] - idx[:-1],
    return ret

max_issues_repo_name: hephaex/cupy

max_forks_repo_name: hephaex/cupy

reason: import cupy


text:
from scipy import shape, dstack, sqrt, floor, array, mean, ones, vstack, hstack, ndarray
from vlfeat import vl_rgb2gray, vl_imsmooth, vl_dsift
from sys import maxint

"""
Python rewrite of https://github.com/vlfeat/vlfeat/blob/master/toolbox/sift/vl_phow.m
### notice no hsv support atm
### comments are largely copied from the code

"""

def vl_phow(im,
            verbose=True,
            fast=True,
            sizes=[4, 6, 8, 10],
            step=2,
            color='rgb',
            floatdescriptors=False,
            magnif=6,
            windowsize=1.5,
            contrastthreshold=0.005):

    opts = Options(verbose, fast, sizes, step, color, floatdescriptors,
                   magnif, windowsize, contrastthreshold)
    dsiftOpts = DSiftOptions(opts)

    # make sure image is float, otherwise segfault
    im = array(im, 'float32')

    # Extract the features
    imageSize = shape(im)
    if im.ndim == 3:
        if imageSize[2] != 3:
            # "IndexError: tuple index out of range" if both if's are checked at the same time
            raise ValueError("Image data in unknown format/shape")
    if opts.color == 'gray':
        numChannels = 1
        if (im.ndim == 2):
            im = vl_rgb2gray(im)
    else:
        numChannels = 3
        if (im.ndim == 2):
            im = dstack([im, im, im])
        if opts.color == 'rgb':
            pass
        elif opts.color == 'opponent':
             # from https://github.com/vlfeat/vlfeat/blob/master/toolbox/sift/vl_phow.m
             # Note that the mean differs from the standard definition of opponent
             # space and is the regular intesity (for compatibility with
             # the contrast thresholding).
             # Note also that the mean is added pack to the other two
             # components with a small multipliers for monochromatic
             # regions.

            mu = 0.3 * im[:, :, 0] + 0.59 * im[:, :, 1] + 0.11 * im[:, :, 2]
            alpha = 0.01
            im = dstack([mu,
                         (im[:, :, 0] - im[:, :, 1]) / sqrt(2) + alpha * mu,
                         (im[:, :, 0] + im[:, :, 1] - 2 * im[:, :, 2]) / sqrt(6) + alpha * mu])
        else:
            raise ValueError('Color option ' + str(opts.color) + ' not recognized')
    if opts.verbose:
        print('{0}: color space: {1}'.format('vl_phow', opts.color))
        print('{0}: image size: {1} x {2}'.format('vl_phow', imageSize[0], imageSize[1]))
        print('{0}: sizes: [{1}]'.format('vl_phow', opts.sizes))

    frames_all = []
    descrs_all = []
    for size_of_spatial_bins in opts.sizes:
        # from https://github.com/vlfeat/vlfeat/blob/master/toolbox/sift/vl_phow.m
        # Recall from VL_DSIFT() that the first descriptor for scale SIZE has
        # center located at XC = XMIN + 3/2 SIZE (the Y coordinate is
        # similar). It is convenient to align the descriptors at different
        # scales so that they have the same geometric centers. For the
        # maximum size we pick XMIN = 1 and we get centers starting from
        # XC = 1 + 3/2 MAX(OPTS.SIZES). For any other scale we pick XMIN so
        # that XMIN + 3/2 SIZE = 1 + 3/2 MAX(OPTS.SIZES).
        # In pracrice, the offset must be integer ('bounds'), so the
        # alignment works properly only if all OPTS.SZES are even or odd.

        off = floor(3.0 / 2 * (max(opts.sizes) - size_of_spatial_bins)) + 1

        # smooth the image to the appropriate scale based on the size
        # of the SIFT bins
        sigma = size_of_spatial_bins / float(opts.magnif)
        ims = vl_imsmooth(im, sigma)

        # extract dense SIFT features from all channels
        frames = []
        descrs = []
        for k in range(numChannels):
            size_of_spatial_bins = int(size_of_spatial_bins)
            # vl_dsift does not accept numpy.int64 or similar
            f_temp, d_temp = vl_dsift(data=ims[:, :, k],
                                      step=dsiftOpts.step,
                                      size=size_of_spatial_bins,
                                      fast=dsiftOpts.fast,
                                      verbose=dsiftOpts.verbose,
                                      norm=dsiftOpts.norm,
                                      bounds=[off, off, maxint, maxint])
            frames.append(f_temp)
            descrs.append(d_temp)
        frames = array(frames)
        descrs = array(descrs)
        d_new_shape = [descrs.shape[0] * descrs.shape[1], descrs.shape[2]]
        descrs = descrs.reshape(d_new_shape)
        # remove low contrast descriptors
        # note that for color descriptors the V component is
        # thresholded
        if (opts.color == 'gray') | (opts.color == 'opponent'):
            contrast = frames[0][2, :]
        elif opts.color == 'rgb':
            contrast = mean([frames[0][2, :], frames[1][2, :], frames[2][2, :]], 0)
        else:
            raise ValueError('Color option ' + str(opts.color) + ' not recognized')
        descrs[:, contrast < opts.contrastthreshold] = 0

        # save only x,y, and the scale
        frames_temp = array(frames[0][0:3, :])
        padding = array(size_of_spatial_bins * ones(frames[0][0].shape))
        frames_all.append(vstack([frames_temp, padding]))
        descrs_all.append(array(descrs))

    frames_all = hstack(frames_all)
    descrs_all = hstack(descrs_all)
    return frames_all, descrs_all

class Options(object):
    def __init__(self, verbose, fast, sizes, step, color,
                 floatdescriptors, magnif, windowsize,
                 contrastthreshold):
        self.verbose = verbose
        self.fast = fast
        if (type(sizes) is not ndarray) & (type(sizes) is not list):
            sizes = array([sizes])
        self.sizes = sizes
        self.step = step
        self.color = color
        self.floatdescriptors = floatdescriptors
        self.magnif = magnif
        self.windowsize = windowsize
        self.contrastthreshold = contrastthreshold

class DSiftOptions(object):
    def __init__(self, opts):
        self.norm = True
        self.windowsize = opts.windowsize
        self.verbose = opts.verbose
        self.fast = opts.fast
        self.floatdescriptors = opts.floatdescriptors
        self.step = opts.step

if __name__ == "__main__":
    from scipy.misc import imread
    im = imread('image_0001.jpg')
    frames, descrs = vl_phow(array(im, 'float32') / 255.0)

max_stars_repo_name: thomasklau/pixy

max_issues_repo_name: thomasklau/pixy

max_forks_repo_name: thomasklau/pixy

reason: from scipy


text:
#!/usr/bin/env python3
import argparse
import numpy as np

from lib import environ, data, models

import torch

import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt

EPSILON = 0.02

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--data", required=True, help="CSV file with quotes to run the model")
    parser.add_argument("-m", "--model", required=True, help="Model file to load")
    parser.add_argument("-b", "--bars", type=int, default=50, help="Count of bars to feed into the model")
    parser.add_argument("-n", "--name", required=True, help="Name to use in output images")
    parser.add_argument("--commission", type=float, default=0.1, help="Commission size in percent, default=0.1")
    parser.add_argument("--conv", default=False, action="store_true", help="Use convolution model instead of FF")
    args = parser.parse_args()

    # 数据及环境参数初始化
    prices = data.load_relative(args.data)
    env = environ.StocksEnv({"TEST": prices}, bars_count=args.bars, reset_on_close=False, commission=args.commission,
                            state_1d=args.conv, random_ofs_on_reset=False, reward_on_close=False, volumes=False)

    # 判断是否使用卷积变体
    if args.conv:
        net = models.DQNConv1D(env.observation_space.shape, env.action_space.n)
    else:
        net = models.SimpleFFDQN(env.observation_space.shape[0], env.action_space.n)

    #
    net.load_state_dict(torch.load(args.model, map_location=lambda storage, loc: storage))

    obs = env.reset()
    start_price = env._state._cur_close()

    total_reward = 0.0
    step_idx = 0
    rewards = []

    while True:
        step_idx += 1
        obs_v = torch.tensor([obs])
        out_v = net(obs_v)
        action_idx = out_v.max(dim=1)[1].item()
        if np.random.random() < EPSILON:
            action_idx = env.action_space.sample()
        action = environ.Actions(action_idx)

        obs, reward, done, _ = env.step(action_idx)
        total_reward += reward
        rewards.append(total_reward)
        if step_idx % 100 == 0:
            print("%d: reward=%.3f" % (step_idx, total_reward))
        if done:
            break

    plt.clf()
    plt.plot(rewards)
    plt.title("Total reward, data=%s" % args.name)
    plt.ylabel("Reward, %")
    plt.savefig("rewards-%s.png" % args.name)


text:
# coding=utf-8
import codecs
import subprocess
from collections import namedtuple

import numpy as np
import pandas as pd
from gensim.models import Doc2Vec
from sklearn.cross_validation import cross_val_score
from sklearn.linear_model import LogisticRegression

import param
import util

############################ 加载数据 ############################
df_all = pd.read_csv(param.data_path + '/output/corpus/all_data.csv', encoding='utf8', nrows=param.train_num).reset_index()
df_all['penalty'] = df_all['penalty'] - 1

############################ 定义函数、类及变量 ############################
def run_cmd(cmd):
    print(cmd)
    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    for t, line in enumerate(iter(process.stdout.readline, b'')):
        line = line.decode('utf8').rstrip()
        print(line)
    process.communicate()
    return process.returncode

SentimentDocument = namedtuple('SentimentDocument', 'words tags')

class Doc_list(object):
    def __init__(self, f):
        self.f = f
    def __iter__(self):
        for i,line in enumerate(codecs.open(self.f,encoding='utf8')):
            words = line.strip().split(' ')
            tags = [int(words[0][2:])]
            words = words[1:]
            yield SentimentDocument(words,tags)

############################ 准备数据 ############################
doc_f = codecs.open(param.data_path + '/output/corpus/doc_for_d2v_12w.txt', 'w', encoding='utf8')
for i, contents in enumerate(df_all.iloc[:param.train_num]['content']):
    words = []
    for word in contents.split(' '):
        words.append(word)
    tags = [i]
    if i % 10000 == 0:
        util.log('iter = %d' % i)
    doc_f.write(u'_*{} {}\n'.format(i, ' '.join(words)))
doc_f.close()

############################ dbow d2v ############################
d2v = Doc2Vec(dm=0, size=300, negative=5, hs=0, min_count=3, window=30, sample=1e-5, workers=8, alpha=0.025, min_alpha=0.025)
doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')
d2v.build_vocab(doc_list)

df_lb = df_all['penalty']

for i in range(5):
    util.log('pass: ' + str(i))
    #     run_cmd('shuf alldata-id.txt > alldata-id-shuf.txt')
    doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')
    d2v.train(doc_list, total_examples=d2v.corpus_count, epochs=d2v.iter)
    X_d2v = np.array([d2v.docvecs[i] for i in range(param.train_num)])
    scores = cross_val_score(LogisticRegression(C=3), X_d2v, df_lb, cv=5)
    util.log('dbow: ' + str(scores) + ' ' + str(np.mean(scores)))
d2v.save(param.data_path + '/output/model/dbow_d2v_12w.model')
util.log('Save done!')

############################ dm d2v ############################
d2v = Doc2Vec(dm=1, size=300, negative=5, hs=0, min_count=3, window=30, sample=1e-5, workers=8, alpha=0.025, min_alpha=0.025)
doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')
d2v.build_vocab(doc_list)

df_lb = df_all['penalty']

for i in range(10):
    util.log('pass: ' + str(i))
    #     run_cmd('shuf alldata-id.txt > alldata-id-shuf.txt')
    doc_list = Doc_list(param.data_path + '/output/corpus/doc_for_d2v_12w.txt')
    d2v.train(doc_list, total_examples=d2v.corpus_count, epochs=d2v.iter)
    X_d2v = np.array([d2v.docvecs[i] for i in range(param.train_num)])
    scores = cross_val_score(LogisticRegression(C=3), X_d2v, df_lb, cv=5)
    util.log('dm: ' + str(scores) + ' ' + str(np.mean(scores)))
d2v.save(param.data_path + '/output/model/dm_d2v_12w.model')
util.log('Save done!')


text:
from abc import ABC, abstractmethod

import numpy as np

class BaseModel(ABC):
    def __init__(self,
                 internal_states_labels: list,
                 internal_params_labels: list,
                 range_delay=None,
                 stochastic=False):
        """This is an abstract class for epidemiological models.

        Parameters
        ----------
        internal_states_labels: list of str
                                Labels for each internal state of the model.
        internal_params_labels: list of str
                                Labels for each parameter of the model.
        range_delay: tuple
            Min and max delay between epidemic start and episode start.
        stochastic: bool
                    Whether to use stochastic models or not.

        Attributes
        ----------
        internal_states_labels
        internal_params_labels
        initial_state: 1D nd.array
                       Initial internal state of the model.
        initial_internal_params: 1D nd.array
                                 Initial internal params of the model.
        current_state: 1D nd.array
                       Current state of the model.

        """

        self.internal_states_labels = internal_states_labels
        self.internal_params_labels = sorted(internal_params_labels)
        self.stochastic = stochastic

        self.initial_state = None
        self.initial_internal_params = None
        self.current_state = None
        self.range_delay = range_delay
        self.reset()  # reset model to initial states and parameters

    def reset(self, delay=None) -> None:
        """Resets the model parameters, and state, add delay.

        Parameters
        ----------
        delay: int, optional
               Number of days the model should be run for before the start of the episode.
               Default is 0.

        """
        self._sample_model_params()
        self._sample_initial_state()
        self._reset_state()
        if self.stochastic:
            if delay is not None:
                self.delay(random=False, delay=delay)
            else:
                self.delay()

    def reset_same_model(self) -> None:
        """
        Resets the model to its former initial conditions with its current parameters.

        """
        self._reset_model_params()
        self._reset_state()

    def delay(self, random=True, delay=None):
        """
        Run the model for 'delay' steps before the beginning of the episode.

        Parameters
        ----------
        delay: int
               Delay before the episode starts in days.

        Returns
        -------
        1D nd.array
            Latest model state
        """
        if not random:
            assert delay is not None
        else:
            if self.range_delay is not None:
                delay = np.random.randint(*self.range_delay)
            else:
                delay = 0
        if delay > 0:
            return self.run_n_steps(n=delay)[-1]

    @abstractmethod
    def _sample_initial_state(self):
        """
        Samples the initial state of the model (abstract)

        """
        pass

    @abstractmethod
    def _sample_model_params(self):
        """
        Samples parameters of the model (abstract)

        """
        pass

    @abstractmethod
    def run_n_steps(self,
                    current_state=None,
                    n=1,
                    t=None):
        """
        Runs the model for n steps.

        Parameters
        ----------
        current_state: 1D nd.array
                       Current state of the model.
        n: int
           Number of steps the model should be run for.

        Returns
        -------
        new_state: 2D nd.array
                   Returns the list of n states from the initial step to the latest step.

        """
        pass

    def _reset_model_params(self) -> None:
        """
        Resets model parameters to their initial values.

        """
        self.current_internal_params = self.initial_internal_params.copy()

    def _get_model_params(self) -> tuple:
        """
        Get current parameters of the model

        Returns
        -------
        tuple
            tuple of the model parameters in the order of the list of labels
        """
        return tuple([self.current_internal_params[k] for k in self.internal_params_labels])

    def _reset_state(self):
        """
        Resets model state to initial state.
        """
        self.current_state = dict(zip(self.internal_states_labels, np.array([self.initial_state['{}0'.format(s)] for s in self.internal_states_labels])))

    def _get_current_state(self):
        """
        Get current state in the order of state labels.

        """
        return np.array([self.current_state['{}'.format(s)] for s in self.internal_states_labels])

    def _set_current_state(self, current_state):
        """
        Set current state to given values.

        Parameters
        ----------
        current_state: 1D nd.array
                       State the current state should be set to.

        """
        self.current_state = dict(zip(self.internal_states_labels, current_state))

    def _convert_to_labelled_states(self, states):
        """
        Converts the state into a dict where keys are state labels
        Parameters
        ----------
        states: 1D nd.array
                Model state.

        Returns
        -------
        dict
            Dict where keys are state labels and values are states.

        """
        return  zip(self.internal_states_labels, states.transpose())

max_stars_repo_name: abdelneuhaus/EpidemiOptim

max_issues_repo_name: abdelneuhaus/EpidemiOptim

max_forks_repo_name: abdelneuhaus/EpidemiOptim


text:
import torch.utils.data as dataloader
from data.datagenerator import CTData
import torch.optim as optim
from miscellaneous.utils import mask_cross_entropy
from imgaug import augmenters as iaa
from config import Config
from models.segmentor_v1 import HybridUNet_single_out_consistency
import os
from miscellaneous.metrics import dice
import numpy as np
import torch
import torch.nn as nn
from tensorboardX import SummaryWriter
import pandas as pd
from skimage.transform import resize
import time

# on line evaluate the result
def online_eval(model, dataloader):
    with torch.no_grad():
        dice_new_list = []
        data_dict_list = []
        last_pre = np.zeros((2,2))
        last_tar = np.zeros((2, 2))
        shape_list = []
        Loss_epoch_val = 0
        for data_val in dataloader:
            images_val, targets_val, subject, slice = data_val
            model.eval()
            images_val = images_val.to(device)
            targets_val = targets_val.to(device)
            outputs_val = model(images_val, config.lamda_sem)
            outputs_val_out = outputs_val.final_out

            # calculate validation loss
            Loss_backbone = mask_cross_entropy(targets_val[:, 1, ...], outputs_val.final_out, alpha=config.alpha)
            Loss_SEM = mask_cross_entropy(targets_val[:, 1, ...], outputs_val.out1_sem_2, alpha=config.alpha) + \
                       mask_cross_entropy(targets_val[:, 1, ...], outputs_val.out1_sem_0, alpha=config.alpha)
            Loss_seg = Loss_backbone + Loss_SEM
            Loss_epoch_val += Loss_seg

            # slice-wise prediction
            outputs_val_1 = outputs_val_out[:,0:2, ...]
            _, predicted_1 = torch.max(outputs_val_1.data, 1)
            predicted_val_1 = predicted_1.data.cpu().numpy()

            subject_val = subject.data.cpu().numpy()
            slice_val = slice.data.cpu().numpy()
            slice_val_1 = slice_val[0][1]

            targets_val = targets_val.data.cpu().numpy()
            targets_val_1 = targets_val[:,1, ...]
            shape_list.append(predicted_val_1.shape)

            '''
                    append to a dict list,
                    "subject": ID of the Patient.
                    "slice": Slice index in the volume of a Patient.
                    "Pre": Prediction of a slice.
                    "target": Label of a slice.
            '''
            data_dict_list.append({"subject": subject_val[0], "slice": slice_val_1, "pre": np.squeeze(predicted_val_1,axis=0),
                                   "target": np.squeeze(targets_val_1, axis=0)})

        # Turn slice-wise predictions to subject-wise predictions
        pd_data = pd.DataFrame(data_dict_list)
        for name, volume_data in pd_data.groupby("subject"):
            pre = volume_data["pre"]
            tar = volume_data["target"]
            pre_array = pre.values
            target_array = tar.values
            pre_temp = np.zeros((len(pre_array), pre_array[0].shape[0], pre_array[0].shape[1]), dtype="int16")
            target_temp = np.zeros((len(pre_array), target_array[0].shape[0], target_array[0].shape[1]), dtype="int16")
            for i in range(len(pre_array)):
                pre_temp[i, :, :] = pre_array[i]

            for i in range(len(target_array)):
                target_temp[i, :, :] = target_array[i]

            dsc_list1 = []

            # calculate dice coefficient
            for i in range(0, config.num_classes):
                dsc_i = dice(pre_temp, target_temp, i)
                dsc_list1.append(dsc_i)
            dice_new_list.append(dsc_list1)

        dice_array = np.array(dice_new_list)
        dice_mean = np.mean(dice_array, axis=0)

    return dice_mean, Loss_epoch_val

os.environ["CUDA_VISIBLE_DEVICES"] = '3'  # '1'
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

if __name__ == '__main__':
    # Select the Nvidia card
    start_time = time.time()
    torch.set_num_threads(2)

    #  ********************training and On-line Validation***************************
    config = Config()
    config.display()

    train_path = os.path.join(config.data_path, "data_train")
    val_path = os.path.join(config.data_path, "data_val")

    if os.path.exists("hybrid_consistency.txt"):
        os.remove("hybrid_consistency.txt")

    with open("hybrid_consistency.txt", "a") as mylog:

        config.write_config(mylog)

        # define a SummaryWriter for show loss curve in the Tensorboard
        best_dice_train = 0
        best_dice_val = 0
        with SummaryWriter("./result") as writer:
            # define the main model
            model_S = HybridUNet_single_out_consistency(backbone_channel=1, backbone_class=2,
                                 drop_rate=config.drop_rate, SEM_channels=3, SEM_class=2).to(device)
            # loss function
            weight_tensor = torch.from_numpy(np.array(config.alpha)).float()
            criterion_S = nn.CrossEntropyLoss(weight_tensor).cuda()

            # whether to pretrain from the backbone network and SEM
            if config.pre_trained:
                print("start pre train from backbone and SEM")
                checkpoint_bb = torch.load(config.pre_trained_model)
                model_S.backbone.load_state_dict(checkpoint_bb)
                checkpoint_sem = torch.load(config.pre_trained_SEM_model)
                model_S.SEM.load_state_dict(checkpoint_sem)
                # # Test freezing parameters of SEM
                # for p in model_S.SEM.parameters():
                #     p.requires_grad = False
                # for name, value in model_S.SEM.named_parameters():
                #     print('name: {},\t grad: {}'.format(name, value.requires_grad))
                # # Test freezing parameters of backbone
                # for p in model_S.backbone.parameters():
                #     p.requires_grad = False
                # for name, value in model_S.backbone.named_parameters():
                #     print('name: {},\t grad: {}'.format(name, value.requires_grad))
                print("Pre train succeed SEM and backbone!")
            # whether load old models
            if config.load_old_model:
                print("start load checkpoint: {}".format(config.note))
                checkpoint = torch.load(config.checkpoint)
                model_S.load_state_dict(checkpoint)
                print("load checkpoint succeed!")

            # set optimizer
            optimizer_S = optim.Adam(model_S.parameters(), lr=config.lr_S, weight_decay=6e-4, betas=(0.97, 0.999))
            # Test freezing parameters of SEM
            # filtered_para = filter(lambda p: p.requires_grad, model_S.parameters())
            # optimizer_S = optim.Adam(filtered_para, lr=config.lr_S,
            #                          weight_decay=6e-4, betas=(0.97, 0.999))
            scheduler_S = optim.lr_scheduler.StepLR(optimizer_S, step_size=config.step_size_S, gamma=0.1)

            # add data augmentation
            augmentation = iaa.Sequential([
                iaa.Fliplr(0.5),
                iaa.Flipud(0.5),
                iaa.OneOf([iaa.Affine(rotate=30),
                           iaa.Affine(rotate=60),
                           iaa.Affine(rotate=330)]),
                iaa.GaussianBlur(sigma=(0.0, 3.0))
            ])
            # data augmentation
            if config.augment:
                ct_data_train = CTData(train_path, augmentation=augmentation)
            else:
                ct_data_train = CTData(train_path, augmentation=None)

            trainloader = dataloader.DataLoader(ct_data_train, batch_size=config.batch_train, shuffle=True)

            ct_data_val = CTData(val_path, augmentation = None)

            valloader = dataloader.DataLoader(ct_data_val, batch_size=1, shuffle=False)
            valloader_train = dataloader.DataLoader(ct_data_train, batch_size=1, shuffle=False)

            test = 0
            print('Rate     | epoch  | Loss seg|DSC_train(bg)| DSC_train(fg)| DSC_val(bg)| DSC_val(fg) ')
            mylog.close()

            for epoch in range (config.num_epoch):
                start_time = time.time()
                loss_epoch = 0
                # loss_epoch_val = 0
                mylog  = open("hybrid_consistencys.txt", "a")
                scheduler_S.step(epoch)
                # zero the parameter gradients
                model_S.train()
                for i, data in enumerate(trainloader):
                    images, targets, _, _ = data
                    # Set mode cuda if it is enable, otherwise mode CPU
                    images = images.to(device)
                    targets = targets.to(device)
                    # mask = mask.to(device)
                    optimizer_S.zero_grad()
                    outputs = model_S(images, config.lamda_sem)

                    loss_backbone = mask_cross_entropy(targets[:,1,...], outputs.final_out, alpha=config.alpha)
                    loss_SEM = mask_cross_entropy(targets[:,1,...], outputs.out1_sem_2, alpha=config.alpha)+\
                               mask_cross_entropy(targets[:,1,...], outputs.out1_sem_0, alpha=config.alpha)
                    loss_seg = loss_backbone+loss_SEM

                    loss_epoch += loss_seg
                    loss_seg.backward()
                    optimizer_S.step()
                    test = i

                end_time = time.time()
                elapsed = end_time-start_time

                # -----------------------Validation------------------------------------
                dice_val, loss_epoch_val = online_eval(model_S, valloader)
                dice_train, _ = online_eval(model_S, valloader_train)
                #-------------------Debug-------------------------
                for param_group in optimizer_S.param_groups:
                    print('%0.6f | %6d | %0.5f | %0.5f| %0.5f| %0.5f| %0.5f ' % (
                            param_group['lr'], epoch,
                            loss_epoch.data.cpu().numpy(),
                            dice_train[0], dice_train[1],
                            dice_val[0], dice_val[1]))

                mylog.write("epoch_{:30} train_loss:  {:3f}  val_loss:  {:3f} elapse:  {:3f} train_dice: {:3f}  {:3f}  val_dice: {:3f}  "
                            "{:3f} \n".format(str(epoch), loss_epoch, loss_epoch_val, elapsed, dice_train[0], dice_train[1],
                                                                                dice_val[0], dice_val[1]))
                mylog.close()
                # save tensorboard files
                writer.add_scalar("train_loss", loss_epoch, epoch)
                writer.add_scalar("val_loss", loss_epoch_val, epoch)

                # save best checkpoint
                if dice_val[1]>best_dice_val:
                    torch.save(model_S.state_dict(), "./checkpoints/best_checkpoint.pth")
                    best_dice_val = dice_val[1]
                if dice_train[1]>best_dice_train:
                    torch.save(model_S.state_dict(), "./checkpoints/best_checkpoint_train.pth")
                    best_dice_train = dice_train[1]

            # export the loss for external use
            writer.export_scalars_to_json("./result/all_scalar.json")
            writer.close()

max_stars_repo_name: JohnleeHIT/SEX-Net

max_issues_repo_name: JohnleeHIT/SEX-Net

max_forks_repo_name: JohnleeHIT/SEX-Net


text:
import warnings

import numpy as np

from ._common import num_nodes_per_cell
from ._exceptions import ReadError
from ._mesh import CellBlock

# https://vtk.org/doc/nightly/html/vtkCellType_8h_source.html
vtk_to_meshio_type = {
    0: "empty",
    1: "vertex",
    # 2: 'poly_vertex',
    3: "line",
    # 4: 'poly_line',
    5: "triangle",
    # 6: 'triangle_strip',
    7: "polygon",
    8: "pixel",
    9: "quad",
    10: "tetra",
    # 11: 'voxel',
    12: "hexahedron",
    13: "wedge",
    14: "pyramid",
    15: "penta_prism",
    16: "hexa_prism",
    21: "line3",
    22: "triangle6",
    23: "quad8",
    24: "tetra10",
    25: "hexahedron20",
    26: "wedge15",
    27: "pyramid13",
    28: "quad9",
    29: "hexahedron27",
    30: "quad6",
    31: "wedge12",
    32: "wedge18",
    33: "hexahedron24",
    34: "triangle7",
    35: "line4",
    42: "polyhedron",
    #
    # 60: VTK_HIGHER_ORDER_EDGE,
    # 61: VTK_HIGHER_ORDER_TRIANGLE,
    # 62: VTK_HIGHER_ORDER_QUAD,
    # 63: VTK_HIGHER_ORDER_POLYGON,
    # 64: VTK_HIGHER_ORDER_TETRAHEDRON,
    # 65: VTK_HIGHER_ORDER_WEDGE,
    # 66: VTK_HIGHER_ORDER_PYRAMID,
    # 67: VTK_HIGHER_ORDER_HEXAHEDRON,
    # Arbitrary order Lagrange elements
    68: "VTK_LAGRANGE_CURVE",
    69: "VTK_LAGRANGE_TRIANGLE",
    70: "VTK_LAGRANGE_QUADRILATERAL",
    71: "VTK_LAGRANGE_TETRAHEDRON",
    72: "VTK_LAGRANGE_HEXAHEDRON",
    73: "VTK_LAGRANGE_WEDGE",
    74: "VTK_LAGRANGE_PYRAMID",
    # Arbitrary order Bezier elements
    75: "VTK_BEZIER_CURVE",
    76: "VTK_BEZIER_TRIANGLE",
    77: "VTK_BEZIER_QUADRILATERAL",
    78: "VTK_BEZIER_TETRAHEDRON",
    79: "VTK_BEZIER_HEXAHEDRON",
    80: "VTK_BEZIER_WEDGE",
    81: "VTK_BEZIER_PYRAMID",
}
meshio_to_vtk_type = {v: k for k, v in vtk_to_meshio_type.items()}

def vtk_to_meshio_order(vtk_type, dtype=int):
    # meshio uses the same node ordering as VTK for most cell types. However, for the
    # linear wedge, the ordering of the gmsh Prism [1] is adopted since this is found in
    # most codes (Abaqus, Ansys, Nastran,...). In the vtkWedge [2], the normal of the
    # (0,1,2) triangle points outwards, while in gmsh this normal points inwards.
    # [1] http://gmsh.info/doc/texinfo/gmsh.html#Node-ordering
    # [2] https://vtk.org/doc/nightly/html/classvtkWedge.html
    if vtk_type == 13:
        return np.array([0, 2, 1, 3, 5, 4], dtype=dtype)
    return None

def meshio_to_vtk_order(meshio_type, dtype=int):
    if meshio_type == "wedge":
        return np.array([0, 2, 1, 3, 5, 4], dtype=dtype)
    return None

def vtk_cells_from_data(connectivity, offsets, types, cell_data_raw):
    # Translate it into the cells array.
    # `connectivity` is a one-dimensional vector with
    # (p00, p01, ... ,p0k, p10, p11, ..., p1k, ...
    # `offsets` is a pointer array that points to the first position of p0, p1, etc.
    if len(offsets) != len(types):
        raise ReadError(f"len(offsets) != len(types) ({len(offsets)} != {len(types)})")

    # identify cell blocks
    breaks = np.where(types[:-1] != types[1:])[0] + 1
    # all cells with indices between start[k] and end[k] have the same type
    start_end = list(
        zip(
            np.concatenate([[0], breaks]),
            np.concatenate([breaks, [len(types)]]),
        )
    )

    cells = []
    cell_data = {}

    for start, end in start_end:
        try:
            meshio_type = vtk_to_meshio_type[types[start]]
        except KeyError:
            warnings.warn(
                f"File contains cells that meshio cannot handle (type {types[start]})."
            )
            continue

        # cells with varying number of points
        special_cells = [
            "polygon",
            "VTK_LAGRANGE_CURVE",
            "VTK_LAGRANGE_TRIANGLE",
            "VTK_LAGRANGE_QUADRILATERAL",
            "VTK_LAGRANGE_TETRAHEDRON",
            "VTK_LAGRANGE_HEXAHEDRON",
            "VTK_LAGRANGE_WEDGE",
            "VTK_LAGRANGE_PYRAMID",
        ]
        if meshio_type in special_cells:
            # Polygons have unknown and varying number of nodes per cell.

            # Index where the previous block of cells stopped. Needed to know the number
            # of nodes for the first cell in the block.
            first_node = 0 if start == 0 else offsets[start - 1]

            # Start off the cell-node relation for each cell in this block
            start_cn = np.hstack((first_node, offsets[start:end]))
            # Find the size of each cell except the last
            sizes = np.diff(start_cn)

            # find where the cell blocks start and end
            b = np.diff(sizes)
            c = np.concatenate([[0], np.where(b != 0)[0] + 1, [len(sizes)]])

            # Loop over all cell sizes, find all cells with this size, and assign
            # connectivity
            for cell_block_start, cell_block_end in zip(c, c[1:]):
                items = np.arange(cell_block_start, cell_block_end)
                sz = sizes[cell_block_start]

                new_order = vtk_to_meshio_order(types[start], dtype=offsets.dtype)
                if new_order is None:
                    new_order = np.arange(sz, dtype=offsets.dtype)
                new_order -= sz

                indices = np.add.outer(start_cn[items + 1], new_order)
                cells.append(CellBlock(meshio_type, connectivity[indices]))

                # Store cell data for this set of cells
                for name, d in cell_data_raw.items():
                    if name not in cell_data:
                        cell_data[name] = []
                    cell_data[name].append(d[start + items])
        else:
            # Non-polygonal cell. Same number of nodes per cell makes everything easier.
            n = num_nodes_per_cell[meshio_type]

            new_order = vtk_to_meshio_order(types[start], dtype=offsets.dtype)
            if new_order is None:
                new_order = np.arange(n, dtype=offsets.dtype)
            new_order -= n

            indices = np.add.outer(offsets[start:end], new_order)
            cells.append(CellBlock(meshio_type, connectivity[indices]))
            for name, d in cell_data_raw.items():
                if name not in cell_data:
                    cell_data[name] = []
                cell_data[name].append(d[start:end])

    return cells, cell_data

class Info:
    """Info Container for the VTK reader."""

    def __init__(self):
        self.points = None
        self.field_data = {}
        self.cell_data_raw = {}
        self.point_data = {}
        self.dataset = {}
        self.connectivity = None
        self.offsets = None
        self.types = None
        self.active = None
        self.is_ascii = False
        self.split = []
        self.num_items = 0
        # One of the problem in reading VTK files are POINT_DATA and CELL_DATA fields.
        # They can contain a number of SCALARS+LOOKUP_TABLE tables, without giving and
        # indication of how many there are. Hence, SCALARS must be treated like a
        # first-class section.  To associate it with POINT/CELL_DATA, we store the
        # `active` section in this variable.
        self.section = None

max_stars_repo_name: ginggs/meshio

max_issues_repo_name: ginggs/meshio

max_forks_repo_name: ginggs/meshio


text:
import csv
import sys
import pandas
import numpy
import event

basename = sys.argv[1]
start = int(sys.argv[2])
end = int(sys.argv[3])

total_s = 0
total_n = 0
for j in range(start, end+1):
    fname = basename + str(j) + ".csv"
    df = pandas.read_csv(fname, header=None)
    trace = df.to_numpy()
    trace = trace[:,0]
    mean = numpy.mean(trace)
    # print("avg", mean)
    s = 0
    n = 0
    for i in range(len(trace)):
        if abs(trace[i]-mean) < 40:
            s = s + trace[i]
            n = n + 1
    total_s = total_s + s
    total_n = total_n + n
    print(fname, s/n)
print("total", total_s/total_n)


text:
import argparse
import json
from pathlib import Path
import yaml

import numpy as np
import torch

from clane import similarity
from clane.graph import Graph
from clane.embedder import Embedder
from clane.embedder import IterativeEmbedder

def embedding(args):
    print('[Embedding]', end='\n')

    # parse hyper-params from the given yml file.
    if args.config_file.absolute().exists():
        with open(args.config_file.absolute(), 'r') as config_io:
            hparams = yaml.load(config_io, Loader=yaml.FullLoader)
    else:
        raise FileNotFoundError(f"Config file not found. {args.config_file.absolute()}")

    device = torch.device('cuda') if args.gpu else torch.device('cpu')
    g = Graph(
        data_root=args.data_root,
        **hparams["graph"],
    )

    print("Graph Loaded.")
    print(f" - {len(g)} vertices")
    print(f" - {len(g.E)} edges")
    print(" - Content Embeddings:")
    print(f"     - dim : {g.d:3d}")
    print(f"     - mean: {g.C.mean():5.2f}")
    print(f"     - std : {g.C.std():5.2f}")

    try:
        similarity_measure = getattr(similarity, hparams["similarity"]["method"])
    except AttributeError:
        raise AttributeError(f'Given similarity method {hparams["similarity"]["method"]} not found.')
    except Exception:
        raise

    similarity_measure = similarity_measure(
        **hparams['similarity']['kwargs'],
    )

    if hasattr(similarity_measure, 'parameters'):
        embedder = IterativeEmbedder(
            graph=g,
            similarity_measure=similarity_measure,
            device=device,
            save_history=args.save_history,
            num_workers=args.num_workers,
            **hparams["embedder"],
        )
    else:
        embedder = Embedder(
            graph=g,
            similarity_measure=similarity_measure,
            save_history=args.save_history,
            **hparams["embedder"],
        )
    embedder.iterate()

    print("Saving the results.")
    if not args.output_root.exists():
        args.output_root.mkdir(parents=True, exist_ok=True)

    if args.save_history:
        for iter, history in enumerate(embedder.history):

            args.output_root.joinpath(f'{iter}').mkdir(parents=True, exist_ok=True)

            if 'Z' in history.keys():
                for iter_Z, Z in enumerate(history["Z"]):
                    np.save(
                        args.output_root.joinpath(f'{iter}/Z_{iter_Z}.npy'),
                        Z.cpu().numpy(),
                    )
            if 'P_loss' in history.keys():
                with open(args.output_root.joinpath(f'{iter}/P_loss.json'), 'w') as io:
                    json.dump([loss.item() for loss in history["P_loss"]], io)
    np.save(
        args.output_root.joinpath('Z.npy'),
        g.Z.cpu().numpy(),
    )

    print(f"The embeddings are stored in {args.output_root.joinpath('Z.npy').absolute()}.")

def get_parser():
    parser = argparse.ArgumentParser(prog="clane")

    parser.add_argument(
        "--data_root", type=Path,
        help="Path to the data root directory."
    )
    parser.add_argument(
        "--output_root", type=Path,
        help="Path to the root for the experiment results to be stored."
    )
    parser.add_argument(
        "--config_file", type=Path,
        help="Path to the training configuration yaml file."
    )
    parser.add_argument(
        "--save_history", action='store_true',
        help="If true, it saves the embeddings for every iteration."
    )
    parser.add_argument(
        "--num_workers", type=int, default=0,
    )
    parser.add_argument(
        "--gpu", action='store_true'
    )
    return parser

def main():
    parser = get_parser()
    args = parser.parse_args()
    embedding(args)

if __name__ == "__main__":
    main()

max_stars_repo_name: helloybz/CLANE

max_issues_repo_name: helloybz/CLANE

max_forks_repo_name: helloybz/CLANE