text:
# %%
import pandas as pd
import numpy as np
import time
import datetime
from datetime import datetime as dt
from datetime import timezone
from spacepy import coordinates as coord
from spacepy.time import Ticktock
from astropy.constants import R_earth
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from plotly.offline import iplot
from kamodo_ccmc.flythrough.utils import ConvertCoord

def SatPlot4D(var,time,lon,lat,alt,vard,varu,inCoordName,inCoordType,plotCoord,groupby,model,
              displayplot=True,type='3D',body='black',divfile='',htmlfile=''):
    """New 4D plotting for satellite trajectories using plotly by Darren De Zeeuw

    __Required variables__

    var: string of variable name
    time: time formatted as a timestamp in UTC
    lat: latitude in deg
    lon: longitude in deg
    alt: altitude in km
    vard: data of variable var, same size array as positions
    varu: string of variable var units
    inCoordName: string for incoming coordinate system.  GDZ, GEO, GSM, GSE, SM, GEI, MAG, RLL
    inCoordType: string for incoming coordinate type.  car, sph
    plotCoord: string for coordinate system used in 3D plot. Assumes cartesian type.
    groupby: grouping of data for animation, values include
        all, day, hour, minute, N, orbitE, orbitM
    model: string of name of model the data was extracted from

    __Optional variables__

    displayplot: logical to show/hide displayed plot (may want false when saving htmlfile)
    type: string for choice of plot type, values: 3D, 1D, 2D, 2DLT
    body: string for choice of 3D inner body, values: black, earth (only GEO), none
    divfile: string with filename to save a html div file of the plot
    htmlfile: string with filename to save a full html file of the plot
    """

    REkm = (R_earth.value/1000.)

    if type == "3D":
        #Convert incoming coordinates into plot coordinages (cartesian)
        xx,yy,zz,units = ConvertCoord(time,lon,lat,alt,inCoordName,inCoordType,plotCoord,'car')

        # Create dictionary block to pass to plotting with selected options
        plot_dict=dict(
            title = 'Satellite extraction from model: '+model+"
"+plotCoord+" coordinates",  # Displayed title for plot, can use
 for new lines
            sats = ["Sat1"],            # Array of satellites to include in plot
            Sat1 = dict(
                display_name = "",
                time = dict(format='timestamp', data=time),  # possible formats: datetime, timestamp (assumes UTC)
                vars = dict(
                    x = dict(units=units[0], data=xx),
                    y = dict(units=units[1], data=yy),
                    z = dict(units=units[2], data=zz),
                    Lat = dict(units='deg', data=lat),
                    Lon = dict(units='deg', data=lon),
                    Alt = dict(units='km', data=alt),
                ),
                position_variables = ["x", "y", "z"],     # three variables to use for position
            ),
            options = dict(
                position_units = "R_E", # possible values: R_E, km, ""
                var = var,            # variable to use for colorscale
                hover_vars = ["Lat", "Lon", "Alt"],     # other information for hoverinfo display
                quiver = False,         # logical value to display or hide quivers
                quiver_scale = 0.1,     # length scale of quivers
                quiver_skip = 0,        # points to skip between displaying quivers
                groupby = groupby,        # possible values: all, day, hour, minute, N (integer, show N values at a time)
                                    #   orbitE (break at equator crossing S->N), orbitM (break at prime meridian crossing)
                body = body,         # possible values: black, earth, and any other value is no body
                colorscale = "Viridis", # named colorscale
                REkm = REkm,  # Earth radius in km
                coord = coord,  # Coordinate system of plot
            ),
        )
        # Fixed position variables already included, now add passed in variable to dictionary
        plot_dict['Sat1']['vars'][var]=dict(units=varu, data=vard)

        # Execute creation and display of figure
        fig=custom3Dsat(plot_dict,vbose=0)
        if divfile != '':
            print('-saving html div file: ',divfile)
            fig.write_html(divfile,full_html=False)
        if htmlfile != '':
            print('-saving full html file: ',htmlfile)
            fig.write_html(htmlfile,full_html=True)
        if displayplot:
            iplot(fig)

    if type == "1D" or type == "2D" or type == "2DLT":
        #Convert incoming coordinates into GDZ sph
        xx,yy,zz,units = ConvertCoord(time,lon,lat,alt,inCoordName,inCoordType,'GDZ','sph')
        xx[xx<0.] += 360.

        # Create dictionary block to pass to plotting with selected options
        plot_dict=dict(
            title = 'Satellite extraction from model: '+model,  # Displayed title for plot, can use
 for new lines
            sats = ["Sat1"],            # Array of satellites to include in plot
            Sat1 = dict(
                display_name = "",
                time = dict(format='timestamp', data=time),  # possible formats: datetime, timestamp (assumes UTC)
                vars = dict(
                    Lon = dict(units=units[0], data=xx),
                    Lat = dict(units=units[1], data=yy),
                    Alt = dict(units=units[2], data=zz),
                ),
                position_variables = ["Lon", "Lat", "Alt"],     # three variables to use for position
            ),
            options = dict(
                position_units = "", # possible values: R_E, km, ""
                var = var,            # variable to use for colorscale
                hover_vars = ["Lon", "Lat", "Alt"],     # other information for hoverinfo display
                groupby = groupby,        # possible values: all, day, hour, minute, N (integer, show N values at a time)
                                    #   orbitE (break at equator crossing S->N), orbitM (break at prime meridian crossing)
            ),
        )
        # Fixed position variables already included, now add passed in variable to dictionary
        plot_dict['Sat1']['vars'][var]=dict(units=varu, data=vard)

        # Execute creation and display of figure
        if type == "1D":
            fig=custom1Dsat(plot_dict,vbose=0)
        elif type == "2D":
            fig=custom2Dsat(plot_dict,vbose=0)
        elif type == "2DLT":
            fig=custom2Dsat(plot_dict,useLT=True,vbose=0)

        if divfile != '':
            print('-saving html div file: ',divfile)
            fig.write_html(divfile,full_html=False)
        if htmlfile != '':
            print('-saving full html file: ',htmlfile)
            fig.write_html(htmlfile,full_html=True)
        if displayplot:
            iplot(fig)

# ===============================================================================================
# ===============================================================================================
def custom3Dsat(datad, vbose=1):
    '''
    This function creates a custom 3D satellite plot, returning a plotly figure object.

    Parameters
    ----------
    datad: This is a data dictionary with the data used to create the plot
    vbose: An optional verbosity value, 0 will only print out warnings and errors. Default is 1.

    Returns
    -------
    fig: A plotly figure object that can then be visualized.

    Other
    -----
    The python code block below will setup and display a working demo.

import numpy as np
import datetime
from datetime import timezone
from plotly.offline import iplot

# Build a datetime array to use in the dictionary
base = datetime.datetime(2000, 1, 1).replace(tzinfo=timezone.utc)
arr = np.array([base + datetime.timedelta(minutes=30*i) for i in range(8)])

sample=dict(
    title = 'Plot Title Here',  # Displayed title for plot, can use
 for new lines
    sats = ["Sat1"],            # Array of satellites to include in plot
    Sat1 = dict(
        display_name = "Fake Satellite",
        time = dict(format='datetime', data=arr),  # possible formats: datetime, timestamp (assumes UTC)
        vars = dict(
            x = dict(units='R_E', data=np.array(np.arange(1.,9.))),
            y = dict(units='R_E', data=np.array(np.arange(1.,9.))),
            z = dict(units='R_E', data=np.array(np.arange(1.,9.))),
            p = dict(units='nP', data=np.array(np.arange(11.,19.))),
            U_x = dict(units='km/s', data=np.array(-1.*np.arange(11.,19.))),
            U_y = dict(units='km/s', data=np.array(np.arange(21.,29.))),
            U_z = dict(units='km/s', data=np.array(np.arange(31.,39.))),
        ),
        position_variables = ["x", "y", "z"],     # three variables to use for position
        vector_variables = ["U_x", "U_y", "U_z"], # three variables to use for quiver if quiver is True
    ),
    options = dict(
        position_units = "R_E", # possible values: R_E, km, ""
        var = "p",              # variable to use for colorscale
        hover_vars = ["U_x"],   # other information for hoverinfo display
        quiver = True,          # logical value to display or hide quivers
        quiver_scale = 0.1,     # length scale of quivers
        quiver_skip = 0,        # points to skip between displaying quivers
        groupby = "orbitM",     # possible values: all, day, hour, minute, N (integer, show N values at a time)
                                #   orbitE (break at equator crossing S->N), orbitM (break at prime meridian crossing)
        body = "black",         # possible values: black, earth, and any other value is no body
        colorscale = "Viridis", # named colorscale
        REkm = 6.3781E3,        # Earth radius in km
    ),
)

fig=custom3Dsat(sample)
iplot(fig)
    '''

    # ===============================================================================================
    # Start timer
    tic = time.perf_counter()

    # Start with error checking ...
    if 'title' not in datad:
        print("Warning, no title given for plot.")
        txttop = "No Title"
    else:
        txttop = datad['title']
    if 'var' not in datad['options']:
        print("ERROR, no variable selected to plot, returning.")
        return None
    var=datad['options']['var']
    if var == "time":
        varu=""
    else:
        varu=datad[datad['sats'][0]]['vars'][var]['units']
    if 'REkm' in datad['options']:
        REkm = datad['options']['REkm']
    else:
        REkm=6.3781E3
    scale=datad['options']['position_units']
    if 'groupby' in datad['options']:
        groupby = datad['options']['groupby']
    else:
        groupby = "all"
    if 'quiver' in datad['options']:
        quiver=datad['options']['quiver']
    else:
        quiver=False
    if quiver:
        quiverscale=datad['options']['quiver_scale']
        if scale == "km":
            quiverscale=quiverscale*REkm
        quiverskip=int(datad['options']['quiver_skip'])
    if 'body' in datad['options']:
        body=datad['options']['body']
    else:
        body = "none"
    if 'colorscale' in datad['options']:
        colorscale = datad['options']['colorscale']
    else:
        colorscale = "Viridis"
    if 'coord' in datad['options']:
        coord = datad['options']['coord']
    else:
        coord = ""

    # set initial values used later, including loop over all sats
    xmin=0.
    xmax=0.
    ymin=0.
    ymax=0.
    zmin=0.
    zmax=0.
    cmin= 1.e99
    cmax=-1.e99
    localts=dict()
    localtimestring=dict()
    agroup=dict()
    ugroup=()
    for sat in datad['sats']:
        sPts=len(datad[sat]['vars'][datad[sat]['position_variables'][0]]['data'])
        # Set localtimestring values
        notime=False
        if 'time' in datad[sat]:
            if datad[sat]['time']['format'] == "datetime":
                localts[sat]=np.array([d.timestamp() for d in datad[sat]['time']['data']])
            elif datad[sat]['time']['format'] == "timestamp":
                localts[sat]=datad[sat]['time']['data'].copy()
            else:
                print("ERROR, Unknown time format.")
                return None
            localtimestring[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d %H:%M:%S") for d in localts[sat]])
        else:
            notime=True
            if var == "time":
                print("ERROR, no time given and plot var selected is time")
                return None
            localtimestring[sat]=np.array(["point "+str(i+1) for i in range(sPts)])

        # Find global contour min/max
        if var == "time":
            c=localts[sat]
        else:
            c=datad[sat]['vars'][var]['data']
        cmin=min(cmin,min(c))
        cmax=max(cmax,max(c))

        # Create array of possible 'groupby' value
        if groupby == "day":
            if notime:
                print("ERROR, no time given and groupby value is",groupby)
                return None
            agroup[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d") for d in localts[sat]])
        elif groupby == "hour":
            if notime:
                print("ERROR, no time given and groupby value is",groupby)
                return None
            agroup[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d %H") for d in localts[sat]])
        elif groupby == "minute":
            if notime:
                print("ERROR, no time given and groupby value is",groupby)
                return None
            agroup[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d %H:%M") for d in localts[sat]])
        elif groupby == "orbitM":
            # Satellite path crosses prime meridian
            x=datad[sat]['vars'][datad[sat]['position_variables'][0]]['data']
            y=datad[sat]['vars'][datad[sat]['position_variables'][1]]['data']
            bgroup = ['orbit'] * len(x)
            j=1
            for i in range(sPts):
                if i != 0:
                    if x[i] > 0. and (y[i]*y[i-1]) < 0.:
                        j+=1
                bgroup[i] = "orbit "+str(j)
            agroup[sat]=np.array(bgroup)
        elif groupby == "orbitE":
            # Satellite path crosses equator going North
            z=datad[sat]['vars'][datad[sat]['position_variables'][2]]['data']
            bgroup = ['orbit'] * len(z)
            j=1
            for i in range(sPts):
                if i != 0:
                    if (z[i]>0. and z[i-1]<0.):
                        j+=1
                bgroup[i] = "orbit "+str(j)
            agroup[sat]=np.array(bgroup)
        elif groupby.isdigit():
            gb=int(groupby)
            agroup[sat] = np.array(["points "+str(int(i/gb)*gb+1)+" - "+str(int(i/gb)*gb+gb) for i in range(sPts)])
        else:
            agroup[sat] = np.array(["all" for i in range(sPts)])

        # Use pandas unique function rather than numpy. Its faster and does not sort the results.
        ugroup=pd.unique(np.append(ugroup, pd.unique(agroup[sat])))

    ngroup = len(ugroup)

    # Build DUMMY data block to insert as needed.
    data_dict_dummy = {
        "type": "scatter3d",
        "name": "dummy", "x": [0.], "y": [0.], "z": [0.],
        "mode": "lines", "line": {"width": 1},
        "hoverinfo": "none",
    }

    # ===============================================================================================  AAA
    # make figure dictionary pieces
    fig_dict = {"data": [], "layout": {}, "frames": []}
    fig_data_saved = {"data": []}
    sliders_dict = {
        "active": 0,
        "yanchor": "top", "xanchor": "left",
        "currentvalue": {
            "prefix": "Currently showing: ",
            "visible": True,
            "xanchor": "left"
        },
        "transition": {"duration": 0},
        "pad": {"b": 10, "t": 10},
        "len": 0.9,
        "x": 0.1,
        "y": 0,
        "steps": []
    }

    # Actual plot creation loop
    for date in ugroup:
        frame = {"data": [], "name": date}
        for sat in datad['sats']:
            x=datad[sat]['vars'][datad[sat]['position_variables'][0]]['data']
            y=datad[sat]['vars'][datad[sat]['position_variables'][1]]['data']
            z=datad[sat]['vars'][datad[sat]['position_variables'][2]]['data']
            sc=1.
            if scale == "km" and datad[sat]['vars'][datad[sat]['position_variables'][0]]['units'] == "R_E":
                sc=REkm
            elif scale == "R_E" and datad[sat]['vars'][datad[sat]['position_variables'][0]]['units'] == "km":
                sc=1./REkm
            if var == "time":
                c=localts[sat]
                varline=""
            else:
                c=datad[sat]['vars'][var]['data']
                varline=var+": %{marker.color:.4g} "+varu+"
"
            if quiver:
                qxvar=datad[sat]['vector_variables'][0]
                qyvar=datad[sat]['vector_variables'][1]
                qzvar=datad[sat]['vector_variables'][2]
                qx=datad[sat]['vars'][qxvar]['data']
                qy=datad[sat]['vars'][qyvar]['data']
                qz=datad[sat]['vars'][qzvar]['data']

            # Update position min/max values
            if date == ugroup[0]:
                xmin=min(xmin,min(x*sc))
                xmax=max(xmax,max(x*sc))
                ymin=min(ymin,min(y*sc))
                ymax=max(ymax,max(y*sc))
                zmin=min(zmin,min(z*sc))
                zmax=max(zmax,max(z*sc))

            # Compute mask to restrict all data in trace
            mask = date == agroup[sat]

            # Create hover information, including extras passed in. Quiver shows additional variables.
            Nhv = len(datad['options']['hover_vars'])
            cd=[]
            cd.append(localtimestring[sat][mask])
            qline=""
            Ndv=1
            if quiver:
                cd.append(qx[mask])
                cd.append(qy[mask])
                cd.append(qz[mask])
                qline+=qxvar+": %{customdata[1]:.2f}
"
                qline+=qyvar+": %{customdata[2]:.2f}
"
                qline+=qzvar+": %{customdata[3]:.2f}
"
                Ndv+=3
            for i in range(Nhv):
                cd.append(datad[sat]['vars'][datad['options']['hover_vars'][i]]['data'][mask])
                qline+=datad['options']['hover_vars'][i]+": %{customdata["+str(Ndv)+"]:.2f} "+\
                    datad[sat]['vars'][datad['options']['hover_vars'][i]]['units']+"
"
                Ndv+=1
            cd=np.asarray(cd).T
            dateline="%{customdata[0]}
"

            # Build data block with mask
            data_dict = {
                "type": "scatter3d",
                "name": date,
                "x": list(x[mask]*sc), "y": list(y[mask]*sc), "z": list(z[mask]*sc),
                "mode": "markers+lines",
                "marker": {
                    "size": 4, "cmin": cmin, "cmax": cmax, "color": list(c[mask]),
                    "showscale": True, "colorscale": colorscale,
                    "colorbar": { "title": ""+var+"
["+varu+"]", "tickformat": ".3g" }
                },
                "line": {"width": 3, "color": "rgba(22,22,22,0.2)"},
                "customdata": cd,
                "hovertemplate": ""+datad[sat]['display_name']+""+
                    "
X: %{x:.4f} "+scale+"
Y: %{y:.4f} "+scale+"
Z: %{z:.4f} "+scale+"
"+
                    qline+varline+dateline+"<extra></extra>",
            }

            # If time is colorbar variable, hide labels by selecting ticks out of range
            if var == "time":
                data_dict.marker.colorbar['tickvals']=(0,1)

            # Put each part of sequence in frame data block
            frame["data"].append(data_dict)

            # First in sequence, put dummy in main data block
            if date == ugroup[0]:
                fig_dict["data"].append(data_dict_dummy)

            # Start quiver
            if quiver:
                # Compute values to put in quiver trace
                # Make array max possible size (triple len(x)), fill, and trim as needed
                xx=np.concatenate([x,x,x])
                yy=np.concatenate([y,y,y])
                zz=np.concatenate([z,z,z])
                qxx=qx
                qyy=qy
                qzz=qz
                # Build new position array, element by element
                j=0
                for i in range(len(mask)):
                    if mask[i]:
                        if i%(quiverskip+1) == 0:
                            xx[j]=x[i]*sc
                            yy[j]=y[i]*sc
                            zz[j]=z[i]*sc
                            xx[j+1]=x[i]*sc+quiverscale*qx[i]
                            yy[j+1]=y[i]*sc+quiverscale*qy[i]
                            zz[j+1]=z[i]*sc+quiverscale*qz[i]
                            xx[j+2]=None
                            yy[j+2]=None
                            zz[j+2]=None
                            j+=3
                xx=np.array(xx[0:j], dtype=np.float64)
                yy=np.array(yy[0:j], dtype=np.float64)
                zz=np.array(zz[0:j], dtype=np.float64)

                # Update position min/max values
                xmin=min(xmin,min(xx))
                xmax=max(xmax,max(xx))
                ymin=min(ymin,min(yy))
                ymax=max(ymax,max(yy))
                zmin=min(zmin,min(zz))
                zmax=max(zmax,max(zz))

                # Build data block
                data_dict = {
                    "type": "scatter3d",
                    "name": "positions", "x": list(xx), "y": list(yy), "z": list(zz),
                    "mode": "lines", "line": {"width": 3, "color": "rgba(22,22,22,0.2)"},
                    "hoverinfo": "none",
                }
                # Put each part of sequence in frame data block
                frame["data"].append(data_dict)

                # First in sequence, put in main data block
                if date == ugroup[0]:
                    fig_dict["data"].append(data_dict_dummy)

        fig_dict["frames"].append(frame)
        slider_step = {"args": [
            [date],
            {"frame": {"duration": 300, "redraw": True},
             "mode": "immediate",
             "transition": {"duration": 0}}
        ],
            "label": date,
            "method": "animate"}
        sliders_dict["steps"].append(slider_step)

    # Assemble frame and slider pieces
    fig_dict["layout"]["sliders"] = [sliders_dict]

    # ===============================================================================================  BBB
    if ngroup > 1:
        for sat in datad['sats']:
            # Add trace if more than one group.
            # This shows the whole trajectory when a subsection of data is showing.
            x=datad[sat]['vars'][datad[sat]['position_variables'][0]]['data']
            y=datad[sat]['vars'][datad[sat]['position_variables'][1]]['data']
            z=datad[sat]['vars'][datad[sat]['position_variables'][2]]['data']
            sc=1.
            if scale == "km" and datad[sat]['vars'][datad[sat]['position_variables'][0]]['units'] == "R_E":
                sc=REkm
            elif scale == "R_E" and datad[sat]['vars'][datad[sat]['position_variables'][0]]['units'] == "km":
                sc=1./REkm

            # Build data block
            data_dict = {
                "type": "scatter3d",
                "name": "positions", "x": list(x*sc), "y": list(y*sc), "z": list(z*sc),
                "mode": "lines", "line": {"width": 3, "color": "rgba(22,22,22,0.2)"},
                "hoverinfo": "none",
            }

            # Put into main data block
            fig_dict["data"].append(data_dict)

    # ===============================================================================================  CCC
    ticE = time.perf_counter()
    # Load points and add 1 RE sphere, padded to cover all data positions
    if body == "black":
        dataXYZ = pd.read_csv('https://ccmc.gsfc.nasa.gov/Kamodo/demo/sphereXYZ.csv')
        dataIJK = pd.read_csv('https://ccmc.gsfc.nasa.gov/Kamodo/demo/sphereIJK.csv')
        if scale == "km":
            dataXYZ *= REkm
        # Build data block
        data_dict = {
            "type": "mesh3d",
            "name": '1 R_E sphere',
            "x": list(np.append(dataXYZ['x'],(xmin,xmax))),
            "y": list(np.append(dataXYZ['y'],(ymin,ymax))),
            "z": list(np.append(dataXYZ['z'],(zmin,zmax))),
            "i": list(dataIJK['i']),
            "j": list(dataIJK['j']),
            "k": list(dataIJK['k']),
            "facecolor": list(dataIJK['c']),
            "flatshading": True,
            "hovertemplate": "Earth<extra></extra>",
        }
        # Put in main data block
        fig_dict["data"].append(data_dict)
    elif body == "earth" and coord == "GEO":
        dataXYZ = pd.read_csv('https://ccmc.gsfc.nasa.gov/ungrouped/GM_IM/EarthXYZ.csv')
        dataIJK = pd.read_csv('https://ccmc.gsfc.nasa.gov/ungrouped/GM_IM/EarthIJKRGB.csv')
        if scale == "km":
            dataXYZ *= REkm
        color=np.array(["rgb("+str(dataIJK['r'][i])+","+str(dataIJK['g'][i])+","+str(dataIJK['b'][i])+")" \
                        for i in range(len(dataIJK['r']))])
        # Need to reverse x,y from file to be in proper GEO coords (180 degree rotation)
        xe=-dataXYZ['x']
        ye=-dataXYZ['y']
        ze= dataXYZ['z']
        # Build data block
        data_dict = {
            "type": "mesh3d",
            "name": '1 R_E sphere',
            "x": np.append(xe,(xmin,xmax)),
            "y": np.append(ye,(ymin,ymax)),
            "z": np.append(ze,(zmin,zmax)),
            "i": dataIJK['i'],
            "j": dataIJK['j'],
            "k": dataIJK['k'],
            "facecolor": color,
            "hovertemplate": "Earth<extra></extra>",
        }
        # Put in main data block
        fig_dict["data"].append(data_dict)
    tocE = time.perf_counter()
    if vbose > 0:
        print(f"  -time loading Earth: {tocE - ticE:0.4f} seconds")

    # ===============================================================================================  DDD
    # Set layout values
    fig_dict["layout"]["height"] = 700
    fig_dict["layout"]["width"] = 800
    fig_dict["layout"]["scene_aspectmode"] = "data"
    fig_dict["layout"]["scene"] = dict(xaxis=dict(title=dict(text="X ["+scale+"]")),
                                       yaxis=dict(title=dict(text="Y ["+scale+"]")),
                                       zaxis=dict(title=dict(text="Z ["+scale+"]")))
    fig_dict["layout"]["title_text"] = txttop
    fig_dict["layout"]["showlegend"] = False
    fig_dict["layout"]["scene_camera"] = dict(center=dict(x=0, y=0, z=0))
    fig_dict["layout"]["hoverlabel_align"] = 'right'
    if ngroup > 1:
        fig_dict["layout"]["updatemenus"] = [
            {
                "buttons": [
                    {
                        "args": [None, {"frame": {"duration": 500, "redraw": True},
                                        "fromcurrent": True, "transition": {"duration": 0}}],
                        "label": "Play",
                        "method": "animate"
                    },
                    {
                        "args": [[None], {"frame": {"duration": 0, "redraw": True},
                                          "mode": "immediate",
                                          "transition": {"duration": 0}}],
                        "label": "Pause",
                        "method": "animate"
                    }
                ],
                "direction": "left",
                "pad": {"r": 10, "t": 35},
                "showactive": False,
                "type": "buttons",
                "x": 0.1,
                "xanchor": "right",
                "y": 0,
                "yanchor": "top"
            }
        ]

    # end timer
    toc = time.perf_counter()
    if vbose > 0:
        print(f"Total time creating figure object: {toc - tic:0.4f} seconds")

    fig3 = go.Figure(fig_dict)

    return fig3

# ===============================================================================================
# ===============================================================================================
def custom1Dsat(datad, vbose=1):
    """
    This function creates a custom 1D satellite plot, returning a plotly figure object.

    Parameters
    ----------
    datad: This is a data dictionary with the data used to create the plot
    vbose: Optional verbosity value, 0 will only print out warnings and errors. Default is 1.

    Returns
    -------
    fig: A plotly figure object that can then be visualized.

    """

    # Start 1D fig
    var=datad['options']['var']
    varu=datad[datad['sats'][0]]['vars'][var]['units']
    Nhv = len(datad['options']['hover_vars'])

    localts=dict()
    localtimestring=dict()
    localdt=dict()
    txttop = datad['title']

    # For now this only makes the plot for the last sat in the dictionary.
    for sat in datad['sats']:
        sPts=len(datad[sat]['time']['data'])
        # Set localtimestring values
        notime=False
        if 'time' in datad[sat]:
            if datad[sat]['time']['format'] == "datetime":
                localts[sat]=np.array([d.timestamp() for d in datad[sat]['time']['data']])
            elif datad[sat]['time']['format'] == "timestamp":
                localts[sat]=datad[sat]['time']['data'].copy()
            else:
                print("ERROR, Unknown time format.")
                return None
            localtimestring[sat] = np.array([datetime.datetime.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d %H:%M:%S") for d in localts[sat]])
            localdt[sat] = np.array([datetime.datetime.fromtimestamp(int(d),tz=timezone.utc) for d in localts[sat]])
        else:
            notime=True
            if var == "time":
                print("ERROR, no time given and plot var selected is time")
                return None
            localtimestring[sat]=np.array(["point "+str(i+1) for i in range(sPts)])

        # Find global contour min/max
        if var == "time":
            c=localts[sat]
        else:
            c=datad[sat]['vars'][var]['data']

        fig1 = make_subplots(rows=(Nhv+1), cols=1, shared_xaxes=True, vertical_spacing=0.04)

        fig1.add_trace(go.Scatter(x=localdt[sat], y=c, name=var,
                                  mode='lines', line= dict(shape='linear', color='black'),
                                  hovertemplate=var+': %{y:.4g}
%{x}<extra></extra>',
                                ),
                       row=1, col=1)
        fig1.update_yaxes(title_text=''+var+'
['+varu+']', exponentformat='e', row=1, col=1)
        fig1.update_layout(yaxis=dict(title=dict(font=dict(size=12))))

        for i in range(Nhv):
            tmpv=datad['options']['hover_vars'][i]
            fig1.add_trace(go.Scatter(x=localdt[sat], y=datad[sat]['vars'][tmpv]['data'], name=tmpv,
                                      mode='lines', line= dict(shape='linear', color='black'),
                                      hovertemplate=tmpv+': %{y:.4g}
%{x}<extra></extra>',
                                     ),
                           row=(i+2), col=1)
            tmpu=""
            if tmpv == "Alt":
                tmpu=" [km]"
            if tmpv == "Lon":
                tmpu=" [deg]"
                fig1.update_yaxes(tick0=0., dtick=90., row=(i+2), col=1)
            if tmpv == "Lat":
                tmpu=" [deg]"
                fig1.update_yaxes(tick0=0., dtick=30., row=(i+2), col=1)
            ya='yaxis'+str(i+2)
            ys="dict(text='"+tmpv+""+tmpu+"',font=dict(size=12))"
            fig1['layout'][ya]['title']=eval(ys)

        fig1.update_layout(height=600, width=800, title_text=txttop, showlegend = False,)

    return fig1

# ===============================================================================================
# ===============================================================================================
def custom2Dsat(datad, useLT=False, vbose=1):
    '''
    This function creates a custom 2D satellite plot in lat/lon or lat/LT, returning a plotly figure object.

    Parameters
    ----------
    datad: This is a data dictionary with the data used to create the plot
    useLT: Optional logical to modify plot to use local time instead of longitude on the X axis.
    vbose: Optional verbosity value, 0 will only print out warnings and errors. Default is 1.

    Returns
    -------
    fig: A plotly figure object that can then be visualized.

    '''

    # ===============================================================================================
    # Start timer
    tic = time.perf_counter()

    # Start with error checking ...
    if 'title' not in datad:
        print("Warning, no title given for plot.")
        txttop = "No Title"
    else:
        txttop = datad['title']
    if 'var' not in datad['options']:
        print("ERROR, no variable selected to plot, returning.")
        return None
    var=datad['options']['var']
    if var == "time":
        varu=""
    else:
        varu=datad[datad['sats'][0]]['vars'][var]['units']
    if 'REkm' in datad['options']:
        REkm = datad['options']['REkm']
    else:
        REkm=6.3781E3
    scale=datad['options']['position_units']
    if 'groupby' in datad['options']:
        groupby = datad['options']['groupby']
    else:
        groupby = "all"
    if 'body' in datad['options']:
        body=datad['options']['body']
    else:
        body = "none"
    if 'colorscale' in datad['options']:
        colorscale = datad['options']['colorscale']
    else:
        colorscale = "Viridis"

    # set initial values used later, including loop over all sats
    cmin= 1.e99
    cmax=-1.e99
    localts=dict()
    localtimestring=dict()
    agroup=dict()
    ugroup=()
    for sat in datad['sats']:
        sPts=len(datad[sat]['vars'][datad[sat]['position_variables'][0]]['data'])
        # Set localtimestring values
        notime=False
        if 'time' in datad[sat]:
            if datad[sat]['time']['format'] == "datetime":
                localts[sat]=np.array([d.timestamp() for d in datad[sat]['time']['data']])
            elif datad[sat]['time']['format'] == "timestamp":
                localts[sat]=datad[sat]['time']['data'].copy()
            else:
                print("ERROR, Unknown time format.")
                return None
            localtimestring[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d %H:%M:%S") for d in localts[sat]])
        else:
            notime=True
            if var == "time" or useLT:
                print("ERROR, no time given and plot var selected is time")
                return None
            localtimestring[sat]=np.array(["point "+str(i+1) for i in range(sPts)])

        # Find global contour min/max
        if var == "time":
            c=localts[sat]
        else:
            c=datad[sat]['vars'][var]['data']
        cmin=min(cmin,min(c))
        cmax=max(cmax,max(c))

        # Create array of possible 'groupby' value
        if groupby == "day":
            if notime:
                print("ERROR, no time given and groupby value is",groupby)
                return None
            agroup[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d") for d in localts[sat]])
        elif groupby == "hour":
            if notime:
                print("ERROR, no time given and groupby value is",groupby)
                return None
            agroup[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d %H") for d in localts[sat]])
        elif groupby == "minute":
            if notime:
                print("ERROR, no time given and groupby value is",groupby)
                return None
            agroup[sat] = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime\
                ("%Y-%m-%d %H:%M") for d in localts[sat]])
        elif groupby == "orbitM":
            # Satellite path crosses prime meridian
            lon=datad[sat]['vars'][datad[sat]['position_variables'][0]]['data']
            bgroup = ['orbit'] * len(lon)
            j=1
            for i in range(sPts):
                if i != 0:
                    if abs(lon[i]-lon[i-1]) > 180.:
                        j+=1
                bgroup[i] = "orbit "+str(j)
            agroup[sat]=np.array(bgroup)
        elif groupby == "orbitE":
            # Satellite path crosses equator going North
            lat=datad[sat]['vars'][datad[sat]['position_variables'][1]]['data']
            bgroup = ['orbit'] * len(lat)
            j=1
            for i in range(sPts):
                if i != 0:
                    if (lat[i]>0. and lat[i-1]<0.):
                        j+=1
                bgroup[i] = "orbit "+str(j)
            agroup[sat]=np.array(bgroup)
        elif groupby.isdigit():
            gb=int(groupby)
            agroup[sat] = np.array(["points "+str(int(i/gb)*gb+1)+" - "+str(int(i/gb)*gb+gb) for i in range(sPts)])
        else:
            agroup[sat] = np.array(["all" for i in range(sPts)])

        # Use pandas unique function rather than numpy. Its faster and does not sort the results.
        ugroup=pd.unique(np.append(ugroup, pd.unique(agroup[sat])))

    ngroup = len(ugroup)

    # Build DUMMY data block to insert as needed.
    data_dict_dummy = {
        "type": "scatter",
        "name": "dummy", "x": [0.], "y": [0.],
        "mode": "lines", "line": {"width": 1},
        "hoverinfo": "none",
    }

    # ===============================================================================================  AAA
    # make figure dictionary pieces
    fig_dict = {"data": [], "layout": {}, "frames": []}
    fig_data_saved = {"data": []}
    sliders_dict = {
        "active": 0,
        "yanchor": "top", "xanchor": "left",
        "currentvalue": {
            "prefix": "Currently showing: ",
            "visible": True,
            "xanchor": "left"
        },
        "transition": {"duration": 0},
        "pad": {"b": 10, "t": 40},
        "len": 0.9,
        "x": 0.1,
        "y": 0,
        "steps": []
    }

    # Actual plot creation loop
    for date in ugroup:
        frame = {"data": [], "name": date}
        for sat in datad['sats']:
            lon=datad[sat]['vars'][datad[sat]['position_variables'][0]]['data']
            lat=datad[sat]['vars'][datad[sat]['position_variables'][1]]['data']

            h = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime("%H") for d in localts[sat]])
            m = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime("%M") for d in localts[sat]])
            s = np.array([dt.fromtimestamp(int(d),tz=timezone.utc).strftime("%S.%f") for d in localts[sat]])
            lt=np.array(h, dtype=np.float64)
            for i in range(len(h)):
                lt[i]=(float(h[i]))+(float(m[i])/60.)+(float(s[i])/3600.)+lon[i]*24./360.
            lt[lt > 24.]-=24.

            if var == "time":
                c=localts[sat]
                varline=""
            else:
                c=datad[sat]['vars'][var]['data']
                varline=var+": %{marker.color:.4g} "+varu+"
"

            # Compute mask to restrict all data in trace
            mask = date == agroup[sat]

            # Create hover information, including extras passed in.
            Nhv = len(datad['options']['hover_vars'])
            cd=[]
            cd.append(localtimestring[sat][mask])
            qline=""
            Ndv=1
            for i in range(Nhv):
                hovv=datad['options']['hover_vars'][i]
                if hovv != "Lat" and hovv != "Lon":
                    cd.append(datad[sat]['vars'][hovv]['data'][mask])
                    qline+=hovv+": %{customdata["+str(Ndv)+"]:.2f} "+\
                        datad[sat]['vars'][hovv]['units']+"
"
                    Ndv+=1
            if useLT:
                hovv='Lon'
                cd.append(datad[sat]['vars'][hovv]['data'][mask])
                qline+=hovv+": %{customdata["+str(Ndv)+"]:.2f} "+\
                    datad[sat]['vars'][hovv]['units']+"
"
                Ndv+=1
            else:
                cd.append(lt[mask])
                qline+="LT: %{customdata["+str(Ndv)+"]:.2f}
"
                Ndv+=1
            cd=np.asarray(cd).T
            dateline="%{customdata[0]}
"

            # Build data block with mask
            if useLT:
                data_dict = {
                    "type": "scatter",
                    "name": date,
                    "x": list(lt[mask]), "y": list(lat[mask]),
                    "mode": "markers",
                    "marker": {
                        "size": 9, "cmin": cmin, "cmax": cmax, "color": list(c[mask]),
                        "showscale": True, "colorscale": colorscale,
                        "colorbar": { "title": ""+var+"
["+varu+"]", "tickformat": ".3g" }
                    },
                    "customdata": cd,
                    "hovertemplate": ""+datad[sat]['display_name']+""+
                        "
LT: %{x:.4f}
Lat: %{y:.4f}
"+
                        qline+varline+dateline+"<extra></extra>",
                }
            else:
                data_dict = {
                    "type": "scatter",
                    "name": date,
                    "x": list(lon[mask]), "y": list(lat[mask]),
                    "mode": "markers",
                    "marker": {
                        "size": 9, "cmin": cmin, "cmax": cmax, "color": list(c[mask]),
                        "showscale": True, "colorscale": colorscale,
                        "colorbar": { "title": ""+var+"
["+varu+"]", "tickformat": ".3g" }
                    },
                    "customdata": cd,
                    "hovertemplate": ""+datad[sat]['display_name']+""+
                        "
Lon: %{x:.4f}
Lat: %{y:.4f}
"+
                        qline+varline+dateline+"<extra></extra>",
                }

            # If time is colorbar variable, hide labels by selecting ticks out of range
            if var == "time":
                data_dict.marker.colorbar['tickvals']=(0,1)

            if ngroup == 1:
                # Put full sequence in main data block
                fig_dict["data"].append(data_dict)

            else:
                # Put each part of sequence in frame data block
                frame["data"].append(data_dict)
                # First in sequence, put dummy in main data block
                if date == ugroup[0]:
                    fig_dict["data"].append(data_dict_dummy)

        if ngroup > 1:
            fig_dict["frames"].append(frame)
        slider_step = {"args": [
            [date],
            {"frame": {"duration": 300, "redraw": True},
             "mode": "immediate",
             "transition": {"duration": 0}}
        ],
            "label": date,
            "method": "animate"}
        sliders_dict["steps"].append(slider_step)

    if ngroup > 1:
        # Assemble frame and slider pieces
        fig_dict["layout"]["sliders"] = [sliders_dict]

    # ===============================================================================================  BBB
    if ngroup > 1:
        for sat in datad['sats']:
            # Add trace if more than one group.
            # This shows the whole trajectory when a subsection of data is showing.
            lon=datad[sat]['vars'][datad[sat]['position_variables'][0]]['data']
            lat=datad[sat]['vars'][datad[sat]['position_variables'][1]]['data']

            # Make array larger than needed (double len(lat)), fill, and trim as needed
            xx=np.concatenate([lon,lon])
            yy=np.concatenate([lat,lat])
            # Build new position array, element by element
            j=0
            for i in range(len(lat)):
                xx[j]=lon[i]
                yy[j]=lat[i]
                j+=1
                if i < (len(lat)-1) and abs(lon[i]-lon[i+1]) > 180.:
                    L=min(abs(lon[i]),abs(360-lon[i]))
                    R=min(abs(lon[i+1]),abs(360-lon[i+1]))
                    Llat=lat[i]+(lat[i+1]-lat[i])*L/(L+R)
                    Rlat=lat[i]+(lat[i+1]-lat[i])*R/(L+R)
                    xx[j]=360.
                    yy[j]=Llat
                    xx[j+1]=None
                    yy[j+1]=None
                    xx[j+2]=0.
                    yy[j+2]=Rlat
                    j+=3
            xx=np.array(xx[0:j], dtype=np.float64)
            yy=np.array(yy[0:j], dtype=np.float64)

            # Build data block
            data_dict = {
                "type": "scatter",
                "name": "positions", "x": list(xx), "y": list(yy),
                "mode": "lines", "line": {"width": 1, "color": "rgba(22,22,22,0.2)"},
                "hoverinfo": "none",
            }

            # Put into main data block
            if not useLT:
                fig_dict["data"].append(data_dict)

    # ===============================================================================================  DDD
    # Set layout values
    if useLT:
        fig_dict["layout"]["xaxis"] = {'dtick': 3.0, 'range': [0.0, 24.0], 'tick0': 0.0,
                                       'title': {'text': 'LT [hrs]'}}
    else:
        fig_dict["layout"]["xaxis"] = {'dtick': 30.0, 'range': [0.0, 360.0], 'tick0': 0.0,
                                       'title': {'text': 'Lon [deg]'}}
    fig_dict["layout"]["yaxis"] = {'dtick': 30.0, 'range': [-90.0, 90.0], 'tick0': 0.0,
                                   'title': {'text': 'Lat [deg]'}}
    fig_dict["layout"]["title_text"] = txttop
    fig_dict["layout"]["showlegend"] = False
    fig_dict["layout"]["hoverlabel_align"] = 'right'
    if ngroup > 1:
        fig_dict["layout"]["updatemenus"] = [
            {
                "buttons": [
                    {
                        "args": [None, {"frame": {"duration": 500, "redraw": True},
                                        "fromcurrent": False, "transition": {"duration": 0}}],
                        "label": "Play",
                        "method": "animate"
                    },
                    {
                        "args": [[None], {"frame": {"duration": 0, "redraw": True},
                                          "mode": "immediate",
                                          "transition": {"duration": 0}}],
                        "label": "Pause",
                        "method": "animate"
                    }
                ],
                "direction": "left",
                "pad": {"r": 10, "t": 65},
                "showactive": False,
                "type": "buttons",
                "x": 0.1,
                "xanchor": "right",
                "y": 0,
                "yanchor": "top"
            }
        ]

    # end timer
    toc = time.perf_counter()
    if vbose > 0:
        print(f"Total time creating figure object: {toc - tic:0.4f} seconds")

    fig2 = go.Figure(fig_dict)
    return fig2

# %%

reason: import numpy,from astropy


text:
# from __future__ import absolute_import, division, print_function, unicode_literals
# 导入TensorFlow和tf.keras
import os
os.environ['CUDA_VISIBLE_DEVICES']='1'
import tensorflow as tf
from tensorflow import keras
from keras import initializers
from keras import optimizers
from keras.callbacks import *
from keras.models import Sequential, load_model
from keras.layers import Conv2D, LSTM, Flatten, Dense, BatchNormalization, Dropout, Reshape, MaxPooling2D
from keras.callbacks import LearningRateScheduler, TensorBoard, ModelCheckpoint, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from keras.regularizers import l1, l2
from keras.utils import multi_gpu_model
# from networks.train_plot import PlotLearning

# 导入辅助库
import numpy as np
import matplotlib.pyplot as plt
import datetime

# 检验tensorflow版本
print(tf.__version__)

# strategy = tf.distribute.MirroredS    trategy(devices=["/device:GPU:0", "/device:GPU:1"],
#                                           cross_device_ops=tf.contrib.distribute.AllReduceCrossDeviceOps(
#                                               all_reduce_alg="hierarchical_copy")
#                                           )
# print('Number of devices: {}'.format(strategy.num_replicas_in_sync))

# 自动创建文件以保存模型
def mkdir(path):
    # 引入模块
    import os

    # 去除首位空格
    path = path.strip()
    # 去除尾部 \ 符号
    path = path.rstrip("\\")

    # 判断路径是否存在
    # 存在     True
    # 不存在   False
    isExists = os.path.exists(path)

    # 判断结果
    if not isExists:
        # 如果不存在则创建目录
        # 创建目录操作函数
        os.makedirs(path)

        print(path + ' 创建成功')
        return True
    else:
        # 如果目录存在则不创建，并提示目录已存在
        print(path + ' 目录已存在')
        return False

# 并没有成功实现多gpu训练，目前只能做单gpu
# class ParallelModelCheckpoint(ModelCheckpoint):
#     def __init__(self, model, filepath, monitor='val_loss', verbose=0,
#                  save_best_only=False, save_weights_only=False,
#                  mode='auto', period=1):
#         self.single_model = model
#         super(ParallelModelCheckpoint, self).__init__(filepath, monitor, verbose, save_best_only, save_weights_only,
#                                                       mode, period)

#     def set_model(self, model):
#         super(ParallelModelCheckpoint, self).set_model(self.single_model)

# HAPPY
class happy_model:
    def __init__(self, epochs=1000, batch_size=2048, load_weights=True):
        self.name = 'happy'
        self.model_filename = './happy.h5'
        self.num_classes = 8
        self.input_shape = [28, 28, 1]
        self.epochs = epochs#
        self.batch_size = batch_size#
        self.weight_decay = 0.0001
        self.log_filepath = r'./happy_tensorboard/'
        self.conv_l1_regularizer = 0.003#
        self.lstm_l1_regularizer = 0.0011#
        self.start_lr = 0.001#
        self.end_lr = 0.000001#
        self.patience = 50#
        self.epoch_1 = 1
        self.epoch_2 = 2
        self.epoch_3 = 3
        self.lr_1 = 0.001
        self.lr_2 = 0.001
        self.lr_3 = 0.001

        # 如果有训练好的模型就直接加载
        if load_weights:
            try:
                self._model = load_model(self.model_filename)
                print('Successfully loaded', self.name)
            except (ImportError, ValueError, OSError) as e:
                print(e)
                print('Failed to load', self.name)

    def count_params(self):
        return self._model.count_params()

    def build_model(self):
        # 此模型的结构
        # self.batch_size = self.batch_size * strategy.num_replicas_in_sync
        # with strategy.scope():
        model = Sequential([

            # # FLATTEN Finishedsparse_
            Reshape((-1, 784, 1), input_shape=self.input_shape),
            #
            # # CONV 1 Finished
            Conv2D(32, (1, 25,), padding='SAME', strides=[1, 1, ], activation='relu',
                   kernel_initializer=initializers.random_normal(stddev=0.1),
                   kernel_regularizer=l1(self.conv_l1_regularizer)),
            BatchNormalization(),
            MaxPooling2D((1, 3), strides=(1, 3), padding='SAME'),
            #
            # # CONV 2 Finished
            Conv2D(64, (1, 25,), padding='SAME', strides=[1, 1, ], activation='relu',
                   kernel_initializer=initializers.random_normal(stddev=0.1),
                   kernel_regularizer=l1(self.lstm_l1_regularizer)),
            BatchNormalization(),
            MaxPooling2D((1, 3), strides=(1, 3), padding='SAME'),
            #
            # # DENSE 1 / Dropout Finished
            Flatten(),
            Dense(1024, activation='relu'),
            Dropout(0.5),
            Reshape((32, 32)),
            #
            # # LSTM 1-3 Finished
            LSTM(256, dropout=0.5, return_sequences=True, kernel_regularizer=l1(self.lstm_l1_regularizer)),
            LSTM(256, dropout=0.5, return_sequences=True, kernel_regularizer=l1(self.lstm_l1_regularizer)),
            LSTM(256, dropout=0.5, return_sequences=False, kernel_regularizer=l1(self.lstm_l1_regularizer)),

            # DENSE 2 / SOFTMAX Finished
            # Dense(100, activation='relu', kernel_initializer=initializers.random_normal(stddev=0.01)),
            # Flatten(),
            Dense(8, activation='softmax', kernel_initializer=initializers.random_normal(stddev=0.01)),

        ])
        adam = optimizers.Adam(lr=self.start_lr, beta_1=0.9, beta_2=0.999, )  # 7.28增大10 times训练步长
        model.compile(optimizer=adam,
                      loss='categorical_crossentropy',
                      metrics=['accuracy'])

        # sparse_
        return model

    def scheduler(self, epoch):
        # 采用了调度器来开始更改学习率，此代码中并没有开启，而是采用该指数递减法；
        # print(epoch, '--------------------------')
        if epoch <= self.epoch_1:
            return self.lr_1
        if epoch <= self.epoch_2:
            return self.lr_2
        if epoch <= self.epoch_3:
            return self.lr_3
        return self.lr_3

    def train(self):
        data_path = '../../0_AEEA_dataset/8class_of_traffic_dataset/data_norm8CLS_ALL.npy'
        label_path = '../../0_AEEA_dataset/8class_of_traffic_dataset/label_norm8CLS_ALL.npy'
        data = np.load(data_path)
        data = data.reshape([-1, 28, 28, 1])
        label_n = np.load(label_path)
        print('data的数量', data.shape)
        print('label的数量', label_n.shape)
        print(label_n[1:10])
        print("label的格式为：", type(label_n))

        x_train = data[:12417]
        y_train = label_n[:12417]
        x_test = data[12417:]
        y_test = label_n[12417:]

        # 数据归一化到【0：255】
        x_test = x_test * 256
        self.x_test = x_test.astype(int)
        x_train = x_train * 256
        self.x_train = x_train.astype(int)
        y_train = keras.utils.to_categorical(y_train, self.num_classes)
        y_test = keras.utils.to_categorical(y_test, self.num_classes)
        self.y_test = y_test.astype(int)
        self.y_train = y_train.astype(int)

        # 模型
        model = self.build_model()
        model.summary()

        # 参数文件夹保存
        mkdir(self.model_filename + 'date_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))

        # 训练
        change_lr = LearningRateScheduler(self.scheduler)

        checkpoint = ModelCheckpoint(
            self.model_filename + 'date_' + datetime.datetime.now().strftime(
                "%Y%m%d-%H%M%S") + '/' + 'epoch_' + '{epoch:02d}' + '_val_acc_' + '{val_acc:.4f}' + '.h5',
            monitor='val_acc',
            verbose=0,
            save_best_only=True,
            mode='auto',
            period=5)
        # plot_callback = PlotLearning()
        # 记录结果，并采用tensorboard可视化；
        tb_cb = TensorBoard(
            log_dir=self.log_filepath + 'date_' + datetime.datetime.now().strftime(
                "%Y%m%d-%H%M%S") + '_conv_l1_' + str(self.conv_l1_regularizer) + '_lstm_l1_' + str(
                self.lstm_l1_regularizer),
            histogram_freq=0)

        # lr change
        reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, verbose=1,
                                      patience=self.patience, min_lr=self.end_lr)

        #SGDR_lr = LR_Cycle(4000, 1.5)
        cbks = [checkpoint, tb_cb, reduce_lr]

        # start traing
        model.fit(x=self.x_train, y=self.y_train,
                  batch_size=self.batch_size,
                  epochs=self.epochs,
                  callbacks=cbks,
                  verbose=2,
                  validation_data=(self.x_test, self.y_test),
                  )
        # save model
        model.save(self.model_filename + '.h5')

        self._model = model

    def predict(self, img):
        # 预测结果
        return self._model.predict(img, batch_size=self.batch_size)

    def predict_one(self, img):
        return self.predict(img)[0]

    def accuracy(self):
        return self._model.evaluate(self.x_test, self.y_test, verbose=0)[1]

if __name__ == '__main__':
    happy = happy_model()
    happy.train()
    print(happy.accuracy())

    # best(val_acc:97): 0。003 0。001 0.000001/
    # goaled: 0.01 0.0005 0.0000001/0.003 0.003 0.000001/0。01 0。005 /0。003 0。001 0.000001/0.001 0.00013 0.0001
    # failed: 0.01 0.001 0.000001/

ext: py

reason: import numpy


text:
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import sklearn
from sklearn.metrics import confusion_matrix
import seaborn as sns

from general_utilities import create_dir

# utility to display a row of images with their predictions and true labels
def display_images(image, predictions, labels, n, save_img_path, all_eval=False):

    save_img_path = save_img_path + '/model_predictions_using_eval_data'
    if all_eval:
        save_img_path = save_img_path + '_all'
    create_dir(save_img_path)

    image = np.reshape(image, [n, 224, 224])
    for i in np.arange(n):

        plt.figure(figsize=(8,8))
        img = image[i,:,:]
        plt.imshow(tf.keras.preprocessing.image.array_to_img(np.rot90(img[:,:,np.newaxis],3))) # Rotate Nifti (by 90^o three times) to normal way of visualization
        if labels[i] == predictions[i]:
            color = 'green'
        else:
            color = 'red'
        plt.xlabel('Actual Label: {}; Predicted Label: {}'.format(labels[i], predictions[i]), color=color)
        plt.savefig(save_img_path + '/model_predictions_using_eval_data_' + str('{0:04d}'.format(i)) + '.png')
        plt.close()

def pred_plots_for_evaluation(class_names, eval_dataset, model, NO_OF_IMAGES, save_img_path, all_eval=False):

    dataset = eval_dataset
    x_batches, y_pred_batches, y_true_batches = [], [], []

    for x, y, _ in dataset:
        y_pred = model(x)
        y_pred_batches = y_pred.numpy()
        y_true_batches = y.numpy()
        x_batches = x.numpy()

    # generate 'size' random numbers in range 0 to 'length of y_pred_batches':
    indexes = np.random.choice(len(y_pred_batches), size=NO_OF_IMAGES)

    images_to_plot = x_batches[indexes]
    y_pred_to_plot = y_pred_batches[indexes]
    y_true_to_plot = y_true_batches[indexes]

    y_pred_labels = [class_names[int(np.around(sel_y_pred.item()))] for sel_y_pred in y_pred_to_plot]
    y_true_labels = []
    for sel_y_true in y_true_to_plot:
        i = np.around(np.asscalar(sel_y_true))
        y_true_labels.append(class_names[i.astype(int)])

    display_images(images_to_plot, y_pred_labels, y_true_labels, NO_OF_IMAGES, save_img_path, all_eval=all_eval)

def obtain_ytrue_ypred(dataset, model, all_eval=False):

    if all_eval:
        y_pred_batches, y_true_batches = np.array([]), np.array([])
        for x, y, _ in dataset:
            y_pred = model(x)
            y_pred_batches = np.append( y_pred_batches, y_pred.numpy())
            y_true_batches = np.append( y_true_batches, y.numpy())
    else:

        y_pred_batches, y_true_batches = [], []
        for x, y, _ in dataset.take(1):
            y_pred = model(x)
            y_pred_batches = y_pred.numpy()
            y_true_batches = y.numpy()

    y_true_batches_ = [int(np.around(y_true_i.item())) for y_true_i in y_true_batches]
    y_pred_batches_ = [int(np.around(y_pred_i.item())) for y_pred_i in y_pred_batches]

    return y_true_batches_, y_pred_batches_

def plot_cm(train_dataset, test_dataset, eval_dataset, model, save_img_path):

    '''Confusion Matrix Plots'''

    save_img_path = save_img_path + '/model_evaluation_confusion_matrix_plots'
    create_dir(save_img_path)

    train_y_true_batches, train_y_pred_batches = obtain_ytrue_ypred(train_dataset, model)
    test_y_true_batches, test_y_pred_batches = obtain_ytrue_ypred(test_dataset, model)
    eval_y_true_batches, eval_y_pred_batches = obtain_ytrue_ypred(eval_dataset, model)

    def plot_and_print(labels, predictions, title):

        print('\n')
        print('Confusion matrix using {} dataset'.format(title))

        cm = confusion_matrix(labels, predictions)
        plt.figure(figsize=(5,5))
        sns.heatmap(cm, annot=True, fmt="d")
        plt.title('Confusion matrix using {} dataset'.format(title))
        plt.ylabel('Actual label')
        plt.xlabel('Predicted label')

        tn = cm[0][0]
        fp = cm[0][1]
        fn = cm[1][0]
        tp = cm[1][1]
        print('True Negatives: ', tn)
        print('False Positives: ', fp)
        print('False Negatives: ', fn)
        print('True Positives: ', tp)

        # Calculate precision
        if (tp + fp == 0):
            precision = 1.0
        else:
            precision = tp / (tp + fp)
        print('Precision: ', precision)

        # Calculate recall
        if (tp + fn == 0):
            recall = 1.0
        else:
            recall = tp / (tp + fn)
        print('Recall: ', recall)

        # Calculate F1-Score
        f1_score = 2 * ((precision * recall) / (precision + recall))
        print('F1 Score: ', f1_score)

        plt.savefig(save_img_path + '/model_evaluation_confusion_matrix_plots_' + title + '.png')
        plt.close()

    plot_and_print(train_y_true_batches, train_y_pred_batches, 'training')
    plot_and_print(test_y_true_batches, test_y_pred_batches, 'testing')
    plot_and_print(eval_y_true_batches, eval_y_pred_batches, 'evaluation')

def plot_cm_eval(dataset, model, save_img_path, title):

    '''Confusion Matrix Plots'''

    save_img_path = save_img_path + '/model_evaluation_confusion_matrix_plots_all'
    create_dir(save_img_path)

    y_true_batches, y_pred_batches = obtain_ytrue_ypred(dataset, model, all_eval=True)

    def plot_and_print(labels, predictions, title):

        print('\n')
        print('Confusion matrix using {} dataset'.format(title))

        cm = confusion_matrix(labels, predictions)
        plt.figure(figsize=(5,5))
        sns.heatmap(cm, annot=True, fmt="d")
        plt.title('Confusion matrix using {} dataset'.format(title))
        plt.ylabel('Actual label')
        plt.xlabel('Predicted label')

        tn = cm[0][0]
        fp = cm[0][1]
        fn = cm[1][0]
        tp = cm[1][1]
        print('True Negatives: ', tn)
        print('False Positives: ', fp)
        print('False Negatives: ', fn)
        print('True Positives: ', tp)

        # Calculate precision
        if (tp + fp == 0):
            precision = 1.0
        else:
            precision = tp / (tp + fp)
        print('Precision: ', precision)

        # Calculate recall
        if (tp + fn == 0):
            recall = 1.0
        else:
            recall = tp / (tp + fn)
        print('Recall: ', recall)

        # Calculate F1-Score
        f1_score = 2 * ((precision * recall) / (precision + recall))
        print('F1 Score: ', f1_score)

        plt.savefig(save_img_path + '/model_evaluation_confusion_matrix_plots_' + title + '.png')
        plt.close()

    plot_and_print(y_true_batches, y_pred_batches, title)

def plot_roc(train_dataset, test_dataset, eval_dataset, model, save_img_path):

    save_img_path = save_img_path + '/model_evaluation_roc_curve_plots'
    create_dir(save_img_path)

    colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

    def plot_roc_curve(ax, labels, predictions, title, **kwargs):

        fp, tp, _ = sklearn.metrics.roc_curve(labels, predictions)

        ax.plot(100*fp, 100*tp, label=title, linewidth=2, **kwargs)
        ax.set_xlabel('False positives [%]')
        ax.set_ylabel('True positives [%]')
        #ax.set_xlim([-0.5,20])
        #ax.set_ylim([80,100.5])
        ax.grid(True)
        ax.set_aspect('equal')

        return ax

    train_y_true_batches, train_y_pred_batches = obtain_ytrue_ypred(train_dataset, model)
    test_y_true_batches, test_y_pred_batches = obtain_ytrue_ypred(test_dataset, model)
    eval_y_true_batches, eval_y_pred_batches = obtain_ytrue_ypred(eval_dataset, model)

    plt.figure(figsize=(5,5))
    ax = plt.gca()
    ax = plot_roc_curve(ax, train_y_true_batches, train_y_pred_batches, 'training', color=colors[0])
    ax = plot_roc_curve(ax, test_y_true_batches, test_y_pred_batches, 'testing', color=colors[0], linestyle='--')
    ax1 = ax
    ax.legend(loc='lower right')
    ax.figure.savefig(save_img_path + '/model_evaluation_roc_curve_plots_1_train_test.png')

    ax1 = plot_roc_curve(ax1, eval_y_true_batches, eval_y_pred_batches, 'evaluation', color=colors[1], linestyle='-.')
    ax1.legend(loc='lower right')
    ax1.figure.savefig(save_img_path + '/model_evaluation_roc_curve_plots_2_train_test_eval.png')

    plt.close('all')

def plot_roc_eval(test_dataset, eval_dataset, model, save_img_path):

    save_img_path = save_img_path + '/model_evaluation_roc_curve_plots_all'
    create_dir(save_img_path)

    colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

    def plot_roc_curve(ax, labels, predictions, title, **kwargs):

        fp, tp, _ = sklearn.metrics.roc_curve(labels, predictions)

        ax.plot(100*fp, 100*tp, label=title, linewidth=2, **kwargs)
        ax.set_xlabel('False positives [%]')
        ax.set_ylabel('True positives [%]')
        #ax.set_xlim([-0.5,20])
        #ax.set_ylim([80,100.5])
        ax.grid(True)
        ax.set_aspect('equal')

        return ax

    test_y_true_batches, test_y_pred_batches = obtain_ytrue_ypred(test_dataset, model, all_eval=True)
    eval_y_true_batches, eval_y_pred_batches = obtain_ytrue_ypred(eval_dataset, model, all_eval=True)

    plt.figure(figsize=(5,5))
    ax = plt.gca()
    ax = plot_roc_curve(ax, test_y_true_batches, test_y_pred_batches, 'testing', color=colors[0])
    ax = plot_roc_curve(ax, eval_y_true_batches, eval_y_pred_batches, 'evaluation', color=colors[1], linestyle='--')
    ax.legend(loc='lower right')
    ax.figure.savefig(save_img_path + '/model_evaluation_roc_curve_plots_test_and_eval.png')

    plt.close('all')


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/datasets/datasets.movielens.ipynb (unless otherwise specified).

__all__ = ['ML1mDataset', 'ML1mDataModule', 'ML1mDataset_v2', 'ML1mDataModule_v2', 'ML1mDataset_v3',
           'ML1mDataModule_v3', 'ML1mDataset_v4', 'ML100kDataset', 'sparseFeature', 'create_ml_1m_dataset',
           'create_implicit_ml_1m_dataset']

# Cell
from typing import Any, Iterable, List, Optional, Tuple, Union, Callable
import os
import json

import pandas as pd
import numpy as np

import torch

from ..utils.common_utils import *
from .bases.common import Dataset
from .bases.interactions import InteractionsDataset, InteractionsDataModule
from .bases.sequential import SequentialDataset, SequentialDataModule
from ..utils.splitting import stratified_split_v2

# Cell
class ML1mDataset(InteractionsDataset):
    url = "http://files.grouplens.org/datasets/movielens/ml-1m.zip"

    @property
    def raw_file_names(self):
        return 'ratings.dat'

    def download(self):
        path = download_url(self.url, self.raw_dir)
        extract_zip(path, self.raw_dir)
        from shutil import move, rmtree
        move(os.path.join(self.raw_dir, 'ml-1m', self.raw_file_names), self.raw_dir)
        rmtree(os.path.join(self.raw_dir, 'ml-1m'))
        os.unlink(path)

    def load_ratings_df(self):
        df = pd.read_csv(self.raw_paths[0], sep='::', header=None, engine='python')
        df.columns = ['uid', 'sid', 'rating', 'timestamp']
        # drop duplicate user-item pair records, keeping recent ratings only
        df.drop_duplicates(subset=['uid', 'sid'], keep='last', inplace=True)
        return df

# Cell
class ML1mDataModule(InteractionsDataModule):
    dataset_cls = ML1mDataset

# Cell
class ML1mDataset_v2(SequentialDataset):
    url = "http://files.grouplens.org/datasets/movielens/ml-1m.zip"

    @property
    def raw_file_names(self):
        return 'ratings.dat'

    def download(self):
        path = download_url(self.url, self.raw_dir)
        extract_zip(path, self.raw_dir)
        from shutil import move, rmtree
        move(os.path.join(self.raw_dir, 'ml-1m', self.raw_file_names), self.raw_dir)
        rmtree(os.path.join(self.raw_dir, 'ml-1m'))
        os.unlink(path)

    def load_ratings_df(self):
        df = pd.read_csv(self.raw_paths[0], sep='::', header=None, engine='python')
        df.columns = ['uid', 'sid', 'rating', 'timestamp']
        return df

# Cell
class ML1mDataModule_v2(SequentialDataModule):
    dataset_cls = ML1mDataset_v2

# Cell
class ML1mDataset_v3(SequentialDataset):
    url = "http://files.grouplens.org/datasets/movielens/ml-1m.zip"

    def __init__(self, data_dir, data_type='train', *args, **kwargs):
        super().__init__(data_dir, data_type, *args, **kwargs)
        if data_type == 'train':
            self.ratings_frame = pd.read_csv(self.processed_paths[0], delimiter=",")
        elif data_type == 'valid':
            self.ratings_frame = pd.read_csv(self.processed_paths[1], delimiter=",")
        elif data_type == 'test':
            self.ratings_frame = pd.read_csv(self.processed_paths[2], delimiter=",")

    @property
    def raw_file_names(self):
        return ['ratings.dat', 'movies.dat', 'users.dat']

    @property
    def processed_file_names(self):
        return ['train.csv', 'valid.csv', 'test.csv']

    def download(self):
        path = download_url(self.url, self.raw_dir)
        extract_zip(path, self.raw_dir)
        from shutil import move, rmtree
        for raw_file_name in self.raw_file_names:
            move(os.path.join(self.raw_dir, 'ml-1m', raw_file_name), self.raw_dir)
        rmtree(os.path.join(self.raw_dir, 'ml-1m'))
        os.unlink(path)

    def load_ratings_df(self):
        df = pd.read_csv(self.raw_paths[0], sep='::', header=None, engine='python')
        df.columns = ['uid', 'sid', 'rating', 'timestamp']
        return df

    def load_movies_df(self):
        df = pd.read_csv(self.raw_paths[1], sep='::', header=None, engine='python')
        df.columns = ["sid", "title", "genres"]
        return df

    def load_users_df(self):
        df = pd.read_csv(self.raw_paths[2], sep='::', header=None, engine='python')
        df.columns = ["uid", "sex", "age_group", "occupation", "zip_code"]
        return df

    def process(self):
        ## movies
        movies = self.load_movies_df()
        movies["year"] = movies["title"].apply(lambda x: x[-5:-1])
        movies.year = pd.Categorical(movies.year)
        movies["year"] = movies.year.cat.codes
        movies["sid"] = movies["sid"].astype(str)

        genres = ["Action","Adventure","Animation","Children's","Comedy","Crime",
                "Documentary","Drama","Fantasy","Film-Noir","Horror","Musical",
                "Mystery","Romance","Sci-Fi","Thriller","War","Western"]

        for genre in genres:
            movies[genre] = movies["genres"].apply(
                lambda values: int(genre in values.split("|"))
            )

        ## users
        users = self.load_users_df()
        users.sex = pd.Categorical(users.sex)
        users["sex"] = users.sex.cat.codes
        users.age_group = pd.Categorical(users.age_group)
        users["age_group"] = users.age_group.cat.codes
        users.occupation = pd.Categorical(users.occupation)
        users["occupation"] = users.occupation.cat.codes
        users.zip_code = pd.Categorical(users.zip_code)
        users["zip_code"] = users.zip_code.cat.codes
        users["uid"] = users["uid"].astype(str)

        # ratings
        ratings = self.load_ratings_df()
        ratings['timestamp'] = pd.to_datetime(ratings['timestamp'], unit='s')
        ratings["sid"] = ratings["sid"].astype(str)
        ratings["uid"] = ratings["uid"].astype(str)

        # Transform the movie ratings data into sequences
        # First, let's sort the the ratings data using the unix_timestamp,
        # and then group the movie_id values and the rating values by user_id.
        # The output DataFrame will have a record for each user_id, with two
        # ordered lists (sorted by rating datetime): the movies they have rated,
        # and their ratings of these movies.

        ratings_group = ratings.sort_values(by=["timestamp"]).groupby("uid")

        ratings_data = pd.DataFrame(
            data={
                "uid": list(ratings_group.groups.keys()),
                "sids": list(ratings_group.sid.apply(list)),
                "ratings": list(ratings_group.rating.apply(list)),
                "timestamps": list(ratings_group.timestamp.apply(list)),
            }
        )

        # Now, let's split the movie_ids list into a set of sequences of a fixed
        # length. We do the same for the ratings. Set the sequence_length variable
        # to change the length of the input sequence to the model. You can also
        # change the step_size to control the number of sequences to generate for
        # each user.
        ratings_data.sids = ratings_data.sids.apply(
            lambda ids: self.create_sequences(ids, self.history_size, self.step_size)
        )
        ratings_data.ratings = ratings_data.ratings.apply(
            lambda ids: self.create_sequences(ids, self.history_size, self.step_size)
        )
        del ratings_data["timestamps"]

        # After that, we process the output to have each sequence in a separate
        # records in the DataFrame. In addition, we join the user features with
        # the ratings data.
        ratings_data_movies = ratings_data[["uid", "sids"]].explode(
            "sids", ignore_index=True
        )
        ratings_data_rating = ratings_data[["ratings"]].explode("ratings", ignore_index=True)
        ratings_data_transformed = pd.concat([ratings_data_movies, ratings_data_rating], axis=1)
        ratings_data_transformed = ratings_data_transformed.join(
            users.set_index("uid"), on="uid"
        )
        ratings_data_transformed.sids = ratings_data_transformed.sids.apply(
            lambda x: ",".join(x)
        )
        ratings_data_transformed.ratings = ratings_data_transformed.ratings.apply(
            lambda x: ",".join([str(v) for v in x])
        )
        del ratings_data_transformed["zip_code"]
        ratings_data_transformed.rename(
            columns={"sids": "sequence_sids", "ratings": "sequence_ratings"},
            inplace=True,
        )
        # Finally, we split the data into training and testing splits, with 85%
        # and 15% of the instances, respectively, and store them to CSV files.
        random_selection = np.random.rand(len(ratings_data_transformed.index)) <= 0.85
        train_data = ratings_data_transformed[random_selection]
        test_data = ratings_data_transformed[~random_selection]

        # save
        train_data.to_csv(self.processed_paths[0], index=False, sep=",")
        test_data.to_csv(self.processed_paths[1], index=False, sep=",")
        test_data.to_csv(self.processed_paths[2], index=False, sep=",")

    def __len__(self):
        return len(self.ratings_frame)

    def __getitem__(self, idx):
        data = self.ratings_frame.iloc[idx]
        user_id = data.uid
        movie_history = eval(data.sequence_sids)
        movie_history_ratings = eval(data.sequence_ratings)
        target_movie_id = movie_history[-1:][0]
        target_movie_rating = movie_history_ratings[-1:][0]
        movie_history = torch.LongTensor(movie_history[:-1])
        movie_history_ratings = torch.LongTensor(movie_history_ratings[:-1])
        sex, age_group, occupation = data.sex, data.age_group, data.occupation
        output = (user_id, movie_history, target_movie_id, movie_history_ratings,
                  target_movie_rating, sex, age_group, occupation)
        return output

# Cell
class ML1mDataModule_v3(SequentialDataModule):
    dataset_cls = ML1mDataset_v3

# Cell
class ML1mDataset_v4(InteractionsDataset):
    url = "http://files.grouplens.org/datasets/movielens/ml-1m.zip"

    @property
    def raw_file_names(self):
        return 'ratings.dat'

    @property
    def processed_file_names(self):
        return 'data.json'

    def download(self):
        path = download_url(self.url, self.raw_dir)
        extract_zip(path, self.raw_dir)
        from shutil import move, rmtree
        move(os.path.join(self.raw_dir, 'ml-1m', self.raw_file_names), self.raw_dir)
        rmtree(os.path.join(self.raw_dir, 'ml-1m'))
        os.unlink(path)

    def load_ratings_df(self):
        df = pd.read_csv(self.raw_paths[0], sep='::', header=None, engine='python')
        df.columns = ['uid', 'sid', 'rating', 'timestamp']
        # drop duplicate user-item pair records, keeping recent ratings only
        df.drop_duplicates(subset=['uid', 'sid'], keep='last', inplace=True)
        return df

    def process(self):
        df = self.load_ratings_df()
        train, test = stratified_split_v2(df, ratio=0.7, col_user='uid', col_item='sid')

        min_session_length = 2
        session_len = test.groupby('uid').size()
        test = test[np.in1d(test['uid'], session_len[session_len >= min_session_length].index)]

        last_items = test.sort_values(by=['uid', 'timestamp']).groupby('uid').timestamp.idxmax()
        y_test = test.loc[last_items]
        x_test = test[~test.index.isin(y_test.index)]

        data = dict()

        def formatRecords(g):
            keys = ['uid','sid','rating']
            result = []
            for item in g.values.tolist():
                item = dict(zip(keys, item))
                result.append(item)
            return result

        data['x_train'] = list(train.groupby('uid').apply(lambda g: formatRecords(g)).to_dict().values())
        data['x_test'] = list(x_test.groupby('uid').apply(lambda g: formatRecords(g)).to_dict().values())
        data['y_test'] = list(y_test.groupby('uid').apply(lambda g: formatRecords(g)).to_dict().values())

        with open(self.processed_paths[0], "w") as f:
            json.dump(data, f)

    def load(self):
        with open(self.processed_paths[0]) as f:
            data = json.load(f)
        return data

# Cell
class ML100kDataset(Dataset):
    url = 'https://files.grouplens.org/datasets/movielens/ml-100k.zip'

    def __init__(self, root):
        super().__init__(root)

    @property
    def raw_file_names(self) -> str:
        return ['u1.base', 'u1.test', 'u4.test', 'allbut.pl', 'u.item',
                'ua.test', 'u.occupation', 'u3.test', 'u5.base', 'ub.test',
                'u2.test', 'u3.base', 'u.genre', 'u.data', 'u4.base',
                'u5.test', 'u.info', 'README', 'ub.base', 'mku.sh', 'u2.base',
                'u.user', 'ua.base']

    @property
    def processed_file_names(self) -> str:
        raise NotImplementedError

    def download(self):
        path = download_url(self.url, self.raw_dir)
        extract_zip(path, self.raw_dir)
        from shutil import move, rmtree
        file_names = os.listdir(osp.join(self.raw_dir, 'ml-100k'))
        for file_name in file_names:
            move(osp.join(self.raw_dir, 'ml-100k', file_name), self.raw_dir)
        rmtree(osp.join(self.raw_dir, 'ml-100k'))
        os.unlink(path)

    def process(self):
        raise NotImplementedError

# Cell
import pandas as pd
import numpy as np
import random
from tqdm import tqdm
from collections import defaultdict

# Cell
def sparseFeature(feat, feat_num, embed_dim=4):
    """
    create dictionary for sparse feature
    :param feat: feature name
    :param feat_num: the total number of sparse features that do not repeat
    :param embed_dim: embedding dimension
    :return:
    """
    return {'feat': feat, 'feat_num': feat_num, 'embed_dim': embed_dim}

# Cell
def create_ml_1m_dataset(file, trans_score=2, embed_dim=8, test_neg_num=100):
    """
    :param file: A string. dataset path.
    :param trans_score: A scalar. Greater than it is 1, and less than it is 0.
    :param embed_dim: A scalar. latent factor.
    :param test_neg_num: A scalar. The number of test negative samples
    :return: user_num, item_num, train_df, test_df
    """
    print('==========Data Preprocess Start=============')
    data_df = pd.read_csv(file, sep="::", engine='python',
                          names=['user_id', 'item_id', 'label', 'Timestamp'])
    # filtering
    data_df['item_count'] = data_df.groupby('item_id')['item_id'].transform('count')
    data_df = data_df[data_df.item_count >= 5]
    # trans score
    data_df = data_df[data_df.label >= trans_score]
    # sort
    data_df = data_df.sort_values(by=['user_id', 'Timestamp'])
    # split dataset and negative sampling
    print('============Negative Sampling===============')
    train_data, val_data, test_data = defaultdict(list), defaultdict(list), defaultdict(list)
    item_id_max = data_df['item_id'].max()
    for user_id, df in tqdm(data_df[['user_id', 'item_id']].groupby('user_id')):
        pos_list = df['item_id'].tolist()

        def gen_neg():
            neg = pos_list[0]
            while neg in set(pos_list):
                neg = random.randint(1, item_id_max)
            return neg

        neg_list = [gen_neg() for i in range(len(pos_list) + test_neg_num)]
        for i in range(1, len(pos_list)):
            hist_i = pos_list[:i]
            if i == len(pos_list) - 1:
                test_data['user_id'].append(user_id)
                test_data['pos_id'].append(pos_list[i])
                test_data['neg_id'].append(neg_list[i:])
            elif i == len(pos_list) - 2:
                val_data['user_id'].append(user_id)
                val_data['pos_id'].append(pos_list[i])
                val_data['neg_id'].append(neg_list[i])
            else:
                train_data['user_id'].append(user_id)
                train_data['pos_id'].append(pos_list[i])
                train_data['neg_id'].append(neg_list[i])
    # feature columns
    user_num, item_num = data_df['user_id'].max() + 1, data_df['item_id'].max() + 1
    feat_col = [sparseFeature('user_id', user_num, embed_dim),
                sparseFeature('item_id', item_num, embed_dim)]
    # shuffle
    random.shuffle(train_data)
    random.shuffle(val_data)
    train = [np.array(train_data['user_id']), np.array(train_data['pos_id']),
               np.array(train_data['neg_id'])]
    val = [np.array(val_data['user_id']), np.array(val_data['pos_id']),
             np.array(val_data['neg_id'])]
    test = [np.array(test_data['user_id']), np.array(test_data['pos_id']),
              np.array(test_data['neg_id'])]
    print('============Data Preprocess End=============')
    return feat_col, train, val, test

# Cell
def create_implicit_ml_1m_dataset(file, trans_score=2, embed_dim=8, maxlen=40):
    """
    :param file: A string. dataset path.
    :param trans_score: A scalar. Greater than it is 1, and less than it is 0.
    :param embed_dim: A scalar. latent factor.
    :param maxlen: A scalar. maxlen.
    :return: user_num, item_num, train_df, test_df
    """
    print('==========Data Preprocess Start=============')
    data_df = pd.read_csv(file, sep="::", engine='python',
                          names=['user_id', 'item_id', 'label', 'Timestamp'])
    # implicit dataset
    data_df = data_df[data_df.label >= trans_score]

    # sort
    data_df = data_df.sort_values(by=['user_id', 'Timestamp'])

    train_data, val_data, test_data = [], [], []

    item_id_max = data_df['item_id'].max()
    for user_id, df in tqdm(data_df[['user_id', 'item_id']].groupby('user_id')):
        pos_list = df['item_id'].tolist()

        def gen_neg():
            neg = pos_list[0]
            while neg in pos_list:
                neg = random.randint(1, item_id_max)
            return neg

        neg_list = [gen_neg() for i in range(len(pos_list) + 100)]
        for i in range(1, len(pos_list)):
            hist_i = pos_list[:i]
            if i == len(pos_list) - 1:
                test_data.append([user_id, hist_i, pos_list[i], 1])
                for neg in neg_list[i:]:
                    test_data.append([user_id, hist_i, neg, 0])
            elif i == len(pos_list) - 2:
                val_data.append([user_id, hist_i, pos_list[i], 1])
                val_data.append([user_id, hist_i, neg_list[i], 0])
            else:
                train_data.append([user_id, hist_i, pos_list[i], 1])
                train_data.append([user_id, hist_i, neg_list[i], 0])
    # item feature columns
    user_num, item_num = data_df['user_id'].max() + 1, data_df['item_id'].max() + 1
    feature_columns = [sparseFeature('user_id', user_num, embed_dim),
                       sparseFeature('item_id', item_num, embed_dim)]

    # shuffle
    random.shuffle(train_data)
    random.shuffle(val_data)
    # random.shuffle(test_data)

    # create dataframe
    train = pd.DataFrame(train_data, columns=['user_id', 'hist', 'target_item', 'label'])
    val = pd.DataFrame(val_data, columns=['user_id', 'hist', 'target_item', 'label'])
    test = pd.DataFrame(test_data, columns=['user_id', 'hist', 'target_item', 'label'])

    print('==================Padding===================')
    from tensorflow.keras.preprocessing.sequence import pad_sequences
    train_X = [train['user_id'].values, pad_sequences(train['hist'], maxlen=maxlen), train['target_item'].values]
    train_y = train['label'].values
    val_X = [val['user_id'].values, pad_sequences(val['hist'], maxlen=maxlen), val['target_item'].values]
    val_y = val['label'].values
    test_X = [test['user_id'].values, pad_sequences(test['hist'], maxlen=maxlen), test['target_item'].values]
    test_y = test['label'].values.tolist()
    print('============Data Preprocess End=============')
    return feature_columns, (train_X, train_y), (val_X, val_y), (test_X, test_y)

max_stars_repo_name: sparsh-ai/recohut

max_issues_repo_name: sparsh-ai/recohut

max_forks_repo_name: RecoHut-Projects/recohut


text:
import numpy as np
import pandas as pd
from bokeh import mpl

ts = pd.Series(np.random.randn(1000), index=pd.date_range('1/1/2000', periods=1000))
ts = ts.cumsum()
p = ts.plot()

mpl.to_bokeh(name="series")

max_stars_repo_name: tswicegood/bokeh