```python
import numpy as np
from sympy import Rational,symbols,exp,lambdify,sqrt,tanh,log,pi
from RadioArray import RadioArray
from ENUFrame import ENU
import astropy.coordinates as ac
import astropy.units as au
import astropy.time as at
from sympy.vector import CoordSysCartesian

def neQuick(h,No=2.2e11,hmax=368.,Ho=50.):
    '''Ne quick model for one layer'''
    res = np.zeros_like(h)
    g = 1./7.
    rfac = 100.
    dh = h - hmax
    g1 = g * dh
    z = dh / (Ho * (1. + rfac * g1 / (rfac * Ho + g1)))
    ee = np.exp(z)
    dial = ee / 1e7 - 1
    sig = 1./(1. + np.exp(-dial))
    res = No*4.0*ee/(1.0 + ee)**2
    #res[ee>1e7] = No*4.0/ee[ee>1e7]
    #res[ee<=1e7] = No*4.0*ee[ee<=1e7]/(1.0 + ee[ee<=1e7])**2
    #res[z > 40] = 0
    return res

def xef1f2(h):
    f2 = neQuick(h,No=2.2e11,hmax=359.,Ho=46.)
    f1 = neQuick(h,No=3e9,hmax=195.,Ho=20.)
    e = neQuick(h,No=3e9,hmax=90.,Ho=10.)
    return f2 + f1 + e

def symbolicChapman(h, nepeak,hmax,width):
    y = (h-hmax)/width
    return nepeak*exp(Rational(1,2)*(Rational(1) - y - exp(-y)))

def symbolicChapman_def1f2(h = None,cond=0):
    if h is None:
        h = symbols('h')
    if cond == 0:
        p = 1
        w = 1
    if cond == 1:
        p = 5
        w = 1.5
    if cond == 2:
        p = 10
        w = 2.
    #f = symbolicChapman(h,Rational(1e10),Rational(450),Rational(250))
    f2 = symbolicChapman(h,Rational(p*2.2e11),Rational(359),Rational(w*46))
    f1 = symbolicChapman(h,Rational(3e9),Rational(195),Rational(20))
    e = symbolicChapman(h,Rational(3e9),Rational(90),Rational(10))
    #d = symbolicChapman(h,Rational(3e9),Rational(80),Rational(50))
    return f2 + f1 + e  + Rational(4.5e8)

def ExampleIRI():
    d = np.genfromtxt('exampleIRI.txt',names=True)
    profile = d['ne']
    return d['height'],d['ne']

def plotModels():
    '''Plot various models'''
    import pylab as plt
    h = np.linspace(0,3000,10000)
    ### from IRI
    h1,ne1 = ExampleIRI()
    plt.plot(h1,ne1,c='black',label='Ex. IRI')
    #neQuick layers
    ne3 = xef1f2(h1)
    plt.plot(h1,ne3,c='blue',label='neQuick_ef1f2')
    lam = lambdify(symbols('h'),symbolicChapman_def1f2(),'numpy')
    ne4 = lam(h)
    plt.plot(h,ne4,c='green',label='Chapman_def1f2')
    plt.legend(frameon=False)
    plt.xlabel('Height above surface (km)')
    plt.ylabel(r'Electron density $n_e$ (${\rm m}^{-3}$)')
    #plt.yscale('log')
    plt.grid()
    plt.title('Ionosphere Models')
    plt.show()

class Model(object):
    '''Symbolic model object. '''
    def makeOrderList(self,paramDict):
        orderList = []
        for key in paramDict.keys():
            orderList.append(key)
        return orderList
    def makeParamDict(self,paramVec,orderList):
        paramDict = {}
        N = np.size(paramVec)
        i = 0
        while i < N:
            paramDict[orderList[i]] = paramVec[i]
            i += 1
        return paramDict
    def makeParamVec(self,paramDict,orderList):
        N = len(orderList)
        paramVec = np.zeros(N)
        i = 0
        while i < N:
            paramVec[i] = paramDict[orderList[i]]
            i += 1
        return paramVec

class ElectronContentModel(Model):
    def __init__(self,radioArray = None,**kwargs):
        super(ElectronContentModel,self).__init__(**kwargs)
        if radioArray is None:
            radioArray = RadioArray(arrayFile='arrays/lofar.hba.antenna.cfg')
        self.radioArray = radioArray
        self.enu = ENU(location=radioArray.getCenter().earth_location)
    def saveNeFunc(self,neFunc):
        f = tempfile.SpooledTemporaryFile()
        np.savez(f,neFunc=neFunc)
        f.flush()
        return f

class IriModel(ElectronContentModel):
    def __init__(self,**kwargs):
        '''The ionosphere is modelled as the IRI plus a set of solitons
        with time coherence imposed via linear coherence over short time intervals'''
        super(IriModel,self).__init__(**kwargs)
        self.initIriParams = self.createInitIriParams()
        self.iriParamDict = self.initIriParams.copy()
        self.iriOrder = self.makeOrderList(self.initIriParams)
        self.numIriParams = len(self.iriOrder)
        self.iriFunc = self.makeSymbolicIri()
        print("Generated IRI symbolic function with {0} params".format(self.numIriParams))

    def setIriParams(self,paramVec):
        '''Set the paramDict for iri from vector'''
        self.iriParamDict = self.makeParamDict(paramVec,self.iriOrder)

    def getIriParams(self):
        return self.makeParamVec(self.iriParamDict,self.iriOrder)

    def createInitIriParams(self):
        '''Create an initial random param for a soliton'''
        init = {'fhm' : np.random.uniform(low = 400, high = 600),
                'fw' : np.random.uniform(low = 200,high = 300),
                'nefm' : 10**np.random.uniform(low = 9,high=11),
                'f2hm' : np.random.uniform(low = 300, high = 400),
                'f2w' : np.random.uniform(low = 150,high = 200),
                'nef2m' : 10**np.random.uniform(low = 11,high=12),
                'f1hm' : np.random.uniform(low = 100, high = 140),
                'f1w' : np.random.uniform(low = 60,high = 100),
                'nef1m' : 10**np.random.uniform(low = 9,high=11),
                'ehm' : np.random.uniform(low = 80, high = 120),
                'ew' : np.random.uniform(low = 50,high = 80),
                'neem' : 10**np.random.uniform(low = 9,high=10),
                'dhm' : np.random.uniform(low = 50, high = 80),
                'dw' : np.random.uniform(low = 50,high = 60),
                'nedm' : 10**np.random.uniform(low = 8,high=10)}
        init = {'f2hm' : 360,
                'f2w' : 46,
                'nef2m' : 2.2e11,
                'f1hm' : 195,
                'f1w' : 20,
                'nef1m' : 3e9,
                'ehm' : 90,
                'ew' : 10,
                'neem' : 3e9}
        return init

    def makeSymbolicIri(self):
        x,y,z = symbols('x y z')
        #R = Rational(6371)
        R = Rational(int(self.radioArray.getCenter().spherical.distance.to(au.km).value))
        r = sqrt(x**(Rational(2))+y**(Rational(2))+z**(Rational(2)))
        h = r - R
        #f = symbolicChapman(h,*symbols('nefm fhm fw'))
        f2 = symbolicChapman(h,*symbols('nef2m f2hm f2w'))
        f1 = symbolicChapman(h,*symbols('nef1m f1hm f1w'))
        e = symbolicChapman(h,*symbols('neem ehm ew'))
        #d = symbolicChapman(h,*symbols('nedm dhm dw'))
        self.iriFunc = f2 + f1 + e + Rational(4.5e8)
        return self.iriFunc

    def generateIri(self,paramVec=None):
        '''Sustitute paramDict into symbolic function'''
        if paramVec is not None:
            self.iriParamDict = self.setIriParams(paramVec)
        self.iriModel = self.iriFunc.subs(self.iriParamDict)
        return self.iriModel

    def evaluate(self,X,Y,Z):
        iri = lambdify(symbols('x y z'),self.generateIri(),'numpy')
        ne = iri(X.flatten(),Y.flatten(),Z.flatten()).reshape(X.shape)
        return ne

    def plotModel(self):
        import pylab as plt
        iri = lambdify(symbols('x y z'),self.generateIri(),'numpy')
        h = np.linspace(6300,9000,1000)
        plt.plot(h,iri(h,0,0))
        plt.xlabel('geocentric radius (km)')
        plt.ylabel('ne m^-3')
        plt.yscale('log')
        plt.grid()
        plt.show()

def IntegrateChapman(Npeak,hmax,width):
    y0 = -hmax/width
    ymax = 10#(h+3*w-h)/w
    from scipy.special import erf
    return -Npeak*np.sqrt(2*np.exp(1)*np.pi)*(erf(np.exp(-ymax/2)/np.sqrt(2)) - erf(np.exp(-y0/2)/np.sqrt(2)))

def IntegrateChapmanNumerical(Npeak,hmax,width,var=1e10):
    h = np.linspace(0,3000,10000)
    lam = lambdify(symbols('h'),symbolicChapman(symbols('h'), Npeak,hmax,width),'numpy')
    ne = lam(h)
    #import pylab as plt
    #plt.plot(h,ne)
    #plt.show()
    from scipy.integrate import simps
    res = []
    for i in range(100):
        res.append(simps(ne+np.random.normal(loc=0,scale=var,size=h.size),h))
    #import pylab as plt
    #plt.hist(res)
    #plt.show()
    return np.mean(res)

def R2upperbound(frequency=100e6,cond=0):
    n_p = 1.24e-2 * frequency**2
    h = np.linspace(0,3000,10000)
    lam = lambdify(symbols('h'),symbolicChapman_def1f2(h = symbols('h'),cond=cond),'numpy')
    if cond == 0:
        var = 1e10
    if cond == 1:
        var = 1e11
    if cond == 2:
        var = 1e12
    ne = lam(h)
    #import pylab as plt
    #plt.plot(h,ne)
    #plt.show()
    from scipy.integrate import simps
    res = []
    alpha = (ne + var)/n_p
    num = alpha**3
    integrand = num/((1-alpha)**(5./2.))
    print("freq:",frequency,"cond:",cond,"alphahat:{0:1.1e}".format(np.mean(alpha)),"sigma_alpha^2:{0:1.1e}".format(np.mean(alpha**2)-np.mean(alpha)**2))
    return "{0:1.1e}".format(simps(integrand,h)*n_p/8./1e16)

def aPrioriModel(h,zenith):
    '''Return a stratified reference ionosphere based on fitted data depending on zenith angle of the sun in degrees'''
    def peakDensity(n0,dn,tau,b,zenith):
        y = zenith/tau
        return n0 + dn*np.exp(-y**2)/(1. + y**(2*b))
    def peakHeight(z0,dz,rho,chi0,zenith):
        return z0 + dz/(1.+np.exp(-(zenith - chi0)/rho))
    def layerDensity(nm,zm,H,z):
        y = (z - zm)/H
        return nm*np.exp(1./2. * (1. - y - np.exp(-y)))

    #D layer
    #nm_d = peakDensity(4e8,5.9e8,58.,1300.,zenith)
    y = zenith/58.
    if y < 1:
        nm_d = 4e8 + 5.9e8*np.exp(-y**2)
    else:
        nm_d = 4e8
    zm_d = peakHeight(81.,7.,7.46,100.,zenith)
    H_d = 8.
    n_d = layerDensity(nm_d,zm_d,H_d,h)
    #E layer
    nm_e = peakDensity(1.6e9,1.6e11,87.,8.7,zenith)
    zm_e = 110.
    H_e = 11.
    n_e = layerDensity(nm_e,zm_e,H_e,h)
    #F1 layer
    nm_f1 = peakDensity(2.0e11,9.1e10,54.,13.6,zenith)
    zm_f1 = 185.
    H_f1 = 40.
    n_f1 = layerDensity(nm_f1,zm_f1,H_f1,h)
    #F2 layer
    nm_f2 = peakDensity(7.7e10,4.4e11,111.,4.8,zenith)
    zm_f2 = peakHeight(242.,75.,7.46,96.,zenith)
    H_f2 = 55.
    n_f2 = layerDensity(nm_f2,zm_f2,H_f2,h)

    n = np.atleast_1d(n_d + n_e + n_f1 + n_f2)
    return n

def fitExampleIonospheres(folder):
    import glob
    files = glob.glob("{0}/*".format(folder))

    # 0 - DOY
    # 1 - Hour
    # 2 - Solar_zenith_angle, degree
    # 3 - Height, km
    # 4 - Electron_density_Ne, m-3
    # 5 - hmF2, km
    # 6 - hmF1, km
    # 7 - hmE, km
    # 8 - hmD, km
    # 9 - NmF2, m-3
    # 10 - NmF1, m-3
    # 11 - NmE, m-3
    # 12 - NmD, m-3
    # 13 - E_valley_width, km
    d = []
    for f in files:
        d.append(np.genfromtxt(f))
    d = np.vstack(d)

    mask = d[:,10] == d[:,10]
    doy = d[mask,0]
    hr = d[mask,1]
    chi = d[mask,2]
    nf2 = d[mask,9]
    nf1 = d[mask,10]
    ne = d[mask,11]
    nd = d[mask,12]
    hf2 = d[mask,5]
    hf1 = d[mask,6]
    he = d[mask,7]
    hd = d[mask,8]
    Z = d[mask,3]
    N = d[mask,4]

    from scipy.optimize import curve_fit
    import pylab as plt

    def gauss(x, *p):
        hr = x[0,:]
        doy = x[1,:]
        chi = x[2,:]
        A, mu, sigma, C = p
        return A*np.exp(-(hr-mu)**2/(2.*sigma**2)) + C

    def peakDensity(chi,A,C,tau,b):
        return np.abs(C) + (np.abs(A)) *np.exp(-(chi/np.abs(tau))**2)/(1 +  (chi/tau)**(2*np.abs(b)))

    def peakDensityFit(chi,*p):
        A,C,tau,b = p
        return peakDensity(chi,A,C,tau,b)

    def layerDensity(z,nm,zm,H):
        y = (z-zm)/np.abs(H)
        return np.abs(nm) * np.exp(0.5*(1-y-np.exp(-y)))

    def peakHeightFit(pred,*p):
        #doy = pred[0,:]
        #hr = pred[1,:]
        chi = pred[:]
        C,D,ACHI,BCHI = p
        return C + (D)/(1 + np.exp(-(chi-ACHI)/BCHI)) #np.exp((doy - ADOY)**2/(2*BDOY**2)) * np.arctan(BCHI*(chi-ACHI)) * (AHR + BHR*hr)

    p0=(1e8,1e8,45,4)
    coeffnd, var_matrix = curve_fit(peakDensityFit, chi, nd, p0=p0)
    print("nd:",coeffnd)
    plt.scatter(chi,nd,c='blue')
    plt.scatter(chi,peakDensityFit(chi,*coeffnd),c='red')
    plt.show()
    p0=(1e9,1e9,45,4)
    coeffne, var_matrix = curve_fit(peakDensityFit, chi, ne, p0=p0)
    print("ne:",coeffne)
    plt.scatter(chi,ne,c='blue')
    plt.scatter(chi,peakDensityFit(chi,*coeffne),c='red')
    plt.show()
    p0=(1e12,1e11,45,4)
    coeffnf2, var_matrix = curve_fit(peakDensityFit, chi, nf2, p0=p0)
    print("nf2:",coeffnf2)
    plt.scatter(chi,nf2,c='blue')
    plt.scatter(chi,peakDensityFit(chi,*coeffnf2),c='red')
    plt.show()
    p0=(1e12,1e11,45,4)
    mask = hf1 != -1
    coeffnf1, var_matrix = curve_fit(peakDensityFit, chi[mask], nf1[mask], p0=p0)
    print("nf1:",coeffnf1)
    plt.scatter(chi[mask],nf1[mask],c='blue')
    plt.scatter(chi,peakDensityFit(chi,*coeffnf1),c='red')
    plt.show()

    pred = chi
    p0 = (np.min(hd),np.max(hd),100,1)
    plt.scatter(chi,hd)
    plt.scatter(chi,peakHeightFit(pred,*p0),color='red')
    plt.show()

    coeffhd, var_matrix = curve_fit(peakHeightFit, pred, hd, p0=p0)
    print(coeffhd)
    plt.scatter(doy,hd)
    plt.scatter(doy,peakHeightFit(pred,*coeffhd),color='red')
    plt.show()

    p0 = (np.min(he),np.max(he),100,1)
    coeffhe, var_matrix = curve_fit(peakHeightFit, pred, he, p0=p0)
    print(coeffhe)
    plt.scatter(doy,he)
    plt.scatter(doy,peakHeightFit(pred,*coeffhe),color='red')
    plt.show()

    p0 = (np.min(hf2),np.max(hf1),100,1)
    coeffhf2, var_matrix = curve_fit(peakHeightFit, pred, hf2, p0=p0)
    print(coeffhf2)
    plt.scatter(doy,hf2)
    plt.scatter(doy,peakHeightFit(pred,*coeffhf2),color='red')
    plt.show()

    mask = hf1 != -1
    pred = chi[mask]
    p0 = (np.min(hf1[mask]),np.max(hf1[mask]),100,1)
    coeffhf1, var_matrix = curve_fit(peakHeightFit, pred, hf1[mask], p0=p0)
    print(coeffhf1)
    plt.scatter(doy[mask],hf1[mask])
    plt.scatter(doy[mask],peakHeightFit(pred,*coeffhf1),color='red')
    plt.show()

    def DEFDensity(pred,*p):
        chi = pred[0,:]
        z = pred[1,:]
        Hd,He,Hf2,Hf1 = p
        nmd = peakDensityFit(chi,*coeffnd)
        nme = peakDensityFit(chi,*coeffne)
        nmf2 = peakDensityFit(chi,*coeffnf2)
        nmf1 = peakDensityFit(chi,*coeffnf1)
        zmd = peakHeightFit(chi,*coeffhd)
        zme = peakHeightFit(chi,*coeffhe)
        zmf2 = peakHeightFit(chi,*coeffhf2)
        zmf1 = peakHeightFit(chi,*coeffhf1)

        return layerDensity(z,nmd,zmd,Hd)+layerDensity(z,nme,zme,He)+layerDensity(z,nmf2,zmf2,Hf2)+layerDensity(z,nmf1,zmf1,Hf1)

    pred = np.vstack([chi,Z])
    # p0 is the initial guess for the fitting coefficients (A, mu and sigma above)
    #Hd,He,Hf,Ad,Cd,taud,bd,Ae,Ce,taue,be,Af,Cf,tauf,bf
    p0 = [  8,11,55,40]

    coeffH, var_matrix = curve_fit(DEFDensity, pred, N, p0=p0)
    print("H:",coeffH)
    plt.scatter(chi,N)
    plt.scatter(chi,DEFDensity(pred,*coeffH),color='red')
    plt.show()
    plt.scatter(Z,N)
    plt.scatter(Z,DEFDensity(pred,*coeffH),color='red')
    plt.show()

if __name__=='__main__':
    h = np.linspace(0,2000,1000)
    import pylab as plt
    plt.plot(h,aPrioriModel(h,0))
    plt.plot(h,aPrioriModel(h,20))
    plt.plot(h,aPrioriModel(h,45))
    plt.plot(h,aPrioriModel(h,90))
    plt.show()
    #fitExampleIonospheres("IriData")
    #print( IntegrateChapman(2.2e12,350,50))
    #print( IntegrateChapmanNumerical(2.2e12,350,50))
    #print(R2upperbound(frequency=25e6,cond=0))
    #print( IntegrateChapman(2.2e12,350,100))

    #plotModels()
    #from ENUFrame import *
    #iri = IriModel()
    #print(iri.iriFunc)
    #print (iri.generateIri())
    #xvec = [0]
    #yvec = [0]
    #zvec = np.linspace(0,3000,10000)
    #X,Y,Z = np.meshgrid(xvec,yvec,zvec,indexing='ij')
    #points = ac.SkyCoord(X.flatten()*au.km,Y.flatten()*au.km,Z.flatten()*au.km,frame=iri.enu).transform_to('itrs').cartesian.xyz.to(au.km).value

    #for x,y,z in zip(X.flatten(),Y.flatten(),Z.flatten()):
    #    points.append(ac.SkyCoord(x*au.km,y*au.km,y*au.km,frame=iri.enu).transform_to('itrs').cartesian.xyz.to(au.km).value)
    #points = np.array(points)
    #xvec = np.linspace(np.min(points[0,:]),np.max(points[0,:]),len(xvec))
    #yvec = np.linspace(np.min(points[1,:]),np.max(points[1,:]),len(yvec))
    #zvec = np.linspace(np.min(points[2,:]),np.max(points[2,:]),len(zvec))
    #X,Y,Z = np.meshgrid(xvec,yvec,zvec)
    #X = points[0,:].reshape(X.shape)
    #Y = points[1,:].reshape(Y.shape)
    #Z = points[2,:].reshape(Z.shape)
    #X,Y,Z=np.meshgrid(xvec,yvec,zvec,indexing='ij')
    #ne = iri.evaluate(points[0,:],points[1,:],points[2,:])
    #print (ne)
    #import pylab as plt
    #plt.plot(np.linspace(0,3000,10000),ne)
    #plt.show()

```


# CF Part 6 - Non-negative Matrix factorization method
> Collaborative Filtering on MovieLens Latest-small Part 6 - Finding recommendations using model based non-negative matrix factorization method

- toc: true
- badges: true
- comments: true
- categories: [movie, collaborative]
- image:

Jusl like Matrix Factorization (MF) [(Yehuda Koren et al., 2009)](https://ieeexplore.ieee.org/document/5197422), Non-negative Matrix Factorization (NMF in short) factors the rating matrix $R$ in two matrices in such a way that $R = PQ^{\top}$.

### One limitation of Matrix Factorization

$P$ and $Q$ values in MF are non interpretable since their components can take arbitrary (positive and negative) values.

### Particulariy of Non-negative Matrix Factorization

NMF [(Lee and Seung, 1999)](http://www.dm.unibo.it/~simoncin/nmfconverge.pdf) allows the reconstruction of $P$ and $Q$ in such a way that $P,Q \ge 0$. Constraining $P$ and $Q$ values to be taken from $[0,1]$ allows  a probabilistic interpretation

- Latent factors represent groups of users who share the same tastes,
- The value  $P_{u,l}$  represents the probability that user $u$ belongs to the group $l$ of users and
- The value $Q_{l,i}$ represents the probability that users in the group $l$  likes item $i$.

### Objective function

With the Euclidian distance, the NMF objective function is defined by

\begin{equation}
J = \frac{1}{2}\sum_{(u,i) \in \kappa}||R_{u,i} - P_uQ_i^{\top}||^2 + \lambda_P||P_u||^2 + \lambda_Q||Q_i||^2
\end{equation}

The goal is to minimize the cost function $J$ by optimizing parameters $P$ and $Q$, with $\lambda_P$ and $\lambda_Q$ the regularizer parameters.

### Multiplicative update rule

According [(Lee and Seung, 1999)](https://www.nature.com/articles/44565.pdf), to the multiplicative update rule for $P$ and $Q$ are as follows :

\begin{equation}
P \leftarrow P \cdot \frac{RQ}{PQ^{\top}Q}
\end{equation}

\begin{equation}
Q \leftarrow Q \cdot \frac{R^{\top}P}{QP^{\top}P}
\end{equation}

However, since $R$ is a sparse matrix, we need to update each $P_u$ according to existing ratings of user $u$. Similarily, we need to update $Q_i$ according to existing ratings on item $i$. Hence :

\begin{equation}
P_{u,k} \leftarrow P_{u,k} \cdot \frac{\sum_{i \in I_u}Q_{i,k}\cdot r_{u,i}}{\sum_{i \in I_u}Q_{i,k}\cdot \hat{r}_{u,i} + \lambda_P|I_u|P_{u,k}}
\end{equation}

\begin{equation}
Q_{i,k} \leftarrow Q_{i,k} \cdot \frac{\sum_{u \in U_i}P_{u,k}\cdot r_{u,i}}{\sum_{u \in U_i}P_{u,k}\cdot \hat{r}_{u,i} + \lambda_Q|U_i|Q_{i,k}}
\end{equation}

Where
- $P_{u,k}$ is the $k^{th}$ latent factor of $P_u$
- $Q_{i,k}$ is the $k^{th}$ latent factor of $Q_i$
- $I_u$ the of items rated by user $u$
- $U_i$ the set of users who rated item $i$

```python
import os

if not (os.path.exists("recsys.zip") or os.path.exists("recsys")):
    !wget https://github.com/nzhinusoftcm/review-on-collaborative-filtering/raw/master/recsys.zip
    !unzip recsys.zip
```

### Install and import useful packages

### requirements

```
matplotlib==3.2.2
numpy==1.19.2
pandas==1.0.5
python==3.7
scikit-learn==0.24.1
scikit-surprise==1.1.1
scipy==1.6.2
```

```python
from recsys.preprocessing import mean_ratings
from recsys.preprocessing import normalized_ratings
from recsys.preprocessing import ids_encoder
from recsys.preprocessing import train_test_split
from recsys.preprocessing import rating_matrix
from recsys.preprocessing import get_examples
from recsys.preprocessing import scale_ratings

from recsys.datasets import ml1m
from recsys.datasets import ml100k

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

import os
```

### Load and preprocess rating

```python
# load data
ratings, movies = ml100k.load()

# prepare data
ratings, uencoder, iencoder = ids_encoder(ratings)

# convert ratings from dataframe to numpy array
np_ratings = ratings.to_numpy()

# get examples as tuples of userids and itemids and labels from normalize ratings
raw_examples, raw_labels = get_examples(ratings, labels_column="rating")

# train test split
(x_train, x_test), (y_train, y_test) = train_test_split(examples=raw_examples, labels=raw_labels)
```

### Non-negative Matrix Factorization Model

```python
class NMF:

    def __init__(self, ratings, m, n, uencoder, iencoder, K=10, lambda_P=0.01, lambda_Q=0.01):

        np.random.seed(32)

        # initialize the latent factor matrices P and Q (of shapes (m,k) and (n,k) respectively) that will be learnt
        self.ratings = ratings
        self.np_ratings = ratings.to_numpy()
        self.K = K
        self.P = np.random.rand(m, K)
        self.Q = np.random.rand(n, K)

        # hyper parameter initialization
        self.lambda_P = lambda_P
        self.lambda_Q = lambda_Q

        # initialize encoders
        self.uencoder = uencoder
        self.iencoder = iencoder

        # training history
        self.history = {
            "epochs": [],
            "loss": [],
            "val_loss": [],
        }

    def print_training_parameters(self):
        print('Training NMF ...')
        print(f'k={self.K}')

    def mae(self, x_train, y_train):
        """
        returns the Mean Absolute Error
        """
        # number of training examples
        m = x_train.shape[0]
        error = 0
        for pair, r in zip(x_train, y_train):
            u, i = pair
            error += abs(r - np.dot(self.P[u], self.Q[i]))
        return error / m

    def update_rule(self, u, i, error):
        I = self.np_ratings[self.np_ratings[:, 0] == u][:, [1, 2]]
        U = self.np_ratings[self.np_ratings[:, 1] == i][:, [0, 2]]

        num = self.P[u] * np.dot(self.Q[I[:, 0]].T, I[:, 1])
        dem = np.dot(self.Q[I[:, 0]].T, np.dot(self.P[u], self.Q[I[:, 0]].T)) + self.lambda_P * len(I) * self.P[u]
        self.P[u] = num / dem

        num = self.Q[i] * np.dot(self.P[U[:, 0]].T, U[:, 1])
        dem = np.dot(self.P[U[:, 0]].T, np.dot(self.P[U[:, 0]], self.Q[i].T)) + self.lambda_Q * len(U) * self.Q[i]
        self.Q[i] = num / dem

    @staticmethod
    def print_training_progress(epoch, epochs, error, val_error, steps=5):
        if epoch == 1 or epoch % steps == 0:
            print(f"epoch {epoch}/{epochs} - loss : {round(error, 3)} - val_loss : {round(val_error, 3)}")

    def fit(self, x_train, y_train, validation_data, epochs=10):

        self.print_training_parameters()
        x_test, y_test = validation_data
        for epoch in range(1, epochs+1):
            for pair, r in zip(x_train, y_train):
                u, i = pair
                r_hat = np.dot(self.P[u], self.Q[i])
                e = abs(r - r_hat)
                self.update_rule(u, i, e)
            # training and validation error  after this epochs
            error = self.mae(x_train, y_train)
            val_error = self.mae(x_test, y_test)
            self.update_history(epoch, error, val_error)
            self.print_training_progress(epoch, epochs, error, val_error, steps=1)

        return self.history

    def update_history(self, epoch, error, val_error):
        self.history['epochs'].append(epoch)
        self.history['loss'].append(error)
        self.history['val_loss'].append(val_error)

    def evaluate(self, x_test, y_test):
        error = self.mae(x_test, y_test)
        print(f"validation error : {round(error,3)}")
        print('MAE : ', error)
        return error

    def predict(self, userid, itemid):
        u = self.uencoder.transform([userid])[0]
        i = self.iencoder.transform([itemid])[0]
        r = np.dot(self.P[u], self.Q[i])
        return r

    def recommend(self, userid, N=30):
        # encode the userid
        u = self.uencoder.transform([userid])[0]

        # predictions for users userid on all product
        predictions = np.dot(self.P[u], self.Q.T)

        # get the indices of the top N predictions
        top_idx = np.flip(np.argsort(predictions))[:N]

        # decode indices to get their corresponding itemids
        top_items = self.iencoder.inverse_transform(top_idx)

        # take corresponding predictions for top N indices
        preds = predictions[top_idx]

        return top_items, preds

```

### Train the NMF model with ML-100K dataset

model parameters :

- $k = 10$ : (number of factors)
- $\lambda_P = 0.6$
- $\lambda_Q = 0.6$
- epochs = 10

Note that it may take some time to complete the training on 10 epochs (around 7 minutes).

```python
m = ratings['userid'].nunique()   # total number of users
n = ratings['itemid'].nunique()   # total number of items

# create and train the model
nmf = NMF(ratings, m, n, uencoder, iencoder, K=10, lambda_P=0.6, lambda_Q=0.6)
history = nmf.fit(x_train, y_train, epochs=10, validation_data=(x_test, y_test))
```

    Training NMF ...
    k=10
    epoch 1/10 - loss : 0.916 - val_loss : 0.917
    epoch 2/10 - loss : 0.915 - val_loss : 0.917
    epoch 3/10 - loss : 0.915 - val_loss : 0.917
    epoch 4/10 - loss : 0.915 - val_loss : 0.917
    epoch 5/10 - loss : 0.915 - val_loss : 0.917
    epoch 6/10 - loss : 0.915 - val_loss : 0.917
    epoch 7/10 - loss : 0.915 - val_loss : 0.917
    epoch 8/10 - loss : 0.915 - val_loss : 0.917
    epoch 9/10 - loss : 0.915 - val_loss : 0.917
    epoch 10/10 - loss : 0.915 - val_loss : 0.917

```python
nmf.evaluate(x_test, y_test)
```

    validation error : 0.917
    MAE :  0.916504134301954

    0.916504134301954

## Evaluation of NMF with Scikit-suprise

We can use the [scikt-suprise](https://surprise.readthedocs.io/en/stable/) package to train the NMF model. It  is an easy-to-use Python scikit for recommender systems.

1. Import the NMF class from the suprise scikit.
2. Load the data with the built-in function
3. Instanciate NMF with ```k=10``` (```n_factors```) and we use 10 epochs (```n_epochs```)
4. Evaluate the model using cross-validation with 5 folds.

```python
from surprise import NMF
from surprise import Dataset
from surprise.model_selection import cross_validate

# Load the movielens-100k dataset (download it if needed).
data = Dataset.load_builtin('ml-100k')

# Use the NMF algorithm.
nmf = NMF(n_factors=10, n_epochs=10)

# Run 5-fold cross-validation and print results.
history = cross_validate(nmf, data, measures=['MAE'], cv=5, verbose=True)
```

    Evaluating MAE of algorithm NMF on 5 split(s).

                      Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std
    MAE (testset)     0.9626  0.9435  0.9777  0.9607  0.9510  0.9591  0.0116
    Fit time          0.63    0.69    0.74    0.65    0.71    0.68    0.04
    Test time         0.15    0.11    0.14    0.10    0.15    0.13    0.02

As result, the mean MAE on the test set is **mae = 0.9510** which is equivalent to the result we have obtained on *ml-100k* with our own implementation **mae = 0.9165**

#### ML-1M

The mean MAE on a 5-fold cross-validation is **mae = 0.9567**

This may take arount 2 minutes

```python
data = Dataset.load_builtin('ml-1m')
nmf = NMF(n_factors=10, n_epochs=10)
history = cross_validate(nmf, data, measures=['MAE'], cv=5, verbose=True)
```

    Evaluating MAE of algorithm NMF on 5 split(s).

                      Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std
    MAE (testset)     0.9559  0.9580  0.9566  0.9639  0.9490  0.9567  0.0047
    Fit time          6.37    6.55    6.49    6.45    6.51    6.47    0.06
    Test time         1.68    1.31    1.64    1.46    1.75    1.57    0.16

## Explainable Matrix Factorization

NMF introduced explainability to MF by constraining $P$ and $Q$ values in $[0,1]$. It can be considered is an inner explainable model. It's possible to inject external informations into the model in order to bring explainability. This is what the **Explainable Matrix Factorization (EMF)** algorithm does. It computes explainable scores from user or item similarities taken from the user-based or item-based CF.

Click [here](https://github.com/nzhinusoftcm/review-on-collaborative-filtering/blob/master/7.Explainable_Matrix_Factorization.ipynb) to open the EMF notebook.

## Reference

1. Daniel D. Lee & H. Sebastian Seung (1999). [Learning the parts of objects by non-negative matrix factorization](https://www.nature.com/articles/44565)
2. Deng Cai et al. (2008). [Non-negative Matrix Factorization on Manifold](https://ieeexplore.ieee.org/document/4781101)
3. Yu-Xiong Wang and Yu-Jin Zhang (2011). [Non-negative Matrix Factorization: a Comprehensive Review](https://ieeexplore.ieee.org/document/6165290)
4. Nicolas Gillis (2014). [The Why and How of Nonnegative Matrix Factorization](https://arxiv.org/pdf/1401.5226.pdf)

## Author

[Carmel WENGA](https://www.linkedin.com/in/carmel-wenga-871876178/),

PhD student at Université de la Polynésie Française,

Applied Machine Learning Research Engineer,

[ShoppingList](https://shoppinglist.cm), NzhinuSoft.


# Local approximation of a bivariate polynomial expression using linear regression

Given two features, extract y (the label), perform linear regression.
Coming from the single feature model, 2 features is adding some complexity not only on the maths, but also on the display through charts.

### Learning goals:
- Analiticaly solve the linear regression and implement it with simple "raw" Python
- Use powers of the features to increase the model capacity
- Use some off the shelve methods of Numpy
- Switch to iterative methods with gradient descent (GD) and stochastic gradient descent (SGD)
- SGD with powers of the features
- Add L2 regularizer to the SGD with powers to improve convergence and fit

```python
import numpy as np
from numpy import random
import matplotlib.pyplot as plt
from sklearn import metrics
from sklearn.preprocessing import normalize as skNormalize
import pandas
```

In 2D, it would require many points to uniformly cover the x plan

It is then prefered to draw x points from a 2D uniform distribution

Ploting y as function of $x_0, x_1$ is then more challenging as $x_0$ and $x_1$ are not continuous monotonic vectors

$$
\begin{align}
f(x) &= (x_0−0.2)^4 + (x_0−0.1)^3 + 0.1 x_1^2 + 0.35 \\
&= x_0^4 + 0.2 x_0^3 − 0.06 x_0^2 − 0.002 x_0 + 0.1 x_1^2 + 0.3506
\end{align}
$$

```python
# f(x) as a bivariate polynom
fPoly = np.array([[-0.002, 0], [-0.06, 0.1], [0.2, 0], [1, 0]])
# Generator
def generateBatch(N):
    #
    xMin = np.array([0, -0.5])
    xMax = np.array([0.5, 0.5])
    #
    b = 0.35
    std = 0.01
    #
    x = random.uniform(xMin, xMax, (N, 2))
    yClean = (x[:,0]-0.2)**4 + (x[:,0]-0.1)**3 + 0.1*x[:,1]**2 + b
    y =  yClean + random.normal(0, std, N)
    return (x, y, yClean)
```

```python
N = 100000
xTrain, yTrain, yTrainClean = generateBatch(N);
print('x', xTrain.shape, ', yClean', yTrainClean.shape, ', y', yTrain.shape);

xTest, yTest, yTestClean = generateBatch(N);
```

    x (100000, 2) , yClean (100000,) , y (100000,)

```python
fig = plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.scatter(xTrain[:,0], yTrainClean, marker='.');
plt.xlabel('x0')
plt.subplot(1,2,2)
plt.scatter(xTrain[:,1], yTrainClean, marker='.');
plt.xlabel('x1')
plt.title('Bivariate model without noise');
```

It looks like a tobogan from the side ($x_0$) and from the front ($x_1$)

Remove biases

```python
xUnB = xTrain - np.mean(xTrain, axis=0)
yUnB = yTrain - np.mean(yTrain)
```

## Analiticaly / Closed form

Linear, 1st degree approximation of Y:
\begin{align}
Y = X W + b
\end{align}

- $X$ is a $N_{sample} \times N_{feature}$ matrix
- $W$ is a $N_{feature}$ vector
- $Y$ is a $N_{sample}$ vector
- $b$ is a scalar

To simplify calculus, let's append b to W, X is modified to append $\mathbb {1}_n$ vector on first column:
\begin{align}
\Theta &=
\begin{bmatrix}
b \\
W \\
\end{bmatrix} \\
X_m &=
\begin{bmatrix}
\mathbb{1}_n & X \\
\end{bmatrix} \\
Y &= X_m \Theta \\
\end{align}

Using mean square error (Euclidian norm), we are looking for $w_{est}$ such that:

\begin{align}
\Theta_{est} \in argmin_\Theta \lVert X_m \Theta - Y \rVert_2^2
\end{align}

Let's define f the function to minimize:
\begin{align}
f(\Theta) & = \lVert X_m \Theta - Y \rVert_2^2 \\
& = (X_m \Theta - Y_u)^T(X_m \Theta - Y) \\
& = \Theta^T X_m^T X_m \Theta + Y^T Y - 2 X_m^T \Theta^T Y
\end{align}

Lookup the minimum through the gradient:

$$\nabla_\Theta f(\Theta) = 2 X_m^T X_m \Theta - 2 X_m^T  Y$$

If $X^T X$ can be inverted:
$$\nabla_\Theta f(\Theta) = 0 \iff \Theta = (X_m^T X_m)^{-1} X_m^T Y$$

## With numpy "primitive" operations

```python
xTrainW1 = np.concatenate((np.ones(N).reshape(-1, 1), xTrain), axis=1)

xTxInv = np.linalg.inv(np.matmul(xTrainW1.T, xTrainW1))
wEstWb = np.matmul(xTxInv, np.matmul(xTrainW1.T, yTrain))
bEst = wEstWb[0]
wEst = wEstWb[1:]
print("Intercept: %.4f, coefficients: " % bEst, wEst)
```

    Intercept: 0.3440, coefficients:  [0.11291328 0.00015331]

### Test model

```python
yEst1 = np.matmul(xTest, wEst) + bEst
```

```python
fig = plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTest, marker='.');
plt.scatter(xTest[:,0], yEst1, marker='.', alpha=0.05);
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTest, marker='.');
plt.scatter(xTest[:,1], yEst1, marker='.', alpha=0.05);
# Mean square error 'a la mano'
mse1 = np.dot(yTest-yEst1, yTest-yEst1) / N
print('Linear regression MSE = {:.3e}'.format(mse1));
```

The tobogan has been approximated by a plan

## Home made with powers
Adding some flexibility to the model using powers of x

```python
xTrain2 = np.concatenate((np.ones(N).reshape(-1, 1), xTrain, xTrain**2, xTrain**3, xTrain**4), axis=1)
xTrain2.shape
```

    (100000, 9)

```python
x2Tx2Inv = np.linalg.inv(np.matmul(xTrain2.T, xTrain2))
w2Wb = np.matmul(x2Tx2Inv,np.matmul(xTrain2.T, yTrain))
b2 = w2Wb[0]
w2 = w2Wb[1:]
print("Intercept: %.4f, coefficients: " % b2, w2)
```

    Intercept: 0.3505, coefficients:  [-1.63771045e-03 -1.90875981e-04 -5.14544930e-02  9.95880746e-02
      1.37758809e-01  1.80340031e-03  1.09411993e+00  5.54438162e-03]

### Test model

```python
xTest2 = np.concatenate((xTest, xTest**2, xTest**3, xTest**4), axis=1)
yEst2 = np.matmul(xTest2, w2) + b2
```

```python
fig = plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.scatter(xTest2[:,0], yTest, marker='.');
plt.scatter(xTest2[:,0], yEst2, marker='.', alpha=0.05);
plt.subplot(1,2,2)
plt.scatter(xTest2[:,1], yTest, marker='.');
plt.scatter(xTest2[:,1], yEst2, marker='.', alpha=0.05);
mse2 = np.dot(yTest-yEst2, yTest-yEst2) / N
mse2Clean = np.dot(yTestClean-yEst2, yTestClean-yEst2) / N
print('Linear regression with powers, MSE = {:.3e}, MSE to clean test data = {:.3e}'.format(mse2, mse2Clean));
```

The polynomial of y has been well approximated.

The MSE on the noisy sample should not be lower than the variance of the noise which is $(10^{-2})^2=10^{-4}$

## NumPy least square

```python
fit4, residues, rank, s = np.linalg.lstsq(xUnB, yUnB, rcond=None)
fit4, rank
```

    (array([0.11291328, 0.00015331]), 2)

### Test model

```python
yEst4 = np.matmul( xTest, fit4) + bEst
```

```python
fig = plt.figure(figsize=(10,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTest, marker='.');
plt.scatter(xTest[:,0], yEst4, marker='.', alpha=0.05);
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTest, marker='.');
plt.scatter(xTest[:,1], yEst4, marker='.', alpha=0.05);
mse4 = metrics.mean_squared_error(yTest, yEst4)
print('Numpy least square, MSE = {:.3e}'.format(mse4) );
```

# Gradient descent

Attempt to find a value of w such that the gradient is null (indication of minima, or saddle point)

Loop on the training data until the gradient is below a threshold

Gradient on $w$ for a linear regression is:
\begin{align}
\nabla_{w} = X^T (X w - y)
\end{align}

```python
# Gradient computation
def calcGradient(x, y, w):
    gradient = np.matmul(x.T, np.matmul(x, w) - y)
    return gradient, np.sum(gradient**2)

# Plot helper to show target reference value
def plotToRef(label, subNum, subRow, subCol, ref, values):
    nIter = len(values)
    r = range(nIter)
    plt.subplot(subNum, subRow, subCol)
    plt.title(label)
    plt.plot(r, values, r, np.ones((nIter))*ref, alpha=0.5)
    plt.grid();
```

## Standard gradient descent

Loop on the same train data

```python
w6 = np.ones(3)
threshold = 1e-6
learningRate = 1e-6
gradient, gradientNorm = calcGradient(xTrainW1, yTrain, w6)

print('w start =', w6, ', Gradient norm =', gradientNorm)
w6Learn = [np.concatenate((w6, [gradientNorm]))]

while gradientNorm > threshold:
    w6 = w6 - learningRate * gradient
    gradient, gradientNorm = calcGradient(xTrainW1, yTrain, w6)
    w6Learn.append(np.concatenate((w6, [gradientNorm])))

print('w end =', w6, ', Gradient norm =', gradientNorm, ', num iteration =', len(w6Learn))
df6 = pandas.DataFrame(w6Learn, columns = ('b', 'w0', 'w1', 'Gradient norm'))
```

    w start = [1. 1. 1.] , Gradient norm = 8332553305.380094
    w end = [3.44007156e-01 1.12913773e-01 1.53310758e-04] , Gradient norm = 9.96689182973102e-07 , num iteration = 7196

```python
w6.shape
```

    (3,)

```python
fig = plt.figure(figsize=(16,10))
plotToRef('$b$', 2,3,1, bEst, df6['b'])
plotToRef('$w_0$', 2,3,2, wEst[0], df6['w0'])
plotToRef('$w_1$', 2,3,3, wEst[1], df6['w1'])
plt.subplot(2,3,4)
plt.semilogy(df6["Gradient norm"])
plt.grid()
plt.legend();
```

```python
fig = plt.figure(figsize=(16,4))
plt.subplot(1,2,1)
plt.semilogy(df6['b'], df6['Gradient norm'])
plt.xlabel('b')
plt.ylabel('Gradient norm')
plt.grid();
plt.title('Trajectory of the Gradient norm during convergence')
plt.subplot(1,2,2)
plt.semilogy(df6['w0'], df6['Gradient norm'])
plt.semilogy(df6['w1'], df6['Gradient norm'])
plt.grid();
plt.xlabel('w')
plt.ylabel('Gradient norm')
plt.legend(("$w_0$", "$w_1$"));
```

### Test model

```python
yEst6 = np.matmul(xTest, w6[1:]) + w6[0]
```

```python
fig = plt.figure(figsize=(12,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTestClean, marker='.')
plt.scatter(xTest[:,0], yEst6, marker='.', alpha=0.05)
plt.title('x0')
plt.legend(('ori clean', 'estimated'))
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTestClean, marker='.')
plt.scatter(xTest[:,1], yEst6, marker='.', alpha=0.05)
plt.title('x1')
plt.legend(('ori clean', 'estimated'))
mse6 = metrics.mean_squared_error(yTest, yEst6)
print('Gradient descent MSE = {:.3e}'.format(mse6));
```

## Stochastic gradient descent

Loop on new data, smaller batch

```python
Nbatch = 1000
w7 = np.ones(3)
threshold = 1e-4
learningRate = 1e-4
nIterMax = 1e5

onesBatch = np.ones((Nbatch, 1))

# Initial batch
xBatch, yBatch, yBC = generateBatch(Nbatch)
gradient, gradientNorm = calcGradient(np.concatenate((onesBatch, xBatch), axis=1), yBatch, w7)

print('w start =', w7, ', Gradient norm =', gradientNorm)
w7Learn = [np.concatenate((w7, [gradientNorm]))]

# Continue
nIter = 0
while (gradientNorm > threshold) & (nIter <= nIterMax):
    w7 = w7 - learningRate * gradient
    xBatch, yBatch, yBC = generateBatch(Nbatch)
    gradient, gradientNorm = calcGradient(np.concatenate((onesBatch, xBatch), axis=1), yBatch, w7)
    w7Learn.append(np.concatenate((w7, [gradientNorm])))
    learningRate = learningRate * 0.9999
    nIter += 1

print('w end =', w7, ', Gradient norm =', gradientNorm, ', num iteration =', len(w7Learn))
df7 = pandas.DataFrame(w7Learn, columns = ('b', 'w0', 'w1', 'Gradient norm'))
```

    w start = [1. 1. 1.] , Gradient norm = 812489.7498501091
    w end = [3.43957080e-01 1.12969836e-01 2.40469033e-05] , Gradient norm = 6.611670633228906e-05 , num iteration = 33932

```python
fig = plt.figure(figsize=(16,10))
plotToRef('$b$', 2,3,1, bEst, df7['b'])
plotToRef('$w_0$', 2,3,2, wEst[0], df7['w0'])
plotToRef('$w_1$', 2,3,3, wEst[1], df7['w1'])
plt.subplot(2,3,4)
plt.semilogy(df7['Gradient norm'])
plt.grid()
plt.legend();
```

```python
fig = plt.figure(figsize=(16,4))
plt.subplot(1,2,1)
plt.semilogy(df7['b'], df7['Gradient norm'])
plt.xlabel('b')
plt.ylabel('Gradient norm')
plt.grid();
plt.title('Trajectory of the Gradient norm during convergence')
plt.subplot(1,2,2)
plt.semilogy(df7['w0'], df7['Gradient norm'])
plt.semilogy(df7['w1'], df7['Gradient norm'])
plt.grid();
plt.xlabel('w')
plt.ylabel('Gradient norm')
plt.legend(("$w_0$", "$w_1$"));
```

### Test model

```python
yEst7 = np.matmul(xTest, w7[1:]) + w7[0]
```

```python
fig = plt.figure(figsize=(12,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTestClean, marker='.')
plt.scatter(xTest[:,0], yEst7, marker='.', alpha=0.05)
plt.title('$x_0$')
plt.legend(('ori clean', 'estimated'))
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTestClean, marker='.')
plt.scatter(xTest[:,1], yEst7, marker='.', alpha=0.05)
plt.title('$x_1$')
plt.legend(('ori clean', 'estimated'))
mse7 = metrics.mean_squared_error(yTest, yEst7)
print('Stochastic gradient descent MSE = {:.3e}'.format(mse7));
```

## SGD with powers

Adding powers of x to the stochastic gradient descent

```python
# Batch size
nBatch = 1000
# Initial coefficients (fixed and tuned manually for this trial)
w8 = np.ones(9)* 1.5
# Gradient descent end threshold (on the gradient)
threshold = 1e-2
# Initial learning rate
learningRate = 1e-4
# Maximum number of iterations
nIterMax = 1e5

onesBatch = np.ones((Nbatch, 1))

# Initial batch
xBatch, yBatch, yBC = generateBatch(nBatch)
xBatch2 = np.concatenate((onesBatch, xBatch, xBatch**2, xBatch**3, xBatch**4), axis = 1)
gradient, gradientNorm = calcGradient(xBatch2, yBatch, w8)

print('w start =', w8, ', Gradient norm = {:.3e}'.format(gradientNorm))
w8Learn = [np.concatenate((w8, [gradientNorm]))]

# Continue
nIter = 0
while (gradientNorm > threshold) & (nIter < nIterMax):
    w8 = w8 - learningRate * gradient
    xBatch, yBatch, yBC = generateBatch(nBatch)
    xBatch2 = np.concatenate((onesBatch, xBatch, xBatch**2, xBatch**3, xBatch**4), axis = 1)
    gradient, gradientNorm = calcGradient(xBatch2, yBatch, w8)
    w8Learn.append(np.concatenate((w8, [gradientNorm])))
    learningRate = learningRate * 0.9999
    nIter += 1

print('w end =', w8, ', Gradient norm = {:.3e}'.format(gradientNorm), ', num iteration =', len(w8Learn))
df8 = pandas.DataFrame(w8Learn, columns = ('b', 'w0_1', 'w1_1', 'w0_2', 'w1_2', 'w0_3', 'w1_3', 'w0_4', 'w1_4', 'Gradient norm'))
```

    w start = [1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5] , Gradient norm = 3.702e+06
    w end = [ 0.38864066 -0.31918763 -0.13231316  0.24717002 -0.13794083  0.82083255
      0.87821587  1.1595413   1.12787625] , Gradient norm = 3.119e-01 , num iteration = 100001

```python
fig = plt.figure(figsize=(12,20))
plotToRef('$w_{0,1}$', 5,2,1, fPoly[0,0], df8['w0_1'])
plotToRef('$w_{1,1}$', 5,2,2, fPoly[0,1], df8['w1_1'])
plotToRef('$w_{0,2}$', 5,2,3, fPoly[1,0], df8['w0_2'])
plotToRef('$w_{1,2}$', 5,2,4, fPoly[1,1], df8['w1_2'])
plotToRef('$w_{0,3}$', 5,2,5, fPoly[2,0], df8['w0_3'])
plotToRef('$w_{1,3}$', 5,2,6, fPoly[2,1], df8['w1_3'])
plotToRef('$w_{0,4}$', 5,2,7, fPoly[3,0], df8['w0_4'])
plotToRef('$w_{1,4}$', 5,2,8, fPoly[3,1], df8['w1_4'])
plotToRef('$b$', 5,2,9, bEst, df8['b'])
plt.subplot(5,2,10)
plt.semilogy(df8['Gradient norm'])
plt.grid()
plt.legend();
```

Convergence is not correct on $w_{1}$ that should only feature a small coefficient on the 2nd order term and no coefficient on the fourth order $w_{1,4}$

### Test model

```python
yEst8 = np.matmul(xTest2, w8[1:]) + w8[0]
```

```python
fig = plt.figure(figsize=(12,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTestClean, marker='.')
plt.scatter(xTest[:,0], yEst8, marker='.', alpha=0.05)
plt.title('$x_0$')
plt.legend(('ori clean', 'estimated'))
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTestClean, marker='.')
plt.scatter(xTest[:,1], yEst8, marker='.', alpha=0.05)
plt.title('$x_1$')
plt.legend(('ori clean', 'estimated'))
mse8 = metrics.mean_squared_error(yTest, yEst8)
print('Stochastic gradient descent MSE = {:.3e}'.format(mse8));
```

### Conclusion on SGD with powers

The fitting of the 4th and 3rd degree powers takes more time, leading to a initial  w shape on the right hand graph ($x_1$).

The MSE is improved, but the optimizer is harder to tune

## SGD with powers and regularizer

Let's add a regularizer to the SGD with powers, using the Euclidian norm (L2)

Gradient on $w$ for a linear regression with L2 regularizer is:
\begin{align}
\nabla_{w} = X^T (X w - y) + \lambda w
\end{align}

```python
# Calculate gradient with regularizer and normalization
def calcGradientRegularizeL2(x, y, w, lam):
    gradient = np.matmul(x.T, np.matmul(x, w) - y) + lam * w
    return gradient, np.sum(gradient**2)
```

```python
nBatch = 1000
w9 = np.ones(9)* 1.5
threshold = 1e-2
learningRate = 1e-4
nIterMax = 1e5
# Regularizer param
lambdaReg = 0.01

# Initial batch
xBatch, yBatch, yBC = generateBatch(nBatch)
xBatch2 = np.concatenate((onesBatch, xBatch, xBatch**2, xBatch**3, xBatch**4), axis = 1)
gradient, gradientNorm = calcGradientRegularizeL2(xBatch2, yBatch, w9, lambdaReg)
print('w start =', w9, ', Gradient norm = {:.3e}'.format(gradientNorm))

w9Learn = [np.concatenate((w9, [gradientNorm]))]
# Continue
nIter = 0
while (gradientNorm > threshold) & (nIter < nIterMax):
    w9 = w9 - learningRate * gradient
    xBatch, yBatch, yBC = generateBatch(nBatch)
    xBatch2 = np.concatenate((onesBatch, xBatch, xBatch**2, xBatch**3, xBatch**4), axis = 1)
    gradient, gradientNorm = calcGradientRegularizeL2(xBatch2, yBatch, w9, lambdaReg)
    w9Learn.append(np.concatenate((w9, [gradientNorm])))
    # learningRate = learningRate * 0.9999
    nIter += 1

print('w end =', w9, ', Gradient norm = {:.3e}'.format(gradientNorm), ', num iteration =', len(w9Learn))
df9 = pandas.DataFrame(w9Learn, columns = ('b', 'w0_1', 'w1_1', 'w0_2', 'w1_2', 'w0_3', 'w1_3', 'w0_4', 'w1_4', 'Gradient norm'))
```

    w start = [1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5] , Gradient norm = 3.619e+06
    w end = [ 0.36046531 -0.03535633 -0.03482748 -0.10542564 -0.11390478  0.47877567
      0.23068354  0.92126672  0.99334058] , Gradient norm = 8.573e-03 , num iteration = 46918

```python
fig = plt.figure(figsize=(12,20))
plotToRef('$w0,1$', 5,2,1, fPoly[0,0], df9['w0_1'])
plotToRef('$w1,1$', 5,2,2, fPoly[0,1], df9['w1_1'])
plotToRef('$w0,2$', 5,2,3, fPoly[1,0], df9['w0_2'])
plotToRef('$w1,2$', 5,2,4, fPoly[1,1], df9['w1_2'])
plotToRef('$w0,3$', 5,2,5, fPoly[2,0], df9['w0_3'])
plotToRef('$w1,3$', 5,2,6, fPoly[2,1], df9['w1_3'])
plotToRef('$w0,4$', 5,2,7, fPoly[3,0], df9['w0_4'])
plotToRef('$w1,4$', 5,2,8, fPoly[3,1], df9['w1_4'])
plotToRef('$b$', 5,2,9, bEst, df9['b'])
plt.subplot(5,2,10)
plt.semilogy(df9['Gradient norm'])
plt.grid()
plt.legend();
```

### Test model

```python
yEst9 = np.matmul(xTest2, w9[1:]) + w9[0]
```

```python
fig = plt.figure(figsize=(12,4))
plt.subplot(1,2,1)
plt.scatter(xTest[:,0], yTestClean, marker='.')
plt.scatter(xTest[:,0], yEst9, marker='.', alpha=0.05)
plt.title('x0')
plt.legend(('ori clean', 'estimated'))
plt.subplot(1,2,2)
plt.scatter(xTest[:,1], yTestClean, marker='.')
plt.scatter(xTest[:,1], yEst9, marker='.', alpha=0.05)
plt.title('x1')
plt.legend(('ori clean', 'estimated'))
mse9 = metrics.mean_squared_error(yTest, yEst9)
print('Stochastic gradient descent with L2 regularizer, MSE = {:.3e}'.format(mse9));
```

### Conclusion on SGD with regularizer

The regularizer has a larger effect on 3rd and 4th degree terms, leading to a fitting that mostly use 1st and 2nd degree (parabolic)

# Where to go from here ?

__Other _two feature_ linear implementation using Keras__ ([HTML](LinearRegressionBivariate-Keras.html) / [Jupyter](LinearRegressionBivariate-Keras.ipynb))

__Theoretical and practical quality aspects of linear regression with the Gaussian Model__ ([HTML](LinearRegressionBivariateQuality.html) / [Jupyter](LinearRegressionBivariateQuality.ipynb))

More complex linear regression __with Feature engineering or feature learning__ ([HTML](LinearRegressionFeatureEngineering-Keras.html) / [Jupyter](LinearRegressionFeatureEngineering-Keras.ipynb))

__Compare with the two feature binary classification with logistic regression using "raw" Python or libraries__ ([HTML](../classification/ClassificationContinuous2Features.html) / [Jupyter](../classification/ClassificationContinuous2Features.ipynb]))