text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Sep 22 13:09:50 2018

@author: chenxiaoxu
"""

from matplotlib import pyplot as plt
import camera as cam
import geo, os
import numpy as np
from matplotlib import pyplot as plt
#import stat_tools as st
from PIL import Image
#import geopy
from geopy.distance import VincentyDistance
import pysolar.solar as ps
import datetime
import pandas as pd
import geopy

inpath = '~/data/images/'
inpathUndistort = '~/pipeLineOutput/final/undistorted/undistorted/'
outpathStitched = '~/pipeLineOutput/final/Stitched/'

PRINT = True
PLOT = True
WRITE = True

##############################################################################
#global parameters for stitching
deg2rad = np.pi/180
max_theta = 70 * deg2rad
max_tan = np.tan(max_theta)

Sensor_ind = 'GHI1'

GHI_Coor = {'GHI1':   [40.868972, -72.852225]}

coordinate = {'HD1A':[40.8580088,  -72.8575717],
              'HD1B':[40.8575056,  -72.8547344],
              'HD1C':[40.85785,    -72.8597],
              'HD2B':[40.872341,   -72.874354],
              'HD2C':[40.87203321, -72.87348295],
              'HD3A':[40.897122,   -72.879053],
              'HD3B':[40.8975,     -72.877497],
              'HD4A':[40.915708,   -72.892406],
              'HD4B':[40.917275,   -72.891592],
              'HD5A':[40.947353,   -72.899617],
              'HD5B':[40.948044,   -72.898372],
             }
#########
#HD2C is the center camera and it must be put in the first place
camIDs4Stitching = ['HD2C', 'HD1A', 'HD1B', 'HD2B', 'HD3A', 'HD4A']
#camIDs4Stitching = ['HD2C', 'HD2B', 'HD1A', 'HD3A']

#########
camCenterID = 'HD2C'
##############################################################################

def height_computing(args):
    imager, neighbors, Date, TimeStamp, DocumFilename = args

    f = inpath + imager.camID + '/' + Date + '/' + imager.camID + '_' + Date + TimeStamp + '.jpg'

    if not os.path.exists(f):
        return -1

    img = cam.image(imager, f)
    img.undistort(imager, rgb = True)

    if img.red is not None:
        img.cloud_mask(imager)
    else:
#    if img.red is None or img.layers <= 0:
        if WRITE:
            Document = open(DocumFilename, 'a')
            Document.write("img.red is None or img.layers <=0")
            Document.close()
        return -1

    h = [[]]*img.layers

    for neighbor in neighbors:
        f_nb = f.replace(imager.camID, neighbor.camID)

        img_nb = cam.image(neighbor, f_nb)
        img_nb.undistort(neighbor, rgb = True)
        if img_nb.red is not None:
            img_nb.cloud_mask(neighbor)
        else:
            if PRINT:
                print("img_nb.red is None")
            continue

        distance = 6367e3*geo.distance_sphere(img.lat, img.lon, img_nb.lat, img_nb.lon)
        for ih in range(img.layers):
            if (len(h[ih])) >= 1:
                continue
            res = cam.cloud_height(img, img_nb, layer = ih, distance = distance)

            if len(res) >= 1 and res[0] < 30*distance and res[0]>0.5*distance:
                h[ih] = res[0]
                if PRINT:
                    print("with camera " + imager.camID + "neighbor " + neighbor.camID)
                    print("the cloud height is")
                    print(res[0])
                return res[0]

      #  if len(h) >= img.layers:
      #      if PRINT:
      #          print("with camera " + imager.camID + " neighbor " + neighbor.camID)
      #          print("len(h) = " + str(len(h)))
      #          print("img.layers =" + str(img.layers))
      #      break

    if (len(h[0]) == 0):
        if PRINT:
            print("len(h[0]) == 0")
    return -1
#    return h[0][0]

def stitching(args):

    Date, TimeStamp, CBH, vy, vx, forwardtimeList, CamImgSize, DocumFilename, = args

    if CBH < 900:
        WindowSize = 100
        DestImg_shp = (8000, 6000 , 3)
    else:
        WindowSize = 150
        DestImg_shp = (12000, 9000 , 3)

    DestImg_shp = (12000,9000,3)
    #Destination Image
    DestImg = np.zeros(DestImg_shp, dtype = np.uint8)

    for camID in camIDs4Stitching:
        print(camID)
        Img_name = inpathUndistort + '/' +  camID + '/' + Date +  '/' + camID + '_' + Date + TimeStamp + '.jpg'

        if not os.path.exists(Img_name):
            if WRITE:
                Document = open(DocumFilename, 'a')
                Document.write(Img_name + '\n' + 'file not found\n')
                Document.close()
            return [[-1]*len(forwardtimeList), [[]]*len(forwardtimeList)]

        CamImg = plt.imread(Img_name)
        CamImg = np.flip(CamImg, 0)
        CamImg = np.flip(CamImg, 1)

        CamImgSize = CamImg.shape[0]

        if (camID == camCenterID):
            Center = (int(DestImg_shp[0]/3), int(DestImg_shp[1]/3*2))
            CamCenterCoor = (coordinate[camID][0], coordinate[camID][1])

            LatiStt = Center[0] - int(CamImgSize/2)
            LatiEnd = Center[0] + int(CamImgSize/2)
            LongStt = Center[1] - int(CamImgSize/2)
            LongEnd = Center[1] + int(CamImgSize/2)

            #Camera Covrage in km
            TotalCoverage_km = 2*0.001*CBH*max_tan
            #km for every pixel
            Km1Pixel = TotalCoverage_km/CamImgSize
            GeoPoint = geopy.Point(CamCenterCoor[0], CamCenterCoor[1])
            #Coord for every Pixel
            Coor1Pixel = (VincentyDistance(kilometers = Km1Pixel).destination(GeoPoint, 0).latitude - CamCenterCoor[0],
                          VincentyDistance(kilometers = Km1Pixel).destination(GeoPoint, 90).longitude - CamCenterCoor[1])

            for j in range(0,3):
                DestImg[LatiStt:LatiEnd, LongStt:LongEnd, j] = CamImg[:,:,j]

            continue

        #Coordinate of Camera
        CamCoor = (coordinate[camID][0], coordinate[camID][1])
        #Difference between this Camera and the Center Camera in Coordinate
        DeltaCoor = (CamCoor[0] - CamCenterCoor[0], CamCoor[1] - CamCenterCoor[1])
        #Difference between this Camera and the Center Camera in Pixel
        DeltaPixel = (int(DeltaCoor[0]/Coor1Pixel[0]), int(DeltaCoor[1]/Coor1Pixel[1]))

        CamCenter = (Center[0] + DeltaPixel[0], Center[1] + DeltaPixel[1])

        LatiStt = CamCenter[0] - int(CamImgSize/2)
        LatiEnd = CamCenter[0] + int(CamImgSize/2)
        LongStt = CamCenter[1] - int(CamImgSize/2)
        LongEnd = CamCenter[1] + int(CamImgSize/2)

        for j in range(0, 3):
            DestImg_Ori = DestImg[LatiStt : LatiEnd, LongStt : LongEnd, j].copy()
            DestImg_Ori = DestImg_Ori.reshape(-1)

            OriNonzeroInd = np.nonzero(DestImg_Ori)[0]

            if (len(OriNonzeroInd) >0):
                CamImgPad = CamImg[:,:,j].copy()
                CamImgPad =  CamImgPad.reshape(-1)
                CamImgPad[OriNonzeroInd] = DestImg_Ori[OriNonzeroInd]
                CamImgPad = np.reshape(CamImgPad, (CamImgSize, CamImgSize))

                DestImg[LatiStt:LatiEnd, LongStt:LongEnd, j] = CamImgPad
            else:
                DestImg[LatiStt:LatiEnd, LongStt:LongEnd, j] = CamImg[:,:,j]

####################################################################################
#    for k in range(1, 26):
#        sensor = 'GHI' + str(k)
#        Sen_Coor = (GHI_Coor[sensor][0], GHI_Coor[sensor][1])
#        Sen_Coor = ((Sen_Coor[0] - Mu[0])/Sigma[0], (Sen_Coor[1] - Mu[1])/Sigma[1])
#        Mark_Lati = np.where(DestGrid[0] >= Sen_Coor[0])[0][0]
#        Mark_Long = np.where(DestGrid[1] >= Sen_Coor[1])[0][0]
#        plt.scatter(Mark_Long, Mark_Lati, s=1, c='yellow', marker='o')
####################################################################################
    print('a')
    Sensor_Coor = (GHI_Coor[Sensor_ind][0], GHI_Coor[Sensor_ind][1])
    Delta_Sensor_Coor = (Sensor_Coor[0] - CamCenterCoor[0], Sensor_Coor[1] - CamCenterCoor[1])
    Delta_Sensor_Pixel = (int(Delta_Sensor_Coor[0]/Coor1Pixel[0]), int(Delta_Sensor_Coor[1]/Coor1Pixel[1]))

    Sensor_loc = (Center[0] + Delta_Sensor_Pixel[0], Center[1] + Delta_Sensor_Pixel[1])

    year = Date[:4]
    month =  Date[4:6]
    day = Date[6:]

    hour = TimeStamp[:2]
    minute = TimeStamp[2:4]
    second = TimeStamp[4:]

    time = datetime.datetime(int(year), int(month), int(day), int(hour), int(minute), int(second), 000000, tzinfo = datetime.timezone.utc)
    sun_altitude = ps.get_altitude(Sensor_Coor[0], Sensor_Coor[1], time)
    sun_azimuth = ps.get_azimuth(Sensor_Coor[0], Sensor_Coor[1], time)

    hemi = -1

    #hemi  = 0 means west, hemi = 1 means east
    if ((sun_azimuth <= 0) | (sun_azimuth > -90)):
        hemi = 0
    elif (sun_azimuth <-270 | (sun_azimuth > -360)):
        hemi = 1
    else:
        if PRINT:
            print("problem in sun_azimuth\n")
            print(camID)
            print(sun_azimuth)

        if WRITE:
            Document = open(DocumFilename, 'a')
            Document.write('problem in sun_azimuth\n' + camID + str(sun_azimuth))
            Document.close()
        STL = -1
        return [[STL]*len(forwardtimeList), [[]]*len(forwardtimeList)]

    if (hemi == 0):
        sun_azimuth = (-sun_azimuth)*deg2rad
        distance_km = (CBH*0.001) * np.tan((90 - sun_altitude) * deg2rad)
        delta_south_km = distance_km*np.cos(sun_azimuth)
        delta_west_km = distance_km*np.sin(sun_azimuth)

        delta_south_pixel = int(delta_south_km / Km1Pixel)
        delta_west_pixel = int(delta_west_km / Km1Pixel)

        Sensor_loc_img = (Sensor_loc[0] - delta_south_pixel, Sensor_loc[1] - delta_west_pixel)

    else:
        sun_azimuth = (360 + sun_azimuth) * deg2rad
        distance_km = (CBH*0.001) * np.tan((90 - sun_altitude) * deg2rad)
        delta_south_km = distance_km*np.cos(sun_azimuth)
        delta_east_km = distance_km*np.sin(sun_azimuth)

        delta_south_pixel = int(delta_south_km / Km1Pixel)
        delta_east_pixel =  int(delta_east_km / Km1Pixel)

        Sensor_loc_img = (Sensor_loc[0] - delta_south_pixel, Sensor_loc[1] + delta_east_pixel)

    FeatureStt1 = (int(Sensor_loc_img[0] - WindowSize/2), int(Sensor_loc_img[1] - WindowSize/2))
    FeatureEnd1 = (int(Sensor_loc_img[0] + WindowSize/2), int(Sensor_loc_img[1] + WindowSize/2))

    if ((FeatureEnd1[0] < 0) | (FeatureStt1[0] > DestImg.shape[0]) |  (FeatureStt1[1] > DestImg.shape[1] | (FeatureEnd1[1] < 0))):
        STL = 0
        return [[STL]*len(forwardtimeList), [[]]*len(forwardtimeList)]

    else:
        FeatureImage1 = DestImg[max(0, FeatureStt1[0]) : min(DestImg_shp[0], FeatureEnd1[0]), max(0, FeatureStt1[1]) : min(DestImg_shp[1], FeatureEnd1[1]), :]

        ZeroCounter = 0

        FeatureChannel = FeatureImage1[:, :, 0]
        FeatureVector = FeatureChannel.reshape(-1)
        NonZeroInd  = np.nonzero(FeatureVector)[0]

        if (len(NonZeroInd) == 0):
            ZeroCounter = ZeroCounter + 1
            Rave1 = 0
            Rmin1 = 0
            Rmax1 = 0
        else:
            Rave1 = np.average(FeatureVector[NonZeroInd])/255
            Rmin1 = np.min(FeatureVector[NonZeroInd])/255
            Rmax1 = np.max(FeatureVector[NonZeroInd])/255

        FeatureChannel = FeatureImage1[:, :, 1]
        FeatureVector = FeatureChannel.reshape(-1)
        NonZeroInd  = np.nonzero(FeatureVector)[0]

        if (len(NonZeroInd) == 0):
            ZeroCounter = ZeroCounter + 1
            Gave1 = 0
            Gmin1 = 0
            Gmax1 = 0
        else:
            Gave1 = np.average(FeatureVector[NonZeroInd])/255
            Gmin1 = np.min(FeatureVector[NonZeroInd])/255
            Gmax1 = np.max(FeatureVector[NonZeroInd])/255

        FeatureChannel = FeatureImage1[:, :, 2]
        FeatureVector = FeatureChannel.reshape(-1)
        NonZeroInd  = np.nonzero(FeatureVector)[0]

        if (len(NonZeroInd) == 0):
            ZeroCounter = ZeroCounter + 1
            Bave1 = 0
            Bmin1 = 0
            Bmax1 = 0
        else:
            Bave1 = np.average(FeatureVector[NonZeroInd])/255
            Bmin1 = np.min(FeatureVector[NonZeroInd])/255
            Bmax1 = np.max(FeatureVector[NonZeroInd])/255

        if (ZeroCounter == 3):
            STL = 0
            return [[STL]*len(forwardtimeList), [[]]*len(forwardtimeList)]

        RBR1 = (Rave1 - Bave1) / (Rave1 + Bave1)

    STLList = []
    featureVectorList = []
    #Since we flip CamImg before, the vy and vx become the opposite.
    for i in range(0, len(forwardtimeList)):

        dy = vy*forwardtimeList[i]
        dx = vx*forwardtimeList[i]

        BacktrackPoint = (Sensor_loc_img[0] + dy, Sensor_loc_img[1] + dx)

        FeatureStt2 = (int(BacktrackPoint[0] - WindowSize/2), int(BacktrackPoint[1] - WindowSize/2))
        FeatureEnd2 = (int(BacktrackPoint[0] + WindowSize/2), int(BacktrackPoint[1] + WindowSize/2))

        if PLOT and i == len(forwardtimeList) - 1:
            print("plotting")
            print(i)
            outpathfile = outpathStitched + Date + '/predict' + str(forwardtimeList[i]) + '/'
            if not os.path.isdir(outpathfile):
                os.makedirs(outpathfile)
                os.chmod(outpathfile, 0o755)

            savename = outpathfile + TimeStamp + '.jpg'
            DestImgPlot = DestImg.copy()
            for kk in range(0,3):
                DestImgPlot[FeatureStt2[0]:FeatureEnd2[0], FeatureStt2[1]:FeatureEnd2[1],kk] = 0
                DestImgPlot[FeatureStt1[0]:FeatureEnd1[0], FeatureStt1[1]:FeatureEnd1[1],kk] = 0

            DestImgPlot[FeatureStt1[0]:FeatureEnd1[0], FeatureStt1[1]:FeatureEnd1[1],0] = 255
            DestImgPlot[FeatureStt2[0]:FeatureEnd2[0], FeatureStt2[1]:FeatureEnd2[1],1] = 255

            print(FeatureStt2)
            print(FeatureEnd2)

            DestImgPlot = np.flip(DestImgPlot, 0)
            FinalImage = Image.fromarray(DestImgPlot)
            FinalImage.save(savename)

        if ((FeatureEnd2[0] < 0) | (FeatureStt2[0] > DestImg.shape[0]) |  (FeatureStt2[1] > DestImg.shape[1] | (FeatureEnd2[1] < 0))):
            STL = 0
            STLList.append(STL)
            featureVectorList.append([])
            continue

        else:
            FeatureImage2 = DestImg[max(0, FeatureStt2[0]) : min(DestImg_shp[0], FeatureEnd2[0]), max(0, FeatureStt2[1]) : min(DestImg_shp[1], FeatureEnd2[1]), :]

            ZeroCounter = 0

            FeatureChannel = FeatureImage2[:, :, 0]
            FeatureVector = FeatureChannel.reshape(-1)
            NonZeroInd  = np.nonzero(FeatureVector)[0]

            if (len(NonZeroInd) == 0):
                ZeroCounter = ZeroCounter + 1
                Rave2 = 0
                Rmin2 = 0
                Rmax2 = 0
            else:
                Rave2 = np.average(FeatureVector[NonZeroInd])/255
                Rmin2 = np.min(FeatureVector[NonZeroInd])/255
                Rmax2 = np.max(FeatureVector[NonZeroInd])/255

            FeatureChannel = FeatureImage2[:, :, 1]
            FeatureVector = FeatureChannel.reshape(-1)
            NonZeroInd  = np.nonzero(FeatureVector)[0]

            if (len(NonZeroInd) == 0):
                ZeroCounter = ZeroCounter + 1
                Gave2 = 0
                Gmin2 = 0
                Gmax2 = 0
            else:
                Gave2 = np.average(FeatureVector[NonZeroInd])/255
                Gmin2 = np.min(FeatureVector[NonZeroInd])/255
                Gmax2 = np.max(FeatureVector[NonZeroInd])/255

            FeatureChannel = FeatureImage2[:, :, 2]
            FeatureVector = FeatureChannel.reshape(-1)
            NonZeroInd  = np.nonzero(FeatureVector)[0]

            if (len(NonZeroInd) == 0):
                ZeroCounter = ZeroCounter + 1
                Bave2 = 0
                Bmin2 = 0
                Bmax2 = 0
            else:
                Bave2 = np.average(FeatureVector[NonZeroInd])/255
                Bmin2 = np.min(FeatureVector[NonZeroInd])/255
                Bmax2 = np.max(FeatureVector[NonZeroInd])/255

            if (ZeroCounter == 3):
                STL = 0
                STLList.append(STL)
                featureVectorList.append([])
                continue

            RBR2 = (Rave2 - Bave2) / (Rave2 + Bave2)

            STL = 1
            featureVector = [Rave1, Rmin1, Rmax1,
                             Gave1, Gmin1, Gmax1,
                             Bave1, Bmin1, Bmax1,
                             Rave2, Rmin2, Rmax2,
                             Gave2, Gmin2, Gmax2,
                             Bave2, Bmin2, Bmax2,
                             RBR1, RBR2]
            STLList.append(STL)
            featureVectorList.append(featureVector)

    return [STLList, featureVectorList]


text:
from simplefit.cleaner import cleaner
import pandas as pd
import numpy as np
import pytest
from pytest import raises

def df(lower_case=True):
    dataf = pd.DataFrame({'num_legs': [2, 4, 8, 0],
                   'num_  wings ': ['bad', 'Good', None, 'bad'],
                   'Num specimen seen ': [10, 2, 1, 8]},
                  index=['falcon', 'dog', 'spider', 'fish'])

    return dataf

def test_cleaner():
    """
    Test function to check the clean data.
    """
    # Tests whether a not dataframe input raises TypeError
    with raises(TypeError):
        cleaner(np.array([1, 2, 3, 4, 5]))

    # Tests whether a not dataframe input raises TypeError
    with raises(TypeError):
         cleaner(df(),lower_case=1)

def test_correct_output():
    # Tests that cleaner returns a dataframe
    output = cleaner(df())

    assert isinstance(output, pd.DataFrame), "Output is not a pandas DataFrame"

def test_correct_columns():
    #TEST 2: Test that the dataframe has the appropriate column names
    output= cleaner(df(),lower_case=True)
    expected_columns = ['num_legs', 'num_  wings', 'num specimen seen']
    assert list(output.columns) == list(expected_columns), "not equal"

ext: py

max_stars_repo_name: UBC-MDS/simplefit

max_issues_repo_name: UBC-MDS/simplefit

max_forks_repo_name: UBC-MDS/simplefit

reason: import numpy


text:
import numpy as np

def KRLS_RFF(u,d,h,D,alpha_0,beta=1.0,l=1.0):

    err = []
    P = np.eye(D)/l
    alpha = alpha_0
    for n in range(len(d)):
        h_n = h.T[n].reshape((D,1))
        err.append((d[n] - h_n.T @ alpha).item())
        a = h_n.T @ P
        k = (P @ h_n)/(beta + a @ h_n)
        P = (P - (k @ a))/beta
        alpha = alpha + k * err[-1]
    return err,alpha


text:
import referenceFrames;
import pyStk;
import re;
import numpy as np;

class pyRFException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class pyRF():

    def __init__(self,refFrame):

        self.refStnList = list();
        self.refData = np.array([]);

        # get the reference frame definition
        if refFrame.lower() in ('itrf08','itrf2008'):
            self.refFrame = referenceFrames.itrf2008;
        elif refFrame.lower() in ('itrf05','itrf2005'):
            self.refFrame = referenceFrames.itrf2005;
        else:
            raise pyRFException('Reference frame '+refFrame+' not recognized');

        # save the id
        self.refFrameId = refFrame;

        # to be determined ...
        self.stnList = [];
        self.npv = np.array([]);
        self.nvv = np.array([]);
        self.refEpoch = np.array([]);

        self.__parse();

    def __parse(self):

        # parse the line on spaces
        lineParts = re.split('\s+',self.refFrame);
        del lineParts[0];

        for i in range(0,len(lineParts)/8):

            # cast the line parts
            stnId = lineParts[i*8];

            x = float(lineParts[i*8+1]);
            y = float(lineParts[i*8+2]);
            z = float(lineParts[i*8+3]);

            vx = float(lineParts[i*8+4]);
            vy = float(lineParts[i*8+5]);
            vz = float(lineParts[i*8+6]);

            refEpoch = float(lineParts[i*8+7]);

            # add the stnid to reference list
            self.refStnList.append(stnId.lower());

            # add the data
            if self.refData.size == 0:
                self.refData = np.array([x,y,z,vx,vy,vz,refEpoch]);
            else:
                self.refData = np.vstack((self.refData,np.array([x,y,z,vx,vy,vz,refEpoch])));

    def initForStnList(self,stnList):

        self.stnList = stnList;

        # mem alloc
        numStns       = len(stnList);
        self.npv      = np.zeros(3*numStns,dtype='float')*np.nan;
        self.nvv      = np.zeros(3*numStns,dtype='float')*np.nan;
        self.refEpoch = np.zeros(3*numStns,dtype='float')*np.nan;

        for i in range(0,len(stnList)):

            stn = stnList[i];

            if stn in self.refStnList:

                refIndx = self.refStnList.index(stn);
                indx    = 3*i;

                self.npv[indx:indx+3]      = self.refData[refIndx,[0,1,2]];
                self.nvv[indx:indx+3]      = self.refData[refIndx,[3,4,5]];
                self.refEpoch[indx:indx+3] = self.refData[refIndx,[6]].repeat(3);

        return self;

    def init(self):

        self.initForStnList(self.refStnList);
        return self;

    def npvForEpoch(self,fyear):

        dt  = fyear - self.refEpoch;
        npv = self.npv + (self.nvv * dt);
        npv = npv.transpose().reshape(npv.size,1);

        return npv;

    def transformForEpoch(self,npv,fyear):

        npvTarget    = self.npvForEpoch(fyear);
        T,npvT,stats = pyStk.helmert(npv, npvTarget);
        return T,npvT,stats

    def transformWithScaleForEpoch(self,npv,fyear):

        npvTarget    = self.npvForEpoch(fyear);
        T,npvT,stats = pyStk.helmert(npv, npvTarget,withScale='yes');
        return T,npvT,stats

max_stars_repo_name: softwarespartan/pyStk

max_issues_repo_name: softwarespartan/pyStk

max_forks_repo_name: softwarespartan/pyStk


text:
# -------------------------------------------------------------------------------------------------
# scientific
import numpy as np
# -------------------------------------------------------------------------------------------------
# system
from copy import copy
# -------------------------------------------------------------------------------------------------
# PyQuantum.Tools
from PyQuantum.Tools.Distributions import Expectation, GeometricDistribution
from PyQuantum.Tools.Pickle import *
from PyQuantum.Tools.Print import *
from PyQuantum.Tools.Units import *
# -------------------------------------------------------------------------------------------------

# =================================================================================================
class Sink:
    def __init__(self, P=[], T=[]):
        self.data = {
            'P': copy(P),
            'T': copy(T),
        }

    def set_P(self, P):
        self.data['P'] = copy(P)

    def set_T(self, T):
        self.data['T'] = copy(T)

    def print(self):
        for i in range(len(self.data['P'])):
            print("{:3f}".format(self.data['T'][i]),
                  ': ', self.data['P'][i], sep='')

        print()
# =================================================================================================

# -------------------------------------------------------------------------------------------------
n_dots = 500

dt = 10

n_trials = 100

print('-'*100)
print('n_dots:', n_dots)
print('dt:', dt)
print('n_trials:', n_trials)
print('-'*100)
print()
# -------------------------------------------------------------------------------------------------

# -------------------------------------------------------------------------------------------------
p_sink = np.abs(np.random.random_sample(n_dots))

p_sink = sorted(p_sink)

T = list(range(1, len(p_sink)+1))
T_dt = T[dt-1::dt]
# T_dt = list(range(dt, len(T), dt))
p_sink_dt = p_sink[dt-1::dt]

data = Sink(P=p_sink, T=T)

if __debug__:
    data.print()
# -------------------------------------------------------------------------------------------------

# BEGIN----------------------------------------- EXPERIMENT ---------------------------------------
T_click = []

max_t = n_dots

if __debug__:
    print('-'*50)

click = []

for _ in range(n_trials):
    T_click_trial = []

    click_trial = {
        'T': [],
        'step': [],
    }

    t_ = T_dt[0]
    t_step = 1

    while t_ < max_t:
        coin = np.random.random_sample(1)[0]

        if __debug__:
            print('(', t_step, ', ', t_, ')', sep='')
            print('p_sink_dt =', p_sink_dt[t_step])
        # print('t_ =', t_)
        # print('t_step =', t_step)

        if coin <= p_sink_dt[t_step]:
            T_click_trial.append(T_dt[t_step] / dt)
            click_trial['T'].append(T_dt[t_step] / dt)
            click_trial['step'].append(t_step)

            if __debug__:
                cprint(str(coin) + ' <= ' +
                       str(p_sink_dt[t_step]), color='green', attrs=['bold'])
            t_step = 0
        else:
            if __debug__:
                cprint(str(coin) + ' > ' +
                       str(p_sink_dt[t_step]), color='red', attrs=['bold'])

        t_step += 1
        t_ += dt

        if __debug__:
            print('-'*50)

    T_click.append(T_click_trial)
    click.append(click_trial)

T_clicks_total = []

N_clicks_total = 0

# print(click)
# exit(0)
for i in click:
    print(i['step'])
    N_clicks_total += len(i['step'])

    for step in i['step']:
        T_clicks_total.append(step * dt)

N_click_avg = np.sum([len(i['step']) for i in click]) / len(T_click)
# END------------------------------------------- EXPERIMENT ---------------------------------------

data_dt = Sink(p_sink_dt, T_dt)

if __debug__:
    data_dt.print()

# -------------------------------------------------------------------------------------------------
p_, t_ = GeometricDistribution(data_dt.data['P'], T)
data_theor = Sink(p_, t_)

E, E_normed = Expectation(data_theor.data['P'], data_theor.data['T'])
# -------------------------------------------------------------------------------------------------

# =================================================================================================
cprint('EXPERIMENT:', color='yellow', attrs=['bold'])

print("N_clicks_total:", N_clicks_total)
print("T_click_total_sum:", np.sum(T_clicks_total))
T_click_avg = np.sum(T_clicks_total) / N_clicks_total
print("T_click_avg:", T_click_avg)
print('\tN_click_avg_avg: ', N_click_avg, sep='')
print('\tT_click_avg_avg: ', n_dots / N_click_avg, sep='')
print('\tT_click_steps_avg: ', n_dots / dt / N_click_avg, sep='')

cprint('THEORY:', color='yellow', attrs=['bold'])

print('T_click_avg: ', E * dt)
# print('theor: ', P_M, ', ', (n_dots) / T_M, sep='')
# print('theor: ', P_M, ', ', len(T_dt), sep='')
# =================================================================================================
# print("abs_err:", abs(P_M * dt - T_click_avg))
# =================================================================================================

max_stars_repo_name: alexfmsu/pyquantum

max_issues_repo_name: alexfmsu/pyquantum

max_forks_repo_name: alexfmsu/pyquantum


text:
import sys
import numpy as np
import copy
import glob
import os
import pandas as pd

from collections import defaultdict

#write components extra
def extract_components(memberlist):
	cs =[]
	for comp in memberlist:

		if comp[0] == 'c':
			if len(comp.split('_fixed')) > 1:
				comp = comp.split('_fixed')[0]
			cs.append(comp)
	return(cs)

def extract_singletons(memberlist):
	ss = []
	for comp in memberlist:
		if comp[0] == 's':
			ss.append(comp)
	return(ss)

#from indexlist to nodes
def to_nodes(indices):
	nodes = []
	for index in indices:
		node = df.loc[[index],'node'].values[0]
		nodes.append(node)
	return(nodes)

def main():
	#assign each node ID to a component
	path = sys.argv[1]
	node_to_comp = {}
	comp_to_nodes = {}
	for filename in glob.glob(os.path.join(path, '*.gfa')):
		with open(os.path.join(os.getcwd(), filename)) as f:
			component = filename.split(path)[1].split('.gfa')[0].strip('/')
			comp_to_nodes[component] = []
			for i,line in enumerate(f):
				if line[0] == "S":
					node = line.strip().split('\t')[1]
					node_to_comp[node] = component
					comp_to_nodes[component].append(node)

	print("number of nodes assigned to a component: ", len(node_to_comp.keys()))

	#read in the graph file to get a list of all nodes
	#read in the dosage file to extract nodes with dosage 1

	dosagefile = sys.argv[2]
	gfafile = sys.argv[3]

	countfile = sys.argv[4]
	outfile = sys.argv[5]

	allnodes = []
	with open(gfafile) as gfa:
		for i,l in enumerate(gfa):
			if l[0] == 'S':
				allnodes.append(l.strip().split('\t')[1])

	haplotigs, diplotigs, triplotigs, tetraplotigs, replotigs = [],[],[],[], []
	nodes = []
	with open(dosagefile) as dosage:
		for i,line in enumerate(dosage):
			if i==0:
				continue
			else:
				node = line.strip().split(',')[0]
				nodes.append(node)

				if int(line.strip().split(',')[2]) == 1 or int(line.strip().split(',')[2]) == 0 :
					haplotigs.append(node)
				if int(line.strip().split(',')[2]) == 2:
					diplotigs.append(node)
				if int(line.strip().split(',')[2]) == 3:
					triplotigs.append(node)
				if int(line.strip().split(',')[2]) == 4:
					tetraplotigs.append(node)
				if int(line.strip().split(',')[2]) > 4:
					replotigs.append(node)
	print("allnodes: ", len(allnodes))
	#nodes that have not been assigned to a component should be singletons
	singletons = [node for node in allnodes if not node in node_to_comp.keys()]
	print("number of single nodes: ", len(singletons))
	assert(len(singletons)+len(node_to_comp) == len(allnodes))
	for node in singletons:
		node_to_comp[node] = 'single'+node
	print("haplotigs: ", len(haplotigs))
	print("diplotigs: ", len(diplotigs))
	print("triplotigs: ", len(triplotigs))
	print("tetraplotigs: ", len(tetraplotigs))
	print("replotigs: ", len(replotigs))

	#add nodes with dosage 0 if those are not in dosage file
	haplotigs.extend([node for node in allnodes if not node in nodes])

	#compute correlation
	df = pd.read_csv(countfile, sep='\t')

	df = df.dropna(axis='columns')

	norm = df.iloc[:,2:]
	norm_t = norm.T

	print("computing correlation")
	corrtable = norm_t.corr(method='spearman')
	print("correlation matrix computed")

	corr_values = defaultdict(list)

	for i in range(len(corrtable.columns)-1):
		corr_values[i] = list(corrtable.iloc[:,i])

	index_to_cluster = defaultdict(list)
	index_to_nodes = defaultdict(list)

	possible_starts = [i for i in corr_values.keys() if float(max(corrtable.iloc[:,i]))>0.5]

	clustersets = []
	for startindex in possible_starts:
		testcluster = []
		maxlist = [i for i in range(len(corr_values[startindex])) if corr_values[startindex][i]>0.5]
		firstmax_id = corr_values[startindex].index(max(corr_values[startindex]))
		firstmax = max(corr_values[startindex])
		testcluster.append(startindex)
		testcluster.extend(maxlist)
		for cur in maxlist:
			col = corr_values[cur]
			testcluster.extend([i for i in range(len(col)) if col[i]>0.5])
			testcluster = list(set(testcluster))

		index_to_cluster[startindex] =  testcluster
		clustersets.append(set(testcluster))
	grouped_sets = {}
	for cur_set in sorted(clustersets, key=len,reverse=True):
		is_subset = False
		for group, sets in grouped_sets.items():
			if not cur_set.isdisjoint(set().union(*sets)):
				sets.append(cur_set)
				is_subset = True
		if not is_subset:
			grouped_sets[tuple(cur_set)] = [cur_set]

	union_sets = {}
	all_ids = []
	for group, sets in grouped_sets.items():
		union_sets[group] = tuple(set().union(*sets))
		all_ids.extend(list(set().union(*sets)))

	#transform sets of nodes into sets of component IDs/singleton IDs
	comp_sets = {}
	for group, indices in union_sets.items():
		comps = []
		for index in indices:
			node = df.loc[[index],'node'].values[0]
			comps.append(node_to_comp[node])
		comp_sets[group] = list(set(comps))

	samesets = []
	longer_sets= []
	merged_compcluster = {}
	visited = set()
	for testgroup, testcomps in comp_sets.items():
		if testgroup not in visited:
			merged_compcluster[testgroup] = []
			same = []
			same_single = []
			cs = extract_components(testcomps)
			same.extend(cs)
			ss = extract_singletons(testcomps)
			same_single.extend(ss)
			for group, comps in comp_sets.items():
					if group != testgroup:
						new_cs = extract_components(comps)
						new_ss = extract_singletons(comps)
						if len(new_cs) > 1:
							longer_sets.append(group)
						if len(new_cs) > 0:
							intersect_frac = float(len(set(new_cs).intersection(set(cs)))/len(new_cs))
							if intersect_frac >0.5:
									same.extend(new_cs)
									visited.add(group)
						else:
							intersect_frac = float(len(set(new_ss).intersection(set(ss)))/len(new_ss))
							if intersect_frac >0.5:
									same_single.extend(new_ss)
									visited.add(group)
			merged_compcluster[testgroup] = list(set(same))+list(set(same_single))

	print("clusters: ")
	with open(outfile, 'w') as outf:
		for i,c in merged_compcluster.items():
			if len(c) > 0:
				print(sorted([int(co.split('component')[1]) for co in c if 'component' in co]), "singletons: ", [co.split('single')[1] for co in c if 'single' in co])
				print(sorted([int(co.split('component')[1]) for co in c if 'component' in co]), "singletons: ", [co.split('single')[1] for co in c if 'single' in co], file=outf)
	allcomponents = [i for key,c in merged_compcluster.items() for i in c]

#	singletongroups = [i for i in merged_compcluster.keys() if len(i) ==1]

	#compute list of nodesets for each merged group
	clusters = {}
	for group,compgroups in merged_compcluster.items():
		nodes = []
		for g in compgroups:
			if g in comp_to_nodes.keys():
				nodes.extend(comp_to_nodes[g])
			else:
				nodes.extend([g])
		clusters[group] = list(set(nodes))

	print("number of clusters: ", len(clusters))
	print([len(c) for key,c in clusters.items()])

if __name__ == '__main__':
	main()


text:
import os
import numpy as np
import pybullet as p
from .robot import Robot

class Sawyer(Robot):
    def __init__(self, controllable_joints='right'):
        right_arm_joint_indices = [3, 8, 9, 10, 11, 13, 16]  # Controllable arm joints
        left_arm_joint_indices = right_arm_joint_indices  # Controllable arm joints
        wheel_joint_indices = []
        right_end_effector = 19  # Used to get the pose of the end effector
        left_end_effector = right_end_effector  # Used to get the pose of the end effector
        right_gripper_indices = [20, 22]  # Gripper actuated joints
        left_gripper_indices = right_gripper_indices  # Gripper actuated joints
        right_tool_joint = 18  # Joint that tools are attached to
        left_tool_joint = right_tool_joint  # Joint that tools are attached to
        right_gripper_collision_indices = [18, 20, 21, 22, 23]  # Used to disable collision between gripper and tools
        left_gripper_collision_indices = right_gripper_collision_indices  # Used to disable collision between gripper and tools

        gripper_pos = {'scratch_itch': [0.015, -0.015],  # Gripper open position for holding tools
                       'feeding': [0, 0],
                       'drinking': [0.025, -0.025],
                       'bed_bathing': [0.0125, -0.0125],
                       'dressing': [0, 0],
                       'arm_manipulation': [0.01, -0.01]}
        tool_pos_offset = {'scratch_itch': [0, 0.125, 0],  # Position offset between tool and robot tool joint
                           'feeding': [-0.1, 0.12, -0.02],
                           'drinking': [0.05, 0.125, 0],
                           'bed_bathing': [0, 0.1175, 0],
                           'arm_manipulation': [0.075, 0.235, 0]}
        tool_orient_offset = {'scratch_itch': [0, 0, np.pi/2.0],  # RPY orientation offset between tool and robot tool joint
                              'feeding': [np.pi/2.0-0.1, 0, np.pi/2.0],
                              'drinking': [0, 0, np.pi/2.0],
                              'bed_bathing': [np.pi/2.0, 0, np.pi/2.0],
                              'arm_manipulation': [0, 0, np.pi/2.0]}
        toc_base_pos_offset = {'scratch_itch': [-0.1, 0, 0.975],  # Robot base offset before TOC base pose optimization
                               'feeding': [-0.1, 0.2, 0.975],
                               'drinking': [-0.1, 0.2, 0.975],
                               'bed_bathing': [-0.2, 0, 0.975],
                               'dressing': [1.8, 0.7, 0.975],
                               'arm_manipulation': [-0.3, 0.6, 0.975]}
        toc_ee_orient_rpy = {'scratch_itch': [0, np.pi/2.0, 0],  # Initial end effector orientation
                             'feeding': [np.pi/2.0, 0, np.pi/2.0],
                             'drinking': [0, -np.pi/2.0, np.pi],
                             'bed_bathing': [0, np.pi/2.0, 0],
                             'dressing': [[0, -np.pi/2.0, 0], [np.pi/2.0, -np.pi/2.0, 0]],
                             'arm_manipulation': [0, -np.pi/2.0, np.pi]}

        wheelchair_mounted = False

        super(Sawyer, self).__init__(controllable_joints, right_arm_joint_indices, left_arm_joint_indices, wheel_joint_indices, right_end_effector, left_end_effector, right_gripper_indices, left_gripper_indices, gripper_pos, right_tool_joint, left_tool_joint, tool_pos_offset, tool_orient_offset, right_gripper_collision_indices, left_gripper_collision_indices, toc_base_pos_offset, toc_ee_orient_rpy, wheelchair_mounted, half_range=False)

    def init(self, directory, id, np_random, fixed_base=True):
        self.body = p.loadURDF(os.path.join(directory, 'sawyer', 'sawyer.urdf'), useFixedBase=fixed_base, basePosition=[-1, -1, 0.975], flags=p.URDF_USE_SELF_COLLISION, physicsClientId=id)
        super(Sawyer, self).init(self.body, id, np_random)

        # Remove collisions between the various arm links for stability
        for i in range(3, 24):
            for j in range(3, 24):
                p.setCollisionFilterPair(self.body, self.body, i, j, 0, physicsClientId=id)

        for i in range(0, 3):
            for j in range(0, 9):
                p.setCollisionFilterPair(self.body, self.body, i, j, 0, physicsClientId=id)


text:
import unittest
from pathlib import Path
import numpy as np
from osd import Problem
from osd.classes import (
    MeanSquareSmall,
    TimeSmoothPeriodicEntryClose,
    LinearTrend
)
from osd.classes.wrappers import make_columns_equal

rms = lambda x: np.sqrt(np.average(np.power(x, 2)))

VERBOSE = False

class TestVectorPeriodicProblem(unittest.TestCase):
    def test_cvx(self):
        y, X_real = make_data()
        period = 17
        classes = [
            MeanSquareSmall(size=y.size),
            TimeSmoothPeriodicEntryClose(
                lambda1=1e2, lambda2=1e-2, weight=5e-3 / y.size, period=period
            ),
            make_columns_equal(LinearTrend)(first_val=0),
        ]
        problem = Problem(y, classes=classes)
        problem.decompose(how='cvx', verbose=VERBOSE)
        np.testing.assert_(np.isclose(problem.problem.value,
                                      problem.objective_value))
        np.testing.assert_(
            problem.objective_value <= 0.0167,
            'actual value: {:.3e}'.format(problem.objective_value)
        )

    def test_admm(self):
        y, X_real = make_data()
        period = 17
        classes = [
            MeanSquareSmall(size=y.size),
            TimeSmoothPeriodicEntryClose(
                lambda1=1e2, lambda2=1e-2, weight=5e-3 / y.size, period=period
            ),
            make_columns_equal(LinearTrend)(first_val=0),
        ]
        problem = Problem(y, classes=classes)
        problem.decompose(how='admm', verbose=VERBOSE)
        np.testing.assert_(
            problem.objective_value <= 0.0167,
            'actual value: {:.3e}'.format(problem.objective_value)
        )

    def test_bcd(self):
        y, X_real = make_data()
        period = 17
        classes = [
            MeanSquareSmall(size=y.size),
            TimeSmoothPeriodicEntryClose(
                lambda1=1e2, lambda2=1e-2, weight=5e-3 / y.size, period=period
            ),
            make_columns_equal(LinearTrend)(first_val=0),
        ]
        problem = Problem(y, classes=classes)
        problem.decompose(how='bcd', verbose=VERBOSE)
        np.testing.assert_(
            problem.objective_value <= 0.0167,
            'actual value: {:.3e}'.format(problem.objective_value)
        )

def make_data():
    filepath = Path(__file__).parent.parent
    data_file_path = (
            filepath
            / "fixtures"
            / "vector_smooth_periodic_low_var.txt"
    )
    with open(data_file_path) as file:
        data = np.loadtxt(file)
    T, p = data.shape
    np.random.seed(110100100)
    X1 = 0.15 * np.random.randn(T, p)
    X2 = data
    X3 = np.tile(np.linspace(0, 2, T), (p, 1)).T
    X_real = np.array([X1, X2, X3])
    y = np.sum(X_real, axis=0)
    use_set = np.random.uniform(size=(T, p)) >= 0.25
    use_set[40:50] = False
    y[~use_set] = np.nan
    return y, X_real

max_stars_repo_name: bmeyers/optimal-signal-decomposition

max_issues_repo_name: cvxgrp/signal-decomposition

max_forks_repo_name: cvxgrp/signal-decomposition


text:
import numpy as np
import numpy.typing as npt

def my_fft(x: npt.NDArray) -> npt.NDArray:
    """FFT処理をします
    Args:
        x(npt.NDArray):信号の配列

    Returns:
        npt.NDArray:FFT結果
    """

    n = x.shape[0]  # xの長さをを調べ、Nに代入
    hat_x = np.zeros(n, dtype="complex")  # 複素数型の0をN個用意(フーリエ変換後の変数)
    k = np.arange(0, n // 2)  # 0から2/n-1まで,2/N個並べた配列(n//2で整数化) バタフライ演算の過程で半分のk
    w = np.exp(-1j * 2 * np.pi * k / n)  # 回転因子

    if n == 2:
        # 2点DFTのバタフライ演算
        hat_x[0] = x[0] + x[1]
        hat_x[1] = x[0] - x[1]
        return hat_x

    if n >= 4:
        # バタフライ演算
        even = my_fft(x[0:n:2])  # 偶数
        odd = my_fft(x[1:n:2])  # 奇数

        hat_x[0 : n // 2] = even + w * odd
        hat_x[n // 2 : n] = even - w * odd
        return hat_x

    return np.zeros(n)


text:
# -*- coding: utf-8 -*-
# MegEngine is Licensed under the Apache License, Version 2.0 (the "License")
#
# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

from ctypes import *

import numpy as np

from .base import _Ctensor, _lib, _LiteCObjBase
from .struct import LiteDataType, LiteDeviceType, LiteIOType, Structure

MAX_DIM = 7

_lite_type_to_nptypes = {
    LiteDataType.LITE_INT: np.int32,
    LiteDataType.LITE_FLOAT: np.float32,
    LiteDataType.LITE_UINT8: np.uint8,
    LiteDataType.LITE_INT8: np.int8,
    LiteDataType.LITE_INT16: np.int16,
    LiteDataType.LITE_UINT16: np.uint16,
    LiteDataType.LITE_HALF: np.float16,
}

_nptype_to_lite_type = {val: key for key, val in _lite_type_to_nptypes.items()}

_str_nptypes_to_lite_nptypes = {
    np.dtype("int32"): LiteDataType.LITE_INT,
    np.dtype("float32"): LiteDataType.LITE_FLOAT,
    np.dtype("uint8"): LiteDataType.LITE_UINT8,
    np.dtype("int8"): LiteDataType.LITE_INT8,
    np.dtype("int16"): LiteDataType.LITE_INT16,
    np.dtype("uint16"): LiteDataType.LITE_UINT16,
    np.dtype("float16"): LiteDataType.LITE_HALF,
}

ctype_to_lite_dtypes = {
    c_int: LiteDataType.LITE_INT,
    c_uint: LiteDataType.LITE_INT,
    c_float: LiteDataType.LITE_FLOAT,
    c_ubyte: LiteDataType.LITE_UINT8,
    c_byte: LiteDataType.LITE_INT8,
    c_short: LiteDataType.LITE_INT16,
    c_ushort: LiteDataType.LITE_UINT16,
}

class LiteLayout(Structure):
    """
    the simple layout description
    """

    _fields_ = [
        ("shapes", c_size_t * MAX_DIM),
        ("ndim", c_size_t),
        ("data_type", c_int),
    ]

    def __init__(self, shape=None, dtype=None):
        if shape:
            shape = list(shape)
            assert len(shape) <= MAX_DIM, "Layout max dim is 7."
            self.shapes = (c_size_t * MAX_DIM)(*shape)
            self.ndim = len(shape)
        else:
            self.shapes = (c_size_t * MAX_DIM)()
            self.ndim = 0
        if not dtype:
            self.data_type = LiteDataType.LITE_FLOAT
        elif isinstance(dtype, LiteDataType):
            self.data_type = dtype
        elif type(dtype) == str:
            self.data_type = _str_nptypes_to_lite_nptypes[np.dtype(dtype)]
        elif isinstance(dtype, np.dtype):
            ctype = np.ctypeslib.as_ctypes_type(dtype)
            self.data_type = ctype_to_lite_dtypes[ctype]
        elif isinstance(dtype, type):
            self.data_type = _nptype_to_lite_type[dtype]
        else:
            raise RuntimeError("unkonw data type")

    def __repr__(self):
        data = {
            "shapes": list(self.shapes)[0 : self.ndim],
            "ndim": self.ndim,
            "data_type": _lite_type_to_nptypes[LiteDataType(self.data_type)],
        }
        return data.__repr__()

class _LiteTensorDesc(Structure):
    """
    warpper of the MegEngine Tensor

    :is_pinned_host: when set, the storage memory of the tensor is pinned memory,
    this is used to Optimize the H2D or D2H memory copy, if the device or layout
    is not set, when copy form other device(CUDA) tensor, this tensor
    will be automatically set to pinned tensor
    """

    _fields_ = [
        ("is_pinned_host", c_int),
        ("layout", LiteLayout),
        ("device_type", c_int),
        ("device_id", c_int),
    ]

    def __init__(self):
        self.layout = LiteLayout()
        self.device_type = LiteDeviceType.LITE_CPU
        self.is_pinned_host = False
        self.device_id = 0

    def __repr__(self):
        data = {
            "is_pinned_host": self.is_pinned_host,
            "layout": LiteLayout(self.layout),
            "device_type": LiteDeviceType(self.device_type.value),
            "device_id": self.device_id,
        }
        return data.__repr__()

class _TensorAPI(_LiteCObjBase):
    """
    get the api from the lib
    """

    _api_ = [
        ("LITE_make_tensor", [_LiteTensorDesc, POINTER(_Ctensor)]),
        ("LITE_set_tensor_layout", [_Ctensor, LiteLayout]),
        ("LITE_reset_tensor_memory", [_Ctensor, c_void_p, c_size_t]),
        ("LITE_reset_tensor", [_Ctensor, LiteLayout, c_void_p]),
        ("LITE_tensor_reshape", [_Ctensor, POINTER(c_int), c_int]),
        (
            "LITE_tensor_slice",
            [
                _Ctensor,
                POINTER(c_size_t),
                POINTER(c_size_t),
                POINTER(c_size_t),
                c_size_t,
                POINTER(_Ctensor),
            ],
        ),
        (
            "LITE_tensor_concat",
            [POINTER(_Ctensor), c_int, c_int, c_int, c_int, POINTER(_Ctensor),],
        ),
        ("LITE_tensor_fill_zero", [_Ctensor]),
        ("LITE_tensor_copy", [_Ctensor, _Ctensor]),
        ("LITE_tensor_share_memory_with", [_Ctensor, _Ctensor]),
        ("LITE_get_tensor_memory", [_Ctensor, POINTER(c_void_p)]),
        ("LITE_get_tensor_total_size_in_byte", [_Ctensor, POINTER(c_size_t)]),
        ("LITE_get_tensor_layout", [_Ctensor, POINTER(LiteLayout)]),
        ("LITE_get_tensor_device_type", [_Ctensor, POINTER(c_int)]),
        ("LITE_get_tensor_device_id", [_Ctensor, POINTER(c_int)]),
        ("LITE_destroy_tensor", [_Ctensor]),
        ("LITE_is_pinned_host", [_Ctensor, POINTER(c_int)]),
    ]

class LiteTensor(object):
    """
    the tensor to hold a block of data
    """

    _api = _TensorAPI()._lib

    def __init__(
        self,
        layout=None,
        device_type=LiteDeviceType.LITE_CPU,
        device_id=0,
        is_pinned_host=False,
    ):
        """
        create a Tensor with layout, device, is_pinned_host param
        """
        self._tensor = _Ctensor()
        if layout:
            self._layout = layout
        else:
            self._layout = LiteLayout()
        self._device_type = device_type
        self._device_id = device_id
        self._is_pinned_host = is_pinned_host

        tensor_desc = _LiteTensorDesc()
        tensor_desc.layout = self._layout
        tensor_desc.device_type = device_type
        tensor_desc.device_id = device_id
        tensor_desc.is_pinned_host = is_pinned_host
        self._api.LITE_make_tensor(tensor_desc, byref(self._tensor))

    def __del__(self):
        self._api.LITE_destroy_tensor(self._tensor)

    def fill_zero(self):
        """
        fill the buffer memory with zero
        """
        self._api.LITE_tensor_fill_zero(self._tensor)
        self.update()

    def share_memory_with(self, src_tensor):
        """
        share the same memory with the src_tensor, the self memory will be freed
        """
        assert isinstance(src_tensor, LiteTensor)
        self._api.LITE_tensor_share_memory_with(self._tensor, src_tensor._tensor)
        self.update()

    @property
    def layout(self):
        self._api.LITE_get_tensor_layout(self._tensor, byref(self._layout))
        return self._layout

    @layout.setter
    def layout(self, layout):
        assert isinstance(layout, LiteLayout)
        self._layout = layout
        self._api.LITE_set_tensor_layout(self._tensor, layout)

    @property
    def is_pinned_host(self):
        """
        whether the tensor is pinned tensor
        """
        pinned = c_int()
        self._api.LITE_is_pinned_host(self._tensor, byref(pinned))
        self._is_pinned_host = pinned
        return bool(self._is_pinned_host)

    @property
    def device_type(self):
        """
        get device of the tensor
        """
        device_type = c_int()
        self._api.LITE_get_tensor_device_type(self._tensor, byref(device_type))
        self._device_type = device_type
        return LiteDeviceType(device_type.value)

    @property
    def device_id(self):
        """
        get device id of the tensor
        """
        device_id = c_int()
        self._api.LITE_get_tensor_device_id(self._tensor, byref(device_id))
        self._device_id = device_id.value
        return device_id.value

    @property
    def is_continue(self):
        """
        whether the tensor memory is continue
        """
        is_continue = c_int()
        self._api.LITE_is_memory_continue(self._tensor, byref(is_continue))
        return bool(is_continue.value)

    @property
    def nbytes(self):
        """
        get the length of the meomry in byte
        """
        self.update()
        length = c_size_t()
        self._api.LITE_get_tensor_total_size_in_byte(self._tensor, byref(length))
        return length.value

    def update(self):
        """
        update the member from C, this will auto used after slice, share
        """
        pinned = c_int()
        self._api.LITE_is_pinned_host(self._tensor, byref(pinned))
        self._is_pinned_host = pinned
        device_type = c_int()
        self._api.LITE_get_tensor_device_type(self._tensor, byref(device_type))
        self._device_type = device_type
        self._api.LITE_get_tensor_layout(self._tensor, byref(self._layout))

    def copy_from(self, src_tensor):
        """
        copy memory form the src_tensor
        """
        assert isinstance(src_tensor, LiteTensor)
        self._api.LITE_tensor_copy(self._tensor, src_tensor._tensor)
        self.update()

    def reshape(self, shape):
        """
        reshape the tensor with data not change, only change the shape
        :param shape: int arrary of dst_shape
        """
        shape = list(shape)
        length = len(shape)
        c_shape = (c_int * length)(*shape)
        self._api.LITE_tensor_reshape(self._tensor, c_shape, length)
        self.update()

    def slice(self, start, end, step=None):
        """
        slice the tensor with gaven start, end, step
        :param start: silce begin index of each dim
        :param end: silce end index of each dim
        :param step: silce step of each dim
        """
        start = list(start)
        end = list(end)
        length = len(start)
        assert length == len(end), "slice with different length of start and end."
        if step:
            assert length == len(step), "slice with different length of start and step."
            step = list(step)
        else:
            step = [1 for i in range(length)]
        c_start = (c_size_t * length)(*start)
        c_end = (c_size_t * length)(*end)
        c_step = (c_size_t * length)(*step)
        slice_tensor = LiteTensor()
        self._api.LITE_tensor_slice(
            self._tensor, c_start, c_end, c_step, length, byref(slice_tensor._tensor)
        )
        slice_tensor.update()
        return slice_tensor

    def get_ctypes_memory(self):
        """
        get the memory of the tensor, return c_void_p of the tensor memory
        """
        self.update()
        mem = c_void_p()
        self._api.LITE_get_tensor_memory(self._tensor, byref(mem))
        return mem

    def set_data_by_share(self, data, length=0, layout=None):
        """
        share the data to the tensor
        param data: the data will shared to the tensor, it should be a
        numpy.ndarray or ctypes data
        """
        self.update()
        if isinstance(data, np.ndarray):
            assert (
                self.is_continue
            ), "set_data_by_share can only apply in continue tensor."
            assert (
                self.is_pinned_host or self.device_type == LiteDeviceType.LITE_CPU
            ), "set_data_by_share can only apply in cpu tensor or pinned tensor."

            np_type = _lite_type_to_nptypes[LiteDataType(self._layout.data_type)]
            c_type = np.ctypeslib.as_ctypes_type(np_type)

            if self.nbytes != data.nbytes:
                self.layout = LiteLayout(data.shape, ctype_to_lite_dtypes[c_type])

            self._shared_data = data
            data = data.ctypes.data_as(POINTER(c_type))

        if layout is not None:
            self.layout = layout
        else:
            assert length == 0 or length == self.nbytes, "the data length is not match."
        self._api.LITE_reset_tensor_memory(self._tensor, data, self.nbytes)

    def set_data_by_copy(self, data, data_length=0, layout=None):
        """
        copy the data to the tensor
        param data: the data to copy to tensor, it should be list,
        numpy.ndarraya or ctypes with length
        """
        self.update()
        if layout is not None:
            self.layout = layout

        assert self.is_continue, "set_data_by_copy can only apply in continue tensor."
        assert (
            self.is_pinned_host or self.device_type == LiteDeviceType.LITE_CPU
        ), "set_data_by_copy can only apply in cpu tensor or pinned tensor."

        np_type = _lite_type_to_nptypes[LiteDataType(self._layout.data_type)]
        c_type = np.ctypeslib.as_ctypes_type(np_type)

        tensor_memory = c_void_p()

        if type(data) == list:
            length = len(data)
            self._api.LITE_get_tensor_memory(self._tensor, byref(tensor_memory))
            tensor_length = self.nbytes
            assert (
                length * sizeof(c_type) <= tensor_length
            ), "the length of input data to set to the tensor is too large."
            arr = (c_type * length)(*data)
            memmove(tensor_memory, arr, sizeof(c_type) * length)

        elif type(data) == np.ndarray:
            if self.nbytes != data.nbytes:
                self.layout = LiteLayout(data.shape, data.dtype)
            arr = data.ctypes.data_as(POINTER(c_type))
            self._api.LITE_get_tensor_memory(self._tensor, byref(tensor_memory))
            assert self.nbytes == data.nbytes
            memmove(tensor_memory, arr, self.nbytes)
        else:
            assert (
                data_length == self.nbytes or layout is not None
            ), "when input data is ctypes, the length of input data or layout must set"
            self._api.LITE_get_tensor_memory(self._tensor, byref(tensor_memory))
            memmove(tensor_memory, data, data_length)

    def to_numpy(self):
        """
        get the buffer of the tensor
        """
        self.update()
        if self.nbytes <= 0:
            return np.array([])
        if self.is_continue and (
            self.is_pinned_host or self.device_type == LiteDeviceType.LITE_CPU
        ):
            ptr = c_void_p()
            self._api.LITE_get_tensor_memory(self._tensor, byref(ptr))

            np_type = _lite_type_to_nptypes[LiteDataType(self._layout.data_type)]
            shape = [self._layout.shapes[i] for i in range(self._layout.ndim)]
            np_arr = np.zeros(shape, np_type)
            if np_arr.nbytes:
                memmove(np_arr.ctypes.data_as(c_void_p), ptr, np_arr.nbytes)
            return np_arr
        else:
            tmp_tensor = LiteTensor(self.layout)
            tmp_tensor.copy_from(self)
            return tmp_tensor.to_numpy()

    def __repr__(self):
        self.update()
        data = {
            "layout": self._layout,
            "device_type": LiteDeviceType(self._device_type.value),
            "device_id": int(self.device_id),
            "is_pinned_host": bool(self._is_pinned_host),
        }
        return data.__repr__()

def LiteTensorConcat(
    tensors, dim, device_type=LiteDeviceType.LITE_DEVICE_DEFAULT, device_id=-1
):
    """
    concat tensor in input dim to one tensor
    dim : the dim to act concat
    device_type: the result tensor device type
    device_id: the result tensor device id
    """
    api = _TensorAPI()._lib
    length = len(tensors)
    c_tensors = [t._tensor for t in tensors]
    c_tensors = (_Ctensor * length)(*c_tensors)
    result_tensor = LiteTensor()
    api.LITE_tensor_concat(
        cast(byref(c_tensors), POINTER(c_void_p)),
        length,
        dim,
        device_type,
        device_id,
        byref(result_tensor._tensor),
    )
    result_tensor.update()
    return result_tensor

max_stars_repo_name: gedoensmax/MegEngine

max_issues_repo_name: gedoensmax/MegEngine

max_forks_repo_name: gedoensmax/MegEngine


text:
import eel
from geneticalgorithm import geneticalgorithm as ga
import numpy as np
import csv
import os
from io import StringIO
eel.init("web")
# Student class

class Student:
    def __init__(self, name, gpa, skill, time):
        self.name = name
        self.gpa = gpa  # Range [0,4]
        # 1 Writing, 2 = Leadership, 3 = Problem Solving, 4 = Creativity, 5 = Planning
        self.skill = skill
        self.time = time  # 1 = Morning, 2 = Afternoon, 3 = Night

    def __repr__(self):
        return 'Name: {},       GPA: {:0.2f},       Skill: {},      Time: {}'.format(self.name, self.gpa, self.skill, self.time)

    def __str__(self):
        return 'Name: {},       GPA: {:0.2f},       Skill: {},      Time: {}'.format(self.name, self.gpa, self.skill, self.time)

###################################################################################################
students = []
teamSize = 0
resultsString = ''

# Fitness function

def f(x):  # lower fitness value is better
    numOfTeams = round(len(x)/teamSize)
    # Create teams container
    teams = []
    for i in range(numOfTeams):
        teams.append([])

    for i in range(len(x)):  # Add each student to his related team
        teams[(int)(x[i])-1].append(students[i])

    # Constraint to check if team sizes are not equal
    if len(x) % teamSize == 0:
        for i in range(numOfTeams):
            if teamSize != len(teams[i]):
                return 500

    gpaAverage = []  # List of gpa average for each team
    timeConflicts = []  # List of time conflicts between team members for each team
    skillRepeats = []  # List of number of repeated skills for each team
    teamsSizes = []

    for team in teams:
        if len(team) == 0:
            return 500
        teamsSizes.append(len(team))

        # Calculate gpa average for each team
        avg = sum(student.gpa for student in team)/len(team)
        gpaAverage.append(avg)

        # Calculate time conflicts for each team
        conflict = 0
        for i in range(len(team)-1):

            j = i+1
            while j < len(team):
                if team[i].time != team[j].time:
                    conflict += 1
                j += 1
        timeConflicts.append(conflict)

        # Calculate number of repeated skills for each team
        repeat = 0
        for i in range(len(team)-1):
            j = i+1
            while j < len(team):
                if team[i].skill == team[j].skill:
                    repeat += 1
                j += 1
        skillRepeats.append(repeat)

    # All fitness values for each feature is in range [0,100]
    # Calcualte fitness based on GPA
    gpaFitnessValue = np.std(gpaAverage)/np.average(gpaAverage)*100.0

    # Calcualte fitness based on Time conflicts
    maxConflicts = ((teamSize*(teamSize-1))/2)*numOfTeams
    timeFitnessValue = sum(timeConflicts)/maxConflicts*100.0

    # Calcualte fitness based on repeated skills
    maxRepeates = ((teamSize*(teamSize-1))/2)*numOfTeams
    skillFitnessValue = sum(skillRepeats)/maxRepeates*100.0

    sizeFitnessValue = np.std(teamsSizes)/np.average(teamsSizes)*500.0

    return gpaFitnessValue + timeFitnessValue + skillFitnessValue + sizeFitnessValue
###################################################################################################

teams = []

@eel.expose
def generateTeams(studentsData, teamS):
    global teamSize
    teamSize = int(teamS)
    numOfTeams = round(len(studentsData)/teamSize)

    # Create students objects
    for i in range(len(studentsData)):
        students.append(Student(studentsData[i][0], float(
            studentsData[i][1]), studentsData[i][2], studentsData[i][3]))

    varbound = np.array([[1, numOfTeams]] * len(studentsData))
    algorithm_param = {'max_num_iteration': 500,
                       'population_size': 250,
                       'mutation_probability': 0.1,
                       'elit_ratio': 0.01,
                       'crossover_probability': 0.5,
                       'parents_portion': 0.3,
                       'crossover_type': 'uniform',
                       'max_iteration_without_improv': 250}

    model = ga(function=f, dimension=len(studentsData), variable_type='int',
               variable_boundaries=varbound, algorithm_parameters=algorithm_param)
    model.run()

    global teams
    solution = model.best_variable

    # Add grouping results to string
    global resultsString
    for i in range(numOfTeams):
        resultsString += 'Team {}:\n'.format(i+1)
        for j in range(len(solution)):
            if solution[j] == (i+1):
                resultsString += students[j].name + "\n"
        resultsString += "\n"
    # Add students names according to their teams
    for i in range(numOfTeams):
        teams.append([])
        for j in range(len(solution)):
            if solution[j] == (i+1):
                teams[i].append(str(students[j]))

@eel.expose
def readFile(text, size):
    fi = StringIO(text)
    reader = csv.reader(fi, delimiter=',')
    data = list(reader)
    generateTeams(data[1:], int(size))

@eel.expose
def exportData():
    f = open("Generated-Teams.txt", "w", encoding='utf-8')
    f.write(resultsString)
    f.close

@eel.expose
def getTeams():
    global teams
    return teams

eel.start("index.html", size=(1350, 800))

max_stars_repo_name: bamukhier/teams-former

max_issues_repo_name: bamukhier/teams-former

max_forks_repo_name: bamukhier/teams-former


text:
# Copyright(c) 2020 Jake Fowler
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

import pandas as pd
import numpy as np
from datetime import datetime
import ctypes
import clr
import System as dotnet
import System.Collections.Generic as dotnet_cols_gen
from pathlib import Path

clr.AddReference(str(Path("cmdty_storage/lib/Cmdty.TimePeriodValueTypes")))
import Cmdty.TimePeriodValueTypes as net_tp

clr.AddReference(str(Path('cmdty_storage/lib/Cmdty.TimeSeries')))
import Cmdty.TimeSeries as ts

clr.AddReference(str(Path('cmdty_storage/lib/Cmdty.Storage')))
import Cmdty.Storage as net_cs

from datetime import date, datetime
import dateutil
import typing as tp
import re

def from_datetime_like(datetime_like: tp.Union[datetime, date, str, pd.Period], time_period_type):
    """ Converts either a pandas Period, str, datetime or date to a .NET Time Period"""
    date_time = py_date_like_to_net_datetime(datetime_like)
    return net_tp.TimePeriodFactory.FromDateTime[time_period_type](date_time)

def py_date_like_to_net_datetime(datetime_like: tp.Union[datetime, date, str, pd.Period]):
    """Converts either a pandas Period, str, datetime or date to a .NET DateTime."""
    if isinstance(datetime_like, str):
        datetime_like = dateutil.parser.parse(datetime_like)
    if hasattr(datetime_like, 'hour'):
        time_args = (datetime_like.hour, datetime_like.minute, datetime_like.second)
    else:
        time_args = (0, 0, 0)
    return dotnet.DateTime(datetime_like.year, datetime_like.month, datetime_like.day, *time_args)

def net_datetime_to_py_datetime(net_datetime):
    return datetime(net_datetime.Year, net_datetime.Month, net_datetime.Day, net_datetime.Hour, net_datetime.Minute,
                    net_datetime.Second, net_datetime.Millisecond * 1000)

def net_time_period_to_pandas_period(net_time_period, freq):
    start_datetime = net_datetime_to_py_datetime(net_time_period.Start)
    return pd.Period(start_datetime, freq=freq)

def series_to_double_time_series(series, time_period_type):
    """Converts an instance of pandas Series to a Cmdty.TimeSeries.TimeSeries type with Double data type."""
    return series_to_time_series(series, time_period_type, dotnet.Double, lambda x: x)

def series_to_time_series(series, time_period_type, net_data_type, data_selector):
    """Converts an instance of pandas Series to a Cmdty.TimeSeries.TimeSeries."""
    series_len = len(series)
    net_indices = dotnet.Array.CreateInstance(time_period_type, series_len)
    net_values = dotnet.Array.CreateInstance(net_data_type, series_len)

    for i in range(series_len):
        net_indices[i] = from_datetime_like(series.index[i], time_period_type)
        net_values[i] = data_selector(series.values[i])

    return ts.TimeSeries[time_period_type, net_data_type](net_indices, net_values)

def net_time_series_to_pandas_series(net_time_series, freq):
    """Converts an instance of class Cmdty.TimeSeries.TimeSeries to a pandas Series"""
    if net_time_series.IsEmpty:
        return pd.Series()
    curve_start = net_time_series.Indices[0].Start
    curve_start_datetime = net_datetime_to_py_datetime(curve_start)
    index = pd.period_range(start=curve_start_datetime, freq=freq, periods=net_time_series.Count)
    prices = [net_time_series.Data[idx] for idx in range(0, net_time_series.Count)]
    return pd.Series(prices, index)

def is_scalar(arg):
    return isinstance(arg, int) or isinstance(arg, float)

def raise_if_none(arg, error_message):
    if arg is None:
        raise ValueError(error_message)

def raise_if_not_none(arg, error_message):
    if arg is not None:
        raise ValueError(error_message)

FREQ_TO_PERIOD_TYPE = {
    "15min": net_tp.QuarterHour,
    "30min": net_tp.HalfHour,
    "H": net_tp.Hour,
    "D": net_tp.Day,
    "M": net_tp.Month,
    "Q": net_tp.Quarter
}
""" dict of str: .NET time period type.
Each item describes an allowable granularity of curves constructed, as specified by the
freq parameter in the curves public methods.

The keys represent the pandas Offset Alias which describe the granularity, and will generally be used
    as the freq of the pandas Series objects returned by the curve construction methods.
The values are the associated .NET time period types used in behind-the-scenes calculations.
"""

def wrap_settle_for_dotnet(py_settle_func, freq):
    def wrapper_settle_function(py_function, net_time_period, freq):
        pandas_period = net_time_period_to_pandas_period(net_time_period, freq)
        py_function_result = py_function(pandas_period)
        net_settle_day = from_datetime_like(py_function_result, net_tp.Day)
        return net_settle_day

    def wrapped_function(net_time_period):
        return wrapper_settle_function(py_settle_func, net_time_period, freq)

    time_period_type = FREQ_TO_PERIOD_TYPE[freq]
    return dotnet.Func[time_period_type, net_tp.Day](wrapped_function)

def wrap_on_progress_for_dotnet(py_on_progress):
    if py_on_progress is None:
        return None
    return dotnet.Action[dotnet.Double](py_on_progress)

# TODO get rid of TimePeriodSpecType or ForwardPointType?
# TODO check that each type definition is correct still
TimePeriodSpecType = tp.Union[str, datetime, date, pd.Period]
ForwardPointType = tp.Union[str, date, datetime, pd.Period]
CurveType = tp.Union[pd.Series, tp.Dict[ForwardPointType, float]]
TimeFunctionType = tp.Callable[[tp.Union[date, datetime], tp.Union[date, datetime]], float]
FwdContractType = tp.Union[date, datetime, pd.Period, float,
                           tp.Tuple[date, date], tp.Tuple[datetime, datetime],
                           tp.Tuple[pd.Period, pd.Period]]
FwdContractsType = tp.Iterable[FwdContractType]

def curve_to_net_dict(curve: CurveType, time_period_type):
    """Creates a .NET Dictionary<T, Double> instance from a Python curve, with type defined by CurveType."""
    ret = dotnet_cols_gen.Dictionary[time_period_type, dotnet.Double]()
    for key in curve.keys():
        net_key = from_datetime_like(key, time_period_type)
        ret.Add(net_key, curve[key])
    return ret

_MAP_NP_NET = {
    np.dtype('float32'): dotnet.Single,
    np.dtype('float64'): dotnet.Double,
    np.dtype('int8'): dotnet.SByte,
    np.dtype('int16'): dotnet.Int16,
    np.dtype('int32'): dotnet.Int32,
    np.dtype('int64'): dotnet.Int64,
    np.dtype('uint8'): dotnet.Byte,
    np.dtype('uint16'): dotnet.UInt16,
    np.dtype('uint32'): dotnet.UInt32,
    np.dtype('uint64'): dotnet.UInt64,
    np.dtype('bool'): dotnet.Boolean,
}

def as_net_array(np_array: np.ndarray):
    """
    Given a `numpy.ndarray` returns a CLR `System.Array`.  See _MAP_NP_NET for
    the mapping of Numpy dtypes to CLR types.

    Note: `complex64` and `complex128` arrays are converted to `float32`
    and `float64` arrays respectively with shape [m,n,...] -> [m,n,...,2]
    """
    dims = np_array.shape
    dtype = np_array.dtype
    # For complex arrays, we must make a view of the array as its corresponding
    # float type.
    if dtype == np.complex64:
        dtype = np.dtype('float32')
        dims.append(2)
        np_array = np_array.view(np.float32).reshape(dims)
    elif dtype == np.complex128:
        dtype = np.dtype('float64')
        dims.append(2)
        np_array = np_array.view(np.float64).reshape(dims)

    net_dims = dotnet.Array.CreateInstance(dotnet.Int32, np_array.ndim)
    for idx in range(np_array.ndim):
        net_dims[idx] = dotnet.Int32(dims[idx])

    if not np_array.flags.c_contiguous:
        np_array = np_array.copy(order='C')
    assert np_array.flags.c_contiguous

    try:
        net_array = dotnet.Array.CreateInstance(_MAP_NP_NET[dtype], *net_dims)
    except KeyError:
        raise NotImplementedError("asNetArray does not yet support dtype {}".format(dtype))

    try:  # Memmove
        dest_handle = dotnet.Runtime.InteropServices.GCHandle.Alloc(net_array,
                                                                    dotnet.Runtime.InteropServices.GCHandleType.Pinned)
        source_ptr = np_array.__array_interface__['data'][0]
        dest_ptr = dest_handle.AddrOfPinnedObject().ToInt64()
        ctypes.memmove(dest_ptr, source_ptr, np_array.nbytes)
    finally:
        if dest_handle.IsAllocated:
            dest_handle.Free()
    return net_array

_MAP_NET_NP = {
    'Single': np.dtype('float32'),
    'Double': np.dtype('float64'),
    'SByte': np.dtype('int8'),
    'Int16': np.dtype('int16'),
    'Int32': np.dtype('int32'),
    'Int64': np.dtype('int64'),
    'Byte': np.dtype('uint8'),
    'UInt16': np.dtype('uint16'),
    'UInt32': np.dtype('uint32'),
    'UInt64': np.dtype('uint64'),
    'Boolean': np.dtype('bool'),
}

def as_numpy_array(net_array) -> np.ndarray:
    """
    Given a CLR `System.Array` returns a `numpy.ndarray`.  See _MAP_NET_NP for
    the mapping of CLR types to Numpy dtypes.
    """
    dims = np.empty(net_array.Rank, dtype=int)
    for idx in range(net_array.Rank):
        dims[idx] = net_array.GetLength(idx)
    net_type = net_array.GetType().GetElementType().Name

    try:
        np_array = np.empty(dims, order='C', dtype=_MAP_NET_NP[net_type])
    except KeyError:
        raise NotImplementedError("asNumpyArray does not yet support System type {}".format(net_type))

    try:  # Memmove
        source_handle = dotnet.Runtime.InteropServices.GCHandle.Alloc(
            net_array, dotnet.Runtime.InteropServices.GCHandleType.Pinned)
        source_ptr = source_handle.AddrOfPinnedObject().ToInt64()
        dest_ptr = np_array.__array_interface__['data'][0]
        ctypes.memmove(dest_ptr, source_ptr, np_array.nbytes)
    finally:
        if source_handle.IsAllocated:
            source_handle.Free()
    return np_array

def to_period_range(freq: str,
                    fwd_contract: FwdContractType) -> tp.Tuple[pd.Period, pd.Period]:
    if isinstance(fwd_contract, pd.Period):
        return fwd_contract.asfreq(freq, 's'), _last_period(fwd_contract, freq)
    if isinstance(fwd_contract, tuple):
        start = fwd_contract[0]
        end = fwd_contract[1]
    else:
        start = fwd_contract
        end = fwd_contract
    if isinstance(start, pd.Period):
        start_period = start.asfreq(freq, 's')
    else:
        start_period = pd.Period(start, freq=freq)
    if isinstance(end, pd.Period):
        end_period = _last_period(end, freq)
    else:
        end_period = pd.Period(end, freq=freq)
    return start_period, end_period

def _last_period(period: pd.Period, freq: str) -> pd.Period:
    """Find the last pandas Period instance of a specific frequency within a Period instance"""
    if not freq[0].isdigit():
        return period.asfreq(freq, 'e')
    m = re.match("(\d+)(\w+)", freq)
    num = int(m.group(1))
    sub_freq = m.group(2)
    return (period.asfreq(sub_freq, 'e') - num + 1).asfreq(freq)

def numerics_provider() -> str:
    return net_cs.StorageHelper.LinearAlgebraProvider()

def net_panel_to_data_frame(net_panel, freq: str) -> pd.DataFrame:
    np_array = as_numpy_array(net_panel.RawData)
    np_array.resize((net_panel.NumRows, net_panel.NumCols))
    sim_periods = [net_time_period_to_pandas_period(p, freq) for p in net_panel.RowKeys]
    period_index = pd.PeriodIndex(data=sim_periods, freq=freq)
    return pd.DataFrame(data=np_array, index=period_index)

def create_net_log_adapter(logger, net_logger_type):
    def log(level, msg):
        logger.log(level, msg)

    is_enabled = dotnet.Func[dotnet.Int32, dotnet.Boolean](lambda level: logger.isEnabledFor(level))
    py_log = dotnet.Action[dotnet.Int32, dotnet.String](log)
    return net_cs.PythonHelpers.PythonLoggerAdapter[net_logger_type](is_enabled, py_log)

max_stars_repo_name: eliot-tabet/storage

max_issues_repo_name: eliot-tabet/storage

max_forks_repo_name: eliot-tabet/storage


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Dec  4 15:00:17 2020

@author: kahg8
"""

import os
# Helper libraries
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

from tensorflow.keras.models import load_model

from preprocessing import get_test_data

PATH_LIST = 'C:/Users/kahg8/Documents/GitHub/speech_recognition/lists/'
PATH_DATA = 'C:/Users/kahg8/Documents/GitHub/speech_recognition/data/'
PATH_MODELS= 'C:/Users/kahg8/Documents/GitHub/speech_recognition/models/'

############################
test_size = 'all'
use_raw = False

use_mfcc = True

use_ssc = True

add_silence = True

add_noise = False

data_augmentation = False
labels = ["yes", "no", "up", "down", "left",
"right", "on", "off", "stop", "go", "zero", "one", "two", "three", "four",
"five", "six", "seven", "eight", "nine","silence","unknown"]
unknown_labels = ["bed", "bird", "cat", "dog", "happy", "house", "marvin", "sheila",
"tree","wow"]

def transform(l):
    n = len(labels[l])
    str_l =''
    max_n = 7
    to_add = max_n + 2 - n
    middle = to_add // 2
    for i in range(to_add):
        if not i == middle:
            str_l += ' '
        else:
            str_l += labels[l]
    return str_l

def cust_round(n):
    str_n = str(round(n,2))
    ret_s = str(round(n,2))
    if len(str_n) == 3:
        ret_s += "0"
    elif len(str_n) == 1:
        ret_s +=".00"
    return ret_s

def metrics(tp,fp,tn,fn):
    recall = tp/(tp+fn)
    if fp + fn != 0:
        fp_rate = fp/(fp+tn)
    else:
        fp_rate = 0
    precision = tp/(tp+fp)
    accuracy = (tp+tn)/(tp+tn+fp+fn)
    f1 = 2/((1/recall)+(1/precision))
    return recall,fp_rate,precision,accuracy,f1

def heatmap(data_mfcc, row_labels, col_labels, ax=None,
            cbar_kw={}, cbarlabel="", **kwargs):
    """
    Create a heatmap from a numpy array and two lists of labels.

    Parameters
    ----------
    data
        A 2D numpy array of shape (N, M).
    row_labels
        A list or array of length N with the labels for the rows.
    col_labels
        A list or array of length M with the labels for the columns.
    ax
        A `matplotlib.axes.Axes` instance to which the heatmap is plotted.  If
        not provided, use current axes or create a new one.  Optional.
    cbar_kw
        A dictionary with arguments to `matplotlib.Figure.colorbar`.  Optional.
    cbarlabel
        The label for the colorbar.  Optional.
    **kwargs
        All other arguments are forwarded to `imshow`.
    """

    fig, ax = plt.subplots()

    # Plot the heatmap
    im1 = ax.imshow(data_mfcc, **kwargs)

    # Create colorbar
    cbar = ax.figure.colorbar(im1, ax=ax, **cbar_kw)
    cbar.ax.set_ylabel(cbarlabel+ '_MFCC', rotation=-90, va="bottom")

    # We want to show all ticks...
    ax.set_xticks(np.arange(data_mfcc.shape[1]))
    ax.set_yticks(np.arange(data_mfcc.shape[0]))
    # ... and label them with the respective list entries.
    ax.set_xticklabels(col_labels)
    ax.set_yticklabels(row_labels)

    # Let the horizontal axes labeling appear on top.
    ax.tick_params(top=True, bottom=False,
                   labeltop=True, labelbottom=False)

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=-30, ha="right",
             rotation_mode="anchor")

    # Turn spines off and create white grid.
    for edge, spine in ax.spines.items():
        spine.set_visible(False)

    ax.set_xticks(np.arange(data_mfcc.shape[1]+1)-.5, minor=True)
    ax.set_yticks(np.arange(data_mfcc.shape[0]+1)-.5, minor=True)
    ax.grid(which="minor", color="w", linestyle='-', linewidth=3)
    ax.tick_params(which="minor", bottom=False, left=False)
    return im1

def annotate_heatmap(im, data=None, valfmt="{x:.2f}",
                     textcolors=["black", "white"],
                     threshold=None, **textkw):
    """
    A function to annotate a heatmap.

    Parameters
    ----------
    im
        The AxesImage to be labeled.
    data
        Data used to annotate.  If None, the image's data is used.  Optional.
    valfmt
        The format of the annotations inside the heatmap.  This should either
        use the string format method, e.g. "$ {x:.2f}", or be a
        `matplotlib.ticker.Formatter`.  Optional.
    textcolors
        A list or array of two color specifications.  The first is used for
        values below a threshold, the second for those above.  Optional.
    threshold
        Value in data units according to which the colors from textcolors are
        applied.  If None (the default) uses the middle of the colormap as
        separation.  Optional.
    **kwargs
        All other arguments are forwarded to each call to `text` used to create
        the text labels.
    """

    if not isinstance(data, (list, np.ndarray)):
        data = im.get_array()

    # Normalize the threshold to the images color range.
    if threshold is not None:
        threshold = im.norm(threshold)
    else:
        threshold = im.norm(data.max())/2.

    # Set default alignment to center, but allow it to be
    # overwritten by textkw.
    kw = dict(horizontalalignment="center",
              verticalalignment="center")
    kw.update(textkw)

    # Get the formatter in case a string is supplied
    if isinstance(valfmt, str):
        valfmt = matplotlib.ticker.StrMethodFormatter(valfmt)

    # Loop over the data and create a `Text` for each "pixel".
    # Change the text's color depending on the data.
    texts = []
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])
            text = im.axes.text(j, i, valfmt(data[i, j], None), **kw)
            texts.append(text)

    return texts

def hist(data_mfcc,data_ssc,list_label,name):

    x = np.arange(len(list_label))  # the label locations
    width = 0.35  # the width of the bars

    fig, ax = plt.subplots()
    rects1 = ax.bar(x- width/2 , data_mfcc, width, label=name + ' MFCC')
    rects2 = ax.bar(x+ width/2 , data_ssc, width, label=name + ' SSC')

    # Add some text for labels, title and custom x-axis tick labels, etc.
    ax.set_ylabel('labels')
    ax.set_title(name + ' by label')
    ax.set_xticks(x)
    ax.set_xticklabels(list_label)
    ax.legend()

    fig.tight_layout()

    plt.show()

############################
        # MAIN
############################

test_data ,test_labels  = get_test_data(labels, unknown_labels,test_size, use_raw, use_mfcc, use_ssc, add_silence, data_augmentation,add_noise)
print(test_data['mfcc'].shape)
print(test_data['ssc'].shape)
directory_path = PATH_MODELS
for file in os.listdir(directory_path):
    if not '.h5' in file:
        continue
    print('Loading model :',file)

    model_name = 'small_cnn_mfcc_60epochs_50batchsize.h5'
    model = load_model(directory_path+'/bests_silence0/' + model_name)
    model_name_ssc = 'small_cnn_ssc_80epochs_50batchsize.h5'
    model_ssc = load_model(directory_path+'/bests_silence0/' + model_name_ssc)

    true_positive = {}
    false_positive = {}
    true_negative = {}
    false_negative = {}
    confusion_matrix = {}

    true_positive_ssc = {}
    false_positive_ssc = {}
    true_negative_ssc = {}
    false_negative_ssc = {}
    confusion_matrix_ssc = {}

    data_type = 'raw'
    if 'mfcc' in model_name:
        data_type = 'mfcc'
    if 'ssc' in model_name:
        data_type = 'ssc'

    somme_test = {}
    count = 0
    success = 0

    success_ssc = 0
    for label in range(0,len(labels)):
        true_positive[label] = 0
        false_positive[label] = 0
        true_negative[label] = 0
        false_negative[label] = 0
        confusion_matrix[label] = {}
        true_positive_ssc[label] = 0
        false_positive_ssc[label] = 0
        true_negative_ssc[label] = 0
        false_negative_ssc[label] = 0
        confusion_matrix_ssc[label] = {}
        for l in range(0,len(labels)):
            confusion_matrix[label][l] = 0
            confusion_matrix_ssc[label][l] = 0
    for i in range(len(test_data['mfcc'])):
        test_mfcc = test_data['mfcc'][i]
        test_ssc = test_data['ssc'][i]
        count +=1
        # test2 = test_data2[i]
        # test3 = test_data3[i]
        #inputs = preprocess_live_data(test,16000)
        # prediction_mfcc = model.predict(np.array([test_mfcc]).reshape(-1,98,13,1))
        # prediction_ssc = model_ssc.predict(np.array([test_ssc]).reshape(-1,98,26,1))
        prediction_mfcc = model.predict(np.array([test_mfcc]))
        prediction_ssc = model_ssc.predict(np.array([test_ssc]))

        predicted_label_mfcc = np.where(prediction_mfcc == np.amax(prediction_mfcc))
        predicted_label_ssc = np.where(prediction_ssc == np.amax(prediction_ssc))
        true_label = np.where(test_labels['mfcc'][i] == np.amax(test_labels['mfcc'][i]))

        if predicted_label_mfcc[1][0] == true_label[0][0]:

            true_positive[predicted_label_mfcc[1][0]] += 1
            success +=1
            for label in range(0,len(labels)):
                if not label == predicted_label_mfcc[1][0]:
                    true_negative[label] += 1
        else:

            false_positive[predicted_label_mfcc[1][0]] += 1
            false_negative[true_label[0][0]] += 1
            confusion_matrix[predicted_label_mfcc[1][0]][true_label[0][0]] += 1

        if predicted_label_ssc[1][0] == true_label[0][0]:

            true_positive_ssc[predicted_label_ssc[1][0]] += 1
            success_ssc +=1
            for label in range(0,len(labels)):
                if not label == predicted_label_ssc[1][0]:
                    true_negative_ssc[label] += 1
        else:

            false_positive_ssc[predicted_label_ssc[1][0]] += 1
            false_negative_ssc[true_label[0][0]] += 1
            confusion_matrix_ssc[predicted_label_ssc[1][0]][true_label[0][0]] += 1

        #print('For label {} : \n Recall is {} % \n False positive rate is : {} % \n Precision is : {} % \n Accuracy is : {} % '.format(label,r,f,p,a))

    print('Success rate MFCC : {} %'.format( 100*success/count))
    print('Success rate SSC : {} %'.format( 100*success_ssc/count))
    matrix = []
    matrix_ssc = []
    unchanged = []
    unchanged_ssc = []
    for k1 , v1 in confusion_matrix.items():
        line = []
        line_unchanged = []
        somme = 0
        for k2 , v2 in v1.items():
            if false_positive[k1] != 0:
                line.append(100*v2/(false_positive[k1]))
                line_unchanged.append(int(v2))
            else:
                line.append(0)
                line_unchanged.append(int(0))
        matrix.append(line)
        unchanged.append(line_unchanged)
    for k1 , v1 in confusion_matrix_ssc.items():
        line = []
        line_unchanged = []
        somme = 0
        for k2 , v2 in v1.items():
            if false_positive_ssc[k1] != 0:
                line.append(100*v2/(false_positive_ssc[k1]))
                line_unchanged.append(int(v2))
            else:
                line.append(0)
                line_unchanged.append(int(0))
        matrix_ssc.append(line)
        unchanged_ssc.append(line_unchanged)

    R = []
    FPR = []
    P = []
    A = []
    F1 = []

    R_ssc = []
    FPR_ssc = []
    P_ssc = []
    A_ssc = []
    F1_ssc = []
    print('|================================================================|')
    print('|        | FPR  |  R   |  A   |  P   | FPR  |  R   |  A   |  P   |')
    print('|========|-----------MFCC------------|------------SSC------------| ')
    for label in range(0,len(labels)):
        print('==========')
        r,fpr,p,a,f1 = metrics(true_positive[label],false_positive[label], true_positive[label], false_negative[label])
        r_ssc,fpr_ssc,p_ssc,a_ssc,f1_ssc = metrics(true_positive_ssc[label],false_positive_ssc[label], true_positive_ssc[label], false_negative_ssc[label])
        str_label = transform(label)
        print('|{}| {} | {} | {} | {} | {} | {} | {} | {} |'.format(str_label,cust_round(fpr),cust_round(r),cust_round(a),cust_round(p),cust_round(fpr_ssc),cust_round(r_ssc),cust_round(a_ssc),cust_round(p_ssc)))

        R.append(r)
        A.append(a)
        P.append(p)
        F1.append(f1)
        FPR.append(fpr)
        R_ssc.append(r_ssc)
        A_ssc.append(a_ssc)
        P_ssc.append(p_ssc)
        F1_ssc.append(f1_ssc)
        FPR_ssc.append(fpr_ssc)

    # hist(R,R_ssc,labels,'Recall')
    # hist(FPR,FPR_ssc,labels,'False positive rate')
    # hist(A,A_ssc,labels,'Accuracy')
    # hist(P,P_ssc,labels,'Precision')
    # hist(F1,F1_ssc,labels,'F1 score')

    print('|================================================================|')

    im1 = heatmap(np.array(matrix),labels,labels,ax=None,cmap='YlGn',cbarlabel="Confusion level MFCC")

    annotate_heatmap(im1,data = np.array(unchanged), valfmt="{x:d}", size=7, threshold=20)
    im2 = heatmap(np.array(matrix_ssc),labels,labels,ax=None,cmap='YlGn',cbarlabel="Confusion level SSC")
    annotate_heatmap(im2,data = np.array(unchanged_ssc), valfmt="{x:d}", size=7, threshold=20)
    break