text:
# Web Scraping
import requests
from bs4 import BeautifulSoup
import pathlib

# Merging Data
import time
import gzip
import pandas as pd
import numpy as np
import os

def retrieve_html():
    """Function that locates the exact files needed for data extraction"""

    page = requests.get("http://insideairbnb.com/get-the-data.html")

    if page.status_code != 200:
        print("Error: Request to InsideAirBnB is failing")

    soup = BeautifulSoup(page.content, 'html.parser')

    td_tags = soup.find_all('td')

    # To ensure only the latest data for a particular city is used.
    city_set = set()

    # To maintain city level summary for data fetched.
    city_data = []

    for td_tag in td_tags:
        link_list = [a['href'] for a in td_tag.find_all('a', href=True)]

        if len(link_list) > 0 and link_list[0].find('listings.csv.gz') != -1:
            # Summary for each city is got by parsing the url itself.
            url_split = link_list[0].split('/')

            if len(url_split) != 9:
                print(f"Error: URL not following the "  # noqa: E999
                      f"format {link_list[0]}")

            if url_split[3] == 'united-states':
                country = url_split[3]
                region = url_split[4]
                city = url_split[5]
                date = url_split[6]
                url = link_list[0]

                if city not in city_set:
                    city_set.add(city)
                    city_data.append([country, region, city, date, url])

    # Check summary information of each city.
    print(f"Info: Total number of city information fetched: {len(city_data)}")
    print("Info: Start summary information of each city")
    for city in city_data:
        print(city)
    print("Info: Completed summary information of each city ")

    return city_data

def download_dataframe(city_data):
    # Consolidated data frame to hold data for all cities together.
    df_all = pd.DataFrame()

    for city in city_data:
        city_name = city[2]
        url = city[4]
        print(f"Info: Downloading data for {city_name} with url {url}")

        r = requests.get(url)

        # Retrieve HTTP meta-data.
        if r.status_code != 200:
            print(f"Error: Request to {url} failed with status "
            "{r.status_code}")  # noqa: E999
            continue

        # Fetch the data locally.
        file_name = f"{city_name}_listings.csv.gz"
        with open(file_name, 'wb') as f:
            f.write(r.content)

        # Unzip and load the file to data frame.
        with gzip.open(file_name) as f:
            df = pd.read_csv(f, error_bad_lines=False, chunksize=1000)

            chunk_list = []  # append each chunk df here

                # Each chunk is in df format
            for chunk in df:

                chunk_list.append(chunk)

            # concat the list into dataframe
            df = pd.concat(chunk_list)

            print(f"Info: Shape of data within {file_name}: {df.shape}")

            if df_all.empty:
                df_all = df
            else:
                df_all = pd.concat([df_all, df])

            print(f"Info: Shape of concatenated dataframe: {df_all.shape}")

        # Remove file
        os.remove(file_name)
        print(f"Info: Removed {file_name}!")

        # Sleep for short duration to ensure server is not loaded
        time.sleep(10)

    return df_all

def reduce_df(df):

    columns = ['zipcode',
               'property_type',
               'room_type',
               'accommodates',
               'bathrooms',
               'bedrooms',
               'beds',
               'bed_type',
               'price',
               'cleaning_fee']
    df = df[columns]

    return df

def data_retrieval():
    print("Info: Step 1 - Data Retrieval start ...")

    # Remove old data if present
    file = pathlib.Path("data.csv")
    if file.exists():
        print("Info: Removing data.csv")
        os.remove("data.csv")

    df = reduce_df(download_dataframe(retrieve_html()))
    df.to_csv('data.csv', index=False)
    print(f"Info: Shape of dataframe after merging: {df.shape}")

    print(f"Info: Data Retrieval completed ...")

if __name__ == '__main__':
    data_retrieval()


text:
import numpy as np
import json
from scipy.spatial.transform import Rotation as R
from utils.humanoid_kin import HumanoidSkeleton
from sim import engine_builder, Plane, Character

def get_mocap_frames(path):
    frames = []
    with open(path) as f:
        j = json.load(f)
        data = j["Frames"]
        for row in data:
            frames.append(row[1:])
    return frames

'''
id  name        parent      dof         mirror
0   root        -1          -1          0
1   chest       0           3           1
2   neck        1           3           2
3   rhip        0           3           9
4   rknee       3           1           10
5   rankle      4           3           11
6   rshoulder   1           3           12
7   relbow      6           1           13
8   rwrist      7           0           14
9   lhip        0           3           3
10  lknee       9           1           4
11  lankle      10          3           5
12  lshoulder   1           3           6
13  lelbow      12          1           7
14  lwrist      13          0           8
'''

class FeatureConverter:
    def __init__(self):
        char_file = "data/characters/humanoid3d.txt"
        ctrl_file = "data/controllers/humanoid3d_ctrl.txt"
        self.__skeleton = HumanoidSkeleton(char_file, ctrl_file)
        '''
        self.__engine = engine_builder("pybullet", 1/600.0)
        self.__character = Character(self.__skeleton, True)
        self.__engine.add_object(self.__character)
        '''
        self.__parent_table = [-1, 0, 1, 0, 3, 4, 1, 6, 7, 0, 9, 10, 1, 12, 13]
        self.__dof = [0, 3, 3, 3, 1, 3, 3, 1, 0, 3, 1, 3, 3, 1, 0]
        self.__mirror = [0, 1, 2, 9, 10, 11, 12, 13, 14, 3, 4, 5, 6, 7, 8]

    def quat_to_pos(self, frame):
        self.__skeleton.set_pose(np.array(frame))
        return list(self.__skeleton.get_feature())

    def pos_to_quat(self, pos):
        return list(self.__skeleton.inv_feature(pos))

    def exp_to_quat(self, exp):
        return self.__skeleton.exp_to_action(exp)

    def quat_to_exp(self, quat, root=[0,0,0,1,0,0,0]):
        return self.__skeleton.targ_pose_to_action(np.array(root + quat))

    def pos_to_exp(self, pos):
        quat = self.pos_to_quat(pos)
        return self.quat_to_exp(quat)

    def mirror(self, pos):
        mirrored = list(pos).copy()
        for i in range(15):
            idx = i*12
            mirroridx = self.__mirror[i]*12
            for j in range(12):
                mirrored[mirroridx + j] = pos[idx + j] * (-1 if j % 3 == 2 else 1)
        return mirrored

    def mirror_root(self, root):
        rot = R.from_quat([root[4], root[5], root[6], root[3]])
        expmap = rot.as_rotvec()
        expmap[0] = -expmap[0]
        expmap[1] = -expmap[1]
        q = R.from_rotvec(expmap).as_quat()
        return [root[0], root[1], -root[2], q[3], q[0], q[1], q[2]]

ext: py

max_stars_repo_name: arpspoof/Jump

max_issues_repo_name: squalidux/Jump

max_forks_repo_name: squalidux/Jump

reason: import numpy,from scipy


text:
import numpy as np
import pandas as pd
try:
    import pandas.core.computation.expressions as expr
except ImportError:
    import pandas.computation.expressions as expr

class Eval:

    params = [['numexpr', 'python'], [1, 'all']]
    param_names = ['engine', 'threads']

    def setup(self, engine, threads):
        self.df = pd.DataFrame(np.random.randn(20000, 100))
        self.df2 = pd.DataFrame(np.random.randn(20000, 100))
        self.df3 = pd.DataFrame(np.random.randn(20000, 100))
        self.df4 = pd.DataFrame(np.random.randn(20000, 100))

        if threads == 1:
            expr.set_numexpr_threads(1)

    def time_add(self, engine, threads):
        pd.eval('self.df + self.df2 + self.df3 + self.df4', engine=engine)

    def time_and(self, engine, threads):
        pd.eval('(self.df > 0) & (self.df2 > 0) & '
                '(self.df3 > 0) & (self.df4 > 0)', engine=engine)

    def time_chained_cmp(self, engine, threads):
        pd.eval('self.df < self.df2 < self.df3 < self.df4', engine=engine)

    def time_mult(self, engine, threads):
        pd.eval('self.df * self.df2 * self.df3 * self.df4', engine=engine)

    def teardown(self, engine, threads):
        expr.set_numexpr_threads()

class Query:

    def setup(self):
        N = 10**6
        halfway = (N // 2) - 1
        index = pd.date_range('20010101', periods=N, freq='T')
        s = pd.Series(index)
        self.ts = s.iloc[halfway]
        self.df = pd.DataFrame({'a': np.random.randn(N), 'dates': index},
                               index=index)
        data = np.random.randn(N)
        self.min_val = data.min()
        self.max_val = data.max()

    def time_query_datetime_index(self):
        self.df.query('index < @self.ts')

    def time_query_datetime_column(self):
        self.df.query('dates < @self.ts')

    def time_query_with_boolean_selection(self):
        self.df.query('(a >= @self.min_val) & (a <= @self.max_val)')

from .pandas_vb_common import setup  # noqa: F401

max_stars_repo_name: developing-coder/pandas

max_issues_repo_name: developing-coder/pandas

max_forks_repo_name: developing-coder/pandas

reason: import numpy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on June 20th 17:28:17 2021

@author: yaleman
"""
"""
% Syntax :
%
% Description
%
%
% See also:
%__________________________________________________
% Authors: Yasser Aleman Gomez
% Radiology Department
% CHUV, Lausanne
% Created on %(date)s
% Version $1.0

%% ======================= Importing Libraries ============================== %
"""
import numpy as np
import nibabel as nib
import h5py
import time
import os
from pathlib import Path
import argparse
import sys
import csv

class SmartFormatter(argparse.HelpFormatter):

    def _split_lines(self, text, width):
        if text.startswith('R|'):
            return text[2:].splitlines()
        # this is the RawTextHelpFormatter._split_lines
        return argparse.HelpFormatter._split_lines(self, text, width)

def _build_args_parser():
    formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=52)

    from argparse import ArgumentParser

    p = argparse.ArgumentParser(formatter_class=SmartFormatter, description='\n Help \n')

    requiredNamed = p.add_argument_group('Required arguments')
    requiredNamed.add_argument('--method', '-m', action='store', choices=['connectivity', 'consfile'], required=True,
                metavar='METHOD', type=str, nargs=1, default='connectivity',
                help="R|Method selection (connectivity or consfile). Another file should be supplied if connectivity is \n"
                     "                                             selected as method. This file should be supplied \n"
                     "                                             through the flags --map -p. \n"
                     "\n"
                     "   connectivity: Compute the mean value of a supplied scalar map along all the bundles included in the \n"
                     "                 atlas. It outputs a connectivity matrix where the connection strength is the mean \n"
                     "                 value of the scalar map along each bundle. \n"
                     " \n"
                     "    consfile   : Extract and save some especific bundles. The selected bundles should be supplied \n"
                     "                 through a nifti-1 file or a csv text file using the flags --bfile -bf. If a scalar \n"
                     "                 map is supplied, a table with the mean values along these bundles is stored as well.\n"
                     "                 - If the file is a Nifti image the bundles intercepting its non-zero values will \n"
                     "                 be extracted and saved. \n"
                     "                 - If the file is a readable text file, each row of the txt file should contain \n"
                     "                 the source and target regions separated by a comma (ie. 10,57). \n"
                     "\n")
    requiredNamed.add_argument('--out', '-o', action='store', required=True, metavar='OUT', type=str, nargs=1,
                help= "R| Output basename. \n"
                      "                    If connectivity was selected as method then the output filename will be: \n"
                      "                    <outbasename>-connmat-scaleId' \n"
                      "\n"
                      "                    Otherwise, it will be: \n"
                      "                    <outbasename>-<sourceId_targetId>-scaleId')\n")
    requiredNamed.add_argument('--scale', '-s', action='store', required=False, choices=['scale1', 'scale2', 'scale3', 'scale4'],
               metavar='SCALE', type=str, nargs=1,
               help= "R| Scale Id (scale1, scale2, scale3 or scale4).\n", default=['scale1'])
    requiredNamed.add_argument('--bfile', '-bf', action='store', required=False, metavar='FILE', type=str, nargs=1,
                help="R| It can be either a binary mask or a Comma Separated text file.\n"
                     "     - If this file is a Nifti image the bundles intercepting its non-zero values will be \n"
                     "       extracted and saved \n"
                     "     - If the flie is a text file, each row of the txt file should contain the source and target \n"
                     "       regions separated by a comma (ie. 10,57). The first row of this file will be ignored. \n"
                     "\n", default='None')
    requiredNamed.add_argument('--map', '-p', action='store', required=False,
                metavar='MAP', type=str, nargs=1,
                help="R| Scalar map to compute its mean value along the selected bundles.\n"
                    "\n" , default='None')
    requiredNamed.add_argument('--subth', '-st', action='store', required=False,
                               metavar='SUBTHRESH', type=float, nargs=1,
                               help="R| Subject-level consistency threshold in percentage (0-100). Bundles appearing in \n"
                                    " a percentage of subjects lower than the selected threshold will not be taken into \n"
                                    " account (default = 0). \n"
                                    "\n", default=[0])
    requiredNamed.add_argument('--voxth', '-vt', action='store', required=False,
                               metavar='VOXTHRESH', type=float, nargs=1,
                               help="R| Voxel-level consistency threshold. Voxels with probability (0-1) values lower \n"
                                    " than the supplied threshold will be set to 0 in the selected bundles (default = 0).\n"
                                    "\n", default=[0])
    requiredNamed.add_argument('--extract', '-e', action='store_true', required=False,
                               help="R| Save individual bundles in Nifti-1 format. \n"
                                    "\n")
    requiredNamed.add_argument('--all', '-a', action='store_true', required=False,
                               help="R| Save individual bundles intercepting any of the regions of interest inside the mask\n"
                                    "\n")
    requiredNamed.add_argument('--collapse', '-c', action='store_true', required=False,
                               help="R| Collapse the bundles into a 4D color-coded Nifti-1 file. \n"
                                    "\n")
    requiredNamed.add_argument('--force', '-f', action='store_true', required=False,
                               help="R| Overwrite the results. \n"
                                    "\n")
    p.add_argument('--verbose', '-v', action='store', required=False,
                type=int, nargs=1,
                help='verbosity level: 1=low; 2=debug')

    args   = p.parse_args()
    method = args.method[0]
    map    = args.map[0]
    file   = args.bfile[0]

    # boolmap = bool(map) + bool(event_id) + bool(broker_id) + bool(product_id) + bool(event_keyword) + bool(metadata)
    if method == 'connectivity' and not os.path.isfile(map):
        print("\n")
        print("Please, supply a valid scalar map using the flags --map or -p.")
        p.print_help()
        sys.exit()
    elif method != 'connectivity' and not os.path.isfile(file):
        print(" ")
        print(" ")
        print("Please, supply a valid txt or binary mask to select the bundles. Use the flags --bfile or -f.")
        print(" ")
        print(" ")

        p.print_help()
        sys.exit()

    return p

def save_matrix(connMat, matFilename, lnames, regCodes, regCoords, regColors):
    """
    Save the connectivity matrix to a HDF5 file.
    @params:
        connMat        - Required  : Numpy array with the connectivity matrix (Float)
        matFilename    - Required  : Output filename for the connectivity matrix (Str)
        lnames         - Required  : Region names (Str)
        regCodes       - Required  : Region codes (position in the matrix) (Int)
        regCoords      - Required  : Coordinates of the regions center-of mass in mm (MNI space)(Float)
        regColors      - Required  : Region RGB (red, green and blue) colors triplets. It can be used for networks visualization.(Float)
    """

    # import networkx as nx
    import h5py
    # G = nx.from_numpy_matrix(connMat)
    # nx.write_gpickle(G, matFilename)

    indnan = np.isnan(connMat)
    connMat[indnan] = 0

    hm = h5py.File(matFilename, 'w')
    # Saving the Connectivity Matrix
    g1 = hm.create_group('connmat')
    g1.create_dataset('matrix', data=connMat)
    g1.create_dataset('gmregions', data=np.array(lnames, dtype='S'))
    g1.create_dataset('gmcodes', data=regCodes)
    g1.create_dataset('gmcoords', data=regCoords)
    g1.create_dataset('gmcolors', data=regColors)
    hm.close()

    return matFilename

# Print iterations progress
def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█', printEnd = "\r"):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd)
    # Print New Line on Complete
    if iteration == total:
        print()

def bundles_from_mask(hfName, maskFilename, outBasename, scaleId, mapFilename, voxth=0, subth=0, boolall=False, boolbund=False, boolcoll=False):
    """
    Compute mean, median and a standard deviation of a certain scalar map along the bundles reaching specified ROIs
    @params:
        hfName            - Required  : h5 filename (Str)
        maskFilename      - Required  : Nifti-1 filename containing the ROIs. Each ROI should have a different value (Str)
        outBasename       - Required  : Output basename (Str)
        scaleId           - Required  : ScaleId: scale1, scale2, scale3 or scale4  (Str)
        mapFilename       - Required  : Scalar map filename (Int)
        voxth             - Optional  : Voxel-wise consistency threshold (Float)
        subth             - Optional  : Subject-wise consistency threshold (Float)
        boolall           - Optional  : Boolean variable to save all the bundles reaching any of the selected ROIs(Bool)
        boolbund          - Optional  : Boolean variable to save the selected bundles (Bool)
        boolcoll          - Optional  : Boolean variable to collapse the selected bundles (Bool)
    """

    outDir = os.path.dirname(outBasename)
    fname  = os.path.basename(outBasename)

    # Creating the folder if it does not exist
    if not os.path.isdir(outDir):
        path = Path(outDir)
        try:
            path.mkdir()
        except OSError:
            print("Failed to make nested directory")
        else:
            print("Nested directory made")

    # Reading the mask image
    maskI = nib.load(maskFilename)
    dataP = maskI.get_fdata()

    if boolall:
        dataP[dataP != 0] = 1

    nrois_mask = np.unique(dataP)
    nrois_mask = nrois_mask[nrois_mask != 0]

    # Reading the HDF5 file
    hf        = h5py.File(hfName, 'r')
    tempVar   = hf.get('header/affine')
    affine    = np.array(tempVar)
    tempVar   = hf.get('header/dim')
    dim       = np.array(tempVar)
    tempVar   = hf.get('header/gmregions')
    stnames   = np.array(tempVar)
    tempVar   = hf.get('header/nsubjects')
    nsubj     = np.array(tempVar)
    tempVar   = hf.get('header/gmcolors')
    gmcolors  = np.array(tempVar)

    # Reading the consistency matrix. It can be used for thresholding
    tempVar   = hf.get('matrices/consistency')
    consistM  = np.array(tempVar)/nsubj

    # tempVar = hf.get('header/scale')
    # scaleId = np.array(tempVar)
    if boolcoll:
        collMask  = np.zeros((dim[0], dim[1], dim[2]), dtype='float')  # Collapsing all the bundles
        collMaskc = np.zeros((dim[0], dim[1], dim[2], 3), dtype='float')

    # Bundles to be used
    bundloc   = np.argwhere(np.triu(consistM, 1) >= subth/100)
    X         = bundloc[:, 0] + 1
    Y         = bundloc[:, 1] + 1

    # Reading the scalar map
    booltab     = False
    if os.path.isfile(mapFilename):
        mapI    = nib.load(mapFilename)
        dataM   = mapI.get_fdata()
        booltab = True

    tabList   = [['BundleId', 'SourceROI', 'TargetROI', 'Consistency' , 'MeanValue', 'StdValue', 'MedianValue']]
    tic       = time.perf_counter()

    # Reading the bundles keys
    # tempVar       = hf.get('atlas')
    # bundIds       = list(tempVar.keys())
    # tbund         = len(bundIds)
    tbund     = len(X)
    # Loop along the bundles
    for i in np.arange(0, tbund - 1):
        # tempVar = hf.get("atlas/" + bundIds[i])
        tempS      = str(X[i]) + "_" + str(Y[i])
        tempVar    = hf.get("atlas/" + tempS)
        voxIds     = np.array(tempVar)

        bdcolor    = np.mean(gmcolors[bundloc[i, :],:],axis=0)/255

        # Creating the empty volume
        array_data = np.zeros((dim[0], dim[1], dim[2]), dtype=float)

        # Assigning the probability
        array_data[voxIds[:, 0], voxIds[:, 1], voxIds[:, 2]] = voxIds[:, 3] / nsubj

        # Thresholding using the concurrency value
        if voxth > 0:
            array_data[array_data < voxth] = 0

        # Interception between the mask and the probabilistic atlas
        tempM      = np.multiply(array_data != 0, dataP)

        nrois_bund = np.unique(tempM)
        nrois_bund = nrois_bund[nrois_bund != 0]

        savebund = False
        if len(nrois_mask) == 1 and len(nrois_bund) == 1:
            savebund = True
        elif len(nrois_mask) != 1 and len(nrois_bund) > 1:
            savebund = True

        if savebund:
            if boolcoll:
                collMask            = collMask  + np.asarray(array_data >0)
                collMaskc[:,:,:,0]  = collMaskc[:,:,:,0] + array_data*bdcolor[0]
                collMaskc[:,:,:,1]  = collMaskc[:,:,:,1] + array_data*bdcolor[1]
                collMaskc[:,:,:,2]  = collMaskc[:,:,:,2] + array_data*bdcolor[2]

            # Saving the bundle image
            if boolbund:
                bundFilename   = os.path.join(outDir, fname + "-fromMask-" + tempS + "-" + scaleId + ".nii.gz")
                array_img      = nib.Nifti1Image(array_data, affine)
                nib.save(array_img, bundFilename)

            # Creating the table
            if booltab:
                # tempS = bundIds[i]
                # roiIds = tempS.split('_')

                mVal    = np.mean(dataM[array_data >= voxth])
                sVal    = np.std(dataM[array_data >= voxth])
                dVal    = np.median(dataM[array_data >= voxth])
                sourROI = stnames[int(X[i])-1].decode('UTF-8')
                targROI = stnames[int(Y[i])-1].decode('UTF-8')
                cVal    = consistM[X[i]-1, Y[i]-1]*100
                tabList.append([tempS, sourROI, targROI, cVal, mVal, sVal, dVal])

        printProgressBar(i + 1, tbund , 'Processing bundle: ' + '(' + str(i+1) + '/' + str(tbund) + ')')

    # Saving the table
    if booltab:
        tabFilename     = outBasename + "-table-fromMask-" + scaleId + ".csv"
        with open(tabFilename, mode='w') as bundTab:
            bund = csv.writer(bundTab, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            for i in np.arange(len(tabList)):
                bund.writerow(tabList[i])
    # Saving the collapsed image
    if boolcoll:
        collFilename           = os.path.join(outDir, fname + "-fromMask-bundcollap-" + scaleId + ".nii.gz")
        # collMask        = collMask != 0
        # array_img              = nib.Nifti1Image(collMask.astype(int), affine)
        # nib.save(array_img, collFilename)
        collMaskc[:, :, :, 0] = np.divide(collMaskc[:, :, :, 0], collMask, where=True)
        collMaskc[:, :, :, 1] = np.divide(collMaskc[:, :, :, 1], collMask, where=True)
        collMaskc[:, :, :, 2] = np.divide(collMaskc[:, :, :, 2], collMask, where=True)
        collMaskc[np.isnan(collMaskc)] = 0

        imgcoll               = nib.Nifti1Image(collMaskc, affine)
        nib.save(imgcoll, collFilename)

    # Timer
    toc                 = time.perf_counter()
    print(f"Processed in {toc - tic:0.4f} seconds")

def bundles_from_textfile(hfName, textFilename, outBasename, scaleId, mapFilename, voxth=0, subth=0, boolbund=False, boolcoll=False):
    """
    Compute mean, median and a standard deviation of a certain scalar map along the bundles specified in a csv
    file.
    @params:
        hfName            - Required  : h5 filename (Str)
        textFilename      - Required  : Comma Separated Value file with the bundles of interest (i.e. 10_57, bundle between regions 10 and 57) (Str)
        outBasename       - Required  : Output basename (Str)
        scaleId           - Required  : ScaleId: scale1, scale2, scale3 or scale4  (Str)
        mapFilename       - Required  : Scalar map filename (Int)
        voxth             - Optional  : Voxel-wise consistency threshold (range: 0-1)(Float)
        subth             - Optional  : Subject-wise consistency threshold in precentage (range: 0-100)(Float)
        boolbund          - Optional  : Boolean variable to save the selected bundles (Bool)
        boolcoll          - Optional  : Boolean variable to collapse the selected bundles (Bool)
    """

    outDir = os.path.dirname(outBasename)
    fname = os.path.basename(outBasename)

    # Creating the folder if it does not exist
    if not os.path.isdir(outDir):
        path = Path(outDir)
        try:
            path.mkdir()
        except OSError:
            print("Failed to make nested directory")
        else:
            print("Nested directory made")

    # Reading the txt file
    bundIds = np.loadtxt(textFilename, skiprows=1, delimiter=',', dtype='u4', ndmin=2)
    bundIds = np.sort(bundIds, axis=1)

    # Reading the HDF5 file
    hf = h5py.File(hfName, 'r')
    tempVar = hf.get('header/affine')
    affine = np.array(tempVar)
    tempVar = hf.get('header/dim')
    dim = np.array(tempVar)
    tempVar = hf.get('header/gmregions')
    stnames = np.array(tempVar)
    tempVar = hf.get('header/nsubjects')
    nsubj = np.array(tempVar)
    tempVar   = hf.get('header/gmcolors')
    gmcolors  = np.array(tempVar)

    # Reading the consistency matrix. It can be used for thresholding
    tempVar  = hf.get('matrices/consistency')
    consistM = np.array(tempVar) / nsubj

    if boolcoll:
        collMask  = np.zeros((dim[0], dim[1], dim[2]), dtype='float')  # Collapsing all the bundles
        collMaskc = np.zeros((dim[0], dim[1], dim[2], 3), dtype='float')

    # If the csv file contains only one column then all the bundles involving the selected regions are extracted
    if np.shape(bundIds)[1] == 1:
        nstr     = len(consistM)
        sbundIds = np.zeros([1, 2], dtype='u4')
        for i in np.arange(0, len(bundIds)):
            a1 = np.ones([nstr, 1], dtype='u4') * bundIds[i]
            a2 = np.linspace(1, nstr, nstr, dtype='u4')
            a2 = a2[:, None]
            out_mat = np.concatenate((a1, a2), axis=1)
            sbundIds = np.concatenate((sbundIds, out_mat), axis=0)
        sbundIds = np.sort(sbundIds, axis=1)
        t = sbundIds[:, 0] - sbundIds[:, 1] != 0
        bundIds = sbundIds[t, :]

    # Bundles to be used
    bundloc  = np.argwhere(np.triu(consistM, 1) >= subth / 100)
    X        = bundloc[:, 0] + 1
    Y        = bundloc[:, 1] + 1

    # Reading the scalar map
    booltab = False
    if os.path.isfile(mapFilename):
        mapI = nib.load(mapFilename)
        dataM = mapI.get_fdata()
        booltab = True

    # Loop along the bundles
    # p = ProgressBar(len(bundIds))
    tabList   = [['BundleId', 'SourceROI', 'TargetROI', 'Consistency' ,'MeanValue', 'StdValue', 'MedianValue']]
    tic = time.perf_counter()
    tbund = len(bundIds)

    for i in np.arange(tbund):
        t = gmcolors[bundIds[i, :]-1, :]
        bdcolor = np.mean(t, axis=0) / 255

        # Reading the selected bundles
        tempVar = hf.get("atlas/" + str(bundIds[i, 0]) + "_" + str(bundIds[i, 1]))

        if tempVar != None:
            voxIds = np.array(tempVar)

            # Creating the empty volume
            array_data = np.zeros((dim[0], dim[1], dim[2]), dtype=float)

            # Assigning the probability
            array_data[voxIds[:, 0], voxIds[:, 1], voxIds[:, 2]] = voxIds[:, 3] / nsubj

            # Thresholding using the currency value
            if voxth > 0:
                array_data[array_data < voxth] = 0

            # Writing the nifti file
            if np.any(array_data):
                collMask            = collMask  + np.asarray(array_data >0)
                collMaskc[:,:,:,0]  = collMaskc[:,:,:,0] + array_data*bdcolor[0]
                collMaskc[:,:,:,1]  = collMaskc[:,:,:,1] + array_data*bdcolor[1]
                collMaskc[:,:,:,2]  = collMaskc[:,:,:,2] + array_data*bdcolor[2]
                if boolbund:
                    # Saving only the bundles intercepting the mask
                    bundFilename = os.path.join(outDir, fname + "-fromCSV-" + str(bundIds[i, 0]) + "_" + str(bundIds[i, 1]) + "-" + scaleId + ".nii.gz")
                    array_img = nib.Nifti1Image(array_data, affine)
                    nib.save(array_img, bundFilename)

                # Creating the table
                if booltab:
                    mVal      = np.mean(dataM[array_data >= voxth])
                    sVal      = np.std(dataM[array_data >= voxth])
                    dVal      = np.median(dataM[array_data >= voxth])
                    sourROI   = stnames[bundIds[i, 0]-1].decode('UTF-8')
                    targROI   = stnames[bundIds[i, 1]-1].decode('UTF-8')
                    cVal = consistM[bundIds[i, 0] - 1, bundIds[i, 1]  - 1] * 100
                    tabList.append([str(bundIds[i, 0]) + "_" + str(bundIds[i, 1]), sourROI, targROI, cVal, mVal, sVal, dVal])

            printProgressBar(i+1, tbund, 'Processing bundle: ' + '(' + str(i + 1) + '/' + str(tbund) + ')')

    # Saving the table
    if booltab:
        tabFilename = outBasename + "-table-fromCSV-" + scaleId + ".csv"
        with open(tabFilename, mode='w') as bundTab:
            bund = csv.writer(bundTab, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            for i in np.arange(len(tabList)):
                bund.writerow(tabList[i])

    # Saving the collapsed image
    if boolcoll:
        collFilename = os.path.join(outDir, fname + "-fromCSV-bundcollap-" + scaleId + ".nii.gz")
        collMaskc[:, :, :, 0] = np.divide(collMaskc[:, :, :, 0], collMask, where=True)
        collMaskc[:, :, :, 1] = np.divide(collMaskc[:, :, :, 1], collMask, where=True)
        collMaskc[:, :, :, 2] = np.divide(collMaskc[:, :, :, 2], collMask, where=True)
        collMaskc[np.isnan(collMaskc)] = 0

        imgcoll               = nib.Nifti1Image(collMaskc, affine)
        nib.save(imgcoll, collFilename)

    # Timer
    toc = time.perf_counter()
    print(f"Processed in {toc - tic:0.4f} seconds")

def bundles_to_connectivity(hfName, outBasename, scaleId, mapFilename, voxth=0, subth=0, boolbund=False, boolforce=False):
    """
     Compute connectivity matrix for a specified scale where the streng of connection is mean, median and standard deviation of a certain scalar map .
     @params:
         hfName            - Required  : h5 filename (Str)
         textFilename      - Required  : Comma Separated Value file with the bundles of interest (i.e. 10_57, bundle between regions 10 and 57) (Str)
         outBasename       - Required  : Output basename (Str)
         scaleId           - Required  : ScaleId: scale1, scale2, scale3 or scale4  (Str)
         mapFilename       - Required  : Scalar map filename (Int)
         voxth             - Optional  : Voxel-wise consistency threshold (range: 0-1)(Float)
         subth             - Optional  : Subject-wise consistency threshold in precentage (range: 0-100)(Float)
         boolbund          - Optional  : Boolean variable to save the selected bundles (Bool)
         boolforce         - Optional  : Rewrite the connectivity matrices (Bool)
     """
    outDir = os.path.dirname(outBasename)
    fname = os.path.basename(outBasename)

    # Creating the folder if it does not exist
    if not os.path.isdir(outDir):
        path = Path(outDir)
        try:
            path.mkdir()
        except OSError:
            print("Failed to make nested directory")
        else:
            print("Nested directory made")

    outBasename + "-connmat-mean-" + scaleId + ".h5"

    if os.path.isfile(outBasename + "-connmat-mean-" + scaleId + ".h5") and boolforce == True:
        print(" The file " + outBasename + "-connmat-mean-" + scaleId + ".h5" + " will be overwritten.")
        os.remove(outBasename + "-connmat-mean-" + scaleId + ".h5")
        print("  ")
    if os.path.isfile(outBasename + "-connmat-std-" + scaleId + ".h5") and boolforce == True:
        print(" The file " + outBasename + "-connmat-std-" + scaleId + ".h5" + " will be overwritten.")
        os.remove(outBasename + "-connmat-std-" + scaleId + ".h5")
        print("  ")
    if os.path.isfile(outBasename + "-connmat-median-" + scaleId + ".h5") and boolforce == True:
        print(" The file " + outBasename + "-connmat-median-" + scaleId + ".h5" + " will be overwritten.")
        os.remove(outBasename + "-connmat-median-" + scaleId + ".h5")
        print("  ")

    # Reading the HDF5 file
    hf        = h5py.File(hfName, 'r')
    tempVar   = hf.get('header/affine')
    affine    = np.array(tempVar)
    tempVar   = hf.get('header/dim')
    dim       = np.array(tempVar)

    # Reading the fields to plot the network
    tempVar   = hf.get('header/gmregions')
    stnames   = np.array(tempVar)
    tempVar = hf.get('header/gmcodes')
    regCodes = np.array(tempVar)
    tempVar = hf.get('header/gmcoords')
    regCoords = np.array(tempVar)
    tempVar = hf.get('header/gmcolors')
    regColors = np.array(tempVar)

    # Reading the number of subjects used to create the atlas
    tempVar   = hf.get('header/nsubjects')
    nsubj     = np.array(tempVar)

    # Reading the consistency matrix. It can be used for thresholding
    tempVar   = hf.get('matrices/consistency')
    consistM  = np.array(tempVar)/nsubj

    # Bundles to be used
    bundloc = np.argwhere(np.triu(consistM, 1) > subth/100)
    X = bundloc[:, 0] + 1
    Y = bundloc[:, 1] + 1

    # tempVar = hf.get('header/scale')
    # scaleId = np.array(tempVar)

    # Reading the scalar map
    booltab = False
    if os.path.isfile(mapFilename):
        mapI = nib.load(mapFilename)
        dataM = mapI.get_fdata()
        booltab = True

    # Creating empty connectivity matrices
    nstr        = len(stnames)
    meanMat     = np.zeros([nstr, nstr])  # Number of fibers
    stdMat      = np.zeros([nstr, nstr])  # Fiber Length
    medMat      = np.zeros([nstr, nstr])  # Number of Points. Important to compute the mean along fibers

    tic     = time.perf_counter()

    # Reading the bundles keys
    # tempVar       = hf.get('atlas')
    # bundIds       = list(tempVar.keys())
    # tbund         = len(bundIds)

    tbund = len(X)
    # Loop along the bundles
    for i in np.arange(0, tbund - 1):
        # tempS   = bundIds[i]
        tempS   = str(X[i]) + "_" + str(Y[i])
        tempVar = hf.get("atlas/" + tempS)
        voxIds  = np.array(tempVar)
        # roiIds  = tempS.split('_')

        # Creating the empty volume
        array_data = np.zeros((dim[0], dim[1], dim[2]), dtype=float)

        # Assigning the probability
        array_data[voxIds[:, 0], voxIds[:, 1], voxIds[:, 2]] = voxIds[:, 3] / nsubj

        # # Thresholding using the currency value
        # if voxth > 0:
        #     array_data[array_data < voxth] = 0

        # Writing the nifti file
        if np.any(array_data):

            if boolbund:
                # Saving only the bundles intercepting the mask
                # bundFilename = os.path.join(outDir, fname + "-atlas-" + str(roiIds[0]) + "_" + str(roiIds[1]) + "-" + scaleId + ".nii.gz")
                bundFilename = os.path.join(outDir, fname + "-atlas-" + str(X[i]) + "_" + str(Y[i]) + "-" + scaleId + ".nii.gz")
                array_img = nib.Nifti1Image(array_data, affine)
                nib.save(array_img, bundFilename)

            # Creating the connectivity matrices
            # meanMat[int(roiIds[0]) - 1, int(roiIds[1]) - 1] = np.mean(dataM[array_data >= voxth])
            # meanMat[int(roiIds[1]) - 1, int(roiIds[0]) - 1] = meanMat[int(roiIds[0]) - 1, int(roiIds[1]) - 1]
            meanMat[int(X[i]) - 1, int(Y[i]) - 1] = np.mean(dataM[array_data >= voxth])
            meanMat[int(Y[i]) - 1, int(X[i]) - 1] = meanMat[int(X[i]) - 1, int(Y[i]) - 1]

            # stdMat[int(roiIds[0]) - 1, int(roiIds[1]) - 1] = np.std(dataM[array_data >= voxth])
            # stdMat[int(roiIds[1]) - 1, int(roiIds[0]) - 1] = stdMat[int(roiIds[0]) - 1, int(roiIds[1]) - 1]
            stdMat[int(X[i]) - 1, int(Y[i]) - 1] = np.mean(dataM[array_data >= voxth])
            stdMat[int(Y[i]) - 1, int(X[i]) - 1] = stdMat[int(X[i]) - 1, int(Y[i]) - 1]

            # medMat[int(roiIds[0]) - 1, int(roiIds[1]) - 1] = np.median(dataM[array_data >= voxth])
            # medMat[int(roiIds[1]) - 1, int(roiIds[0]) - 1] = medMat[int(roiIds[0]) - 1, int(roiIds[1]) - 1]
            medMat[int(X[i]) - 1, int(Y[i]) - 1] = np.mean(dataM[array_data >= voxth])
            medMat[int(Y[i]) - 1, int(X[i]) - 1] = medMat[int(X[i]) - 1, int(Y[i]) - 1]

        printProgressBar(i+1, tbund, 'Processing bundle: ' + '(' + str(i+1) + '/' + str(tbund) + ')')

    # Saving the connectivity matrices

    # Mean
    meanFilename = outBasename + "-connmat-mean-" + scaleId + ".h5"
    # save_matrix(meanMat, meanFilename, gmcoords, , stnames)
    save_matrix(meanMat, meanFilename, stnames, regCodes, regCoords, regColors)

    # Std
    stdFilename = outBasename + "-connmat-std-" + scaleId + ".h5"
    # save_matrix(stdMat, stdFilename)
    save_matrix(stdMat, stdFilename, stnames, regCodes, regCoords, regColors)

    # Median
    medFilename = outBasename + "-connmat-median-" + scaleId + ".h5"
    # save_matrix(medMat, medFilename)
    save_matrix(medMat, medFilename, stnames, regCodes, regCoords, regColors)

    # Timer
    toc = time.perf_counter()
    print(f"Processed in {toc - tic:0.4f} seconds")

def main():
    import sys
    # 0. Handle inputs
    parser = _build_args_parser()
    args = parser.parse_args()

    print(args)
    if args.verbose is not None:
        v = np.int(args.verbose[0])
    else:
        v = 0
        print('- Verbose set to 0\n')
    if v:
        print('\nInputs\n')
#

    # Getting the path of the current running python file
    cwd = os.getcwd()

    scaleid   = args.scale[0]
    file      = args.bfile[0]
    fmeth     = args.method[0]
    fout      = args.out[0]
    fmap      = args.map[0]
    voxth     = args.voxth[0]
    subth     = args.subth[0]
    boolall   =  args.all
    boolbund  = args.extract
    boolcoll  = args.collapse
    boolforce = args.force

    hfName = os.path.join(cwd,'probconnatlas', "wm.connatlas." + scaleid + ".h5")

    if fmeth == 'connectivity':
        bundles_to_connectivity(hfName, fout, scaleid, fmap, voxth, subth, boolbund, boolforce)

    elif fmeth == 'consfile':
        # fid = open(file, "r+")
        imask = False
        if file[-7:]== '.nii.gz' or file[-4:] == '.nii':
            imask = True

        if imask == True:
            bundles_from_mask(hfName, file, fout, scaleid, fmap, voxth, subth, boolall, boolbund, boolcoll)
        else:
            bundles_from_textfile(hfName, file, fout, scaleid, fmap, voxth, subth, boolbund, boolcoll)
    else:
        print("Please enter a correct method (connectivity or consfile). ")
        sys.exit()

if __name__ == "__main__":
    main()

max_stars_repo_name: connectomicslab/probconnatlas

max_issues_repo_name: connectomicslab/probconnatlas

max_forks_repo_name: connectomicslab/probconnatlas

reason: import numpy,import networkx


text:
import bob.db.iris
import bob.learn.em
import bob.learn.linear
import matplotlib.pyplot as plt
import numpy
numpy.random.seed(2)  # FIXING A SEED

def train_ubm(features, n_gaussians):
    """
    Train UBM

     **Parameters**
       features: 2D numpy array with the features

       n_gaussians: Number of Gaussians

    """

    input_size = features.shape[1]

    kmeans_machine = bob.learn.em.KMeansMachine(int(n_gaussians), input_size)
    ubm = bob.learn.em.GMMMachine(int(n_gaussians), input_size)

    # The K-means clustering is firstly used to used to estimate the initial
    # means, the final variances and the final weights for each gaussian
    # component
    kmeans_trainer = bob.learn.em.KMeansTrainer('RANDOM_NO_DUPLICATE')
    bob.learn.em.train(kmeans_trainer, kmeans_machine, features)

    # Getting the means, weights and the variances for each cluster. This is a
    # very good estimator for the ML
    (variances, weights) = kmeans_machine.get_variances_and_weights_for_each_cluster(features)
    means = kmeans_machine.means

    # initialize the UBM with the output of kmeans
    ubm.means = means
    ubm.variances = variances
    ubm.weights = weights

    # Creating the ML Trainer. We will adapt only the means
    trainer = bob.learn.em.ML_GMMTrainer(
        update_means=True, update_variances=False, update_weights=False)
    bob.learn.em.train(trainer, ubm, features)

    return ubm

def jfa_train(features, ubm):
    """
     Trains U and V matrix

     **Parameters**
       features: List of :py:class:`bob.learn.em.GMMStats` organized by class

       n_gaussians: UBM (:py:class:`bob.learn.em.GMMMachine`)

     """

    stats = []
    for user in features:
        user_stats = []
        for f in user:
            s = bob.learn.em.GMMStats(ubm.shape[0], ubm.shape[1])
            ubm.acc_statistics(f, s)
            user_stats.append(s)
        stats.append(user_stats)

    subspace_dimension_of_u = 1
    subspace_dimension_of_v = 1

    jfa_base = bob.learn.em.JFABase(
        ubm, subspace_dimension_of_u, subspace_dimension_of_v)
    trainer = bob.learn.em.JFATrainer()
    # trainer.rng = bob.core.random.mt19937(int(self.init_seed))
    bob.learn.em.train_jfa(trainer, jfa_base, stats, max_iterations=50)

    return jfa_base

# GENERATING DATA
data_per_class = bob.db.iris.data()
setosa = numpy.column_stack(
    (data_per_class['setosa'][:, 0], data_per_class['setosa'][:, 3]))
versicolor = numpy.column_stack(
    (data_per_class['versicolor'][:, 0], data_per_class['versicolor'][:, 3]))
virginica = numpy.column_stack(
    (data_per_class['virginica'][:, 0], data_per_class['virginica'][:, 3]))
data = numpy.vstack((setosa, versicolor, virginica))

# TRAINING THE PRIOR
ubm = train_ubm(data, 3)
jfa_base = jfa_train([setosa, versicolor, virginica], ubm)

# Variability direction U
u0 = jfa_base.u[0:2, 0] / numpy.linalg.norm(jfa_base.u[0:2, 0])
u1 = jfa_base.u[2:4, 0] / numpy.linalg.norm(jfa_base.u[2:4, 0])
u2 = jfa_base.u[4:6, 0] / numpy.linalg.norm(jfa_base.u[4:6, 0])

# Variability direction V
v0 = jfa_base.v[0:2, 0] / numpy.linalg.norm(jfa_base.v[0:2, 0])
v1 = jfa_base.v[2:4, 0] / numpy.linalg.norm(jfa_base.v[2:4, 0])
v2 = jfa_base.v[4:6, 0] / numpy.linalg.norm(jfa_base.v[4:6, 0])

figure, ax = plt.subplots()
plt.scatter(setosa[:, 0], setosa[:, 1], c="darkcyan", label="setosa")
plt.scatter(versicolor[:, 0], versicolor[:, 1],
            c="goldenrod", label="versicolor")
plt.scatter(virginica[:, 0], virginica[:, 1], c="dimgrey", label="virginica")

plt.scatter(ubm.means[:, 0], ubm.means[:, 1], c="blue",
            marker="x", label="centroids - mle")
# plt.scatter(ubm.means[:, 0], ubm.means[:, 1], c="blue",
#             marker=".", label="within class varibility", s=0.01)

# U
ax.arrow(ubm.means[0, 0], ubm.means[0, 1], u0[0], u0[1],
         fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[1, 0], ubm.means[1, 1], u1[0], u1[1],
         fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[2, 0], ubm.means[2, 1], u2[0], u2[1],
         fc="k", ec="k", head_width=0.05, head_length=0.1)
plt.text(ubm.means[0, 0] + u0[0], ubm.means[0, 1] +
         u0[1] - 0.1, r'$\mathbf{U}_1$', fontsize=15)
plt.text(ubm.means[1, 0] + u1[0], ubm.means[1, 1] +
         u1[1] - 0.1, r'$\mathbf{U}_2$', fontsize=15)
plt.text(ubm.means[2, 0] + u2[0], ubm.means[2, 1] +
         u2[1] - 0.1, r'$\mathbf{U}_3$', fontsize=15)

# V
ax.arrow(ubm.means[0, 0], ubm.means[0, 1], v0[0], v0[1],
         fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[1, 0], ubm.means[1, 1], v1[0], v1[1],
         fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[2, 0], ubm.means[2, 1], v2[0], v2[1],
         fc="k", ec="k", head_width=0.05, head_length=0.1)
plt.text(ubm.means[0, 0] + v0[0], ubm.means[0, 1] +
         v0[1] - 0.1, r'$\mathbf{V}_1$', fontsize=15)
plt.text(ubm.means[1, 0] + v1[0], ubm.means[1, 1] +
         v1[1] - 0.1, r'$\mathbf{V}_2$', fontsize=15)
plt.text(ubm.means[2, 0] + v2[0], ubm.means[2, 1] +
         v2[1] - 0.1, r'$\mathbf{V}_3$', fontsize=15)

plt.xticks([], [])
plt.yticks([], [])

# plt.grid(True)
plt.xlabel('Sepal length')
plt.ylabel('Petal width')
plt.legend(loc=2)
plt.ylim([-1, 3.5])

plt.tight_layout()
# plt.show()


text:
from PIL import Image
import model
import seamCarver as sc
import numpy as np
import random as rand

# img = Image.open(r"C:\users\sunny\OneDrive\Desktop\ImageEditor\test.png")
# seam = sc.findLowestSeam(np.array(img), 0)
# print(seam)
# img = sc.carveSeam(img, 2, 3)
# seam = sc.findLowestSeam(img, True)
# print(seam)
# img = sc.remove_seam(img, seam, 1)
# img.save(r"C:\users\sunny\OneDrive\Desktop\ImageEditor\test2.png")
# print(img.size)

img = np.array([[[255, 255, 0], [0, 255, 0], [0, 255, 255]],
[[0, 255, 0], [255, 0, 255], [255, 0, 0]],
[[0, 255, 255], [255, 255, 0], [0, 255, 0]]])
img = sc.getEnergyMap(img)
print(img)

# img = img.resize((300, 300))
# img.save(r"C:\users\sunny\OneDrive\Desktop\ImageEditor\test.png")


text:
# Copyright (c) Facebook, Inc. and its affiliates.

import torch
from mmf.common.registry import registry
from mmf.models.pythia import Pythia
from mmf.models.memo_net import MemoNet
from mmf.models.gmn import GraphMemoNet
from mmf.modules.embeddings import (
    ImageFeatureEmbedding,
    MultiHeadImageFeatureEmbedding,
    PreExtractedEmbedding,
    TextEmbedding,
)
from mmf.modules.layers import MemNNLayer
# from mmf.modules.layers import EdgeLayer
# from mmf.modules.layers import NodeLayer
# from mmf.modules.layers import GlobalLayer
# from mmf.modules.layers import GraphMemoryLayer
from mmf.modules.layers import GraphLayer
from mmf.modules.fusions import MCB
# from mmf.modules.layers import MemoLayer
# from torch_scatter import scatter_mean
# from torch_geometric.nn import MetaLayer
from torch import nn
import numpy as np
import cv2
torch.autograd.set_detect_anomaly(True)

@registry.register_model("gmn2")
class GraphMemoNet2(Pythia):
    def __init__(self, config):
        super().__init__(config)

    @classmethod
    def config_path(cls):
        return "configs/models/gmn2/defaults.yaml"

    def build(self):
        super().build()

        self._build_word_embedding()
        self._init_text_embeddings("text")
        self._init_feature_encoders("image")
        self._init_feature_embeddings("image")
        self._init_GMN()
        self._init_MCB()
        # self._init_memo_layer()
        # self._init_MN("image")
        self._init_combine_layer("image", "text")
        self._init_classifier(self._get_classifier_input_dim())
        self._init_extras()

    def _init_GMN(self):
        self.visual_graph = GraphLayer("image", hidden_dim=2048, node_dim=2048, edge_dim=2048)
        # self.textual_graph = GraphLayer("text", hidden_dim=2048, node_dim=2048, edge_dim=2048)
        # 1204 2048
        # self.memory =

    def _init_MCB(self):
        input_dims = [self.config.mcb.visual_input_dim, self.config.mcb.textual_input_dim]
        output_dim = self.config.mcb.output_dim
        self.mcb = MCB(input_dims, output_dim)

    def is_adjcent(self, bbox1, bbox2):
        if ((bbox1==[0,0,0,0]) or (bbox2==[0,0,0,0])):
            adj = False
        else:
            d_ecu_convert = (bbox1[0]+bbox1[2]-bbox2[0]-bbox2[2])*(bbox1[0]+bbox1[2]-bbox2[0]-bbox2[2])+(bbox1[1]+bbox1[3]-bbox2[1]-bbox2[3])*(bbox1[1]+bbox1[3]-bbox2[1]-bbox2[3])
            adj = (d_ecu_convert<1.0)
        return adj

    # def _init_memo_layer(self, H, W):
    #     self.memo_layer = MemoLayer()

    def process_feature_embedding(
        self, attr, sample_list, text_embedding_total, extra=None, batch_size_t=None
    ):
        if extra is None:
            extra = []
        feature_embeddings = []
        feature_attentions = []   #useless var
        features = []
        batch_size_t = (
            sample_list.get_batch_size() if batch_size_t is None else batch_size_t
        )
        # print(batch_size_t)

        ## metadata
        bs = batch_size_t
        ## Convert list of keys to the actual values
        extra = sample_list.get_fields(extra)
        ## bboxes
        bbox_feature = torch.tensor(sample_list.image_info_0["bbox"]).cuda()  # [bs, 100, 4]

        # ====print img with bbox==========
        # for i in range (bs):
        #     img_src = sample_list.region_description["image_url"][i]
        #     cap = cv2.VideoCapture(img_src)
        #     if( cap.isOpened() ) :
        #         ret,img = cap.read()
        #         # cv2.imshow("image",img)
        #         # cv2.waitKey()
        #     for j in bbox_feature[i]:
        #         cv2.rectangle(img, (j[0],j[1]), (j[2],j[3]), (0,255,0), 1)
        #     for k in bbox_phrase[i]:
        #         p1 = (int(k[0]*sample_list.image_info_0["image_width"][i]),int(k[1]*sample_list.image_info_0["image_height"][i]))
        #         p2 = (int(k[2]*sample_list.image_info_0["image_width"][i]),int(k[3]*sample_list.image_info_0["image_height"][i]))
        #         print(p1,p2)
        #         cv2.rectangle(img, p1, p2, (255,0,0), 1)
        #     print(sample_list.region_description["image_id"][i])
        #     cv2.imwrite(str(sample_list.region_description["image_id"][i].item())+".jpg", img)
        for i in range (bs):
            bbox_feature[i, :, 0] /= sample_list.image_info_0["image_width"][i]
            bbox_feature[i, :, 1] /= sample_list.image_info_0["image_height"][i]
            bbox_feature[i, :, 2] /= sample_list.image_info_0["image_width"][i]
            bbox_feature[i, :, 3] /= sample_list.image_info_0["image_height"][i]
        # print("bbox_feature", bbox_feature)

        feature_idx = 0

        # Get all of the features, which are in the form, "image_feature_0"
        # "image_feature_1" ...
        while True:
            feature = getattr(sample_list, f"{attr}_feature_{feature_idx:d}", None)
            if feature is None:
                break
            # print("feature_idx", feature_idx)
            # print("feature1", feature.size()) # torch.Size([4, 100/196, 2048])
            feature_idx += 1
            feature = feature[:batch_size_t]
            # print("feature2", feature.size())
            features.append(feature)

        feature_encoders = getattr(self, attr + "_feature_encoders")

        # print("=====feat encoding=====")
        # print("feat_encoders", feature_encoders)
        # result:
        # feat_encoders ModuleList(
        #     (0): ImageFeatureEncoder(
        #         (module): FinetuneFasterRcnnFpnFc7(
        #         (lc): Linear(in_features=2048, out_features=2048, bias=True)
        #         )
        #     )
        #     (1): ImageFeatureEncoder(
        #         (module): Identity()
        #     )
        # )

        # Each feature should have a separate image feature encoders
        assert len(features) == len(feature_encoders), (
            "Number of feature encoders, {} are not equal "
            "to number of features, {}.".format(len(feature_encoders), len(features))
        )

        # Now, iterate to get final attended image features
        encoded_feature = []
        for i, feature in enumerate(features):
            # Get info related to the current feature. info is generally
            # in key of format "image_info_0" for 0th feature
            feature_info = getattr(sample_list, f"{attr}_info_{i:d}", {})
            # print("feature_i: ", i, feature.size())   # torch.Size([4, 100/196, 2048])
            # print("feature_info", feature_info)  # {}

            # For Pythia, we need max_features to mask attention
            feature_dim = getattr(feature_info, "max_features", None)
            if feature_dim is not None:
                feature_dim = feature_dim[:batch_size_t]
            # print("max_feat_dim_in_gmn2_processor", feature_dim)  # [batch, max_feat], none

            # Attribute in which encoders are saved, for "image" it
            # will be "image_feature_encoders", other example is
            # "context_feature_encoders"
            encoders_attr = attr + "_feature_encoders"
            feature_encoder = getattr(self, encoders_attr)[i]
            # Encode the features
            encoded_feature.append(feature_encoder(feature))

            # print("encoded_feat:", i, encoded_feature.size()) # torch.Size([64, 100/196, 2048])
            #feature1--finetune; feat2--identity
            list_attr = attr + "_feature_embeddings_list"
            feature_embedding_models = getattr(self, list_attr)[i]  # image_feature_embeddings_list

        # print("=====feat_embedding===== ")
        # Forward through these embeddings one by one
        # current data: encoded_feature[0,1], text_embedding_total, embedding_phrase total, feature_dim, extra, bbox_phrase, bbox_feature

        # init graph data
        visual_node_features = [[] for i in range (bs)]
        visual_edge_ends = [[] for i in range (bs)]
        visual_edge_features = [[] for i in range (bs)]

        visual_edge_feat_dim = 2048 # 2048

        visual_global_features = encoded_feature[1][:,0,:] #?? last layer of resnet 101

        ## visual graph init
        for i in range(bs):
            for j in range (len(encoded_feature[0][i])):
                # j-th feature in i-th batch
                # print("mcb", encoded_feature[0][i][j].size(), text_embedding_total[i].size()) # torchsize([2048])
                visual_node_features[i].append((self.mcb( [encoded_feature[0][i][j], text_embedding_total[i]] )).unsqueeze(0)) # [bs, 100, 2048]
                for k in range(len(bbox_feature[i])):
                    if self.is_adjcent(bbox_feature[i][j], bbox_feature[i][k]):
                        visual_edge_ends[i].append([j,k]) # [bs, num_edges, 2]
                        visual_edge_features[i].append(torch.zeros(1,visual_edge_feat_dim))# [bs, num_edges, 2048]
            visual_edge_features[i] = torch.cat(visual_edge_features[i]).cuda(0)
            visual_node_features[i] = torch.cat(visual_node_features[i])
                # visual_nodes[i] = VisualNodeModel(visual_node_features[i])
                # visual_edges[i] = VisualEdgeModel()

            # ## textual graph init
            # # print(embedding_phrase_total[i]) # [1,4,50/49,2048]
            # embedding_phrase_total[0][i]=torch.cat(embedding_phrase_total[0][i], dim = 0)
            # # print("embedding_phrase_total", embedding_phrase_total[0][0].size())
            # # print("text_embedding_total", text_embedding_total[i].size())
            # textual_node_features[i]= (embedding_phrase_total[0][i]*text_embedding_total[i]).unsqueeze(0) # [49, 2048] * [2048] =>[49,2048]
            # #  textual_node_features = torch.mmf(embedding_phrase_total,text_embedding_total)
            # # print("num_regions_vg", num_regions_vg[i], len(bbox_phrase[i]))
            # ## edge init
            # for j in range (num_regions_vg[i]):
            #     # textual_node_features[i].append(multiply(embedding_phrase_total[i][j], text_embedding_total[i]))
            #     for k in range (len(bbox_phrase[i])):
            #         if self.is_adjcent(bbox_phrase[i][j], bbox_phrase[i][k]):
            #             textual_edge_ends[i].append([j,k]) # [bs, num_edges, 2]
            #             textual_edge_features[i].append(torch.zeros(1,textual_edge_feat_dim))# [bs, num_edges, 2048]
            # textual_edge_features[i] = torch.cat(textual_edge_features[i]).cuda(0)
            # # textual_nodes[i] = TextualNodeModel(extual_node_features[i])
            # # textual_edges[i] = TextualEdgeModel()

            visual_node_features[i], visual_edge_features[i], visual_global_features[i] = self.visual_graph(visual_node_features[i], visual_edge_ends[i], visual_edge_features[i], visual_global_features[i])
            # textual_node_features[i], textual_edge_features[i], textual_global_features[i] = self.textual_graph(textual_node_features[i], textual_edge_ends[i], textual_edge_features[i], textual_global_features[i])
            # visual_node_features, visual_edge_features, visual_global_features = self.visual_graph(visual_node_features, visual_edge_ends, visual_edge_features, visual_global_features)
            # textual_node_features, textual_edge_features, textual_global_features = self.textual_graph(textual_node_features, textual_edge_ends, textual_edge_features, textual_global_features)

        # print("len vis node feat", len(visual_node_features)) #[4*[100, 2048]]
        # print("vis node feat[0]", visual_node_features[1].size())
        if feature_dim is not None:
            encoded_feature[0] = torch.cat(visual_node_features, dim = 0).reshape(len(visual_node_features),feature_dim,2048)
        # print(encoded_feature[0].size())

        for i, feature in enumerate(features):
            for feature_embedding_model in feature_embedding_models:
                inp = (encoded_feature[i], text_embedding_total, feature_dim, extra)
                # torch.Size([64, 100, 2048]), [64,2048], none, samplelist()
                # print(feature_embedding_model) #attn & identity as listed in yml
                # print(encoded_feature[i].size())
                # print(text_embedding_total.size())
                # print("erxtra", extra) # infos in samplelist with key extra

                embedding, attention = feature_embedding_model(*inp)

                # out = self.GMNs[i](encoded_feature, text_embedding_total, embedding_phrase_total, )
                # memo = self.MNs[i](encoded_feature, text_embedding_total, embedding_phrase_total)  # torch.Size([bs, 2048])
                # memo = self.MNs[i](encoded_feature_512, text_embedding_total)  # torch.Size([bs, 2048])
                # print("memo:", memo.size())
                # print("embedding", embedding.size())

                # embedding_memo_superpos = embedding + memo
                # feature_embeddings.append(embedding_memo_superpos)
                # feature_embeddings.append(memo)
                feature_embeddings.append(embedding)

        # Concatenate all features embeddings and return along with attention
        feature_embedding_total = torch.cat(feature_embeddings, dim=1)

        # print("feature_embeddings_tot", feature_embedding_total.size()) #[bs*4096]

        return feature_embedding_total, feature_attentions

    def get_optimizer_parameters(self, config):
        combine_layer = self.image_text_multi_modal_combine_layer
        params = [
            {"params": self.word_embedding.parameters()},
            {"params": self.image_feature_embeddings_list.parameters()},
            {"params": self.text_embeddings.parameters()},
            {"params": self.mcb.parameters()},
            {"params": self.visual_graph.parameters()},
            # {"params": self.textual_graph.parameters()},
            {"params": combine_layer.parameters()},
            {"params": self.classifier.parameters()},
            {
                "params": self.image_feature_encoders.parameters(),
                "lr": (config.optimizer.params.lr * 0.1),
            },
        ]

        return params

    def print_sample_list(self, sample_list):
        print ("=====sample_list=====")
        print(sample_list.fields())
        for key in sample_list.keys():
            print(key+":")
            # print(type(sample_list[key]))
            if isinstance(sample_list[key],str) :
                print("str:", sample_list[key])
            elif isinstance(sample_list[key],dict) : # region description: dict
                for key2 in sample_list[key].keys():
                    print("    "+key2+":")
                    # if type(sample_list[key][key2]) is np.ndarray:
                    #     print(sample_list[key][key2].shape)
                    # else:
                    #     print(sample_list[key][key2])
                    print(sample_list[key][key2])
            elif isinstance(sample_list[key],list) :
                for i in sample_list[key]:  # image info 1: [none, ]
                    if i != None:
                        print(i.keys(), i.values)
                    else:
                        print (i)
            else:
                print(sample_list[key].size())

    def forward(self, sample_list):
        # self.print_sample_list(sample_list)

        ## metadata
        bs = len(sample_list.question_id)

        ## question & caption word embedding(word->300D vec)
        # print("text", sample_list.text)  #tensor[4,20]
        sample_list.text = self.word_embedding(sample_list.text)
        # print("text", sample_list.text.size()) # torch.Size([4, 20, 300])

        # question & caption GRU embedding
        text_embedding_total = self.process_text_embedding(sample_list)
        # print("text_embedding", text_embedding_total.size())

        # image feature embedding with MCB+graph
        image_embedding_total, _ = self.process_feature_embedding(
            "image", sample_list, text_embedding_total
        )
        # print("img_embedding", image_embedding_total.size())

        if self.inter_model is not None:
            image_embedding_total = self.inter_model(image_embedding_total)

        # print("image_embedding", image_embedding_total.size()) # [batch*4096]
        # print("text_embedding:" , image_embedding_total.size()) # [batch*4096]

        # print("=======memory======")

        # print("=====combine layer=====")

        joint_embedding = self.combine_embeddings(
            ["image", "text"], [image_embedding_total, text_embedding_total]
        )

        # print("joint_embedding:", joint_embedding.size()) # [batch*5000]

        model_output = {"scores": self.calculate_logits(joint_embedding)}

        # print("model_output:", model_output['scores'].size())
        # for name, param in self.MNs.named_parameters():
        #     print (name, param)

        return model_output


text:
import numpy as np
import scipy

try:
    import librosa  # noqa
    import librosa.effects  # noqa
    import librosa.feature  # noqa
except ImportError:
    raise ImportError("Please install librosa with: `pip install librosa`")
import soundfile as sf

try:
    from fastdtw import fastdtw  # noqa
except ImportError:
    raise ImportError("Please install fastdtw with: `pip install fastdtw`")
from pororo.models.tts.tacotron.params import Params as hp

def load(path):
    """Load a sound file into numpy array."""
    data, sample_rate = sf.read(path)
    assert (
        hp.sample_rate == sample_rate
    ), f"Sample rate do not match: given {hp.sample_rate}, expected {sample_rate}"
    return data

def save(data, path):
    """Save numpy array as sound file."""
    sf.write(path, data, samplerate=hp.sample_rate)

def ms_to_frames(ms):
    """Convert milliseconds into number of frames."""
    return int(hp.sample_rate * ms / 1000)

def trim_silence(data, window_ms, hop_ms, top_db=50, margin_ms=0):
    """Trim leading and trailing silence from an audio signal."""
    wf = ms_to_frames(window_ms)
    hf = ms_to_frames(hop_ms)
    mf = ms_to_frames(margin_ms)
    if mf != 0:
        data = data[mf:-mf]
    return librosa.effects.trim(data,
                                top_db=top_db,
                                frame_length=wf,
                                hop_length=hf)

def duration(data):
    """Return duration of an audio signal in seconds."""
    return librosa.get_duration(data, sr=hp.sample_rate)

def amplitude_to_db(x):
    """Convert amplitude to decibels."""
    return librosa.amplitude_to_db(x, ref=np.max, top_db=None)

def db_to_amplitude(x):
    """Convert decibels to amplitude."""
    return librosa.db_to_amplitude(x)

def preemphasis(y):
    """Preemphasize the signal."""
    # y[n] = x[n] - perc * x[n-1]
    return scipy.signal.lfilter([1, -hp.preemphasis], [1], y)

def spectrogram(y, mel=False):
    """Convert waveform to log-magnitude spectrogram."""
    if hp.use_preemphasis:
        y = preemphasis(y)
    wf = ms_to_frames(hp.stft_window_ms)
    hf = ms_to_frames(hp.stft_shift_ms)
    S = np.abs(librosa.stft(y, n_fft=hp.num_fft, hop_length=hf, win_length=wf))
    if mel:
        S = librosa.feature.melspectrogram(S=S,
                                           sr=hp.sample_rate,
                                           n_mels=hp.num_mels)
    return amplitude_to_db(S)

def mel_spectrogram(y):
    """Convert waveform to log-mel-spectrogram."""
    return spectrogram(y, True)

def linear_to_mel(S):
    """Convert linear to mel spectrogram (this does not return the same spec. as mel_spec. method due to the db->amplitude conversion)."""
    S = db_to_amplitude(S)
    S = librosa.feature.melspectrogram(S=S,
                                       sr=hp.sample_rate,
                                       n_mels=hp.num_mels)
    return amplitude_to_db(S)

def normalize_spectrogram(S, is_mel):
    """Normalize log-magnitude spectrogram."""
    if is_mel:
        return (S - hp.mel_normalize_mean) / hp.mel_normalize_variance
    else:
        return (S - hp.lin_normalize_mean) / hp.lin_normalize_variance

def denormalize_spectrogram(S, is_mel):
    """Denormalize log-magnitude spectrogram."""
    if is_mel:
        return S * hp.mel_normalize_variance + hp.mel_normalize_mean
    else:
        return S * hp.lin_normalize_variance + hp.lin_normalize_mean

reason: import numpy,import scipy


text:
import networkx as NX

def snowball(net, seed, max_depth = 3, maxnodes=1000):
    seen = set()
    queue = set()

    queue.add(seed)
    queue2 = set()

    for _ in range(max_depth+1):
	    while queue:
	        user_id = queue.pop()
	        seen.add(user_id)

	        NN = net.neighbors(user_id)

	        for node in NN:
	            if node not in seen:
	                queue2.add(node)

	    queue.update(queue2)
	    queue2 = set()

    return seen

net = NX.connected_watts_strogatz_graph(10000, 4, 0.01)
neve = snowball(net, 0)

print(neve)

reason: import networkx


text:
# demonstrates how to do gradient descent with numpy matrices,
# with regularization.
#
# the notes for this class can be found at:
# https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
# https://www.udemy.com/data-science-logistic-regression-in-python

from __future__ import print_function, division
from builtins import range
# Note: you may need to update your version of future
# sudo pip install -U future

import numpy as np

N = 100
D = 2

X = np.random.randn(N,D)

# center the first 50 points at (-2,-2)
X[:50,:] = X[:50,:] - 2*np.ones((50,D))

# center the last 50 points at (2, 2)
X[50:,:] = X[50:,:] + 2*np.ones((50,D))

# labels: first 50 are 0, last 50 are 1
T = np.array([0]*50 + [1]*50)

# add a column of ones
# ones = np.array([[1]*N]).T
ones = np.ones((N, 1))
Xb = np.concatenate((ones, X), axis=1)

# randomly initialize the weights
w = np.random.randn(D + 1)

# calculate the model output
z = Xb.dot(w)

def sigmoid(z):
    return 1/(1 + np.exp(-z))

Y = sigmoid(z)

# calculate the cross-entropy error
def cross_entropy(T, Y):
    E = 0
    for i in range(len(T)):
        if T[i] == 1:
            E -= np.log(Y[i])
        else:
            E -= np.log(1 - Y[i])
    return E

# let's do gradient descent 100 times
learning_rate = 0.1
for i in range(100):
    if i % 10 == 0:
        print(cross_entropy(T, Y))

    # gradient descent weight udpate with regularization
    w += learning_rate * ( Xb.T.dot(T - Y) - 0.1*w )

    # recalculate Y
    Y = sigmoid(Xb.dot(w))

print("Final w:", w)

max_stars_repo_name: JouniVatanen/NLP-and-Deep-Learning


text:
# Co-authored-by: Nitesh Yadav <niteshyadav585@gmail.com>
# Co-authored-by: John Andersen <johnandersenpdx@gmail.com>
import pathlib
from typing import AsyncIterator

import joblib
import numpy as np
import pandas as pd
from xgboost import XGBClassifier

from dffml.record import Record
from dffml.base import config, field
from dffml.source.source import Sources
from dffml.util.entrypoint import entrypoint
from dffml.feature.feature import Feature, Features
from dffml.model.model import SimpleModel, ModelNotTrained

@config
class XGBClassifierModelConfig:
    location: pathlib.Path = field("Location where model should be saved")
    features: Features = field("Features on which we train the model")
    predict: Feature = field("Value to be predicted")
    learning_rate: float = field(
        "Learning rate to train with", mutable=True, default=0.3
    )
    n_estimators: int = field(
        "Number of gradient boosted trees. Equivalent to the number of boosting rounds",
        default=100,
        mutable=True,
    )
    max_depth: int = field(
        "Maximium tree depth for base learners", default=6, mutable=True
    )
    objective: str = field("Objective in training", default="multi:softmax")
    subsample: float = field(
        "Subsample ratio of the training instance", default=1
    )
    gamma: float = field(
        "Minimium loss reduction required to make a furthre partition on a leaf node",
        default=0,
    )
    n_jobs: int = field(
        "Number of parallel threads used to run xgboost", default=-1
    )
    colsample_bytree: float = field(
        "Subsample ratio of columns when constructing each tree", default=1
    )
    booster: str = field(
        "Specify which booster to use: gbtree, gblinear or dart",
        default="gbtree",
    )
    min_child_weight: float = field(
        "Minimum sum of instance weight(hessian) needed in a child", default=1
    )
    reg_lambda: float = field(
        "L2 regularization term on weights. Increasing this value will make model more conservative",
        default=1,
    )
    reg_alpha: float = field(
        "L1 regularization term on weights. Increasing this value will make model more conservative",
        default=0,
    )

@entrypoint("xgbclassifier")
class XGBClassifierModel(SimpleModel):
    r"""
    Model using xgboost to perform classification prediction via gradient boosted trees.
    XGBoost is a leading software library for working with standard tabular data (the type of data you store in Pandas DataFrames,
    as opposed to more exotic types of data like images and videos). With careful parameter tuning, you can train highly accurate models.

    Examples
    --------

    Command line usage

    First download the training and test files, change the headers to DFFML
    format. The first row is an encoding of the classifications, we want CSV
    headers for the column names.

    .. code-block::
        :test:

        $ wget http://download.tensorflow.org/data/iris_training.csv
        $ wget http://download.tensorflow.org/data/iris_test.csv
        $ sed -i 's/.*setosa,versicolor,virginica/SepalLength,SepalWidth,PetalLength,PetalWidth,classification/g' iris_training.csv iris_test.csv

    Run the train command

    .. code-block:: console
        :test:

        $ dffml train \
            -sources train=csv \
            -source-filename iris_training.csv \
            -model xgbclassifier \
            -model-features \
              SepalLength:float:1 \
              SepalWidth:float:1 \
              PetalLength:float:1 \
              PetalWidth:float:1 \
            -model-predict classification \
            -model-location model \
            -model-max_depth 3 \
            -model-learning_rate 0.01 \
            -model-learning_rate 0.01 \
            -model-n_estimators 200 \
            -model-reg_lambda 1 \
            -model-reg_alpha 0 \
            -model-gamma 0 \
            -model-colsample_bytree 0 \
            -model-subsample 1

    Assess the accuracy

    .. code-block:: console
        :test:

        $ dffml accuracy \
            -sources train=csv \
            -source-filename iris_test.csv \
            -model xgbclassifier \
            -model-features \
              SepalLength:float:1 \
              SepalWidth:float:1 \
              PetalLength:float:1 \
              PetalWidth:float:1 \
            -model-predict classification \
            -model-location model \
            -features classification \
            -scorer clf

    Make predictions

    .. code-block:: console
        :test:

        $ dffml predict all \
            -sources train=csv \
            -source-filename iris_test.csv \
            -model xgbclassifier \
            -model-features \
              SepalLength:float:1 \
              SepalWidth:float:1 \
              PetalLength:float:1 \
              PetalWidth:float:1 \
            -model-predict classification \
            -model-location model

    Python usage

    .. literalinclude:: /../model/xgboost/examples/iris_classification.py

    Output

    .. code-block::

        Test accuracy: 0.933333333333333
        Training accuracy: 0.9703703703703703

    """

    CONFIG = XGBClassifierModelConfig

    def __init__(self, config) -> None:
        super().__init__(config)
        # The saved model
        self.saved = None

    async def __aenter__(self):
        await super().__aenter__()
        self.saved_filepath = pathlib.Path(
            self.parent.config.location
            if not hasattr(self.parent, "temp_dir")
            else self.parent.temp_dir,
            "model.joblib",
        )
        # Load saved model if it exists
        if self.saved_filepath.is_file():
            self.saved = joblib.load(str(self.saved_filepath))
            self.is_trained = True
        return self

    async def __aexit__(self, exc_type, exc_value, traceback):
        if self.saved:
            # Save the trained model
            joblib.dump(self.saved, str(self.saved_filepath))
        await super().__aexit__(exc_type, exc_value, traceback)

    async def train(self, sources: Sources) -> None:
        """
        Trains and saves a model using the source data, and the config attributes
        """
        # Get data into memory
        xdata = []
        ydata = []
        async for record in sources.with_features(
            self.features + [self.parent.config.predict.name]
        ):
            record_data = []
            for feature in record.features(self.features).values():
                record_data.extend(
                    [feature] if np.isscalar(feature) else feature
                )
            xdata.append(record_data)
            ydata.append(record.feature(self.parent.config.predict.name))
        x_data = pd.DataFrame(xdata)
        y_data = pd.DataFrame(ydata)

        self.saved = XGBClassifier(
            n_estimators=self.config.n_estimators,
            learning_rate=self.config.learning_rate,
            max_depth=self.config.max_depth,
            objective=self.config.objective,
            subsample=self.config.subsample,
            gamma=self.config.gamma,
            n_jobs=self.config.n_jobs,
            colsample_bytree=self.config.colsample_bytree,
            booster=self.config.booster,
            min_child_weight=self.config.min_child_weight,
            reg_lambda=self.config.reg_lambda,
            reg_alpha=self.config.reg_alpha,
        )

        self.saved.fit(x_data, y_data, eval_metric="merror")
        self.is_trained = True

    async def predict(self, sources: Sources) -> AsyncIterator[Record]:
        """
        Uses saved model to make prediction off never seen before data
        """
        if not self.is_trained:
            raise ModelNotTrained(
                "Train the model first before getting predictions"
            )
        # Grab records and input data (X data)
        input_data = await self.get_input_data(sources)
        # Make predictions
        xdata = []
        for record in input_data:
            record_data = []
            for feature in record.features(self.features).values():
                record_data.extend(
                    [feature] if np.isscalar(feature) else feature
                )
            xdata.append(record_data)

        predictions = self.saved.predict(pd.DataFrame(xdata))
        # Update records and yield them to caller
        for record, prediction in zip(input_data, predictions):
            record.predicted(
                self.config.predict.name, float(prediction), float("nan")
            )
            yield record

    async def get_input_data(self, sources: Sources) -> list:
        saved_records = []
        async for record in sources.with_features(
            self.config.features.names()
        ):
            saved_records.append(record)
        return saved_records

max_stars_repo_name: agriyakhetarpal/dffml

max_issues_repo_name: agriyakhetarpal/dffml

max_forks_repo_name: agriyakhetarpal/dffml


text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import numpy as np
import paddle
import scipy.stats

from config import ATOL, DEVICES, RTOL
from parameterize import TEST_CASE_NAME, parameterize_cls, place, xrand

paddle.enable_static()

@place(DEVICES)
@parameterize_cls((TEST_CASE_NAME, 'concentration'),
                  [('test-one-dim', np.random.rand(89) + 5.0)])
class TestDirichlet(unittest.TestCase):
    def setUp(self):
        self.program = paddle.static.Program()
        self.executor = paddle.static.Executor()
        with paddle.static.program_guard(self.program):
            conc = paddle.static.data('conc', self.concentration.shape,
                                      self.concentration.dtype)
            self._paddle_diric = paddle.distribution.Dirichlet(conc)
            self.feeds = {'conc': self.concentration}

    def test_mean(self):
        with paddle.static.program_guard(self.program):
            [out] = self.executor.run(self.program,
                                      feed=self.feeds,
                                      fetch_list=[self._paddle_diric.mean])
            np.testing.assert_allclose(
                out,
                scipy.stats.dirichlet.mean(self.concentration),
                rtol=RTOL.get(str(self.concentration.dtype)),
                atol=ATOL.get(str(self.concentration.dtype)))

    def test_variance(self):
        with paddle.static.program_guard(self.program):
            [out] = self.executor.run(self.program,
                                      feed=self.feeds,
                                      fetch_list=[self._paddle_diric.variance])
            np.testing.assert_allclose(
                out,
                scipy.stats.dirichlet.var(self.concentration),
                rtol=RTOL.get(str(self.concentration.dtype)),
                atol=ATOL.get(str(self.concentration.dtype)))

    def test_prob(self):
        with paddle.static.program_guard(self.program):
            random_number = np.random.rand(*self.concentration.shape)
            random_number = random_number / random_number.sum()
            feeds = dict(self.feeds, value=random_number)
            value = paddle.static.data('value', random_number.shape,
                                       random_number.dtype)
            out = self._paddle_diric.prob(value)
            [out] = self.executor.run(self.program,
                                      feed=feeds,
                                      fetch_list=[out])
            np.testing.assert_allclose(
                out,
                scipy.stats.dirichlet.pdf(random_number, self.concentration),
                rtol=RTOL.get(str(self.concentration.dtype)),
                atol=ATOL.get(str(self.concentration.dtype)))

    def test_log_prob(self):
        with paddle.static.program_guard(self.program):
            random_number = np.random.rand(*self.concentration.shape)
            random_number = random_number / random_number.sum()
            feeds = dict(self.feeds, value=random_number)
            value = paddle.static.data('value', random_number.shape,
                                       random_number.dtype)
            out = self._paddle_diric.log_prob(value)
            [out] = self.executor.run(self.program,
                                      feed=feeds,
                                      fetch_list=[out])
            np.testing.assert_allclose(
                out,
                scipy.stats.dirichlet.logpdf(random_number, self.concentration),
                rtol=RTOL.get(str(self.concentration.dtype)),
                atol=ATOL.get(str(self.concentration.dtype)))

    def test_entropy(self):
        with paddle.static.program_guard(self.program):
            [out] = self.executor.run(
                self.program,
                feed=self.feeds,
                fetch_list=[self._paddle_diric.entropy()])
            np.testing.assert_allclose(
                out,
                scipy.stats.dirichlet.entropy(self.concentration),
                rtol=RTOL.get(str(self.concentration.dtype)),
                atol=ATOL.get(str(self.concentration.dtype)))

max_stars_repo_name: RangeKing/Paddle

max_issues_repo_name: RangeKing/Paddle

max_forks_repo_name: RangeKing/Paddle


text:
from Project.src.NeuralNetworks.neural_networks import NN
import numpy as np

classFeatures = [("mon_intensity", np.float),
                 ("mon_duration", np.float),
                 ("mon_hr", np.float),
                 ("tue_intensity", np.float),
                 ("tue_duration", np.float),
                 ("tue_hr", np.float),
                 ("wed_intensity", np.float),
                 ("wed_duration", np.float),
                 ("wed_hr", np.float),
                 ("thu_intensity", np.float),
                 ("thu_duration", np.float),
                 ("thu_hr", np.float),
                 ("fri_intensity", np.float),
                 ("fri_duration", np.float),
                 ("fri_hr", np.float),
                 ("sat_intensity", np.float),
                 ("sat_duration", np.float),
                 ("sat_hr", np.float),
                 ("sun_intensity", np.float),
                 ("sun_duration", np.float),
                 ("sun_hr", np.float),
                 ("weekIntensity", np.float),
                 ("weekDuration", np.float),
                 ("weekHearthrate", np.float),
                 ("weekOveral", np.float)
                 ]
classes = {
    "bad": 0,
    "good": 1
}
classificationResult = "bestAtlethes"

trainPath = "../../../Data/week_riders_random_train.csv"
testPath = "../../../Data/week_riders_random_test.csv"

nn = NN(classFeatures, classificationResult, classes, trainPath, testPath, ";")

solvers = ['lbfgs', 'sgd', 'adam']
activations = ['identity', 'logistic', 'tanh', 'relu']

itters = 100
result = {}

for iter in range(0, itters):

    for solver in solvers:
        for activation in activations:
            result.setdefault(solver + '-' + activation, {'score': 0, 'f1_score': 0})
            res_nn = nn.run(solver, activation)
            result[solver + '-' + activation]['score'] += res_nn['score']
            result[solver + '-' + activation]['f1_score'] += res_nn['f1_score']

    print("Hello " + str(iter) + "%")

result2 = {}
for key, value in result.items():
    val = value
    val['score'] = value['score'] / itters
    val['f1_score'] = value['f1_score'] / itters
    result2[key] = val

print(result)


text:
import numpy as np
from scipy.io import loadmat

def get_unimib_data(s="acc"):
    print("Loading UniMiB set ", s)
    X_flat = loadmat("data/UniMiB-SHAR/data/" + s + "_data.mat")[s + "_data"]
    y = loadmat("data/UniMiB-SHAR/data/" + s + "_labels.mat")[s + "_labels"][:,0]
    if(s=="acc"):
        labels = loadmat("data/UniMiB-SHAR/data/" + s + "_names.mat")[s + "_names"][0,:]
    else:
        labels = loadmat("data/UniMiB-SHAR/data/" + s + "_names.mat")[s + "_names"][:,0]
    print("Num instances: ", len(X_flat))
    print("Instance length: ", len(X_flat[0]))

    y = np.array(y - 1)
    X = np.zeros((len(X_flat), 3, 151), dtype='float')
    X[:,0,0:151]=X_flat[:,0:151]
    X[:,1,0:151]=X_flat[:,151:302]
    X[:,2,0:151]=X_flat[:,302:453]
    print(labels)
    return X, y, labels

def get_uci_data():
    print("Loading UCI HAR Dataset")
    X_x = np.genfromtxt("data/UCI HAR Dataset/train/Inertial Signals/total_acc_x_train.txt" )
    X_y = np.genfromtxt("data/UCI HAR Dataset/train/Inertial Signals/total_acc_y_train.txt" )
    X_z = np.genfromtxt("data/UCI HAR Dataset/train/Inertial Signals/total_acc_z_train.txt" )
    X = np.zeros((7352, 3, 128))
    X[:,0,:]=X_x[:,:]
    X[:,1,:]=X_y[:,:]
    X[:,2,:]=X_z[:,:]
    y = np.genfromtxt("data/UCI HAR Dataset/train/y_train.txt")
    y = np.array(y - 1)
    with open("data/UCI HAR Dataset/activity_labels.txt") as f:
        labels = f.read().split('\n')
    labels = labels[:-1]
    return X, y, labels

def get_synthetic_set(num):
    filename = "data/synthetic/synthetic_set{}".format(num)
    #print(filename)
    X = np.genfromtxt(filename + "_data.csv", dtype="float64", delimiter=",")
    #print(X)
    X = np.reshape(X, (len(X), 1, len(X[0])))
    y = np.genfromtxt(filename+"_labels.csv")
    y = np.array(y, dtype="int")
    #print(y)
    labels = []
    for i in range(len(y)):
        labels.append("Class {}".format(i))
    return X, y, labels

if __name__ == "__main__":
    X, y, labels = get_unimib_data("two_classes")
    X, y, labels = get_uci_data()

max_stars_repo_name: gentry-atkinson/tsar

max_issues_repo_name: gentry-atkinson/tsar

max_forks_repo_name: gentry-atkinson/tsar


text:
import numpy as np
import matplotlib.pyplot as plt

def inverse_activation_function(input):
    a = 10
    b = 0
    c = 1
    x = input/21.9                                                  #21.9 is the mean distance from queries to all prototypes for unknown classes for the trained classifier for the whas dataset
    gaussian_function = a*np.exp(-(x-b)*(x-b)/(2*c))
    return gaussian_function

def computeOpenMaxProbability(openmax_embeddingLayer, openmax_score_u):

    print('openmax_embeddingLayer')
    print(openmax_embeddingLayer)
    #print(np.array(openmax_embeddingLayer).shape)
    print('openmax_score_u')
    print(openmax_score_u)
    #print(np.array(openmax_score_u).shape)
    #print(np.sum(openmax_score_u))

    class_scores = np.exp(openmax_embeddingLayer)
    total_denominator = np.sum(np.exp(openmax_embeddingLayer)) + np.exp(np.sum(openmax_score_u))
    prob_scores = class_scores/total_denominator
    prob_unknowns = np.exp(np.sum(openmax_score_u))/total_denominator

    print('total_denominator')
    print(total_denominator)
    #print('prob_scores')
    #print(prob_scores)
    #print('prob_unknowns')
    #print(prob_unknowns)
    print('sum')
    print(np.sum(prob_scores) + prob_unknowns)

    return prob_scores, prob_unknowns

def recalibrate_scores(weibull_model, query2prototype_distance, alpharank=3, distance_type='l2'):
    num_classes = len(weibull_model)

    ranked_list = (np.amax(query2prototype_distance)/query2prototype_distance).argsort()[::-1]
    alpha_weights = [((alpharank+1) - i)/float(alpharank) for i in range(1, alpharank+1)]
    ranked_alpha = np.ones(num_classes)
    activation_vector = inverse_activation_function(query2prototype_distance)
    #for i in range(len(alpha_weights)):
    #    ranked_alpha[ranked_list[i]] = alpha_weights[i]

    print('ranked_alpha')
    print(ranked_alpha)

    openmax_embeddingLayer = []
    openmax_score_u = []

    print('Distance')
    print(query2prototype_distance)
    print('Activation vector')
    print(activation_vector)

    for class_id in range(num_classes):
        class_weibull = weibull_model[class_id]

        w_score = class_weibull['weibull_model'].w_score(query2prototype_distance[class_id])
        #modified_fc8_score = (np.amax(query2prototype_distance)/query2prototype_distance[class_id])*( 1 - w_score*ranked_alpha[class_id] )
        modified_fc8_score = (activation_vector[class_id])*( 1 - w_score*ranked_alpha[class_id] )
        openmax_embeddingLayer += [modified_fc8_score]
        openmax_score_u += [activation_vector[class_id] - modified_fc8_score]
        #openmax_score_u += [np.amax(query2prototype_distance)/query2prototype_distance[class_id] - modified_fc8_score]

        #print('Class ID')
        #print(class_id)
    print('w_score')
        #print(w_score)

    print([weibull_model[class_id]['weibull_model'].w_score(query2prototype_distance[class_id]) for class_id in range(num_classes)])

    #openmax_score_u += [-query2prototype_distance[np.argmax(openmax_embeddingLayer)] + modified_fc8_score]

    openmax_class_prob, openmax_unknown_prob = computeOpenMaxProbability(openmax_embeddingLayer, openmax_score_u)

    print('openMax scores: ')
    print(openmax_class_prob, openmax_unknown_prob)

    return openmax_class_prob, openmax_unknown_prob

def unknown_prob_histogram(unknown_probs, unknown_probs_open_world):
    print("HEHERHERHEHRHEHR")
    print(unknown_probs)
    print('DSUAIHJODIASJ')
    print(unknown_probs_open_world)
    fig, ax = plt.subplots()
    n, bins, patches = ax.hist([unknown_probs, unknown_probs_open_world], color=['b', 'g'])
    #n, bins, patches = ax.hist(unknown_probs_open_world, color='g')
    ax.set_title('Unknown probs')
    plt.show()