text:
# -*- coding: utf-8 -*-
"""new_feature_creation.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/11UcvGyRWukMPwtrXWxkJbaQeYuf3UekE

# Set up Environment
Update Python and install prerequisite software. Most of this code can be skipped if you are running the Notebook locally, simply make sure you have all the right libraries. I use the following code cell to mount a folder from Google Drive.
"""

!apt-get install -y -qq software-properties-common python-software-properties module-init-tools > /dev/null
!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null
!apt-get update -qq 2>&1 > /dev/null
!apt-get -y install -qq google-drive-ocamlfuse fuse > /dev/null
from google.colab import auth
auth.authenticate_user()
from oauth2client.client import GoogleCredentials
creds = GoogleCredentials.get_application_default()
import getpass
!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL
vcode = getpass.getpass()
!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}

!mkdir -p drive
!google-drive-ocamlfuse drive
!pip install -q keras
!pip install -q intervaltree

"""Change to the Google Drive learning directory, or your directory with the libraries and data files"""

import os
os.chdir("/content/drive/learning")

"""# Import Pandas and Numpy"""

import pandas
import numpy

"""# Set up filenames

Define input and ouput filenames for the files
"""

input_filename = "merged.csv"
output_filename = "output_features_07_26_everything.csv"
output_file = open(output_filename, "w", newline = '')

"""# Load CSV File"""

df = pandas.read_csv(input_filename, dtype=str, keep_default_na=False)

"""# Define Helper Function for splitting Entrez IDs"""

def split_entrez_ids(ids):
    entrez_ids = []
    if ids.strip() != '':
        for entrez_id in ids.strip().split(';'):
            entrez_ids.append(int(entrez_id))
    return entrez_ids

"""# Usage
Simply click the run button for every feature you would like added to the CSV. Once you are done adding the features, go to the last line, called Save CSV, and click the run button for that.

# Keep track of features added
"""

list_of_features_added = []

"""# MPO Feature"""

# import the mpo library from the ./libraries folder
from libraries import mpo

#the mpo database is in the ./libraries folder
mpo_class = mpo.Gene_to_MPO_Features(
    "./libraries/MPO_topPh_GP20180216.tsv")

# keep track of feature name for column naming
feature_name = 'mpo'

# warn if feature is already added!
if feature_name in list_of_features_added:
    print(__name__ + ": Warning: feature '" +
          feature_name + "' already added to dataframe")
list_of_features_added.append(feature_name)

# create dictionary of columns -> list of values for each row
final_mpo_dict = {}

# These lists represent one column each

# multi_entrez_to_num_phenotypes is the total sum of the phenotypes matched
list_multi_entrez_to_num_phenotypes = []

# multi_entrez_to_num_phenotypes_using_thresh is a more conservative sum
# where a gene that matches the same phenotype multitple times is counted only
# once.
list_multi_entrez_to_num_phenotypes_using_thresh = []

# create a key for each column in the dictionary
for column in mpo_class.get_all_phenotypes():
    final_mpo_dict[feature_name + '_' + column] = []

# iterate through every row
for ids in df['genes_in_proximity']:
    list_multi_entrez_to_num_phenotypes.append(
        mpo_class.multi_entrez_to_num_phenotypes(split_entrez_ids(ids)))
    list_multi_entrez_to_num_phenotypes_using_thresh.append(
        mpo_class.multi_entrez_to_num_phenotypes_using_thresh(
            split_entrez_ids(ids)))
    for key, value in mpo_class.multi_entrez_to_phenotypes_using_thresh(
        split_entrez_ids(ids)).items():
        final_mpo_dict[feature_name + '_' + key].append(value)

# add all columns from dictionary
df = pandas.concat([df, pandas.DataFrame.from_dict(final_mpo_dict)], axis=1)

# add new column
df[feature_name + "_multi_entrez_to_num_phenotypes"] = pandas.DataFrame(
    list_multi_entrez_to_num_phenotypes)

# add new column
df[feature_name +
   "_multi_entrez_to_num_phenotypes_using_thresh"] = pandas.DataFrame(
    list_multi_entrez_to_num_phenotypes_using_thresh)

"""# OMIM Feature"""

from libraries import omim
omim_class = omim.Gene_to_OMIM_Features(
    "./libraries/OMIMdiseasePhenotypeOnly_gene_info_GP20180213.tsv")
feature_name = 'omim'
if feature_name in list_of_features_added:
    print(__name__ + ": Warning: feature '" +
          feature_name + "' already added to dataframe")

list_of_features_added.append(feature_name)

# only one column, how many genes are listed in OMIM?
list_omim_num_diseases = []
for ids in df['genes_in_proximity']:
    list_omim_num_diseases.append(
        omim_class.multi_entrez_to_num_diseases(split_entrez_ids(ids)))
df[feature_name + "_num_diseases"] = pandas.DataFrame(list_omim_num_diseases)

"""# pLI Feature"""

# we need two libraries, one to convert from the entrez ID to the Gene Symbol
# and the other to actually give us the pLI of the gene(s)
from libraries import entrez_symbol
entrez_symbol_class = entrez_symbol.Entrez_Symbol_Lookup(
    "./libraries/map.gsymbol.to.enzid.tsv")
from libraries import gene_pli
pli_class = gene_pli.Gene_to_pLI_Features(
    "./libraries/pLI_EXac_broad_institure_2016_03.txt")
feature_name = 'pli'
if feature_name in list_of_features_added:
    print(__name__ + ": Warning: feature '"+
          feature_name + "' already added to dataframe")
list_of_features_added.append(feature_name)

# the pLI values are binned to be quite granular
columns = ["pli_0.0_to_0.1", "pli_0.1_to_0.2", "pli_0.2_to_0.3",
           "pli_0.3_to_0.4", "pli_0.4_to_0.5", "pli_0.5_to_0.6",
           "pli_0.6_to_0.7", "pli_0.7_to_0.8", "pli_0.8_to_0.9",
           "pli_0.9_to_1.0"]

# once agains we keep a dictionary where each key/column -> list of values
final_pli_dict = {}
for column in columns:
    final_pli_dict[feature_name + '_' + column] = []

# iterate through every row
for ids in df['genes_in_proximity']:
    # create another temporary dictionary just for the current row,
    # so that every key/column -> a single value
    row_pli_dict = {}
    # initialize all values to 0
    for column in columns:
        row_pli_dict[column] = 0
    for entrez_id in split_entrez_ids(ids):
        # there is more than one symbol returned for a given Entrez ID ...
        list_symbols = entrez_symbol_class.entrez_id_to_symbols(entrez_id)
        max_pli = -2
        # find the max pLI of all the "same" gene symbols
        for symbol in list_symbols:
            max_pli = max(max_pli, pli_class.gene_symbol_to_pLI(symbol))
        # the library returns '-1' if the gene is not found, so if none of the
        # genes are found or no entrez ID translation existed, then max_pli < 0
        if max_pli < 0:
            continue
        elif max_pli <= 0.1:
            row_pli_dict["pli_0.0_to_0.1"] += 1
        elif max_pli <= 0.2:
            row_pli_dict["pli_0.1_to_0.2"] += 1
        elif max_pli <= 0.3:
            row_pli_dict["pli_0.3_to_0.4"] += 1
        elif max_pli <= 0.4:
            row_pli_dict["pli_0.3_to_0.4"] += 1
        elif max_pli <= 0.5:
            row_pli_dict["pli_0.4_to_0.5"] += 1
        elif max_pli <= 0.6:
            row_pli_dict["pli_0.5_to_0.6"] += 1
        elif max_pli <= 0.7:
            row_pli_dict["pli_0.6_to_0.7"] += 1
        elif max_pli <= 0.8:
            row_pli_dict["pli_0.7_to_0.8"] += 1
        elif max_pli <= 0.9:
            row_pli_dict["pli_0.8_to_0.9"] += 1
        elif max_pli <= 1.0:
            row_pli_dict["pli_0.9_to_1.0"] += 1
    # finally add the current row dictionary to the final dictionary
    for key, value in row_pli_dict.items():
        final_pli_dict[feature_name + '_' + key].append(value)
df = pandas.concat([df, pandas.DataFrame.from_dict(final_pli_dict)], axis=1)

"""# Repetitive Elements
Warning: creating this feature may take around an hour.
Even loading the library takes a few minutes
"""

from libraries import repeat_elem
repeat_class = repeat_elem.Gene_Interval_to_Repetitive_Elements(
    "./libraries/RLCRs_DNN-CNV.txt")
feature_name = 'repeat'
if feature_name in list_of_features_added:
    print(__name__ + ": Warning: feature '" +
          feature_name + "' already added to dataframe")
list_of_features_added.append(feature_name)

final_repeat_dict = {}
# make a column for every repetitive element, e.g. "LINE", "SINE"
for key in repeat_class.set_of_element_types:
    final_repeat_dict[feature_name + '_' + key] = []
for index, row in df.iterrows():
    # this process takes so long, we need a progress update
    if index % 100 == 0:
        print(" cnv:" + str(index), end = '')

    # add to the dictionary for every repetitive element
    for key, value in repeat_class.get_all_intersecting_elements(
                    row['chr'], int(row['start']), int(row['end'])).items():
        final_repeat_dict[feature_name + '_' + key].append(value)
df = pandas.concat([df, pandas.DataFrame.from_dict(final_repeat_dict)], axis=1)

"""# Pathways"""

from libraries import pathways
pathways_class = pathways.Gene_to_Pathways(
    "./libraries/allSizes_GOincludingIEA_pathways_20180213.GMT")
feature_name = 'pathways'
if feature_name in list_of_features_added:
    print(__name__ + ": Warning: feature '" +
          feature_name + "' already added to dataframe")
list_of_features_added.append(feature_name)

# Get all the Pathways Databases once
databases = pathways_class.get_all_databases()
# NCI is no longer supported and GO has too many genesets
databases.remove("NCI")
databases.remove("GO")

hugedf = pandas.DataFrame(numpy.zeros((len(df), 1)), columns = ['drop'])
for database in databases:
    for line_num in pathways_class._dict_databases_to_lines[database]:
        hugedf[database + '_' + str(line_num)] = 0
    for index, row in df.iterrows():
        for key, descrip, value in pathways_class.multi_entrez_to_gene_sets(
                              split_entrez_ids(row['genes_in_proximity']),
                              database, filter_min = 30, filter_max = 1000):
            hugedf.at[index, database + '_' + str(key)] = str(value)
hugedf.drop(['drop'], axis=1)
df = pandas.concat([df, hugedf], axis=1)

"""# Save CSV"""

df.to_csv(output_filename)

max_stars_repo_name: yoonsikp/cnv-pathogenicity-prediction

max_issues_repo_name: yoonsikp/cnv-pathogenicity-prediction

max_forks_repo_name: yoonsikp/cnv-pathogenicity-prediction


text:
import cv2
import numpy as np

img = cv2.imread('watch.jpeg', cv2.IMREAD_COLOR)

px = img[25, 25]

print(px)

img[250, 250] = [255, 0, 255]

watch_face = img[117:166, 352:560]
img[0:235, 0:394] = watch_face

roi = img[100:150, 100:150] = [0, 0, 0]

cv2.imshow("The images", img)

cv2.waitKey(0)
cv2.destroyAllWindows()

ext: py

max_stars_repo_name: uysalserkan/OpenCV-Samples

max_issues_repo_name: uysalserkan/OpenCV-Samples

max_forks_repo_name: uysalserkan/OpenCV-Samples

reason: import numpy


text:
"""
Skimmer for ParticleNet tagger inputs.

Author(s): Cristina Mantilla Suarez, Raghav Kansal
"""

import numpy as np
import awkward as ak
import pandas as pd

from coffea.processor import ProcessorABC, dict_accumulator
from coffea.analysis_tools import PackedSelection
from coffea.nanoevents.methods.base import NanoEventsArray
from coffea.nanoevents.methods.nanoaod import FatJetArray

from typing import Dict
import os

from .utils import add_selection_no_cutflow, pad_val

P4 = {
    "eta": "eta",
    "phi": "phi",
    "mass": "mass",
    "pt": "pt",
}

"""
Features
--------

Graph-level:
 - gen-jet p4
 - gen parton flavour

Node-level:
 - pfcand p4
 - pfcand pid
 - pfcand charge?

"""

class DatasetProcessor(ProcessorABC):
    """
    Produces a flat training ntuple from PFNano.
    """

    def __init__(self, radius="AK8", num_jets=1, num_particles=150):
        self.radius = radius

        if self.radius == "AK8":
            self.fatjet_label = "FatJet"
            self.subjet_label = "SubJet"
            self.pfcands_label = "FatJetPFCands"
            self.svs_label = "FatJetSVs"

        elif self.radius == "AK15":
            self.fatjet_label = "FatJetAK15"
            self.subjet_label = "FatJetAK15SubJet"
            self.pfcands_label = "FatJetAK15PFCands"
            self.svs_label = "JetSVsAK15"

        self.skim_vars = {
            "RecoJet": {
                **P4,
                "msoftdrop": "msd",
            },
            "GenJet": {**P4, "partonFlavour": "partonFlavour"},
            "RecoPart": {
                **P4,
                "pdgId": "pid",
                "charge": "charge",
            },
        }

        self._columns = [
            f"{vartype}_{var}" for vartype in self.skim_vars for var in self.skim_vars[vartype]
        ]
        print(self._columns)

        self.num_jets = num_jets
        self.num_particles = num_particles

        # self._accumulator = dict_accumulator({})

    @property
    def columns(self):
        return self._columns

    @property
    def accumulator(self):
        return self._accumulator

    def dump_table(self, pddf: pd.DataFrame, fname: str) -> None:
        """
        Saves pandas dataframe events to './outparquet'
        """
        import pyarrow.parquet as pq
        import pyarrow as pa

        local_dir = os.path.abspath(os.path.join(".", "outparquet"))
        os.system(f"mkdir -p {local_dir}")

        # need to write with pyarrow as pd.to_parquet doesn't support different types in
        # multi-index column names
        table = pa.Table.from_pandas(pddf)
        pq.write_table(table, f"{local_dir}/{fname}")

    def to_pandas(self, events: Dict[str, np.array]) -> pd.DataFrame:
        """
        Convert our dictionary of numpy arrays into a pandas data frame.
        Uses multi-index columns for numpy arrays with >1 dimension
        (e.g. FatJet arrays with two columns)
        """
        return pd.concat(
            [pd.DataFrame(v.reshape(v.shape[0], -1)) for k, v in events.items()],
            axis=1,
            keys=list(events.keys()),
        )

    def get_genjet_vars(
        self,
        events: NanoEventsArray,
        fatjets: FatJetArray,
        ak15: bool = True,
        match_dR: float = 1.0,
    ):
        """Matched fat jet to gen-level jet and gets gen jet vars"""
        if ak15:
            gen_dr = fatjets.delta_r(events.GenJetAK15)
            matched_gen_jet = events.GenJetAK15[ak.argmin(gen_dr, axis=1, keepdims=True)]
            matched_gen_jet_mask = fatjets.delta_r(matched_gen_jet) < match_dR

            GenJetVars = {
                f"GenJet_{key}": matched_gen_jet[var] * ak.values_astype(matched_gen_jet_mask, int)
                + ak.values_astype(~matched_gen_jet_mask, int) * (-99999)
                for (var, key) in self.skim_vars["GenJet"].items()
            }
        else:
            # NanoAOD automatically matched ak8 fat jets
            # No soft dropped gen jets however

            GenJetVars = {
                f"GenJet_{key}": ak.fill_none(fatjets.matched_gen[var], -99999)
                for (var, key) in self.skim_vars["GenJet"].items()
            }

        return GenJetVars

    def get_recoparticles_vars(self, events: NanoEventsArray, fatjets: FatJetArray, jet_idx: int):
        jet_ak_pfcands = events[self.pfcands_label][events[self.pfcands_label].jetIdx == jet_idx]
        jet_pfcands = events.PFCands[jet_ak_pfcands.pFCandsIdx]

        feature_dict = {
            f"RecoPart_{key}": jet_pfcands[var] for (var, key) in self.skim_vars["RecoPart"].items()
        }

        feature_dict["RecoPart_mask"] = (
            ~(
                ak.pad_none(
                    # padding to have at least one pf candidate in the graph
                    pad_val(
                        feature_dict["RecoPart_eta"], 1, -1, axis=1, to_numpy=False, clip=False
                    ),
                    self.num_particles,
                    axis=1,
                    clip=True,
                )
                .to_numpy()
                .mask
            )
        ).astype(np.float32)

        # if no padding is needed, mask will = 1.0
        if isinstance(feature_dict["RecoPart_mask"], np.float32):
            feature_dict["RecoPart_mask"] = np.ones(
                (len(feature_dict["RecoPart_eta"]), self.num_particles)
            ).astype(np.float32)

        # convert to numpy arrays and normalize features
        for var in feature_dict:
            a = (
                ak.pad_none(feature_dict[var], self.num_particles, axis=1, clip=True)
                .to_numpy()
                .filled(fill_value=0)
            ).astype(np.float32)

            feature_dict[var] = a

        return feature_dict

    def process(self, events: ak.Array):

        jet_vars = []

        for jet_idx in range(self.num_jets):
            # objects
            fatjets = ak.pad_none(events[self.fatjet_label], self.num_jets, axis=1)[:, jet_idx]

            # selection
            selection = PackedSelection()
            preselection_cut = fatjets.pt > 200
            add_selection_no_cutflow("preselection", preselection_cut, selection)

            # variables
            RecoJetVars = {
                f"RecoJet_{key}": ak.fill_none(fatjets[var], -99999)
                for (var, key) in self.skim_vars["RecoJet"].items()
            }

            JetVars = {
                **RecoJetVars,
                **self.get_genjet_vars(events, fatjets, ak15=self.radius == "AK15"),
            }

            PartVars = self.get_recoparticles_vars(events, fatjets, jet_idx)

            if np.sum(selection.all(*selection.names)) == 0:
                print("No jets pass selections")
                continue

            skimmed_vars = {**JetVars, **PartVars}
            # apply selections
            skimmed_vars = {
                key: np.squeeze(np.array(value[selection.all(*selection.names)]))
                for (key, value) in skimmed_vars.items()
            }

            jet_vars.append(skimmed_vars)

        if len(jet_vars) > 1:
            # stack each set of jets
            jet_vars = {
                var: np.concatenate([jet_var[var] for jet_var in jet_vars], axis=0)
                for var in jet_vars[0]
            }
        elif len(jet_vars) == 1:
            jet_vars = jet_vars[0]
        else:
            print("No jets passed selection")
            return {}

        fname = events.behavior["__events_factory__"]._partition_key.replace("/", "_")

        # convert output to pandas
        df = self.to_pandas(jet_vars)

        # save to parquet
        self.dump_table(df, fname + ".parquet")

        return {}

    def postprocess(self, accumulator):
        return accumulator


text:
"""Module to vectorize text and save vectorizer.

Developed by EricZhu-42 in June, 2019.
"""

import os
import pickle

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer

local_path = os.path.split(__file__)[0]
feature_path = os.path.join(local_path,r'models\TfifVectorizer.pkl')
matrix_path = os.path.join(local_path,r'models\Matrix.pkl')
sep_sentences_path = os.path.join(local_path,r'data\Words_from_content.txt')
max_features_num = 4000

def save_model(sep_sentences_pos:str):

    with open(sep_sentences_pos,'r',encoding='utf-8') as f:
        sep_sentences = f.readlines()

    vectorizer = TfidfVectorizer(max_features=max_features_num)
    vectorizer.fit(sep_sentences)
    X = vectorizer.transform(sep_sentences)
    print(X.toarray())
    print(vectorizer.get_feature_names())

    with open(feature_path,'wb') as fw:
        pickle.dump(vectorizer,fw)
    with open(matrix_path,'wb') as fw:
        pickle.dump(X,fw)

def load_vectorizer(feature_pos:str):
    with open(feature_pos,'rb') as fw:
        vectorizer = pickle.load(fw)
    return vectorizer

def load_sentences(matrix_pos:str):
    with open(matrix_pos,'rb') as fw:
        sentences = pickle.load(fw)
    return sentences

if __name__ == "__main__":
    save_model(sep_sentences_path)


text:
import zmq, sys
from zmq import ssh
from matplotlib import pyplot as plt
import numpy as np
import time
import cv2
from ipdb import set_trace as tt

from share_control.ui_cv import Slider
from .joints_registry import joint_info
# from environments.inmoov.inmoov_p2p_client_ready import InmoovGymEnv

SERVER_PORT = 7777
HOSTNAME = 'localhost'
SSH_NAME = "283a60820s.wicp.vip" # SSH ip
SSH_PORT = 17253 # SSH port
SSH_PWD = "SJJLPPsunte95" # SSH passwords
USER_NAME = "tete" # SSH username

def server_connection():
    context = zmq.Context()
    socket = context.socket(zmq.PAIR)
    socket.bind("tcp://*:{}".format(SERVER_PORT))
    print("Waiting for the client")
    line = "Hello Big Man"
    socket.send_json({'msg': line})
    msg = socket.recv_json()
    assert 'msg' in msg and msg['msg'] == line, "Connection failed, check server and client configuration"
    print("Client Connected")
    return socket

def client_connection():
    context = zmq.Context()
    socket = context.socket(zmq.PAIR)
    socket.connect("tcp://{}:{}".format(HOSTNAME, SERVER_PORT))
    print("Waiting for server")
    msg = socket.recv_json()
    # resend message to ensure the integrity of the msg
    socket.send_json(msg)
    print("Server Connected")
    return socket

def client_ssh_connection():
    context = zmq.Context()
    socket = context.socket(zmq.PAIR)
    password = SSH_PWD
    username = USER_NAME
    server_port = SSH_PORT
    server_name = SSH_NAME
    server = "{}@{}:{}".format(username, server_name, server_port)
    addr = "tcp://{}:{}".format(HOSTNAME, SERVER_PORT)
    tunnuel = ssh.tunnel_connection(socket, addr=addr, server=server, password=password)
    print("Waiting for server")
    msg = socket.recv_json()
    # resend message to ensure the integrity of the msg
    socket.send_json(msg)
    print("send: {}".format(msg))
    print("Server Connected")
    return socket

def recv_array(socket, flags=0, copy=True, track=False):
    """recv a numpy array"""
    md = socket.recv_json(flags=flags)
    msg = socket.recv(flags=flags, copy=copy, track=track)
    buf = memoryview(msg)
    A = np.frombuffer(buf, dtype=md['dtype'])
    return A.reshape(md['shape'])

def plot_robot_view(left_px, right_px):
    plt.ion()
    figsize = np.array([2, 1]) * 3
    fig = plt.figure("Inmoov", figsize=figsize)
    ax1 = fig.add_subplot(121)
    ax1.imshow(left_px)
    ax1.set_title("Left")
    ax2 = fig.add_subplot(122)
    ax2.imshow(right_px)
    ax2.set_title("Right")
    fig.suptitle('Inmoov Simulation: Two Cameras View', fontsize=20)
    plt.draw()
    plt.pause(0.00001)

def plot_robot_view_cv(left_px, right_px):
    """
    Use opencv to create the windows
    :param left_px:
    :param right_px:
    :return:
    """
    name = "Inmoov"
    cv2.namedWindow(name, cv2.WINDOW_NORMAL)
    cv2.resizeWindow(name, 800, 400)
    image = np.hstack([left_px, right_px])
    cv2.imshow(name, image)
    cv2.waitKey(0) & 0xFF
    return

def joint_controller(socket, joint_info):
    joint_name = [n[1] for n in joint_info]
    joint_info = np.array([p[:1]+p[2:] for p in joint_info])
    # joint_low_limit, joint_high_limit = np.zeros(shape=(53,)), np.ones(shape=(53,))
    joint_low_limit, joint_high_limit = joint_info[:, 1], joint_info[:, 2]
    step = 0
    # TODO: please modify this part to control the robot
    slider = Slider(win_name="Inmoov", dimension=len(joint_low_limit),
                    sample_freq=100,
                    slider_name=joint_name,
                    low=joint_low_limit, high=joint_high_limit)
    position = None
    old_position = None
    while True:
        # time.sleep(0.2)
        print("Step: {}".format(step))
        step += 1
        slider_position = slider.get_slider_data()
        if slider_position is None:
            break
        # position = np.random.uniform(low=joint_low_limit*0.1, high=joint_high_limit*0.1)
        old_position = position
        position = slider_position
        # to print the change of the position
        if old_position is not None and position is not None:
            old_data, data_new = old_position, position
            for j, e in enumerate(data_new):
                if e != old_data[j]:
                    print("Change at {}, from {:.2f} to {:.2f}".format(joint_name[j], old_data[j], data_new[j]))
        msg = {"command":"position", "position": position.tolist()}
        socket.send_json(msg)
        step_data = []
        for i in range(5):
            step_data.append(recv_array(socket))
        joint_state = step_data[0]
        left_px, right_px = step_data[3][0], step_data[3][1]
        reward = np.squeeze(step_data[1])
        done = np.squeeze(step_data[2])
        effector_position = step_data[4]
        print(effector_position)
        # print(joint_state - position)
        plot_robot_view(left_px, right_px)

    cv2.destroyAllWindows()

def position_controller(socket):
    pos_name = ['x', 'y', 'z']
    low_limit, high_limit = np.array([0, -2, 0]), np.array([1, 2, 2])
    step = 0
    init_position = np.array([0.7, -0.01, 1.6802])
    # TODO: please modify this part to control the robot
    slider = Slider(win_name="Inmoov", dimension=3,
                    sample_freq=100,
                    slider_name=pos_name,
                    low=low_limit, high=high_limit,
                    init_pos=init_position)
    position = None
    old_position = None
    while True:
        # time.sleep(0.2)
        print("Step: {}".format(step))
        step += 1
        slider_position = slider.get_slider_data()
        if slider_position is None:
            break
        # position = np.random.uniform(low=joint_low_limit*0.1, high=joint_high_limit*0.1)
        old_position = position
        position = slider_position
        # to print the change of the position
        if old_position is not None and position is not None:
            old_data, data_new = old_position, position
            for j, e in enumerate(data_new):
                if e != old_data[j]:
                    print("Change at {}, from {:.2f} to {:.2f}".format(pos_name[j], old_data[j], data_new[j]))
        msg = {"command": "position", "position": position.tolist()}
        socket.send_json(msg)
        step_data = []
        for i in range(5):
            step_data.append(recv_array(socket))
        joint_state = step_data[0]
        left_px, right_px = step_data[3][0], step_data[3][1]
        reward = np.squeeze(step_data[1])
        done = np.squeeze(step_data[2])
        effector_position = step_data[4]
        # print(joint_state - position)
        plot_robot_view(left_px, right_px)

    cv2.destroyAllWindows()

if __name__ == "__main__":
    # For remote server #
    # socket = client_ssh_connection()

    #############################
    #### local version ##########
    #############################
    socket = client_connection()

    # joint_controller(socket, joint_info=joint_info)
    position_controller(socket)


text:
import numpy as np
from joblib import parallel_backend
from sklearn.datasets import load_digits
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC

from ray.util.joblib import register_ray
register_ray()

param_space = {
    'C': np.logspace(-6, 6, 10),
    'gamma': np.logspace(-8, 8, 5),
    'tol': np.logspace(-4, -1, 10),
    'kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
    'class_weight': [None, 'balanced'],
}

model = SVC(kernel='rbf')
search = GridSearchCV(model, param_space, cv=5, verbose=1)
digits = load_digits()

with parallel_backend('ray'):
    search.fit(digits.data, digits.target)

max_stars_repo_name: anabranch/sklearn-ray

max_issues_repo_name: anabranch/sklearn-ray

max_forks_repo_name: anabranch/sklearn-ray


text:
"""Bouw de ondergrond discretisatie package"""

from pathlib import Path

import flopy
import numpy as np

from .utils import read_3d_array, read_array

def build_dis_package(gwf, conf):
    idomain = __read_idomain(conf["idomain_file"])
    tops = __read_masked_levels(conf["top_files"], conf["n_layers"])
    bots = __read_masked_levels(conf["bottom_files"], conf["n_layers"])

    top, bottoms = __build_grid(tops, bots)
    idomain = np.broadcast_to(idomain, bottoms.shape)
    # initialize the DIS package
    return flopy.mf6.modflow.mfgwfdis.ModflowGwfdis(
        gwf,
        pname='dis', nlay=(conf["n_layers"]*2 - 1),
        nrow=conf["n_rows"], ncol=conf["n_cols"],
        delr=conf["del_rows"], delc=conf["del_cols"],
        top=top, botm=bottoms,
        idomain=idomain,
    )

def __read_idomain(idomainfile):
    return read_array(idomainfile).filled(0.)

def __read_masked_levels(levelfiles, nlay):
    levelfiles = list(levelfiles.parent / levelfiles.name.format(ilay=i + 1)
                      for i in range(nlay))
    lvls = read_3d_array(levelfiles)
    # mask bad nodata values
    lvls = np.ma.masked_where(lvls.mask | (lvls < -9990.), lvls)
    return lvls

def __build_grid(tops, bots):
    top = tops[0, :, :].filled(0.)
    botm = []
    nlay = bots.shape[0]
    for ilay in range(nlay):
        botm.append(bots[ilay, :, :].filled((2*ilay + 1) * -1e-3))
        if (ilay + 1) < nlay:
            botm.append(tops[ilay + 1, :, :].filled((2*ilay + 2) * -1e-3))
    botm = np.ma.stack(botm)
    return top, botm


text:
import numpy as np

# region 激活函数(activation function)
def sigmoid(x):
    """
    sigmoid函数很早就开始作为激活函数使用
    :param x:
    :return:
    """
    return 1 / (1 + np.exp(-x))

def relu(x):
    """
    ReLU函数(Rectified Linear Unit)
    最近主要使用该函数作为激活函数
    :param x:
    :return:
    """
    return np.maximum(0, x)

# endregion

# region 输出层函数
def identity_function(a):
    """
    恒等函数
    一般用于回归问题
    :param a:
    :return:
    """
    return a

def softmax(a):
    """
    softmax函数
    一般用于分类问题
    输出结果总和为1，体现为概率。
    :param a:
    :return:
    """
    c = np.max(a)
    exp_a = np.exp(a - c)  # 溢出对策
    sum_exp_a = np.sum(exp_a)
    y = exp_a / sum_exp_a

    return y

# endregion

# region 损失函数(loss function)
def mean_squared_error(y, t):
    """
    均方误差
    :param y: 神经网络的输出
    :param t: 监督数据(one-hot)
    :return:
    """
    return 0.5 * np.sum((y - t) ** 2)

def mean_squared_error_mini_batch(y, t):
    """
    均方误差的mini-batch版
    :param y: 神经网络的输出
    :param t: 监督数据(one-hot)
    :return:
    """
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)

    batch_size = y.shape[0]
    return 0.5 * np.sum((y - t) ** 2) / batch_size

def cross_entropy_error(y, t):
    """
    交叉熵误差
    np.log(0)会变为负无穷大-inf，导致后续计算无法进行，
    引入微小值delta，作为保护策略。
    :param y: 神经网络的输出
    :param t: 监督数据(one-hot)
    :return:
    """
    delta = 1e-7  # 保护策略
    return -np.sum(t * np.log(y + delta))

def cross_entropy_error_mini_batch(y, t):
    """
    交叉熵误差的mini-batch版
    :param y: 神经网络的输出
    :param t: 监督数据(one-hot)
    :return:
    """
    delta = 1e-7
    if y.ndim == 1:
        # 如果y为一维，即只有一条测试数据
        # 将y和t转换成1×size的矩阵，适配下面的计算
        t = t.reshape(1, t.size)
        y = y.reshare(1, y.size)

    batch_size = y.shape[0]
    return -np.sum(t * np.log(y + delta)) / batch_size

# endregion

# region 数值微分(numerical differentiation)
def numerical_diff(f, x):
    """
    导数计算
    :param f:
    :param x:
    :return:
    """
    h = 1e-4  # 不能使用太小的数值，否则会产生舍入误差(rounding error)
    return (f(x + h) - f(x - h)) / (2 * h)

def numerical_gradient(f, x):
    """
    梯度计算
    :param f:
    :param x:
    :return:
    """

    def _ng(f, x):
        h = 1e-4
        grad = np.zeros_like(x)  # 生成和x形状相同的数组

        for idx in range(x.size):
            tmp_val = x[idx]
            # f(x+h)的计算
            x[idx] = tmp_val + h
            fxh1 = f(x)

            # f(x-h)的计算
            x[idx] = tmp_val - h
            fxh2 = f(x)

            grad[idx] = (fxh1 - fxh2) / (2 * h)
            x[idx] = tmp_val  # 还原值

        return grad

    if x.ndim == 1:
        return _ng(f, x)
    else:
        grad = np.zeros_like(x)

        for idx, x in enumerate(x):
            grad[idx] = _ng(f, x)

        return grad

# endregion

# region functions
def function_2(x):
    # return x ** 2 + y ** 2
    if x.ndim == 1:
        return np.sum(x ** 2)
    else:
        return np.sum(x ** 2, axis=1)
# endregion


text:
# Copyright 2014 Uri Laserson
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import json
import string
import random
import itertools

from Bio import Alphabet, pairwise2
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.SeqFeature import SeqFeature, FeatureLocation

import numpy as np
import scipy as sp
import scipy.stats

from jellyfish import hamming_distance

from ulutil import unafold
from ulutil.pyutils import as_handle

random.seed()

# ==============================
# = General sequence utilities =
# ==============================

def substitute(seq,pos,sub):
    return seq[:pos] + sub + seq[pos+1:]

complement_table = string.maketrans('ACGTRYSWKMBDHVN','TGCAYRSWMKVHDBN')

def reverse(seq):
    return seq[::-1]

def complement(seq):
    return seq.upper().translate(complement_table)

def reverse_complement(seq):
    """Compute reverse complement of sequence.

    Mindful of IUPAC ambiguities.
    Return all uppercase.
    """
    return reverse(complement(seq))
    # return seq.upper().translate(complement_table)[::-1]

def translate(seq):
    return Seq(seq.replace('-','N'),Alphabet.DNAAlphabet()).translate().tostring()

def gc_content(seq):
    gc = seq.lower().count('g') + seq.lower().count('c')
    return float(gc) / len(seq)

def random_dna_seq(n):
    choice = random.choice
    return reduce(lambda cumul,garbage:cumul+choice('ACGT'),xrange(n),'')

global_align = lambda seq1,seq2: pairwise2.align.globalms(seq1,seq2,0.5,-0.75,-2.,-1.5,one_alignment_only=True)[0]

def percent_id(seq1,seq2):
    alignment = global_align(seq1,seq2)
    return (1. - hamming_distance(alignment[0],alignment[1]) / float(len(alignment[0]))) * 100.

# barcode mapping fns
def barcode_hamming(observed,barcodes):
    """Compute entropy of probabilistic barcode assignment.

    observed -- SeqRecord of the barcode
    barcodes -- list of barcode possibilities (python strings)
    """
    obs_seq = observed.seq.tostring()
    distances = [(barcode,hamming_distance(obs_seq,barcode)) for barcode in barcodes]
    closest = min(distances,key=lambda p: p[1])
    return closest  # tuple of (barcode, distance)

def barcode_probabilities(observed,barcodes):
    """Compute entropy of probabilistic barcode assignment.

    observed -- 'fastq' SeqRecord of the barcode
    barcodes -- list of barcode possibilities (python strings)
    """
    obs_seq = np.array(list(observed.seq.tostring()))
    obs_qual = np.array(observed.letter_annotations['phred_quality'])
    barcodes = np.array([list(bc) for bc in barcodes])

    choice = np.zeros(barcodes.shape, dtype=np.int)
    choice[barcodes == obs_seq] = 1
    choice[barcodes != obs_seq] = 2
    choice[:, obs_seq == 'N'] = 0

    N = np.zeros((1,barcodes.shape[1]))
    E = np.log1p(-np.power(10, -obs_qual / 10.))
    D = -np.log(3) - (obs_qual / 10.) * np.log(3)

    B = np.exp(np.sum(np.choose(choice, [N,E,D]), axis=1))
    return B / np.sum(B)

def barcode_entropy(observed, barcodes):
    """Compute entropy of probabilistic barcode assignment.

    observed -- 'fastq' SeqRecord of the barcode
    barcodes -- list of barcode possibilities (python strings)
    """
    P = barcode_probabilities(observed, barcodes)
    return sp.stats.entropy(P)

# for generating 'safe' filenames from identifiers
cleanup_table = string.maketrans('/*|><+ ','_____p_')
def cleanup_id(identifier):
    return identifier.translate(cleanup_table)

def seqhist(seqlist):
    seqdict = dict()
    for seq in seqlist:
        seqdict[seq] = seqdict.get(seq,0) + 1
    return seqdict

def seqmode(seqs):
    if isinstance(seqs,list):
        seqs = seqhist(seqs)
    return max(seqs.iterkeys(),key=lambda k: seqs[k])

def dimer_dG(seq1,seq2):
    """Compute a primer-dimer score using UNAFOLD hybrid_min"""
    scores = []
    subseqs1 = []
    subseqs2 = []
    for i in xrange( min(len(seq1),len(seq2)) ):
        subseqs1.append( seq1[-i-1:] )
        subseqs2.append( seq2[-i-1:] )
    scores = unafold.hybrid_min_list(subseqs1,subseqs2,NA='DNA')
    return -min(scores)

def dimer_overlap(seq1,seq2,weight_3=10):
    """Compute a primer-dimer score by counting overlaps

    weight_3 is the num of 3' bases to add extra weight to either primer
    """
    # import pdb
    # pdb.set_trace()
    overlap_score = lambda s1,s2: sum(1 if c1.lower() == c2.lower() else -1 for c1, c2 in itertools.izip(s1,s2))
    seq2rc = reverse_complement(seq1)
    scores = []
    for i in xrange( min(len(seq1),len(seq2)) ):
        subseq1 = seq1[-i-1:]
        subseq2 = seq2rc[:i+1]
        score = 0
        if (i+1) <= 2*weight_3:
            score += overlap_score(subseq1,subseq2) * 2
        else:
            score += overlap_score(subseq1[:weight_3],subseq2[:weight_3]) * 2
            score += overlap_score(subseq1[weight_3:-weight_3],subseq2[weight_3:-weight_3])
            score += overlap_score(subseq1[-weight_3:],subseq2[-weight_3:]) * 2
        scores.append(score)
    return max(scores)

# ==========================
# = Manual FASTA iteration =
# ==========================

# taken from biopython

identity = string.maketrans('','')
nonalpha = identity.translate(identity,string.ascii_letters)

def FastaIterator(handleish,title2ids=lambda s: s):
    with as_handle(handleish,'r') as handle:
        while True:
            line = handle.readline()
            if line == '' : return
            if line[0] == '>':
                break

        while True:
            if line[0] != '>':
                raise ValueError("Records in Fasta files should start with '>' character")
            descr = title2ids(line[1:].rstrip())
            fullline = ''
            line = handle.readline()
            while True:
                if not line : break
                if line[0] == '>': break
                fullline += line.translate(identity,nonalpha)
                line = handle.readline()

            yield (descr,fullline)

            if not line : return #StopIteration
        assert False, "Should not reach this line"

# ============================
# = biopython-specific tools =
# ============================

def make_SeqRecord(name,seq):
    return SeqRecord(Seq(seq),id=name,name=name,description=name)

def get_string(seqobj):
    if isinstance(seqobj,SeqRecord):
        seq = seqobj.seq.tostring().upper()
    elif isinstance(seqobj,Seq):
        seq = seqobj.tostring().upper()
    elif isinstance(seqobj,str):
        seq = seqobj.upper()
    return seq

def get_features(feature_list,feature_type):
    target_features = []
    for feature in feature_list:
        if feature.type == feature_type:
            target_features.append(feature)
    return target_features

def advance_to_features(feature_iter,feature_types):
    # note, here feature_types is a list of possible stopping points
    for feature in feature_iter:
        if feature.type in feature_types:
            return feature
    raise ValueError, "didn't find %s in record" % feature_types

def advance_to_feature(feature_iter,feature_type):
    return advance_to_features(feature_iter,[feature_type])

def map_feature( feature, coord_mapping, offset=0, erase=[] ):
    new_feature = copy.deepcopy(feature)
    new_start = coord_mapping[feature.location.start.position][-1] + offset
    new_end   = coord_mapping[feature.location.end.position][0] + offset
    new_location = FeatureLocation(new_start,new_end)
    new_feature.location = new_location
    for qual in erase:
        new_feature.qualifiers.pop(qual,None)
    return new_feature

def copy_features( record_from, record_to, coord_mapping, offset=0, erase=[], replace=False ):
    if replace:
        # index record_to features:
        feature_index = {}
        for (i,feature) in enumerate(record_to.features):
            feature_index.setdefault(feature.type,[]).append(i)

    feat_idx_to_delete = []
    for feature in record_from.features:
        if replace:
            feat_idx_to_delete += feature_index.get(feature.type,[])
        new_feature = map_feature( feature, coord_mapping, offset, erase )
        record_to.features.append(new_feature)

    if replace:
        for idx in sorted(feat_idx_to_delete,reverse=True):
            record_to.features.pop(idx)

def translate_features( record ):
    for feature in record.features:
        offset = int(feature.qualifiers.get('codon_start',[1])[0]) - 1
        feature.qualifiers['translation'] = feature.extract(record.seq)[offset:].translate()

# SeqRecord <-> JSON-serializable

def simplifySeq(seq):
    obj = {}
    obj['__Seq__'] = True
    obj['seq'] = seq.tostring()
    obj['alphabet'] = seq.alphabet.__repr__().rstrip(')').rstrip('(')
    return obj

def complicateSeq(obj):
    if '__Seq__' not in obj:
        raise ValueError, "object must be converable to Bio.Seq"

    # Figure out which alphabet to use
    try:
        alphabet = Alphabet.__getattribute__(obj['alphabet'])()
    except AttributeError:
        pass
    try:
        alphabet = Alphabet.IUPAC.__getattribute__(obj['alphabet'])()
    except AttributeError:
        raise

    seq = Seq(obj['seq'],alphabet=alphabet)
    return seq

def simplifySeqFeature(feature):
    obj = {}
    obj['__SeqFeature__'] = True
    obj['location'] = (feature.location.nofuzzy_start,feature.location.nofuzzy_end)
    obj['type'] = feature.type
    obj['strand'] = feature.strand
    obj['id'] = feature.id
    obj['qualifiers'] = feature.qualifiers
    return obj

def complicateSeqFeature(obj):
    if '__SeqFeature__' not in obj:
        raise ValueError, "object must be converable to Bio.SeqFeature"
    location = FeatureLocation(*obj['location'])
    feature = SeqFeature(location=location,type=obj['type'],strand=obj['strand'],id=obj['id'],qualifiers=obj['qualifiers'])
    return feature

def simplifySeqRecord(record):
    obj = {}
    obj['__SeqRecord__'] = True
    obj['seq'] = simplifySeq(record.seq)
    obj['id'] = record.id
    obj['name'] = record.name
    obj['description'] = record.description
    obj['dbxrefs'] = record.dbxrefs
    obj['annotations'] = record.annotations
    obj['letter_annotations'] = record.letter_annotations   # should work because it is actually a _RestrictedDict obj which subclasses dict
    obj['features'] = map(simplifySeqFeature,record.features)
    return obj

def complicateSeqRecord(obj):
    if '__SeqRecord__' not in obj:
        raise ValueError, "object must be converable to Bio.SeqRecord"
    features = map(complicateSeqFeature,obj['features'])
    record = SeqRecord(seq=complicateSeq(obj['seq']),id=obj['id'],name=obj['name'],description=obj['description'],dbxrefs=obj['dbxrefs'],features=features,annotations=obj['annotations'],letter_annotations=obj['letter_annotations'])
    return record

max_stars_repo_name: churchlab/ulutil

max_issues_repo_name: churchlab/ulutil

max_forks_repo_name: churchlab/ulutil

reason: import numpy,import scipy


text:
from gensim.models import Word2Vec
import numpy as np
from numba import njit

from tqdm import tqdm

@njit
def set_seed(seed):
    np.random.seed(seed)

class Node2Vec(Word2Vec):
    def __init__(
        self,
        graph,
        dim,
        walk_length,
        context,
        p=1.0,
        q=1.0,
        workers=1,
        batch_walks=None,
        seed=None,
        **args,
    ):
        # <https://github.com/RaRe-Technologies/gensim/issues/2801>
        assert walk_length < 10000
        if batch_walks is None:
            batch_words = 10000
        else:
            batch_words = min(walk_length * batch_walks, 10000)

        super().__init__(
            sg=1,
            epochs=1,
            vector_size=dim,
            window=context,
            min_count=1,
            workers=workers,
            batch_words=batch_words,
            **args,
        )
        self.build_vocab(([w] for w in graph.node_names))
        self.graph = graph
        self.walk_length = walk_length
        self.p = p
        self.q = q
        self.seed = seed

    def train(self, epochs, *, progress_bar=True, **kwargs):
        def gen_nodes(epochs):
            if self.seed is not None:
                np.random.seed(self.seed)
            for _ in range(epochs):
                for i in np.random.permutation(len(self.graph.node_names)):
                    # dummy walk with same length
                    yield [i] * self.walk_length

        if progress_bar:

            def pbar(it):
                return tqdm(
                    it, desc="Training", total=epochs * len(self.graph.node_names)
                )

        else:

            def pbar(it):
                return it

        super().train(
            pbar(gen_nodes(epochs)),
            total_examples=epochs * len(self.graph.node_names),
            epochs=1,
            **kwargs,
        )

    def generate_random_walk(self, t):
        return self.graph.generate_random_walk(self.walk_length, self.p, self.q, t)

    def _do_train_job(self, sentences, alpha, inits):
        if self.seed is not None:
            set_seed(self.seed)
        sentences = [self.generate_random_walk(w[0]) for w in sentences]
        return super()._do_train_job(sentences, alpha, inits)

reason: import numpy,from numba


text:
""" Contains the base class for agents.

Agent class should be inherited to create agents.
It is possible to define custom Agent with
body parts, sensors and corresponding Keyboard commands.

Examples can be found in simple_playgrounds/agents/agents.py

"""
from __future__ import annotations

from typing import List, Optional, Dict, Union, TYPE_CHECKING, Tuple

if TYPE_CHECKING:
    from simple_playgrounds.device.sensor import SensorDevice
    from simple_playgrounds.agent.actuators import ActuatorDevice
    from simple_playgrounds.agent.controllers import Controller, Keyboard
    from simple_playgrounds.playground.playground import Playground
    from pymunk import Shape
    from pygame import Surface
    from ..common.entity import Entity
    from simple_playgrounds.device.communication import CommunicationDevice

from abc import ABC
import numpy as np
import pygame
from PIL import Image, ImageDraw, ImageFont

from simple_playgrounds.common.position_utils import CoordinateSampler, Coordinate
from simple_playgrounds.agent.parts import Part, Platform, AnchoredPart
from simple_playgrounds.agent.actuators import Grasp
from simple_playgrounds.element.elements.teleport import TeleportElement

# pylint: disable=too-many-instance-attributes
# pylint: disable=no-member

_BORDER_IMAGE = 3

class Agent(ABC):
    """
    Base class for building agents.
    Agents are composed of a base_platform and parts which are anchored with each other.
    Actuators are applied to parts to allow discrete or continuous control.
    Sensors can be anchored to parts of an agent.

    Attributes:
        name: name of the agent. Either provided by the user or generated using internal counter.
        base_platform: Main part of the agent. Can be mobile or fixed.
        parts: Different parts attached to the base or to other parts.
        actuators:
        sensors:
        initial_coordinates:

    """
    _index_agent: int = 0

    def __init__(
        self,
        base_platform: Part,
        name: Optional[str] = None,
    ):
        """
        Base class for agents.

        Args:
            base_platform: Platform object, required to initialize an agent.
                All agents have a Platform.
            name: Name of the agent. If not provide, a name will be added by default.

        """

        self.name: str
        if name is None:
            self.name = 'agent_' + str(Agent._index_agent)
        else:
            self.name = name
        Agent._index_agent += 1

        # List of sensors
        self.sensors: List[SensorDevice] = []

        # Body parts
        self.base_platform: Part = base_platform
        self.parts: List[Part] = []
        self.add_part(base_platform)

        # Default starting position
        self.initial_coordinates: Optional[Union[Coordinate,
                                                 CoordinateSampler]] = None

        # Keep track of the actions for display
        self._current_actions: Optional[Dict[ActuatorDevice, float]] = None

        # Actuators
        self.actuators: List[ActuatorDevice] = []

        self._controller: Optional[Controller] = None

        # Reward
        self.reward: float = 0

        # Teleport
        self._teleported_to: Optional[Coordinate, TeleportElement] = None
        self._has_teleported: bool = False

        # Used to set an element which is not supposed to overlap
        self._allow_overlapping: bool = False
        self._overlapping_strategy_set: bool = False
        self._max_attempts: int = 100

        # Communication
        self.communication: Optional[CommunicationDevice] = None
        self._can_communicate = False

        self._playground: Optional[Playground] = None
        self._screen = None
        self._quit_key_ready = True
        self._surface_screen = None

    @property
    def playground(self):
        return self._playground

    @playground.setter
    def playground(self, pg):
        for part in self.parts:
            part.playground = pg
        for act in self.actuators:
            act.playground = pg
        for sens in self.sensors:
            sens.playground = pg

        self._playground = pg

    @property
    def in_playground(self):
        return bool(self._playground)

    # COMMUNICATION
    def add_communication(self,
                          communication: CommunicationDevice,
                          ):

        if self._can_communicate:
            raise ValueError('Communication Device already added')

        self.communication = communication

    # CONTROLLER
    @property
    def controller(self):
        """
        The controller allows to select actions for each actuator of the agent.
        """
        return self._controller

    @controller.setter
    def controller(self, controller: Controller):

        self._controller = controller
        self._controller.controlled_actuators = self.actuators
        self._current_actions = controller.generate_null_actions()

        self._controller.agent = self

    def add_actuator(self, actuator: ActuatorDevice):

        if self.in_playground:
            raise ValueError('Add actuators outside of a playground.')

        self.actuators.append(actuator)
        actuator.playground = self._playground

    @property
    def grasped_elements(self):

        list_hold = []

        for act in self.actuators:
            if isinstance(act, Grasp) and act.grasped_element:
                list_hold.append(act.grasped_element)

        return list_hold

    # OVERLAPPING STRATEGY
    @property
    def overlapping_strategy(self):
        if self._overlapping_strategy_set:
            return self._allow_overlapping, self._max_attempts
        return None

    @overlapping_strategy.setter
    def overlapping_strategy(
        self,
        strategy: Tuple[bool, int],
    ):
        self._allow_overlapping, self._max_attempts = strategy
        self._overlapping_strategy_set = True

    # POSITION / VELOCITY
    @property
    def initial_coordinates(self):
        """
        Initial position can be fixed (tuple) or a PositionAreaSampler.
        """

        if isinstance(self._initial_coordinates, tuple):
            return self._initial_coordinates
        if isinstance(self._initial_coordinates, CoordinateSampler):
            return self._initial_coordinates.sample()

        return self._initial_coordinates

    @initial_coordinates.setter
    def initial_coordinates(self, coordinates: Union[Coordinate,
                                                     CoordinateSampler]):
        self._initial_coordinates = coordinates

    @property
    def coordinates(self):
        return self.base_platform.position, self.base_platform.angle

    @coordinates.setter
    def coordinates(self, coord: Coordinate):

        position, angle = coord

        for part in self.parts:
            if isinstance(part, Platform):
                part.position, part.angle = position, angle
            elif isinstance(part, AnchoredPart):
                part.set_relative_coordinates()
            else:
                raise ValueError("Part type not implemented")

    def reindex_shapes(self):

        for part in self.parts:
            part.pm_body.space.reindex_shapes_for_body(part.pm_body)

    @property
    def position(self):
        """
        Position of the agent.
        In case of an Agent with multiple Parts, its position is the position of the base_platform.
        """
        return self.base_platform.position

    @position.setter
    def position(self, pos: Tuple[float, float]):
        self.coordinates = pos, self.angle

    @property
    def angle(self):
        """
        Angle of the agent.
        In case of an Agent with multiple Parts, its angle is the angle of the base_platform.
        Setter only used during initialization
        """
        return self.base_platform.angle

    @angle.setter
    def angle(self, theta: float):
        self.coordinates = self.position, theta

    @property
    def velocity(self):
        """
        Velocity of the agent.
        In case of an Agent with multiple Parts, its velocity is the velocity of the base_platform.
        """
        return self.base_platform.velocity

    @velocity.setter
    def velocity(self, velocity: Tuple[float, float]):
        for part in self.parts:
            part.velocity = velocity

    @property
    def angular_velocity(self):
        """
        Velocity of the agent.
        In case of an Agent with multiple Parts, its velocity is the velocity of the base_platform.
        """
        return self.base_platform.angular_velocity

    @angular_velocity.setter
    def angular_velocity(
        self,
        angular_velocity: float,
    ):
        for part in self.parts:
            part.angular_velocity = angular_velocity

    # SENSORS

    def add_sensor(self, new_sensor: SensorDevice):
        """
        Add a Sensor to an agent.
        Should be done outside of a playground.
        Else, it will raise an error.

        Args:
            new_sensor: Sensor.

        Returns:

        """
        if self.in_playground:
            raise ValueError('Add sensors outside of a playground.')

        self.sensors.append(new_sensor)

        # If already in playground, add
        if self._playground:
            new_sensor.playground = self._playground

    @property
    def observations(self):
        return {sensor: sensor.sensor_values for sensor in self.sensors}

    def generate_sensor_image(self,
                              width_sensor: int = 200,
                              height_sensor: int = 30,
                              plt_mode: bool = False):
        """
        Generate a full image containing all the sensor representations of an Agent.
        Args:
            width_sensor: width of the display for drawing.
            height_sensor: when applicable (1D sensor), the height of the display.
            plt_mode: if True, returns images compatible with pyplot.

        Returns:

        """

        list_sensor_images = []
        for sensor in self.sensors:
            list_sensor_images.append(sensor.draw(width_sensor, height_sensor))

        full_height = sum([im.shape[0] for im in list_sensor_images])\
                      + len(list_sensor_images) * (_BORDER_IMAGE + 1)

        full_img = np.ones((full_height, width_sensor, 3))

        current_height = 0
        for sensor_image in list_sensor_images:
            current_height += _BORDER_IMAGE
            full_img[current_height:sensor_image.shape[0] + current_height, :, :] \
                = sensor_image[:, :, :]
            current_height += sensor_image.shape[0]

        if plt_mode:
            full_img = full_img[:, :, ::-1]

        return full_img

    # BODY PARTS

    def add_part(self, part: Part):
        """
        Add a Part to the agent
        Args:
            part: Part to add to the agent.

        """
        self.parts.append(part)
        part.agent = self

    def apply_actions_to_actuators(self, actions_dict: Dict[ActuatorDevice, float]):
        """
        Apply actions to each body part of the agent.

        Args:
            actions_dict: dictionary of body_part_name, Action.
        """

        self._current_actions = actions_dict

        for actuator, value in actions_dict.items():
            actuator.apply_action(value)

    def owns_shape(self, pm_shape: Shape):
        """
        Verifies if a pm_shape belongs to an agent.

        Args:
            pm_shape: pymunk_shape.

        Returns: True if pm_shape belongs to the agent.

        """
        all_shapes = [part.pm_visible_shape for part in self.parts]
        if pm_shape in all_shapes:
            return True
        return False

    def get_part_from_shape(self, pm_shape: Shape):
        """
        Returns the body part that correspond to particular Pymunk shape.

        Args:
            pm_shape: pymunk shape.

        Returns: Body part.

        """
        return next(
            iter([
                part for part in self.parts
                if part.pm_visible_shape == pm_shape
            ]), None)

    # DYNAMICS

    @property
    def teleported_to(self):
        return self._teleported_to

    def has_teleported_to(self, destination):
        self._teleported_to = destination
        self._has_teleported = True

    @property
    def has_teleported(self):
        return self._has_teleported

    def pre_step(self):
        """
        Reset actuators and reward to 0 before a new step of the environment.
        """

        self.reward = 0

        self._update_teleport()

        for actuator in self.actuators:
            actuator.pre_step()

    def _update_teleport(self):

        self._has_teleported = False

        if isinstance(self._teleported_to, TeleportElement):
            if self._overlaps(self._teleported_to):
                return

        self._teleported_to = None

    def _overlaps(
        self,
        entity: Entity,
    ) -> bool:

        for part in self.parts:

            if entity.pm_visible_shape and part.pm_visible_shape.shapes_collide(entity.pm_visible_shape):
                return True

            if entity.pm_invisible_shape and part.pm_visible_shape.shapes_collide(entity.pm_invisible_shape):
                return True

        return False

    def reset(self):
        """
        Resets all body parts
        """
        self.velocity = [0, 0]

        for part in self.parts:
            part.reset()

    def draw(self,
             surface: Surface,
             excluded: Optional[Union[List[Entity], Entity]] = None):
        """
        Draw the agent on a pygame surface.

        Args:
            surface: Pygame Surface to draw on.
            excluded: parts that should not be drawn on the surface.
        """

        if not excluded:
            list_excluded = []
        elif not isinstance(excluded, list):
            list_excluded = [excluded]
        else:
            list_excluded = excluded

        for part in self.parts:
            if part not in list_excluded:
                part.draw(surface, viewpoint=(0, 0))

    def generate_actions_image(self,
                               width_action: int = 100,
                               height_action: int = 30,
                               plt_mode: bool = False):
        """
        Function that draws all action values of the agent.

        Args:
            width_action: width of the action image in pixels.
            height_action: height of a single action image in pixels.
            plt_mode: if True, returns a pyplot-compatible image.

        Returns:
            Image of the agent's current actions.

        """
        # pylint: disable=too-many-locals

        number_parts_with_actions = len(self.parts)

        assert isinstance(self._current_actions, dict)
        count_all_actions = len(self._current_actions)

        total_height_actions = number_parts_with_actions * (_BORDER_IMAGE + height_action) \
                               + (_BORDER_IMAGE + height_action) * count_all_actions + _BORDER_IMAGE

        img_actions = Image.new("RGB", (width_action, total_height_actions),
                                (255, 255, 255))
        drawer_action_image = ImageDraw.Draw(img_actions)

        fnt = ImageFont.truetype("Pillow/Tests/fonts/FreeMono.ttf",
                                 int(height_action * 2 / 3))

        current_height = 0

        for part in self.parts:

            w_text, h_text = fnt.getsize(text=part.name.upper())
            drawer_action_image.text(
                (width_action / 2.0 - w_text / 2,
                 current_height + height_action / 2 - h_text / 2),
                part.name.upper(),
                font=fnt,
                fill=(0, 0, 0))

            start = (0, current_height)
            end = (width_action, current_height + height_action)
            drawer_action_image.rectangle([start, end],
                                          outline=(0, 0, 0),
                                          width=2)

            current_height += _BORDER_IMAGE + height_action

            all_action_parts = [
                (action, value)
                for action, value in self._current_actions.items()
                if action.part.name == part.name
            ]

            for actuator, _ in all_action_parts:

                actuator.draw(drawer_action_image, width_action,
                              current_height, height_action, fnt)
                current_height += _BORDER_IMAGE + height_action

        img_actions = np.asarray(img_actions) / 255.

        if plt_mode:
            img_actions = img_actions[:, :, ::-1]

        return img_actions

max_stars_repo_name: embaba/simple-playgrounds

max_issues_repo_name: embaba/simple-playgrounds

max_forks_repo_name: embaba/simple-playgrounds


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb

file_names = [["Softsync_s_{0}_log_1_scale=8_#agents=8:itr=0:spcl=False_mom.log",
         "Softsync_mcasS_{0}_log_8_scale=8_#agents=8:itr=0:spcl=False_mom.log",
         "Softsync_mcasSawr_{0}_log_8_scale=1_#agents=8:itr=0:spcl=True_mom.log",
         "Softsync_mcasSawr2_{0}_log_8_scale=1_#agents=8:itr=0:spcl=True_mom.log",
         "Softsync_mcasSbin_{0}_log_8_scale=8_#agents=8:itr=0:spcl=False_mom.log"]

# List a color for Agent we want to plot
colours = ["black", "blue", "red", ]#"orange","yellow","pink","blue","grey"] #can also just give range(0, number_of_colours), if we set auto_colorus = True
# if true use default seaborn colours
auto_colours = False
# number of epochs
epochs = 100
#if there is more than one evaluation of the metrics per epoch
evals_per_epoch = 1
# how many times the simulator was rerun with the same parameters.
repeats = 5
#column in the csv file that coresponds to named metric
indexes = {'Train Loss':2, 'Train Accuracy':3, 'Test Loss':4, 'Test Accuracy':5}

names = ["Baseline", "Asynchronous 8 agents", "Asynchronous 8 agents, scaled learning rate"]
fontsize_titles=14
fontsize_labels = 12

#List to store the read data points
train_loss = []
train_accuracy = []
test_loss = []
test_accuracy = []

for name in file_names:
    res2 = []
    res3 = []
    res0 = []
    res1 = []
    for j in range(1, repeats+1):
        #change the file name here
        data = pd.read_csv(name.format(j), header = None)
        res0.append(data.get_values()[:, indexes['Train Loss']])
        res1.append(data.get_values()[:, indexes['Train Accuracy']])
        res2.append(data.get_values()[:, indexes['Test Loss']])
        res3.append(data.get_values()[:, indexes['Test Accuracy']])
    train_loss.append(res0)
    train_accuracy.append(res1)
    test_loss.append(res2)
    test_accuracy.append(res3)

plt.figure(figsize=[15,10])
plots = [221,222,223,224]
metrics_ensemble = [train_loss,train_accuracy,test_loss,test_accuracy]

nr_agents = len(file_names)
stride = epochs*evals_per_epoch

for key, p in enumerate(plots):
    plt.subplot(p)

    x = np.zeros([repeats * nr_agents * stride])
    for i in range(0, nr_agents * repeats):
        x[i * stride:(i + 1) * stride] = np.arange(1./evals_per_epoch, epochs + 1./evals_per_epoch, 1./evals_per_epoch)

    h = []
    for c, i in enumerate(colours):
        for j in range(repeats * stride):
            h.append(c)

    y = np.zeros([repeats * nr_agents * stride])
    for i, d in enumerate(metrics_ensemble[key]):
        for j, dd in enumerate(d):
            y[i * stride * repeats + j * stride:i * stride * repeats + (j + 1) * stride] = dd

    plt.title(list(indexes.keys())[key], fontsize=fontsize_titles)
    if auto_colours:
        plt_sb = sb.lineplot(x=x, y=y, hue=h, ci="sd")
    else:
        plt_sb = sb.lineplot(x=x, y=y, hue=h, palette=colours, ci="sd")
    agent = names
    plt.legend(agent, loc="center right",fontsize = fontsize_labels)
    plt.xlabel("Epochs", fontsize = fontsize_labels)
    #plt.ylabel("Accuracy", fontsize = fontsize_labels)
plt.tight_layout()
plt.show()

max_stars_repo_name: jwbrun/ASGDSimulator

max_issues_repo_name: jwbrun/ASGDSimulator

max_forks_repo_name: jwbrun/ASGDSimulator


text:
"""
Example 3. Texture-only style transfer.
"""
from __future__ import division
import os
import argparse
import glob

import torch
import torch.nn as nn
import numpy as np
import torchvision.transforms
from skimage.io import imread, imsave
from tqdm.auto import tqdm
import imageio

import neural_renderer as nr
from vgg_loss import GatysVggLoss

current_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = os.path.join(current_dir, 'data')

class Model(nn.Module):
    def __init__(self, filename_obj, filename_style, vgg_path,
                 lambda_content=1, lambda_style=1, lambda_tv=0.01,
                 image_size=224):
        super(Model, self).__init__()
        vertices, faces = nr.load_obj(filename_obj, normalization=False)
        self.register_buffer('vertices', vertices[None, :, :])
        self.register_buffer('faces', faces[None, :, :])

        # create textures
        texture_size = 4
        textures = torch.zeros(1, self.faces.shape[1], texture_size, texture_size, texture_size, 3, dtype=torch.float32)
        self.textures = nn.Parameter(textures)

        # load reference image
        style_image = torch.from_numpy(imread(filename_style).astype('float32') / 255.).permute(2,0,1)[None, ::]
        style_image = torch.nn.functional.interpolate(style_image, (image_size, image_size), mode='bilinear')
        self.register_buffer('style_image', style_image)

        # setup renderer
        renderer = nr.Renderer(camera_mode='look_at', image_size=image_size, perspective=False)
        renderer.light_intensity_directional = 0.0
        renderer.light_intensity_ambient = 1.0
        self.renderer = renderer

        # setup content and style losses
        self.vgg_loss = GatysVggLoss(vgg_path)
        self.lambda_content = lambda_content
        self.lambda_style = lambda_style
        self.lambda_tv = lambda_tv

        # setup graphics projection matrix
        self.P = None

    def forward(self, calc_loss=True):
        self.renderer.eye = nr.get_points_from_angles(2.732, 0, np.random.uniform(0, 360))
        image, _, mask = self.renderer(self.vertices, self.faces, torch.sigmoid(self.textures))  # [batch_size, RGB, image_size, image_size]

        if not calc_loss:
            return image

        s, c, tv = self.vgg_loss(image, None, self.style_image, mask)
        loss = self.lambda_content * c + self.lambda_style * s + self.lambda_tv * tv

        #loss = torch.nn.MSELoss()(image, self.style_image)

        """
        #print(f"C: {c}, S: {s}, TV: {tv}")
        import matplotlib.pyplot as plt
        fig, ax = plt.subplots(1, 2)
        ax[0].imshow(torchvision.transforms.ToPILImage()(image.squeeze().detach().cpu()))
        ax[1].imshow(mask.squeeze().cpu().detach().numpy())
        plt.show()
        """

        return loss, image

def make_gif(filename):
    with imageio.get_writer(filename, mode='I') as writer:
        for filename in sorted(glob.glob('/tmp/_tmp_*.png')):
            writer.append_data(imageio.imread(filename))
            os.remove(filename)
    writer.close()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-io', '--mesh', type=str, default=os.path.join(data_dir, 'teapot.obj'))
    parser.add_argument('-is', '--filename_style', type=str, default="/home/hoellein/datasets/styles/3style/14-2.jpg")
    parser.add_argument('-or', '--filename_output', type=str, default=os.path.join(data_dir, 'style_transfer_teapot_result.gif'))
    parser.add_argument('-vgg', '--vgg_model_path', type=str, default="/home/hoellein/models/vgg_conv.pth")
    parser.add_argument('-lc', '--lambda_content', type=float, default=0)
    parser.add_argument('-ls', '--lambda_style', type=float, default=1)
    parser.add_argument('-ltv', '--lambda_tv', type=float, default=1e7)
    parser.add_argument('-lr', '--learning_rate', type=float, default=0.1)
    parser.add_argument('-e', '--epochs', type=int, default=300)
    parser.add_argument('--size', type=int, default=224)
    parser.add_argument('-g', '--gpu', type=int, default=0)
    args = parser.parse_args()

    model = Model(args.mesh, args.filename_style, args.vgg_model_path,
                  lambda_content=args.lambda_content,
                  lambda_style=args.lambda_style,
                  lambda_tv=args.lambda_tv,
                  image_size=args.size)
    model.cuda()

    # optimize
    optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
    loop = tqdm(range(args.epochs))
    for _ in loop:
        loop.set_description('Optimizing')
        optimizer.zero_grad()
        loss, _ = model()
        loop.set_postfix({"loss": loss.detach().cpu().numpy().item()})
        loss.backward()
        optimizer.step()

    # draw object
    loop = tqdm(range(0, 360, 4))
    for num, azimuth in enumerate(loop):
        loop.set_description('Drawing')
        model.renderer.eye = nr.get_points_from_angles(2.732, 0, azimuth)
        images, _, _ = model.renderer(model.vertices, model.faces, torch.sigmoid(model.textures))
        image = images.detach().cpu().numpy()[0].transpose((1, 2, 0))
        imsave('/tmp/_tmp_%04d.png' % num, image)
    make_gif(args.filename_output)
    nr.save_obj(args.filename_output + ".obj", model.vertices[0], model.faces[0], model.textures[0])

if __name__ == '__main__':
    main()


text:
#coding: utf-8

import os
import time
import random
import random
import torch
import torchaudio

import numpy as np
import soundfile as sf
import torch.nn.functional as F

from torch import nn
from torch.utils.data import DataLoader

import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

np.random.seed(1)
random.seed(1)

SPECT_PARAMS = {
    "n_fft": 2048,
    "win_length": 1200,
    "hop_length": 800
}
MEL_PARAMS = {
    "n_mels": 80,
    "n_fft": 2048,
    "win_length": 1200,
    "hop_length": 800
}

class MelDataset(torch.utils.data.Dataset):
    def __init__(self,
                 data_list,
                 sr=24000,
                 validation=False,
                 ):

        _data_list = [l[:-1].split('|') for l in data_list]
        self.data_list = [(path, int(label)) for path, label in _data_list]
        self.data_list_per_class = {
            target: [(path, label) for path, label in self.data_list if label == target] \
            for target in list(set([label for _, label in self.data_list]))}

        self.sr = sr
        self.to_melspec = torchaudio.transforms.MelSpectrogram(**MEL_PARAMS)

        self.mean, self.std = -4, 4
        self.validation = validation
        self.max_mel_length = 96

    def __len__(self):
        return len(self.data_list)

    def __getitem__(self, idx):
        data = self.data_list[idx]
        mel_tensor, label, initlmk, motion, input_lmks = self._load_data(data)
        ref_data = random.choice(self.data_list)
        ref_mel_tensor, ref_label, ref_initlmk, ref_motion, _ = self._load_data(ref_data)
        ref2_data = random.choice(self.data_list_per_class[ref_label])
        ref2_mel_tensor, _, ref2_initlmk, ref2_motion, _ = self._load_data(ref2_data)
        return mel_tensor, label, ref_mel_tensor, ref2_mel_tensor, ref_label, initlmk, motion, input_lmks

    def _load_data(self, path):
        wave_tensor, label = self._load_tensor(path)
        initlmk, motion, input_lmks = self._load_motion(path)

        # left track == right track in 5 samples
        if len(wave_tensor.shape) > 1:
            wave_tensor = wave_tensor[:, 0]

        ##TODO: Release when training -by gy.
        if not self.validation: # random scale for robustness
            random_scale = 0.5 + 0.5 * np.random.random()
            wave_tensor = random_scale * wave_tensor

        mel_tensor = self.to_melspec(wave_tensor)
        mel_tensor = (torch.log(1e-5 + mel_tensor) - self.mean) / self.std
        mel_length = mel_tensor.size(1)
        if mel_length > self.max_mel_length:

            random_start = np.random.randint(0, mel_length - self.max_mel_length)

            # random_start = 0
            mel_tensor = mel_tensor[:, random_start:random_start + self.max_mel_length]
            # motion = motion = [:, random_start]
            motion = motion[random_start:random_start + self.max_mel_length, :, :]
            input_lmks = input_lmks[random_start:random_start + self.max_mel_length, :, :]
            if random_start >= 1:
                for i in range(random_start-1):
                    initlmk = initlmk + motion[i, :, :]
        else:
            motion = motion[:mel_length-1, :, :]
            input_lmks = input_lmks[:mel_length-1, :, :]
        # print(mel_tensor.shape, motion.shape, input_lmks.shape)
        return mel_tensor, label, initlmk, motion, input_lmks

    def _preprocess(self, wave_tensor, ):
        mel_tensor = self.to_melspec(wave_tensor)
        mel_tensor = (torch.log(1e-5 + mel_tensor) - self.mean) / self.std
        return mel_tensor

    def _load_tensor(self, data):
        wave_path, label = data
        label = int(label)
        wave, sr = sf.read(wave_path)
        wave_tensor = torch.from_numpy(wave).float()
        return wave_tensor, label

    def _load_motion(self, data):
        bname = os.path.basename(data[0])
        dsp = data[0].replace('wav/', 'video/front/')
        initlmk_path = dsp.replace(bname, 'initlmk_'+bname.replace('wav', 'npy'))
        initlmk = np.load(initlmk_path)
        motion_path= dsp.replace(bname, 'motion_'+bname.replace('wav', 'npy'))
        motion = np.load(motion_path)

        input_lmks = [initlmk]
        for i in motion:
            input_lmks.append(input_lmks[-1]+i)
        input_lmks = np.stack(input_lmks, axis=0)

        return torch.from_numpy(initlmk), torch.from_numpy(motion), torch.from_numpy(input_lmks)

class MelDatasetPCA(torch.utils.data.Dataset):
    def __init__(self,
                 data_list,
                 sr=24000,
                 validation=False,
                 ):

        _data_list = [l[:-1].split('|') for l in data_list]
        self.data_list = [(path, int(label)) for path, label in _data_list]
        self.data_list_per_class = {
            target: [(path, label) for path, label in self.data_list if label == target] \
            for target in list(set([label for _, label in self.data_list]))}

        self.sr = sr
        self.to_melspec = torchaudio.transforms.MelSpectrogram(**MEL_PARAMS)

        self.mean, self.std = -4, 4
        self.validation = validation
        self.max_mel_length = 96

        self.mean_mead = torch.from_numpy(np.load('./PCA/mean_mead.npy'))
        self.U = torch.from_numpy(np.load('./PCA/U_mead.npy'))

    def __len__(self):
        return len(self.data_list)

    def __getitem__(self, idx):
        data = self.data_list[idx]
        mel_tensor, label, initlmk, motion, output_lmks = self._load_data(data)
        ref_data = random.choice(self.data_list)
        ref_mel_tensor, ref_label, ref_initlmk, ref_motion, _ = self._load_data(ref_data)
        ref2_data = random.choice(self.data_list_per_class[ref_label])
        ref2_mel_tensor, _, ref2_initlmk, ref2_motion, _ = self._load_data(ref2_data)
        return mel_tensor, label, ref_mel_tensor, ref2_mel_tensor, ref_label, initlmk, motion, output_lmks

    def _load_data(self, path):
        wave_tensor, label = self._load_tensor(path)
        initlmk, motion, output_lmks = self._load_motion(path)
        # left track == right track in 5 samples
        if len(wave_tensor.shape) > 1:
            wave_tensor = wave_tensor[:, 0]

        ##TODO: Release when training -by gy.
        if not self.validation: # random scale for robustness
            random_scale = 0.5 + 0.5 * np.random.random()
            wave_tensor = random_scale * wave_tensor

        mel_tensor = self.to_melspec(wave_tensor)
        mel_tensor = (torch.log(1e-5 + mel_tensor) - self.mean) / self.std
        # print(initlmk.shape, output_lmks.shape, mel_tensor.shape, path)

        ##TODO: This is a bug
        if motion.shape[0] < (mel_tensor.shape[1]-2):
            # print(path, ' has problem.')
            # print('motion shape: ', motion.shape, 'mel_tensor shape: ', mel_tensor.shape)
            # assert(0)
            mel_length = min(motion.shape[0], mel_tensor.shape[1])
        else:
            mel_length = mel_tensor.size(1)

        if mel_length > self.max_mel_length:
            random_start = np.random.randint(0, mel_length - self.max_mel_length)

            # random_start = 0
            mel_tensor = mel_tensor[:, random_start:random_start + self.max_mel_length]
            # motion = motion = [:, random_start]

            if random_start >= 1:
                for i in range(random_start-1):
                    initlmk = initlmk + motion[i, :, :]
            motion = motion[random_start : random_start + self.max_mel_length, :, :]
            output_lmks = output_lmks[random_start : random_start + self.max_mel_length, :]
        else:
            mel_tensor = mel_tensor[:, :mel_length-1]
            motion = motion[:mel_length-1, :, :]
            output_lmks = output_lmks[:mel_length-1, :]
        # print(mel_tensor.shape, motion.shape, input_lmks.shape)
        return mel_tensor, label, initlmk, motion, output_lmks

    def _preprocess(self, wave_tensor, ):
        mel_tensor = self.to_melspec(wave_tensor)
        mel_tensor = (torch.log(1e-5 + mel_tensor) - self.mean) / self.std
        return mel_tensor

    def _load_tensor(self, data):
        wave_path, label = data
        label = int(label)
        wave, sr = sf.read(wave_path)
        wave_tensor = torch.from_numpy(wave).float()
        # print('wave: ', wave_tensor.shape, wave_path)
        return wave_tensor, label

    def _load_motion(self, data):
        bname = os.path.basename(data[0])
        dsp = data[0].replace('wav/', 'video/front/')
        # print(dsp)
        initlmk_path = dsp.replace(bname, 'initlmk_'+bname[:-4]+'_multiland.npy')
        initlmk = np.load(initlmk_path)
        # print(initlmk.shape)
        motion_path= dsp.replace(bname, 'motion_'+bname[:-4]+'_multiland.npy')
        motion = np.load(motion_path)
        input_lmks = [initlmk]
        # print('mot: ', motion.shape)
        for i in motion:
            input_lmks.append(input_lmks[-1]+i)
        output_lmks = np.stack(input_lmks[1:], axis=0)[:,:,:2]
        b = output_lmks.shape[0]
        output_lmks = np.reshape(output_lmks, [b, 468*2])
        output_lmks = torch.from_numpy(output_lmks)

        # print(((input_lmks[1:] - input_lmks[0:-1]) == motion).any()==True)
        output_lmks = torch.mm(output_lmks - self.mean_mead.expand_as(output_lmks), self.U[:,:32])

        return torch.from_numpy(initlmk), torch.from_numpy(motion), output_lmks

class Collater(object):
    """
    Args:
      adaptive_batch_size (bool): if true, decrease batch size when long data comes.
    """

    def __init__(self, return_wave=False):
        self.text_pad_index = 0
        self.return_wave = return_wave
        self.max_mel_length = 96
        self.mel_length_step = 16
        self.latent_dim = 16

    def __call__(self, batch):
        batch_size = len(batch)
        nmels = batch[0][0].size(0)
        mels = torch.zeros((batch_size, nmels, self.max_mel_length)).float()
        labels = torch.zeros((batch_size)).long()
        ref_mels = torch.zeros((batch_size, nmels, self.max_mel_length)).float()
        ref2_mels = torch.zeros((batch_size, nmels, self.max_mel_length)).float()
        ref_labels = torch.zeros((batch_size)).long()
        inits = torch.zeros((batch_size, 68, 3))
        mots = torch.zeros((batch_size, self.max_mel_length, 68, 3))
        length_mots = torch.zeros((batch_size)).long()
        input_lmkss = torch.zeros((batch_size, self.max_mel_length, 68, 3))

        for bid, (mel, label, ref_mel, ref2_mel, ref_label, init, mot, input_lmks) in enumerate(batch):
            mel_size = mel.size(1)
            mels[bid, :, :mel_size] = mel
            # print('mels shape:', mels.shape, mel.shape)

            ref_mel_size = ref_mel.size(1)
            ref_mels[bid, :, :ref_mel_size] = ref_mel

            ref2_mel_size = ref2_mel.size(1)
            ref2_mels[bid, :, :ref2_mel_size] = ref2_mel

            labels[bid] = label
            ref_labels[bid] = ref_label

            inits[bid] = init

            mot_size = min(mot.size(0), self.max_mel_length)
            length_mots[bid] = mot_size
            mots[bid,:mot_size,:,:] = mot[:mot_size,:,:]
            input_lmkss[bid,:mot_size,:,:] = input_lmks

        z_trg = torch.randn(batch_size, self.latent_dim)
        z_trg2 = torch.randn(batch_size, self.latent_dim)

        mels, ref_mels, ref2_mels = mels.unsqueeze(1), ref_mels.unsqueeze(1), ref2_mels.unsqueeze(1)
        return mels, labels, ref_mels, ref2_mels, ref_labels, z_trg, z_trg2, inits, mots, length_mots, input_lmkss

def build_dataloader(path_list,
                     validation=False,
                     batch_size=4,
                     num_workers=1,
                     device='cpu',
                     collate_config={},
                     dataset_config={}):

    dataset = MelDataset(path_list, validation=validation)
    collate_fn = Collater(**collate_config)
    data_loader = DataLoader(dataset,
                             batch_size=batch_size,
                             shuffle=(not validation),
                             num_workers=num_workers,
                             drop_last=(not validation),
                             collate_fn=collate_fn,
                             pin_memory=(device != 'cpu'))

    return data_loader

class CollaterPCA(object):
    """
    Args:
      adaptive_batch_size (bool): if true, decrease batch size when long data comes.
    """

    def __init__(self, return_wave=False):
        self.text_pad_index = 0
        self.return_wave = return_wave
        self.max_mel_length = 96
        self.mel_length_step = 16
        self.latent_dim = 16

    def __call__(self, batch):
        batch_size = len(batch)
        nmels = batch[0][0].size(0)
        mels = torch.zeros((batch_size, nmels, self.max_mel_length)).float()
        labels = torch.zeros((batch_size)).long()
        ref_mels = torch.zeros((batch_size, nmels, self.max_mel_length)).float()
        ref2_mels = torch.zeros((batch_size, nmels, self.max_mel_length)).float()
        ref_labels = torch.zeros((batch_size)).long()
        inits = torch.zeros((batch_size, 468, 3))
        mots = torch.zeros((batch_size, self.max_mel_length, 468, 3))
        length_mots = torch.zeros((batch_size)).long()
        output_lmkss = torch.zeros((batch_size, self.max_mel_length, 32))

        for bid, (mel, label, ref_mel, ref2_mel, ref_label, init, mot, output_lmks) in enumerate(batch):
            mel_size = mel.size(1)
            mels[bid, :, :mel_size] = mel
            # print('mels shape:', mels.shape, mel.shape)

            ref_mel_size = ref_mel.size(1)
            ref_mels[bid, :, :ref_mel_size] = ref_mel

            ref2_mel_size = ref2_mel.size(1)
            ref2_mels[bid, :, :ref2_mel_size] = ref2_mel

            labels[bid] = label
            ref_labels[bid] = ref_label

            inits[bid] = init

            mot_size = min(mot.size(0), self.max_mel_length)
            length_mots[bid] = mot_size
            mots[bid,:mot_size,:,:] = mot[:mot_size,:,:]
            output_lmkss[bid,:mot_size,:] = output_lmks

        z_trg = torch.randn(batch_size, self.latent_dim)
        z_trg2 = torch.randn(batch_size, self.latent_dim)

        mels, ref_mels, ref2_mels = mels.unsqueeze(1), ref_mels.unsqueeze(1), ref2_mels.unsqueeze(1)
        return mels, labels, ref_mels, ref2_mels, ref_labels, z_trg, z_trg2, inits, mots, length_mots, output_lmkss

def build_dataloader_pca(path_list,
                     validation=False,
                     batch_size=4,
                     num_workers=1,
                     device='cpu',
                     collate_config={},
                     dataset_config={}):

    dataset = MelDatasetPCA(path_list, validation=validation)
    collate_fn = CollaterPCA(**collate_config)
    data_loader = DataLoader(dataset,
                             batch_size=batch_size,
                             shuffle=(not validation),
                             num_workers=num_workers,
                             drop_last=(not validation),
                             collate_fn=collate_fn,
                             pin_memory=(device != 'cpu'))

    return data_loader


text:
'''
Created on Jun 13, 2018

@author: rch
'''

from bmcs.mats.mats_damage_fn import \
    IDamageFn, LiDamageFn, JirasekDamageFn, AbaqusDamageFn,\
    PlottableFn, DamageFn
from traits.api import Float, Property
from traitsui.api import View, Item, VGroup, UItem

import numpy as np
import pylab as p

class GfDamageFn(DamageFn):
    '''Class defining the damage function coupled with the fracture
    energy of a cohesive crack model.
    '''
    L_s = Float(1.0,
                MAT=True,
                input=True,
                label="L_s",
                desc="Length of the softening zone",
                enter_set=True,
                auto_set=False)

    E = Float(34000.0,
              MAT=True,
              input=True,
              label="E",
              desc="Young's modulus",
              enter_set=True,
              auto_set=False)

    f_t = Float(4.5,
                MAT=True,
                input=True,
                label="f_t",
                desc="Tensile strength",
                enter_set=True,
                auto_set=False)

    G_f = Float(0.004,
                MAT=True,
                input=True,
                label="G_f",
                desc="Fracture energy",
                enter_set=True,
                auto_set=False)

    eps_0 = Property()

    def _get_eps_0(self):
        return self.f_t / self.E

    eps_ch = Property()

    def _get_eps_ch(self):
        return self.G_f / self.f_t

    plot_max = Property(depends_on='G_f,f_t,E')

    def _get_plot_max(self):
        return self.eps_ch * self.L_s * 3.0

    def __call__(self, eps):
        L_s = self.L_s
        f_t = self.f_t
        G_f = self.G_f
        E = self.E
        eps_0 = self.eps_0
        g_eps = 1 - f_t * np.exp(-f_t * (eps - eps_0) * L_s / G_f) \
            / (E * eps)
        g_eps[np.where(eps - eps_0 < 0)] = 0
        return g_eps

    def diff(self, eps):
        L_s = self.L_s
        f_t = self.f_t
        G_f = self.G_f
        E = self.E
        eps_0 = self.eps_0
        d_g_eps = (f_t * np.exp(L_s * (eps_0 - eps) * f_t / G_f)
                   / (E * G_f * eps**2) *
                   (G_f + L_s * eps * f_t))
        d_g_eps[np.where(eps - eps_0 < 0)] = 0.0
        return d_g_eps

    traits_view = View(
        VGroup(
            VGroup(
                Item('s_0', style='readonly',
                     full_size=True, resizable=True),
                Item('f_t'),
                Item('G_f'),
                Item('E'),
            ),
            VGroup(
                UItem('fn@', height=300)
            )
        )
    )

    tree_view = traits_view

f_t = 2.4
G_f = 0.090
E = 20000.0
eps_0 = f_t / E
eps_ch = G_f / f_t
L_ch = E * G_f / f_t**2

print('L_ch', L_ch)

L = 45.0
u_max = 0.15
eps_max = eps_ch
eps = np.linspace(0, eps_max, 100)

omega_fn_abaqus = AbaqusDamageFn(s_0=eps_0, s_u=0.03)
omega_fn_gf = GfDamageFn(G_f=G_f, f_t=f_t, E=E)

omega_fn = omega_fn_gf

n_T = 100
K_max = 200

for N in [1.00001, 3, 4, 5, 9]:
    sig_t = []
    eps_t = []
    L_el = (N - 1.0) / N * L
    L_s = 1.0 / N * L
    eps_s = 0.0
    eps_s_arr = np.array([eps_s], dtype=np.float_)
    u_t = np.linspace(0, u_max, n_T)
    omega_fn.L_s = L_s
    for u in u_t:
        for K in range(K_max):
            R = 1 / L_el * (u - eps_s_arr * L_s) - \
                (1 - omega_fn(eps_s_arr)) * eps_s_arr
            if np.fabs(R) < 1e-8:
                break
            dR = -L_s / L_el + omega_fn.diff(eps_s_arr) * \
                eps_s_arr - (1 - omega_fn(eps_s_arr))
            d_eps_s = -R / dR
            eps_s_arr += d_eps_s
            if K == K_max - 1:
                raise ValueError('No convergence')
        sig = ((1.0 - omega_fn(eps_s_arr)) * E * eps_s_arr)[0]
        sig_t.append(sig)
        eps_t.append(u / L)
    p.plot(eps_t, sig_t)
p.show()


text:
import os
import csv
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, accuracy_score, recall_score, roc_auc_score, classification_report, roc_curve,auc
import warnings
warnings.filterwarnings('ignore')

def model(algorithm, dtrain_X, dtrain_Y, dtest_X, dtest_Y, blind_X, blind_Y, algo = 0):
    cols = dtrain_X.columns
    algorithm.fit(dtrain_X[cols],dtrain_Y)
    predictions = algorithm.predict(dtest_X[cols]) # Predictions for the test dataset
    blindPredictions = algorithm.predict(blind_X[cols]) # Predictions for the blind dataset
    print (algorithm)

    print ("Accuracy score : ", round(accuracy_score(dtest_Y, predictions), 3))
    print ("Recall score   : ", round(recall_score(dtest_Y, predictions), 3))
    print ("classification report :\n", classification_report(dtest_Y, predictions))

    if(algo == 1):
        fname = "logit"
    elif(algo == 2):
        fname = "randomForest"

    s0 = dtest_Y.reset_index(level=0)
    s1 = pd.Series(predictions, name = 'predicted')
    pd.concat([s0, s1], axis = 1).to_csv(fname + "modelPred.csv", sep=',', index = False) # Save predictions to file
    print("Model predictions for test data written to " + fname +"modelPred.csv")

    s2 = blind_Y.reset_index(level=0)
    s3 = pd.Series(blindPredictions, name = 'predicted')
    pd.concat([s2.CustomerID, s3], axis = 1).to_csv(fname + "blindPred.csv", sep=',', index = False) # Save blind test predictions to file
    print("Model predictions for blind data written to "+ fname + "blindPred.csv")

    return classification_report(dtest_Y, predictions, output_dict = True)

def missingImpute(data, numTypes, objTypes):
    data_new  = data[[i for i in data.columns if i not in numTypes + objTypes]]

    ## Impute median value for features with missing numerical data
    for j, col in enumerate(numTypes):
        data_new[j] = data[col].fillna(int(data[col].median()))
        data_new = data_new.rename(columns = {j: col})

    ## Impute mode (most frequent) value for features with missing categorical data
    for k, col in enumerate(objTypes):
        data_new[k] = data[col].fillna(data[col].mode()[0])
        data_new = data_new.rename(columns={k: col})

    return data_new

def main():
    print("+--------------+")
    print("Reading the Data")
    path = os.getcwd()
    applicationFile = path + '/ds-app.tsv'
    borrowerFile = path + '/ds-borrower.tsv'
    creditFile = path + '/ds-credit.tsv'
    resultFile = path + '/ds-result.tsv'

    files = [applicationFile, borrowerFile, creditFile, resultFile]
    holder = {}
    for file in files:
        with open(file) as fp:
            reader = csv.reader(fp, delimiter='\t')
            header = next(reader, None)
            table = []
            for row in reader:
                elements = row[0].split()
                table.append(elements)
            df = pd.DataFrame(table, columns = header)
            holder[file] = df

    applicationdf = holder[applicationFile]
    borrowerdf = holder[borrowerFile]
    creditdf = holder[creditFile]
    resultdf = holder[resultFile]

    print("+-----------------+")
    print("Setting the indices")
    applicationdf.set_index('CustomerID', inplace=True)
    borrowerdf.set_index('CustomerID', inplace=True)
    creditdf.set_index('CustomerID', inplace=True)
    resultdf.set_index('CustomerID', inplace=True)

    print("+-------------------+")
    print("Merging the data sets")
    join1 = pd.merge(applicationdf,
                      borrowerdf,
                      right_index=True,
                      left_index=True,
                      how = 'left')

    join2 = pd.merge(join1,
                      creditdf,
                      right_index=True,
                      left_index=True,
                      how = 'left')

    join3 = pd.merge(join2,
                      resultdf,
                      right_index=True,
                      left_index=True,
                      how = 'left')

    join3.index = join3.index.map(int)
    join3.sort_index(inplace=True)
    join3[["LoanPayoffPeriodInMonths","RequestedAmount","InterestRate",
           "YearsInCurrentResidence","Age","NumberOfDependantsIncludingSelf",
           "CurrentOpenLoanApplications"]] = join3[["LoanPayoffPeriodInMonths","RequestedAmount","InterestRate",
                                                    "YearsInCurrentResidence","Age","NumberOfDependantsIncludingSelf",
                                                    "CurrentOpenLoanApplications"]].apply(pd.to_numeric)

    blindTestdf = join3[join3['WasTheLoanApproved'].isnull()]
    combineddf = join3[-join3['WasTheLoanApproved'].isnull()]

    print("+---------------+")
    print("Train Test split.")
    train, test = train_test_split(combineddf, test_size = 0.25)
    num_dtypes = [i for i in train.select_dtypes(include = np.number).columns if i not in applicationdf.columns]
    obj_dtypes = [i for i in train.select_dtypes(include = np.object).columns if i not in applicationdf.columns]
    obj_dtypes = [x for x in obj_dtypes if x != 'WasTheLoanApproved']

    print("+--------------------+")
    print("Handling missing data.")
    # train_new  = train[[i for i in train.columns if i not in num_dtypes + obj_dtypes]]
    # test_new  = test[[i for i in test.columns if i not in num_dtypes + obj_dtypes]]
    # blind_new = blindTestdf[[i for i in blindTestdf.columns if i not in num_dtypes + obj_dtypes]]
    train_new = missingImpute(train, num_dtypes, obj_dtypes)
    test_new = missingImpute(test, num_dtypes, obj_dtypes)
    blind_new = missingImpute(blindTestdf, num_dtypes, obj_dtypes)

    ## Impute median value for features with missing numerical data
    # for j, col in enumerate(num_dtypes):
    #     train_new[j] = train[col].fillna(int(train[col].median()))
    #     train_new = train_new.rename(columns = {j: col})
    #     test_new[j] = test[col].fillna(int(test[col].median()))
    #     test_new = test_new.rename(columns = {j: col})
        # blind_new[j] = blindTestdf[col].fillna(int(blindTestdf[col].median()))
        # blind_new = blind_new.rename(columns = {j: col})

    ## Impute mode (most frequent) value for features with missing categorical data
    # for k, col in enumerate(obj_dtypes):
    #     train_new[k] = train[col].fillna(train[col].mode()[0])
    #     train_new = train_new.rename(columns={k: col})
    #     test_new[k] = test[col].fillna(test[col].mode()[0])
    #     test_new = test_new.rename(columns={k: col})
        # blind_new[j] = blindTestdf[col].fillna(blindTestdf[col].mode()[0])
        # blind_new = blind_new.rename(columns = {j: col})

    ## Converting 'Y'/'N' values to 1/0
    blind_new["WasTheLoanApproved"].fillna(0, inplace=True)
    train_new.WasTheLoanApproved.replace(('Y', 'N'), (1, 0), inplace=True)
    test_new.WasTheLoanApproved.replace(('Y', 'N'), (1, 0), inplace=True)

    train_new[['WasTheLoanApproved']] = train_new[['WasTheLoanApproved']].apply(pd.to_numeric)
    test_new[['WasTheLoanApproved']] = test_new[['WasTheLoanApproved']].apply(pd.to_numeric)

    print("+--------------------+")
    print("Label and hot encoding")
    le = LabelEncoder()
    obj_dtypes = [i for i in train_new.select_dtypes(include = np.object).columns]
    for i in obj_dtypes:
        train_new[i] = le.fit_transform(train_new[i])
        test_new[i] = le.fit_transform(test_new[i])
        blind_new[i] = le.fit_transform(blind_new[i])
    train_new = pd.get_dummies(data=train_new,columns=obj_dtypes)
    test_new = pd.get_dummies(data=test_new,columns=obj_dtypes)
    blind_new = pd.get_dummies(data=blind_new,columns=obj_dtypes)

    ## Splitting training data into X, Y
    x_train = train_new[[i for i in train_new.columns if i not in ['WasTheLoanApproved']]]
    y_train = train_new.WasTheLoanApproved

    ## Splitting testing data into X, Y
    x_test = test_new[[i for i in test_new.columns if i not in ['WasTheLoanApproved']]]
    y_test = test_new.WasTheLoanApproved

    ## Splitting blind test data into X, Y
    x_blind = blind_new[[i for i in blind_new.columns if i not in ['WasTheLoanApproved']]]
    y_blind = blind_new.WasTheLoanApproved

    print("+---------------------------------------+")
    print("Running a Logistic Regression Classifier.")
    logit = LogisticRegression()
    logitModel = model(logit, x_train, y_train, x_test, y_test, x_blind, y_blind, algo = 1)
    # print(logitModel)

    print("+---------------------------------+")
    print("Running a Random Forest Classifier.")
    rfc = RandomForestClassifier()
    rfcModel = model(rfc, x_train, y_train, x_test, y_test, x_blind, y_blind, algo = 2)
    # print(rfcModel)

if __name__ == '__main__':
    main()

max_stars_repo_name: NeilBardhan/logistic-regression

max_issues_repo_name: NeilBardhan/logistic-regression

max_forks_repo_name: NeilBardhan/logistic-regression