text:
"""
Created by Rohan Paleja on September 23, 2019
Display performance of Sammut et. al. benchmark
"""

import torch
import sys
import torch.nn as nn
import torch.nn.functional as F

sys.path.insert(0, '/home/Anonymous/PycharmProjects/bayesian_prolo')
import numpy as np
from torch.autograd import Variable
from utils.naive_utils import  load_in_naive_data
from utils.hri_utils import save_performance_results, save_trained_nets
# sys.path.insert(0, '../')

torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(50)
np.random.seed(50)

class NNSmall(nn.Module):
    """
    Sammut et. al. benchmark
    """

    def __init__(self):
        super(NNSmall, self).__init__()
        self.fc1 = nn.Linear(242, 128)
        self.relu1 = nn.ReLU()
        self.fc2 = nn.Linear(128, 128)
        self.relu2 = nn.ReLU()
        self.fc21 = nn.Linear(128, 32)
        self.relu21 = nn.ReLU()
        self.fc22 = nn.Linear(32, 32)
        self.relu22 = nn.ReLU()
        self.fc23 = nn.Linear(32, 32)
        self.relu23 = nn.ReLU()
        self.fc3 = nn.Linear(32, 20)
        self.soft = nn.LogSoftmax(dim=0)

    def forward(self, x):
        """
        forward pass
        :param x: i_minus_j or vice versa
        :return:
        """
        x = self.fc1(x)
        x = self.relu1(x)
        x = self.fc2(x)
        x = self.relu2(x)
        x = self.fc21(x)
        x = self.relu21(x)
        x = self.fc22(x)
        x = self.relu22(x)
        x = self.fc23(x)
        x = self.relu23(x)
        x = self.fc3(x)
        # x = self.soft(x)

        return x

# noinspection PyTypeChecker,PyArgumentList
class NNTrain:
    """
    class structure to train the NN for a certain amount of schedules.
    This class handles training the NN, evaluating the NN, and saving the results
    """

    def __init__(self):
        self.num_schedules = 150
        self.num_test_schedules = 100
        self.total_loss_array = []

        self.X_train_naive, self.Y_train_naive, self.schedule_array_train_naive, self.X_test_naive, self.Y_test_naive, self.schedule_array_test_naive = load_in_naive_data(
            250,250)

        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.model = NNSmall().to(device)

        print(self.model.state_dict())
        self.opt = torch.optim.Adam(self.model.parameters())
        self.when_to_save = 1000

        self.X_train_naive, \
        self.Y_train_naive, \
        self.schedule_array_train_naive,  = self.sample_data(150)

        self.X_test_naive, \
        self.Y_test_naive, \
        self.schedule_array_test_naive,  = self.sample_test_data(100)
        self.num_test_schedules = 100

    def sample_data(self, size):
        # return self.X_train_naive[0:size * 20 * 20], \
        #        self.Y_train_naive[0:size * 20 * 20], \
        #        self.schedule_array_train_naive[0:size], \
        #        self.start_of_each_set_twenty_train[0:size * 20]
        if size == 250:
            set_of_twenty = 0
        else:
            set_of_twenty = np.random.randint(250-size)
        self.sample_min = set_of_twenty * 20
        return self.X_train_naive[set_of_twenty*20:set_of_twenty*20 + size * 20], \
               self.Y_train_naive[set_of_twenty*20:set_of_twenty*20 + size * 20], \
               self.schedule_array_train_naive[set_of_twenty:set_of_twenty+size]

    def sample_test_data(self, size):
        # return self.X_train_naive[0:size * 20 * 20], \
        #        self.Y_train_naive[0:size * 20 * 20], \
        #        self.schedule_array_train_naive[0:size], \
        #        self.start_of_each_set_twenty_train[0:size * 20]
        if size == 250:
            set_of_twenty = 0
        else:
            set_of_twenty = np.random.randint(250-size)
        self.sample_test_min = set_of_twenty * 20
        return self.X_test_naive[set_of_twenty*20:set_of_twenty*20 + size * 20], \
               self.Y_test_naive[set_of_twenty*20:set_of_twenty*20 + size * 20], \
               self.schedule_array_test_naive[set_of_twenty:set_of_twenty+size]

    def train(self):
        """
        Trains NN.
        Randomly samples a schedule and timestep within that schedule, and passes in the corresponding data in an attempt to classify which task was scheduled
        :return:
        """

        epochs = 400000
        for epoch in range(epochs):

            # sample a timestep before the cutoff for cross_validation
            rand_timestep_within_sched = np.random.randint(len(self.X_train_naive))
            input_nn = self.X_train_naive[rand_timestep_within_sched]
            truth_nn = self.Y_train_naive[rand_timestep_within_sched]

            # iterate over pairwise comparisons
            if torch.cuda.is_available():
                input_nn = Variable(torch.Tensor(np.asarray(input_nn).reshape(1, 242)).cuda())  # change to 5 to increase batch size
                truth = Variable(torch.Tensor(np.asarray(truth_nn).reshape(1)).cuda().long())
            else:
                input_nn = Variable(torch.Tensor(np.asarray(input_nn).reshape(1, 242)))
                truth = Variable(torch.Tensor(np.asarray(truth_nn).reshape(1)).long())

            self.opt.zero_grad()
            output = self.model.forward(input_nn)
            loss = F.cross_entropy(output, truth)

            loss.backward()

            # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5)
            self.opt.step()

            self.total_loss_array.append(loss.item())

            if epoch % 1000 == 999:
                print('loss at', epoch, ', total loss (average for each 100, averaged)', np.mean(self.total_loss_array[-100:]))
                # print(self.model.state_dict())
                # save_trained_nets(self.model, 'naive_NN' + str(epoch))

    def evaluate_on_test_data(self):

        """
        Evaluate performance of a trained network.
        This is tested on 20% of the data and will be stored in a text file.
        :return:
        """

        prediction_accuracy = [0, 0]
        percentage_accuracy_top1 = []
        percentage_accuracy_top3 = []

        for i, schedule in enumerate(self.schedule_array_test_naive):
            for count in range(schedule[0]-self.sample_test_min, schedule[1] -self.sample_test_min + 1):

                net_input = self.X_test_naive[count]
                truth = self.Y_test_naive[count]

                if torch.cuda.is_available():
                    input_nn = Variable(torch.Tensor(np.asarray(net_input).reshape(1, 242)).cuda())
                    truth = Variable(torch.Tensor(np.asarray(truth).reshape(1)).cuda().long())
                else:
                    input_nn = Variable(torch.Tensor(np.asarray(net_input).reshape(1, 242)))
                    truth = Variable(torch.Tensor(np.asarray(truth).reshape(1)))

                # forward
                output = self.model.forward(input_nn)

                index = torch.argmax(output).item()

                # confusion_matrix[truth][index] += 1
                # top 3
                _, top_three = torch.topk(output, 3)

                if index == truth.item():
                    prediction_accuracy[0] += 1

                if truth.item() in top_three.detach().cpu().tolist()[0]:
                    prediction_accuracy[1] += 1

            print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20)

            print('schedule num:', i)

            percentage_accuracy_top1.append(prediction_accuracy[0] / 20)
            percentage_accuracy_top3.append(prediction_accuracy[1] / 20)
            prediction_accuracy = [0, 0]

        print(np.mean(percentage_accuracy_top1))
        # save_performance_results(percentage_accuracy_top1, percentage_accuracy_top3, 'results_NN_naive')
        return np.mean(percentage_accuracy_top1)

def main():
    """
    entry point for file
    :return:
    """
    res = []
    for i in range(3):
        trainer = NNTrain()
        trainer.train()
        out = trainer.evaluate_on_test_data()
        res.append(out)
    print(np.mean(res))
    print(np.std(res))

if __name__ == '__main__':
    main()


text:
import Quicksort
import numpy as np
# using quicksort here as it has small overhead and each user only has a small number of transactions

def get_user_purchase_deets(shoppers,brands):
    # we shall get deets: total_num_transactions, start_row, end_row, brands as a list
    # add more deets if you need more

    end_row = 0
    total_transactions = len(shoppers)
    # Initialize
    user_deets = {}
    users_list = []
    num_transactions_list = []
    start_row_list =[]
    end_row_list = []
    list_of_transactions = []

    # Fill in values
    while end_row < total_transactions:
        num_transactions = 0
        transactions_list = []
        start_row = end_row
        start_user = shoppers[start_row]
        while(end_row < total_transactions and shoppers[end_row] == start_user):
            num_transactions+=1
            transactions_list.append(brands[end_row])
            end_row+=1
        Quicksort.quicksort(transactions_list)

        # input values into main lists
        users_list.append(start_user)
        num_transactions_list.append(num_transactions)
        start_row_list.append(start_row)
        end_row_list.append(end_row)
        list_of_transactions.append(transactions_list)

    user_deets["user_id"] = np.array(users_list,dtype = np.int)
    user_deets["num_transactions"] = np.array(num_transactions_list, dtype=np.int16)
    user_deets["start_row"] = np.array(start_row_list, dtype=np.int32)
    user_deets["end_row"] = np.array(end_row_list, dtype=np.int32)
    user_deets["list_of_transactions"] = list_of_transactions
    user_deets["meta"] = {"size": len(num_transactions_list),
                          "num_columns": 5,
                          "column_type_list": [("user_id",'int'),
                                               ("num_transactions",'int16'),
                                               ("start_row",'int32'),
                                               ("end_row",'int32'),
                                               ("list_of_transactions",'list')]}

    return user_deets

# Earlier was using row wise storage, converting to columnwise keeping with most databases/R etc.
def get_user_purchase_deets_old(shoppers,brands):
    # we shall get deets: total_num_transactions, start_row, end_row, brands as a list
    # add more deets if you need more
    user_deets = []
    end_row = 0
    total_transactions = len(shoppers)
    while end_row < total_transactions:
        num_transactions = 0
        transactions_list = []
        start_row = end_row
        start_user = shoppers[start_row]
        while(end_row < total_transactions and shoppers[end_row] == start_user):
            num_transactions+=1
            transactions_list.append(brands[end_row])
            end_row+=1
        Quicksort.quicksort(transactions_list)
        user_deets.append((start_user,num_transactions,start_row,end_row,transactions_list))
    return user_deets

if __name__== "__main__":
    customers = [0,0,1,1,1,2,2,2,2]
    purchases = [0,3,5,1,2,4,1,3,5]
    print(get_user_purchase_deets(customers, purchases))

ext: py

reason: import numpy


text:
from abc import ABC, abstractmethod
import json
from enum import Enum

import torch
import numpy as np

from ptyolov3.models import Darknet
from ptyolov3.utils.utils import process_detections
from ptyolov3.utils.datasets import image_tensor_from_array

from .utils import draw_bounding_box

class PytorchObjectDetector:

    def __init__(self, model_path, config_path, labels_path, img_size):
        """
        :param model_path: location of serialised state_dict
        :param config_path: location of model config (darknet format)
        :param labels_path: location of category labels mapping
        :param img_size: image size used in input layer to model
        """
        self.model_path = model_path
        self.config_path = config_path
        self.labels_path = labels_path
        self.img_size = img_size
        self.model = self._load_model()

    def predict(self, image_array, conf_threshold, nms_threshold):
        """
        Run object detection inference on a a single image

        :param image_array: numpy array with shape [?,?,3]
        :param conf_threshold: object confidence probability (P(object), not P(class|object))
        :param nms_threshold: iou threshold used in non-maximum supression
        :returns: dict containing lists of bounding box co-ordinates (ymin, xmin, ymax, xmax),
            object_conf (object probabilities), scores (class probabilities) and class ids (integers)
        """
        image_tensor = image_tensor_from_array(image_array, self.img_size)

        with torch.no_grad():
            model_output = self.model(image_tensor)

        detections = process_detections(
            raw_detections=model_output,
            conf_threshold=conf_threshold,
            nms_threshold=nms_threshold,
            img_size=self.img_size,
            orig_img_shape=image_array.shape[:2]
        ).numpy()

        output_dict = {}
        output_dict['boxes'] = self.map_boxes(detections[:, :4])
        output_dict['object_conf'] = detections[:, 4]
        output_dict['scores'] = detections[:, 5]
        output_dict['category_ids'] = detections[:, 6].astype(np.uint8)

        return output_dict

    def _load_model(self):
        ''' Deserialise state_dict and insantiate model '''
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        model = Darknet(self.config_path, self.img_size)
        model.load_state_dict(torch.load(self.model_path, map_location=device))

        return model

    @staticmethod
    def map_boxes(boxes):
        ''' Convert list of box coords in PT format to dict '''
        return [dict(zip(['xmin', 'ymin', 'xmax', 'ymax'], box)) for box in boxes]

    def draw_bounding_boxes(self, image_array, boxes, object_conf, scores, category_ids, **draw_kwargs):
        """
        Draw all bounding boxes for a single image

        :param image_array: numpy array with shape [?,?,3]
        :param boxes: list of list of (ymin, xmin, ymax, xmax) box co-ordinates
        :param scores: list of class probabilities for boxes
        :param category_ids: list of class ids for boxes
        :returns: numpy array with all boxes annootated
        """

        for box, object_conf, score, category_id in zip(boxes, object_conf, scores, category_ids):

            label = f'{object_conf:.0%}'
            image_array = draw_bounding_box(image_array=image_array, label=label,
                                            **box, **draw_kwargs)

        return image_array

max_stars_repo_name: jarulsamy/mini-metro

max_issues_repo_name: jarulsamy/mini-metro

max_forks_repo_name: jarulsamy/mini-metro


text:
# -*- coding: utf-8 -*-
# mock pieces for testing
from __future__ import annotations

import random
from typing import Any

from pioreactor.config import config
from pioreactor.whoami import am_I_active_worker
from pioreactor.whoami import is_testing_env

class MockI2C:
    def __init__(self, SCL: int, SDA: int) -> None:
        pass

    def writeto(self, *args, **kwargs) -> None:
        return

    def try_lock(self, *args, **kwargs) -> bool:
        return True

    def unlock(self, *args, **kwargs) -> None:
        pass

    def __enter__(self) -> MockI2C:
        return self

    def __exit__(self, *args: Any) -> None:
        return

class MockAnalogIn:
    INIT_STATE = 0.01
    state = INIT_STATE
    _counter = 0.0

    def __init__(self, ads, channel, **kwargs) -> None:

        # import pandas as pd
        # self.source = pd.read_csv(f"/Users/camerondavidson-pilon/code/pioreactor/demo_od{channel}.csv", index_col=0)

        # subscribe to dosing events
        assert channel in [0, 1], "channel must be in 0, 1"
        self.max_gr = 0.25 + 0.1 * random.random()
        self.scale_factor = 0.00035 + 0.00005 * random.random()
        self.lag = 2 * 60 * 60 - 1 * 60 * 60 * random.random()

    def growth_rate(self, duration_as_seconds: int) -> float:
        import numpy as np

        return (
            self.max_gr
            / (1 + np.exp(-self.scale_factor * (duration_as_seconds - self.lag)))
            * (
                1
                - 1
                / (
                    1
                    + np.exp(
                        -self.scale_factor * 2 * (duration_as_seconds - 3 * self.lag)
                    )
                )
            )
        )

    @property
    def voltage(self) -> float:
        import random
        import numpy as np

        self.gr = self.growth_rate(
            self._counter / config.getfloat("od_config", "samples_per_second")
        )
        self.state *= np.exp(
            self.gr
            / 60
            / 60
            / config.getfloat("od_config", "samples_per_second")
            / 25  # divide by 25 from oversampling_count
        )
        self._counter += 1.0 / 25.0  # divide by 25 from oversampling_count
        return self.state + random.normalvariate(0, sigma=self.state * 0.001)

    @property
    def value(self) -> int:
        return round(self.voltage * 2**17)

class MockDAC43608:

    _DEVICE_CONFIG = 1
    _STATUS = 2
    _BRDCAST = 3
    A = 8
    B = 9
    C = 10
    D = 11
    E = 12
    F = 13
    G = 14
    H = 15

    def __init__(self, *args, **kwargs) -> None:
        pass

    def set_intensity_to(self, channel: str, intensity: float) -> None:
        assert 0 <= intensity <= 1, "intensity should be between 0 and 1"
        assert channel in list(range(8, 16)), "register should be in 8 to 15"
        # TODO: this should update MQTT too
        return

    def power_up(*args) -> None:
        pass

    def power_down(*args) -> None:
        pass

class MockTMP1075:
    def __init__(*args, address=0x4F) -> None:
        pass

    def get_temperature(self) -> float:
        import time, math, random

        return 3 * math.sin(0.1 * time.time() / 60) + 25 + 0.2 * random.random()

if am_I_active_worker() or is_testing_env():

    from rpi_hardware_pwm import HardwarePWM

    class MockHardwarePWM(HardwarePWM):
        def __init__(self, pwm_channel: int, hz: float) -> None:
            self.pwm_channel = pwm_channel
            self._hz = hz
            self.pwm_dir = ""

        def is_overlay_loaded(self) -> bool:
            return True

        def is_export_writable(self) -> bool:
            return True

        def does_pwmX_exists(self) -> bool:
            return True

        def echo(self, m: int, fil: str) -> None:
            pass

max_stars_repo_name: CamDavidsonPilon/morbidostat

max_issues_repo_name: CamDavidsonPilon/morbidostat

max_forks_repo_name: CamDavidsonPilon/morbidostat


text:
"""
Defines a Dataset object. Datasets contain one or more data arrays, QC info, etc.
"""

# Third-party
import numpy as np

class Dataset:
    """
    Base Dataset class
    """
    def __init__(self, data_type=None):
        self.data_type = data_type
        self.dates_with_files_not_loaded = set()
        self.files_not_loaded = set()
        self.dates_loaded = set()

class Observation(Dataset):
    """
    Observation Dataset
    """
    def __init__(self, obs=None):
        Dataset.__init__(self, data_type='observation')
        self.data_type = 'observation'
        self.obs = obs

class Forecast(Dataset):
    """
    Forecast Dataset
    """
    def __init__(self):
        Dataset.__init__(self, data_type='forecast')
        self.data_type = 'forecast'

class EnsembleForecast(Forecast):
    """
    Ensemble Forecast Dataset
    """
    def __init__(self, ens=None, ens_mean=None, ens_spread=None):
        Forecast.__init__(self)
        self.ens = ens
        self._ens_mean = ens_mean
        self._ens_spread = ens_spread

    def get_ens_mean(self):
        """
        Returns the ensemble mean

        Since ens_mean is defined as a property which calls this method, it won't take up memory
        by default

        ### Returns

        - array: ensemble mean
        """
        return np.nanmean(self.ens, axis=1) if self._ens_mean is None else self._ens_mean

    ens_mean = property(get_ens_mean)

    def get_ens_spread(self):
        """
        Returns the ensemble spread

        Since ens_spread is defined as a property which calls this method, it won't take up memory
        by default

        ### Returns

        - array: ensemble spread
        """
        return np.nanstd(self.ens, axis=1) if self._ens_spread is None else self._ens_spread

    ens_spread = property(get_ens_spread)

class DeterministicForecast(Forecast):
    """
    Ensemble Forecast Dataset
    """
    def __init__(self, fcst=None):
        Forecast.__init__(self)
        self.fcst = fcst

class Climatology(Dataset):
    """
    Climatology Dataset
    """
    def __init__(self, climo=None):
        Dataset.__init__(self, data_type='climatology')
        self.data_type = 'climatology'
        self.climo = climo


text:
"""
Code to make a nice plot of JdF Canyon bathymetry.

"""

# imports

#%% setup
import os, sys
sys.path.append(os.path.abspath('../alpha'))
import Lfun
import zrfun
Ldir = Lfun.Lstart()
import pfun
from mpl_toolkits.axes_grid1.inset_locator import inset_axes

import numpy as np
import netCDF4 as nc
import matplotlib.pyplot as plt
import pandas as pd

# output location (assume it exists)
out_dir = Ldir['parent'] + 'LiveOcean_NOTES/Figures Slides Movies/'

sys.path.append(os.path.abspath('../x_moor'))
import moor_lists

# get all mooring locations
moor_sta_dict, junk, junk, junk = moor_lists.get_sta_dict('shelf_moorings_mccabe')
moor_df = pd.DataFrame(index = moor_sta_dict.keys(), columns=['Longitude','Latitude'])
for sn in moor_sta_dict.keys():
    x, y = moor_sta_dict[sn]
    moor_df.loc[sn,'Longitude'] = x
    moor_df.loc[sn,'Latitude'] = y

# get map fields
dstr = '2019.07.04'
fn = Ldir['roms'] + 'output/cas6_v3_lo8b/f'+dstr+'/ocean_his_0020.nc'
G = zrfun.get_basic_info(fn, only_G = True)
lon = G['lon_psi']
lat = G['lat_psi']
z = -G['h']

zz = np.ma.masked_where(G['mask_rho']==False, z)

# plotting
plt.close('all')
fs=18
plt.rc('font', size=fs)

fig = plt.figure(figsize=(14,12))

ax = fig.add_subplot(121)
cs = ax.pcolormesh(lon, lat, zz[1:-1,1:-1], vmin=-3000, vmax = 100, cmap='terrain')
pfun.add_coast(ax)
pfun.dar(ax)
ax.axis([-130, -122, 42, 52])
ax.set_xticks([-128, -126, -124])
ax.set_yticks([44, 46, 48, 50])
# Inset colorbar
cbaxes = inset_axes(ax, width="40%", height="4%", loc='upper right')
cb = fig.colorbar(cs, cax=cbaxes, orientation='horizontal')
cb.set_ticks([-2000, -200])
for t in cb.ax.get_xticklabels():
     t.set_fontsize(.8*fs)
aa1 = [-125.5, -124.5, 47.5, 49]
pfun.draw_box(ax, aa1, linestyle='-', color='r', alpha=1, linewidth=3, inset=0)

ax = fig.add_subplot(122)
cs = ax.pcolormesh(lon, lat, zz[1:-1,1:-1], vmin=-500, vmax = 0, cmap='terrain')
pfun.add_coast(ax)
pfun.dar(ax)
ax.axis(aa1)
ax.set_xticks([-125.5, -125, -124.5])
ax.set_yticks([47.5, 48, 48.5, 49])
# Inset colorbar
cbaxes = inset_axes(ax, width="40%", height="4%", loc='upper right')
cb = fig.colorbar(cs, cax=cbaxes, orientation='horizontal')
cb.set_ticks([-500, -200, 0])
for t in cb.ax.get_xticklabels():
     t.set_fontsize(.8*fs)
cs = ax.contour(G['lon_rho'], G['lat_rho'], zz, [-300, -200, -100],
    colors='k', linestyles='-', linewidths=1)

moor_df.plot(x='Longitude', y='Latitude',marker='o',ls='',mec='k',
    c='orange', ms=10,ax=ax,legend=False)

plt.show()

plt.rcdefaults()

max_stars_repo_name: parkermac/LiveOcean

max_issues_repo_name: parkermac/LiveOcean

max_forks_repo_name: parkermac/LiveOcean


text:
import pysmurf
import numpy as np
import matplotlib.pyplot as plt

S = pysmurf.SmurfControl(make_logfile=False, setup=True)

S.set_noise_select(3, 1, write_log=True)
S.get_noise_select(3, write_log=True)

n_sample = 2**19
n = 4

dat = np.zeros((n, n_sample), dtype=complex)
for i in np.arange(n):
    dat[i] = S.read_adc_data(band, n_samples, hw_trigger=True)
    time.sleep(.5)

S.set_noise_select(3, 0, write_log=True)
S.get_noise_select(3, write_log=True)

fig, ax = plt.subplots(1)
end = 250
for i in np.arange(n):
    ax.plot(dat[n,:end])

plt.show()


text:
"""
@author: Peter Corke
"""

import numpy.testing as nt
import numpy as np
import roboticstoolbox as rp
import spatialmath as sm
import unittest

class TestRobot(unittest.TestCase):
    def test_init(self):
        l0 = rp.PrismaticDH()
        l1 = rp.RevoluteDH()

        with self.assertRaises(TypeError):
            rp.DHRobot([l0, l1], keywords=1)

        with self.assertRaises(TypeError):
            rp.Robot(l0)

        with self.assertRaises(TypeError):
            rp.Robot([l0, 1])

    def test_configurations_str(self):
        r = rp.models.DH.Puma560()
        r.configurations_str()

        r2 = rp.models.ETS.Frankie()
        r2.configurations_str()

    def test_dyntable(self):
        r = rp.models.DH.Puma560()
        r.dynamics()

    def test_linkcolormap(self):
        r = rp.models.DH.Puma560()
        r.linkcolormap()

        r.linkcolormap(["r", "r", "r", "r", "r", "r"])

    # def test_tool_error(self):
    #     r = rp.models.DH.Puma560()

    #     with self.assertRaises(ValueError):
    #         r.tool = 2

    def test_links(self):

        l0 = rp.PrismaticDH()
        l1 = rp.RevoluteDH()
        l2 = rp.PrismaticDH()
        l3 = rp.RevoluteDH()

        r0 = rp.DHRobot([l0, l1, l2, l3])

        self.assertIs(r0[0], l0)
        self.assertIs(r0[1], l1)
        self.assertIs(r0[2], l2)
        self.assertIs(r0[3], l3)

    def test_ikine_LM(self):
        panda = rp.models.DH.Panda()
        q = np.array([0, -0.3, 0, -2.2, 0, 2.0, np.pi / 4])
        T = panda.fkine(q)
        Tt = sm.SE3([T, T])

        qr = [0.0342, 1.6482, 0.0312, 1.2658, -0.0734, 0.4836, 0.7489]

        sol1 = panda.ikine_LM(T)
        sol2 = panda.ikine_LM(Tt)
        sol3 = panda.ikine_LM(T, q0=[0, 1.4, 0, 1, 0, 0.5, 1])

        self.assertTrue(sol1.success)
        self.assertAlmostEqual(np.linalg.norm(T - panda.fkine(sol1.q)), 0, places=4)

        self.assertTrue(sol2.success[0])
        self.assertAlmostEqual(
            np.linalg.norm(T - panda.fkine(sol2.q[0, :])), 0, places=4
        )
        self.assertTrue(sol2.success[0])
        self.assertAlmostEqual(
            np.linalg.norm(T - panda.fkine(sol2.q[1, :])), 0, places=4
        )

        self.assertTrue(sol3.success)
        self.assertAlmostEqual(np.linalg.norm(T - panda.fkine(sol3.q)), 0, places=4)

        with self.assertRaises(ValueError):
            panda.ikine_LM(T, q0=[1, 2])

    def test_ikine_LM_mask(self):

        # simple RR manipulator, solve with mask

        l0 = rp.RevoluteDH(a=2.0)
        l1 = rp.RevoluteDH(a=1)

        r = rp.DHRobot([l0, l1])  # RR manipulator
        T = sm.SE3(-1, 2, 0)
        sol = r.ikine_LM(T, mask=[1, 1, 0, 0, 0, 0])

        self.assertTrue(sol.success)
        self.assertAlmostEqual(
            np.linalg.norm(T.t[:2] - r.fkine(sol.q).t[:2]), 0, places=4
        )

        # test initial condition search, drop iteration limit so it has to do
        # some searching
        sol = r.ikine_LM(T, mask=[1, 1, 0, 0, 0, 0], ilimit=8, search=True)

        self.assertTrue(sol.success)
        self.assertAlmostEqual(
            np.linalg.norm(T.t[:2] - r.fkine(sol.q).t[:2]), 0, places=4
        )

    def test_ikine_LM_transpose(self):
        # need to test this on a robot with squarish Jacobian, choose Puma

        r = rp.models.DH.Puma560()
        T = r.fkine(r.qn)

        sol = r.ikine_LM(T, transpose=0.5, ilimit=1000, tol=1e-6)

        self.assertTrue(sol.success)
        self.assertAlmostEqual(np.linalg.norm(T - r.fkine(sol.q)), 0, places=4)

    def test_ikine_con(self):
        panda = rp.models.DH.Panda()
        q = np.array([0, -0.3, 0, -2.2, 0, 2.0, np.pi / 4])
        T = panda.fkine(q)
        Tt = sm.SE3([T, T, T])

        # qr = [7.69161412e-04, 9.01409257e-01, -1.46372859e-02,
        #       -6.98000000e-02, 1.38978915e-02, 9.62104811e-01,
        #       7.84926515e-01]

        sol1 = panda.ikine_min(T, qlim=True, q0=np.zeros(7))
        sol2 = panda.ikine_min(Tt, qlim=True)

        self.assertTrue(sol1.success)
        self.assertAlmostEqual(np.linalg.norm(T - panda.fkine(sol1.q)), 0, places=4)
        nt.assert_array_almost_equal(
            T.A - panda.fkine(sol1.q).A, np.zeros((4, 4)), decimal=4
        )

        self.assertTrue(sol2[0].success)
        nt.assert_array_almost_equal(
            T.A - panda.fkine(sol2[0].q).A, np.zeros((4, 4)), decimal=4
        )
        self.assertTrue(sol2[1].success)
        nt.assert_array_almost_equal(
            T.A - panda.fkine(sol2[1].q).A, np.zeros((4, 4)), decimal=4
        )

    def test_ikine_unc(self):
        puma = rp.models.DH.Puma560()
        q = puma.qn
        T = puma.fkine(q)
        Tt = sm.SE3([T, T])

        sol1 = puma.ikine_min(Tt)
        sol2 = puma.ikine_min(T)
        sol3 = puma.ikine_min(T)

        self.assertTrue(sol1[0].success)
        nt.assert_array_almost_equal(
            T.A - puma.fkine(sol1[0].q).A, np.zeros((4, 4)), decimal=4
        )
        self.assertTrue(sol1[1].success)
        nt.assert_array_almost_equal(
            T.A - puma.fkine(sol1[1].q).A, np.zeros((4, 4)), decimal=4
        )

        self.assertTrue(sol2.success)
        nt.assert_array_almost_equal(
            T.A - puma.fkine(sol2.q).A, np.zeros((4, 4)), decimal=4
        )

        self.assertTrue(sol3.success)
        nt.assert_array_almost_equal(
            T.A - puma.fkine(sol3.q).A, np.zeros((4, 4)), decimal=4
        )

    # def test_plot_swift(self):
    #     r = rp.models.Panda()

    #     env = r.plot(r.q, block=False)
    #     env.close()

if __name__ == "__main__":  # pragma nocover
    unittest.main()
    # pytest.main(['tests/test_SerialLink.py'])

    # import roboticstoolbox as rtb
    # from spatialmath import SE3

    # l0 = rtb.RevoluteDH(a=2.0)
    # l1 = rtb.RevoluteDH(a=1)
    # # l1 = rp.PrismaticDH(theta=1.0)
    # # l2 = rp.PrismaticDH(theta=1, qlim=[0, 2])

    # # r0 = rtb.DHRobot([l0, l1])  # RP manipulator
    # r1 = rtb.DHRobot([l0, l1])  # RR manipulator
    # print(r1)
    # T = SE3(-1, 2, 0)
    # print(T)
    # sol = r1.ikine_LM(T, mask=[1, 1, 0, 0, 0, 0])
    # print(sol)
    # print(r1.fkine(sol.q))

    # # sol = r1.ikine_LM(T, transpose=0.05, tol=1e-2, ilimit=1000, mask=[1, 1, 0, 0, 0, 0])
    # # print(sol)
    # # print(r1.fkine(sol.q))

    # sol = r1.ikine_LM(
    #     T, mask=[1, 1, 0, 0, 0, 0],
    #     ilimit=8, search=True)
    # print(sol)

    # r = rtb.models.DH.Puma560()

    # T = r.fkine(r.qn)

    # sol = r.ikine_LM(T, transpose=0.5, ilimit=1000, tol=1e-6)
    # print(sol)


text:
import azure
import pickle
import os.path as op
import numpy as np
import pandas as pd
from glob import glob
from azure.cognitiveservices.vision.face.models._models_py3 import Coordinate

def load_landmarks(stim, api='google', frames_dir='../FEED_stimulus_frames'):

    stims = sorted(glob(f'{frames_dir}/{stim}/*/texmap/frame*.png'))

    # number of landmarks: 27 (azure), 34 (google)
    n_lm = 27 if api == 'azure' else 34
    df = pd.DataFrame(columns=['frame', 'landmark', 'coords'], index=range(n_lm * 30))

    row = 0
    for i in range(30):  # loop across frames
        info = stims[i].replace('.png', f'_api-{api}_annotations.pkl')
        with open(info, 'rb') as f_in:
            info = pickle.load(f_in)

        if api == 'azure':
            info = info[0].face_landmarks
            ii = 0
            for attr in dir(info):
                this_attr = getattr(info, attr)
                if isinstance(this_attr, Coordinate):
                    df.loc[row, 'frame'] = i
                    df.loc[row, 'landmark'] = ii
                    df.loc[row, 'coords'] = (this_attr.x, this_attr.y)
                    ii += 1
                    row += 1

        elif api == 'google':
            info = info.face_annotations[0]
            for ii in range(len(info.landmarks)):
                df.loc[row, 'frame'] = i
                df.loc[row, 'landmark'] = ii
                df.loc[row, 'coords'] = (
                    np.round(info.landmarks[ii].position.x, 1),
                    np.round(info.landmarks[ii].position.y, 1)
                )
                row += 1
        else:
            raise ValueError("Choose api from 'google' and 'azure'.")

    df['stim'] = stim
    df['api'] = api
    df = df.loc[:, ['stim', 'frame', 'landmark', 'coords', 'api']]
    return df

if __name__ == '__main__':

    from tqdm import tqdm
    from joblib import Parallel, delayed

    stims = [op.basename(d) for d in sorted(glob('../FEED_stimulus_frames/id*'))]
    dfs = Parallel(n_jobs=15)(delayed(load_landmarks)(s, api)
        for s in tqdm(stims) for api in ['azure', 'google']
    )
    df = pd.concat(dfs, axis=0).set_index('stim')
    df.to_csv('data/landmarks.tsv', sep='\t')


text:
__author__ = "Mihaela Rosca"
__contact__ = "mihaela.c.rosca@gmail.com"

import heapq
import matplotlib.pyplot as plt
import numpy
import os
import scipy
import scipy.linalg

from os.path import isfile, join
from scipy import misc

# Import all common functions
from common import *

# The directory path to the images
PICTURE_PATH = "/pics/cambrdige_pics/"

# The current directory where the script is ran
currentDir = os.path.dirname(os.path.abspath(__file__))

"""
  Converts the data to zero mean data.
"""
def convertDataToZeroMean(data):
  means = scipy.mean(data, axis=0)
  rows, cols = data.shape
  zeroMean = numpy.zeros((rows, cols))
  for i in xrange(rows):
    zeroMean[i] = data[i] - means

  assert zeroMean.shape == data.shape

  return zeroMean

"""
  Uses a heuristic to evaluate how many dimensions should the data be reduced
    to.

Arguments:
  eigenValues:
    The eigen values of the covariance matrix, or numbers proportional to them.
    Should be a numpy 1-D array.
Returns:
  The dimension the data should be reduced to.
"""
def dimensionFromEigenIndividualVariance(eigenValues):
  threshold = 0.01
  dimension = 0

  s = numpy.sum(eigenValues)
  print "sum eigen" + str(s)

  for eigen in eigenValues:
    r = eigen / s
    if r > threshold:
      dimension += 1

  return dimension

# requires the eigen values to be sorted before
def dimensionFromEigenTotalVariance(eigenValues):
  threshold = 0.95
  dimension = 0

  s = numpy.sum(eigenValues)
  print "sum eigen" + str(s)
  current = 0
  for eigen in eigenValues:
    r = (eigen / s)
    current += r
    if current >= threshold:
      break
    dimension += 1

  return dimension

"""
This method uses the  Karhunen Lowe transform to fastly compute the
eigen vaues of the data.

It is faster than the SVD method below, but can be prone to floating point
errors more than the SVD one.
Arguments:
  train:
    Numpy array of arrays
  dimension: the dimension to which to reduce the size of the data set.

Returns:
  The principal components of the data.
"""
# Returns the principal components of the given training
# data by commputing the principal eigen vectors of the
# covariance matrix of the data
def pca(train, dimension):
  # Use the Karhunen Lowe transform to fastly compute
  # the principal components.
  rows, cols = train.shape
  # Step1: Get the mean of each column of the data
  # Ie create the average image
  u = convertDataToZeroMean(train)

  # Step2: Compute the eigen values of the U * U^T matrix
  # the size of U * U^T is rows * rows (ie the number of data points you have
  # in your training)
  eigVals, eigVecs = scipy.linalg.eig(u.dot(u.T))

  # Step3: Compute the eigen values of U^T*U from the eigen values of U * U^T
  bigEigVecs = numpy.zeros((rows, cols))
  for i in xrange(rows):
    bigEigVecs[i] = u.T.dot(eigVecs[:, i])

  # Step 4: Normalize the eigen vectors to get orthonormal components
  bigEigVecs = map(lambda x: x / scipy.linalg.norm(x), bigEigVecs)

  eigValsBigVecs = zip(eigVals, bigEigVecs)
  sortedEigValsBigVecs = sorted(eigValsBigVecs, key=lambda x : x[0], reverse=True)

  index = 0
  if dimension == None:
    # Get the eigen values
    # Note that these are not the eigen values of the covariance matrix
    # but the eigen values of U * U ^T
    # however, this is fine because they just differ by a factor
    # so the ratio between eigen values will be preserved
    eigenValues = map(lambda x : x[0], sortedEigValsBigVecs)
    dimension = dimensionFromEigenTotalVariance(eigenValues)
    print "Using PCA dimension " + str(dimension)

  result = np.empty(rows, dimension)
  for eigVal, vector in sortedEigValsBigVecs:
    if index >= dimension:
      break

    if eigVal <=0:
      print "Warning: Non-positive eigen value"

    result[:, index] = vector
    index = index + 1

  return result

"""
Arguments:
  train:
    Numpy array of arrays
  dimension: the dimension to which to reduce the size of the data set.

Returns:
  The principal components of the data.

  This method should be preferred over the above: it is well known that the
  SVD methods are more stable than the ones that require the computation of
  the eigen values and eigen vectors.
  For more detail see:
  http://math.stackexchange.com/questions/3869/what-is-the-intuitive-relationship-between-svd-and-pca
"""
def pcaWithSVD(train, dimension=None):
  zeroMean = convertDataToZeroMean(train)

  # SVD guaranteed that the singular values are in non-increasing order
  # this means that the u's are already ordered as required, according
  # to the magnitute of the eigen values
  u, s, vh = scipy.linalg.svd(zeroMean)

  if dimension == None:
    # Get the eigen values from the singular values
    eigenValues = s ** 2;
    dimension = dimensionFromEigenTotalVariance(eigenValues)
    print "Using PCA dimension " + str(dimension)

  return vh[0:dimension-1]

"""
Arguments:
  pcaMethod: a method to use for PCA.
  images: A python list of images that have to be of the same size.
  dimension: the dimension to which to reduce the size of the data set.
Returns:
  A tuple:
    The first element of the tuple is formed from the eigen faces of given
      images.
    The second element of the tuple if formed from the vector version of the
      eigen faces. This is kept for optimization reasons.
"""
def getEigenFaces(pcaMethod, images, dimension=None):

  imgSize = images[0].shape;
  # this call should not be here: the code should assume that the images have
  # been transofrmed to vectors before
  imgs = imagesToVectors(images)

  vectors = pcaMethod(imgs, dimension)
  eigenFaces = map(lambda x: vectorToImage(x, imgSize), vectors)

  return (eigenFaces, vectors)

def reduce(principalComponents, vectors):
  assert len(principalComponents) > 0

  print principalComponents[0].shape

  principalComponents = np.array(principalComponents)

  lowDimRepresentation  = np.dot(vectors, principalComponents.T)
  # lowDimRepresentation = map(lambda x : vectors.dot(x), principalComponents)
  # sameDimRepresentation = \
  #   sum([ x * y for x, y in zip(principalComponents, lowDimRepresentation)])
  # TODO: do this with einsum
  sameDimRepresentation = lowDimRepresentation[:, np.newaxis] * principalComponents.T
  sameDimRepresentation = sameDimRepresentation.sum(axis=2)
  # TODO: create the proper thing here so that you can
  # easily see what the ouput is
  return  (lowDimRepresentation, sameDimRepresentation)

"""
 Reduces a 2D image represented by a numpy 2D array of integer values(pixels)
 to a lower dimension, dictated by the number of principal components.
"""
def reduceImageToLowerDimensions(principalComponents, image2D):
  assert len(principalComponents) > 0

  size = principalComponents[0].shape
  vector = vectorToImage(image2D, size)

  lowDimRepresentation = map(lambda x : x.T.dot(vector), principalComponents)
  sameDimRepresentation = \
    sum([ x * y for x, y in zip(principalComponents, lowDimRepresentation)])
  return  (lowDimRepresentation, sameDimRepresentation)

def main():
  # Load all the image files in the current directory
  picFiles = []
  path = currentDir + PICTURE_PATH
  for root, dirs, files in os.walk(path):
    if root != path:
      picFiles += map(lambda x: os.path.join(root, x), files)

  print len(picFiles)

  imgs = map(lambda x: misc.imread(x, flatten=True), picFiles)

  eigenFaces, principalComponents = getEigenFaces(pca, imgs)
  # plt.imshow(eigenFaces[0], cmap=plt.cm.gray)
  # plt.show()

  lowDimRepresentation, sameDimRepresentation = \
      reduceImageToLowerDimensions(principalComponents, imgs[0])

  plt.imshow(imgs[0], cmap=plt.cm.gray)
  plt.show()

  image2D = vectorToImage(sameDimRepresentation, imgs[0].shape)
  plt.imshow(image2D, cmap=plt.cm.gray)
  plt.show()
  print "done"

if __name__ == '__main__':
  main()

reason: import numpy,import scipy,from scipy


text:
from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple, Union
from copy import deepcopy
from pathlib import Path
from functools import partial

import numpy as np
import torch as th
import torch.nn.functional as F

from stable_baselines3 import SAC, TD3, DQN
from stable_baselines3.common.off_policy_algorithm import OffPolicyAlgorithm
from stable_baselines3.common.on_policy_algorithm import OnPolicyAlgorithm
from stable_baselines3.common.buffers import DictReplayBuffer, ReplayBuffer, RolloutBuffer, DictRolloutBuffer
from stable_baselines3.common.type_aliases import (
    DictReplayBufferSamples,
    DictRolloutBufferSamples,
    ReplayBufferSamples,
    RolloutBufferSamples,
)
try:
    from sb3_contrib import TQC, QRDQN
except ImportError:
    TQC, QRDN = None, None

from sbil.demo.buffer import double_buffer
from sbil.demo.utils import get_demo_buffer, state_action, all_state_action
from sbil.utils import set_method, set_restore, train_off

class RCESamples(NamedTuple):
    observations: Union[Dict[str, th.Tensor], th.Tensor]
    actions: th.Tensor
    next_observations: Union[Dict[str, th.Tensor], th.Tensor]
    dones: th.Tensor
    n_step: th.Tensor
    futur_obs: Union[Dict[str, th.Tensor], th.Tensor]
    demo_obs: Union[Dict[str, th.Tensor], th.Tensor]
    demo_act: th.Tensor

def critic_loss(self, data, use_behaviour_policy):
    r = data['replay']
    # get actions
    if hasattr(self.actor, "action_log_prob"): # stochastic policy
        demo_act = self.actor.action_log_prob(r.demo_obs)[0] if use_behaviour_policy else r.demo_act
        next_actions, next_log_prob = self.actor.action_log_prob(r.next_observations)
        futur_actions, futur_log_prob = self.actor.action_log_prob(r.futur_obs)
    else: # determinitic policy
        def get_action(obs):
            act = self.actor_target(obs)
            noise = act.clone().data.normal_(0, self.target_policy_noise)
            noise = noise.clamp(-self.target_noise_clip, self.target_noise_clip)
            return (act + noise).clamp(-1, 1)
        demo_act = get_action(r.demo_obs) if use_behaviour_policy else r.demo_act
        next_actions = get_action(r.next_observations)
        futur_actions = get_action(r.futur_obs)

    # concatenate next and futur for efficency
    if isinstance(r.observations, dict):
        nf_obs = {k: th.cat((v, r.futur_obs[k])) for k,v in r.next_observations.items()}
    else:
        nf_obs = th.vstack((r.next_observations, r.futur_obs))
    nf_act = th.vstack((next_actions, futur_actions))
    nf_dones = th.cat((r.dones, r.dones))
    nf_q = self.critic_target(nf_obs, nf_act)

    if isinstance(self, (TD3, SAC)):
        with th.no_grad():
            nf_q, _ = th.min(th.sigmoid(th.cat(nf_q, dim=1)), dim=1, keepdim=True)
            nf_w = (1 - nf_dones) * nf_q
            nf_w /= (1.000001-nf_w)
            nf_γw = self.gamma**th.cat((th.ones_like(r.n_step), r.n_step)).view(-1,1) * nf_w
            # split
            next_γw, futur_γw = th.tensor_split(nf_γw, 2)
            w, futur_w = th.tensor_split(nf_w, 2)
            # The prediction is the mean
            y = (next_γw/(next_γw+1) + futur_γw/(futur_γw+1)) / 2

        value = th.cat(self.critic(r.observations, r.actions), dim=1)
        demo_value = th.cat(self.critic(r.demo_obs, demo_act), dim=1)

        # reshape to do pairwise operations (double sum)
        input = th.cat((demo_value, value), dim=0)
        target = th.cat((th.ones_like(demo_value), y.repeat(1,2)), dim=0)

    elif TQC is not None and isinstance(self, TQC):
        n_target_quantiles = self.critic.quantiles_total - self.top_quantiles_to_drop_per_net * self.critic.n_critics
        with th.no_grad():
            nf_q = th.sigmoid(nf_q)
            nf_q, _ = th.sort(nf_q.reshape(2*data['batch_size'], -1)) # sort
            nf_q = nf_q[:, :n_target_quantiles] # drop
            nf_w = ((1 - nf_dones) * nf_q)[:, None, None, :] # discard transitions with done
            nf_w = nf_w / (1.000001-nf_w) # classifier’s prediction (ratio) at the next time step and futur
            nf_γ = self.gamma**th.vstack((th.ones_like(r.n_step), r.n_step))
            nf_γw = nf_γ.view(2*data['batch_size'], 1, 1, 1) * nf_w
            # split
            next_γw, futur_γw = th.tensor_split(nf_γw, 2)
            w, futur_w = th.tensor_split(nf_w, 2)
            # The prediction is the mean
            y = (next_γw/(next_γw+1) + futur_γw/(futur_γw+1)) / 2

        current_quantiles = self.critic(r.observations, r.actions)
        demo_quantiles = self.critic(r.demo_obs, demo_act)

        # reshape to do pairwise operations (double sum)
        input = th.cat((demo_quantiles, current_quantiles), dim=0).unsqueeze(-1).repeat(1, 1, 1, n_target_quantiles)
        target = th.cat((th.ones_like(y), y), dim=0).repeat(1, self.critic.n_critics, self.critic.n_quantiles, 1)

    else:
        raise NotImplementedError(f"critic_loss not implemented for {self}.")

    weight = th.cat((th.ones_like(w)*(1-self.gamma), 1+self.gamma*w), dim=0)
    critic_loss = F.binary_cross_entropy_with_logits(input=input, target=target, weight=weight)

    data['critic_loss'] = critic_loss
    return {'critic_loss': critic_loss.item()}

def sample(self, batch_size, env, *args, demo_buffer, n_step, super_=None, **kwargs) -> Union[RCESamples]:
    if self.full:
        batch_inds = (np.random.randint(0, self.buffer_size, size=batch_size) + self.pos) % self.buffer_size
    else:
        batch_inds = np.random.randint(0, max(self.pos-n_step, 1), size=batch_size)

    # get next_obs
    if isinstance(self.observations, dict):
        get_obs = lambda i: self._normalize_obs({key: obs[i, 0, :] for key, obs in self.observations.items()}, env)
        is_dict = True
    else:
        get_obs = lambda i: self._normalize_obs(self.observations[i, 0, :], env)
        is_dict = False

    next_batch_inds = (batch_inds + 1) % self.buffer_size # batch indexe

    # fetch the greatest n_step value without reaching done
    n_step_, ok = np.ones(batch_size, dtype=int), np.ones(batch_size, dtype=bool)
    for i in range(1, n_step):
        ok = np.logical_and(np.logical_not(self.dones[(next_batch_inds + i) % self.buffer_size, 0]), ok)
        n_step_ += ok

    # get futur
    futur_batch_inds = (batch_inds + n_step_) % self.buffer_size

    # sample demo
    demo_sample = demo_buffer.sample(batch_size=batch_size, env=env)

    data = (
        get_obs(batch_inds),
        self.actions[batch_inds, :] if is_dict else self.actions[batch_inds, 0, :],
        get_obs(next_batch_inds),
        self.dones[batch_inds] * (1 - self.timeouts[batch_inds]),
        n_step_,
        get_obs(futur_batch_inds),
        demo_sample.observations,
        demo_sample.actions,
    )
    def to_torch(x):
        if isinstance(x, dict):
            return {k:self.to_torch(v) for k,v in x.items()}
        return self.to_torch(x)

    return RCESamples(*tuple(map(to_torch, data)))

def rce(
    learner: OffPolicyAlgorithm,
    demo_buffer: Union[DictReplayBuffer, ReplayBuffer, str, Path],
    n_step=10,
    use_behaviour_policy: bool = False,
) -> OffPolicyAlgorithm:
    """
    Recursive classification of examples: https://arxiv.org/abs/2103.12656
    We suppose observations are stored sequentially: o_0, o_1, o_2, ...
    so we can access to futur observations for n_step

    :param learner: Stable baselines learner object
    :param demo_buffer: Demonstration replay buffer containing success examples (not trajectories).
    :param n_step: Number of futur step for the n_step return.
    :param use_behaviour_policy: Choose to use the current policy to estimate the action
        of the demonstration. Defaut to False.
    :return learner: Decorated learner
    """
    is_dqn = hasattr(learner, "q_net") or hasattr(learner, "quantile_net")
    assert not is_dqn, "DQN like learner is not supported."

    demo_buffer_ = get_demo_buffer(demo_buffer, learner)
    # assert demo_buffer_.optimize_memory_usage, "optimize_memory_usage must be set to True in order to use n_step."
    set_restore(learner)

    set_method(
        learner,
        old="train",
        new=train_off,
        critic_loss=partial(critic_loss, use_behaviour_policy=use_behaviour_policy),
    )

    set_method(
        learner.replay_buffer,
        old="sample",
        new=sample,
        demo_buffer=demo_buffer_,
        n_step=n_step,
    )

    return learner

max_stars_repo_name: Yakumoo/sbil

max_issues_repo_name: Yakumoo/sbil

max_forks_repo_name: Yakumoo/sbil


text:
import os
import time
import copy
import torch
import numpy as np
from tqdm import tqdm
import torch.nn as nn
from apex import amp

NUM_CLASSES = int(os.environ["NUM_CLASSES"])

def fbeta_score(y_pred, y_true, thresh, device, beta=2, eps=1e-9, sigmoid=True):
    "Computes the f_beta between `preds` and `targets`"
    y_pred = torch.from_numpy(y_pred).float().to(device)
    y_true = torch.from_numpy(y_true).float().to(device)
    beta2 = beta ** 2
    if sigmoid: y_pred = y_pred.sigmoid()
    y_pred = (y_pred > thresh).float()
    y_true = y_true.float()
    TP = (y_pred * y_true).sum(dim=1)
    prec = TP / (y_pred.sum(dim=1) + eps)
    rec = TP / (y_true.sum(dim=1) + eps)
    res = (prec * rec) / (prec * beta2 + rec + eps) * (1 + beta2)
    return res.mean()

def find_best_fixed_threshold(preds, targs, device):
    score = []
    thrs = np.arange(0, 0.5, 0.01)
    for thr in tqdm(thrs):
        score.append(fbeta_score(preds, targs, thresh=thr, device=device))
    score = np.array(score)
    pm = score.argmax()
    best_thr, best_score = thrs[pm], score[pm].item()
    print('thr={} F2={}'.format(best_thr, best_score))
    return best_thr, best_score

def train_model(model, data_loader, dataset_sizes, device, optimizer, scheduler, num_epochs, fold_name, use_amp=True):
    since = time.time()
    criterion = nn.BCEWithLogitsLoss()
    best_model_wts = copy.deepcopy(model.state_dict())
    best_loss = 99999999
    all_scores = []
    best_score = -np.inf
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        valid_preds = np.zeros((dataset_sizes["val"], NUM_CLASSES))
        valid_labels = np.zeros((dataset_sizes["val"], NUM_CLASSES))
        val_bs = data_loader["val"].batch_size
        for phase in ['train', 'val']:
            if phase == 'train':
                scheduler.step(best_score)
                model.train()  # Set model to training mode
            else:
                model.eval()  # Set model to evaluate mode

            running_loss = 0.0

            # Iterate over data.
            tk0 = tqdm(data_loader[phase], total=int(dataset_sizes[phase] / data_loader[phase].batch_size))
            counter = 0
            for bi, d in enumerate(tk0):
                inputs = d["image"]
                labels = d["labels"]
                inputs = inputs.to(device, dtype=torch.float)
                labels = labels.to(device, dtype=torch.float)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        if use_amp is True:
                            with amp.scale_loss(loss, optimizer) as scaled_loss:
                                scaled_loss.backward()
                        else:
                            loss.backward()
                        optimizer.step()

                running_loss += loss.item() * inputs.size(0)
                counter += 1
                tk0.set_postfix(loss=(running_loss / (counter * data_loader[phase].batch_size)))

                if phase == "val":
                    valid_labels[bi * val_bs:(bi + 1) * val_bs, :] = labels.detach().cpu().squeeze().numpy()
                    valid_preds[bi * val_bs:(bi + 1) * val_bs, :] = outputs.detach().cpu().squeeze().numpy()

            epoch_loss = running_loss / dataset_sizes[phase]
            if phase == "val":
                best_thr, score = find_best_fixed_threshold(valid_preds, valid_labels, device)
                all_scores.append(score)
                if score > best_score:
                    best_score = score
                    best_model_wts = copy.deepcopy(model.state_dict())
                    torch.save(model.state_dict(), os.path.join(fold_name, "model.bin"))

            print('{} Loss: {:.4f}'.format(phase, epoch_loss))

        if len(all_scores[-5:]) == 5:
            if best_score not in all_scores[-5:]:
                break
            if len(np.unique(all_scores)) == 1:
                break
            if abs(min(all_scores[-5:]) - max(all_scores[-5:])) < 0.001:
                break
        print(all_scores[-5:])

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val loss: {:4f}'.format(best_loss))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

max_stars_repo_name: abhishekkrthakur/imet-collection

max_issues_repo_name: abhishekkrthakur/imet-collection

max_forks_repo_name: abhishekkrthakur/imet-collection


text:
#  Copyright 2020 Forschungszentrum Jülich GmbH and Aix-Marseille Université
# "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements;
# and to You under the Apache License, Version 2.0. "
import numpy
import sys
import os
import pickle
import base64

from common.utils.security_utils import check_integrity
from actions_adapters.parameters import Parameters
import cosim_example_demos.TVB_NEST_demo.tvb_sim.wrapper_TVB_mpi as Wrapper
from EBRAINS_RichEndpoint.Application_Companion.common_enums import SteeringCommands
from EBRAINS_RichEndpoint.Application_Companion.common_enums import INTEGRATED_SIMULATOR_APPLICATION as SIMULATOR
from EBRAINS_ConfigManager.global_configurations_manager.xml_parsers.default_directories_enum import DefaultDirectories
from EBRAINS_ConfigManager.global_configurations_manager.xml_parsers.configurations_manager import ConfigurationsManager
from EBRAINS_ConfigManager.workflow_configuraitons_manager.xml_parsers.xml2class_parser import Xml2ClassParser

import tvb.simulator.lab as lab
import matplotlib.pyplot as plt
from tvb.contrib.cosimulation.cosimulator import CoSimulator
from tvb.contrib.cosimulation.cosim_monitors import CosimCoupling

numpy.random.seed(125)

class TVBAdapter:

    def __init__(self, p_configurations_manager=None, p_log_settings=None, p_sci_params_xml_path_filename=None):
        self.__simulator = None
        self._log_settings = p_log_settings
        self._configurations_manager = p_configurations_manager
        self.__logger = self._configurations_manager.load_log_configurations(
            name="TVB_Adapter",
            log_configurations=self._log_settings,
            target_directory=DefaultDirectories.SIMULATION_RESULTS)
        self.__path_to_parameters_file = self._configurations_manager.get_directory(
            directory=DefaultDirectories.SIMULATION_RESULTS)

        # Loading scientific parameters into an object
        self.__sci_params = Xml2ClassParser(p_sci_params_xml_path_filename, self.__logger)

        self.__parameters = Parameters(self.__path_to_parameters_file)
        print(f'__ric__,self.__path_to_parameters_file={self.__path_to_parameters_file}')
        self.__logger.info("initialized")

    #
    # def __configure(self, time_synch=0.1, id_nest_region=None, dt=0.1):
    def __configure(self):
        """
        configure TVB before the simulation
        modify example of https://github.com/the-virtual-brain/tvb-root/blob/master/tvb_documentation/tutorials/tutorial_s1_region_simulation.ipynb
        based on: https://github.com/multiscale-cosim/TVB-NEST-demo/blob/main/nest_elephant_tvb/launcher/run.py
        :param: no parameters required
        :return: simulator
        """
        # :param time_synch: time of synchronization between simulator
        # :param id_nest_region: id of the region simulated with NEST
        # :param dt: size of the integration step
        # :return:
        # """
        oscillator = lab.models.Generic2dOscillator()
        #
        white_matter = lab.connectivity.Connectivity.from_file()
        # white_matter.speed = numpy.array([4.0])
        white_matter.speed = self.__sci_params.white_matter_speed
        #
        # white_matter_coupling = lab.coupling.Linear(a=numpy.array([0.154]))
        white_matter_coupling = lab.coupling.Linear(a=self.__sci_params.lab_coupling_linear_a)
        #
        # heunint = lab.integrators.HeunDeterministic(dt=dt)
        heunint = lab.integrators.HeunDeterministic(dt=self.__sci_params.heun_deterministic_dt)
        #
        what_to_watch = (lab.monitors.Raw(),)
        # special monitor for MPI
        simulator = CoSimulator(
            voi=numpy.array([0]),  # coupling with Excitatory firing rate
            synchronization_time=self.__sci_params.synchronization_time,  #  synchronization_time=time_synch,  # time of synchronization time between simulators
            # monitor for the coupling between simulators
            cosim_monitors=(CosimCoupling(coupling=white_matter_coupling),),
            proxy_inds=self.__sci_params.proxy_inds,   #  proxy_inds=numpy.array(id_nest_region, dtype=int),  # id of the proxy node
            model=oscillator,
            connectivity=white_matter,
            coupling=white_matter_coupling,
            integrator=heunint,
            monitors=what_to_watch
        )
        simulator.configure()
        self.__logger.info(f'TVB simulator has been configured...')
        return simulator

    def execute_init_command(self):
        self.__logger.debug("executing INIT command")
        # self.__simulator = self.__configure(
        #     self.__parameters.time_synch,
        #     self.__parameters.id_nest_region,
        #     self.__parameters.resolution)
        self.__simulator = self.__configure()
        self.__simulator.simulation_length = self.__parameters.simulation_time
        self.__logger.debug("INIT command is executed")
        return self.__parameters.time_synch  # minimum step size for simulation

    def execute_start_command(self):
        self.__logger.debug("executing START command")
        (r_raw_results,) = Wrapper.run_mpi(
            self.__simulator,
            self.__parameters.path,
            self.__logger)
        self.__logger.debug('TVB simulation is finished')
        return r_raw_results

    def execute_end_command(self, p_raw_results=None):
        self.__logger.info("plotting the result")
        plt.figure(1)
        plt.plot(p_raw_results[0], raw_results[1][:, 0, :, 0] + 3.0)
        plt.title("Raw -- State variable 0")
        plt.savefig(self.__parameters.path + "/figures/plot_tvb.png")

        self.__logger.debug("post processing is done")

if __name__ == "__main__":
    # unpickle configurations_manager object
    configurations_manager = pickle.loads(base64.b64decode(sys.argv[2]))
    # unpickle log_settings
    log_settings = pickle.loads(base64.b64decode(sys.argv[3]))
    # security check of pickled objects
    # it raises an exception, if the integrity is compromised
    check_integrity(configurations_manager, ConfigurationsManager)
    check_integrity(log_settings, dict)
    # everything is fine, run simulation
    tvb_adapter = TVBAdapter(p_configurations_manager=configurations_manager,
                             p_log_settings=log_settings,
                             p_sci_params_xml_path_filename=sys.argv[4])
    local_minimum_step_size = tvb_adapter.execute_init_command()
    # send local minimum step size to Application Manager as a response to INIT
    # NOTE Application Manager expects a string in the following format:
    # {'PID': <int>, 'LOCAL_MINIMUM_STEP_SIZE': <float>}
    pid_and_local_minimum_step_size = \
        {SIMULATOR.PID.name: os.getpid(),
         SIMULATOR.LOCAL_MINIMUM_STEP_SIZE.name: local_minimum_step_size}

    # Application Manager will read the stdout stream via PIPE
    # NOTE the communication with Application Manager via PIPES will be
    # changed to some other mechanism
    print(f'{pid_and_local_minimum_step_size}')
    user_action_command = input()
    # execute if steering command is START
    if SteeringCommands[user_action_command] == SteeringCommands.START:
        raw_results = tvb_adapter.execute_start_command()
        tvb_adapter.execute_end_command(raw_results)
        sys.exit(0)
    else:
        # TODO raise and log the exception with traceback and terminate with
        # error if received an unknown steering command
        print(f'unknown steering command: '
              f'{SteeringCommands[user_action_command]}',
              file=sys.stderr)
        sys.exit(1)


text:
#!/usr/bin/env python3

import argparse
from secrets import randbelow
from time import time
from typing import Tuple

import cv2
import numpy as np
from screeninfo import Monitor, get_monitors

from util import (
    CameraModel,
    CameraParams,
    Hyperion,
    VideoCapture,
    arg_non_neg_int,
    arg_pos_float,
    arg_pos_int,
    arg_scale_float,
    imshow,
    show_preview,
)

def parse_args():
    parser = argparse.ArgumentParser(
        description="Measure intrinsic and distortion coefficients for a camera.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        "-br",
        "--board-rows",
        type=arg_pos_int,
        default=9,
        help="Number of calibration board rows",
    )
    parser.add_argument(
        "-bc",
        "--board-cols",
        type=arg_pos_int,
        default=6,
        help="Number of calibration board columns",
    )
    parser.add_argument(
        "-bf",
        "--board-frames",
        type=arg_pos_int,
        default=20,
        help="How many frames to use for the calibration",
    )
    parser.add_argument(
        "-sm",
        "--screen-monitor",
        type=arg_non_neg_int,
        default=0,
        help="Which monitor number to use, for multi-monitor setups",
    )
    parser.add_argument(
        "-sb",
        "--screen-board-scale",
        type=arg_scale_float,
        default=0.5,
        help="Scale of the calibration board relative to screen size",
    )
    parser.add_argument(
        "-sp",
        "--screen-preview-scale",
        type=arg_scale_float,
        default=0.5,
        help="Scale of the camera preview window",
    )
    parser.add_argument(
        "-fd",
        "--frame-delay-ms",
        type=arg_pos_int,
        default=1,
        help="Minimum delay between camera captures, in milliseconds",
    )
    parser.add_argument(
        "-fs",
        "--frame-success-delay-ms",
        type=arg_pos_int,
        default=1000,
        help="Minimum delay between successful camera captures, in milliseconds",
    )
    parser.add_argument(
        "-ci",
        "--cam-index",
        type=arg_non_neg_int,
        default=0,
        help="Index of the camera capture device",
    )
    parser.add_argument(
        "-cx",
        "--cam-res-x",
        type=arg_pos_int,
        default=1280,
        help="Horizontal resolution for camera capture, in pixels",
    )
    parser.add_argument(
        "-cy",
        "--cam-res-y",
        type=arg_pos_int,
        default=720,
        help="Vertical resolution for camera capture, in pixels",
    )
    parser.add_argument(
        "-cf",
        "--cam-fps",
        type=arg_pos_int,
        default=5,
        help="Frames-per-second for camera capture, in pixels (used only during calibration)",
    )
    parser.add_argument(
        "-cm",
        "--cam-model",
        type=CameraModel,
        choices=list(CameraModel),
        default=CameraModel.PINHOLE,
        help="Mathematical model of the camera to use",
    )
    parser.add_argument(
        "-ic",
        "--iter-calib",
        type=arg_pos_int,
        default=30,
        help="Maximum iterations for camera calibration",
    )
    parser.add_argument(
        "-is",
        "--iter-subpix",
        type=arg_pos_int,
        default=30,
        help="Maximum iterations for corner subpixel calculation",
    )
    parser.add_argument(
        "-ec",
        "--epsilon-calib",
        type=arg_pos_float,
        default=1e-6,
        help="Epsilon accuracy for camera calibration",
    )
    parser.add_argument(
        "-es",
        "--epsilon-subpix",
        type=arg_pos_float,
        default=1e-3,
        help="Epsilon accuracy for corner subpixel calculation",
    )
    parser.add_argument(
        "-o",
        "--output-params-file",
        type=str,
        default="params.json",
        help="Path to the JSON file for storing the output parameters",
    )
    parser.add_argument(
        "-ha",
        "--hyperion-api",
        type=str,
        default="http://localhost:8090/json-rpc",
        help="URL of Hyperion HTTP/S JSON API",
    )
    return parser.parse_args()

def get_board_img(mon: Monitor, board_dims: Tuple[int, int], screen_board_scale: float):
    board_square = min(
        round(mon.height * screen_board_scale / (board_dims[0] + 3)),
        round(mon.width * screen_board_scale / (board_dims[1] + 3)),
    )
    img_height = board_square * (board_dims[0] + 3)
    img_width = board_square * (board_dims[1] + 3)

    black = 0
    white = 255

    board_img = np.full((img_height, img_width), white, np.uint8)

    black_row = False
    black_col = False

    for row in range(1, board_dims[0] + 2):
        black_row = not black_row
        black_col = black_row

        y = row * board_square

        for col in range(1, board_dims[1] + 2):
            x = col * board_square

            board_img[y : y + board_square, x : x + board_square] = (
                black if black_col else white
            )
            black_col = not black_col

    return board_img

def show_board(mon: Monitor, board_img: np.array, delay_ms: int):
    y0 = randbelow(mon.height - board_img.shape[0])
    x0 = randbelow(mon.width - board_img.shape[1])
    imshow("Calibration", board_img, (y0, x0), board_img.shape[:2])
    cv2.waitKey(delay_ms)

def calibrate(
    cap: VideoCapture,
    mon: Monitor,
    cam_model: CameraModel,
    board_dims: Tuple[int, int],
    board_img: np.array,
    board_frames: int,
    frame_delay_ms: int,
    frame_success_delay_ms: int,
    iter_calib: int,
    iter_subpix: int,
    epsilon_calib: float,
    epsilon_subpix: float,
):
    board_flags = (
        cv2.CALIB_CB_ADAPTIVE_THRESH
        | cv2.CALIB_CB_FAST_CHECK
        | cv2.CALIB_CB_NORMALIZE_IMAGE
    )
    if cam_model == CameraModel.PINHOLE:
        calib_func = cv2.calibrateCamera
        calib_flags = 0
    elif cam_model == CameraModel.FISHEYE:
        calib_func = cv2.fisheye.calibrate
        calib_flags = (
            cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC
            | cv2.fisheye.CALIB_CHECK_COND
            | cv2.fisheye.CALIB_FIX_SKEW
        )
    term_criteria = cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_MAX_ITER
    calib_criteria = (term_criteria, iter_calib, epsilon_calib)
    subpix_criteria = (term_criteria, iter_subpix, epsilon_subpix)

    objpoints = []  # 3d point in real world space
    imgpoints = []  # 2d points in image plane

    objp = np.zeros((1, board_dims[0] * board_dims[1], 3), np.float32)
    objp[0, :, :2] = np.mgrid[0 : board_dims[0], 0 : board_dims[1]].T.reshape(-1, 2)

    found = True

    while len(imgpoints) < board_frames:
        if found or (time() - start) * 1000 > frame_success_delay_ms:
            show_board(mon, board_img, frame_success_delay_ms)
            start = time()

        img = cap.read()
        if img is None:
            cv2.waitKey(frame_delay_ms)
            continue

        img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        found, corners = cv2.findChessboardCorners(
            img_gray,
            board_dims,
            board_flags,
        )

        if found:
            cv2.cornerSubPix(img_gray, corners, (3, 3), (-1, -1), subpix_criteria)
            cv2.drawChessboardCorners(img, board_dims, corners, found)

            objpoints.append(objp)
            imgpoints.append(corners)

            print("Detected frame: ", len(imgpoints), "out of", board_frames)

    dims = img_gray.shape[::-1]
    rms, k, d, _, _ = calib_func(
        objpoints,
        imgpoints,
        dims,
        None,
        None,
        flags=calib_flags,
        criteria=calib_criteria,
    )

    print(f"Used {len(objpoints)} valid images for calibration. RMS error: {rms}")

    return CameraParams(cam_model, dims, k, d)

def main():
    args = parse_args()

    mon = get_monitors()[args.screen_monitor]

    if args.hyperion_api:
        Hyperion(args.hyperion_api)

    cap = VideoCapture(args.cam_index, (args.cam_res_y, args.cam_res_x), args.cam_fps)
    show_preview(cap, mon, args.screen_preview_scale, args.frame_delay_ms)

    board_dims = (args.board_rows, args.board_cols)
    board_img = get_board_img(mon, board_dims, args.screen_board_scale)

    params = calibrate(
        cap,
        mon,
        args.cam_model,
        board_dims,
        board_img,
        args.board_frames,
        args.frame_delay_ms,
        args.frame_success_delay_ms,
        args.iter_calib,
        args.iter_subpix,
        args.epsilon_calib,
        args.epsilon_subpix,
    )
    params.save(args.output_params_file)

if __name__ == "__main__":
    main()

max_stars_repo_name: dinvlad/HyperVision

max_issues_repo_name: dinvlad/HyperVision

max_forks_repo_name: dinvlad/HyperVision


text:
from sklearn.pipeline import Pipeline
import pandas as pd
import numpy as np

def pipeline(X):
    """
    Simple 2-step sklearn pipeline containing a transform and an estimator steps implemented using custom classes
    It can be used as a calibration task for a regression: add it in the very end of a blueprint,  and it will
    multiply predictions by a fixed coefficients so that, on training, avg(predicted) = avg(actuals)
    """
    return Pipeline(steps=[("preprocessing", Calibrator(X)), ("model", EmptyEstimator())])

class Calibrator:
    """
    During fit(), it computes and stores the calibration coefficient that is equal to
    avg(actuals) / avg(predicted) on training data
    During transform(), it multiplies incoming data by the calibration coefficient
    """

    def __init__(self, X):
        self.multiplier = None  # calibration coefficient
        if len(X.columns) != 1:
            raise Exception(
                "As an input, this task must receive a single column containing predictions of a calibrated estimator. Instead, multiple columns have been passed."
            )

    def fit(self, X, y=None, **kwargs):
        self.multiplier = sum(y) / sum(X[X.columns[0]])
        return self

    def transform(self, X):
        return np.array(X[X.columns[0]] * self.multiplier).reshape(-1, 1)

class EmptyEstimator:
    """
    [Empty] estimator:
    - during fit, it does nothing
    - during predict, it passes incoming data as a prediction
    """

    def fit(self, X, y):
        return self

    def predict(self, data: pd.DataFrame):
        return data[:, 0]

max_stars_repo_name: andreakropp/datarobot-user-models

max_issues_repo_name: andreakropp/datarobot-user-models

max_forks_repo_name: andreakropp/datarobot-user-models


text:
import numpy as np
import gc
import torch
import sys
import os
import psutil

# By @ smth
def mem_report():
    for obj in gc.get_objects():
        if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
            print(type(obj), obj.size())

def cpu_stats():
    print(sys.version)
    print(psutil.cpu_percent())
    print(psutil.virtual_memory())  # physical memory usage
    pid = os.getpid()
    py = psutil.Process(pid)
    memoryUse = py.memory_info()[0] / 2. ** 30  # memory use in GB...I think
    print('memory GB:', memoryUse)

def write_line_to_file(s, f_path="progress.txt", new_file=False, verbose=False):
    code = "w" if new_file else "a"
    if verbose: print(s)
    with open(f_path, code, encoding='utf-8') as f:
        f.write(s)
        f.write("\n")

class UnbufferedStdOut:

    def __init__(self, stream, filename=None):
        self.stream = stream
        self.filename = filename

    def write(self, data):
        self.stream.write(data)
        self.stream.flush()
        if self.filename is not None:
            write_line_to_file(str(data), self.filename)

# sys.stdout=UnbufferedStdOut(sys.stdout)


text:
# A group-selection model of the evolution of cooperation with deme extinction
# Loosely based on chapter 7.1 of Bowles and Gintis, "A Cooperative Species"

#===============
# SETUP
#===============

from helipad import Helipad, MultiLevel, Agent
from random import choice
import numpy.random as nprand
from math import exp, sqrt

heli = Helipad()
heli.name = 'Deme Selection'
heli.order = ['linear', 'linear', 'match']
heli.stages = 3 #Stage 1 for intra-demic competition, stage 2 for reproduction, stage 3 for war

heli.addPrimitive('deme', MultiLevel, dflt=20, priority=1)
heli.removePrimitive('agent')
heli.addParameter('b', 'Benefit conferred', 'slider', dflt=6, opts={'low': 0, 'high': 10, 'step': 1})
heli.addParameter('c', 'Cost incurred', 'slider', dflt=2, opts={'low': 0, 'high': 10, 'step': 1})
heli.addParameter('k', 'Likelihood of war', 'slider', dflt=0.25, opts={'low': 0, 'high': 1, 'step': 0.01})

#===============
# BEHAVIOR
#===============

def strength(self):
	return sum([a.stocks['payoff'] for a in self.agents['agent']])
MultiLevel.strength = property(strength)

def agentStep(agent, deme, stage):
	if agent.breed == 'altruist':
		beneficiary = choice(deme.agents['agent'])
		agent.stocks['payoff'] -= deme.model.param('c')
		beneficiary.stocks['payoff'] += deme.model.param('b')

#deme is the MultiLevel object, which inherits from both baseAgent and Helipad.
@heli.hook
def demeInit(deme, model):
	deme.param('num_agent', 20)
	deme.addBreed('altruist', '#009900')
	deme.addBreed('selfish', '#990000')
	deme.addGood('payoff', '#000099', 1)
	deme.addHook('agentStep', agentStep)

#Odds of winning follow a sigmoid distribution in the difference in strength
@heli.hook('demeMatch')
def war(demes, primitive, model, stage):
	k = model.param('k')
	if not nprand.choice(2, 1, p=[1-k, k])[0]: return

	#Fight
	diff = (demes[0].strength - demes[1].strength)/10
	prob = 1/(1+exp(-diff))
	result = nprand.choice(2, 1, p=[prob, 1-prob])[0]

	#Colonize
	pop = len(demes[not result].agents['agent'])
	for a in demes[not result].agents['agent']: a.die()
	for i in range(pop):
		choice(demes[result].agents['agent']).reproduce().transfer(demes[not result])

@heli.hook
def demeStep(deme, model, stage):
	if stage > 1: deme.dontStepAgents = True

	#Reproduce
	if stage==2:
		for a in deme.agents['agent']:
			for i in range(a.stocks['payoff']): a.reproduce()
			a.die()

#Normalize population at the beginning of stage 3
@heli.hook
def modelStep(model, stage):
	if stage==3:
		for d in model.agents['deme']:
			targetDpop = 20
			while len(d.agents['agent']) > 20: choice(d.agents['agent']).die()
			while len(d.agents['agent']) < 20: choice(d.agents['agent']).reproduce()

#===============
# DATA AND VISUALIZATION
#===============

from helipad.visualize import TimeSeries
viz = heli.useVisual(TimeSeries)

@heli.reporter
def population(model): return sum([len(d.agents['agent']) for d in model.agents['deme']])
@heli.reporter
def altruists(model): return sum([len(d.agent('altruist')) for d in model.agents['deme']])/population(model)
@heli.reporter
def selfish(model): return sum([len(d.agent('selfish')) for d in model.agents['deme']])/population(model)
@heli.reporter
def fitness(model): return sum([sum([a.stocks['payoff'] for a in d.agents['agent']]) for d in model.agents['deme']])/population(model)

viz.addPlot('pop', 'Population', selected=False)
viz.addPlot('pheno', 'Phenotypes', stack=True)
viz.addPlot('fitness', 'Fitness')
viz.plots['pop'].addSeries('population', 'Total population', '#000000')
viz.plots['pheno'].addSeries('altruists', 'Altruists', '#33CC33')
viz.plots['pheno'].addSeries('selfish', 'Selfish', '#BB2222')
viz.plots['fitness'].addSeries('fitness', 'Average Fitness', '#000099')

#===============
# LAUNCH THE GUI
#===============

heli.launchCpanel()

max_stars_repo_name: vishalbelsare/helipad

max_issues_repo_name: vishalbelsare/helipad

max_forks_repo_name: vishalbelsare/helipad


text:
"""
    该py文件是将5个学期的成绩进行分类 再进行作图处理
"""
import pandas as pd
import numpy as np
from matplotlib.figure import Figure

def score_map(tem):
    if tem == "优":
        tem = 90
    elif tem == '良':
        tem = 80
    elif tem == '中':
        tem = 70
    elif tem == '及格' or tem == '合格':
        tem = 60
    elif tem == '不及格':
        tem = 0
    return tem

def scoreSub(series):
    temp = float(series["成绩"]) * float(series["学分"])
    return temp

def countScore(x):
    return sum(x["scoreSub"]) / sum(x["学分"])

def avgPlot():
    # 此函数不需要传入函数 我们使用读入的本地文件 test.csv
    # 我们先读入csv文件
    df = pd.read_csv("../GPA/data.csv",index_col=False)
    # 我们针对成绩当中的“优 良 中 及格 ....”等等我们需要先进行处理
    # 优：90 良：80 中：70 及格：60 不及格：0

    df["成绩"] = df["成绩"].map(score_map)

    # print(df["成绩"])

    # 这里我认为分组之前可以先给数据新添加一列 成绩乘以学分 计算我们

    df["scoreSub"] = df.apply(scoreSub, axis=1)
    # print(df["scoreSub"])
    # print(df)
    # 我们已经完成了上述的操作 以及处理好了成绩中的文字 并且我们使用apply函数增加了一列学分乘以成绩的值

    # 接下来我们需要进行分组

    avgScore = df.groupby("首修学期", as_index=False).apply(countScore)
    # 我们现在相当于已经得到了这个五个学期的绩点成绩
    # 接下我们需要针对这五个成绩的绩点进行作图

    # 我们需要进行作图

    # 把变量放好
    # Semster = tuple(df["首修学期"].unique())
    # print(len(Semster))
    Score = avgScore.values.tolist()
    Semster, Grade = map(list, zip(*Score))
    Semster = tuple(Semster)
    # Grade = np.array(Grade)
    demoGrade = []
    for i in Grade:
        i = round(i,2)
        demoGrade.append(i)
    print(Semster)
    print(Grade)

    fig = Figure()
    ax = fig.add_subplot(1, 1, 1)

    # Example data
    y_pos = np.arange(len(Semster))

    hbars = ax.barh(y_pos, demoGrade , align='center')
    ax.set_yticks(y_pos)
    ax.set_yticklabels(Semster)
    ax.invert_yaxis()  # labels read top-to-bottom
    ax.set_xlabel('Score')
    ax.set_title('Average Score For Individual Semster')

    # # Label with specially formatted floats
    ax.bar_label(hbars, fmt='%.2f')
    ax.set_xlim(right=100)  # adjust xlim to fit labels
    fig.tight_layout()
    return fig


text:
import os
import gzip
import pickle
import numpy as np

def _print_info(name, data_set):
    x, y = data_set
    print("""{}
  X::
    shape:{}
    min:{} mean:{:5.2f} max:{:5.2f}
  Y::
    shape:{}
    min:{} mean:{:5.2f} max:{}
    """.format(name,
               x.shape, x.min(), x.mean(), x.max(),
               y.shape, y.min(), y.mean(), y.max(),))
def _load_mnist():
    data_dir = os.path.dirname(os.path.abspath(__file__))
    data_file = os.path.join(data_dir, "scenes4.pkl")
    f = open(data_file,'rb')
    up = pickle.Unpickler(f)

    training_x, training_y, testing_x, testing_y = up.load()
    f.close()
    training_x= training_x.astype('float32')
    training_y=training_y.astype('float32')
    testing_x= testing_x.astype('float32')
    testing_y=testing_y.astype('float32')

    return training_x, training_y, testing_x, testing_y

training_x, training_y, testing_x, testing_y = _load_mnist()

if __name__ == '__main__':
    _print_info("Training Data Set:", (training_x, training_y))
    _print_info("Test Data Set:", (testing_x, testing_y))


text:
import numpy as np
import pandas as pd
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier

def abs_sum(y_true, y_pred):
    return sum(sum(abs(y_true - y_pred)))

def convert_to_prob(y):
    y_prob = []
    for label in y:
        row = []
        for i in range(4):
            if i == label:
                row.append(1)
            else:
                row.append(0)
        y_prob.append(row)
    return np.array(y_prob)

if __name__ == '__main__':
    df = pd.read_csv('data/train.csv')
    df_list = []
    for items in df.values:
        df_list.append([items[0]] + [float(i) for i in items[1].split(',')] + [items[2]])

    df = pd.DataFrame(np.array(df_list))
    df.columns = ['id'] + ['signal_'+str(i) for i in range(len(df_list[0])-2)] + ['label']

    x = df.drop(['id', 'label'], axis=1).to_numpy()
    y = df['label'].to_numpy()

    pca = PCA(n_components=0.95)
    x = pca.fit_transform(x)

    test_set_size = 0.1
    X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=test_set_size, random_state=3)

    knn = KNeighborsClassifier(n_neighbors=1, weights='distance').fit(X_train, y_train)
    y_pred_prob = knn.predict_proba(X_test)
    y_true_prob = convert_to_prob(y_test)
    loss = abs_sum(y_pred_prob, y_true_prob)
    print(loss)


text:
import streamlit as st
import warnings
warnings.filterwarnings("ignore")
# EDA Pkgs
import pandas as pd
import numpy as np
import tweepy
import json
from tweepy import OAuthHandler
import re
import textblob
from textblob import TextBlob
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import openpyxl
import time
import tqdm
import boto3
from io import StringIO
import datetime
import re

#To Hide Warnings
st.set_option('deprecation.showfileUploaderEncoding', False)
st.set_option('deprecation.showPyplotGlobalUse', False)
# Viz Pkgs

################# Twitter API Connection #######################
consumer_key = "xxx"
consumer_secret = "xxx"
access_token = "xxx-xxx"
access_token_secret = "xx"

# Use the above credentials to authenticate the API.

auth = tweepy.OAuthHandler( consumer_key , consumer_secret )
auth.set_access_token( access_token , access_token_secret )
api = tweepy.API(auth)
################################################################

df = pd.DataFrame(columns=["Tweet_Date","Twitter_User","IsVerified","Tweet","Likes","RT",'User_location'])

# Write a Function to extract tweets:
def get_tweets(Topic,Count):
    i=0
    #my_bar = st.progress(100) # To track progress of Extracted tweets
    for tweet in tweepy.Cursor(api.search, q=Topic,count=100, lang="en",exclude='retweets').items():
        #time.sleep(0.1)
        #my_bar.progress(i)
        df.loc[i,"Tweet_Date"] = tweet.created_at
        df.loc[i,"Twitter_User"] = tweet.user.name
        df.loc[i,"IsVerified"] = tweet.user.verified
        df.loc[i,"Tweet"] = tweet.text
        df.loc[i,"Likes"] = tweet.favorite_count
        df.loc[i,"RT"] = tweet.retweet_count
        df.loc[i,"User_location"] = tweet.user.location
        #df.to_csv("TweetDataset.csv",index=False)
        #df.to_excel('{}.xlsx'.format("TweetDataset"),index=False)   ## Save as Excel
        i=i+1
        if i>Count:
            break
        else:
            pass

# Function to Clean the Tweet.
def clean_tweet(tweet):
    return ' '.join(re.sub('(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)|([RT])', ' ', tweet.lower()).split())

# Funciton to analyze Sentiment
def analyze_sentiment(tweet):
    analysis = TextBlob(tweet)
    if analysis.sentiment.polarity > 0:
        return 'Positive'
    elif analysis.sentiment.polarity == 0:
        return 'Neutral'
    else:
        return 'Negative'

lst = ['AMZN','MSFT','GOOG','CVS','UNH','HUM','COST','WMT','TGT','DELL','AAPL','HPQ','GM','TSLA']
dff = pd.DataFrame()

for i in lst:

    get_tweets(i , Count=200)

    df['clean_tweet'] = df['Tweet'].apply(lambda x : clean_tweet(x))

    # Call function to get the Sentiments
    df["Sentiment"] = df["Tweet"].apply(lambda x : analyze_sentiment(x))

    df["Company"] = i

    dff = dff.append(df)

    dff = dff.fillna(0)

    # df.Tweet.fillna(value=0, inplace=True)

    # df.a.fillna(value=0, inplace=True)

dff = dff.astype(str).apply(lambda x: x.str.encode('ascii', 'ignore').str.decode('ascii'))

dff['Company'] = dff['Company'].str.replace(',', '')
dff['Company'] = dff['Company'].str.replace('\r', '')
dff['Company'] = dff['Company'].str.replace("\\", '')
dff['Company'].replace('', np.nan, inplace=True)

dff.dropna(subset=['Company'], inplace=True)

dff['Tweet'] = dff['Tweet'].apply(lambda x: re.split('https:\/\/.*', str(x))[0])
dff['Tweet'] = dff['Tweet'].str.replace(',', '')
dff['Tweet'] = dff['Tweet'].str.replace('\r', '')
dff['Tweet'] = dff['Tweet'].str.replace("\\", '')
dff['Tweet'] = dff['Tweet'].str.strip().replace("", 'Unkown')
dff['Tweet'].replace('', np.nan, inplace=True)

dff.dropna(subset=['Tweet'], inplace=True)

dff['Twitter_User'] = dff['Twitter_User'].apply(lambda x: re.split('https:\/\/.*', str(x))[0])
dff['Twitter_User'] = dff['Twitter_User'].str.replace(',', '')
dff['Twitter_User'] = dff['Twitter_User'].str.replace('\r', '')
dff['Twitter_User'] = dff['Twitter_User'].str.replace("\\", '')
dff['Twitter_User'] = dff['Twitter_User'].str.strip().replace("", 'Unkown')
dff['Twitter_User'].replace('', np.nan, inplace=True)

dff.dropna(subset=['Twitter_User'], inplace=True)

dff['User_location'] = dff['User_location'].apply(lambda x: re.split('https:\/\/.*', str(x))[0])
dff['User_location'] = dff['User_location'].str.replace(',', '')
dff['User_location'] = dff['User_location'].str.replace('\r', '')
dff['User_location'] = dff['User_location'].str.replace("\\", '')
dff['User_location'] = dff['User_location'].str.strip().replace("", 'Unkown')
dff['User_location'].replace('', np.nan, inplace=True)

dff.dropna(subset=['User_location'], inplace=True)

print(dff)

# Upload to S3 Bucket

ACCESS_KEY_ID = 'xx'
ACCESS_SECRET_KEY = 'xx'

s3 = boto3.resource(
    service_name='s3',
    region_name='us-east-1',
    aws_access_key_id= ACCESS_KEY_ID,
    aws_secret_access_key=ACCESS_SECRET_KEY
)
bucket = 'scrappednewsdata' # already created on S3

csv_buffer = StringIO()
dff.to_csv(csv_buffer, index = False, encoding='utf-8')
s3_resource = boto3.resource('s3')
s3_resource.Object(bucket,'TwitterCSVFiles/TwitterData-'+str(datetime.datetime.now())+'.csv').put(Body=csv_buffer.getvalue())
#s3_resource.Object(bucket,'TwitterData.csv').put(Body=csv_buffer.getvalue())

print('Successful!!!!!')


text:
import numpy as np

class KMeansClustering:
    """
    K-Means Clustering Model

    Args:
        n_clusters: Number of clusters(int)
    """

    def __init__(self, n_clusters, n_init=10, max_iter=1000, delta=0.001):

        self.n_cluster = n_clusters
        self.n_init = n_init
        self.max_iter = max_iter
        self.delta = delta

    def init_centroids(self, data):
        idx = np.random.choice(
            data.shape[0], size=self.n_cluster, replace=False)
        self.centroids = np.copy(data[idx, :])

    def fit(self, data):
        """
        Fit the model to the training dataset.
        Args:
            data: M x D Matrix(M data points with D attributes each)(numpy float)
        Returns:
            The object itself
        """
        if data.shape[0] < self.n_cluster:
            raise ValueError(
                'Number of clusters is grater than number of datapoints')

        best_centroids = None
        m_score = float('inf')

        for _ in range(self.n_init):
            self.init_centroids(data)

            for _ in range(self.max_iter):
                cluster_assign = self.e_step(data)
                old_centroid = np.copy(self.centroids)
                self.m_step(data, cluster_assign)

                if np.abs(old_centroid - self.centroids).sum() < self.delta:
                    break

            cur_score = self.evaluate(data)

            if cur_score < m_score:
                m_score = cur_score
                best_centroids = np.copy(self.centroids)

        self.centroids = best_centroids

        return self
    def assign_cluster(self, point):
        """
        Assign a cluster to a point
        Args:
            point: A single point (1 x D) vector (numpy float)
        Returns:
            cluster_id: Cluster ID (int)
        """
        distances = np.linalg.norm(point - self.centroids, axis=1)
        return np.argmin(distances)

    def e_step(self, data):
        """
        Expectation Step.
        Finding the cluster assignments of all the points in the data passed
        based on the current centroids
        Args:
            data: M x D Matrix (M training samples with D attributes each)(numpy float)
        Returns:
            Cluster assignment of all the samples in the training data
            (M) Vector (M number of samples in the train dataset)(numpy int)
        """
        cluster_assign = np.zeros(data.shape[0], dtype=int)

        for idx, point in enumerate(data):
            cluster_assign[idx] = self.assign_cluster(point)

        return cluster_assign
        # done

    def m_step(self, data, cluster_assgn):
        """
        Maximization Step.
        Compute the centroids
        Args:
            data: M x D Matrix(M training samples with D attributes each)(numpy float)
        Change self.centroids
        """
        for cluster_id in range(self.n_cluster):
            cluster_points = data[cluster_assgn == cluster_id]
            self.centroids[cluster_id] = np.mean(cluster_points, axis=0)
        # done

    def evaluate(self, data, cluster_assign):
        """
        K-Means Objective
        Args:
            data: Test data (M x D) matrix (numpy float)
        Returns:
            metric : Summation of number of clusters, summation of number of cases square of difference between centroid of cluster j and ith case of x
        """
        # Initialize the distance array
        distance = np.zeros((data.shape[0]))
        # SSE
        for k in range(self.n_cluster):
            # Calucluate error
            squared = np.square(data[cluster_assign==k] - self.centroids[k])
            distance[cluster_assign == k] = np.sqrt(np.sum(squared, axis=1))

        # Sum and Square errors
        return np.sum(np.square(distance))