text:
# Copyright (c) Facebook, Inc. and its affiliates.

import pkg_resources
import os
import errno
import math
from pathlib import Path
import pickle
import re
import sys

import numpy as np

from collections import defaultdict

DATA_PATH = 'data/wikidata_big/kg/tkbc_processed_data/'

# DATA_PATH = pkg_resources.resource_filename('tkbc', 'data/')

def get_be(begin, end):
    if begin is None:
        begin = (-math.inf, 0, 0)
    else:
        begin = (int(begin), 0, 0)

    if end is None:
        end = (math.inf, 0, 0)
    else:
        end = (int(end), 0, 0)

    return begin, end

def prepare_dataset_rels(path, name):
    """
    Given a path to a folder containing tab separated files :
     train, test, valid
    In the format :
    (lhs)\t(rel)\t(rhs)\t(type)\t(timestamp)\n
    Maps each entity, relation+type and timestamp to a unique id, create corresponding folder
    name in pkg/data, with mapped train/test/valid files.
    Also create to_skip_lhs / to_skip_rhs for filtered metrics and
    rel_id / ent_id for analysis.
    """
    files = ['train', 'valid', 'test']
    entities, relations, timestamps = set(), set(), set()
    for f in files:
        file_path = os.path.join(path, f)
        to_read = open(file_path, 'r')
        for line in to_read.readlines():
            v = line.strip().split('\t')
            lhs, rel, rhs, begin, end = v
            begin, end = get_be(begin, end)
            timestamps.add(begin)
            timestamps.add(end)
            entities.add(lhs)
            entities.add(rhs)
            relations.add(rel)
        to_read.close()
    timestamps_int = set()
    for t in timestamps:
        timestamps_int.add(t[0])
    min_t = min(timestamps_int)
    max_t = max(timestamps_int)
    print('Minimum, maximum times:', min_t, max_t)
    timestamps = [(i, 0, 0) for i in range(min_t, max_t + 1)]
    print(f"{len(timestamps)} timestamps")

    entities_to_id = {x: i for (i, x) in enumerate(sorted(entities))}
    relations_to_id = {x: i for (i, x) in enumerate(sorted(relations))}

    # we need to sort timestamps and associate them to actual dates

    # all_ts = sorted(timestamps)[1:-1]
    all_ts = sorted(timestamps)
    timestamps_to_id = {x: i for (i, x) in enumerate(all_ts)}
    # print(timestamps_to_id)

    print("{} entities, {} relations over {} timestamps".format(len(entities), len(relations), len(timestamps)))
    n_relations = len(relations)
    n_entities = len(entities)

    try:
        os.makedirs(os.path.join(DATA_PATH, name))
    except OSError as e:
        r = input(f"{e}\nContinue ? [y/n]")
        if r != "y":
            sys.exit()

    # write ent to id / rel to id
    for (dic, f) in zip([entities_to_id, relations_to_id, timestamps_to_id], ['ent_id', 'rel_id', 'ts_id']):
        ff = open(os.path.join(DATA_PATH, name, f), 'wb')
        pickle.dump(dic, ff)
        ff.close()

    # dump the time differences between timestamps for continuity regularizer
    # ignores number of days in a month but who cares
    # ts_to_int = [x[0] * 365 + x[1] * 30 + x[2] for x in all_ts]
    ts_to_int = [x[0] for x in all_ts]
    ts = np.array(ts_to_int, dtype='float')
    diffs = ts[1:] - ts[:-1]  # remove last timestamp from time diffs. it's not really a timestamp
    out = open(os.path.join(DATA_PATH, name, 'ts_diffs.pickle'), 'wb')
    pickle.dump(diffs, out)
    out.close()

    # map train/test/valid with the ids
    event_list = {
        'all': [],
    }
    for f in files:
        file_path = os.path.join(path, f)
        to_read = open(file_path, 'r')
        examples = []
        ignore = 0
        total = 0
        full_intervals = 0
        half_intervals = 0
        point = 0
        for line in to_read.readlines():
            v = line.strip().split('\t')
            lhs, rel, rhs, begin, end = v
            begin_t, end_t = get_be(begin, end)
            total += 1

            begin = begin_t
            end = end_t

            if begin_t[0] == -math.inf:
                begin = all_ts[0]
                if not end_t[0] == math.inf:
                    half_intervals += 1
            if end_t[0] == math.inf:
                end = all_ts[-1]
                if not begin_t[0] == -math.inf:
                    half_intervals += 1

            if begin_t[0] > -math.inf and end_t[0] < math.inf:
                if begin_t[0] == end_t[0]:
                    point += 1
                else:
                    full_intervals += 1

            begin = timestamps_to_id[begin]
            end = timestamps_to_id[end]

            if begin > end:
                ignore += 1
                continue

            lhs = entities_to_id[lhs]
            rel = relations_to_id[rel]
            rhs = entities_to_id[rhs]

            event_list['all'].append((begin, -1, (lhs, rel, rhs)))
            event_list['all'].append((end, +1, (lhs, rel, rhs)))

            try:
                examples.append([lhs, rel, rhs, begin, end])
            except ValueError:
                continue
        out = open(Path(DATA_PATH) / name / (f + '.pickle'), 'wb')
        pickle.dump(np.array(examples).astype('uint64'), out)
        out.close()
        print(f"Ignored {ignore} events.")
        print(f"Total : {total} // Full : {full_intervals} // Half : {half_intervals} // Point : {point}")

    for k, v in event_list.items():
        out = open(Path(DATA_PATH) / name / ('event_list_' + k + '.pickle'), 'wb')
        print("Dumping all events", len(v))
        pickle.dump(sorted(v), out)
        out.close()

if __name__ == "__main__":
    datasets = ['wikidata_big']
    for d in datasets:
        print("Preparing dataset {}".format(d))
        try:
            dataset_location = 'data/wikidata_big/kg/'
            prepare_dataset_rels(
                dataset_location,
                d
            )
            # prepare_dataset_rels(
            #     os.path.join(
            #         os.path.dirname(os.path.realpath(__file__)), 'src_data', d
            #     ),
            #     d
            # )
        except OSError as e:
            if e.errno == errno.EEXIST:
                print(e)
                print("File exists. skipping...")
            else:
                raise


text:
'''
memewizard
~~~~~~~~~~
One good source for all things memes
    >>> from memewizard import meme_object_yt
    >>> from memewizard.cli import main
    >>> from memewizard.visual import make_pie
'''

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup
import pytrends.request
import pandas as pd
import numpy as np
import requests
import warnings
import random
import json
import re

from memewizard.helpers import (subjectify, chunkify)

warnings.filterwarnings('ignore')
stopwords = ['Why Is', 'Why Does', 'Everyone', 'EVERYONE', 'Why', 'Is The', 'How', 'What Are', 'What\'s Up', 'With', '?', '.', 'Are You', 'FINISHED']

def funnywords() -> 'list[str]':
    words = requests.get('https://raw.githubusercontent.com/snguyenthanh/better_profanity/master/better_profanity/profanity_wordlist.txt') \
        .text.split('\n')
    while '' in words: words.remove('')
    return words

regex = re.compile('|'.join(map(re.escape, stopwords)))
nsfw_regex = re.compile('|'.join(funnywords())+'|rule 34|b*tches|d*ck', re.IGNORECASE)

class meme_object:
    '''The meme_object class to handle KnowYourMeme data'''
    def fetch_memes(page: str) -> list:
        '''Scrape any memes page on knowyourmemes.com'''

        r = requests.get(
            'https://knowyourmeme.com/memes/page/{}'
            .format(page),
            headers={'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36'}
        ).text
        soup = BeautifulSoup(r, 'html.parser')

        # parsing perfection
        m = soup.find('tbody', {'class':['entry-grid-body', 'infinite']}).text.split('    ')

        for i, s in enumerate(m):
            m[i] = s.strip()
            if m[i] == '': del m[i]
        for i, s in enumerate(m): m[i] = s.strip()
        for i, s in enumerate(m):
            if 'NSFW' in m[i] or m[i] == 'NSFW':
                del m[i+1]
                del m[i]
        while '' in m:
            m.remove('')
        for i, s in enumerate(m):
            if 'Updated' in m[i]: del m[i]

        return [i.strip() for i in m if not nsfw_regex.search(i)]
    def fetch_meme_info(url: str) -> dict:
        '''Fetch more information on a meme with a given URL.'''
        r = requests.get(url,
        headers={'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36'})
        s = BeautifulSoup(r.text, 'html.parser')
        title = s.find('title').text.split(' |')[0]

        try:
            st = s.find('div', {'class': ['details']}).text.split('\n\n')
        except AttributeError:
            st = ['']
        while '' in st:
            st.remove('')
        st = [_st.replace(':', '').replace('\n', '') for _st in st]
        st = list(chunkify(st, 2))
        st.insert(0, ['Name', title])

        if invalids.RESEARCH in st:
            st.remove(invalids.RESEARCH)
        return {title: st}
    def fetch_meme_images(meme_list: list) -> list:
        '''Find relevant images for a list of memes.'''
        meme_images = []
        for meme in meme_list:
            r = requests.get(
                'https://www.google.com/search?q={}&rlz=1CAOUAQ_enUS980&source=lnms&tbm=isch&biw=1517&bih=750&dpr=0.9&surl=1&safe=active&ssui=on#imgrc=1QhXmjkgq_MRQM'
                .format(meme)
            ).text
            soup = BeautifulSoup(r, 'html.parser')

            images = soup.find_all('img')
            meme_images.append(images[random.randrange(1, len(images))]['src'])

        return meme_images
    def fetch_trend_history(memes: list) -> list:
        '''Fetch trend history of a given list of memes. The list of memes should contain less than 6 items.'''
        trend = pytrends.request.TrendReq()
        trend.build_payload(memes, timeframe='today 1-m', cat='0', geo='US')

        search = trend.interest_over_time()

        df = pd.DataFrame(search)
        df.reset_index(inplace=True, drop=True)
        df = df.to_dict()

        meme = []
        for k in df:
            submeme = []
            v = df[k]
            for e in v:
                if not isinstance(v[e], (bool, str)):
                    submeme.append(v[e])
            meme.append(submeme)
        while [] in meme:
            meme.remove([])

        return meme

class meme_object_yt:
    '''The meme_object class to handle data from YouTube'''
    def fetch_memes() -> list:
        '''Fetches memes from LessonsInMemeCulture on YouTube and does a ton of parsing magic.'''
        r = requests.get('https://www.youtube.com/c/LessonsinMemeCulture/videos')
        big_response = json.loads(r.text.split('var ytInitialData =')[1].split(';</script>')[0])
        filtere = big_response['contents']['twoColumnBrowseResultsRenderer']['tabs'][1]['tabRenderer']['content']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['gridRenderer']['items']
        res = []
        for i in range(len(filtere)):
            try:
                string = filtere[i]['gridVideoRenderer']['title']['runs'][0]['text'].replace('“', '"').replace('”', '"')
                token = regex.sub('', string)
                words = re.findall('"([^"]*)"', string)
                if len(words) > 0:
                    meme = ''.join([s.split('_')[0] for s in words[0]])
                    res.append(meme)
                else:
                    res.append(subjectify(token))
            except KeyError:
                break

        return res

    def fetch_meme_dates() -> list:
        '''Fetch "meme" dates from LessonsInMemeCulture'''
        r = requests.get('https://www.youtube.com/c/LessonsinMemeCulture/videos')
        big_response = json.loads(r.text.split('var ytInitialData =')[1].split(';</script>')[0])
        filtere = big_response['contents']['twoColumnBrowseResultsRenderer']['tabs'][1]['tabRenderer']['content']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['gridRenderer']['items']

        dates = []

        for i in range(len(filtere)):
            try:
                if not '*' in str(filtere[i]['gridVideoRenderer']['title']['runs'][0]['text']):
                    dates.append(filtere[i]['gridVideoRenderer']['publishedTimeText']['simpleText'])
            except KeyError:
                break
        return dates

class invalids:
  RESEARCH = ['This submission is currently being researched and evaluated.',
              'You can help confirm this entry by contributing facts, media, and other evidence of notability and mutation.']
  NOTFOUND = 'Page Not Found (404) - Know Your Meme'
  GALLERY = 'Trending Videos Gallery'

def predict(meme: str) -> None:
  '''Predict the popularity of a meme'''
  memes = meme_object.fetch_trend_history([meme])[0]
  data = [[i, m] for i,m in enumerate(memes)]

  y_data = np.array(data)[:,1]

  X = np.array(range(len(y_data))).reshape(-1,1)

  X_train,X_test,y_train,y_test=train_test_split(X,y_data,test_size=0.25,random_state=0)

  logreg = LogisticRegression(solver='lbfgs', max_iter=1000)
  logreg.fit(X_train,y_train)
  y_pred=logreg.predict(X_test)

  y = [*y_train, *y_test, *y_pred]

  plt.plot(range(len(y)),y,c='black')
  plt.axvline(len(y)-len(y_pred), c='blue', alpha=0.2)
  plt.title('History for \'{}\''.format(meme))

  plt.savefig('figure.png')

ext: py

max_stars_repo_name: themysticsavages/memewizard

max_issues_repo_name: themysticsavages/memewizard

max_forks_repo_name: themysticsavages/memewizard

reason: import numpy


text:
import numpy as np
import glob
import os
import re
import matplotlib.pyplot as plt
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator

c = np.zeros(0)
a = np.array([1, 2, 3])
b = np.array([5, 6, 7])

print(np.concatenate((a, b), axis = 0))
stack_a_b = np.stack((a, b), axis = 0)
print("stack_a_b:\n ", stack_a_b)
print("stack_a_b.mean():\n ", stack_a_b.mean(axis = 0))
print(np.mean((a, b), axis = 0))
print(np.std((a, b), axis = 0))

print("b[3:4]: ", b[1:1])

thresholds = np.zeros(0)
print(thresholds)

max_stars_repo_name: NemoHimma/ZeroRL

max_issues_repo_name: NemoHimma/ZeroRL

max_forks_repo_name: NemoHimma/ZeroRL


text:
from typing import List
import numpy as np
from src.data import Problem, Case, Matter

class MapColor:

    def __init__(self):
        pass

    @classmethod
    def array(cls, x_arr: np.array, background: np.int = 0) -> List[np.array]:
        """
        :param x_arr: np.array(int), array to split
        :param background: int, must be one of 0-9
        :return: List[np.array(np.int)]
        """

        res_list = []
        for c in range(10):
            if c == background:
                continue
            if (x_arr == c).sum() == 0:
                continue
            new_m_value = np.ones(x_arr.shape, dtype=np.int) * background
            new_m_value[x_arr == c] = c
            res_list.append((new_m_value, c))

        assert len(res_list) > 0
        return list(sorted(res_list, key=lambda res: (res[0] != background).sum()))

    @classmethod
    def matter(cls, m: Matter) -> List[Matter]:

        arr_list = cls.array(m.values, m.background_color)
        res_list = []
        for arr in arr_list:
            x_arr = arr[0]
            # trim
            x_sum = (x_arr != m.background_color).sum(axis=1)
            y_sum = (x_arr != m.background_color).sum(axis=0)

            min_x = min([i for i in range(x_arr.shape[0]) if x_sum[i]])
            max_x = max([i for i in range(x_arr.shape[0]) if x_sum[i]])
            min_y = min([i for i in range(x_arr.shape[1]) if y_sum[i]])
            max_y = max([i for i in range(x_arr.shape[1]) if y_sum[i]])

            new_values = x_arr[min_x:max_x + 1, min_y:max_y + 1].copy()
            m = Matter(new_values, min_x, min_y, background_color=m.background_color, new=True)
            m.color = arr[1]
            res_list.append(m)

        return res_list

    @classmethod
    def case(cls, c: Case) -> Case:
        assert len(c.matter_list) == 1
        new_case = c.copy()
        new_case.matter_list = cls.matter(c.matter_list[0])
        return new_case

    @classmethod
    def problem(cls, p: Problem) -> Problem:
        q: Problem = p.copy()
        q.train_x_list = [cls.case(c) for c in p.train_x_list]
        q.test_x_list = [cls.case(c) for c in p.test_x_list]
        return q

if __name__ == "__main__":
    x = np.array([[2, 2, 3], [2, 0, 3]])
    print(MapColor.array(x))


text:
import numpy as np
import sklearn.decomposition
import gc

import cv2
import imageio
import copy
import time
from tqdm.notebook import tqdm, trange

from matplotlib import pyplot as plt

from face_rhythm.util import helpers

def make_distance_matrix(center_idx, vid_height, vid_width):
    """
    creates a matrix of cartesian coordinate distances from the center

    Args:
        center_idx (list): chosen center index
        vid_height (int): height of the video in pixels
        vid_width (int): width of the video in pixels

    Returns:
        distance_matrix (np.ndarray): array of distances to the center index

    """

    x, y = np.meshgrid(range(vid_width), range(vid_height))  # note dim 1:X and dim 2:Y
    return np.sqrt((y - int(center_idx[1])) ** 2 + (x - int(center_idx[0])) ** 2)

def create_kernel(config_filepath, point_idxs):
    """
    creates convolutional kernel

    Args:
        config_filepath (Path): path to the config file
        point_idxs (np.ndarray): point indices

    Returns:
        cos_kernel (np.ndarray): cosine kernel
        cos_kernel_mean (np.ndarray): mean of cosine kernel
    """

    config = helpers.load_config(config_filepath)
    cdr = config['CDR']
    video = config['Video']

    width_cos_kernel = cdr['width_cosKernel']
    num_dots = cdr['num_dots']
    vid_height = video['height']
    vid_width = video['width']
    cos_kernel = np.zeros((vid_height, vid_width, num_dots))
    cos_kernel_mean = np.zeros(num_dots)
    for ii in tqdm(range(num_dots),desc="creating kernel"):
        x = make_distance_matrix(np.squeeze(point_idxs)[ii], vid_height, vid_width)
        x_norm = x / width_cos_kernel
        x_clipped = np.minimum(x_norm, 1)
        cos_kernel[:, :, ii] = (np.cos(x_clipped * np.pi) + 1) / 2
        tmp = copy.deepcopy(cos_kernel[:, :, ii])
        tmp[tmp == 0] = np.nan
        cos_kernel_mean[ii] = np.nanmean(tmp)
    return cos_kernel, cos_kernel_mean

def space_points(config_filepath, pts_all):
    """
    spaces out the points

    Args:
        config_filepath (Path): path to the config file
        pts_all (dict): dict of point arrays

    Returns:
        pts_spaced_convDR (np.ndarray): spaced out point array
    """

    config = helpers.load_config(config_filepath)
    cdr = config['CDR']

    spacing = cdr['spacing']

    bbox_subframe_displacement = pts_all['bbox_subframe_displacement']
    pts_x_displacement = pts_all['pts_x_displacement']
    pts_y_displacement = pts_all['pts_y_displacement']

    pts_spaced_convDR = np.ones((np.int64(bbox_subframe_displacement[3] * bbox_subframe_displacement[2] / spacing),
                                 2)) * np.nan  ## preallocation
    cc = 0  ## set idx counter
    # make spaced out points
    for ii in range(len(pts_x_displacement)):
        if (pts_x_displacement[ii] % spacing == 0) and (pts_y_displacement[ii] % spacing == 0):
            pts_spaced_convDR[cc, 0] = pts_x_displacement[ii]
            pts_spaced_convDR[cc, 1] = pts_y_displacement[ii]
            cc = cc + 1

    pts_spaced_convDR = np.expand_dims(pts_spaced_convDR, 1).astype('single')
    pts_spaced_convDR = np.delete(pts_spaced_convDR, np.where(np.isnan(pts_spaced_convDR[:, 0, 0])), axis=0)
    return pts_spaced_convDR

def points_show(config_filepath, session, pts_all, pts_spaced_convDR, cosKernel):
    """
    shows the points with the cosKernel overlayed

    Args:
        config_filepath (Path): path to the config file
        session (dict): current session dictionary
        pts_all (dict): dict of point arrays
        pts_spaced_convDR (np.ndarray): array of spaced points
        cosKernel (np.ndarray): cosine kernel

    Returns:

    """

    config = helpers.load_config(config_filepath)
    cdr = config['CDR']
    vidNum_toUse = cdr['vidNum']
    frameNum_toUse = cdr['frameNum']
    dot_size = cdr['dot_size']
    kernel_pixel = cdr['kernel_pixel']
    path_vid_allFiles = session['videos']

    kernel_example = np.zeros_like(cosKernel[...,:3])
    kernel_example[...,2] = cosKernel[...,kernel_pixel]
    alpha = cdr['kernel_alpha']

    color_tuples = helpers.load_data(config_filepath, 'color_tuples')

    vid = imageio.get_reader(path_vid_allFiles[vidNum_toUse], 'ffmpeg')
    frame = vid.get_data(
        frameNum_toUse)  # Get a single frame to use as the first 'previous frame' in calculating optic flow
    pointInds_tuple = list(np.arange(pts_spaced_convDR.shape[0]))
    for ii in range(pts_spaced_convDR.shape[0]):
        pointInds_tuple[ii] = tuple(np.squeeze(pts_spaced_convDR[ii, 0, :]))
        cv2.circle(frame, pointInds_tuple[ii], dot_size, color_tuples[ii], -1)
    plt.imshow(cv2.cvtColor(np.float32((frame*(1-alpha)+255*kernel_example*alpha)/255), cv2.COLOR_BGR2RGB))
    plt.show()

def makeConvDR(ii, input_traces, cos_kernel, cos_kernel_mean, pca, rank_reduced, dots_new):
    """
    performs the convolutional dimensionality reduction
    called within the multithreading

    Args:
        ii ():
        input_traces ():
        cos_kernel ():
        cos_kernel_mean ():
        pca ():
        rank_reduced ():
        dots_new ():

    Returns:
        positions_convDR_meanSub ():

    """

    influence_weightings = cos_kernel[int(dots_new[ii][0][1]), int(dots_new[ii][0][0]), :]

    idx_nonZero = np.array(np.where(influence_weightings != 0))[0, :]

    displacements_preConvDR_x = input_traces[idx_nonZero, 0, :] * influence_weightings[idx_nonZero][:, None]
    displacements_preConvDR_x = displacements_preConvDR_x - np.mean(displacements_preConvDR_x, axis=1)[:, None]
    displacements_preConvDR_y = input_traces[idx_nonZero, 1, :] * influence_weightings[idx_nonZero][:, None]
    displacements_preConvDR_y = displacements_preConvDR_y - np.mean(displacements_preConvDR_y, axis=1)[:, None]
    pca.fit(displacements_preConvDR_x)
    output_PCA_loadings0 = pca.components_.T
    pca.fit(displacements_preConvDR_y)
    output_PCA_loadings1 = pca.components_.T

    output_PCA_scores0 = np.dot(displacements_preConvDR_x, output_PCA_loadings0)
    output_PCA_scores1 = np.dot(displacements_preConvDR_y, output_PCA_loadings1)
    positions_convDR_meanSub = np.zeros((2, input_traces.shape[2]))
    positions_convDR_meanSub[0, :] = np.mean(
        np.dot(output_PCA_loadings0[:, :rank_reduced], output_PCA_scores0[:, :rank_reduced].T), axis=1) / \
                                     cos_kernel_mean[ii]
    positions_convDR_meanSub[1, :] = np.mean(
        np.dot(output_PCA_loadings1[:, :rank_reduced], output_PCA_scores1[:, :rank_reduced].T), axis=1) / \
                                     cos_kernel_mean[ii]
    return positions_convDR_meanSub

def compute_influence(config_filepath, pointInds_toUse, pts_spaced_convDR, cosKernel, cosKernel_mean, positions_new_sansOutliers):
    """
    performs single-threaded convolutional dimensionality reduction

    Args:
        config_filepath (Path): path to current config file
        pointInds_toUse (np.ndarray): original point indices
        pts_spaced_convDR (np.ndarray): point locations after dimensionality reduction
        cosKernel (np.ndarray): cosine kernel
        cosKernel_mean (np.ndarray): mean of the cosine kernel
        positions_new_sansOutliers (np.ndarray): integrated positions

    Returns:
        positions_convDR_meanSub (np.ndarray): positions after dim red with mean substracted
        positions_convDR_absolute (np.ndarray): absolute positions after dim red
    """

    config = helpers.load_config(config_filepath)
    num_components = config['CDR']['num_components']

    input_traces = np.float32(positions_new_sansOutliers)
    # num_components = 3
    rank_reduced = num_components

    dots_old = pointInds_toUse
    dots_new = pts_spaced_convDR

    pca = sklearn.decomposition.PCA(n_components=num_components)

    positions_convDR_meanSub = np.zeros((dots_new.shape[0] , 2 , input_traces.shape[2]))
    output_PCA_loadings = np.zeros((dots_new.shape[0] , 2 , input_traces.shape[2] , num_components))
    output_PCA_scores = list(np.zeros(dots_new.shape[0]))
    for ii in trange(dots_new.shape[0] , mininterval=1):
    #     print(ii)
        influence_weightings = cosKernel[int(dots_new[ii][0][1]) , int(dots_new[ii][0][0]) , :]

        idx_nonZero = np.array(np.where(influence_weightings !=0))[0,:]

        displacements_preConvDR_x = input_traces[idx_nonZero , 0 , :] * influence_weightings[idx_nonZero][:,None]
        displacements_preConvDR_x = displacements_preConvDR_x - np.mean(displacements_preConvDR_x , axis=1)[:,None]
        displacements_preConvDR_y = input_traces[idx_nonZero , 1 , :] * influence_weightings[idx_nonZero][:,None]
        displacements_preConvDR_y = displacements_preConvDR_y - np.mean(displacements_preConvDR_y , axis=1)[:,None]
        pca.fit(displacements_preConvDR_x)
        output_PCA_loadings[ii,0,:,:] = pca.components_.T
        pca.fit(displacements_preConvDR_y)
        output_PCA_loadings[ii,1,:,:] = pca.components_.T

        output_PCA_scores[ii] = np.zeros((2,displacements_preConvDR_y.shape[0] , num_components))
        output_PCA_scores[ii][0,:,:] = np.dot( displacements_preConvDR_x  ,  output_PCA_loadings[ii,0,:,:] )
        output_PCA_scores[ii][1,:,:] = np.dot( displacements_preConvDR_y  ,  output_PCA_loadings[ii,1,:,:] )
        positions_convDR_meanSub[ii,0,:] = np.mean(np.dot( output_PCA_loadings[ii,0,:,:rank_reduced] , output_PCA_scores[ii][0,:,:rank_reduced].T ) , axis=1) / cosKernel_mean[ii]
        positions_convDR_meanSub[ii,1,:] = np.mean(np.dot( output_PCA_loadings[ii,1,:,:rank_reduced] , output_PCA_scores[ii][1,:,:rank_reduced].T ) , axis=1) / cosKernel_mean[ii]

    positions_convDR_absolute = (positions_convDR_meanSub + np.squeeze(pts_spaced_convDR)[:, :, None])
    return positions_convDR_meanSub, positions_convDR_absolute

def conv_dim_reduce_workflow(config_filepath):
    """
    sequences the steps for performing convolutional dimensionality reduction on the points

    Args:
        config_filepath (Path): path to the current config file

    Returns:

    """

    print(f'== Beginning convolutional dimensionality reduction ==')
    tic_all = time.time()

    config = helpers.load_config(config_filepath)
    general = config['General']
    video = config['Video']

    for session in general['sessions']:
        pointInds_toUse = helpers.load_nwb_ts(session['nwb'], 'Optic Flow', 'pointInds_toUse')
        pts_all = helpers.get_pts(session['nwb'])

        # first let's make the convolutional kernel. I like the cosine kernel because it goes to zero.
        tic = time.time()
        cosKernel, cosKernel_mean = create_kernel(config_filepath, pointInds_toUse)
        helpers.print_time('Kernel created', time.time() - tic)

        # let's make new dots with wider spacing
        tic = time.time()
        pts_spaced_convDR = space_points(config_filepath, pts_all)
        print(f'number of points: {pts_spaced_convDR.shape[0]}')
        helpers.print_time('Points spaced out', time.time() - tic)

        tic_session = time.time()
        if config['CDR']['display_points']:
            points_show(config_filepath, session, pts_all, pts_spaced_convDR, cosKernel)
        positions_new_sansOutliers = helpers.load_nwb_ts(session['nwb'], 'Optic Flow', 'positions_cleanup')
        positions_convDR_meanSub, positions_convDR_absolute = compute_influence(config_filepath, pointInds_toUse, pts_spaced_convDR,
                                                                 cosKernel, cosKernel_mean, positions_new_sansOutliers)

        helpers.create_nwb_ts(session['nwb'], 'Optic Flow', 'positions_convDR_meanSub', positions_convDR_meanSub, video['Fs'])
        helpers.create_nwb_ts(session['nwb'], 'Optic Flow', 'positions_convDR_absolute', positions_convDR_absolute, video['Fs'])
        helpers.create_nwb_ts(session['nwb'], 'Optic Flow', 'pts_spaced_convDR', pts_spaced_convDR, video['Fs'])
        helpers.print_time(f'Session {session["name"]} completed', time.time() - tic_session)

        del positions_new_sansOutliers, positions_convDR_meanSub, positions_convDR_absolute, cosKernel, cosKernel_mean, pts_spaced_convDR, pts_all, pointInds_toUse

    helpers.print_time('total elapsed time', time.time() - tic_all)
    print(f'== End convolutional dimensionality reduction ==')

    gc.collect()

max_stars_repo_name: bernardosabatinilab/face-rhythm

max_issues_repo_name: bernardosabatinilab/face-rhythm

max_forks_repo_name: bernardosabatinilab/face-rhythm


text:
import matplotlib
matplotlib.use('Agg')  # Must be before importing matplotlib.pyplot or pylab!
import matplotlib.pyplot as plt

import os
import argparse
import sys
import pickle
from tqdm import tqdm

import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
from torch.optim.lr_scheduler import ReduceLROnPlateau

from utils import epoch
from models import MLP

parser = argparse.ArgumentParser()
parser.add_argument('data_path', metavar='DATA_PATH', help='path to datasets')
parser.add_argument('--output_dir', type=str, default='./', help='output directory. Default=Current folder')
parser.add_argument('--epochs', type=int, default=200, help='number of epochs. Default=200')
parser.add_argument('--batch_size', type=int, default=512, help='batch size. Default=512')
parser.add_argument('--eval_batch_size', type=int, default=512, help='batch size for eval mode. Default=512')
parser.add_argument('--lr', type=float, default=1e-1, help='initial learning rate. Default=1e-1')
parser.add_argument('--momentum', type=float, default=0.9, help='momentum. Default=0.9')
parser.add_argument('--layers', type=int, default=5, help='number of hidden layers. Default=5')
parser.add_argument('--units', type=int, default=256, help='number of hidden units in each layer. Default=256')
parser.add_argument('--dropout', type=float, default=0.5, help='Dropout rate at input layer. Default=0.5')
parser.add_argument('--l1', type=float, default=0.0, help='L1 regularization. Default=0')
parser.add_argument('--clr', type=float, default=0.0, help='Cross-Lipschitz regularization. Default=0')
parser.add_argument('--no-cuda', dest='cuda', action='store_false', help='NOT use cuda')
parser.add_argument('--seed', type=int, default=0, help='random seed to use. Default=0')
parser.set_defaults(cuda=True)

def train_mlp(args, loaders, model, criterion, optimizer, scheduler, l1_factor=0.0, clr_factor=0.0, model_name='model'):
	train_loader = loaders['train_loader']
	valid_loader = loaders['valid_loader']
	test_loader = loaders['test_loader']

	if args.cuda:
		model = model.cuda()

	best_valid_loss = sys.float_info.max

	train_losses = []
	valid_losses = []

	if not os.path.exists(args.output_dir):
		os.makedirs(args.output_dir)

	for i_epoch in tqdm(range(args.epochs), desc='Epochs'):

		# Train
		train_labels, train_preds, train_loss = epoch(train_loader, model, train=True, criterion=criterion, optimizer=optimizer, l1_factor=l1_factor, clr_factor=clr_factor)
		train_losses.append(train_loss)

		# Validation
		valid_labels, valid_preds, valid_loss = epoch(valid_loader, model, criterion=criterion)

		# Learning rate decay
		if scheduler is not None:
			scheduler.step(valid_loss)

		valid_losses.append(valid_loss)

		# remember best valid loss and save checkpoint
		is_best = valid_loss < best_valid_loss

		if is_best:
			best_valid_loss = valid_loss

			# evaluate on test set
			test_labels, test_preds, test_loss = epoch(test_loader, model, criterion=criterion)

			with open(args.output_dir + model_name + '_result.txt', 'w') as f:
				f.write('Best Validation Epoch: {}\n'.format(i_epoch))
				f.write('Best Validation Loss: {}\n'.format(best_valid_loss))
				f.write('Train Loss: {}\n'.format(train_loss))
				f.write('Test Loss: {}\n'.format(test_loss))

			# Save entire model
			torch.save(model, args.output_dir + model_name + '.pth')
			# Save model params
			torch.save(model.state_dict(), args.output_dir + model_name + '_params.pth')

	# plot
	plt.figure()
	plt.plot(np.arange(len(train_losses)), np.array(train_losses), label='Training Loss')
	plt.plot(np.arange(len(valid_losses)), np.array(valid_losses), label='Validation Loss')
	plt.xlabel('epoch')
	plt.ylabel('Loss')
	plt.legend(loc="best")
	plt.savefig(args.output_dir + model_name + '_loss.eps', format='eps')

def load_data(data_path):
	with open(data_path + 'train.data_csr', 'rb') as f:
		X_train = pickle.load(f)
	with open(data_path + 'train.labels', 'rb') as f:
		y_train = pickle.load(f)
	with open(data_path + 'valid.data_csr', 'rb') as f:
		X_valid = pickle.load(f)
	with open(data_path + 'valid.labels', 'rb') as f:
		y_valid = pickle.load(f)
	with open(data_path + 'test.data_csr', 'rb') as f:
		X_test = pickle.load(f)
	with open(data_path + 'test.labels', 'rb') as f:
		y_test = pickle.load(f)

	print('Train: {}, Validation: {}, Test: {}'.format(X_train.shape[0], X_valid.shape[0], X_test.shape[0]))

	train_set = TensorDataset(torch.from_numpy(X_train.todense().astype('float32')),
							  torch.from_numpy(np.array(y_train).astype('int')))
	valid_set = TensorDataset(torch.from_numpy(X_valid.todense().astype('float32')),
							  torch.from_numpy(np.array(y_valid).astype('int')))
	test_set = TensorDataset(torch.from_numpy(X_test.todense().astype('float32')),
							 torch.from_numpy(np.array(y_test).astype('int')))

	return train_set, valid_set, test_set

if __name__ == '__main__':
	args = parser.parse_args()

	np.random.seed(args.seed)
	torch.manual_seed(args.seed)
	if args.cuda:
		torch.cuda.manual_seed(args.seed)

	train_set, valid_set, test_set = load_data(args.data_path)

	train_loader = DataLoader(dataset=train_set, batch_size=args.batch_size, shuffle=True)
	valid_loader = DataLoader(dataset=valid_set, batch_size=args.eval_batch_size, shuffle=False)
	test_loader = DataLoader(dataset=test_set, batch_size=args.eval_batch_size, shuffle=False)
	loaders = {'train_loader': train_loader, 'valid_loader': valid_loader, 'test_loader': test_loader}

	input_dim = train_set.data_tensor.size()[1]

	weight_class0 = torch.mean(train_set.target_tensor.float())
	weight_class1 = 1.0 - weight_class0
	weight = torch.FloatTensor([weight_class0, weight_class1])

	criterion = nn.CrossEntropyLoss(weight=weight)
	if args.cuda:
		criterion = criterion.cuda()

	model = MLP(input_dim=input_dim, num_hidden_layers=args.layers, hidden_dim=args.units, dropout=args.dropout)

	optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
	scheduler = ReduceLROnPlateau(optimizer, 'min')

	train_mlp(args, loaders, model, criterion, optimizer, scheduler, l1_factor=args.l1, clr_factor=args.clr)


text:
import os
import timeit
import argparse

from joblib import Parallel, delayed, cpu_count
from sklearn.preprocessing import minmax_scale, FunctionTransformer, LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.neighbors import KNeighborsClassifier

import pandas as pd
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

from algorithms.relief import Relief
from algorithms.local_search import LocalSearch

def evaluate_partition(X_train, y_train, X_test, y_test, transformer,
                       make_trace):
    # Beginning of timing zone
    t_start = timeit.default_timer()
    transformer.fit(X_train, y_train)
    X_train = transformer.transform(X_train)
    X_test = transformer.transform(X_test)
    knn = KNeighborsClassifier(n_neighbors=1)
    knn.fit(X_train, y_train)
    # End of timing zone
    t_stop = timeit.default_timer()
    # Gathering information
    accuracy = accuracy_score(knn.predict(X_test), y_test)
    time = t_stop - t_start
    reduction = transformer.reduction
    if make_trace:
        result = (accuracy, time, reduction, transformer.trace)
    else:
        result = (accuracy, time, reduction)
    return result

def pipeline(X, y, transformer, seed, make_trace, n_jobs=4):
    X = minmax_scale(X)
    kfold = StratifiedKFold(5, shuffle=True, random_state=seed)
    results = Parallel(n_jobs=n_jobs)(delayed(evaluate_partition)(
        X[train], y[train], X[test], y[test], transformer, make_trace)
        for train, test in kfold.split(X, y))
    if len(results[0]) > 3:
        accuracies, times, reductions, traces = zip(*results)
    else:
        accuracies, times, reductions = zip(*results)
        traces = []
    return np.array(accuracies), np.array(times), np.array(reductions), traces

def create_dataframe(accuracies, times, reductions):
    columns = ['Accuracy', 'Reduction', 'Aggregation', 'Time']
    index = ['Partition ' + str(i + 1) for i in range(5)]
    agreggations = (accuracies + reductions) / 2
    data = np.array([accuracies, reductions, agreggations, times]).T
    return pd.DataFrame(data=data, columns=columns, index=index)

def evaluate_algorithm(algorithm, X, Y, seed, make_trace, n_jobs):
    if algorithm == 'relief':
        transformer = Relief()
        make_trace = False
    elif algorithm == 'local-search':
        transformer = LocalSearch(seed=seed)
    else:
        make_trace = False
        transformer = FunctionTransformer(lambda x: x, validate=False)
        transformer.reduction = 0
    results = pipeline(X, Y, transformer, seed, make_trace, n_jobs)
    return create_dataframe(*results[:-1]), results[-1]

def pretty_print(dataset, algorithm, seed, results):
    summary = results.describe().loc[['mean', 'std', '50%']]
    summary.index = ['Mean', 'Std.Dev', 'Median']
    output = """
=======================================================
    %s     |     %s      |  SEED = %d
=======================================================\n%s\n\n%s
    """ % (dataset.upper(), algorithm.upper(), seed, results.to_string(),
           summary)
    return output

def generate_graphics(filename, results, traces):
    """Generates some bloxplots based on the parameter --results--
    and save it on a file. Optionally, it creates a trace line plot.
    """
    _, axes = plt.subplots(2, len(results.columns) // 2, figsize=(10, 6))
    plt.suptitle('Algorithm Results', fontsize='x-large')
    for col, axis in zip(results.columns, axes.flatten()):
        results.boxplot(column=col, ax=axis)
    plt.savefig('%s_results.png' % filename)
    plt.clf()
    if traces:
        plt.title('Local Search fitness function trace')
        for i, t in enumerate(traces):
            plt.plot(t, label='Partition %d' % (i + 1))
        plt.legend()
        plt.savefig('%s_trace.png' % filename)

def create_directory(path):
    """Creates a directory if it doesn't exist."""
    if not os.path.isdir(path):
        os.makedirs(path, exist_ok=True)

def load_dataset(dataset):
    df = None
    if dataset in DATASETS:
        df = pd.read_csv('../BIN/%s.csv' % dataset)
    elif dataset.endswith('.csv'):
        df = pd.read_csv(dataset)
    X = df.iloc[:, :-1].values
    y = df.iloc[:, -1].values
    y = LabelEncoder().fit_transform(y)
    return X, y

def get_filename(filepath):
    base = os.path.basename(filepath)
    return os.path.splitext(base)[0]

def main(dataset, algorithm, seed, trace, n_jobs, to_excel):
    X, y = load_dataset(dataset)
    dataset = get_filename(dataset)
    results, traces = evaluate_algorithm(algorithm, X, y, seed, trace, n_jobs)
    filename = 'output/%s/%s_%s' % (algorithm, dataset, seed)
    create_directory('output/%s/' % algorithm)
    generate_graphics(filename, results, traces)
    output = pretty_print(dataset, algorithm, seed, results)
    if to_excel:
        results.to_excel(filename + '.xlsx')
    print(output)

DATASETS = ['colposcopy', 'texture', 'ionosphere']
ALGORITHMS = ['relief', 'local-search', 'knn']
N_JOBS_RANGE = list(range(1, min(4, cpu_count()) + 1))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        'dataset', type=str, help='Predefined datasets or a csv file')
    parser.add_argument(
        'algorithm',
        type=str,
        choices=ALGORITHMS,
        help='Algorithm to use for feature weighting')
    parser.add_argument(
        '--seed',
        type=int,
        default=77766814,
        help='Seed to initialize the random generator')
    parser.add_argument(
        '--n_jobs',
        type=int,
        choices=N_JOBS_RANGE,
        default=1,
        help='Number of jobs to run in parallel to evaluate partitions.')
    parser.add_argument(
        '--trace', help='Generate trace for local search', action='store_true')
    parser.add_argument(
        '--to_excel', help='Dump results into xlsx file', action='store_true')
    args = vars(parser.parse_args())
    main(**args)


text:
import numpy as np
import pandas as pd
import yfinance as yf
from data_process import get_revised_yearly_return, get_revised_monthly_return
import plotly.graph_objects as go
import config

def get_returns(prices):
    return prices[-1]/prices[0]-1

def plot_yearly_diff_comparison(df_drl, df_etf, save_path):
    #return in years
    yearly_asset_return_drl, yearly_reward_return_drl = get_revised_yearly_return(df_drl)
    yearly_return_etf = df_etf.groupby([df_etf.index.year]).Close.apply(get_returns)-config.ETF_EXPENSE

    # define histogram colors
    colors_asset_drl = np.where(yearly_asset_return_drl.values > 0, '#06d6a0', '#ef476f')
    colors_reward_drl = np.where(yearly_reward_return_drl.values > 0, '#06d6a0', '#ef476f')
    colors_etf = np.where(yearly_return_etf.values > 0, '#0a9396', '#e63946')

    # plot asset diff
    fig = go.Figure()
    fig.add_trace(
        go.Bar(
            x=yearly_asset_return_drl.index,
            y=yearly_asset_return_drl.values * 100,
            name="DRL",
            marker_color=colors_asset_drl,
            text=yearly_asset_return_drl.values * 100,
            texttemplate='%{text:.2f}',
            textposition='outside'
        ),
    )
    fig.add_trace(
        go.Bar(
            x=yearly_return_etf.index,
            y=yearly_return_etf.values * 100,
            name='ETF',
            marker_color=colors_etf,
            text=yearly_return_etf.values * 100,
            texttemplate='%{text:.2f}',
            textposition='outside'
        ),
    )

    # make the figure prettier
    layout = go.Layout(
        title="Yearly asset return comparison with ETF (%)",
        plot_bgcolor='#ecf8f8',
        font_family='Monospace',
        font_color='#073b4c',
        font_size=10,
        xaxis=dict( rangeslider=dict(visible=False)),
        autosize=True,
    )

    fig.update_layout(layout)
    fig.write_image(save_path+"yearly_asset_comparison.png")

    # plot reward diff
    fig = go.Figure()
    fig.add_trace(
        go.Bar(
            x=yearly_reward_return_drl.index,
            y=yearly_reward_return_drl.values * 100,
            name="DRL",
            marker_color=colors_reward_drl,
            text=yearly_reward_return_drl.values * 100,
            texttemplate='%{text:.2f}',
            textposition='outside'
        ),
    )
    fig.add_trace(
        go.Bar(
            x=yearly_return_etf.index,
            y=yearly_return_etf.values * 100,
            name='ETF',
            marker_color=colors_etf,
            text=yearly_return_etf.values * 100,
            texttemplate='%{text:.2f}',
            textposition='outside'
        ),
    )

    # make the figure prettier
    layout = go.Layout(
        title="Yearly reward return comparison with ETF (%)",
        plot_bgcolor='#ecf8f8',
        font_family='Monospace',
        font_color='#073b4c',
        font_size=10,
        xaxis=dict( rangeslider=dict(visible=False)),
        autosize=True,
    )

    fig.update_layout(layout)
    fig.write_image(save_path+"yearly_reward_comparison.png")

def plot_monthly_heatmap(df_drl, df_etf, save_path):

    years = 6

    monthly_asset_return_drl, monthly_reward_return_drl = get_revised_monthly_return(df_drl)
    monthly_asset_return_drl = np.resize([np.nan]*10 + list(monthly_asset_return_drl.values) + [np.nan]*2, (years, 12))
    monthly_reward_return_drl = np.resize([np.nan]*10 + list(monthly_reward_return_drl.values) + [np.nan]*2, (years, 12))

    monthly_return_etf = df_etf.groupby([df_etf.index.year, df_etf.index.month]).Close.apply(get_returns)-config.ETF_EXPENSE
    monthly_return_etf = np.resize([np.nan] * 10 + list(monthly_return_etf.values) + [np.nan] * 2, (years, 12))

    months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
    years = [i for i in np.arange(2016, 2021 + 1, 1)]
    colorscale = [[0, '#ef476f'], [0.5, 'white'], [1, '#06d6a0']]

    # drl asset return heatmap
    monthly_returns = monthly_asset_return_drl
    annotations = go.Annotations()
    for n, row in enumerate(monthly_returns):
        for m, val in enumerate(row):
            annotations.append(go.Annotation(text=str(round(monthly_returns[n][m] * 100, 2)), x=months[m], y=years[n],
                                             xref='x1', yref='y1', showarrow=False))
    trace = go.Heatmap(x=months, y=years, z=monthly_returns, colorscale=colorscale, showscale=True)
    fig = go.Figure(data=go.Data([trace]))
    fig['layout'].update(
        title="DRL monthly asset returns in a heatmap (%)",
        annotations=annotations,
        xaxis=go.XAxis(ticks='', side='top'),
        yaxis=go.YAxis(ticks='', ticksuffix='  '),  # ticksuffix is a workaround to add a bit of padding
        autosize=True
    )
    fig.write_image(save_path+"heatmap_DRL_asset.png")

    # drl reward return heatmap
    monthly_returns = monthly_reward_return_drl
    annotations = go.Annotations()
    for n, row in enumerate(monthly_returns):
        for m, val in enumerate(row):
            annotations.append(go.Annotation(text=str(round(monthly_returns[n][m] * 100, 2)), x=months[m], y=years[n],
                                             xref='x1', yref='y1', showarrow=False))
    trace = go.Heatmap(x=months, y=years, z=monthly_returns, colorscale=colorscale, showscale=True)
    fig = go.Figure(data=go.Data([trace]))
    fig['layout'].update(
        title="DRL monthly reward returns in a heatmap (%)",
        annotations=annotations,
        xaxis=go.XAxis(ticks='', side='top'),
        yaxis=go.YAxis(ticks='', ticksuffix='  '),  # ticksuffix is a workaround to add a bit of padding
        autosize=True
    )
    fig.write_image(save_path+"heatmap_DRL_reward.png")

    # ETF
    monthly_returns = monthly_return_etf
    annotations = go.Annotations()
    for n, row in enumerate(monthly_returns):
        for m, val in enumerate(row):
            annotations.append(go.Annotation(text=str(round(monthly_returns[n][m] * 100, 2)), x=months[m], y=years[n],
                                             xref='x1', yref='y1', showarrow=False))
    trace = go.Heatmap(x=months, y=years, z=monthly_returns, colorscale=colorscale, showscale=True)
    fig = go.Figure(data=go.Data([trace]))
    fig['layout'].update(
        title="ETF monthly returns in a heatmap (%)",
        annotations=annotations,
        xaxis=go.XAxis(ticks='', side='top'),
        yaxis=go.YAxis(ticks='', ticksuffix='  '),  # ticksuffix is a workaround to add a bit of padding
        autosize=True
    )
    fig.write_image(save_path+"heatmap_etf.png")

    # asset diff
    monthly_returns = monthly_asset_return_drl - monthly_return_etf
    annotations = go.Annotations()
    for n, row in enumerate(monthly_returns):
        for m, val in enumerate(row):
            annotations.append(go.Annotation(text=str(round(monthly_returns[n][m] * 100, 2)), x=months[m], y=years[n],
                                             xref='x1', yref='y1', showarrow=False))
    trace = go.Heatmap(x=months, y=years, z=monthly_returns, colorscale=colorscale, showscale=True)
    fig = go.Figure(data=go.Data([trace]))
    fig['layout'].update(
        title="Monthly asset return diff with ETF in a heatmap (%)",
        annotations=annotations,
        xaxis=go.XAxis(ticks='', side='top'),
        yaxis=go.YAxis(ticks='', ticksuffix='  '),  # ticksuffix is a workaround to add a bit of padding
        autosize=True
    )
    fig.write_image(save_path+"heatmap_diff_asset.png")

    # reward diff
    monthly_returns = monthly_reward_return_drl - monthly_return_etf
    annotations = go.Annotations()
    for n, row in enumerate(monthly_returns):
        for m, val in enumerate(row):
            annotations.append(go.Annotation(text=str(round(monthly_returns[n][m] * 100, 2)), x=months[m], y=years[n],
                                             xref='x1', yref='y1', showarrow=False))
    trace = go.Heatmap(x=months, y=years, z=monthly_returns, colorscale=colorscale, showscale=True)
    fig = go.Figure(data=go.Data([trace]))
    fig['layout'].update(
        title="Monthly reward return diff with ETF in a heatmap (%)",
        annotations=annotations,
        xaxis=go.XAxis(ticks='', side='top'),
        yaxis=go.YAxis(ticks='', ticksuffix='  '),  # ticksuffix is a workaround to add a bit of padding
        autosize=True
    )
    fig.write_image(save_path+"heatmap_diff_reward.png")

def plot_trading_behavior(df, ticker, start, end, save_path):

    candelstick = yf.Ticker(ticker).history(start=start,end=end)[["Open","High","Low","Close"]]
    candelstick.columns = [c.lower() for c in candelstick.columns]

    trading = df[(df["ticker"]==ticker)&(df["date"]>=start)&(df["date"]<=end)]

    fig = go.Figure()
    fig.add_trace(go.Candlestick(
        x=candelstick.index,
        open=candelstick.open,
        high=candelstick.high,
        low=candelstick.low,
        close=candelstick.close,
        increasing_line_color='#ef476f',
        decreasing_line_color='#06d6a0',
        showlegend=False
    ),
                 )
    buying_signal = trading[(trading.signal == 1)&(trading.transaction_cost>0)]
    selling_signal = trading[(trading.signal == -1)&(trading.transaction_cost>0)]

    fig.add_trace(
        go.Scatter(
            x=buying_signal.date,
            y=buying_signal.transaction_price,
            marker = dict(
                color='#073b4c',
                size=10,
                line=dict(
                    color='#118ab2',
                    width=2
                ),
                symbol='triangle-up'

            ),
            mode = "markers",
            name = "Buy",
            showlegend=True,
        ),
    )

    fig.add_trace(
        go.Scatter(
            x=selling_signal.date,
            y=selling_signal.transaction_price,
            marker = dict(
                color='#fb5607',
                size=10,
                line=dict(
                    color='#ffbe0b',
                    width=2
                ),
                symbol='triangle-down'

            ),
            mode = "markers",
            name = "Sell",
            showlegend=True,
        ),
    )
    layout = go.Layout(
        plot_bgcolor='#ecf8f8',
        font_family='Monospace',
        font_color='#073b4c',
        font_size=10,
        xaxis=dict(
            rangeslider=dict(visible=False)
        ),
        autosize=True
    )

    fig.update_xaxes(
        rangebreaks=[
            dict(bounds=['sat', 'mon'])
        ]
    )
    fig.update_layout(layout)
    fig.write_image(save_path+ticker+"_"+start+"_"+end+"_trading_signal.png")


text:
import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
import torch.nn.functional as F
import numpy as np
from thirdparty.two_branches_attention import DotAttention
from handlers.output_handler import FileHandler

class AttentiveDecoder(nn.Module):
    """ Attentive decoder """
    def __init__(self, attn_model, hidden_size: int, output_size: int, embedding_layer,
                 label_embedding_layer,
                 bidirectional: bool = False,
                 n_layers: int = 1, dropout: float = 0.1,
                 use_self_att = False, use_label: bool = False, use_input_feeding: bool = False):
        super(AttentiveDecoder, self).__init__()
        FileHandler.myprint("Using Attention type : %s" % attn_model)
        self.attn_model = attn_model
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.bidirectional = bidirectional

        self.embedding = embedding_layer
        embedding_size = embedding_layer.embedding_dim

        self.label_embedding = label_embedding_layer
        self.embedding_size = embedding_size

        self.use_label = use_label  # for label embedding
        self.label_embedding_size = 0
        self.embedding_dropout = nn.Dropout(dropout)
        self.input_feeding_size = 0
        self.use_input_feeding = use_input_feeding
        if self.use_input_feeding:
            self.input_feeding_size = self.hidden_size # we want it to be same as hidden_size

        self.gru = nn.GRU(self.embedding_size + self.input_feeding_size, self.hidden_size,
                          batch_first = True, bidirectional = self.bidirectional, num_layers = self.n_layers)

        num_direct = 1
        self.concat = nn.Linear(hidden_size * 2 * num_direct, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)
        self.atten_faster = DotAttention(attn_model, num_direct * hidden_size, hidden_size)

    def forward(self, input_seq, last_hidden, encoder_outputs, encoder_labels, input_feeding_prev_time_step):
        """
        Running decoder step by step
        :param input_seq: shape (B, )
        :param last_hidden: including (hidden_state, cell_state)
        :param encoder_outputs: (B, L, H)
        :param input_feeding_prev_time_step: shape (B, self.input_feeding_size)
        :return: output shape (B, Output_size),
        (last_hidden_state, last_cell_state)
        attn_weights with shape (B, S=1, time_steps_in_encoder)
        """
        embedded = self.embedding(input_seq)
        embedded = self.embedding_dropout(embedded) # (batch_size, embedding_size)

        if self.use_input_feeding:
            embedded = torch.cat([embedded, input_feeding_prev_time_step], dim = -1)

        # embedded = embedded.view(1, batch_size, self.embedding_size)  # S=1 x B x N
        embedded = embedded.unsqueeze(1)  # Shape = B x 1 x embedding_size, indicating seq_len = 1.

        # Get current hidden state from input word and last hidden state
        # rnn_output, hidden = self.gru(embedded, last_hidden)
        # We have rnn_output.shape = (batch_size, seq_len = 1, num_direct * hidden_size = 1 * hidden_size)
        last_hidden = last_hidden.contiguous()
        rnn_output, last_hidden_state = self.gru(embedded, last_hidden)

        # Calculate attention from current RNN state and all encoder outputs;
        # apply to encoder outputs to get weighted average
        attn_weights = self.atten_faster(rnn_output, encoder_outputs)  # (B, seq_len = 1, time_steps_in_encoder)
        context = attn_weights.bmm(encoder_outputs)  # shape_of_context = (B, decoder_seq_len = 1, hidden_size)

        # Attentional vector using the RNN hidden state and context vector
        # concatenated together (Luong eq. 5)
        rnn_output = rnn_output.squeeze(1)  # B x S=1 x N -> B x N
        context = context.squeeze(1)  # B x S=1 x N -> B x N
        concat_input = torch.cat((rnn_output, context), dim = -1)
        concat_output = F.tanh(self.concat(concat_input))

        if self.use_input_feeding: input_feeding_prev_time_step = concat_output
        # Finally predict next token (Luong eq. 6, without softmax)
        output = self.out(concat_output)

        # Return final output, hidden state, and attention weights (for visualization)
        return output, last_hidden_state, attn_weights, input_feeding_prev_time_step


text:
from pathlib import Path
import argparse
import pickle

import pandas as pd
import numpy as np

def softmax(x):
    """
    >>> res = softmax(np.array([0, 200, 10]))
    >>> np.sum(res)
    1.0
    >>> np.all(np.abs(res - np.array([0, 1, 0])) < 0.0001)
    True
    >>> res = softmax(np.array([[0, 200, 10], [0, 10, 200], [200, 0, 10]]))
    >>> np.sum(res, axis=1)
    array([ 1.,  1.,  1.])
    >>> res = softmax(np.array([[0, 200, 10], [0, 10, 200]]))
    >>> np.sum(res, axis=1)
    array([ 1.,  1.])
    """
    if x.ndim == 1:
        x = x.reshape((1, -1))
    max_x = np.max(x, axis=1).reshape((-1, 1))
    exp_x = np.exp(x - max_x)
    return exp_x / np.sum(exp_x, axis=1).reshape((-1, 1))

def fuse_scores(scores_dict):
    modalities_combinations = [('rgb', 'flow'), ('rgb', 'spec'),
                               ('flow', 'spec'), ('rgb', 'flow', 'spec')]

    fused_scores = {}
    for mod_comb in modalities_combinations:
        name = '_'.join(mod_comb)
        fused_scores[name] = {'scores': {}}
        for task in ['verb', 'noun']:
            scores_list = [scores_dict[m]['scores'][task] for m in mod_comb]
            scores_list = [softmax(scores.mean(axis=(1, 2))) for scores in scores_list]
            fused_scores[name]['scores'][task] = np.mean(scores_list, axis=0)

    return fused_scores

def main(args):

    for split in ['seen', 'unseen']:
        rgb_scores = pd.read_pickle(args.rgb / ('test_' + split + '.pkl'))
        flow_scores = pd.read_pickle(args.flow / ('test_' + split + '.pkl'))
        spec_scores = pd.read_pickle(args.spec / ('test_' + split + '.pkl'))

        scores_dict = {'rgb': rgb_scores, 'flow': flow_scores, 'spec': spec_scores}
        fused_scores = fuse_scores(scores_dict)

        for key in fused_scores.keys():
            output_dir = args.scores_root / key / ('test_' + split + '.pkl')
            output_dir.parent.mkdir(parents=True, exist_ok=True)
            with open(output_dir, 'wb') as f:
                pickle.dump(fused_scores[key], f, protocol=pickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('scores_root', type=Path)
    parser.add_argument('--rgb', type=Path, help='Directory of the RGB scores')
    parser.add_argument('--flow', type=Path, help='Directory of the Flow scores')
    parser.add_argument('--spec', type=Path, help='Directory of the Spectrogram scores')

    args = parser.parse_args()
    main(args)


text:
import torch
import cma
import numpy as np

from pix2latent.utils.misc import HiddenPrints, cprint
from pix2latent.utils.image import binarize

class _BaseCMAOptimizer():
    """
    Base template for CMA optimization. Should be used jointly with
    _BaseOptimizer.

    NOTE:
        Since the seeds are pre-determined by the CMA code-base, support
        to optimize multiple variables using CMA is disabled. NeverGrad
        optimizers should be able to support this feature.
    """

    def __init__(self):
        self.num_samples = -1
        self.cma_optimizers = {}
        self._sampled = {}
        return

    @torch.no_grad()
    def setup_cma(self, var_manager):
        """
        initializes CMA for variables that have the attribute `grad_free`

        Args
            var_manager (VariableManger): instance of the variable manager

        """

        for var_name, var_dict in var_manager.variable_info.items():

            if var_dict['grad_free'] is False:
                continue

            if type(var_dict['grad_free']) == tuple:
                mu, sigma = var_dict['grad_free']

                if mu is None:
                    mu = np.zeros(var_dict['shape'])

                if sigma is None:
                    sigma = 1.

                cma_opt = CMA(mu, sigma=sigma)

            else:
                mu = np.zeros(var_dict['shape'])
                cma_opt = CMA(mu, sigma=1.0)

            self.cma_optimizers[(var_dict['var_type'], var_name)] = cma_opt

            self.num_samples = max(self.num_samples, cma_opt.batch_size())

        cprint('(cma-es) number of samples: {}'.format(self.num_samples), 'y')

        assert len(self.cma_optimizers.keys()) == 1, \
           'currently only a single input variable can be optimized via CMA '+\
           'but got: {}'.format(self.cma_optimizers.keys())
        return

    @torch.no_grad()
    def cma_init(self, var_manager):
        """
        initializes the provided variable from CMA

        Args:
            var_manager (VariableManger): instance of the variable manager
        """

        vars = var_manager.initialize(num_samples=self.num_samples)

        for (var_type, var_name), cma_opt in self.cma_optimizers.items():
            cma_data = cma_opt.ask()

            for i, d in enumerate(cma_data):
                vars[var_type][var_name].data[i].data = \
                            torch.Tensor(d).data.type_as(
                                vars[var_type][var_name].data[i].data)

            self._sampled[(var_type, var_name)] = cma_data

        return vars

    @torch.no_grad()
    def cma_update(self, variables, loss=None, inverted_loss=False):
        """
        Updates CMA distribution either with the provided loss or loss that
        is recomputed.

        Args:
            variables (dict): a dictionary instance generated from the
                variable manager.
            loss (array or list): a 1-dimensional array or list consisting of
                losses corresponding to each sample. If the loss is not
                provided, uses the variables to recompute the loss.
                [Default: None]
            inverted_loss (bool): if True, the loss is computed after inverting
                the generated images back to the original target. For example
                this is used to compute the loss on the original target.
                [Default: False]
        """

        for (var_type, var_name), cma_opt in self.cma_optimizers.items():

            cma_data = self._sampled[(var_type, var_name)]

            if loss is None:
                out, loss, _ = self.step(variables, optimize=False)

            if inverted_loss and hasattr(variables, 'transform'):

                target_type = \
                        self.var_manager.variable_info['target']['var_type']
                weight_type = \
                        self.var_manager.variable_info['weight']['var_type']

                target = self.var_manager.variable_info['target']['default']
                weight = self.var_manager.variable_info['weight']['default']

                target = target.unsqueeze(0).type_as(out)
                weight = weight.unsqueeze(0).type_as(out)

                t_fn = self.transform_fns['target']['fn']
                t_param = torch.stack(variables.transform.t.data)
                out = t_fn(out, t_param, invert=True)

                loss = self.loss_fn(out, target, binarize(weight))
                loss = loss.cpu().detach().numpy()

            cma_opt.tell(cma_data, loss)
        return loss

class CMA():
    def __init__(self, mu=128 * [0], sigma=1.0, seed=None):
        """
        Wrapper class function for PyCMA. Since CMA does not allow for 1
        variable optimization, we will duplicate it to be 2 variables and
        only compute CMA on the first variable.

        Args:
            mu (list): a 1D array of CMA means. [Default: 128 * [0]]
            sigma (float): sigma for the CMA. This sigma is shared on all seeds
                [Default: 1.0]
            seed (int): seed used for reproducibility. [Default: None]

        Attribute:
            is_scalar (boolean): hack to go around the fact that CMA does not
                support scalar optimizer.
            cma (CMA): core CMA optimizer
        """

        options = {}

        if seed is not None:
            options['seed'] = seed
        self.is_scalar = False

        if len(mu) == 1:
            mu = list(mu) * 2
            options['CMA_on'] = 0
            self.is_scalar = True

        with HiddenPrints():
            self.cma = cma.CMAEvolutionStrategy(mu, sigma, options)
        return

    def batch_size(self):
        """ Returns the required batch size for CMA """
        return self.cma.sp.popsize

    def ask(self, batch_size=None):
        """ Asks for samples to evaluate. batch_size must be None to train """
        x = np.array(self.cma.ask(batch_size))

        if self.is_scalar:
            self._x = x
            self._x_proxy = x[:, :1]
            return self._x_proxy

        return x

    def tell(self, x, y):
        """ Apply CMA update """

        if self.is_scalar:
            assert x is self._x_proxy
            return self.cma.tell(self._x, y)

        return self.cma.tell(x, y)

    def mean(self):
        """ Returns the mean of the current CMA distribution """

        x = self.cma.mean

        if self.is_scalar:
            return x[:1]

        return x


text:
import numpy as np

class SeqReplayBuffer:
    buffer_type = "seq_vanilla"

    def __init__(
        self,
        max_replay_buffer_size,
        observation_dim,
        action_dim,
        sampled_seq_len: int,
        sample_weight_baseline: float,
        **kwargs
    ):
        """
        this buffer is used for sequence/trajectory/episode:
                it stored the whole sequence
                into the buffer (not transition), and can sample (sub)sequences
                that has 3D shape (sampled_seq_len, batch_size, dim)
                based on some rules below.
        it still uses 2D size as normal (max_replay_buffer_size, dim)
                but tracks the sequences

        NOTE: it save observations twice, so it is vanilla version of seq replay buffer,
            sufficient to vector-based observations, as RAM is not the bottleneck
        """
        self._max_replay_buffer_size = max_replay_buffer_size
        self._observation_dim = observation_dim
        self._action_dim = action_dim

        self._observations = np.zeros(
            (max_replay_buffer_size, observation_dim), dtype=np.float32
        )
        self._next_observations = np.zeros(
            (max_replay_buffer_size, observation_dim), dtype=np.float32
        )

        self._actions = np.zeros((max_replay_buffer_size, action_dim), dtype=np.float32)
        self._rewards = np.zeros((max_replay_buffer_size, 1), dtype=np.float32)

        # terminals are "done" signals, useful for policy training
        # for each trajectory, it has single 1 like 0000001000 for reaching goal or early stopping
        # 	or simply 0s for timing out.
        # NOTE: so we cannot use terminals to determine the trajectory boundary!
        self._terminals = np.zeros((max_replay_buffer_size, 1), dtype=np.uint8)

        # NOTE: valid_starts are (internal) masks which is 1 (or postive number as weight)
        # 	if we can SAMPLE the (sub)sequence FROM this index else 0.
        # For each trajectory, the first index has valid_start as 1 (or postive number),
        # 	the LAST sampled_seq_len indices are 0s, and the middle ones are 1s (or positive numbers)
        # 	That is to say, if its length <= sampled_seq_len, then the valid_starts looks like 100000000
        # 	else looks like 11111000000 (have sampled_seq_len - 1 zeros)
        # See _compute_valid_starts function for details
        self._valid_starts = np.zeros((max_replay_buffer_size), dtype=np.float32)

        assert sampled_seq_len >= 2
        assert sample_weight_baseline >= 0.0
        self._sampled_seq_len = sampled_seq_len
        self._sample_weight_baseline = sample_weight_baseline

        self.clear()

        RAM = 0.0
        for name, var in vars(self).items():
            if isinstance(var, np.ndarray):
                RAM += var.nbytes
        print(f"buffer RAM usage: {RAM / 1024 ** 3 :.2f} GB")

    def size(self):
        return self._size

    def clear(self):
        self._top = 0  # trajectory level (first dim in 3D buffer)
        self._size = 0  # trajectory level (first dim in 3D buffer)

    def add_episode(self, observations, actions, rewards, terminals, next_observations):
        """
        NOTE: must add one whole episode/sequence/trajectory,
                        not some partial transitions
        the length of different episode can vary, but must be greater than 2
                so that the end of valid_starts is 0.

        all the inputs have 2D shape of (L, dim)
        """
        assert (
            observations.shape[0]
            == actions.shape[0]
            == rewards.shape[0]
            == terminals.shape[0]
            == next_observations.shape[0]
            >= 2
        )

        seq_len = observations.shape[0]  # L
        indices = list(
            np.arange(self._top, self._top + seq_len) % self._max_replay_buffer_size
        )

        self._observations[indices] = observations
        self._actions[indices] = actions
        self._rewards[indices] = rewards
        self._terminals[indices] = terminals
        self._next_observations[indices] = next_observations

        self._valid_starts[indices] = self._compute_valid_starts(seq_len)

        self._top = (self._top + seq_len) % self._max_replay_buffer_size
        self._size = min(self._size + seq_len, self._max_replay_buffer_size)

    def _compute_valid_starts(self, seq_len):
        valid_starts = np.ones((seq_len), dtype=float)

        num_valid_starts = float(max(1.0, seq_len - self._sampled_seq_len + 1.0))

        # compute weights: baseline + num_of_can_sampled_indices
        total_weights = self._sample_weight_baseline + num_valid_starts

        # now each item has even weight, if baseline is 0.0, then it's 1s
        valid_starts *= total_weights / num_valid_starts

        # set the num_valid_starts: indices are zeros
        valid_starts[int(num_valid_starts) :] = 0.0

        return valid_starts

    def random_episodes(self, batch_size):
        """
        return each item has 3D shape (sampled_seq_len, batch_size, dim)
        """
        sampled_episode_starts = self._sample_indices(batch_size)  # (B,)

        # get sequential indices
        indices = []
        for start in sampled_episode_starts:  # small loop
            end = start + self._sampled_seq_len  # continuous + T
            indices += list(np.arange(start, end) % self._max_replay_buffer_size)

        # extract data
        batch = self._sample_data(indices)
        # each item has 2D shape (num_episodes * sampled_seq_len, dim)

        # generate masks (B, T)
        masks = self._generate_masks(indices, batch_size)
        batch["mask"] = masks

        for k in batch.keys():
            batch[k] = (
                batch[k]
                .reshape(batch_size, self._sampled_seq_len, -1)
                .transpose(1, 0, 2)
            )

        return batch

    def _sample_indices(self, batch_size):
        # self._top points at the start of a new sequence
        # self._top - 1 is the end of the recently stored sequence
        valid_starts_indices = np.where(self._valid_starts > 0.0)[0]

        sample_weights = np.copy(self._valid_starts[valid_starts_indices])
        # normalize to probability distribution
        sample_weights /= sample_weights.sum()

        return np.random.choice(valid_starts_indices, size=batch_size, p=sample_weights)

    def _sample_data(self, indices):
        return dict(
            obs=self._observations[indices],
            act=self._actions[indices],
            rew=self._rewards[indices],
            term=self._terminals[indices],
            obs2=self._next_observations[indices],
        )

    def _generate_masks(self, indices, batch_size):
        """
        input: sampled_indices list of len B*T
        output: masks (B, T)
        """

        # get valid_starts of sampled sequences (B, T)
        # each row starts with a postive number, like 11111000, or 10000011, or 1s
        sampled_seq_valids = np.copy(self._valid_starts[indices]).reshape(
            batch_size, self._sampled_seq_len
        )
        sampled_seq_valids[sampled_seq_valids > 0.0] = 1.0  # binarize

        # build masks
        masks = np.ones_like(sampled_seq_valids, dtype=float)  # (B, T), default is 1

        # we want to find the boundary (ending) of sampled sequences
        # 	i.e. the FIRST 1 (positive number) after 0
        # 	this is important for varying length episodes
        # the boundary (ending) appears at the FIRST -1 in diff
        diff = sampled_seq_valids[:, :-1] - sampled_seq_valids[:, 1:]  # (B, T-1)
        # add 1s into the first column
        diff = np.concatenate([np.ones((batch_size, 1)), diff], axis=1)  # (B, T)

        # special case: the sampled sequence cannot cross self._top
        indices_array = np.array(indices).reshape(
            batch_size, self._sampled_seq_len
        )  # (B,T)
        # set the top as -1.0 as invalid starts
        diff[indices_array == self._top] = -1.0

        # now the start of next episode appears at the FIRST -1 in diff
        invalid_starts_b, invalid_starts_t = np.where(
            diff == -1.0
        )  # (1D array in batch dim, 1D array in seq dim)
        invalid_indices_b = []
        invalid_indices_t = []
        last_batch_index = -1

        for batch_index, start_index in zip(invalid_starts_b, invalid_starts_t):
            if batch_index == last_batch_index:
                # for same batch_idx, we only care the first appearance of -1
                continue
            last_batch_index = batch_index

            invalid_indices = list(
                np.arange(start_index, self._sampled_seq_len)
            )  # to the end
            # extend to the list
            invalid_indices_b += [batch_index] * len(invalid_indices)
            invalid_indices_t += invalid_indices

        # set invalids in the masks
        masks[invalid_indices_b, invalid_indices_t] = 0.0

        return masks

if __name__ == "__main__":
    seq_len = 10
    baseline = 1.0
    buffer = SeqReplayBuffer(100, 2, 2, seq_len, baseline)
    for l in range(seq_len - 2, seq_len + 2):
        print(l)
        assert buffer._compute_valid_starts(l)[0] > 0.0
        print(buffer._compute_valid_starts(l))
    for e in range(10):
        data = np.zeros((10, 2))
        buffer.add_episode(
            np.zeros((11, 2)),
            np.zeros((11, 2)),
            np.zeros((11, 1)),
            np.zeros((11, 1)),
            np.zeros((11, 2)),
        )
    print(buffer._size, buffer._valid_starts)

max_stars_repo_name: hai-h-nguyen/pomdp-baselines

max_issues_repo_name: hai-h-nguyen/pomdp-baselines

max_forks_repo_name: hai-h-nguyen/pomdp-baselines


text:
import streamlit as st
from PIL import Image
import numpy as np
from Utilss import Select
from tensorflow.keras import models
import tensorflow as tf

# NM_model = tf.keras.models.load_model('models/CoronaCT_V1')
# Malaria_model = tf.keras.models.load_model('models/MLRA_V1.h5')

Pred_cls = ["Pneumonia", "Malaria", "Brain Tumor"]

p_cls = st.selectbox('Choose', Pred_cls)

@st.cache(allow_output_mutation=True)
def load_model_BRAIN_TUMOR():
    model = tf.keras.models.load_model('models/brain_Tumor_V2.h5')

    return model

@st.cache(allow_output_mutation=True)
def load_model_CORONA():
    model = tf.keras.models.load_model('models/CoronaCT_V1.h5')

    return model

@st.cache(allow_output_mutation=True)
def load_model_MLRA():
    model = tf.keras.models.load_model('models/MLRA_V1.h5')

    return model

model_BT = load_model_BRAIN_TUMOR()
model_CORONA = load_model_CORONA()
model_MLRA = load_model_MLRA()

Pred_cls = ["Pneumonia", "Malaria", "Brain Tumor"]

def ChooseModel(value):
    if value == 'Pneumonia':
        return model_CORONA

    elif value == 'Malaria':
        return model_MLRA

    else:
        return model_BT

mainModel = ChooseModel(p_cls)

img_config = Select(p_cls)

st.write(f" Prediction of {p_cls} ")

# st.set_option('deprecation.showfileUploaderEncoding', False)
IMG_FILE = st.file_uploader("Please Upload Image here....", type=['jpg', 'jpeg', 'png'])

if IMG_FILE:

    img = Image.open(IMG_FILE)

    img = np.array(img.convert("RGB").resize(img_config['size']))
    img = img / 255.0
    img = np.expand_dims(img, 0)

    pred = mainModel.predict(img)

    result = np.argmax(pred)
    score = round(np.max(pred) * 100, 2)

    result2 = img_config['cls_names'][result]

    mystr = f" {result2}  {score}% "
    st.success(mystr)
    st.image(IMG_FILE, use_column_width=True, caption=mystr)


text:
'''
Inverse kinematics with friction cone constraint.

min Dq, f || q - q0 ||**2 + || f ||**2

subject to  q = pin.integrate(q0, Dq)
            sum(f_i) == weight
            com_z >= avg(p_z) # com height higher than the average height of the feet
            f_i X (com_pos - p_i) == 0 # zero angular momentum at the com

            # The friction cone constraint can be one of the following:
            0)  || f_t ||**2 <= mu * || f_n ||**2   # in this case mu = 1. f_t and f_n are the tangential and orthogonal component of the contact force
            1)  || f @ k.T @ k - k @ k.T @ f ||**2 <= k.T @ f @ f.T @ k @k.T @ k    # k is the vector normal to the ground, while f is the vector of contact force
            2)  || f.T @ k ||**2 >= (cos(alpha_k))**2 || f ||**2 * || k ||**2   # here alpha_k is the angle of the friction cone, and it's set to 45 degree,
                                                                                which is equivalent to mu = 1

'''

import matplotlib
import pinocchio as pin
from pinocchio import casadi as cpin
import casadi
import numpy as np
import example_robot_data as robex
import matplotlib.pyplot as plt
from pinocchio.visualize import GepettoVisualizer

plt.style.use('seaborn')

### LOAD AND DISPLAY PENDULUM
robot = robex.load('solo12')
model = robot.model
data = model.createData()

# Either 0, 1 or 2 This is used to choose between the 3 methods
# They are equivalent, but the convergence is slower for some of them
FRICTION_CONE_CONTRAINT_TYPE = 2

try:
    viz = pin.visualize.GepettoVisualizer(robot.model,robot.collision_model,robot.visual_model)
    viz.initViewer()
    viz.loadViewerModel()
    viz.display(robot.q0)
except:
    viz=None

def ground(xy):
    #return (xy[0]/5+xy[1]/8) * 0 # Flat terrain
    #return (xy[0]/5+xy[1]/8)
    #return xy[0]/1+xy[1]/1
    return (np.sin(xy[0]*3)/5 + np.cos(xy[1]**2*3)/20 + np.sin(xy[1]*xy[0]*5)/10)
    #return xy[0]**2/10 - xy[1]**2/20 + xy[0]*xy[1]/15 + .3*xy[0] + .15*xy[1]

def vizGround(viz,elevation,space,name='ground',color=[1,1,1,1]):
    space = 1e-1
    gv = viz.viewer.gui
    for i,x in enumerate(np.arange(-1,1,space)):
        gv.deleteNode(f'world/pinocchio/visuals/{name}_cx{i}',True)
        gv.addCurve(f'world/pinocchio/visuals/{name}_cx{i}',
                    [ [x,y,elevation([x,y])] for y in np.arange(-1,1,space) ], color)
    for i,y in enumerate(np.arange(-1,1,space)):
        gv.deleteNode(f'world/pinocchio/visuals/{name}_cy{i}',True)
        gv.addCurve(f'world/pinocchio/visuals/{name}_cy{i}',
                    [ [x,y,elevation([x,y])] for x in np.arange(-1,1,space) ], color)

if viz is not None: vizGround(viz,ground,.1,color=[1,1,1,1])

cxyz = casadi.SX.sym("xyz",3,1)

groundNormal = casadi.Function('groundNormal',[cxyz],[casadi.gradient(cxyz[2]-ground(cxyz[:2]),cxyz)])

feet = { idx: f.name for idx,f in enumerate(model.frames) if 'FOOT' in f.name }
mass = sum( [ y.mass for y in model.inertias ] )
grav = np.linalg.norm(model.gravity.linear)

### ACTION MODEL
class MX2SX:
    length = .3  # pendulum elongated dimension
    def __init__(self,model):
        self.cmodel = cmodel = cpin.Model(model)
        self.cdata = cdata = cmodel.createData()

        cq = casadi.SX.sym("q",model.nq,1)
        cdq = casadi.SX.sym("dq",model.nv,1)

        cpin.framesForwardKinematics(self.cmodel,self.cdata,cq)
        self.feet = \
            { idx: casadi.Function(f'{name}_pos', [cq], [cdata.oMf[idx].translation])  for idx,name in feet.items() }

        self.com = casadi.Function('com',[cq],[cpin.centerOfMass(cmodel,cdata,cq)])

        self.integrate = casadi.Function('integrate',[ cq,cdq ],[cpin.integrate(cmodel,cq,cdq) ])

### PROBLEM
opti = casadi.Opti()
mx2sx = MX2SX(model)

# Decision variables
# Note that the contact forces are optimization variables
dq = opti.variable(model.nv)
q = mx2sx.integrate(robot.q0,dq)
fs = [ opti.variable(3) for _ in feet.values() ]

# constraint the com_xy position
opti.subject_to(mx2sx.com(q)[:2] == np.array([.1,.2]))
rq = q[7:]-robot.q0[7:]
totalcost = rq.T@rq

for f in mx2sx.feet.values():
    opti.subject_to( f(q)[2] == ground(f(q)[:2]) )  ### Foot in contact with the ground

torque = 0
com = mx2sx.com(q)
for idx,force in zip(mx2sx.feet.keys(),fs):
    pos = mx2sx.feet[idx](q)
    perp = groundNormal(pos)

    if (FRICTION_CONE_CONTRAINT_TYPE == 0):
            normal = perp/ casadi.norm_2(perp)
            fn = force.T@normal
            ft = force - fn*normal
            opti.subject_to(  ft.T@ft <= fn.T@fn  )

    elif (FRICTION_CONE_CONTRAINT_TYPE == 1):
        fp = (force@perp.T@perp- perp@perp.T@force)
        opti.subject_to( fp.T@fp <=  (perp.T@force@force.T@perp@perp.T@perp))

    elif (FRICTION_CONE_CONTRAINT_TYPE == 2):
        opti.subject_to( (force.T@perp)@perp.T@force >= 1/2 * (force.T@force) * (perp.T@perp))
    else:
        raise ValueError("Please select a valid type of friction constraint")

    torque += casadi.cross(force,pos-com)

opti.subject_to( sum(fs) == np.array([ 0,0,mass*grav ]))  # Sum of forces is weight
opti.subject_to( torque == 0 )  # Sum of torques around COM is 0
totalcost += sum( [ f.T@f for f in fs ] )/10 # Add penalty on forces

# Center of mass above the feet
opti.subject_to( mx2sx.com(q)[2] >= (sum([ f(q)[2] for f in mx2sx.feet.values() ]))/len(feet))

### SOLVE
opti.minimize(totalcost)
opti.solver("ipopt") # set numerical backend
sol = opti.solve()
qopt = opti.value(q)
dqopt = opti.value(dq)
try:
    fs_opt = [ opti.value(f) for f in fs ]
except:
    fs_opt = [ np.zeros(3) for _ in feet ]
positions = [ np.array( p(qopt) ).T[0] for p in mx2sx.feet.values() ]
perps = [ np.array(groundNormal(p)).T[0] for p in positions ]
normals = [ n/np.linalg.norm(n) for n in perps ]

if viz is not None: gv=viz.viewer.gui
for idx,force in zip(mx2sx.feet.keys(),fs_opt):
    #opti.subject_to( f[:2].T@f[:2] <= f[2]**2 )
    pos = np.array(mx2sx.feet[idx](qopt))
    perp = np.array(groundNormal(pos))
    print(perp.T)
    normal = perp/(perp.T@perp)

    if viz:
        gv.deleteNode(f'world/pinocchio/visuals/normal{idx}',True)
        gv.addCurve(f'world/pinocchio/visuals/normal{idx}', [ list(pos.flat), list((pos+normal).flat) ] , [ 0,0,1,1 ])

        gv.deleteNode(f'world/pinocchio/visuals/force{idx}',True)
        gv.addCurve(f'world/pinocchio/visuals/force{idx}', [ list(pos.flat), list(pos.flat+force/mass/grav) ] , [ 1,0,0,1 ])

if viz is not None:
    viz.display(qopt)


text:
# pylint: skip-file
import pytest
import numpy as np
import pyomo.environ as aml
from suspect.convexity import Convexity
from suspect.expression import ExpressionType
from galini.pyomo import problem_from_pyomo_model
from galini.suspect import ProblemContext
from galini.expression_relaxation.concave import UnivariateConcaveExpressionRelaxation

@pytest.fixture
def problem():
    m = aml.ConcreteModel()

    m.x = aml.Var(bounds=(2, 20))
    m.y = aml.Var(bounds=(3, 30))
    m.z = aml.Var(bounds=(4, 20))

    m.obj = aml.Objective(expr=m.x)

    m.univariate_concave = aml.Constraint(expr=aml.sqrt(m.x) >= 0)
    m.univariate_concave_2 = aml.Constraint(expr=aml.sqrt(3.0*m.x + 2.0) >= 0)
    m.univariate_concave_3 = aml.Constraint(expr=aml.sqrt(aml.log(3*m.x + 2)) >= 0)
    m.univariate_nonconcave = aml.Constraint(expr=aml.sin(m.x) >= 0)
    m.univariate_nonconcave_2 = aml.Constraint(expr=m.x**3 >= 0)
    m.univariate_nonconcave_3 = aml.Constraint(expr=3**m.x >= 0)

    m.nonunivariate = aml.Constraint(expr=m.x**m.y >= 0)
    m.nonunivariate_2 = aml.Constraint(expr=m.x + m.y + m.z >= 0)
    m.nonunivariate_3 = aml.Constraint(expr=aml.sin(m.x + aml.log(m.y)) >= 0)

    return problem_from_pyomo_model(m)

class TestUnivariateConcaveUnderestimator:
    @pytest.mark.parametrize('constraint_name',
                             ['univariate_concave', 'univariate_concave_2', 'univariate_concave_3'])
    def test_univariate_concave(self, problem, constraint_name):
        ctx = ProblemContext(problem)
        r = UnivariateConcaveExpressionRelaxation()

        constraint_expr = problem.constraint(constraint_name).root_expr

        ctx.set_convexity(constraint_expr, Convexity.Concave)

        assert r.can_relax(problem, constraint_expr, ctx)

        result = r.relax(problem, constraint_expr, ctx)

        assert result.constraints == []

    @pytest.mark.parametrize('name',
                             ['univariate_nonconcave', 'univariate_nonconcave_2',
                              'univariate_nonconcave_3'])
    def test_is_univariate(self, problem, name):
        r = UnivariateConcaveExpressionRelaxation()
        constraint_expr = problem.constraint(name).root_expr
        assert r.is_univariate(constraint_expr)

    @pytest.mark.parametrize('name',
                             ['nonunivariate', 'nonunivariate_2', 'nonunivariate_3'])
    def test_is_not_univariate(self, problem, name):
        r = UnivariateConcaveExpressionRelaxation()
        constraint_expr = problem.constraint(name).root_expr
        assert not r.is_univariate(constraint_expr)

max_stars_repo_name: michaelbynum/galini

max_issues_repo_name: JackaChou/galini

max_forks_repo_name: JackaChou/galini

reason: import numpy,import pyomo


text:
# created at 2018-05-11

# updated at 2018-08-24
# 增加录入多张人脸到CSV的功能

# By        coneypo
# Blog:     http://www.cnblogs.com/AdaminXie
# GitHub:   https://github.com/coneypo/Dlib_face_recognition_from_camera

#   return_128d_features()          获取某张图像的128d特征
#   write_into_csv()                将某个文件夹中的图像读取特征并写入csv
#   compute_the_mean()              从csv中读取128d特征，并计算特征均值

import cv2
import os
import dlib
from skimage import io
import csv
import numpy as np
import pandas as pd

path_faces_rd = "F:/code/python/P_dlib_face_reco/data/faces_from_camera/"
path_csv = "F:/code/python/P_dlib_face_reco/data/csvs_from_camera/"

# detector to find the faces
detector = dlib.get_frontal_face_detector()

# shape predictor to find the face landmarks
predictor = dlib.shape_predictor("shape_predictor_5_face_landmarks.dat")

# face recognition model, the object maps human faces into 128D vectors
facerec = dlib.face_recognition_model_v1("dlib_face_recognition_resnet_model_v1.dat")

# 返回单张图像的128D特征
def return_128d_features(path_img):
    img = io.imread(path_img)
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    dets = detector(img_gray, 1)

    print("检测的人脸图像：", path_img, "\n")

    # 因为有可能截下来的人脸再去检测，检测不出来人脸了
    # 所以要确保是 检测到人脸的人脸图像 拿去算特征
    if len(dets) != 0:
        shape = predictor(img_gray, dets[0])
        face_descriptor = facerec.compute_face_descriptor(img_gray, shape)
    else:
        face_descriptor = 0
        print("no face")

    # print(face_descriptor)
    return face_descriptor

# 将文件夹中照片特征提取出来，写入csv
# 输入input:
#   path_faces_personX:     图像文件夹的路径
#   path_csv:               要生成的csv路径

def write_into_csv(path_faces_personX, path_csv):
    dir_pics = os.listdir(path_faces_personX)
    with open(path_csv, "w", newline="") as csvfile:
        writer = csv.writer(csvfile)
        for i in range(len(dir_pics)):
            # 调用return_128d_features()得到128d特征
            print("正在读的人脸图像：", path_faces_personX + "/" + dir_pics[i])
            features_128d = return_128d_features(path_faces_personX + "/" + dir_pics[i])
            #  print(features_128d)
            # 遇到没有检测出人脸的图片跳过
            if features_128d == 0:
                i += 1
            else:
                writer.writerow(features_128d)

# 读取 某人 所有的人脸图像的数据，写入 person_X.csv
faces = os.listdir(path_faces_rd)
for person in faces:
    print(path_csv + person + ".csv")
    write_into_csv(path_faces_rd + person, path_csv + person + ".csv")

# 从csv中读取数据，计算128d特征的均值
def compute_the_mean(path_csv_rd):
    column_names = []

    # 128列特征
    for i in range(128):
        column_names.append("features_" + str(i + 1))

    # 利用pandas读取csv
    rd = pd.read_csv(path_csv_rd, names=column_names)

    # 存放128维特征的均值
    feature_mean = []

    for i in range(128):
        tmp_arr = rd["features_" + str(i + 1)]
        tmp_arr = np.array(tmp_arr)

        # 计算某一个特征的均值
        tmp_mean = np.mean(tmp_arr)
        feature_mean.append(tmp_mean)
    return feature_mean

# 存放所有特征均值的 CSV 的路径
path_csv_feature_all = "F:/code/python/P_dlib_face_reco/data/features_all.csv"
# 存放人脸特征的csv的路径
path_csv_rd = "F:/code/python/P_dlib_face_reco/data/csvs_from_camera/"

with open(path_csv_feature_all, "w", newline="") as csvfile:
    writer = csv.writer(csvfile)
    csv_rd = os.listdir(path_csv_rd)
    print("特征均值: ")
    for i in range(len(csv_rd)):
        feature_mean = compute_the_mean(path_csv_rd + csv_rd[i])
        # print(feature_mean)
        print(path_csv_rd + csv_rd[i])
        writer.writerow(feature_mean)

max_stars_repo_name: juxiangwu/image-processing

max_issues_repo_name: juxiangwu/image-processing

max_forks_repo_name: juxiangwu/image-processing


text:
# import mat4py
# a = mat4py.loadmat('/Users/chengxiao/Downloads/FGSD-master/MATLAB/data/G_mutag.mat')
# strr = '/Users/chengxiao/Downloads/graph2vec-master/dataset_test/opcua_simpletypes.c.exm_3_sym.json'
# split_ = strr.split('/')[-1]
# strip = split_[:-5]
# print("end")
import os
import json
from collections import Counter
import numpy as np
from imblearn.over_sampling import SMOTE, ADASYN  # 过抽样处理库SMOTE
from imblearn.under_sampling import RandomUnderSampler  # 欠抽样处理库RandomUnderSampler
from imblearn.over_sampling import RandomOverSampler  # 欠抽样处理库RandomUnderSampler
from imblearn.ensemble import EasyEnsemble  # 简单集成方法EasyEnsemble

from imblearn.combine import SMOTEENN
from imblearn.combine import SMOTETomek
import csv
#
inputpath = "/Users/chengxiao/Downloads/SARD.2019-02-28-22-07-31/noerror/result_sym"
inputcsvpath = "../features/test-787-noerror.csv"

def getGraphs(inputpath, inputCSVPath):
    '''得到graphs，key为图向量x和标记target

    :param inputpath: graphs-json
    :param inputCSVPath: graphs-vec
    :return: graph
    '''

    graphs = dict()

    # get target
    for dirpath, dirnames, filenames in os.walk(inputpath):
        # for dir in dirnames:
        #     fulldir = os.path.join(dirpath,dir)
        #     print(fulldir)

        for file in filenames:  # 遍历完整文件
            fullpath = os.path.join(dirpath, file)
            # print (fullpath)
            with open(fullpath, 'r', encoding="utf-8") as f:
                curjson = json.load(f)
                if ("target" not in curjson.keys()):
                    continue
                nodeStrList = curjson["nodes"]

                target = curjson["target"]
                edgeList = curjson["edges"]
                graphs[file] = dict()
                graphs[file]["target"] = target
                # graphs[file]["edgeList"] = edgeList
                # graphs[file]["nodeStrList"] = nodeStrList

    # get x
    with open(inputCSVPath, encoding="utf-8") as f:
        f_csv = csv.reader(f)
        headers = next(f_csv)
        for row in f_csv:
            name = row[0]
            vec = row[1:]
            graphs[name + ".json"]['x'] = vec
    return graphs

def main():
    graphs = getGraphs(inputpath, inputcsvpath)
    X = list()
    Y = list()

    for graph in graphs:
        X.append(graphs[graph]['x'])
        Y.append(graphs[graph]['target'])

    X = np.array(X).astype('float64')
    Y = np.array(Y)

    # 结合采样
    # https://blog.csdn.net/kizgel/article/details/78553009
    # smote_tomek = SMOTEENN(random_state=0)
    smote_tomek = SMOTETomek(random_state=0)
    X_resampled, y_resampled = smote_tomek.fit_sample(X, Y)
    # X_resampled, y_resampled = X,Y
    print(sorted(Counter(y_resampled).items()))
    # print(round(-0.222))

if __name__ == '__main__':
    main()