text:
import sys
import os

import numpy as np
import torch
import matplotlib.pyplot as plt

from PIL import Image, ImageDraw
from scipy.spatial.transform import Rotation as R
import copy
import k3d
import cv2
import io

from visu import save_image
from helper import re_quat
from matplotlib import cm
from torchvision import transforms
import math
from math import pi

jet = cm.get_cmap('jet')
SEG_COLORS = (np.stack([jet(v)
                        for v in np.linspace(0, 1, 22)]) * 255).astype(np.uint8)

def backproject_points(p, fx, fy, cx, cy):
    """
    p.shape = (nr_points,xyz)
    """
    # true_divide
    u = torch.round((torch.div(p[:, 0], p[:, 2]) * fx) + cx)
    v = torch.round((torch.div(p[:, 1], p[:, 2]) * fy) + cy)

    if torch.isnan(u).any() or torch.isnan(v).any():
        u = torch.tensor(cx).unsqueeze(0)
        v = torch.tensor(cy).unsqueeze(0)
        print('Predicted z=0 for translation. u=cx, v=cy')
        # raise Exception

    return torch.stack([v, u]).T

def get_img_from_fig(fig, dpi=180):
    buf = io.BytesIO()
    fig.savefig(buf, format="png", dpi=dpi)
    buf.seek(0)
    img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)
    buf.close()
    img = cv2.imdecode(img_arr, 1)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    return img

class Visualizer():
    def __init__(self, p_visu, writer=None):
        if p_visu[-1] != '/':
            p_visu = p_visu + '/'
        self.p_visu = p_visu
        self.writer = writer

        if not os.path.exists(self.p_visu):
            os.makedirs(self.p_visu)

    def plot_translations(self,
                          tag,
                          epoch,
                          img,
                          delta_v,
                          mask,
                          store=False,
                          jupyter=False):
        """
        img torch.tensor(h,w,3)
        delta_v torch.tensor(h,w,2)
        mask torch.tensor(h,w) BOOL
        """
        delta_v = delta_v * \
            mask.type(torch.float32)[:, :, None].repeat(1, 1, 2)
        # delta_v '[+down/up-], [+right/left-]'

        def bin_dir_amplitude(delta_v):
            amp = torch.norm(delta_v, p=2, dim=2)
            amp = amp / torch.max(amp)  # normalize the amplitude
            dir_bin = torch.atan2(delta_v[:, :, 0], delta_v[:, :, 1])
            nr_bins = 8
            bin_rad = 2 * pi / nr_bins
            dir_bin = torch.round(dir_bin / bin_rad) * bin_rad
            return dir_bin, amp

        rot_bin, amp = bin_dir_amplitude(delta_v)
        s = 20
        a = 2 if s > 15 else 1
        pil_img = Image.fromarray(img.numpy().astype(np.uint8), 'RGB')
        draw = ImageDraw.Draw(pil_img)
        txt = f"""Horizontal, pos right | neg left:
  max = {torch.max(delta_v[mask][:,0])}
  min = {torch.min(delta_v[mask][:,0])}
  mean = {torch.mean(delta_v[mask][:,0])}
Vertical, pos down | neg up:
  max = {torch.max(delta_v[mask][:,1])}
  min = {torch.min(delta_v[mask][:,1])}
  mean = {torch.mean(delta_v[mask][:,1])}
\n"""
        draw.text((10, 60), txt, fill=(255, 255, 255, 255))
        col = (0, 255, 0)
        grey = (207, 207, 207)
        for u in range(int(delta_v.shape[0] / s) - 2):
            u = int(u * s)
            for v in range(int(delta_v.shape[1] / s) - 2):
                v = int(v * s)
                if mask[u, v] == True:
                    du = round(math.cos(rot_bin[u, v])) * s / 2 * amp[u, v]
                    dv = round(math.sin(rot_bin[u, v])) * s / 2 * amp[u, v]
                    try:
                        draw.line([(v, u), (v + dv, u + du)],
                                  fill=col, width=2)
                        draw.ellipse([(v - a, u - a), (v + a, u + a)],
                                     outline=grey, fill=grey, width=2)
                    except:
                        pass
        if jupyter:
            display(pil_img)
        if store:
            pil_img.save(self.p_visu + str(epoch) +
                         '_' + tag + '.png')
        if self.writer is not None:
            img_np = np.array(pil_img).astype(np.uint8)
            self.writer.add_image(
                tag, img_np, global_step=epoch, dataformats='HWC')

    def plot_contour(self,
                     tag,
                     epoch,
                     img,
                     points,
                     cam_cx=0,
                     cam_cy=0,
                     cam_fx=0,
                     cam_fy=0,
                     trans=[[0, 0, 0]],
                     rot_mat=[[1, 0, 0], [0, 1, 0], [0, 0, 1]],
                     store=False,
                     jupyter=False,
                     thickness=2,
                     color=(0, 255, 0)):
        """
        tag := tensorboard tag
        epoch := tensorboard epoche
        store := ture -> stores the image to standard path
        path := != None creats the path and store to it path/tag.png
        img:= original_image, [widht,height,RGB], torch
        points:= points of the object model [length,x,y,z]
        trans: [1,3]
        rot: [3,3]
        """
        rot_mat = np.array(rot_mat)
        trans = np.array(trans)
        img_f = copy.deepcopy(img).astype(np.uint8)
        points = np.dot(points, rot_mat.T)
        points = np.add(points, trans[0, :])
        h = img_f.shape[0]
        w = img_f.shape[1]
        acc_array = np.zeros((h, w, 1), dtype=np.uint8)

        # project pointcloud onto image
        for i in range(0, points.shape[0]):
            p_x = points[i, 0]
            p_y = points[i, 1]
            p_z = points[i, 2]
            u = int(((p_x / p_z) * cam_fx) + cam_cx)
            v = int(((p_y / p_z) * cam_fy) + cam_cy)
            try:
                a = 10
                acc_array[v - a:v + a + 1, u - a:u + a + 1, 0] = 1
            except:
                pass

        kernel = np.ones((a * 2, a * 2, 1), np.uint8)
        erosion = cv2.erode(acc_array, kernel, iterations=1)

        try:  # problem cause by different cv2 version > 4.0
            contours, hierarchy = cv2.findContours(
                np.expand_dims(erosion, 2), cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
        except:  # version < 4.0
            _, contours, hierarchy = cv2.findContours(
                np.expand_dims(erosion, 2), cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
        out = np.zeros((h, w, 3), dtype=np.uint8)
        cv2.drawContours(out, contours, -1, (0, 255, 0), 3)

        for i in range(h):
            for j in range(w):
                if out[i, j, 1] == 255:
                    img_f[i, j, :] = out[i, j, :]
        if jupyter:
            display(Image.fromarray(img_f))

        if store:
            save_image(img_f, tag=str(epoch) + '_' + tag, p_store=self.p_visu)

        if self.writer is not None:
            self.writer.add_image(tag, img_f.astype(
                np.uint8), global_step=epoch, dataformats='HWC')

    def plot_segmentation(self, tag, epoch, label, store):
        if label.dtype == np.float32:
            label = label.round()
        image_out = np.zeros(
            (label.shape[0], label.shape[1], 3), dtype=np.uint8)
        for h in range(label.shape[0]):
            for w in range(label.shape[1]):
                image_out[h, w, :] = SEG_COLORS[int(label[h, w])][:3]
        if store:
            save_image(
                image_out, tag=f"{epoch}_{tag}", p_store=self.p_visu)
        if self.writer is not None:
            self.writer.add_image(
                tag, image_out, global_step=epoch, dataformats="HWC")

    def plot_estimated_pose(self,
                            tag,
                            epoch,
                            img,
                            points,
                            trans=[[0, 0, 0]],
                            rot_mat=[[1, 0, 0], [0, 1, 0], [0, 0, 1]],
                            cam_cx=0, cam_cy=0, cam_fx=0, cam_fy=0,
                            store=False, jupyter=False, w=2):
        """
        tag := tensorboard tag
        epoch := tensorboard epoche
        store := ture -> stores the image to standard path
        path := != None creats the path and store to it path/tag.png
        img:= original_image, [widht,height,RGB]
        points:= points of the object model [length,x,y,z]
        trans: [1,3]
        rot: [3,3]
        """
        if type(rot_mat) == list:
            rot_mat = np.array(rot_mat)
        if type(trans) == list:
            trans = np.array(trans)

        img_d = copy.deepcopy(img)
        points = np.dot(points, rot_mat.T)
        points = np.add(points, trans[0, :])
        for i in range(0, points.shape[0]):
            p_x = points[i, 0]
            p_y = points[i, 1]
            p_z = points[i, 2]
            u = int(((p_x / p_z) * cam_fx) + cam_cx)
            v = int(((p_y / p_z) * cam_fy) + cam_cy)
            try:
                img_d[v - w:v + w + 1, u - w:u + w + 1, 0] = 0
                img_d[v - w:v + w + 1, u - w:u + w + 1, 1] = 255
                img_d[v - w:v + w + 1, u - w:u + w + 1, 0] = 0
            except:
                #print("out of bounce")
                pass

        if jupyter:
            display(Image.fromarray(img_d))

        if store:
            #store_ar = (img_d* 255).round().astype(np.uint8)
            #print("IMAGE D:" ,img_d,img_d.shape )
            save_image(img_d, tag=str(epoch) + '_' + tag, p_store=self.p_visu)
        if self.writer is not None:
            self.writer.add_image(tag, img_d.astype(
                np.uint8), global_step=epoch, dataformats='HWC')

    def plot_bounding_box(self, tag, epoch, img, rmin=0, rmax=0, cmin=0, cmax=0, str_width=2, store=False, jupyter=False, b=None):
        """
        tag := tensorboard tag
        epoch := tensorboard epoche
        store := ture -> stores the image to standard path
        path := != None creats the path and store to it path/tag.png
        img:= original_image, [widht,height,RGB]

        """

        if isinstance(b, dict):
            rmin = b['rmin']
            rmax = b['rmax']
            cmin = b['cmin']
            cmax = b['cmax']

        # ToDo check Input data
        img_d = np.array(copy.deepcopy(img))

        c = [0, 0, 255]
        rmin_mi = max(0, rmin - str_width)
        rmin_ma = min(img_d.shape[0], rmin + str_width)

        rmax_mi = max(0, rmax - str_width)
        rmax_ma = min(img_d.shape[0], rmax + str_width)

        cmin_mi = max(0, cmin - str_width)
        cmin_ma = min(img_d.shape[1], cmin + str_width)

        cmax_mi = max(0, cmax - str_width)
        cmax_ma = min(img_d.shape[1], cmax + str_width)

        img_d[rmin_mi:rmin_ma, cmin:cmax, :] = c
        img_d[rmax_mi:rmax_ma, cmin:cmax, :] = c
        img_d[rmin:rmax, cmin_mi:cmin_ma, :] = c
        img_d[rmin:rmax, cmax_mi:cmax_ma, :] = c
        print("STORE", store)
        img_d = img_d.astype(np.uint8)
        if store:
            #store_ar = (img_d* 255).round().astype(np.uint8)
            save_image(img_d, tag=str(epoch) + tag, p_store=self.p_visu)
        if jupyter:
            display(Image.fromarray(img_d))
        if self.writer is not None:
            self.writer.add_image(tag, img_d.astype(
                np.uint8), global_step=epoch, dataformats='HWC')

    def plot_batch_projection(self, tag, epoch,
                              images, target, cam,
                              max_images=10, store=False, jupyter=False):

        num = min(max_images, target.shape[0])
        fig = plt.figure(figsize=(7, num * 3.5))
        for i in range(num):
            masked_idx = backproject_points(
                target[i], fx=cam[i, 2], fy=cam[i, 3], cx=cam[i, 0], cy=cam[i, 1])

            for j in range(masked_idx.shape[0]):
                try:
                    images[i, int(masked_idx[j, 0]), int(
                        masked_idx[j, 1]), 0] = 0
                    images[i, int(masked_idx[j, 0]), int(
                        masked_idx[j, 1]), 1] = 255
                    images[i, int(masked_idx[j, 0]), int(
                        masked_idx[j, 1]), 2] = 0
                except:
                    pass

            min1 = torch.min(masked_idx[:, 0])
            max1 = torch.max(masked_idx[:, 0])
            max2 = torch.max(masked_idx[:, 1])
            min2 = torch.min(masked_idx[:, 1])

            bb = BoundingBox(p1=torch.stack(
                [min1, min2]), p2=torch.stack([max1, max2]))

            bb_img = bb.plot(
                images[i, :, :, :3].cpu().numpy().astype(np.uint8))
            fig.add_subplot(num, 2, i * 2 + 1)
            plt.imshow(bb_img)

            fig.add_subplot(num, 2, i * 2 + 2)
            real = images[i, :, :, :3].cpu().numpy().astype(np.uint8)
            plt.imshow(real)

        if store:
            #store_ar = (img_d* 255).round().astype(np.uint8)
            plt.savefig(
                f'{self.p_visu}/{str(epoch)}_{tag}_project_batch.png', dpi=300)
            #save_image(img_d, tag=str(epoch) + tag, p_store=self.p_visu)
        if jupyter:
            plt.show()
        if self.writer is not None:
            # you can get a high-resolution image as numpy array!!
            plot_img_np = get_img_from_fig(fig)
            self.writer.add_image(
                tag, plot_img_np, global_step=epoch, dataformats='HWC')
        plt.close()

    def visu_network_input(self, tag, epoch, data, max_images=10, store=False, jupyter=False):
        num = min(max_images, data.shape[0])
        fig = plt.figure(figsize=(7, num * 3.5))

        for i in range(num):

            n_render = f'batch{i}_render.png'
            n_real = f'batch{i}_real.png'
            real = np.transpose(
                data[i, :3, :, :].cpu().numpy().astype(np.uint8), (1, 2, 0))
            render = np.transpose(
                data[i, 3:, :, :].cpu().numpy().astype(np.uint8), (1, 2, 0))

            # plt_img(real, name=n_real, folder=folder)
            # plt_img(render, name=n_render, folder=folder)

            fig.add_subplot(num, 2, i * 2 + 1)
            plt.imshow(real)
            plt.tight_layout()
            fig.add_subplot(num, 2, i * 2 + 2)
            plt.imshow(render)
            plt.tight_layout()

        if store:
            #store_ar = (img_d* 255).round().astype(np.uint8)
            plt.savefig(
                f'{self.p_visu}/{str(epoch)}_{tag}_network_input.png', dpi=300)
            #save_image(img_d, tag=str(epoch) + tag, p_store=self.p_visu)
        if jupyter:
            plt.show()
        if self.writer is not None:
            # you can get a high-resolution image as numpy array!!
            plot_img_np = get_img_from_fig(fig)
            self.writer.add_image(
                tag, plot_img_np, global_step=epoch, dataformats='HWC')
        plt.close()

    def visu_network_input_pred(self, tag, epoch, data, images, target, cam, max_images=10, store=False, jupyter=False):
        num = min(max_images, data.shape[0])
        fig = plt.figure(figsize=(10.5, num * 3.5))

        for i in range(num):
            # real render input
            n_render = f'batch{i}_render.png'
            n_real = f'batch{i}_real.png'
            real = np.transpose(
                data[i, :3, :, :].cpu().numpy().astype(np.uint8), (1, 2, 0))
            render = np.transpose(
                data[i, 3:, :, :].cpu().numpy().astype(np.uint8), (1, 2, 0))
            fig.add_subplot(num, 3, i * 3 + 1)
            plt.imshow(real)
            plt.tight_layout()
            fig.add_subplot(num, 3, i * 3 + 2)
            plt.imshow(render)
            plt.tight_layout()

            # prediction
            masked_idx = backproject_points(
                target[i], fx=cam[i, 2], fy=cam[i, 3], cx=cam[i, 0], cy=cam[i, 1])
            for j in range(masked_idx.shape[0]):
                try:
                    images[i, int(masked_idx[j, 0]), int(
                        masked_idx[j, 1]), 0] = 0
                    images[i, int(masked_idx[j, 0]), int(
                        masked_idx[j, 1]), 1] = 255
                    images[i, int(masked_idx[j, 0]), int(
                        masked_idx[j, 1]), 2] = 0
                except:
                    pass
            min1 = torch.min(masked_idx[:, 0])
            max1 = torch.max(masked_idx[:, 0])
            max2 = torch.max(masked_idx[:, 1])
            min2 = torch.min(masked_idx[:, 1])
            bb = BoundingBox(p1=torch.stack(
                [min1, min2]), p2=torch.stack([max1, max2]))
            bb_img = bb.plot(
                images[i, :, :, :3].cpu().numpy().astype(np.uint8))
            fig.add_subplot(num, 3, i * 3 + 3)
            plt.imshow(bb_img)
            # fig.add_subplot(num, 2, i * 2 + 4)
            # real = images[i, :, :, :3].cpu().numpy().astype(np.uint8)
            # plt.imshow(real)

        if store:
            #store_ar = (img_d* 255).round().astype(np.uint8)
            plt.savefig(
                f'{self.p_visu}/{str(epoch)}_{tag}_network_input_and_prediction.png', dpi=300)
            #save_image(img_d, tag=str(epoch) + tag, p_store=self.p_visu)
        if jupyter:
            plt.show()
        if self.writer is not None:
            # you can get a high-resolution image as numpy array!!
            plot_img_np = get_img_from_fig(fig)
            self.writer.add_image(
                tag, plot_img_np, global_step=epoch, dataformats='HWC')
        plt.close()

def plot_pcd(x, point_size=0.005, c='g'):
    """
    x: point_nr,3
    """
    if c == 'b':
        k = 245
    elif c == 'g':
        k = 25811000
    elif c == 'r':
        k = 11801000
    elif c == 'black':
        k = 2580
    else:
        k = 2580
    colors = np.ones(x.shape[0]) * k
    plot = k3d.plot(name='points')
    plt_points = k3d.points(x, colors.astype(np.uint32), point_size=point_size)
    plot += plt_points
    plt_points.shader = '3d'
    plot.display()

def plot_two_pcd(x, y, point_size=0.005, c1='g', c2='r'):
    if c1 == 'b':
        k = 245
    elif c1 == 'g':
        k = 25811000
    elif c1 == 'r':
        k = 11801000
    elif c1 == 'black':
        k = 2580
    else:
        k = 2580

    if c2 == 'b':
        k2 = 245
    elif c2 == 'g':
        k2 = 25811000
    elif c2 == 'r':
        k2 = 11801000
    elif c2 == 'black':
        k2 = 2580
    else:
        k2 = 2580

    col1 = np.ones(x.shape[0]) * k
    col2 = np.ones(y.shape[0]) * k2
    plot = k3d.plot(name='points')
    plt_points = k3d.points(x, col1.astype(np.uint32), point_size=point_size)
    plot += plt_points
    plt_points = k3d.points(y, col2.astype(np.uint32), point_size=point_size)
    plot += plt_points
    plt_points.shader = '3d'
    plot.display()

class SequenceVisualizer():
    def __init__(self, seq_data, images_path, output_path=None):
        self.seq_data = seq_data
        self.images_path = images_path
        self.output_path = output_path

    def plot_points_on_image(self, seq_no, frame_no, jupyter=False, store=False, pose_type='filtered'):
        seq_data = self.seq_data
        images_path = self.images_path
        output_path = self.output_path
        frame = seq_data[seq_no][frame_no]
        unique_desig = frame['dl_dict']['unique_desig'][0]

        if pose_type == 'ground_truth':
            # ground truth
            t = frame['dl_dict']['gt_trans'].reshape(1, 3)
            rot_quat = re_quat(copy.deepcopy(
                frame['dl_dict']['gt_rot_wxyz'][0]), 'wxyz')
            rot = R.from_quat(rot_quat).as_matrix()
        elif pose_type == 'filtered':
            # filter pred
            t = np.array(frame['filter_pred']['t']).reshape(1, 3)
            rot_quat = re_quat(copy.deepcopy(
                frame['filter_pred']['r_wxyz']), 'wxyz')
            rot = R.from_quat(rot_quat).as_matrix()
        elif pose_type == 'final_pred_obs':
            # final pred
            t = np.array(frame['final_pred_obs']['t']).reshape(1, 3)
            rot_quat = re_quat(copy.deepcopy(
                frame['final_pred_obs']['r_wxyz']), 'wxyz')
            rot = R.from_quat(rot_quat).as_matrix()
        else:
            raise Exception('Pose type not implemented.')

        w = 2
        if type(unique_desig) != str:
            im = np.array(Image.open(
                images_path + unique_desig[0] + '-color.png'))  # ycb
        else:
            im = np.array(Image.open(
                images_path + unique_desig + '.png'))  # laval
        img_d = copy.deepcopy(im)

        dl_dict = frame['dl_dict']
        points = copy.deepcopy(
            seq_data[seq_no][0]['dl_dict']['model_points'][0, :, :])
        points = np.dot(points, rot.T)
        points = np.add(points, t[0, :])

        cam_cx = dl_dict['cam_cal'][0][0]
        cam_cy = dl_dict['cam_cal'][0][1]
        cam_fx = dl_dict['cam_cal'][0][2]
        cam_fy = dl_dict['cam_cal'][0][3]
        for i in range(0, points.shape[0]):
            p_x = points[i, 0]
            p_y = points[i, 1]
            p_z = points[i, 2]
            u = int(((p_x / p_z) * cam_fx) + cam_cx)
            v = int(((p_y / p_z) * cam_fy) + cam_cy)
            try:
                img_d[v - w:v + w + 1, u - w:u + w + 1, 0] = 0
                img_d[v - w:v + w + 1, u - w:u + w + 1, 1] = 255
                img_d[v - w:v + w + 1, u - w:u + w + 1, 0] = 0
            except:
                #print("out of bounds")
                pass

        img_disp = Image.fromarray(img_d)
        if jupyter:
            display(img_disp)

        if store:
            outpath = output_path + \
                '{}_{}_{}.png'.format(pose_type, seq_no, frame_no)
            img_disp.save(outpath, "PNG", compress_level=1)
            print("Saved image to {}".format(outpath))

    def save_sequence(self, seq_no, pose_type='filtered', name=''):
        for fn in range(len(self.seq_data)):
            self.plot_points_on_image(seq_no, fn, False, True, pose_type)
        if name:
            video_name = '{}_{}_{}'.format(name, pose_type, seq_no)
        else:
            video_name = '{}_{}'.format(pose_type, seq_no)
        cmd = "cd {} && ffmpeg -r 10 -i ./filtered_{}_%d.png -vcodec mpeg4 -y {}.mp4".format(
            self.output_path, seq_no, video_name)
        os.system(cmd)

def load_sample_dict():
    # load data
    os.chdir('/home/jonfrey/PLR2')
    sys.path.append('src')
    sys.path.append('src/dense_fusion')

    from loaders_v2 import ConfigLoader, GenericDataset

    exp_cfg = ConfigLoader().from_file('/home/jonfrey/PLR2/yaml/exp/exp_ws_deepim.yml')
    env_cfg = ConfigLoader().from_file(
        '/home/jonfrey/PLR2/yaml/env/env_natrix_jonas.yml')
    generic = GenericDataset(
        cfg_d=exp_cfg['d_train'],
        cfg_env=env_cfg)
    img = Image.open(
        '/media/scratch1/jonfrey/datasets/YCB_Video_Dataset/data/0000/000001-color.png')
    out = generic[0]
    generic.visu = True
    names = ['cloud', 'choose', 'img_masked', 'target', 'model_points',
             'idx', 'add_depth', 'add_mask', 'img', 'cam', 'rot', 'trans', 'desig']

    sample = {}
    print(len(out[0]))
    for i, o in enumerate(out[0]):
        sample[names[i]] = o

    return sample

if __name__ == "__main__":

    sample = load_sample_dict()

    p = "/home/jonfrey/tmp"
    vis = Visualizer(p_visu=p, writer=None)
    vis.plot_contour(tag="visu_contour_test",
                     epoch=0,
                     img=sample['img'],
                     points=sample['target'],
                     cam_cx=sample['cam'][0],
                     cam_cy=sample['cam'][1],
                     cam_fx=sample['cam'][2],
                     cam_fy=sample['cam'][3],
                     store=True)

    vis.plot_estimated_pose(tag="visu_estimated_test",
                            epoch=0,
                            img=sample['img'],
                            points=sample['target'],
                            cam_cx=sample['cam'][0],
                            cam_cy=sample['cam'][1],
                            cam_fx=sample['cam'][2],
                            cam_fy=sample['cam'][3],
                            store=True)

    images = sample['img']
    images = images.unsqueeze(0)
    images = images.repeat(10, 1, 1, 1)

    target = sample['target']
    target = target.unsqueeze(0)
    target = target.repeat(10, 1, 1)

    cam = sample['cam']
    cam = cam.unsqueeze(0)
    cam = cam.repeat(10, 1)

    vis.plot_batch_projection(tag='batch_projection', epoch=0,
                              images=images, target=target, cam=cam,
                              max_images=10, store=True, jupyter=False)
    images = torch.transpose(images, 1, 3)
    images = torch.transpose(images, 2, 3)
    data = torch.cat([images, images], dim=1)

    vis.visu_network_input(tag="network_input",
                           epoch=0, data=data,
                           max_images=10,
                           store=True,
                           jupyter=False)

reason: import numpy,from scipy


text:
import os
import numpy as np
import tensorflow as tf
import matplotlib.pylab as plt
import matplotlib as mp

FLAGS = tf.app.flags.FLAGS

def format_plot(X, epoch=None, title=None, figsize=(15, 10)):

    plt.figure(figsize=figsize)

    if X.shape[-1] == 1:
        plt.imshow(X[:, :, 0], cmap="gray")
    else:
        plt.imshow(X)

    plt.axis("off")
    plt.gca().xaxis.set_major_locator(mp.ticker.NullLocator())
    plt.gca().yaxis.set_major_locator(mp.ticker.NullLocator())

    if epoch is not None and title is None:
        save_path = os.path.join(FLAGS.fig_dir, "current_batch_%s.png" % epoch)
    elif epoch is not None and title is not None:
        save_path = os.path.join(FLAGS.fig_dir, "%s_%s.png" % (title, epoch))
    elif title is not None:
        save_path = os.path.join(FLAGS.fig_dir, "%s.png" % title)
    plt.savefig(save_path, bbox_inches='tight', pad_inches=0)
    plt.clf()
    plt.close()

def save_image(X1, X2, e=None, title=None):

    if FLAGS.data_format == "NCHW":
        X1 = X1.transpose((0, 2, 3, 1))
        X2 = X2.transpose((0, 2, 3, 1))

    Xup = X1[:32]
    Xdown = X2[:32]

    n_cols = 8

    list_rows_f = []
    for i in range(Xup.shape[0] // n_cols):
        Xrow = np.concatenate([Xup[k] for k in range(n_cols * i, n_cols * (i + 1))], axis=1)
        list_rows_f.append(Xrow)
    list_rows_r = []
    for i in range(Xup.shape[0] // n_cols):
        Xrow = np.concatenate([Xdown[k] for k in range(n_cols * i, n_cols * (i + 1))], axis=1)
        list_rows_r.append(Xrow)

    Xup = np.concatenate(list_rows_f, axis=0)
    Xdown = np.concatenate(list_rows_r, axis=0)

    X_ones = 255 * np.ones_like(Xup, dtype=np.uint8)
    X_ones = X_ones[:5, :, :]

    X = np.concatenate((Xup, X_ones, Xdown), axis=0)

    format_plot(X, epoch=e, title=title)

ext: py

max_stars_repo_name: inamori/DeepLearningImplementations

max_issues_repo_name: inamori/DeepLearningImplementations

max_forks_repo_name: inamori/DeepLearningImplementations

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
annotations_line2d module
Created on Thu Sep 10 21:51:23 2015
@author: James Sorenson
"""

import matplotlib
import matplotlib.pyplot as plt
# This is to prevent overlapping annotations from being dragged simultaneously
# due to the multi-threaded nature of the matplotlib gui.
import threading
###########################
# Globals
###########################
attr_name = 'annotations_line2d'

_event= None # Used for debugging

###########################
# Class definitions
###########################
class DraggableAnnotationLine2D(matplotlib.offsetbox.DraggableBase):
    """This class is like Matplotlib’s DraggableAnnotation, but this one actually works.
    Apparently, the original class can't handle annotations that are created
    using 'offset points' from a data point. This class ONLY works with those.
    Left-click to move the annotation without changing the data point.
    Middle-click to slide the annotation to a different data point.
    Right-click to delete the annotation.
    The original annotation artist is in self.ref_artist.
    We save additional info in self.line, self.index, and self.formatter.
    """
    # Class-level lock to make sure only ONE annotation is moved at a time.
    # Due to QT's multi—threaded nature, it‘s best to use a real thread lock.
    _drag_lock=threading.Lock()
    _counter=0 # Just a counter to give each annotation a unique ID.

    def __init__(self, ref_artist, line=None, index=None, formatter=None, use_blit=True):
        # Use the base init (This isn‘t C++ where the parent is called automatically.)
        super().__init__(ref_artist, use_blit=use_blit)
        # Store the other parameters
        self.line=line
        self.index=index
        self.formatter=formatter
        # Create a unique ID for this annotation (for debugging)
        DraggableAnnotationLine2D._counter += 1
        DraggableAnnotationLine2D._counter %= 2**31 # Not too big
        self.id = DraggableAnnotationLine2D._counter
        #print('Init',self.id)
        if formatter is not None:
            # Get and set the text
            self.ref_artist.set_text(self.formatter(line, index))
        #Update the canvas to make sure the annotation is visible
        self.canvas.draw()

    def artist_picker(self, artist, event):
        """
        Determines if the artist should enable move for this mouse button event
        """
        # Make sure this only happens with a click. Ignore scroll.
        # Left or Right click works on all of these annotations
        # Middle click (slide) requires that line and index are assigned
        if (event.button in (1,3)) or \
            (event.button ==2 and self.line is not None and self.index is not None):
            # Good action. We only want to drag if the cursor is inside the
            # box, not the arrow and the area around it.
            # contains(event) returns (bool,attr)
            #print('Picked',self.id)
            drag = self.ref_artist.get_bbox_patch().contains(event)
            if drag[0]:
                #Make sure no other annotation are dragging.
                # wait=False means no block. True if a successful lock.
                if DraggableAnnotationLine2D._drag_lock.acquire(False):
                    # Record the mouse button
                    self.button=event.button
                    #print('Claim',self.id)
                    return drag
        # If we made it here, then we're not moving
        return (False, None)

    def save_offset(self):
        """
        On button-down, this saves the current location of the annotation.
        Annotation object is in self.ref_artist.
        """
        #print('Save',self.id)
        if self.button == 1:
            # Left-click. Move the annotation while pointing at the same data.
            # Get the starting position of the artist in points (relative to data point)
            self.drag_start_text_points = self.ref_artist.get_position()
            # Get the inverted transform so we can convert pixels to paints.
            self.drag_trans_mat = self.ref_artist.get_transform().inverted().get_matrix()
        elif self.button == 2:
            # Middle-click. We need some additional information to slide the data.
            self.xydata=self.line.get_xydata() #just makes it easier (this does NOT copy)
            # we need the pixels of the starting data point (not the cursor)
            self.drag_start_pixels = self.ref_artist.get_axes().transData.transform(self.ref_artist.xy)
            # Get the translation from pixels to data for annotation.xy
            self.drag_trans_pix2dat = self.ref_artist.get_axes().transData.inverted()

    def update_offset(self, dx, dy):
        """
        dx and dy is the total pixel offset from the point where the mouse
        drag started.
        """
        if self.button == 1: # Left—click
            # Scale delta pixels to delta points using parts of annotation transform.
            # The full transform includes the data offset, but set position already does that.
            new_position=(self.drag_start_text_points[0] + dx * self.drag_trans_mat[0,0],
                          self.drag_start_text_points[1] + dy * self.drag_trans_mat[1,1])
            # Apply as delta points from data point
            self.ref_artist.set_position(new_position)
        elif self.button == 2: # Middle—click
            # We may have a logarithmic scale, but update offset only gives us delta pixels.
            # Add the delta to the starting pixels, then convert to data coordinates
            pixels_dxy = matplotlib.numpy.array((dx,dy))
            new_data_xy = self.drag_trans_pix2dat.transform(self.drag_start_pixels+pixels_dxy)
            # Determine if the new data coordinates reach or exceed the next line data point.
            index=self.index
            while (index > 0) and (self.xydata[index-1][0] > new_data_xy[0]):
                #Move left
                index -= 1
            while (index < self.xydata.shape[0] - 1) and (self.xydata[index+1][0] < new_data_xy[0]):
                # Move right
                index += 1
            if index != self.index:
                # we moved an index! Update the annotation
                self.ref_artist.xy=self.xydata[index,:]
                self.index=index
                if self.formatter is not None:
                    # Update the text in the annotation
                    self.ref_artist.set_text(self.formatter(self.line, index))

    def finalize_offset(self):
        """Called when the mouse button is released, if this was picked in the first place."""
        #print('Finalize',self.id)
        if self.button == 2 and self.formatter is not None:
            # Print out annotation text for the user to copy/paste
            self.print_annotation()
        elif self.button == 3:
            # Delete annotation
            self.remove()

    def on_release(self,event):
        """
        Called when the mouse button is released, whether or not this was picked.
        We extend this function so that we are guaranteed to release the thread lock.
        """
        # Call the original
        super().on_release(event)
        #Everyone tries to remove the block, just in case the controlling annotation was removed.
        try:
            DraggableAnnotationLine2D._drag_lock.release()
        except RuntimeError:
            pass # Already released. Not a concern.

    def print_annotation(self):
        """Does exactly what you think it does"""
        print('Annotation: {0}, ind={1}\n{2}'.format(self.line.get_label(), self.index, self.ref_artist.get_text()))

    def remove(self):
        """Disconnect and delete the annotation."""
        #print('Remove',self.id)
        self.disconnect() # Disconnect the callbacks
        self.ref_artist.remove() # Delete the annotation artist
        self.got_artist=False # Tell this class it no longer has an artist
        self.canvas.draw() # Update the whole canvas so the annotation disappears

class AnnotationPicker(object):
    """
    A class to enable convenient annotations to any plot.
    This is meant only for 2D lines.
    Left-click to move the annotation without changing the data point.
    Middle-click to slide the annotation to a different data point.
    Right-click to delete the annotation.
    Optional arguments:
        artists: (default None) A single or list of artists to attach this to as 'artist annotations'
        tolerance : (default 5) Picker tolerance to a line's data point to create an annotation.
        formatter : function to generate the string in the annotation. fcn(Line2D artist, index)
        All other keyword arguments Will be passed to the annotation.
    """

    def __init__(self, artists=None, tolerance=5, formatter=None, button=1, key = 'control', use_blit=True, **kwargs):
        # Parse the arguments
        self.tolerance = tolerance
        self.use_blit = use_blit
        self.button = button
        self.key=key
        if formatter is None: # Use default
            self.formatter=self._annotate_line_str
        else:
            self.formatter = formatter
        # Save the annotation parameters
        self.annotation_kwargs = dict(xycoords='data', textcoords='offset points',
            fontsize=11, picker=True, xytext=(20, 20),
            bbox=dict(boxstyle='round,pad=0.5', fc='yellow', alpha=0.5),
            arrowprops=dict(shrink=0.05, headwidth=5, width=1))
        # Add in additional/modified user parameters
        self.annotation_kwargs.update(kwargs)
        # Apply this annotation instance to the given artists and children
        if artists is not None:
            self.apply(artists)

    def apply(self, artists):
        """
        Enable picker on lines so that annotations are activated.
        This particular Annotation instance will be applied to this artist and
        its children (unless the children already have their own instance.
        Use 'clear annotaions' if you wish to override children settings.
        """
        # This is overly complex, but it allows the user to throw anything at it (figure, axes, line, etc)
        # Make it iterable for convenience
        artists = _make_iterable(artists)
        for artist in artists:
            if artist is None:
                continue
            # Attach this instance to the given artists
            setattr(artist, attr_name, self)
            # Enable picker to any line contained in this artist that is not already enabled.
            if isinstance(artist, matplotlib.lines.Line2D) and not artist.pickable():
                lines = [artist]
            elif isinstance(artist, matplotlib.axes.Axes):
                lines = [line for line in artist.get_lines() if not line.pickable()]
            elif isinstance(artist, matplotlib.figure.Figure):
                lines = [line for ax in artist.get_axes() for line in ax.get_lines() if not line.pickable()]
            else:
                lines=[]
            for line in lines:
                line.set_picker(self.tolerance)

        # Make sure the callbacks are enabled for the parent canvas
        enable_callbacks(artist)

    def annotate(self, line, index, text=None):
        """
        Makes a draggable, interactive annotation on the given line,
        at the given index, with the given text.
        line : Line2D object to annotate
        index : The index of the line to put the annotation
        text : The text to fill the annotation with. If None, then use default.
        Returns a DraggableAnnotationLine2D instance where the annotation artist is in self.ref_artist.
        """
        if text is None:
            # Get the text from the formatter
            formatter=self.formatter
        else:
            # Manual text is given. Don't use the formatter
            formatter = None
        # Create the annotation at the designated point
        ax=line.get_axes()
        annot=ax.annotate(text, line.get_xydata()[index,:], **self.annotation_kwargs)
        # Make it draggable using our class, then return the object
        return DraggableAnnotationLine2D(annot, line, index, formatter, use_blit=self.use_blit)

    def _annotate_line_str(self, line, index):
        """
        The default function to take a Line2D artist and index and generate a
        string for the annotation box.
        """
        xy=line.get_xydata()[index]
        return '{0}[{1}]:\nx={2:.9}\ny:{3:.9}'.format(line.get_label(),index,xy[0],xy[1])

    def _onpick(self,event):
        """Called by canvas pick event."""
        if event.mouseevent.button == self.button and \
            event.mouseevent.key == self.key and \
            isinstance(event.artist, matplotlib.lines.Line2D):
            # More than one index may be in range. Determine the middle index.
            ind = event.ind[len(event.ind)//2]
            global _event
            _event=event
            # Generate the annotation
            self.annotate(event.artist, ind)

###########################
# Module functions
###########################

def enable_callbacks(artist):
    """
    Enable annotation callbacks within this canvas/figure.
    This adds the .annotations attribute to the canvas to hold the callbacks.
    """
    if isinstance(artist, matplotlib.figure.Figure):
        canvas=artist.canvas
    elif hasattr(artist, 'get_figure'):
        canvas=artist.get_figure().canvas
    else:
        canvas=artist

    if not hasattr(canvas,attr_name):
        # Add the callbacks and store as a list in the canvas attribute
        callbacks=[]
        callbacks.append(canvas.mpl_connect('pick_event', _on_pick_event))
        callbacks.append(canvas.mpl_connect('figure_enter_event', _on_figure_enter_event))
        setattr(canvas, attr_name, callbacks)

def disable_callbacks(canvas):
    """
    Disable all annotation callbacks pertaining to this callback.
    We leave the pickers and annotation instances in the artists.
    We just get rid of the callback attached to the canvas.
    """
    if isinstance(canvas, matplotlib.figure.Figure):
        canvas=canvas.canvas # We were given the figure instead
    for callback in getattr(canvas, attr_name, []):
        canvas.mpl_disconnect(callback)
    delattr(canvas, attr_name)
    print('AnnotationPicker callback removed from canvas.')

def annotate(line, index, text=None):
    """
    Wrapper function around AnnotationPicker.annotate()
    This will find the controlling instance of Annotations for the given line
    and create an interactive annotation at the given index with the given text.
    Input:
        line: The matplotlib line object to annotate (plt.figure(1).axes[0].lines[0])
        index: The index of the line to annotate.
        text: The annotation text. It None, then the AnnotationPicker.formatter()
            is used to generate text at the given line and index.
    Returns:
        DraggableAnnotationLine2D object
    """
    annotations_instance = _find_annotations_instance(line)
    if annotations_instance is None:
        # Create a default annotation for this line
        annotations_instance = AnnotationPicker(line)
        setattr(line, attr_name, annotations_instance)
    annotations_instance.annotate(line, index, text)

def subplots(*args, anno=None, **kwargs):
    """
    Identical to plt.subplots(), but  also assigns an AnnotationPicker class
    to the figure.  Use "anno=AnnotationPickerInstance" to use a specific instance
    of the AnnotationPicker.
    """
    # Since we are using plt.subplots, this will show immediately if interactive.
    # gca and gcf will also be updated.
    fig,ax_list=plt.subplots(*args, **kwargs)
    if anno is None:
        # Create default AnnotationPicker that will be connected to the figure
        AnnotationPicker(fig)
    else:
        anno.apply(fig)
    return (fig,ax_list)

###########################
# Private Utilites
###########################

def _make_iterable(obj):
    """Return obj as a list if it is not already an iterable object"""
    if hasattr(obj,'__iter__'):
        return obj
    else:
        # Make it iterable for consistency
        return [obj]

def _find_annotations_instance(artist):
    """
    Find the controlling Annotations instance for this artists.
    It could be attached to the artist itself, or on the parent axes or figure.
    Returns the controlling Annotations instance.
    """
    if hasattr(artist, attr_name):
        # Instance is attached to the artist itself
        return getattr(artist, attr_name)
    elif hasattr(artist, 'get_axes' ) and hasattr(artist.get_axes(), attr_name):
        # Instance is attached to the axes
        return getattr(artist.get_axes(), attr_name)
    elif hasattr(artist, 'get_figure') and hasattr(artist.get_figure(), attr_name):
        # Instance is attached to the figure
        return getattr(artist.get_figure(), attr_name)
        # No instance found
    return None

def _clear_annotations(artist):
    """
    Call this on any artist to clear the annotation instances for that artist
    and all of its children. Mostly useful for debugging.
    """
    artists = _make_iterable(artist)
    for artist in artists:
        if hasattr(artist, attr_name):
            delattr(artist, attr_name)
            if hasattr(artist,'get chlldren'):
                _clear_annotations(artist.get_children())
    print('All annotations in artist and children deleted.')

###########################
# Canvas Callback functions
###########################
def _on_pick_event(event):
    """
    This is what initially gets called when ANY artist in the figure with
    picking enabled is picked.
    Startlng with the artist itself, thls function will determine the closest
    AnnotationPicker instance to call. This permits different settings per
    line or per axes.
    """
    annotations_instance = _find_annotations_instance(event.artist)
    if annotations_instance is not None:
        # Call the controlling Annotations instance
        annotations_instance._onpick(event)

def _on_figure_enter_event(event):
    """
    When the mouse enters the figure, this will make sure all lines have
    picker enabled so that new lines can be annotated.
    """
    fig=event.canvas.figure
    # Only lines that are not already pickable will be updated.
    lines=[line for ax in fig.axes for line in ax.lines if not line.pickable()]
    for line in lines:
        # The controlling Annotations instance is either in the axes or figure.
        annotations_instance=_find_annotations_instance(line)
        if annotations_instance is not None:
            line.set_picker(annotations_instance.tolerance)
    # We may need to update legends if the user manually plotted or deleted a line.
    #legend_update(fig, draw=True) #Draw if a change was detected

###########################
# TEST
###########################
if __name__ == '__main__':
    import numpy as np
    plt.ion()
    # Use our subplots wrapper to make sure annotations are enabled
    fig,ax=subplots(2,1)
    ax[0].set_title('click on points')
    x=np.r_[-5:5:.1]
    y=x**2-5*x+3
    lines=[]
    lines += ax[0].plot(x,x**2-5*x+3, '-.',label='My Line')
    lines += ax[1].plot(x,5*x+4,label='Line2')
    # Enable Annotations
    anno=AnnotationPicker(fig)
    an=anno.annotate(ax[0].lines[0],30, 'A manual annotation')
    # Add a legend
    #leg=legend(ax)
    # Add another line and see if moving the mouse in catches it
    ax[1].plot(x,2*x+7, label='New line')
    # Create custom string for 2nd axes
    def custom_text(line,ind):
        xy=line.get_xydata()[ind]
        custom='Custom text\nData[{0}]: {1:.9}, {2:.9}'.format(ind,xy[0],xy[1])
        return custom

    anno2=AnnotationPicker(ax[1],formatter=custom_text, key=None)
    ax[1].plot(x,y, '.-',label='No picker yet') # See if the picker gets enabled
    ax[1].legend()
    plt.draw()


text:
import os
import shutil
import logging
import numpy as num
from pyrocko import orthodrome

logger = logging.getLogger('pinky.util')

def ensure_dir(p):
    if not os.path.exists(p):
        os.makedirs(p)

class ListEmpty(Exception):
    pass

def ensure_list(l):
    if not len(l):
        raise ListEmpty()

def first_element(x):
    if len(x) <= 1:
        return x[0]
    elif len(x) > 1:
        raise Exception('%s has more than one item' % x)

def filter_oob(sources, targets, config):
    '''Filter sources that will be out of bounds of GF database.'''
    nsources, ntargets = len(sources), len(targets)
    slats, slons = num.empty(nsources), num.empty(nsources)
    sdepth = num.empty(nsources)
    tlats, tlons = num.empty(ntargets), num.empty(ntargets)
    televations = num.empty(ntargets)

    for i_s, s in enumerate(sources):
        slats[i_s], slons[i_s], sdepth[i_s] = *s.effective_latlon, s.depth

    for i_t, t in enumerate(targets):
        tlats[i_t], tlons[i_t] = t.effective_latlon
        televations[i_t] = t.elevation

    # dmax = config.distance_max
    # nsources = len(sources)
    # sources_out = []
    # for i_s, s in enumerate(sources[::-1]):
    #     for t in targets:
    #         print(t.distance_to(s))
    #         if t.distance_to(s) > dmax:
    #             print('break')
    #             break
    #     else:
    #         print('fine')
    #         sources_out.append(s)

    # return sources_out

    dists = num.empty((ntargets, nsources))
    depths = num.empty((ntargets, nsources))
    for i in range(ntargets):
        dists[i] = orthodrome.distance_accurate50m_numpy(
                slats, slons, tlats[i], tlons[i])
        depths[i] = sdepth + televations[i]

    i_dist = num.where(num.any(num.logical_or(
            dists > config.distance_max-100, dists < config.distance_min+100), axis=0))[0]
    i_depth = num.where(num.any(num.logical_or(
            depths > config.source_depth_max-100, depths < config.source_depth_min+100), axis=0))[0]
    print(i_depth)
    i_filter = num.union1d(i_depth, i_dist)

    print(dists)
    logger.warn('Removing %i / %i sources which would be out of bounds' %
            (len(i_filter), nsources))

    for i in i_filter[::-1]:
        # print('.'*10)
        # for t in targets:
        #     print(sources[i].distance_to(t))
        sources.pop(i)
        # del sources[i]
    _d = [t.distance_to(s) for t in targets for s in sources]
    print(max(_d))
    print(min(_d))
    _d = [s.depth+t.elevation for t in targets for s in sources]
    print(min(_d))
    print(max(_d))
    return sources

def delete_if_exists(dir_or_file):
    '''Deletes `dir_or_file` if exists'''
    if os.path.exists(dir_or_file):
        if os.path.isfile(dir_or_file):
            logger.debug('deleting file: %s' % dir_or_file)
            os.remove(dir_or_file)
        else:
            logger.debug('deleting directory: %s' % dir_or_file)
            shutil.rmtree(dir_or_file)

def nsl(tr):
    return tr.nslc_id[:3]

def append_to_dict(d, k, v):
    _m = d.get(k, [])
    _m.append(v)
    d[k] = _m

def snr(chunk, split_factor):
    isplit = int(chunk.shape[1] * split_factor)
    chunk_masked = num.ma.masked_invalid(chunk)
    chunk = chunk**2
    sig = chunk[:, :isplit]
    noise = chunk[:, isplit:]
    s = num.sqrt(num.sum(sig, axis=1))
    n = num.sqrt(num.sum(noise, axis=1))

    return num.nanmean(s/n) / split_factor

max_stars_repo_name: braunfuss/pinky

max_issues_repo_name: braunfuss/pinky

max_forks_repo_name: braunfuss/pinky


text:
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os
a=os.path.join(os.path.dirname(__file__),"train.csv")

def cost(h,y,m):
    pass

data=pd.read_csv(os.path.join(os.path.dirname(__file__),"train.csv"))
data.dropna(axis=0)

y_data=data.SalePrice
y=np.array(y_data,dtype=np.int64)

x1_data=data["LotArea"]
x2_data=data["OverallQual"]
x3_data=data["OverallCond"]
x4_data=data["YearBuilt"]

m=len(y)
x0=np.ones(m)
x=np.array([x0,x1_data,x2_data,x3_data,x4_data],dtype=np.int64).T

theta=np.ones(len(x))
alpha=0.01
epochs=1000

t=((np.linalg.inv((x.T).dot(x))).dot(x.T)).dot(y)
print(t)

fit=np.dot(x,t)

plt.scatter(x[:,4],y)
plt.plot([min(x[:,4]),max(x[:,4])],[min(fit),max(fit)],color="red")
#plt.show()

for i in range(0,5):
    print(t[i])

while(1):
    ls=list(map(int,input("Enter the 4 features: ").split()))
    am=np.array(ls)
    print(sum(np.insert(am,0,1)*t))
    break

max_stars_repo_name: iSriBalaji/MachineLearning

max_issues_repo_name: iSriBalaji/MachineLearning

max_forks_repo_name: iSriBalaji/MachineLearning


text:
# %%
#### imports
from dataclasses import dataclass
from enum import Enum

import numpy as np
from numpy.random import randint, random, uniform, normal, choice

# %%
#### constants
max_tree_size = 8
max_leaf_value = 100

operators = {
    "sum": lambda x,y : x+y,
    "sub": lambda x,y: x-y,
    "mul": lambda x,y: x*y,
    "div": lambda x,y: x/y if y != 0 else np.inf,
    "pow": lambda x,y: x**y,
    "sin": np.sin,
    "cos": np.cos,
    "id": lambda x: x
}

#### structures

@dataclass
class Node:
    value: tuple
    left: "Node" = None
    right: "Node" = None

class ValType(Enum):
    const = 0   # operand argument
    unary = 1   # one operand operator
    binary = 2  # two operand operator
    var = 3    # expression variable

#### methods

def get_operator_type(op):
    if op in ["sum", "sub", "mul", "div", "pow"]:
        return ValType.binary
    if op in ["sin", "cos", "id"]:
        return ValType.unary
    return None

def evaluate_tree(node: Node, args = None):
    if not node:
        return 0

    value_type, value = node.value
    if value_type == ValType.const:
        return value

    if value_type == ValType.var:
        if args is None:
            return 0
        if value >= len(args):
            return 0
        return args[value]

    # convert to proper operator name
    value = list(operators.keys())[int(value)]

    left = evaluate_tree(node.left)
    if value_type == ValType.unary:
        return operators[value](left)

    right = evaluate_tree(node.right)
    if value_type == ValType.binary:
        return operators[value](left, right)

    # or else, error
    print("invalid operator type!")

# %%
#### tree generation

def create_tree(tree_size, node_values, value_types):
    if tree_size == 0:
        return node_values, value_types, None

    node_vt, node_v = value_types[0], node_values[0]
    node_values, value_types = node_values[1:], value_types[1:]
    if node_vt == ValType.var:
        node_v = int(node_v)

    if tree_size == 1:
        return node_values, value_types, Node((node_vt, node_v))

    rts = (tree_size-1)//2      # right tree size
    lts = (tree_size-1) - rts   # left tree size

    node_values, value_types, left = create_tree(lts, node_values, value_types)
    node_values, value_types, right = create_tree(rts, node_values, value_types)

    return node_values, value_types, Node((node_vt, node_v), left, right)

def create_trees(n_trees, max_tree_size, max_leaf_value):
    tree_sizes = randint(1, max_tree_size+1, size= n_trees)     # size of each tree

    n_leaves = ((tree_sizes+np.ones(n_trees))//2).astype(int)   # leaf number of each tree
    leaf_vals = np.zeros(np.sum(n_leaves))                      # leaf values

    is_var_p = np.random.normal(0.3, 0.1)
    lfs = np.sum(n_leaves)                                      # number of all leaf values
    is_var = choice([1, 0], size=lfs, p=[is_var_p, 1-is_var_p])

    n_var_leaves = np.count_nonzero(is_var)                     # number of all leaves containing variable
    n_vars = max(1, int(np.random.exponential(scale=2)))
    leaf_vals[is_var==1] = randint(0, n_vars, size= n_var_leaves)

    n_const_leaves = lfs - n_var_leaves
    leaf_vals[is_var==0] = uniform(low=0, high= max_leaf_value, size= n_const_leaves)

    n_operators = tree_sizes - n_leaves                         # operator number of each tree
    op_vals = randint(0, len(operators.keys()), size= np.sum(n_operators)) # operator types

    # set all node vals
    node_vals = np.zeros(np.sum(tree_sizes))
    no = 0  # nodes offset
    lo = 0  # leaves offset
    oo = 0  # operators offset

    value_types = []
    leaf_types = np.where(is_var==0, ValType.const, ValType.var)
    op_keys = list(operators.keys())

    for i in range(n_trees):
        window = node_vals[no:no+tree_sizes[i]]
        window[0:n_operators[i]] = op_vals[oo:oo+n_operators[i]]
        window[n_operators[i]:tree_sizes[i]] = leaf_vals[lo:lo+n_leaves[i]]

        op_strs = [op_keys[op_vals[i]] for i in range(oo, oo+n_operators[i])]
        value_types += list(map(lambda op: get_operator_type(op), op_strs))

        value_types += list(leaf_types[lo:lo+n_leaves[i]])

        no += tree_sizes[i]
        oo += n_operators[i]
        lo += n_leaves[i]

    # create trees
    trees = []
    for size in tree_sizes:
        node_vals, value_types, tree = create_tree(size, node_vals, value_types)
        trees.append(tree)

    # print(node_vals, value_types)

    return trees

# %%
#### debugging tools

def tree_to_expression(node: Node, use_common_var_names = True, limit_floating_point = True):
    if node is None:
        return "0"

    val_type, value = node.value
    if val_type == ValType.const:
        if not value:
            return "0"
        if limit_floating_point and isinstance(value, float):
            return "%.2f" % value
        return str(value)

    if val_type == ValType.var:
        if not use_common_var_names:
            return "x"+ str(int(value))

        i = int(value)
        if i < 13:
            return ['x','y','z', 'w', 't', 'u', 'v', 'm', 'n', 'p', 'q', 'r', 's'][i]

        if i < 26:
            return chr(97+i-13)

        return "x"+ str(i)

    # fetch name of operator
    value = str(list(operators.keys())[int(value)])

    if val_type == ValType.unary:
        return value + "("+ tree_to_expression(node.left) + ")"

    if val_type == ValType.binary:
        return (
            value + "(" + tree_to_expression(node.left) + ", "
            + tree_to_expression(node.right) + ")"
        )

    print("Unknown value type: " + str(val_type))

# %%

def simulate_one_round(population, ):
    trees = create_trees(population, max_tree_size, max_leaf_value)
    scores = evaluate_tree()

# %%
### DATA GENERATION

def generate_random_data():
    data_size = int(normal(1000, 200))
    max_x = normal(100,20)
    x = uniform(low= -max_x, high= max_x, size= data_size)

    max_value = normal(0, 1000)
    values = uniform(low= -max_value, high= max_value, size= data_size)

    return x, values

def generate_points(f, dsize, interval= (-100,100)):
# generates points for function f = f(x)
    start, end = interval
    x = uniform(low= start, high= end, size= dsize)
    return x,  np.array(list(map(f,x)))

### PERFORMANCE EVALUATION

def split_data(x,y, train_ratio):
    n = int(train_ratio * len(x))
    train_x, train_y, score_x, score_y = x[:n], y[:n], x[n:], y[n:]
    return train_x, train_y, score_x, score_y

# %%
### EVOLUTION
def replicate(node: Node):
    if node is None:
        return None
    return Node(node.value, replicate(node.left), replicate(node.right))

def crossover(xx: Node, xy: Node, p: float = 0.5):
    if xx is None or xy is None:
        return

    x = np.random.random()
    go_right = x > 0.5

    if x > p:
    # then don't crossever in root

        # increase the chance of next crossover
        p = 2 * p * np.random.rand()
        if go_right:
            crossover(xx.right, xy.right, p)
        else:
            crossover(xx.left, xx.left, p)
        return

    # time to crossover
    xx.value, xy.value = xy.value, xx.value
    if go_right:
        xx.right, xy.right = xy.right, xx.right
    else:
        xx.left, xy.left = xy.left, xx.left

def random_operation():
    ops = list(operators.keys())
    x = randint(len(ops))
    op = ops[x]
    return get_operator_type(op), x

def random_of_type(val_type: ValType):
    if val_type is ValType.const:
        return np.random.random()* 100
    if val_type is ValType.var:
        return randint(26)
    if val_type in [ValType.binary, ValType.unary]:
        typ, val = random_operation()
        while typ != val_type:
            typ, val = random_operation()
        return val

def mutate(n: Node, p):
    if n is None:
        return False

    x = np.random.random()
    go_right = x > 0.5
    if x > p:
        # making sure mutation happens!
        p = 1.4 * p

        if go_right:
            return mutate(n.right, p)
        else:
            return mutate(n.left, p)

    typ, val = n.value

    y = np.random.random()
    if y<p:
    # mutate the typ itself
        typ = ValType(randint(len(ValType)))
        val = random_of_type(typ)
    else:
        if typ is ValType.const:
            val = val * np.random.normal(0, 3)
        elif typ is ValType.var:
            if go_right:
                val += 1
            else:
                if val > 0:
                    val -= 1
                else:
                    val += 2
        else:
        # if operation
            _typ = typ
            while _typ == typ:
                _typ, _val = random_operation()
            typ, val = _typ, _val

    n.value = typ, val
    return True

# %%
# x,y = generate_points(lambda x : x*2, 10)
# print(x)
# print(y)

# %%
#### tests

def test_mutation():
    # testintg mutation
    trees = create_trees(20, 9, 10)
    for t in trees:
        before = tree_to_expression(t)
        is_mutated = mutate(t, 0.3)
        if not is_mutated:
            continue
        after = tree_to_expression(t)

        print("from: " + before)
        print("to: " + after)
        # x = evaluate_tree(t, [1,1,1,1,1])
        # print("%.2f" % x)
        print()

def crossover_test():
    trees = create_trees(2, 20, 10)
    t,u = trees
    print(tree_to_expression(t), "\n",tree_to_expression(u))
    print("crossing over..")
    crossover(t,u, 0.5)
    print(tree_to_expression(t), "\n", tree_to_expression(u))

crossover_test()

# %%

# %%

# %%

reason: import numpy,from numpy


text:
from __future__ import print_function, division
import os
from math import ceil
import json
import numpy as np

try:
    import h5py
    WITH_H5PY = True
except ImportError:
    WITH_H5PY = False
try:
    import zarr
    WITH_ZARR = True
except ImportError:
    WITH_ZARR = False
try:
    import z5py
    WITH_Z5PY = True
except ImportError:
    WITH_Z5PY = False

def load_mask(path, key):
    ext = os.path.splitext(path)[-1]
    if ext.lower() in ('.h5', '.hdf', '.hdf'):
        assert WITH_H5PY
        with h5py.File(path, 'r') as f:
            ds = f[key][:]
    elif ext.lower() in ('.zr', '.zarr', '.n5'):
        assert WITH_Z5PY or WITH_ZARR
        if WITH_ZARR:
            f = zarr.open(path)
            ds = f[key][:]
        elif WITH_Z5PY:
            with z5py.File(path) as f:
                ds = f[key][:]
    return ds

def make_prediction_blocks(shape, output_block_shape, mask_file, output_file,
                           mask_key='data', roi_begin=None, roi_end=None):

    # load mask and get mask shape
    assert os.path.exists(mask_file), mask_file
    mask = load_mask(mask_file, mask_key)
    mask_shape = mask.shape

    # compute output block shape
    blocked_shape = [int(ceil(float(fs) / os)) for fs, os in zip(shape, output_block_shape)]

    # compute the scale factors between mask and blocked_shape
    scale_factor_mask = [float(sh) / bs for sh, bs in zip(mask_shape, blocked_shape)]

    if roi_begin is not None:
        assert roi_end is not None
        roi_begin_out = [roib // obs for roib, obs in zip(roi_begin, output_block_shape)]
        roi_end_out = [roie // obs + 1 for roie, obs in zip(roi_end, output_block_shape)]
    else:
        roi_begin_out = [0, 0, 0]
        roi_end_out = blocked_shape

    prediction_mask = np.zeros(blocked_shape, dtype='uint8')
    prediction_blocks = []
    # generate blocks
    for z in range(roi_begin_out[0], roi_end_out[0]):
        print("generating for", z, "/", roi_end_out[0])
        for y in range(roi_begin_out[1], roi_end_out[1]):
            for x in range(roi_begin_out[2], roi_end_out[2]):
                coord = [z, y, x]
                bb_mask = tuple(slice(int(co * sf),
                                      int(ceil((co + 1) * sf)))
                                for co, sf in zip(coord, scale_factor_mask))
                mask_block = mask[bb_mask]

                if np.sum(mask_block) > 0:
                    coord_data = [co * sf for co, sf in zip(coord, output_block_shape)]
                    prediction_blocks.append(coord_data)
                    prediction_mask[z, y, x] = 1

    with open(output_file, 'w') as f:
        json.dump(prediction_blocks, f)

    n_blocks_total = np.prod(blocked_shape)
    print("Number of blocks to predict", len(prediction_blocks))
    print("Percentage of blocks that will be predicted", len(prediction_blocks) / n_blocks_total)

    return prediction_mask

def order_blocks(block_file, out_file, central_coordinate, resolution=(1, 1, 1)):
    assert isinstance(central_coordinate, np.ndarray)
    assert len(central_coordinate) == 3
    with open(block_file, 'r') as f:
        prediction_blocks = np.array(json.load(f))

    distances = np.sum(np.square(np.multiply(prediction_blocks, resolution) -
                                 np.multiply(central_coordinate, resolution)), axis=1)
    sort = np.argsort(distances)

    prediction_blocks = prediction_blocks[sort]

    with open(out_file, 'w') as f:
        json.dump(prediction_blocks.tolist(), f)


text:
#!/usr/bin/python3

import matplotlib.pyplot as plt
import numpy as np

def show_image(image, width, height):
    plt.imshow(image.reshape([width, height]))
    plt.gray()
    plt.show()

def create_sprite_image(images, save_path):
    """Creates and save a sprite image consisting of images passed as argument.

    Arguments:
        images: numpy like array with shape [num_images, height, width].
        save_path: path to save sprite image.

    Returns:
        sprite: sprite image consisting of the images passed as images.
    """
    if isinstance(images, list):
        images = np.array(images)
    height = images.shape[1]
    width = images.shape[2]
    num_images = int(np.ceil(np.sqrt(images.shape[0])))

    sprite = np.ones((height * num_images, width * num_images))

    for i in range(num_images):
        for j in range(num_images):
            img_idx = i * num_images + j
            if img_idx < images.shape[0]:
                image = images[img_idx]
                sprite[i * height:(i + 1) * height,
                       j * width:(j + 1) * width] = image

    plt.imsave(save_path, sprite, cmap='gray')
    return sprite

def invert_grayscale(images):
    """ Makes black white, and white black """
    return 1 - images

max_stars_repo_name: vianasw/fashion-classifier

max_issues_repo_name: vianasw/fashion-classifier

max_forks_repo_name: vianasw/fashion-classifier


text:
# Copyright 2020 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from enum import Enum
from .lp_tools.LP_formulation import get_value_by_name

class line_type(Enum):
    SCLP_x0 = -1
    SCLP_main = 0
    SCLP_sub = 1
    SCLP_orthogonal = 2

class parametric_line():

    def __init__(self, x_0, q_N, theta_bar, T=0, del_T =1, del_x_0=None, del_q_N=None, Kset_0=None, Jset_N=None, B1=None, B2=None, ltype = line_type.SCLP_main):
        self._x_0 = x_0
        self._q_N = q_N
        self._theta_bar = theta_bar
        self._T = T
        self._del_T = del_T
        self._del_x_0 = del_x_0
        self._del_q_N = del_q_N
        self._Kset_0 = Kset_0
        self._Jset_N = Jset_N
        self._B1 = B1
        self._B2 = B2
        self._ltype = ltype
        self._theta = 0
        self._back_direction = False

    def build_boundary_sets(self, klist, jlist):
        if self._del_x_0 is None:
            self._Kset_0 = klist[self._x_0 > 0]
        else:
            self._Kset_0 = klist[np.logical_or(self._x_0 > 0, self._del_x_0 > 0)]
        if self._del_q_N is None:
            self._Jset_N = jlist[self._q_N > 0]
        else:
            self._Jset_N = jlist[np.logical_or(self._q_N > 0, self._del_q_N > 0)]

    def scale(self, factor):
        self._theta_bar *= factor
        self._theta *= factor
        self._del_T /= factor
        if self._del_x_0 is not None:
            self._del_x_0 /= factor
        if self._del_q_N is not None:
            self._del_q_N /= factor

    @property
    def x_0(self):
        return self._x_0

    @property
    def q_N(self):
        return self._q_N

    @property
    def del_x_0(self):
        if self._back_direction and self._del_x_0 is not None:
            return -self._del_x_0
        else:
            return self._del_x_0

    @property
    def del_q_N(self):
        if self._back_direction and self._del_q_N is not None:
            return -self._del_q_N
        else:
            return self._del_q_N

    @property
    def T(self):
        return self._T

    @T.setter
    def T(self, value):
        self._T = value

    @property
    def del_T(self):
        if self._back_direction:
            return -self._del_T
        else:
            return self._del_T

    @property
    def Kset_0(self):
        return self._Kset_0

    @property
    def Jset_N(self):
        return self._Jset_N

    @property
    def theta_bar(self):
        return self._theta_bar

    @theta_bar.setter
    def theta_bar(self, value):
        if value < self._theta:
            self._back_direction = True
        else:
            self._back_direction = False
        self._theta_bar = value

    @property
    def B1(self):
        return self._B1

    @property
    def B2(self):
        return self._B2

    @property
    def theta(self):
        return self._theta

    def is_main(self):
        return self._ltype == line_type.SCLP_main

    def is_sub(self):
        return self._ltype == line_type.SCLP_sub

    def is_orthogonal(self):
        return self._ltype == line_type.SCLP_orthogonal

    def is_end(self, delta):
        if self._back_direction:
            return self._theta - delta <= self._theta_bar
        else:
            return self._theta + delta >= self._theta_bar

    def forward_to_end(self):
        self._forward_to(self._theta_bar - self._theta)

    def _forward_to(self, delta):
        if self._del_x_0 is not None:
            self._x_0 += self._del_x_0 * delta
        if self._del_q_N is not None:
            self._q_N += self._del_q_N * delta
        self._T += self._del_T * delta
        self._theta += delta

    def forward_to(self, delta):
        if self._back_direction:
            self._backward_to(delta)
        else:
            self._forward_to(delta)

    def backward_to(self, delta):
        if self._back_direction:
            self._forward_to(delta)
        else:
            self._backward_to(delta)

    def _backward_to(self, delta):
        if self._del_x_0 is not None:
            self._x_0 -= self._del_x_0 * delta
        if self._del_q_N is not None:
            self._q_N -= self._del_q_N * delta
        self._T -= self._del_T * delta
        self._theta -= delta

    def get_orthogonal_line(self, theta_bar= 1, type=1):
        del_x = None
        del_q = None
        if type == 1 or type == 3:
            del_x = np.zeros_like(self._x_0)
            del_x[self._Kset_0-1] = np.random.rand(len(self._Kset_0)) - 0.5
            theta_bar =min(theta_bar, 1/np.max(np.divide(-del_x, self._x_0, where = np.logical_and(del_x < 0, self._Kset_0))))
        if type == 2 or type == 3:
            del_q = np.zeros_like(self._q_N)
            del_q[self._Jset_N-1] = np.random.rand(len(self._Jset_N)) - 0.5
            theta_bar = min(theta_bar, 1/np.max(np.divide(-del_q, self._q_N, where=np.logical_and(del_q < 0, self._Jset_N))))
        return parametric_line(self._x_0, self._q_N, theta_bar, self._theta, 0, del_x, del_q,
                               self._Kset_0, self._Jset_N, None, None, line_type.SCLP_orthogonal)

    def get_x0_parline(self, solution, v1, v_x0):
        lk1 = solution.klist == v1
        del_x_0 = np.zeros_like(self._x_0)
        del_x_0[lk1] = v_x0
        return parametric_line(self._x_0, self._q_N, self._T, self._T, 0, del_x_0, np.zeros_like(self._q_N),
                               self._Kset_0, self._Jset_N, None, None, line_type.SCLP_x0)

    @staticmethod
    def get_subproblem_parametric_line(basis, solution, v1, v2, AAN1, AAN2, pbaseB1red, pbaseB2red):
        x_0 = np.zeros(solution.KK, order='C')
        q_N = np.zeros(solution.JJ, order='C')
        del_x_0 = np.zeros(solution.KK, order='C')
        del_q_N = np.zeros(solution.JJ, order='C')
        # Boundary values for one sided subproblem, collision at t=0
        if AAN1 is None:
            if not isinstance(v1, list):
                # The case of v1 > 0, collision case iv_a
                if v1 > 0:
                    dx_DD_v1 = get_value_by_name(basis, v1, True)
                    lk1 = solution.klist == v1
                    x_0[lk1] = -dx_DD_v1
                    del_x_0[lk1] = dx_DD_v1
                # The case of v1 < 0, collision case iii_a
                elif v1 < 0:
                    dq_B2_v1 = get_value_by_name(AAN2, v1, False)
                    lj1 = solution.jlist == -v1
                    del_q_N[lj1] = -dq_B2_v1
        #
        # Boundary values for one sided subproblem, collision at t=T
        elif AAN2 is None:
            if not isinstance(v2, list):
                # The case of v2 > 0, collision case iii_b
                if v2 > 0:
                    dx_B1_v2 = get_value_by_name(AAN1, v2, True)
                    lk2 = solution.klist == v2
                    del_x_0[lk2] = -dx_B1_v2
                # The case of v2 < 0, collision case iv_b
                elif v2 < 0:
                    dq_DD_v2 = get_value_by_name(basis, v2, False)
                    lj2 = solution.jlist == -v2
                    q_N[lj2] = -dq_DD_v2
                    del_q_N[lj2] = dq_DD_v2
        #
        # Boundary values for two sided subproblem, collision at 0<t<T
        #  setting boundaries for the second exiting variable v1
        else:
            if not isinstance(v1, list):
                if v1 > 0:
                    dx_DD_v1 = get_value_by_name(basis, v1, True)
                    lk1 = solution.klist == v1
                    x_0[lk1] = -dx_DD_v1
                    dx_B1_v1 = get_value_by_name(AAN1, v1, True)
                    del_x_0[lk1] = -0.5 * dx_B1_v1 + dx_DD_v1
                elif v1 < 0:
                    dq_B2_v1 = get_value_by_name(AAN2, v1, False)
                    lj1 = solution.jlist == -v1
                    del_q_N[lj1] = -0.5 * dq_B2_v1
            #  setting boundaries for the first exiting variable v2
            if not isinstance(v2, list):
                if v2 > 0:
                    dx_B1_v2 = get_value_by_name(AAN1, v2, True)
                    lk2 = solution.klist == v2
                    del_x_0[lk2] = -0.5 * dx_B1_v2
                elif v2 < 0:
                    dq_DD_v2 = get_value_by_name(basis, v2, False)
                    lj2 = solution.jlist == -v2
                    q_N[lj2] = -dq_DD_v2
                    dq_B2_v2 = get_value_by_name(AAN2, v2, False)
                    del_q_N[lj2] = -0.5 * dq_B2_v2 + dq_DD_v2
        par_line = parametric_line(x_0, q_N, 1, 1, 0, del_x_0, del_q_N, None, None, pbaseB1red, pbaseB2red, line_type.SCLP_sub)
        par_line.build_boundary_sets(solution.klist, solution.jlist)
        return par_line

max_stars_repo_name: haroldship/SCLPsolver

max_issues_repo_name: haroldship/SCLPsolver

max_forks_repo_name: haroldship/SCLPsolver


text:
from keras.models import load_model
import numpy as np
from keras import backend as K
from load import add_tweets, classes, binary, model_name, data_folder, subclasses
from sklearn.utils import shuffle
import codecs

def test_model(model_name, classes, subclasses, binary, test_tweets):
	def f1(y_true, y_pred):
		def recall(y_true, y_pred):
			true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
			possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
			recall = true_positives / (possible_positives + K.epsilon())
			return recall
		def precision(y_true, y_pred):
			true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
			predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
			precision = true_positives / (predicted_positives + K.epsilon())
			return precision
		precision = precision(y_true, y_pred)
		recall = recall(y_true, y_pred)
		return 2 * ((precision * recall) / (precision + recall + K.epsilon()))

	model = load_model("../models/" + model_name)

	confusion_matrix = np.zeros((len(classes), len(classes)), dtype=int)
	expanded_confusion_matrix = np.zeros((len(subclasses), len(classes)), dtype=int)
	subclass_counter = 0
	for i in range(len(test_tweets)):
		for j in range(len(test_tweets[i])):
			results = model.predict(np.array(test_tweets[i][j]))
			for k in range(len(test_tweets[i][j])):
				if binary:
					if results[k][0] == 0.5:
						predicted_as = 1 - i
					else:
						predicted_as = int(round(results[k][0]))
				else:
					predicted_as = int(round(results[k][0]))
				expanded_confusion_matrix[subclass_counter, predicted_as] += 1
				confusion_matrix[i, predicted_as] += 1
			subclass_counter += 1
	print(expanded_confusion_matrix)

	return confusion_matrix, expanded_confusion_matrix

def common_metrics(confusion_matrix, expanded_confusion_matrix):
	precisions = []
	recalls = []
	trueps = []
	falsens = []
	falseps = []
	for i in range(len(classes)):
		truep = confusion_matrix[i, i]
		falsen = 0
		falsep = 0
		for j in range(len(classes)):
			if j != i:
				falsen += confusion_matrix[i, j]
				falsep += confusion_matrix[j, i]
		trueps.append(truep)
		falsens.append(falsen)
		falseps.append(falsep)
		if truep == 0:
			precision = 0
		else:
			precision = truep / (truep + falsep)
		recall = truep / (truep + falsen)
		precisions.append(precision)
		recalls.append(recall)
		if precision == 0 and recall == 0:
			f1 = 0
		else:
			f1 = 2 * precision * recall / (precision + recall)
		print("Class", i, "Precision:", precision, "Recall:", recall, "F1-score:", f1)
	micro_avg_precision = sum(trueps) / (sum(trueps) + sum(falseps))
	micro_avg_recall = sum(trueps) / (sum(trueps) + sum(falsens))
	print(micro_avg_precision, micro_avg_recall)
	if micro_avg_precision == 0 and micro_avg_recall == 0:
		micro_avg_f1score = 0
	else:
		micro_avg_f1score = 2 * micro_avg_precision * micro_avg_recall / (micro_avg_precision + micro_avg_recall)
	print("Micro-averaged F1-score:", micro_avg_f1score)
	macro_avg_precision = sum(precisions) / len(classes)
	macro_avg_recall = sum(recalls) / len(classes)
	if macro_avg_precision == 0 and macro_avg_recall == 0:
		macro_avg_f1score = 0
	else:
		macro_avg_f1score = 2 * macro_avg_precision * macro_avg_recall / (macro_avg_precision + macro_avg_recall)
	print("Macro-averaged F1-score:", macro_avg_f1score)
	accuracy = sum(trueps) / confusion_matrix.sum()
	print("Accuracy:", accuracy)
	return micro_avg_f1score, macro_avg_f1score, accuracy

if __name__ == "__main__":
	test_tweets = []

	tweets = []
	for i in range(len(classes)):
		a = []
		length = 0
		for j in range(len(classes[i])):
			t = []
			fp = codecs.open(data_folder + "/" + classes[i][j] + "_test.txt", "r", "utf-8")
			ts = fp.readlines()
			add_tweets(ts, t)
			fp.close()
			a.append(t)
			length += len(t)
		tweets.append(a)
	for i in range(len(classes)):
		test = []
		for j in range(len(classes[i])):
			test.append(tweets[i][j])
		test_tweets.append(test)
	confusion_matrix, expanded_confusion_matrix = test_model(model_name, classes, subclasses, binary, test_tweets)
	print()
	common_metrics(confusion_matrix, expanded_confusion_matrix)


text:
"""
=======================================================
1D contour with multiple data sets by changing settings
=======================================================

Plot good figures with multiple data sets using more settings.
"""
import coplot.plot_contours as plc
import coplot.plots as pl
import matplotlib.pyplot as plt
import numpy as np

a1 = np.random.randn(10000)
a2 = a1 + 1

# If you want to plot good figures, you should use it like this:
fig = plc.Plot_1d([a1,a2]).plot(bins=150,labels=[r'$a$', r'$\rm PDF$'],colors=['r','k'],\
line_styles=['-','--'], smooth=3, ticks_size=10, legend=True, legend_labels=['Data 1', 'Data 2'])
# and you can save the figure like this:
# pl.savefig('test_1d_3.pdf', fig)
plt.show()

max_stars_repo_name: Guo-Jian-Wang/coplot

max_issues_repo_name: Guo-Jian-Wang/coplot

max_forks_repo_name: Guo-Jian-Wang/coplot


text:
import numpy as np
import os.path as osp
from rlkit.data_management.simple_replay_buffer import SimpleReplayBuffer
from gym.spaces import Box, Discrete, Tuple, MultiBinary
from rlkit.torch.multitask.pointmass_rewards import PointMassBestRandomRelabeler
from rlkit.torch.multitask.gym_relabelers import ReacherRelabelerWithGoalAndObs
from rlkit.util.utils import save_video
from rlkit.core import logger

class MultiTaskReplayBuffer(SimpleReplayBuffer):
    def __init__(
            self,
            max_replay_buffer_size,
            env,
            relabeler,
            latent_dim,
            normalize_rewards=False,
            grid_normalize=False,
            on_policy=False,
            plot=False,
            dads=False,
            approx_irl=False,
            hide_latent=False,
            permute_relabeling=False,
            add_random_relabeling=False
    ):
        """
        :param max_replay_buffer_size:
        """
        self.env = env
        self._action_space = env.action_space
        self._observation_space = env.observation_space
        if isinstance(self.env.action_space, Discrete):
            action_dim = env.action_space.n
        else:
            action_dim = env.action_space.low.size

        if isinstance(self.env.observation_space, Discrete) or isinstance(self.env.observation_space, MultiBinary):
            obs_dim = env.observation_space.n
        else:
            obs_dim = env.observation_space.low.size

        self.latent_dim = latent_dim
        self._latents = np.zeros((max_replay_buffer_size, latent_dim))
        self.relabeler = relabeler

        self._observation_dim = obs_dim
        self._action_dim = action_dim
        self._max_replay_buffer_size = max_replay_buffer_size
        self._observations = np.zeros((max_replay_buffer_size, obs_dim))
        # It's a bit memory inefficient to save the observations twice,
        # but it makes the code *much* easier since you no longer have to
        # worry about termination conditions.
        self._next_obs = np.zeros((max_replay_buffer_size, obs_dim))
        self._actions = np.zeros((max_replay_buffer_size, action_dim))
        # Make everything a 2D np array to make it easier for other code to
        # reason about the shape of the data
        self._rewards = np.zeros((max_replay_buffer_size, 1))
        # self._terminals[i] = a terminal was received at time i
        self._terminals = np.zeros((max_replay_buffer_size, 1), dtype='uint8')
        # Define self._env_infos[key][i] to be the return value of env_info[key]
        # at time i
        self.normalize_rewards = normalize_rewards
        self.grid_normalize = grid_normalize
        self.on_policy = on_policy
        self.dads = dads
        self.hide_latent = hide_latent
        self.permute_relabeling = permute_relabeling
        self.add_random_relabeling = add_random_relabeling
        self.permutation_list = []
        if dads:
            self._qpos = np.zeros((max_replay_buffer_size, 2))
            self._next_qpos = np.zeros((max_replay_buffer_size, 2))
        self.plot = plot
        if plot:
            self.trajectory_latents = []
        self.approx_irl = approx_irl

        self._top = 0
        self._size = 0
        self.all_relabeling_info = dict()
        self.original_latents = []
        self.relabeled_latents = []
        self.features = []
        self.original_rewards = []
        self.relabeled_rewards = []
        float_formatter = "{:.2f}".format
        np.set_printoptions(formatter={'float_kind': float_formatter})

        self.epoch = 0
        self.paths_this_epoch = 0

    def add_sample(self, observation, action, reward, terminal, next_observation, **kwargs):
        raise NotImplementedError("Only use add_path")

    def add_single_sample(self, latent, observation, action, reward, terminal, next_observation, **kwargs):
        if isinstance(self._action_space, Discrete):
            action = np.eye(self._action_space.n)[action]
        self._observations[self._top] = observation
        self._actions[self._top] = action
        self._rewards[self._top] = reward
        self._terminals[self._top] = terminal
        self._latents[self._top] = latent
        self._next_obs[self._top] = next_observation
        self._advance()

    def add_single_dads_sample(self, latent, observation, action, terminal, next_observation, qpos, next_qpos):
        if isinstance(self._action_space, Discrete):
            action = np.eye(self._action_space.n)[action]
        self._observations[self._top] = observation
        self._actions[self._top] = action
        self._terminals[self._top] = terminal
        self._latents[self._top] = latent
        self._next_obs[self._top] = next_observation
        self._qpos[self._top] = qpos
        self._next_qpos[self._top] = next_qpos
        self._advance()

    def sample_data(self, indices):
        return dict(
            observations=self._observations[indices],
            actions=self._actions[indices],
            latents=self._latents[indices],
            rewards=self._rewards[indices],
            terminals=self._terminals[indices],
            next_observations=self._next_obs[indices],
        )

    def add_path(self, path):
        # add with original z and resampled z
        original_z = path["latents"][0]

        """
            NEED TO REPLACE GET LATENTS AND REWARDS FOR SKILLS
        """
        resampled_zs, reward_list, ranked_latents = self.relabeler.get_latents_and_rewards(path)
        if self.permute_relabeling:
            self.permutation_list.append((path, resampled_zs[0]))
        if not any([np.array_equal(original_z, z) for z in resampled_zs]):
            resampled_zs.append(original_z)
            reward_list.append(self.relabeler.calculate_path_reward(path, original_z))
        if self.add_random_relabeling:
            random_z = self.relabeler.sample_task()
            resampled_zs.append(random_z)
            reward_list.append(self.relabeler.calculate_path_reward(path, random_z))

        # save video if applicable, definitely save
        original_discounted_reward = self.relabeler.get_discounted_reward(reward_list[-1])
        if 'rgb_array' in path.keys():
            save_path = osp.join(logger.get_snapshot_dir(), "epoch{}_{}.mp4".format(self.epoch, self.paths_this_epoch))
            text_original = "{}, {}, {:.2f}".format(str(original_z),
                                       str(self.relabeler.get_features(path, latent=original_z)),
                                       original_discounted_reward)
            text_relabeled = "{}, {:.2f}".format(str(resampled_zs[0]), self.relabeler.get_discounted_reward(reward_list[0]))
            text = text_original + "; " + text_relabeled
            save_video(path['rgb_array'], save_path, text)
        self.original_latents.append(original_z)
        self.relabeled_latents.append(resampled_zs[:-1])
        self.features.append(self.relabeler.get_features(path))
        self.original_rewards.append(original_discounted_reward)
        self.relabeled_rewards.append([self.relabeler.get_discounted_reward(reward) for reward in reward_list[:-1]])

        if self.hide_latent:
            resampled_zs = [np.zeros_like(z) for z in resampled_zs]
        for z, rewards in zip(resampled_zs, reward_list):
            for i, (
                    obs,
                    action,
                    reward,
                    next_obs,
                    terminal,
                    agent_info,
            ) in enumerate(zip(
                path["observations"],
                path["actions"],
                rewards,
                path["next_observations"],
                path["terminals"],
                path["agent_infos"],
            )):
                self.add_single_sample(
                    z,
                    obs,
                    action,
                    reward,
                    terminal,
                    next_obs,
                )

        self.terminate_episode()
        if self.plot and len(self.trajectory_latents) < 3:
            ranked_latents.insert(0, original_z)
            self.trajectory_latents.append((path, ranked_latents))  # trajectory, original_z, new_z
        self.paths_this_epoch += 1

    def add_dads_path(self, path):
        z = path["latents"][0]
        for i, (
            obs,
            action,
            next_obs,
            terminal,
            agent_info,
            qpos,
            next_qpos,
        ) in enumerate(zip(
            path["observations"],
            path["actions"],
            path["next_observations"],
            path["terminals"],
            path["agent_infos"],
            path['qpos'],
            path['next_qpos']
        )):
            self.add_single_dads_sample(
                z,
                obs,
                action,
                terminal,
                next_obs,
                qpos,
                next_qpos,
            )

        self.terminate_episode()

    def add_path_fixed_latent(self, path, rewards, latent):
        for i, (
                obs,
                action,
                reward,
                next_obs,
                terminal,
                agent_info,
        ) in enumerate(zip(
            path["observations"],
            path["actions"],
            rewards,
            path["next_observations"],
            path["terminals"],
            path["agent_infos"],
        )):
            self.add_single_sample(
                latent,
                obs,
                action,
                reward,
                terminal,
                next_obs,
            )

        self.terminate_episode()

    def add_paths(self, paths):
        if self.dads:
            for path in paths:
                self.add_dads_path(path)
        elif self.normalize_rewards or self.approx_irl:
            assert not self.hide_latent
            if self.normalize_rewards:
                # sample a bunch of latents
                # calculate the rewards of each trajectory under each latent
                # take each of them and normalize
                # label based on the best normalized
                latents, rewards = self.relabeler.normalize_path_returns(paths, use_grid=self.grid_normalize)  #latents should be a list of lists, same for rewards
            elif self.approx_irl:
                latents, rewards = self.relabeler.approx_irl_relabeling(paths)
            else:
                raise RuntimeError

            self.relabeled_latents.extend(latents)
            orig_latents = [[path['latents'][0]] for path in paths]
            orig_rewards = [[self.relabeler.calculate_path_reward(path, z[0])] for path, z in zip(paths, orig_latents)]

            assert len(latents) == len(rewards)
            assert len(orig_latents) == len(orig_rewards)
            assert len(latents) == len(orig_latents)
            if self.plot and isinstance(self.relabeler, PointMassBestRandomRelabeler):
                self.relabeler.plot_paths(paths, orig_latents, latents, title='Epoch {}'.format(str(self.epoch)))
                self.trajectory_latents = None

            for i in range(len(latents)):
                latents[i].extend(orig_latents[i])
                rewards[i].extend(orig_rewards[i])

            # save videos if necessary
            for i, path in enumerate(paths):
                original_z = path['latents'][0]
                original_discounted_reward = self.relabeler.get_discounted_reward(rewards[i][-1])
                if 'rgb_array' in path.keys():
                    save_path = osp.join(logger.get_snapshot_dir(),
                                         "epoch{}_{}.mp4".format(self.epoch, self.paths_this_epoch))
                    text_original = "{}, {}, {:.2f}".format(str(original_z),
                                                        str(self.relabeler.get_features(path, latent=original_z)),
                                                        original_discounted_reward)
                    text_relabeled = "{}, {:.2f}".format(str(latents[i][0]),
                                                        self.relabeler.get_discounted_reward(rewards[i][0]))
                    text = text_original + "; " + text_relabeled
                    save_video(path['rgb_array'], save_path, text)
                self.original_latents.append(original_z)
                self.relabeled_latents.append(latents[i][:-1])
                self.features.append(self.relabeler.get_features(path))
                self.original_rewards.append(original_discounted_reward)
                self.relabeled_rewards.append([self.relabeler.get_discounted_reward(reward) for reward in rewards[i][:-1]])
                self.paths_this_epoch += 1

            for path, reward_list, latent_list in zip(paths, rewards, latents):
                assert len(reward_list) == len(latent_list)
                for r, z in zip(reward_list, latent_list):
                    self.add_path_fixed_latent(path, r, z)
                if self.add_random_relabeling:
                    random_z = self.relabeler.sample_task()
                    random_r = self.relabeler.calculate_path_reward(path, random_z)
                    self.add_path_fixed_latent(path, random_r, random_z)
                if self.permute_relabeling:
                    self.permutation_list.append((path, z))
        else:
            if self.relabeler.sliding_normalization:
                self.relabeler.update_sliding_params(paths)
            for path in paths:
                self.add_path(path)
        if self.permute_relabeling:
            self.handle_permuting()

    def random_batch(self, batch_size):
        indices = np.random.randint(0, self._size, batch_size)
        if self.dads:
            return dict(
                observations=self._observations[indices],
                actions=self._actions[indices],
                terminals=self._terminals[indices],
                next_observations=self._next_obs[indices],
                latents=self._latents[indices],
                qpos=self._qpos[indices],
                next_qpos=self._next_qpos[indices],
            )
        else:
            return dict(
                # observations=np.concatenate([self._observations[indices], self._latents[indices]], axis=1),
                observations=self._observations[indices],
                actions=self._actions[indices],
                rewards=self._rewards[indices],
                terminals=self._terminals[indices],
                next_observations=self._next_obs[indices],
                # next_observations=np.concatenate([self._next_obs[indices], self._latents[indices]], axis=1),
                latents=self._latents[indices],
            )

    def handle_permuting(self):
        latents = [pair[1] for pair in self.permutation_list]
        paths = [pair[0] for pair in self.permutation_list]
        if len(latents) <= 1:
            print("No permuting happening, only {} paths".format(len(latents)))
            return
        print("permuting {} paths".format(len(latents)))
        new_latents = latents[1:] + [latents[0]]
        assert len(new_latents) == len(paths)
        for path, z in zip(paths, new_latents):
            r = self.relabeler.calculate_path_reward(path, z)
            self.add_path_fixed_latent(path, r, z)
        self.permutation_list = []

    def end_epoch(self, epoch):
        if self.on_policy:
            self._top = 0
            self._size = 0
        if self.plot:
            # plot everything
            if self.relabeler.use_adv and False:
                # need to calculate the rewards, q1, q2, and advantage
                disc_rewards = np.array([[self.relabeler.get_discounted_path_reward(path, latent) for latent in latents]
                                for path, latents in self.trajectory_latents])
                values = np.array([self.relabeler.get_both_values(path['observations'][0], np.array(latents))
                                   for path, latents in self.trajectory_latents]).squeeze()
                v1, v2 = values[:,0,:], values[:,1,:]
                if self.relabeler.subtract_final_value:
                    print('subtracting final value')
                    final_values = np.array([self.relabeler.get_both_values(path['next_observations'][-1], np.array(latents))
                                   for path, latents in self.trajectory_latents]).squeeze()
                    v1 -= final_values[:,0,:] * self.relabeler.discount ** \
                                        np.array([len(path['observations']) for path, _ in self.trajectory_latents])[...,np.newaxis]
                    v2 -= final_values[:,1,:] * self.relabeler.discount ** \
                                        np.array([len(path['observations']) for path, _ in self.trajectory_latents])[...,np.newaxis]
                adv = disc_rewards - values.min(axis=1)
                traj_infos = dict(
                    rewards=disc_rewards,
                    v1=v1,
                    v2=v2,
                    adv=adv,
                )
            else:
                traj_infos = None
            if isinstance(self.relabeler, PointMassBestRandomRelabeler):
                self.relabeler.plot_resampling_heatmaps(self.trajectory_latents, "Epoch" + str(self.epoch), traj_infos=traj_infos)
            if isinstance(self.relabeler, ReacherRelabelerWithGoalAndObs):
                self.relabeler.plot_resampling_heatmaps(self.trajectory_latents, "Epoch" + str(self.epoch), traj_infos=traj_infos)
            self.trajectory_latents = []
        self.permutation_list = []

        # save the relabeling information here
        relabeling_info = {"original_latents": np.array(self.original_latents),
                            "relabeled_latents": np.array(self.relabeled_latents),
                            "features": np.array(self.features),
                            "original_rewards": np.array(self.original_rewards),
                            "relabeled_rewards": np.array(self.relabeled_rewards)}
        self.all_relabeling_info['epoch{}'.format(self.epoch)] = relabeling_info
        logger.save_extra_data(self.all_relabeling_info, 'relabeling_info.pkl')
        self.original_latents = []
        self.relabeled_latents = []
        self.features = []
        self.original_rewards = []
        self.relabeled_rewards = []
        self.epoch += 1
        self.paths_this_epoch = 0

    def get_snapshot(self):
        return dict(relabeler=self.relabeler)

max_stars_repo_name: YashBit/accelerate-skillDiscovery

max_issues_repo_name: YashBit/accelerate-skillDiscovery

max_forks_repo_name: YashBit/accelerate-skillDiscovery