#!/usr/bin/env python
import cv2
import rospy
import numpy as np
import PIL.Image as Image
import PIL.ImageOps as ImageOps
from button_recognition.msg import recognition
from button_recognition.msg import recog_result
from button_recognition.srv import *
from ocr_rcnn_lib.button_recognition import ButtonRecognizer
from cv_bridge import CvBridge
from sensor_msgs.msg import Image as Image_msg
from sensor_msgs.msg import CameraInfo
import pyrealsense2 as rs2
from visualization_msgs.msg import Marker

def resize_to_480x680(img):
    print("shape of panel image input: {}".format(img.shape))
    if img.shape != (480, 640):
        img_pil = Image.fromarray(img)
        img_thumbnail = img_pil.thumbnail((640, 480), Image.ANTIALIAS)
        delta_w, delta_h = 640 - img_pil.size[0], 480 - img_pil.size[1]
        padding = (delta_w // 2, delta_h // 2, delta_w - (delta_w // 2), delta_h - (delta_h // 2))
        new_im = ImageOps.expand(img_pil, padding)
        img = np.copy(np.asarray(new_im))

    print('result shape after resize', img.shape)
    return img

class RecognitionService:
  def __init__(self, model):
    self.model = model
    assert isinstance(self.model, ButtonRecognizer)
    self.cv_bridge = CvBridge()
    self.latest_im = None
    rospy.Subscriber("/camera/color/image_raw", Image_msg, self.im_callback)

    self.latest_depth_im = None
    rospy.Subscriber("/camera/aligned_depth_to_color/image_raw", Image_msg, self.depth_callback)

    self.intrinsics = None
    rospy.Subscriber('/camera/aligned_depth_to_color/camera_info', CameraInfo, self.imageDepthInfoCallback)

    self.marker_pub = rospy.Publisher('/button', Marker, queue_size=1, latch=True)

  def depth_callback(self, data):
    depth_image = self.cv_bridge.imgmsg_to_cv2(data, desired_encoding='passthrough')
    depth_array = np.array(depth_image, dtype=np.float32)
    self.latest_depth_im = depth_array

  def im_callback(self, data):
    cv_image = self.cv_bridge.imgmsg_to_cv2(data, desired_encoding='passthrough')
    self.latest_im = cv_image

  def imageDepthInfoCallback(self, cameraInfo):
    try:
      if self.intrinsics:
        return
      self.intrinsics = rs2.intrinsics()
      self.intrinsics.width = cameraInfo.width
      self.intrinsics.height = cameraInfo.height
      self.intrinsics.ppx = cameraInfo.K[2]
      self.intrinsics.ppy = cameraInfo.K[5]
      self.intrinsics.fx = cameraInfo.K[0]
      self.intrinsics.fy = cameraInfo.K[4]
      if cameraInfo.distortion_model == 'plumb_bob':
        self.intrinsics.model = rs2.distortion.brown_conrady
      elif cameraInfo.distortion_model == 'equidistant':
        self.intrinsics.model = rs2.distortion.kannala_brandt4
      self.intrinsics.coeffs = [i for i in cameraInfo.D]
    except CvBridgeError as e:
      print(e)
      return

  def perform_recognition(self, request):
    button_text = request.buttonText
    start = rospy.get_time()

    video = False
    ##should change to subscriber
    # image_name = "/home/zixianzang/Desktop/bag_image/170.jpg"
    # im = cv2.imread(image_name)
    #
    # depth_name = "/home/zixianzang/Desktop/bag_depth_images/170.npy"
    # depth = np.load(depth_name)

    im = self.latest_im.copy()
    depth = self.latest_depth_im.copy()
    image_np = resize_to_480x680(im.copy())

    recognitions = self.model.predict(image_np)

    _ = self.model.predict(image_np, draw=True)
    cv2.imwrite("/home/zixianzang/Desktop/result.jpg", image_np)

    if video:
      for i in range(600, 1100):
        print(i)
        im_dir = "/home/zixianzang/Desktop/bag_image/{}.jpg".format(str(i))
        im_ = cv2.imread(im_dir, cv2.IMREAD_COLOR)
        image_np_ = resize_to_480x680(im_.copy())
        _ = self.model.predict(image_np_, draw=True)
        cv2.imwrite("/home/zixianzang/Desktop/detected_image/{}.jpg".format(str(i)), image_np_)

    recog_resp = recog_result()
    exist = False

    #hack implementation for up and down
    if len(recognitions) == 2:
      y_min_1 = recognitions[0][0][0]
      y_min_2 = recognitions[1][0][0]
      if y_min_2 > y_min_1:
        recognitions[1][2] = ')'
        recognitions[0][2] = '('
      elif y_min_2 < y_min_1:
        recognitions[1][2] = '('
        recognitions[0][2] = ')'

    for item in recognitions:
      if item[2] == button_text:
        sample = recognition()
        ymin = int(item[0][0] * image_np.shape[0])
        xmin = int(item[0][1] * image_np.shape[1])
        ymax = int(item[0][2] * image_np.shape[0])
        xmax = int(item[0][3] * image_np.shape[1])
        print(ymin, ymax)
        exist = True

        originHeight, originWidth = float(im.shape[0]), float(im.shape[1])
        ymin, ymax = float(ymin), float(ymax)
        xmin, xmax = float(xmin), float(xmax)

        if originHeight/480 > originWidth/640:
          print('tall')
          shrink_ratio = originHeight/480
          origin_ymin, origin_ymax = int(ymin*shrink_ratio), int(ymax*shrink_ratio)
          xpad = (640 - originWidth / shrink_ratio)/2
          origin_xmin, origin_xmax = int((xmin-xpad)*shrink_ratio), int((xmax-xpad)*shrink_ratio)
        elif originHeight/480 < originWidth/640:
          print('wide')
          shrink_ratio = originWidth / 640
          origin_xmin, origin_xmax = int(xmin * shrink_ratio), int(xmax * shrink_ratio)
          ypad = (480 - originHeight / shrink_ratio) / 2
          origin_ymin, origin_ymax = int((ymin - ypad) * shrink_ratio), int((ymax - ypad) * shrink_ratio)

        else:
          print('same ratio')
          shrink_ratio = originWidth / 640
          origin_xmin, origin_xmax = int(xmin * shrink_ratio), int(xmax * shrink_ratio)
          origin_ymin, origin_ymax = int(ymin * shrink_ratio), int(ymax * shrink_ratio)

        sample.score = item[1]  # float
        sample.text = item[2]
        sample.belief = item[3]
        sample.x_min = origin_xmin
        sample.y_min = origin_ymin
        sample.x_max = origin_xmax
        sample.y_max = origin_ymax

        x_mid = int((origin_xmin+origin_xmax) / 2)
        y_mid = int((origin_ymin + origin_ymax) / 2)

        result = rs2.rs2_deproject_pixel_to_point(self.intrinsics, [x_mid, y_mid], depth[y_mid, x_mid])
        print(result)
        x = result[0]
        y = result[1]
        z = result[2]
        # print(x, y, z)
        sample.x_3d = x
        sample.y_3d = y
        sample.z_3d = z
        recog_resp.data.append(sample)

        show_marker = True
        if show_marker:
          for iter in range(50):
            marker = Marker()
            marker.header.frame_id = "camera_link"
            marker.header.stamp = rospy.get_rostime()
            marker.id = 0
            marker.type = 2  # sphere
            marker.action = 0
            marker.pose.position.x = result[2]/1000
            marker.pose.position.y = -result[0]/1000
            marker.pose.position.z = -result[1]/1000
            marker.pose.orientation.x = 0
            marker.pose.orientation.y = 0
            marker.pose.orientation.z = 0
            marker.pose.orientation.w = 1.0
            marker.scale.x = 0.1
            marker.scale.y = 0.1
            marker.scale.z = 0.1

            marker.color.r = 0.0
            marker.color.g = 1.0
            marker.color.b = 0.0
            marker.color.a = 1.0

            marker.lifetime = rospy.Duration(0)
            self.marker_pub.publish(marker)
            rospy.sleep(0.1)

    ###visualize
    # for b in recog_resp.data:
    #   copy = im.copy()
    #   im[b.y_min:b.y_max, b.x_min:b.x_max, 0] = 0
    #   im[b.y_min:b.y_max, b.x_min:b.x_max, 1] = 255
    #   im[b.y_min:b.y_max, b.x_min:b.x_max, 2] = 0
    #   linewidth = 5
    #   im[b.y_min+linewidth:b.y_max-linewidth] = copy[b.y_min+linewidth:b.y_max-linewidth]
    #   cv2.imshow("detection", im)
    #   cv2.waitKey(0)

    # ###debug
    # if exist:
    #   for b in recog_resp.data:
    #
    #     originHeight, originWidth = float(im.shape[0]), float(im.shape[1])
    #     print(originHeight, originWidth)
    #
    #     ymin, ymax = float(b.y_min), float(b.y_max)
    #     xmin, xmax = float(b.x_min), float(b.x_max)
    #     #taller
    #     if originHeight/480 > originWidth/640:
    #       print('tall')
    #       shrink_ratio = originHeight/480
    #       origin_ymin, origin_ymax = int(ymin*shrink_ratio), int(ymax*shrink_ratio)
    #       xpad = (640-originWidth/shrink_ratio)/2
    #       origin_xmin, origin_xmax = int((xmin-xpad)*shrink_ratio), int((xmax-xpad)*shrink_ratio)
    #
    #     # im[origin_ymin:origin_ymax, origin_xmin:origin_xmax] = 0
    #     # print('origin_cor', origin_ymin, origin_ymax, origin_xmin, origin_xmax)
    #     # im[0:50, 50:100] = 0
    #     # cv2.imwrite("/home/zixianzang/Desktop/pos_origin.jpg", im)

    # ###debug
    # if exist:
    #   for b in recog_resp.data:
    #     image_np[b.y_min:b.y_max, b.x_min:b.x_max] = 0
    #     print(b.y_min, b.y_max, b.x_min, b.x_max)
    #     cv2.imwrite("/home/zixianzang/Desktop/pos.jpg", image_np)

    if not exist:
      rospy.loginfo('no button with text {} found'.format(button_text))
      print('found', [i[2] for i in recognitions])
    end = rospy.get_time()
    rospy.loginfo('Recognition finished: {} buttons are detected using {} seconds!'.format(
      len(recognitions), end-start))
    return recog_serverResponse(recog_resp)

def button_recognition_server():
  rospy.init_node('button_recognition_server', anonymous=True)
  model = ButtonRecognizer(use_optimized=True)
  recognizer = RecognitionService(model)
  service = rospy.Service('recognition_service',
                          recog_server,
                          recognizer.perform_recognition)

  try:
    rospy.spin()
  except KeyboardInterrupt:
    recognizer.model.clear_session()
    rospy.logdebug('Shutting down ROS button recognition module!')
  cv2.destroyAllWindows()

if __name__ == '__main__':
  button_recognition_server()


import threadpool
from collections import deque
from timeit import default_timer as timer

# Function for workers to run
def f(queue, x):
    queue.append(x)

if __name__ == '__main__':
    WORKER_CNT = 10
    RUN_CNT = 10000

    # Construct thread pool with specified workers
    pool = threadpool.ThreadPool(WORKER_CNT)

    # Construct queue to store values in
    q = deque(maxlen=RUN_CNT)

    # Start feeding workers tasks
    start = timer()

    for i in range(RUN_CNT):
        pool.add_task(f, q, i)

    end = timer()

    # Wait
    pool.wait_completion()

    # Check accuracy
    for i in range(RUN_CNT):
        # Make sure all numbers made it into q
        # i.e. all tasks completed successfully
        if i not in q:
            print('%d missing from queue' % (i))
            exit(1)

    print('Tasks completed successfully!')
    print('Elapsed Task Time: %.6f' % (end - start))


import torch
from torch import nn
from torch.nn.utils.rnn import pack_padded_sequence
from torch.nn.utils.rnn import pad_packed_sequence

class _NonLocalBlockND(nn.Module):
    """PyTorch module for NonLocalBlock"""

    def __init__(self, in_channels, inter_channels=None, dimension=3, sub_sample=True, bn_layer=True):
        super(_NonLocalBlockND, self).__init__()

        assert dimension in [1, 2, 3]

        self.dimension = dimension
        self.sub_sample = sub_sample

        self.in_channels = in_channels
        self.inter_channels = inter_channels

        if self.inter_channels is None:
            self.inter_channels = in_channels // 2
            if self.inter_channels == 0:
                self.inter_channels = 1

        if dimension == 3:
            conv_nd = nn.Conv3d
            max_pool_layer = nn.MaxPool3d(kernel_size=(1, 2, 2))
            bn = nn.BatchNorm3d
        elif dimension == 2:
            conv_nd = nn.Conv2d
            max_pool_layer = nn.MaxPool2d(kernel_size=(2, 2))
            bn = nn.BatchNorm2d
        else:
            conv_nd = nn.Conv1d
            max_pool_layer = nn.MaxPool1d(kernel_size=(2))
            bn = nn.BatchNorm1d

        self.g = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,
                         kernel_size=1, stride=1, padding=0)

        if bn_layer:
            self.W = nn.Sequential(
                conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,
                        kernel_size=1, stride=1, padding=0),
                bn(self.in_channels)
            )
            nn.init.constant_(self.W[1].weight, 0)
            nn.init.constant_(self.W[1].bias, 0)
        else:
            self.W = conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,
                             kernel_size=1, stride=1, padding=0)
            nn.init.constant_(self.W.weight, 0)
            nn.init.constant_(self.W.bias, 0)

        self.theta = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,
                             kernel_size=1, stride=1, padding=0)

        self.phi = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,
                           kernel_size=1, stride=1, padding=0)

        if sub_sample:
            self.g = nn.Sequential(self.g, max_pool_layer)
            self.phi = nn.Sequential(self.phi, max_pool_layer)

    def forward(self, x, return_nl_map=False):
        batch_size = x.size(0)

        g_x = self.g(x).view(batch_size, self.inter_channels, -1)
        g_x = g_x.permute(0, 2, 1)

        theta_x = self.theta(x).view(batch_size, self.inter_channels, -1)
        theta_x = theta_x.permute(0, 2, 1)
        phi_x = self.phi(x).view(batch_size, self.inter_channels, -1)
        f = torch.matmul(theta_x, phi_x)
        N = f.size(-1)
        f_div_C = f / N

        y = torch.matmul(f_div_C, g_x)
        y = y.permute(0, 2, 1).contiguous()
        y = y.view(batch_size, self.inter_channels, *x.size()[2:])
        W_y = self.W(y)
        z = W_y + x

        if return_nl_map:
            return z, f_div_C
        return z

class NONLocalBlock2D(_NonLocalBlockND):
    """PyTorch module for NonLocalBlock2D"""
    def __init__(self, in_channels, inter_channels=None, sub_sample=True, bn_layer=True):
        super(NONLocalBlock2D, self).__init__(in_channels,
                                              inter_channels=inter_channels,
                                              dimension=2, sub_sample=sub_sample,
                                              bn_layer=bn_layer)

# CONVLSTM: https://github.com/ndrplz/ConvLSTM_pytorch/blob/master/convlstm.py
class ConvLSTMCell(nn.Module):
    """PyTorch module for ConvLSTM Cell"""
    def __init__(self, input_dim, hidden_dim):
        super(ConvLSTMCell, self).__init__()

        self.input_dim = input_dim
        self.hidden_dim = hidden_dim

        self.conv = nn.Conv2d(in_channels=self.input_dim + self.hidden_dim,
                              out_channels=4 * self.hidden_dim,
                              kernel_size=3,
                              padding=1)

    def forward(self, input_tensor, cur_state):
        h_cur, c_cur = cur_state

        combined = torch.cat([input_tensor, h_cur], dim=1)

        combined_conv = self.conv(combined)

        cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_dim, dim=1)
        i = torch.sigmoid(cc_i)
        f = torch.sigmoid(cc_f)
        o = torch.sigmoid(cc_o)
        g = torch.tanh(cc_g)
        c_next = f * c_cur + i * g
        h_next = o * torch.tanh(c_next)

        return h_next, c_next

    def init_hidden(self, batch_size, height, width):
        return (torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv.weight.device),
                torch.zeros(batch_size, self.hidden_dim, height, width, device=self.conv.weight.device))

class ConvLSTM(nn.Module):
    """PyTorch module for ConvLSTM"""

    def __init__(self, input_dim, hidden_dim):
        super(ConvLSTM, self).__init__()

        self.input_dim = input_dim
        self.hidden_dim = hidden_dim
        self.cell = ConvLSTMCell(input_dim=self.input_dim, hidden_dim=self.hidden_dim)

    def forward(self, input_tensor,time = None):
        b, _, _, h, w = input_tensor.size()

        hidden_state = self.cell.init_hidden(b, h, w)
        seq_len = input_tensor.size(1)

        h, c = hidden_state
        for t in range(seq_len):
            reset = (time == t).nonzero().view(-1)
            for index in reset:
                h[index] = 0
                c[index] = 0

            h, c = self.cell(input_tensor=input_tensor[:, t, :, :, :],cur_state=[h, c])

        return h

#Conv2d + BN + RELU
class Convbn(nn.Module):
    """PyTorch module for Convolutional layer+Batch Normalization with ReLU"""

    def __init__(self,ins,ous,kernel,padding = 0):
        super(Convbn,self).__init__()

        self.conv = nn.Conv2d(ins,ous,kernel,padding = padding)
        self.bn = nn.BatchNorm2d(ous)
        self.relu = nn.ReLU(inplace = True)

    def forward(self,x):
        return self.relu(self.bn(self.conv(x)))

class TemporalAttension(nn.Module):
    """PyTorch module for RealSmileNet's TSA Block"""
    def __init__(self,channels):
        super(TemporalAttension,self).__init__()

        self.conv = nn.Conv2d(channels,channels,3,padding = 1)

        init = torch.zeros((3,3))
        init[1,1] = 1

        self.conv.weight.data.copy_(init)
        self.conv.bias.data.copy_(torch.zeros(channels))

    def forward(self, x):
        x1 = x[:,:-1]
        x2 = x[:,1:]
        o = x2 - x1

        o = torch.cat((torch.zeros((x.size(0),1,x.size(2),x.size(3),x.size(4)),device = x.device),o),1)
        o = o.view(-1,x.size(2),x.size(3),x.size(4))
        x = self.conv(o).view(x.size()) * x + x

        return x

class DeepSmileNet(nn.Module):
    """PyTorch module for RealSmileNet and AUDA features classification"""

    def __init__(self, f):
        super(DeepSmileNet,self).__init__()
        concat_size_on_last = 0
        self.f = f

        if "videos" in f:
            self.TSA = TemporalAttension(3)
            self.FPN = self._fpn_layers([4,'M', 6, 'M'])
            self.ConvLSTMLayer = ConvLSTM(6,8)
            self.Classification = nn.Sequential(
                NONLocalBlock2D(8),
                nn.AvgPool2d(kernel_size = 2, stride = 2),
                Convbn(8,10,2),
                nn.Dropout(0.5),
                nn.Flatten()  # 250
            )
            concat_size_on_last += 250

        if "aus" in f:
            n = 150
            self.AUsLSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationAUs = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            concat_size_on_last += n

        self.Ddict = {}

        if "d1da27" in f:
            n = 150
            self.Fd1da27LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd1da27 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d1da27": ("dynamics_delta_adjusted_27", self.Fd1da27LSTM, self.ClassificationFd1da27)})
            concat_size_on_last += n

        if "d2da27" in f:
            n = 150
            self.Fd2da27LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd2da27 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d2da27": ("dynamics_2nd_delta_adjusted_27", self.Fd2da27LSTM, self.ClassificationFd2da27)})
            concat_size_on_last += n

        if "d1da9" in f:
            n = 150
            self.Fd1da9LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd1da9 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d1da9": ("dynamics_delta_adjusted_9", self.Fd1da9LSTM, self.ClassificationFd1da9)})
            concat_size_on_last += n

        if "d2da9" in f:
            n = 150
            self.Fd2da9LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd2da9 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d2da9": ("dynamics_2nd_delta_adjusted_9", self.Fd2da9LSTM, self.ClassificationFd2da9)})
            concat_size_on_last += n

        if "d1d27" in f:
            n = 150
            self.Fd1d27LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd1d27 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d1d27": ("dynamics_delta_27", self.Fd1d27LSTM, self.ClassificationFd1d27)})
            concat_size_on_last += n

        if "d2d27" in f:
            n = 150
            self.Fd2d27LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd2d27 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d2d27": ("dynamics_2nd_delta_27", self.Fd2d27LSTM, self.ClassificationFd2d27)})
            concat_size_on_last += n

        if "d1d9" in f:
            n = 150
            self.Fd1d9LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd1d9 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d1d9": ("dynamics_delta_9", self.Fd1d9LSTM, self.ClassificationFd1d9)})
            concat_size_on_last += n

        if "d2d9" in f:
            n = 150
            self.Fd2d9LSTM = nn.LSTM(input_size=17, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationFd2d9 = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            self.Ddict.update({"d2d9": ("dynamics_2nd_delta_9", self.Fd2d9LSTM, self.ClassificationFd2d9)})
            concat_size_on_last += n

        if "si" in f:
            n = 10
            self.SILSTM = nn.LSTM(input_size=1, hidden_size=n, num_layers=1, batch_first=True)
            self.ClassificationSI = nn.Sequential(
                nn.BatchNorm1d(n),
                nn.ReLU(inplace=True)
            )
            concat_size_on_last += n

        if "auwise" in f:
            n = 80
            npart = int(n/5)
            self.AUWISEsimple_apex = nn.Sequential(
                nn.BatchNorm1d(119),
                nn.Linear(119, npart),
                nn.ReLU(inplace = True)
            )

            self.AUWISEsimple_offset = nn.Sequential(
                nn.BatchNorm1d(119),
                nn.Linear(119, npart),
                nn.ReLU(inplace = True)
            )

            self.AUWISEsimple_onset = nn.Sequential(
                nn.BatchNorm1d(119),
                nn.Linear(119, npart),
                nn.ReLU(inplace = True)
            )

            self.AUWISEsimple_whole_sequence = nn.Sequential(
                nn.BatchNorm1d(119),
                nn.Linear(119, npart),
                nn.ReLU(inplace = True)
            )

            self.AUWISEsimple_whole_smile = nn.Sequential(
                nn.BatchNorm1d(119),
                nn.Linear(119, npart),
                nn.ReLU(inplace = True)
            )

            self.AUWISEdict = [
                self.AUWISEsimple_apex,
                self.AUWISEsimple_offset,
                self.AUWISEsimple_onset,
                self.AUWISEsimple_whole_sequence,
                self.AUWISEsimple_whole_smile
            ]
            concat_size_on_last += n

        if "crossau" in f:
            m = 640
            n = 80
            npart = int(n / 5)
            mpart = int(m / 5)

            self.CROSSAUsimple_apex = nn.Sequential(
                nn.BatchNorm1d(1088),
                nn.Linear(1088, mpart),
                nn.ReLU(inplace=True),
                nn.Linear(mpart, npart),
                nn.ReLU(inplace=True)
            )

            self.CROSSAUsimple_offset = nn.Sequential(
                nn.BatchNorm1d(1088),
                nn.Linear(1088, mpart),
                nn.ReLU(inplace=True),
                nn.Linear(mpart, npart),
                nn.ReLU(inplace=True)
            )

            self.CROSSAUsimple_onset = nn.Sequential(
                nn.BatchNorm1d(1088),
                nn.Linear(1088, mpart),
                nn.ReLU(inplace=True),
                nn.Linear(mpart, npart),
                nn.ReLU(inplace=True)
            )

            self.CROSSAUsimple_whole_sequence = nn.Sequential(
                nn.BatchNorm1d(1088),
                nn.Linear(1088, mpart),
                nn.ReLU(inplace=True),
                nn.Linear(mpart, npart),
                nn.ReLU(inplace=True)
            )

            self.CROSSAUsimple_whole_smile = nn.Sequential(
                nn.BatchNorm1d(1088),
                nn.Linear(1088, mpart),
                nn.ReLU(inplace=True),
                nn.Linear(mpart, npart),
                nn.ReLU(inplace=True)
            )

            self.CROSSAUdict = [
                self.CROSSAUsimple_apex,
                self.CROSSAUsimple_offset,
                self.CROSSAUsimple_onset,
                self.CROSSAUsimple_whole_sequence,
                self.CROSSAUsimple_whole_smile
            ]
            concat_size_on_last += n

        self.ClassificationCat = nn.Sequential(
            nn.Linear(concat_size_on_last, 1),
            nn.Sigmoid()
        )

    def _fpn_layers(self,cfg,in_channels = 3):
        """Creates module for RealSmileNet's FPN Block"""
        layers = [nn.BatchNorm2d(in_channels)]
        for x in cfg:
            if x == 'M':
                layers += [nn.AvgPool2d(kernel_size=2, stride=2)]
            else:
                layers += [Convbn(in_channels, x, kernel=3, padding=1)]
                in_channels = x
        layers += [nn.AvgPool2d(kernel_size=1, stride=1)]
        layers += [nn.Dropout2d(0.2)]
        return nn.Sequential(*layers)

    def __forward_au_features(self, aus, aus_len, lstm_layer, cls_layer):
        """Forwards AUDA Package's sequential features"""

        # https://suzyahyah.github.io/pytorch/2019/07/01/DataLoader-Pad-Pack-Sequence.html
        # https://github.com/pytorch/pytorch/issues/43227
        # RuntimeError: 'lengths' argument should be a 1D CPU int64 tensor, but got 1D cuda:0 Long tensor - naprawione - aus_len z cpu podawane
        aus_packed = pack_padded_sequence(aus, aus_len, batch_first=True, enforce_sorted=False)
        aus_packed, (aus_hn, aus_cn) = lstm_layer(aus_packed)
        aus, _ = pad_packed_sequence(aus_packed, batch_first=True)

        # to samo
        # aus01 = aus[0][aus_len[0]-1]
        # aus02 = aus_hn[0][0]
        aus = aus_hn[0]
        aus = cls_layer(aus)
        return aus

    def forward(self, x_videos, s, x_df_dict, frames_len):
        # TSA block
        tocat = []

        if "videos" in self.f:
            x_videos = self.TSA(x_videos)

            # FPN block
            batch_size, timesteps, C, H, W = x_videos.size()
            input_x = []

            for l in range(x_videos.size(0)): # Pozbywanie się pustych klatek
                input_x.append(x_videos[l,s[l]:,:,:,:])
            #Przykładow: MAX S
            input_x = torch.cat(input_x,0)
            out = self.FPN(input_x)

            # ConvLSTM + LSTM blocks
            current = 0
            _,new_c,new_w,new_h = out.size()
            reshape_out = torch.zeros((batch_size, timesteps,new_c,new_w,new_h),device = x_videos.device)

            for index,l in enumerate(s):
                reshape_out[index,l:] = out[current:current + timesteps - l]
                current+= timesteps - l
            x_videos = reshape_out

            x_videos = self.ConvLSTMLayer(x_videos, s)
            x_videos = self.Classification(x_videos)
            tocat.append(x_videos)

        if "aus" in self.f:
            aus = self.__forward_au_features(x_df_dict['action_units'], frames_len, self.AUsLSTM, self.ClassificationAUs)
            tocat.append(aus)

        if "si" in self.f:
            si = self.__forward_au_features(x_df_dict['smile_intensities'], frames_len, self.SILSTM, self.ClassificationSI)
            tocat.append(si)

        d1d2features = [d1d2f for d1d2f in self.f if d1d2f.startswith("d1") or d1d2f.startswith("d2")]
        for d1d2feature in d1d2features:
            x_d1d2 = x_df_dict[self.Ddict.get(d1d2feature)[0]]
            d1d2lstm = self.Ddict.get(d1d2feature)[1]
            d1d2endclassifier = self.Ddict.get(d1d2feature)[2]
            aus = self.__forward_au_features(x_d1d2, frames_len, d1d2lstm, d1d2endclassifier)
            tocat.append(aus)

        if "auwise" in self.f:
            i = 0
            for value in self.AUWISEdict:
                x_auwise = x_df_dict['auwise'][:,i,:]
                x_auwise = value(x_auwise)
                tocat.append(x_auwise)
                i+=1

        if "crossau" in self.f:
            i = 0
            for value in self.CROSSAUdict:
                x_crossau = x_df_dict['crossau'][:,i,:]
                x_crossau = value(x_crossau)
                tocat.append(x_crossau)
                i+=1

        all_features = torch.cat(tocat,dim=1) #dim=1 - dim0 to u nas batch size

        all_features_to_sigm = self.ClassificationCat(all_features)
        return all_features_to_sigm, all_features

class MultipleDeepSmileNet(nn.Module):
    """PyTorch module for multiple DeepSmileNet modules (aka. concatenation models)"""
    def __init__(self, deepSmileNets, variant):
        self.variant = variant
        super(MultipleDeepSmileNet,self).__init__()

        n = 0

        if variant == 1:
            n = len(deepSmileNets)

            for submodel in deepSmileNets:
                for name, param in submodel.named_parameters():
                    param.requires_grad = False
        elif variant == 2:
            for submodel in deepSmileNets:
                for name, param in submodel.named_parameters():
                    param.requires_grad = False
                    if "ClassificationCat" in name and "weight" in name:
                        size = param.size()[1]
                        n+=size

        self.deepSmileNets = nn.ModuleList(deepSmileNets)
        self.Classification = nn.Sequential(
            nn.Linear(n, 1),
            nn.Sigmoid()
        )

        if variant == 1:
            linear = self.Classification[0]
            linear.weight.data.fill_(1/n)
            nn.init.constant_(linear.bias.data, 0)

    def forward(self, x_videos, s, x_df_dict, frames_len):
        preds = []
        for deepSmileNet in self.deepSmileNets:
            deepSmileNet.eval()
            pred, pred_pre_class = deepSmileNet(x_videos, s, x_df_dict, frames_len)

            if self.variant == 1:
                preds.append(pred)
            elif self.variant == 2:
                preds.append(pred_pre_class)
        cat_all = torch.cat(preds, dim=1)

        cat_all = self.Classification(cat_all)

        return cat_all, None


import pyximport
pyximport.install(reload_support=True)

import truncnormal
import cProfile
from timeit import default_timer as timer
from scipy.stats import truncnorm
import numpy as np
import scipy.special as cs
import matplotlib.pyplot as plt
import math

from importlib import reload
reload(truncnormal)

# =============================================================================
# import importlib
# importlib.reload('truncnormal')
# =============================================================================

def tuncnormal(loc, a,  b, scale, size):

    z = np.zeros(size)

    for idx in range(size):

        a_norm = (a[idx] - loc[idx]) / scale
        b_norm = (b[idx] - loc[idx]) / scale

        # Inverse Transformation Sampling
        if (b_norm - a_norm >= 1) and (not (a_norm < -4 and b_norm < -4)) and (not (a_norm > 4 and b_norm > 4)):
            u = np.random.uniform(size = 1, low = cs.ndtr(a_norm), high = cs.ndtr(b_norm))[0]
            z_norm = cs.ndtri(u)
            z[idx] = z_norm * scale + loc[idx]

        # Uniform Proposal Rejection Sampling
        elif b_norm - a_norm < 1:
            if b_norm * a_norm < 0:
                m = 1 / np.sqrt(2 * math.pi)
            else:
                m = np.maximum(1/np.sqrt(2 * math.pi) * np.exp(-0.5 * (a_norm) ** 2), 1/np.sqrt(2 * math.pi) * np.exp(-0.5 * (b_norm) ** 2))

            p = 0
            while p < 1:

                u = np.random.uniform(size = 1, low = a_norm, high = b_norm)[0]
                v = np.random.uniform(size = 1, low = 0, high = 1)[0]

                if v < (1/np.sqrt(2 * math.pi) * np.exp(-0.5 * (u) ** 2)) / m:
                    z[idx] =  u * scale + loc[idx]
                    p = 1

        # Rayleigh Proposal Rejection Sampling
        else:
            flag = 0

            if b_norm < 0:
                # Note the flip of relationship when times -1
                a_norm = (b[idx] - loc[idx]) / scale * (-1)
                b_norm = (a[idx] - loc[idx]) / scale * (-1)
                flag = 1

            c = a_norm ** 2 / 2
            q = 1 - np.exp(c - (b_norm ** 2 / 2))

            p = 0
            while p < 1:

                u = np.random.uniform(size = 1, low = 0, high = 1)[0]
                v = np.random.uniform(size = 1, low = 0, high = 1)[0]
                z_norm = c - np.log(1 - q * u)

                if z_norm * v ** 2 <= a_norm:
                    if flag == 1:
                        z[idx] = (scale * (np.sqrt(2 * z_norm)) - loc[idx]) * (-1)
                    else:
                        z[idx] = scale * (np.sqrt(2 * z_norm)) + loc[idx]
                    p = 1

    return z

mu = np.full(100000,0)
a = np.full(100000, -np.inf)
b = np.full(100000, 3)

start = timer()
sample = ct.tuncnormal(mu, a, b, 1, len(mu))
end = timer()
print(end - start)

_ = np.linspace(-10, b[0]+0.5, 100000)
t = truncnorm.pdf(_, a = a[0], b = b[0])
plt.plot(_, t)
plt.hist(sample, density = True, bins = 20)
plt.show()

tuncnormal(theta_single[one_mask],
            np.full(sum(one_mask.flatten()), -np.inf),
            alpha_R[one_mask],
            1,
            sum(one_mask.flatten()))


# -*- coding: utf-8 -*-
#harvest theses from EPFL
#FS: 2019-10-28

import sys
import os
import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup
import re
import ejlmod3
import time

insttoskip = ['CDH', 'CDM', 'SV', 'SB/ISIC', 'SB/CIBM', 'STI/IBI-STI', 'STI/IMX/LMC', 'STI/IMX/SMAL',
              'IC/IINFCOM/CHILI', 'ENAC']
reharvest = False

#check already harvested
ejldirs = ['/afs/desy.de/user/l/library/dok/ejl/backup',
           '/afs/desy.de/user/l/library/dok/ejl/backup/%i' % (ejlmod3.year(backwards=1)),
           '/afs/desy.de/user/l/library/dok/ejl/backup/%i' % (ejlmod3.year(backwards=2))]
redoki = re.compile('THESES.EPFL.*doki$')
rehttp = re.compile('^I\-\-(http.*\d).*')
bereitsin = []
for ejldir in ejldirs:
    #print(ejldir)
    for datei in os.listdir(ejldir):
        if redoki.search(datei):
            inf = open(os.path.join(ejldir, datei), 'r')
            for line in inf.readlines():
                if len(line) > 1 and line[0] == 'I':
                    if rehttp.search(line):
                        http = rehttp.sub(r'\1', line.strip())
                        if not http in bereitsin:
                            bereitsin.append(http)
            #print('  %6i %s' % (len(bereitsin), datei))

publisher = 'Ecole Polytechnique, Lausanne'
hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
       'Accept-Encoding': 'none',
       'Accept-Language': 'en-US,en;q=0.8',
       'Connection': 'keep-alive'}
recs = {}
for year in [ejlmod3.year(), ejlmod3.year(backwards=1)]:
    prerecs = []
    tocurl = 'https://infoscience.epfl.ch/search?ln=en&cc=Infoscience%2FResearch%2FThesis&p=&f=&rm=&ln=en&sf=year&so=d&rg=500&c=Infoscience%2FResearch%2FThesis&c=&of=hb&fct__3=' + '%i' % (year)
    ejlmod3.printprogress('=', [[year], [tocurl]])
    req = urllib.request.Request(tocurl, headers=hdr)
    tocpage = BeautifulSoup(urllib.request.urlopen(req), features="lxml")
    time.sleep(30)
    for div in tocpage.body.find_all('div', attrs = {'class' : 'result-title'}):
        for a in div.find_all('a'):
            rec = {'tc' : 'T', 'jnl' : 'BOOK', 'year' : str(year), 'keyw' : [], 'inst' : '', 'date' : str(year), 'supervisor' : [], 'note' : []}
            rec['artlink'] = 'https://infoscience.epfl.ch' + a['href']
            rec['tit'] = a.text.strip()
            if ejlmod3.checkinterestingDOI(rec['artlink']):
                if reharvest or not rec['artlink'] in bereitsin:
                    prerecs.append(rec)

    j = 0
    for rec in prerecs:
        keepit = True
        j += 1
        recsstat = [len(recs[inst]) for inst in recs]
        ejlmod3.printprogress('-', [[year], [j, len(prerecs)], [rec['artlink']], recsstat, [sum(recsstat)]])
        try:
            req = urllib.request.Request(rec['artlink'])
            artpage = BeautifulSoup(urllib.request.urlopen(req), features="lxml")
            time.sleep(10)
        except:
            print('wait 10 minutes')
            time.sleep(600)
            req = urllib.request.Request(rec['artlink'])
            artpage = BeautifulSoup(urllib.request.urlopen(req), features="lxml")
            time.sleep(30)
        ejlmod3.metatagcheck(rec, artpage, ['citation_author', 'description', 'citation_keywords', 'citation_doi', 'citation_pdf_url'])
        rec['autaff'][-1].append('Ecole Polytechnique, Lausanne')
        for a in artpage.body.find_all('a'):
            if a.has_attr('href'):
                if re.search('\/collection\/Infoscience\/Research\/[A-Z]', a['href']):
                    inst = re.sub('.*Research\/(.*)\?.*', r'\1', a['href'])
                    if inst != 'Thesis':
                        #print(' . ', inst)
                        if inst in insttoskip:
                            keepit = False
                        elif inst in ['SB/Mathematics', 'SB/MATH']:
                            rec['fc'] = 'm'
                        elif inst in ['SB/IPHYS/C3MP', 'SB/IPHYS/CTMC']:
                            rec['fc'] = 'f'
                        elif inst in ['SB/IPHYS/LASTRO', 'SB/IPHYS/LPPC']:
                            rec['fc'] = 'a'
                        elif inst in ['SB/IPHYS/LQM', 'SB/IPHYS/LQP']:
                            rec['fc'] = 'k'
                        elif inst in ['IC']:
                            rec['fc'] = 'c'
                        if len(inst) > len(rec['inst']):
                            rec['inst'] = inst
                            rec['note'] = [ inst ]
        rec['inst'] = re.sub('\/.*', '', rec['inst'])
        for div in artpage.body.find_all('div', attrs = {'class' : 'metadata-row'}):
            for span in div.find_all('span', attrs = {'class' : 'title'}):
                title = span.text.strip()
            for span in div.find_all('span', attrs = {'class' : 'value'}):
                #pages
                if title == 'Pagination':
                    if re.search('^\d+$', span.text.strip()):
                        rec['pages'] = span.text.strip()
                #supervisor
                elif title == 'Advisor(s)':
                    for a in span.find_all('a'):
                        rec['supervisor'].append([a.text])
        if keepit:
            if rec['inst'] in recs:
                recs[rec['inst']].append(rec)
            else:
                recs[rec['inst']] = [rec]
            rec['note'].append(rec['artlink'])
            ejlmod3.printrecsummary(rec)
        else:
            #print('skip thesis from %s' % (rec['inst']))
            ejlmod3.adduninterestingDOI(rec['artlink'])

for inst in list(recs.keys()):
    jnlfilename = 'THESES-EPFL-%s_%s' % (ejlmod3.stampoftoday(), inst)
    ejlmod3.writenewXML(recs[inst], publisher, jnlfilename)


"""Preprocessing imagenet2012 dataset into TFRecord format

This code is adapted from
https://github.com/google-research/big_vision/blob/main/big_vision/tools/download_tfds_datasets.py
"""

from absl import app
import tensorflow_datasets as tfds
import resource

low, high = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (high, high))

def main(argv):
  if len(argv) > 1 and "download_tfds_datasets.py" in argv[0]:
    datasets = argv[1:]
  else:
    exit(0)
  for d in datasets:
    tfds.load(name=d, download=True, data_dir='')

if __name__ == "__main__":
  app.run(main)


import unittest
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from crispr_scoring_module import CRISPRScoringModule

class TestCRISPRScoringModule(unittest.TestCase):

    def setUp(self):
        self.fasta_sequences = [
            SeqRecord(Seq("AAACCCGGGTTT"), id="seq1"),
            SeqRecord(Seq("GGGCCCAAATTTCGG"), id="seq2")
        ]
        self.pam_sequence = 'CGG'

        # Initialize CRISPRScoringModule with the test data
        self.crispr_tool = CRISPRScoringModule(None, self.pam_sequence)
        self.crispr_tool.target_sites = [
            {'start': 5, 'end': 7, 'sequence': 'CGG', 'score': None},
            {'start': 12, 'end': 14, 'sequence': 'CGG', 'score': None}
        ]

    def test_find_target_sites(self):
        # Here you can test the find_target_sites method.
        # Since this method needs to read from a FASTA file, you can create a temporary FASTA file
        # or mock the file reading part using Python's unittest.mock.
        # It is generally a good practice to use mocking for file I/O in unit tests.
        pass

    def test_score_target_sites(self):
        # Test scoring based on GC content
        self.crispr_tool.score_target_sites()

        # Check the scores
        for target_site in self.crispr_tool.target_sites:
            sequence = target_site['sequence']
            expected_score = 100.0 * sequence.count('G') + sequence.count('C') / len(sequence)
            self.assertEqual(target_site['score'], expected_score)

if __name__ == '__main__':
    unittest.main()


"""Running a WADO-RS download speed benchmark
"""
import logging
from os import environ

import requests
from dicomtrolley.core import StudyReference
from dicomtrolley.qido_rs import QidoRS
from dicomtrolley.trolley import Trolley

from dicomtrolley.wado_rs import WadoRS

from trolleybenchmark.runner import Runner
from trolleybenchmark.experiments import TrolleyDownloadExperiment, \
    WadoRSTrolleyDownloadStudy

logging.basicConfig(level=logging.DEBUG)
logging.getLogger("PIL").setLevel(logging.INFO)  # shut up PIL

session = requests.Session()
trolley = Trolley(downloader=WadoRS(session=session,
                                    url=environ['WADO_RS_URL']),
                  searcher=QidoRS(session=session,
                                  url=environ['WADO_RS_URL']))

studies = [StudyReference('123')]

for study in studies:
        # make sure
        series_list = trolley.ensure_to_series_level([study])

        experiments = []
        for series in series_list:
                experiments.append(TrolleyDownloadExperiment(
                        trolley=trolley,
                        target=series,
                        tmp_dir=f"/tmp/run_experiment/",
                        label=f"series for {study.study_uid}",
                        comment=f"test_comment"))

runner = Runner(experiments=experiments,
                results_path="/tmp/runner")
runner.run()


from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm
from django.contrib.auth.views import LoginView, LogoutView
from django.shortcuts import render
from django.urls import reverse_lazy
from django.views.generic import CreateView

signup = CreateView.as_view(
    form_class=UserCreationForm,
    template_name="partials/form.html",
    extra_context={"form_name": "Sign up", "submit_label": "Sign up"},
    success_url=reverse_lazy("accounts:login"),
)

login = LoginView.as_view(
    template_name="partials/form.html",
    extra_context={"form_name": "로그인", "submit_label": "로그인"},
)

logout = LogoutView.as_view(next_page="accounts:login")

@login_required
def profile(request):
    user = request.user
    return render(
        request,
        "accounts/profile.html",
        {
            "user": user,
        },
    )


#!/bin/python3

import math
import os
import random
import re
import sys
from collections import deque

def median(buckets, d):
    s = 0
    last = 0

    for (val, ctr) in enumerate(buckets):
        if ctr > 0:
            s += ctr
            if s > d//2 and d % 2 == 1:
                return val
            elif s > d//2 and d % 2 == 0:
                if ctr >= 2:
                    return val / 2
                else:
                    return (last + val) / 2

            last = val

def activityNotifications(ex, d):
    ret = 0
    buckets = [0] * 300
    queue = deque()

    for (i,x) in enumerate(ex):
        if i >= d:
            med = median(buckets,d)
            if (x >= med * 2):
                print(buckets)
                print(med)
                print(i, x)
                return 0
            ret += (x >= median(buckets,d) * 2)
            y = queue.popleft()
            buckets[y] -= 1

        buckets[x] += 1
        queue.append(x)

    return ret

if __name__ == '__main__':
    #fptr = open(os.environ['OUTPUT_PATH'], 'w')
    nd = input().split()
    n = int(nd[0])
    d = int(nd[1])
    expenditure = list(map(int, input().rstrip().split()))
    result = activityNotifications(expenditure, d)
    #fptr.write(str(result) + '\n')

    #fptr.close()
    print(result)


import math
import os
import random

import copy
import torch
import pickle
from torch import Tensor
from typing import Tuple, Iterator
from contextlib import contextmanager
from torch.utils.data import Dataset, IterableDataset
import pandas as pd

def GetStartingIndices(full_dataset):
    os.makedirs('./starting_indices', exist_ok=True)
    pkl_file = './starting_indices/index.pkl'
    if os.path.isfile(pkl_file):
        with open(pkl_file, 'rb') as fid:
            index = pickle.load(fid)
        return index

    dl = torch.utils.data.DataLoader(full_dataset, 1, shuffle=False, drop_last=False)
    last_y = -1
    indices = []
    for i, batch in enumerate(dl):
        _, _, y = batch
        y= y.item()
        if y != last_y:
            print("Added index for label "+str(y))
            indices += [i]
        last_y = y

    index = torch.tensor(indices)
    with open(pkl_file, 'wb') as file:
        pickle.dump(index , file)

    return index

def SplitDataset(full_dataset, n_labels, ratios=[]):
    index = GetStartingIndices(full_dataset)
    sets = []
    new_ratios = [0]+ratios
    index_offset = 0
    current_ratio = 1 - sum(new_ratios)
    for i in range(0, len(new_ratios)):
        current_ratio += new_ratios[i]
        last_label = int(n_labels * current_ratio - 1)
        last_image_index = index[last_label+1].item()-1

        set = copy.deepcopy(full_dataset)
        set.n_records = last_image_index-index_offset+1
        set.ds1.index_offset = index_offset
        set.ds2.index_offset = index_offset
        sets += [set]
        index_offset = last_image_index+1
    return sets

def CreateNNLabels(dataset, dataset_name,force_run = False):
    os.makedirs('./nn_spaces', exist_ok=True)
    pkl_nn_space_file = './nn_spaces/nn_space_' + dataset_name + '.pkl'
    if os.path.isfile(pkl_nn_space_file) and not force_run:
        with open(pkl_nn_space_file, 'rb') as fid:
            nn_space_dict = pickle.load(fid)
        return nn_space_dict

    print("Creating NN full space for: ", dataset_name)

    dl = torch.utils.data.DataLoader(dataset, 1, shuffle=False, drop_last=False)
    embedding_space = []
    labels = []
    for i, batch in enumerate(dl):
        _, X2, y = batch
        y = y.item()

        embedding_space += [X2]
        labels += [y]

    print("Dumping embedding space")
    embedding_dict = {'features': torch.stack(embedding_space).squeeze(),
                      'labels': torch.tensor(labels)}
    with open(pkl_nn_space_file, 'wb') as file:
        pickle.dump(embedding_dict, file)
    return embedding_dict

def CreateMeanLabels(dataset, dataset_name, force_create=False):
    os.makedirs('./nn_spaces', exist_ok=True)
    pkl_nn_space_file = './nn_spaces/nn_space_' + dataset_name + '.pkl'
    if os.path.isfile(pkl_nn_space_file) and not force_create:
        with open(pkl_nn_space_file, 'rb') as fid:
            nn_space_dict = pickle.load(fid)
        return nn_space_dict

    print("Creating NN space for: ",dataset_name)
    index = GetStartingIndices(dataset)
    dl = torch.utils.data.DataLoader(dataset, 1, shuffle=False, drop_last=False)
    current_label_features = []
    embedding_space = []
    labels = []
    last_y = None

    for i, batch in enumerate(dl):
        _, X2, y = batch
        y = y.item()
        if last_y is None:
            last_y = y
        if y != last_y:
            print("Processed Label: ", last_y)
            mean_feature = torch.stack(current_label_features).squeeze()
            if len(mean_feature.shape) > 1:
                mean_feature = mean_feature.mean(dim=0)

            embedding_space += [mean_feature]
            labels += [last_y]
            current_label_features = []
            last_y = y

        current_label_features += [X2]

    print("Processed Label: ", last_y)
    mean_feature = torch.stack(current_label_features).squeeze()
    if len(mean_feature.shape) > 1:
        mean_feature = mean_feature.mean(dim=0)

    embedding_space += [mean_feature]
    labels += [last_y]

    print("Dumping embedding space")
    embedding_dict = {'features': torch.stack(embedding_space).squeeze(),
                      'labels': torch.tensor(labels)}
    with open(pkl_nn_space_file, 'wb') as file:
        pickle.dump(embedding_dict , file)
    return embedding_dict

def GetLabelVectors(label, index, lengths, full_dataset):
    i_label_start = index[label].item()
    i_label_stop = i_label_start + lengths[label].item()
    x1_vectors, x2_vectors = [], []
    for i in range(i_label_start, i_label_stop):
        x1, x2, y = full_dataset[i]
        assert y == label
        x1_vectors += [x1]
        x2_vectors += [x2]

    x1 = torch.stack(x1_vectors)
    x2 = torch.stack(x2_vectors)
    return x1,x2

def InterpolateDatasetRandom(dataset, full_dataset, min_interp_num=20, max_interp_num=30,interpolated_div = 2):
    set = copy.deepcopy(dataset)
    set_interpolated_data = copy.deepcopy(dataset)
    index = GetStartingIndices(full_dataset)
    _,_,label_first = set[0]
    _, _, label_last = set[len(set)-1]
    index = index[label_first:label_last+1]
    indices_shift = torch.hstack([index[1:] , torch.tensor(len(set))])
    lengths = indices_shift - index
    [sorted, idx_sorted] = torch.sort(lengths, descending=True)

    current_label = max(set.classes) + 1

    pkl_batch_size =  set.ds1.dataset_batch_size
    n_images = len(full_dataset)
    pkl_file_i = n_images // pkl_batch_size
    max_idx_offset = len(full_dataset) - len(set)
    max_idx = len(set) - 1
    interpolated_cnt = 959872
    if interpolated_cnt==0:
        paths = ['./Insight/insightface/recognition/arcface_torch/r18_features',
                 './Insight/insightface/recognition/arcface_torch/r50_features']

        # ds_features_r18 = [[], []]
        # ds_features_r50 = [[], []]
        ds_features_r18_interpolated = torch.empty(size=(0, 512))
        ds_features_r50_interpolated = torch.empty(size=(0, 512))
        ds_interpolated_lists = [ds_features_r18_interpolated, ds_features_r50_interpolated]

        label_list = []

        # ds_features_list = [ds_features_r18, ds_features_r50]
        # first_even_id = False
        #for i, batch in enumerate(dl):
        for [[prev_num_vectors, prev_label] , [num_vectors, label]]  in zip(zip(sorted, idx_sorted) ,zip(sorted[1:], idx_sorted[1:]) ):
            X1_prev, X2_prev = GetLabelVectors(prev_label, index, lengths, full_dataset)
            X1,X2 = GetLabelVectors(label,index,lengths,full_dataset)
            ds_features_r18 = [X1_prev,X1]
            ds_features_r50 = [X2_prev, X2]
            #X1, X2, y = batch
            #y = y.item()
            #if y > stop_label:
            #    break
            #if y % 2 == 0 and y != 0 and not first_even_id:
                #first_even_id = True
                # Take min size
            ds_features_list = [ds_features_r18, ds_features_r50]
            min_len = min([len(ds_features_list[0][0]), len(ds_features_list[0][1]),max_interp_num])
            max_len = max([len(ds_features_list[0][0]), len(ds_features_list[0][1])])
            if max_len<min_interp_num:

                break
            label_list += [current_label] * min_len
            current_label = current_label + 1

            for i_ds, ds_features in enumerate(ds_features_list):

                ds_features_label_even = ds_features[0][:min_len]
                ds_features_label_odd = ds_features[1][:min_len]
                ds_features_interpolated = (ds_features_label_even + ds_features_label_odd) / 2
                ds_interpolate = ds_interpolated_lists[i_ds]
                ds_interpolated_lists[i_ds] = torch.vstack((ds_interpolate, ds_features_interpolated))

                if ds_interpolated_lists[i_ds].shape[0] >= pkl_batch_size:
                    new_pkl_features = ds_interpolated_lists[i_ds][:pkl_batch_size, :]
                    idx_tensor = torch.tensor(list(range(n_images, n_images + pkl_batch_size)))
                    label_tensor = torch.tensor(label_list[:pkl_batch_size])

                    temp_tensor = torch.hstack((torch.unsqueeze(idx_tensor, dim=1),
                                                torch.unsqueeze(label_tensor, dim=1),
                                                new_pkl_features))

                    pkl_batch = os.path.join(paths[i_ds], str(pkl_file_i) + '.pkl')
                    with open(pkl_batch, 'wb') as file:
                        pickle.dump(temp_tensor, file)

                    ds_interpolated_lists[i_ds] = ds_interpolated_lists[i_ds][pkl_batch_size:, :]
                    if i_ds == 1:
                        print("Dumped ", pkl_batch)
                        label_list = label_list[pkl_batch_size:]
                        n_images = n_images + pkl_batch_size
                        pkl_file_i = pkl_file_i + 1
                        interpolated_cnt += pkl_batch_size

                # ds_features_r18 = [[], []]
                # ds_features_r50 = [[], []]
            #elif y % 2 != 0:
            #    first_even_id = False

            # ds_features_r18[y % 2].append(X1)
            # ds_features_r50[y % 2].append(X2)

    set.n_records = len(set) + (interpolated_cnt//interpolated_div)
    set.ds1.max_idx_offset = max_idx_offset
    set.ds2.max_idx_offset = max_idx_offset
    set.ds1.max_idx = max_idx
    set.ds2.max_idx = max_idx

    set_interpolated_data.n_records = interpolated_cnt // interpolated_div
    set_interpolated_data.ds1.max_idx_offset = len(full_dataset)
    set_interpolated_data.ds2.max_idx_offset = len(full_dataset)
    set_interpolated_data.ds1.max_idx = -1
    set_interpolated_data.ds2.max_idx = -1

    print("n_records", str(set.n_records), "interpolated count", str(interpolated_cnt), "Max Idx", str(max_idx))
    return set,set_interpolated_data

class PklEmbeddingsDataset(Dataset):
    """
    A dataset representing embedded vectors of face recognition
    """

    def __init__(self, pkl_dir_path1, pkl_dir_path2):
        """
        :param pkl_dir_path1:
        :param pkl_dir_path2:
        """
        super().__init__()
        self.ds1 = PklDataset(pkl_dir_path1)
        self.ds2 = PklDataset(pkl_dir_path2)

        self.n_records = len(self.ds2)
        self.classes = self.ds2.classes

    def __getitem__(self, index: int) -> Tuple[Tensor, Tensor]:
        image1, label1, idx1 = self.ds1[index]
        image2, label2, idx2 = self.ds2[index]
        assert idx1 == idx2
        assert label1 == label2
        return image1, image2, label1
        # return next(self.iterator)

    def __len__(self):
        """
        :return: Number of samples in this dataset.
        """

        return self.n_records
        # ========================

class PklDataset(Dataset):
    """
    A dataset representing embedded vectors of face recognition
    """

    def __init__(self, pkl_dir_path, infer_classes_and_n_records=False):
        """
        :param pklFilePath:

        """
        super().__init__()
        print("Loading pkl dataset: " + pkl_dir_path)
        self.pkl_dir_path = pkl_dir_path
        with open(os.path.join(pkl_dir_path, 'metadata.pkl'), 'rb') as pkl_metadata:
            metadata_dict = pickle.load(pkl_metadata)
            self.n_records = metadata_dict["n_images"]
            self.classes = metadata_dict["classes"]
            self.dataset_batch_size = metadata_dict["batch_size"]
        self.files = [None] * (self.n_records // self.dataset_batch_size)
        self.max_idx_offset = 0
        self.index_offset = 0
        self.max_idx = -1
        if infer_classes_and_n_records:
            self._infer_classes_and_n_records()
        # self.iterator = self._get_image()

    def _infer_classes_and_n_records(self):
        labels_set = set()
        n_records = 0
        raw_data = self._load_batch(return_on_failure=True)
        while raw_data is not None:
            n_records += raw_data.shape[0]
            labels_set.update(raw_data[:, 0].cpu().detach().tolist())
            raw_data = self._load_batch(return_on_failure=True)
        self.n_records = n_records
        self.classes = [int(item) for item in labels_set]

    def __getitem__(self, index: int) -> Tuple[Tensor, int, int]:
        index += self.index_offset

        if index > self.max_idx:
            index += self.max_idx_offset
        # i_batch = offset + index // self.dataset_batch_size
        i_batch = index // self.dataset_batch_size
        offset = index % self.dataset_batch_size
        batch_path = os.path.join(self.pkl_dir_path, str(i_batch) + '.pkl')
        # if self.files[i_batch] is None:
        # self.files[i_batch] = open(batch_path, 'rb')
        f = open(batch_path, 'rb')
        batch = pickle.load(f)
        idx = int(batch[offset, 0].item())
        label = int(batch[offset, 1].item())
        image = batch[offset, 2:]
        return image, label, idx
        # return next(self.iterator)

    def _get_image(self) -> Tuple[Tensor, int]:
        while True:
            raw_data = self._load_batch()
            for feature_tensor in raw_data:
                label = int(feature_tensor[0].item())
                image = feature_tensor[1:]
                yield image, label

    def _load_batch(self, return_on_failure=False):

        try:
            raw_data = pickle.load(self.pkl_file)
        except EOFError:
            if return_on_failure:
                return None
            # print("Reopening pkl file...")
            self.pkl_file.close()
            self.pkl_file = open(self.pkl_file_path, 'rb')
            nRecords = pickle.load(self.pkl_file)
            classes = pickle.load(self.pkl_file)
            raw_data = pickle.load(self.pkl_file)
        return raw_data

    def __len__(self):
        """
        :return: Number of samples in this dataset.
        """

        return self.n_records
        # ========================

#
# class ImageStreamDataset(IterableDataset):
#     """
#     A dataset representing an infinite stream of noise images of specified dimensions.
#     """
#
#     def __init__(self, num_classes: int, C: int, W: int, H: int):
#         """
#         :param num_classes: Number of classes (labels)
#         :param C: Number of channels per image
#         :param W: Image width
#         :param H: Image height
#         """
#         super().__init__()
#         self.num_classes = num_classes
#         self.image_dim = (C, W, H)
#
#     def __iter__(self) -> Iterator[Tuple[Tensor, int]]:
#         """
#         :return: An iterator providing an infinite stream of random labelled images.
#         """
#
#         while True:
#             # Generate infinite images as requested and yield each time
#             yield random_labelled_image(self.image_dim, self.num_classes)
#         # ========================
#


# -*- coding: utf-8 -*-
#harvest thesis from UMass Amherst
#FS: 2022-11-10

import getopt
import sys
import os
import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup
import re
import ejlmod3
import time
import undetected_chromedriver as uc

publisher = 'UMass Amherst'
jnlfilename = 'THESES-UMassAmherst-%s' % (ejlmod3.stampoftoday())

pages = 1
years = 2
skipalreadyharvested = True
boring = ['Afro-American Studies', 'Animal Biotechnology & Biomedical Sciences', 'Anthropology',
          'Biomedical Engineering', 'Chemical Engineering', 'Chemistry', 'Civil and Environmental Engineering',
          'Civil Engineering', 'Communication Disorders', 'Communication', 'Comparative Literature',
          'Economics', 'Education', 'Electrical and Computer Engineering', 'English',
          'Environmental Conservation', 'Food Science', 'Geosciences', 'Hispanic Literatures & Linguistics',
          'History', 'Hospitality & Tourism Management', 'Industrial Engineering & Operations Research',
          'Kinesiology', 'Linguistics', 'Management', 'Microbiology', 'Molecular and Cellular Biology',
          'Music', 'Neuroscience and Behavior', 'Nursing', 'Organismic and Evolutionary Biology', 'Philosophy',
          'Plant Biology', 'Political Science', 'Polymer Science and Engineering', 'Psychology',
          'Public Health', 'Regional Planning', 'Resource Economics', 'School Psychology', 'Sociology',
          'Wildlife & Fisheries Conservation', 'American Studies', 'Biochemistry', 'Biology',
          'Education (also CAGS)', 'Education; Children and Family Studies', 'Isenberg School of Management',
          'Education; Language, Literacy and Culture', 'Educational Policy and Leadership',
          'Education Policy, Research and Administration; Educational Policy and Leadership',
          'Education; Teacher Education & Curriculum Studies', 'English - American Studies', 'Entomology',
          'Forest Resources', 'German and Scandinavian Studies', 'Germanic Languages & Literatures',
          'Hispanic Literatures and Linguistics', 'Industrial Engineering and Operations Research',
          'Languages, Literatures, and Cultures; Hispanic Literatures and Linguistics',
          'Lusophone Literatures and Cultures', 'Marine Sciences and Technology',
          'Mechanical and Industrial Engineering', 'Mechanical Engineering', 'Plant and Soil Sciences',
          'Plant, Soil & Insect Sciences', 'Psychology; Clinical Psychology',
          'Psychometric Methods, Education Statistics and Research Methods', 'Sport Management',
          'Wildlife and Fisheries Conservation', 'Animal Biotechnology and Biomedical Sciences',
          'Biostatistics and Epidemiology', 'Clinical Psychology',
          'Educational Planning, Research, and Administration', 'Language, Literacy, & Culture',
          'Educational Policy and Leadership; International Education',
          'Educational Policy, Research, and Administration', 'Education; Educational Policy and Leadership',
          'Education, Educational Policy Research, and Administration Research',
          'Education; Education Policy, Research, and Administration', 'Education; Research and Evaluation Methods',
          'Education; Student Development', 'Education; Teacher Education & School Improvement',
          'Epidemiology and Biostatistics', 'Isenberg School of Management; Operations Management',
          'Mechanical and Industrial Engineering; Industrial Engineering and Operations Research',
          'Neuroscience & Behavior', 'Organismic and Evolutionary Biology; Entomology',
          'Research and Evaluating Methods', 'Social Justice Education',
          'Teacher Education and Curriculum Studies; Language, Literacy, & Culture',
          'Teacher Education and Curriculum Studies', 'Teacher Education']

basetocurl = 'https://scholarworks.umass.edu/dissertations_2/index.'
tocextension = 'html'

options = uc.ChromeOptions()
options.binary_location = '/usr/bin/google-chrome'
options.binary_location='/usr/bin/chromium'
options.add_argument('--headless')
chromeversion = int(re.sub('.*?(\d+).*', r'\1', os.popen('%s --version' % (options.binary_location)).read().strip()))
driver = uc.Chrome(version_main=chromeversion, options=options)

if skipalreadyharvested:
    alreadyharvested = ejlmod3.getalreadyharvested(jnlfilename)

prerecs = []
date = False
for i in range(pages):
    tocurl = basetocurl + tocextension
    ejlmod3.printprogress('=', [[i+1, pages], [tocurl]])
    try:
        tocpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(tocurl), features="lxml")
        time.sleep(3)
    except:
        print("retry %s in 180 seconds" % (tocurl))
        time.sleep(180)
    for div in tocpage.body.find_all('div'):
        if div.has_attr('class') and re.search('locks', div['class'][0]):
            for child in div.children:
                try:
                    name = child.name
                except:
                    continue
                if name == 'h4':
                    for span in child.find_all('span'):
                        date = span.text.strip()
                elif name == 'p':
                    #year = int(re.sub('.*(20\d\d).*', r'\1', rec['date']))
                    if int(date) > ejlmod3.year(backwards=years):
                        if child.has_attr('class') and 'article-listing' in child['class']:
                            rec = {'jnl' : 'BOOK', 'tc' : 'T', 'date' : date, 'note' : []}
                            for a in child.find_all('a'):
                                rec['tit'] = a.text.strip()
                                rec['artlink'] = a['href']
                                a.replace_with('')
                            if ejlmod3.checkinterestingDOI(rec['artlink']):
                                prerecs.append(rec)
    print('  ', len(prerecs))
    tocextension = '%i.html' % (i+2)

i = 0
recs = []
for rec in prerecs:
    keepit = True
    i += 1
    ejlmod3.printprogress('-', [[i, len(prerecs)], [rec['artlink']], [len(recs)]])
    try:
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['artlink']), features="lxml")
        time.sleep(3)
    except:
        print("retry %s in 180 seconds" % (rec['artlink']))
        time.sleep(180)
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['artlink']), features="lxml")
    ejlmod3.metatagcheck(rec, artpage, ['description', 'bepress_citation_author', 'bepress_citation_pdf_url', 'bepress_citation_doi',
                                        'bepress_citation_online_date'])
    if 'doi' in rec:
        rec['doi'] = re.sub('.*doi.org\/', '', rec['doi'])
    #supervisor
    for div in artpage.body.find_all('div', attrs = {'id' : 'advisor1'}):
        for p in div.find_all('p'):
            rec['supervisor'] = [[ re.sub('^Dr. ', '', p.text.strip()) ]]
    for div in artpage.body.find_all('div', attrs = {'id' : 'advisor2'}):
        for p in div.find_all('p'):
            rec['supervisor'].append( [re.sub('^Dr. ', '', p.text.strip())] )
    #ORCID
    for div in artpage.body.find_all('div', attrs = {'id' : 'orcid'}):
        for h4 in div.find_all('h4'):
            if re.search('uthor', h4.text):
                for p in div.find_all('p'):
                    if re.search('orcid', p.text):
                        rec['autaff'][-1].append(re.sub('.*orcid.*?\/', 'ORCID:', p.text.strip()))
                    else:
                        rec['autaff'][-1].append('ORCID:'+p.text.strip())
                if rec['autaff'][0][0] == ',':
                    orcidlink = re.sub('ORCID:', 'https://orcid.org/', rec['autaff'][-1][-1])
                    print('   try to get author name from %s' % (orcidlink))
                    driver.get(orcidlink)
                    orcidpage = BeautifulSoup(driver.page_source, features="lxml")
                    author = ''
                    for meta in orcidpage.head.find_all('meta', attrs = {'property' : 'profile:last_name'}):
                        author = meta['content']
                    for meta in orcidpage.head.find_all('meta', attrs = {'property' : 'profile:first_name'}):
                        author += ', ' + meta['content']
                    if author:
                        rec['autaff'][0][0] = author
                        print('\n   got author name "%s" from ORCID page\n' % (author))
                    else:
                        rec['autaff'][0][0] = 'Doe, John'
                        print('\n   could not get author name from ORCID page\n')
                    time.sleep(10)
            else:
                print(div)
    #keywords
    for div in artpage.body.find_all('div', attrs = {'id' : 'bp_categories'}):
        for p in div.find_all('p'):
            rec['keyw'] = re.split(' *\| *', p.text.strip())
    #department
    for div in artpage.body.find_all('div', attrs = {'id' : 'department'}):
        for p in div.find_all('p'):
            department = p.text.strip()
            if department in boring:
                keepit = False
            elif department in ['Computer Science']:
                rec['fc'] = 'c'
            elif department in ['Astronomy']:
                rec['fc'] = 'a'
            elif department in ['Applied Mathematics', 'Mathematics']:
                rec['fc'] = 'm'
                rec['autaff'][-1].append('UMASS, Amherst, Dept. Math. Stat.')
            elif department in ['Statistics']:
                rec['fc'] = 's'
                rec['autaff'][-1].append('UMASS, Amherst, Dept. Math. Stat.')
            elif department in ['Physics and Astronomy', 'Physics & Astronomy', 'Physics']:
                rec['autaff'][-1].append('Massachusetts U., Amherst')
            else:
                rec['note'].append(department)
    #license
    for link in artpage.find_all('link', attrs = {'rel' : 'license'}):
        rec['license'] = {'url' : link['href']}
    if keepit:
        if len(rec['autaff'][-1]) == 1:
            rec['autaff'][-1].append(publisher)
        if skipalreadyharvested and 'doi' in rec and rec['doi'] in alreadyharvested:
            print('   %s already in backup' % (rec['doi']))
        else:
            ejlmod3.printrecsummary(rec)
            recs.append(rec)
    else:
        ejlmod3.adduninterestingDOI(rec['artlink'])

ejlmod3.writenewXML(recs, publisher, jnlfilename)


# Databricks notebook source
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns # viz
import matplotlib.pyplot as plt # viz
from scipy import stats
import json
from typing import List, Tuple

from sklearn.ensemble import IsolationForest
from sklearn.metrics import f1_score, balanced_accuracy_score, roc_auc_score, precision_recall_fscore_support
from sklearn import metrics, linear_model

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import TensorDataset, DataLoader

import warnings
warnings.filterwarnings('ignore')

# COMMAND ----------

train_df.groupby(['sus', 'evil'])[['timestamp']].count()

# COMMAND ----------

test_df.groupby(['sus', 'evil'])[['timestamp']].count()

# COMMAND ----------

train_df.groupby(['sus'])[['timestamp']].count()

# COMMAND ----------

test_df.groupby(['sus'])[['timestamp']].count()

# COMMAND ----------

train_df.sus.value_counts().plot(kind='bar', title='Label Frequency for sus label in Train Dataset')

# COMMAND ----------

def dataset_to_corr_heatmap(dataframe, title, ax):
    corr = dataframe.corr()
    sns.heatmap(corr, ax = ax, annot=True, cmap="YlGnBu")
    ax.set_title(f'Correlation Plot for {title}')

# COMMAND ----------

datasets = [train_df, test_df, validation_df]

entropy_values = []
for dataset in datasets:
    dataset_entropy_values = []
    for col in dataset.columns:
        if col == 'timestamp':
            pass
        else:
            counts = dataset[col].value_counts()
            col_entropy = stats.entropy(counts)
            dataset_entropy_values.append(col_entropy)

    entropy_values.append(dataset_entropy_values)

plt.boxplot(entropy_values)
plt.title('Boxplot of Entropy Values')
plt.ylabel("entropy values")
plt.xticks([0,1,2,3],labels=['','train', 'test', 'validate'])
plt.show()

# COMMAND ----------

train = train_df[["processId", "parentProcessId", "userId", "mountNamespace", "eventId", "argsNum", "returnValue"]]
train_labels = train_df['sus']

# COMMAND ----------

def process_args_dataframe(df: pd.DataFrame, column_name: str) -> pd.DataFrame:
    """
    Processes the `args` column within the dataset
    """

    processed_dataframes = []
    data = df[column_name].tolist()

    # Debug counter
    counter = 0

    for row in data:
        if row == '[]': # If there are no args
            pass
        else:
            try:
                ret = process_args_row(row)
                processed_dataframes.append(ret)
            except:
                print(f'Error Encounter: Row {counter} - {row}')

            counter+=1

    processed = pd.concat(processed_dataframes).reset_index(drop=True)
    processed.columns = processed.columns.str.lstrip()

    df = pd.concat([df, processed], axis=1)

    return df

def prepare_dataset(df: pd.DataFrame, process_args=False) -> pd.DataFrame:
    """
    Prepare the dataset by completing the standard feature engineering tasks
    """

    df["processId"] = train_df["processId"].map(lambda x: 0 if x in [0, 1, 2] else 1)  # Map to OS/not OS
    df["parentProcessId"] = train_df["parentProcessId"].map(lambda x: 0 if x in [0, 1, 2] else 1)  # Map to OS/not OS
    df["userId"] = pd.to_numeric(df["userId"])  # Convert "userId" column to numeric values
    df["userId"] = train_df["userId"].map(lambda x: 0 if x < 1000 else 1)  # Map to OS/not OS
    df["mountNamespace"] = train_df["mountNamespace"].map(lambda x: 0 if x == 4026531840 else 1)  # Map to mount access to mnt/ (all non-OS users) /elsewhere
    df["eventId"] = train_df["eventId"]  # Keep eventId values (requires knowing max value)
    df["returnValue"] = pd.to_numeric(df["returnValue"], errors="coerce")  # Convert "returnValue" column to numeric values
    df["returnValue"] = train_df["returnValue"].map(lambda x: 0 if x == 0 else (1 if x > 0 else 2))  # Map to success/success with value/error

    if process_args is True:
        df = process_args_dataframe(df, 'args')

    features = df[["processId", "parentProcessId", "userId", "mountNamespace", "eventId", "argsNum", "returnValue"]]
    labels = df['sus']

    return features, labels

# COMMAND ----------

train_no_args_feats, train_no_args_labels = prepare_dataset(train_df)

# COMMAND ----------

train_df_feats, train_df_labels = prepare_dataset(train_df)
test_df_feats, test_df_labels = prepare_dataset(test_df)
val_df_feats, val_df_labels = prepare_dataset(validation_df)

# COMMAND ----------

clf = IsolationForest(contamination=0.1, random_state=0).fit(train_df_feats)

# COMMAND ----------

def metric_printer(y_true, y_pred):
    y_true = np.array([int(label) if label.isdigit() else -1 for label in y_true], dtype=np.int64)
    y_true[y_true == 0] = 1

    metric_tuple = precision_recall_fscore_support(y_true, y_pred, average="weighted", pos_label=-1)
    print(f'Precision:\t{metric_tuple[0]}')
    print(f'Recall:\t\t{metric_tuple[1]:.3f}')
    print(f'F1-Score:\t{metric_tuple[2]:.3f}')

# COMMAND ----------

y_pred= clf.predict(val_df_feats)
y_probas = clf.score_samples(val_df_feats)
metric_printer(val_df_labels, y_pred)

# COMMAND ----------

y_pred= clf.predict(test_df_feats)
y_probas = clf.score_samples(test_df_feats)
metric_printer(test_df_labels, y_pred)


import numpy as np
from cffi import FFI

__all__ = [ 'subsample2' ]

ffi = FFI()
ffi.cdef('''

void downsample2(const double* x,             // shape: n x point_dim
                 const int* idx_selected,     // shape: k
                 int k,
                 const int* idx_not_selected, // shape: n-k
                 unsigned char* selected,     // shape: n
                 double* d_min,               // shape: n
                 int n,                       // number of points
                 int point_dim,
                 int n_pts_to_add);

''')

so = ffi.dlopen('subsample2.so')
downsample = so.downsample2

def const_double_ptr(arr):
    assert arr.dtype == np.float64, arr.dtype
    return ffi.cast('const double*', arr.ctypes.data)

def double_ptr(arr):
    assert arr.dtype == np.float64, arr.dtype
    return ffi.cast('double*', arr.ctypes.data)

def uchar_ptr(arr):
    assert arr.dtype == np.bool_, arr.dtype
    return ffi.cast('unsigned char*', arr.ctypes.data)

def const_int_ptr(arr):
    assert arr.dtype == np.intc, arr.dtype
    return ffi.cast('const int*', arr.ctypes.data)

#-------------------------------------------------------------------------------

def subsample2(x, n_desired_pts):
    # Returns: a boolean array, indicating whether a point in x was selected.
    assert x.ndim == 2, x.shape # x is a 2D array of the points
    n_pts, point_dim = x.shape[0], x.shape[1]
    assert n_pts > 0 and point_dim > 0, (n_pts, point_dim)
    assert np.isfinite(x).all()
    assert 0 < n_desired_pts, n_desired_pts
    if n_desired_pts > n_pts:
        msg = '*** Warning: We have less points than desired! ({} < {}) ***'
        print(msg.format(n_pts, n_desired_pts))
        return np.full(n_pts, np.True_, dtype=np.bool_)
    else:
        # Best effort: try to pick the point, closest to the middle:
        mean = np.mean(x, axis=0)
        idx = np.argmin(np.linalg.norm(x-mean, axis=1, ord=1))
        selected = np.full(n_pts, np.False_, dtype=np.bool_)
        selected[idx] = np.True_
        # Then call subsampling to fill up the rest, if any
        return subsample2_(x, selected, n_desired_pts)

def subsample2_(x, selected, n_desired_pts):
    # Similar to subsample but some points are already selected
    assert x.ndim == 2, x.shape # x is a 2D array of the points
    n_pts, point_dim = x.shape[0], x.shape[1]
    assert n_pts > 0 and point_dim > 0, (n_pts, point_dim)
    assert np.isfinite(x).all()
    assert x.shape[0] == selected.shape[0], (x.shape, selected.shape)
    assert 0 < n_desired_pts, n_desired_pts
    if n_desired_pts > n_pts:
        msg = '*** Warning: We have less points than desired! ({} < {}) ***'
        print(msg.format(n_pts, n_desired_pts))
        return np.full(n_pts, np.True_, dtype=np.bool_)
    selected = selected.copy()
    all_idx = np.arange(n_pts)
    idx = all_idx[selected]
    k = len(idx)
    to_add = n_desired_pts - k
    if to_add <= 0:
        return selected
    idx_not_selected = all_idx[~selected]
    assert len(idx_not_selected) == n_pts - k
    assert selected.any()
    x = np.ascontiguousarray(x)
    idx = idx.astype(np.intc)
    idx_not_selected = idx_not_selected.astype(np.intc)
    # d_nn: temporary work array for distances to the nearest neighbors
    d_nn = np.full(n_pts, np.nan)
    downsample(const_double_ptr(x), # shape: (n_pts, point_dim)
               const_int_ptr(idx),  # shape: (k,)
               k,
               const_int_ptr(idx_not_selected), # shape: (n_pts-k,)
               uchar_ptr(selected), # shape: (n_pts,)
               double_ptr(d_nn),    # shape: (n_pts,)
               n_pts,
               point_dim,
               to_add)
    return selected

#-------------------------------------------------------------------------------
# For performance testing and profiling with perf

def _main():
    np.random.seed(1)
    # 10k -> 1k downsampling, point dimension: 25
    subsample2(np.random.random((10000, 25)), 1000)

if __name__ == '__main__':
    _main()


import datetime

from flask import g
from flask_restful import Resource, reqparse, abort, fields, marshal
from sqlalchemy.sql.elements import or_, and_

from App.apis.api_constant import HTTP_OK
from App.apis.movie_user_apis.utils import login_required, required_permmsion
from App.ext import db
from App.modeles.cinema_admin.cinema_hall_model import Hall
from App.modeles.cinema_admin.hall_movie_model import HallMovie
from App.modeles.user.model_constant import VIP_USER
from App.modeles.user.movie_order_model import MovieOrder, ORDER_STATUS_PAYED_NOT_GET, ORDER_STATUS_NOT_PAY, \
    ORDER_STATUS_GET

parse = reqparse.RequestParser()
parse.add_argument("hall_movie_id",required = True,help="请提供排挡信息")
parse.add_argument("o_seats",required=True,help="请选择座位号")

movie_order_fields = {
    "o_price":fields.Float,
    "o_seats":fields.String,
    "o_hall_movie_id":fields.Integer,
    "o_status":fields.String,
    "o_user_id":fields.Integer
}

class MovieUserOrderResource(Resource):

    @login_required
    def post(self):
        user = g.user
        args = parse.parse_args()
        hall_movie_id = args.get("hall_movie_id")
        o_seats = args.get("o_seats")

        """
        座位存在
        座位没有被锁单或者下单
        """
        #可用座位编号

        hall = Hall.query.get(HallMovie.query.get(hall_movie_id).h_id)

        movie_order_buyed = MovieOrder.query.filter(MovieOrder.o_hall_movie_id == hall_movie_id).filter(or_(MovieOrder.o_status == ORDER_STATUS_PAYED_NOT_GET,MovieOrder.o_status == ORDER_STATUS_GET)).all()

        movie_order_locked = MovieOrder.query.filter(MovieOrder.o_hall_movie_id == hall_movie_id).filter(and_(MovieOrder.o_status == ORDER_STATUS_NOT_PAY,MovieOrder.o_time > datetime.datetime.now())).all()

        print("movie_order_buyed",movie_order_buyed)
        print("movie_order_locked",movie_order_locked)

        seats_lock =[]

        for movie_order in movie_order_buyed:
            seats = movie_order.o_seats.split("#")

            seats_lock += seats

        for movie_order in movie_order_locked:
            seats = movie_order.o_seats.split("#")
            seats_lock += seats

        print("seats_lock",seats_lock)

        print("hall",hall)

        seats_can_buy = list(set(hall.h_seate.split("#")) ^ set(seats_lock))

        o_seats_list = o_seats.split("#")
        #判断当前选择的座位号是不是可选择座位号的子集
        if not set(o_seats_list).issubset(set(seats_can_buy)):
            abort("座位选择错误，请重新选择")
        #可用座位编号

        mover_order = MovieOrder()
        mover_order.o_hall_movie_id = hall_movie_id
        mover_order.o_seats = o_seats
        mover_order.o_user_id = user.id
        mover_order.o_time = datetime.datetime.now() + datetime.timedelta(minutes=15)

        '''
        try:
            movie_order1 = MovieOrder.query.get(1)
            movie_order2 = MovieOrder.query.get(2)
            db.session.delete(movie_order1)
            db.session.delete(movie_order2)
        except Exception as e:
            print(e)
            db.session.rollback()

        else:
            db.session.commit()
        '''

        if not mover_order.save():
            abort(400,msg="下单失败")

        data = {
            "msg":"下单成功",
            "status":HTTP_OK,
            "data":marshal(mover_order,movie_order_fields)
        }
        return data

class MovieUserOrdersResource(Resource):

    @required_permmsion(VIP_USER)
    def put(self,orderid):

        return {"msg":"oook"}