```
from google.colab import drive
drive.mount('/content/drive')
```

    Mounted at /content/drive

```
current_path = '/content/drive/My Drive/Colab Notebooks/Project_GANder/gankook_cuda/semantic-segmentation-pytorch/pytorch_room_layout/'

import os
os.chdir(current_path)
!ls
```

    build		      features		     segment_lsun.py
    cv2_requirements.txt  info.json		     segment_rl.py
    datasets	      lsun.pth.tar	     segment_st.py
    data_transforms.py    lu_vp_detect.egg-info  sunrgbd.pth.tar
    dist		      pics		     test_result
    drn_d_105_024_val     __pycache__	     Vanish_Point_Detection
    drn_d_105_024_val_ms  README.md		     visualize.m
    drn.py		      requirements.txt	     XiaohuLuVPDetection

### Get Vanishing point, inliers

```
# !python XiaohuLuVPDetection/setup.py build
# !python XiaohuLuVPDetection/setup.py install
!pip install -r cv2_requirements.txt
```

    Collecting opencv-python==4.0.0.21
      Downloading https://files.pythonhosted.org/packages/37/49/874d119948a5a084a7ebe98308214098ef3471d76ab74200f9800efeef15/opencv_python-4.0.0.21-cp36-cp36m-manylinux1_x86_64.whl (25.4MB)
         |████████████████████████████████| 25.4MB 97.0MB/s
    Collecting opencv-contrib-python==4.0.0.21
      Downloading https://files.pythonhosted.org/packages/1f/c0/0463d91f297521b2e15e3d682d7077557fe773db72a03a0d2dba899ab8a5/opencv_contrib_python-4.0.0.21-cp36-cp36m-manylinux1_x86_64.whl (31.2MB)
         |████████████████████████████████| 31.2MB 180kB/s
    Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from opencv-python==4.0.0.21->-r cv2_requirements.txt (line 1)) (1.18.5)
    ERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.
    Installing collected packages: opencv-python, opencv-contrib-python
      Found existing installation: opencv-python 4.1.2.30
        Uninstalling opencv-python-4.1.2.30:
          Successfully uninstalled opencv-python-4.1.2.30
      Found existing installation: opencv-contrib-python 4.1.2.30
        Uninstalling opencv-contrib-python-4.1.2.30:
          Successfully uninstalled opencv-contrib-python-4.1.2.30
    Successfully installed opencv-contrib-python-4.0.0.21 opencv-python-4.0.0.21

```
import cv2
cv2.__version__
```

    '4.0.0'

### pytorch.room.layout

```
# !pip install -r requirements.txt
# !apt install octave
```

```
# import torch
# torch.__version__
```

```
# !python segment_rl.py train -d datasets/lsun/ -c 4 -s 480 --arch drn_d_105 --batch-size 32 --random-scale 1.75 --random-rotate 15 --epochs 100 --lr 0.01 --momentum 0.9 --lr-mode poly --pretrained sunrgbd.pth.tar
```

```
# !python segment_st.py test -d datasets/sunrgbd/ -c 37 --arch drn_d_105 -s 480 --resume sunrgbd.pth.tar --phase val --batch-size 1 --ms
```

```
# !python segment_lsun.py test -d ../mask_data/wall/ -o ../mask_layout_data/wall/ -c 4 --arch drn_d_105 -s 320 --resume lsun.pth.tar --phase test --batch-size 1 --ms
```

### Functions

```
#             Detect Intersection             #
import math
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.cluster import KMeans
from sympy import Line
from sympy.geometry import Ray

def line(p1, p2):
    A = (p1[1] - p2[1])
    B = (p2[0] - p1[0])
    C = (p1[0]*p2[1] - p2[0]*p1[1])
    return A, B, -C

def intersection2(L1, L2):
    D  = L1[0] * L2[1] - L1[1] * L2[0]
    Dx = L1[2] * L2[1] - L1[1] * L2[2]
    Dy = L1[0] * L2[2] - L1[2] * L2[0]
    if D != 0:
        x = Dx / D
        y = Dy / D
        return x,y
    else:
        return False

def regression(img, x, y, color=(255, 0, 0), axis=1):

  if axis == 1:
    y_at_border = np.array([0, img.shape[0]])
    p = np.polyfit(y, x, deg=1)
    x_at_border = np.poly1d(p)(y_at_border)
  else:
    x_at_border = np.array([0, img.shape[1]])
    p = np.polyfit(x, y, deg=1)
    y_at_border = np.poly1d(p)(x_at_border)

  cv2.line(img, (int(x_at_border[0]), int(y_at_border[0])), (int(x_at_border[1]), int(y_at_border[1])), color, 2)

  return x_at_border, y_at_border

def find_reg_points(img, lines, color=(255,0,0), drawlines=False):

  centroids = list()
  r_xs = list()
  r_ys = list()
  for line_ in lines:
      for rho,theta in line_:
          a = np.cos(theta)
          b = np.sin(theta)
          x0 = a*rho
          y0 = b*rho
          x1 = int(x0 + 1000*(-b))
          y1 = int(y0 + 1000*(a))
          x2 = int(x0 - 1000*(-b))
          y2 = int(y0 - 1000*(a))

          slope = (y1 - y0) / float(x1 - x0)
          angle = math.degrees(math.atan(slope))
          if abs(angle) > 80:
            # print(img.shape[1])
            h_layout = line((0, 0), (img.shape[1], 0))
            h_layout_lower = line((0, img.shape[0]), (img.shape[1], img.shape[0]))
            r = intersection2(h_layout, line((x1, y1), (x2, y2)))
            r_lower = intersection2(h_layout_lower, line((x1, y1), (x2, y2)))
            # cv2.line(img, (int(x1), int(y1)), (int(x2), int(y2)), color, 2)

            center_p = (int((r[0] + r_lower[0]) / 2), int((r[1] + r_lower[1])/ 2))
            centroids.append(center_p)

            r_xs.append((r[0], r_lower[0]))
            r_ys.append((r[1], r_lower[1]))

            if drawlines:
              cv2.line(img, (int(r[0]), int(r[1])), (int(r_lower[0]), int(r_lower[1])), color, 2)
              cv2.circle(img, center_p, 10, (255, 0, 255), -1)

  if drawlines:
    cv2.line(img, (int(0), int(0)), (int(0), int(img.shape[0])), color, 2)
    cv2.line(img, (int(img.shape[1]), int(0)), (int(img.shape[1]), int(img.shape[0])), color, 2)
    cv2.circle(img, (0, int(img.shape[0] / 2)), 10, (255, 0, 255), -1)
    cv2.circle(img, (img.shape[1], int(img.shape[0] / 2)), 10, (255, 0, 255), -1)
  centroids.append((0, int(img.shape[0] / 2)))
  centroids.append((img.shape[1], int(img.shape[0] / 2)))

  return r_xs, r_ys, centroids

from scipy.spatial import distance as sci_dist

def order_points(pts):

  xSorted = pts[np.argsort(pts[:, 0]), :]
  leftMost = xSorted[:2, :]
  rightMost = xSorted[2:, :]
  leftMost = leftMost[np.argsort(leftMost[:, 1]), :]
  (tl, bl) = leftMost
  D = sci_dist.cdist(tl[np.newaxis], rightMost, "euclidean")[0]
  (br, tr) = rightMost[np.argsort(D)[::-1], :]

  return np.array([tl, tr, br, bl], dtype="float32")

def drawLines(img, lines, color=(255,255,255), only_horizontal=False):
    """
    Draw lines on an image
    """
    for line in lines:
        for rho,theta in line:
          a = np.cos(theta)
          b = np.sin(theta)
          x0 = a*rho
          y0 = b*rho
          x1 = int(x0 + 1000*(-b))
          y1 = int(y0 + 1000*(a))
          x2 = int(x0 - 1000*(-b))
          y2 = int(y0 - 1000*(a))

          slope = (y1 - y0) / float(x1 - x0)
          angle = math.degrees(math.atan(slope))
          if only_horizontal:
            if abs(angle) < 70:
              cv2.line(img, (x1,y1), (x2,y2), color, 1)
          else:
            cv2.line(img, (x1,y1), (x2,y2), color, 1)

```

```
from PIL import Image
import os
import matplotlib.pyplot as plt
import numpy as np
from XiaohuLuVPDetection.lu_vp_detect.vp_detection import VPDetection
import time
# import os
import cv2
# import pylab as pl
from skimage import morphology as mp
import sys
from Vanish_Point_Detection import wrapper
# from varname import Wrapper

length_thresh = 50
principal_point = None
focal_length = 1300 # 1102.79
seed = 1300
vpd = VPDetection(length_thresh, principal_point, focal_length, seed)

org_path = 'datasets/lsun/images/' # -> original image
layout_path = 'drn_d_105_024_val_ms/images/' # -> layout image (result image frome above model file)
refer_path = '../refer_data/wall/myxkehu1kfzggursepnk0tfiyps8zbs5umvzv8d92r6hhxejgawebwsufssgov5q-.jpg'

object_ = 'wall'
mask_path = '../mask_data/' + object_ + '/'
# layout_path = '../test_layout_result/' + object_ + '/'
layout_path = '../mask_layout_data/' + object_ + '/'
# refer_path = './refer_data/' + object_ + '/'

val_image_list = os.listdir(layout_path)

one_iter = False
start_index = 0
for image_index, image in enumerate(val_image_list):

  if image_index < start_index:
    continue

  # image = '720x480_20190219162540169_bnvKQG536n.png'
  # print('image_index :', val_image_list.index(image))
  # one_iter = True

  # print(image)
  img = Image.open(layout_path + image)
  # print(type(img))
  img_np = np.invert(np.asarray(img))
  # print(img_np.max(), img_np.min())
  ret, thr = cv2.threshold(img_np, 254, 255, cv2.THRESH_BINARY_INV)

  org = Image.open(mask_path + image)
  org_color_np, org_np = np.split(np.asarray(org), 2, axis=1)
  # org_np = np.asarray(org)
  # gray = cv2.cvtColor(org_np, cv2.COLOR_RGB2GRAY)
  # print(org_np.shape)
  # ret, thr = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY_INV)

  print(image)
  # plt.subplot(131)
  # plt.imshow(org)
  # plt.show()
  img_size = (org_np.shape[1], org_np.shape[0])

  scale_factor = 6
  print('scale_factor :', scale_factor)

  refer = Image.open(refer_path)
  refer = np.asarray(refer)

  #     refer_size와 img_size가 동일하거나 refer_size가 작은 경우를 고려해야한다.     #
  refer = np.tile(refer, (scale_factor, scale_factor, 1))
  size_ratio = math.floor(min((refer.shape[0] / (org_np.shape[0] * 1.5)), (refer.shape[1] / (org_np.shape[1] * 1.5))))
  refer = Image.fromarray(refer).resize((int(refer.shape[1] / size_ratio), int(refer.shape[0] / size_ratio)))

  # tun = thr
  # skl = mp.medial_axis(tun).astype(np.uint8) * 255
  rho = 1
  theta = np.pi/180
  thresh = 50
  # print(skl.shape, skl.dtype)

  # lines = cv2.HoughLines(org_np.astype(np.uint8), rho, theta, thresh)
  # lines = cv2.HoughLines(skl, rho, theta, thresh)

  # print(lines)
  # plt.imshow(skl)
  # plt.show()

  # Draw all Hough lines in red
  # img_with_all_lines = np.copy(skl)
  # img_with_all_lines = cv2.cvtColor(img_with_all_lines, cv2.COLOR_GRAY2RGB)
  # plt.imshow(img_with_all_lines)
  # plt.show()
  org_color_np2 = org_color_np.copy()
  org_np = org_np.astype(np.uint8)
  org_np2 = org_np.copy()
  org_np3 = org_np.copy()

  start = time.time()

  # r_x_list, r_y_list, centroids_list = find_reg_points(org_np, lines,)
  # drawLines(img_with_all_lines, lines)

  #     Edit Your Mask     #
  kernel = np.ones((10, 10), np.uint8)

  # plt.figure(figsize=(20, 15))
  # for plot_i in range(5):
  #   plt.subplot(1, 5, plot_i+1)
  #   if plot_i == 0:
  #     plt.imshow(org_np2)
  #   elif plot_i == 1: #      Dilation     #
  #     plt.imshow(cv2.dilate(org_np2, kernel, iterations=1))
  #   elif plot_i == 2: #      Closing      #
  #     plt.imshow(cv2.morphologyEx(org_np2, cv2.MORPH_CLOSE, kernel))
  #   elif plot_i == 3: #     Erode     #
  #     plt.imshow(cv2.erode(org_np2, kernel, iterations=1))
  #   elif plot_i == 4: #     Opening   #
  #     plt.imshow(cv2.morphologyEx(org_np2, cv2.MORPH_OPEN, kernel))
  # plt.show()

  # mask_prob =

  org_np2_morp = cv2.morphologyEx(org_np2, cv2.MORPH_CLOSE, kernel)
  org_np2_copy = org_np2_morp.copy()
  org_np2_copy_gray = cv2.cvtColor(org_np2_copy, cv2.COLOR_RGB2GRAY)

  vpd.find_vps(org_np2_morp)
  vps = vpd.vps_2D
  vl_org_img, vl_list = vpd.create_debug_VP_image()

  ret, thr_org_np2 = cv2.threshold(org_np2_copy, 127, 255, cv2.THRESH_BINARY)
  thr_org_np2 = cv2.morphologyEx(thr_org_np2, cv2.MORPH_CLOSE, kernel)
  thr_org_np2_copy = thr_org_np2.copy()
  vpd.find_vps(thr_org_np2_copy)
  # vps = vpd.vps_2D
  vl_thresh_org_img, vl_thresh_org_list = vpd.create_debug_VP_image(show_vl=True)

  #       multiply thresh mask to color map      #
  org_color_np = (vl_thresh_org_img / 255. * org_color_np).astype(np.uint8)
  vpd.find_vps(org_color_np)
  vps = vpd.vps_2D
  vl_color_img, vl_color_list = vpd.create_debug_VP_image()

  # plt.imshow(vl_thresh_org_img)
  # plt.show()

  edge_org_np2 = cv2.Canny(org_np2_morp, 20, 300)
  edge_org_np2_copy = edge_org_np2.copy()
  edge_org_np2_copy2 = cv2.Canny((org_np2_copy / 255. * org_np2_copy).astype(np.uint8), 20, 300)

  vpd.find_vps(edge_org_np2)
  # vps = vpd.vps_2D
  edge_vl_img, edge_vl_list = vpd.create_debug_VP_image()

  vpd.find_vps(edge_org_np2_copy2)
  # vps = vpd.vps_2D
  edge_vl_img2, edge_vl_list2 = vpd.create_debug_VP_image()

  #             Hough Lines Detection           #
  # print('org_np2.dtype :', org_np2.dtype)
  # print('org_np2.shape :', org_np2.shape)
  # plt.imshow(org_np2_copy_gray)
  # plt.show()
  # lines = cv2.HoughLines(org_np2_copy_gray, rho, theta, thresh)
  # drawLines(org_np2_copy_gray, lines)
  edge_lines = cv2.HoughLines(edge_org_np2_copy, rho, theta, thresh)
  drawLines(edge_org_np2_copy, edge_lines, only_horizontal=True)
  edge_lines2 = cv2.HoughLines(edge_org_np2_copy2, rho, theta, thresh)
  drawLines(edge_org_np2_copy2, edge_lines2, only_horizontal=True)

  plt.figure(figsize=(15, 10))
  plt.subplot(151)
  plt.imshow(vl_org_img)
  plt.title('vl_org_img')

  plt.subplot(152)
  # plt.imshow(org_color_np)
  plt.imshow(vl_color_img)

  plt.subplot(153)
  plt.imshow(vl_thresh_org_img)
  plt.title('vl_thresh_org_img')

  plt.subplot(154)
  plt.imshow(edge_vl_img)
  plt.title('edge_vl_img')

  plt.subplot(155)
  plt.imshow(edge_vl_img2)
  plt.title('edge_vl_img2')
  plt.show()

  vl_list_for_vline = vl_color_list
  vl_img_for_vline = edge_vl_img2.copy()

  print('#       Used vl_list for vline work       #')
  print('vl_color_list')
  print()

  #         Extend v_lines and draw h_lines       #
  h_border = Line((0, 0), (org_np.shape[1], 0))
  h_border_lower = Line((0, org_np.shape[0]), (org_np.shape[1], org_np.shape[0]))

  reg_xs = list()
  reg_ys = list()
  v_lines = list()
  h_lines_xrange = list()
  for vl in vl_list_for_vline:
      x0, y0, x1, y1 = vl
      slope = (y1 - y0) / float(x1 - x0)
      angle = math.degrees(math.atan(slope))
      # print(angle)
      if abs(angle) > 80:
        # cv2.line(skl_copy, (int(x1), int(y1)), (int(x0), int(y0)), (255, 0, 0), 3,
        #                  cv2.LINE_AA)
        v_line = Line((x0, y0), (x1, y1))
        [top_point] = v_line.intersection(h_border)
        [bot_point] = v_line.intersection(h_border_lower)
        # v_lines.append()
        reg_xs.append((float(top_point[0]), float(bot_point[0])))
        reg_ys.append((float(top_point[1]), float(bot_point[1])))

      if abs(angle) < 70:
        h_lines_xrange.append((min(x0, x1), max(x0, x1)))

  #           Remove Unnecessary Vline        #
  # print("#        Before vline Removing       #")
  # print('reg_xs :', reg_xs)
  # reg_xs_copy = reg_xs.copy()
  # reg_ys_copy = reg_ys.copy()
  # for reg_x, reg_y in zip(reg_xs, reg_ys):
  #   for x_range in h_lines_xrange:
  #     if x_range[0] + 5 < np.mean(reg_x) < x_range[1] - 5:
  #       # print('reg_x :', reg_x)
  #       reg_xs_copy.remove(reg_x)
  #       reg_ys_copy.remove(reg_y)
  #       break
  # reg_xs = reg_xs_copy
  # reg_ys = reg_ys_copy
  # print('#        After Removing      #')
  # print('reg_xs :', reg_xs)

  print('#        Before Sorting      #')
  # print('reg_xs :', reg_xs)

  center_xs = list()
  for reg_x, reg_y in zip(reg_xs, reg_ys):
    #     all_closest_inter가 정렬되지 않았다면 inters type = list()      #
    # sum_x = 0
    # for inter in inters:
    #   sum_x += inter[0] + inter[2]
    center_xs.append(np.mean(reg_x))
    # regression(vl_img_for_vline, reg_x, reg_y)

  # plt.imshow(vl_img_for_vline)
  # plt.title('Divided Session')
  # plt.show()

  # if one_iter:
  #   break
  # else:
  #   continue

  #                       K Means                       #
  centroids_data = np.array(center_xs).reshape(-1, 1)
  # print('len(r_x_list), len(r_y_list) :', len(r_x_list), len(r_y_list))
  print('centroids_data.shape :', centroids_data.shape)
  mms = MinMaxScaler()
  cen_data =mms.fit_transform(centroids_data)
  # print('cen_data :', cen_data)

  # print('cen_data :', cen_data)
  K = range(1, 7)
  s_dist = list()
  for k in K:
    if cen_data.shape[0] < k:
      break
    km = KMeans(n_clusters=k)
    km = km.fit(cen_data)
    inertia = km.inertia_
    s_dist.append(inertia)

    # if inertia <= 0:
    #   print('vertical line number =', k)
    #   break
  mms2 = MinMaxScaler()
  mms_s_dist = mms2.fit_transform(np.array(s_dist).reshape(-1, 1))
  k_thresh = 0.1
  # plt.plot(range(len(mms_s_dist)), mms_s_dist)
  # plt.show()

  if cen_data.shape[0] > 1:
    for i in range(len(mms_s_dist)):
      if mms_s_dist[i] < k_thresh:
        k = i + 1
        break
  else:
    k = 0

  #     k Confirmation : Comparing k.cluster_centers_ dist (Regulation of closed vline's minimum distance)  #
  while True:

    km = KMeans(n_clusters=k)
    km = km.fit(cen_data)
    if k <= 0:
      break

    cluster_centroids = km.cluster_centers_
    # print('cluster_centroids :', cluster_centroids)
    error_exist = False
    for i in range(len(cluster_centroids) - 1):
      for j in range(i + 1, len(cluster_centroids)):
        if abs(cluster_centroids[i] - cluster_centroids[j]) < 0.05:
          error_exist = True
    if error_exist:
      k -= 1
    else:
      break

  print('k is', k)

  # predict_cen = km.predict(cen_data)
  # km = KMeans(n_clusters=k)
  km = km.fit(centroids_data)
  predict_cen = km.cluster_centers_
  print('predict_cen :', predict_cen)

  # keys = list(range(k))
  # for rm in predict_cen[-2:]:
  #   keys.remove(rm)
  # print('keys :', keys)

  #       sorted_predict_cen 로 vline 을 정립한다.      #
  # sorted_index = list()
  sorted_predict_cen = sorted(predict_cen)

  # for center_x in sorted_center_xs:
  #   # print('center_x :', center_x)
  #   sorted_index.append(center_xs.index(center_x))
  # # print('sorted_index :', sorted_index)   #

  # sorted_reg_xs = list()
  # sorted_reg_ys = list()
  # for s_index in sorted_index:
  #   sorted_reg_xs.append(reg_xs[s_index])
  #   sorted_reg_ys.append(reg_ys[s_index])
  # reg_xs = sorted_reg_xs
  # reg_ys = sorted_reg_ys

  print()
  print("#        After Sorting       #")
  # print('reg_xs :', reg_xs)
  print('sorted_predict_cen :', sorted_predict_cen)
  print()

  for reg_x in sorted_predict_cen:
    cv2.line(vl_img_for_vline, (int(reg_x), int(org_np.shape[0])), (int(reg_x), 0), (100, 156, 245), 2)

  plt.figure(figsize=(15, 10))
  plt.subplot(131)
  plt.imshow(vl_img_for_vline)
  plt.title('Divided Session')
  # plt.show()

  plt.subplot(132)
  plt.imshow(edge_org_np2_copy)
  # plt.show()
  plt.subplot(133)
  plt.imshow(edge_org_np2_copy2)
  plt.show()

  #           VL Segmentation             #
  # Angle between line1 and x-axis
  # print(len(vl_list))

  # print('len(v_lines) :', len(v_lines))
  # print('len(h_lines) :', len(h_lines))

  #     vline 이 존재하지 않는 경우도 고려해야한다.     #
  if len(reg_xs) == 0:
    reg_xs.append([0, 0])
    reg_ys.append([0, org_np2.shape[0]])

  #         Divide Session by vline       #
  for reg_index, reg_x in enumerate(sorted_predict_cen):

    #     In a Session     #
    #     1.  find vanishing point    #
    #     2.  Find 3 points (vp, top & bottom points)     #
    #     3.  Do warfine and attach to the black plane      #

    print("#      Session Status      #")
    print('reg_index, reg_x :', reg_index, reg_x)

    #     0.  crop by vline     #
          #   Find Max_x, min_x, (Max_y, min_y = org.shape[0], 0)
          #   1.    우편에 한해서 max_x = 우편 vline max_x & min_x = 현재 vline min_x
          #   2.    좌편에 한해서 max_x = 현재 Max_x & min_x = 좌편 vline min_x

    #      vline 별로 양옆으로 작업을 하면 len(vline) = 1의 작업을 반복할 필요가 없어진다.    #
    iter = False
    while True:

      #                   We need Max, min x & y                #
      find_pair = True

      if not iter:
        #       vline 우측 session      #
        # print(final_xs)
        # four_inters.append([final_xs[0], final_ys[0]])
        # four_inters.append(centroid_inters[0])
        # print(four_inters)

        #       오른쪽 끝 vline 이면        #
        if reg_index == len(sorted_predict_cen) - 1:
          print('Rightest Vline')
          #   1.    우편에 한해서 max_x = 우편 vline max_x & min_x = 현재 vline min_x
          max_x = org_np.shape[1]
          min_x = np.min(reg_x)

        else:
          print('Middle Vline')
          next_reg_x = sorted_predict_cen[reg_index + 1]
          max_x = np.max(next_reg_x)
          if np.min(reg_x) < 0:
            min_x = 0
          else:
            min_x = np.min(reg_x)

      #     i = 0 에 한해서만 왼쪽으로도 refering 진행, 나머지는 오른쪽으로만     #
      else:
          #   2.    좌편에 한해서 max_x = 현재 Max_x & min_x = 좌편 vline min_x
          print('Leftest Vline')
          max_x = np.max(reg_x)
          min_x = 0

      print('min_x, max_x :', min_x, max_x)
      # max_y = org_np.shape[0]
      # min_y = 0

      #     Make Session      #
      session = org_np2[:, int(min_x):int(max_x)]
      # plt.imshow(session)
      # plt.show()

      if (max_x - min_x) > 50:
        length_thresh = 50
      else:
        length_thresh = 20
        if max_x - min_x < 20:
            #     i != 0 인 경우 break     #
          if reg_index == 0 and np.sum(sorted_predict_cen) != 0 and not iter:
            iter = True
            print('iter :', iter)
            continue
          else:
            break

      principal_point = None
      focal_length = 1300 # 1102.79
      seed = None
      vpd = VPDetection(length_thresh, principal_point, focal_length, seed)

      try:
        vpd.find_vps(session)
        # vps = vpd.vps_2D
        vl_img, vl_list = vpd.create_debug_VP_image(show_vl=True)

        #       Closing Morphing by OpenCV      #
        kernel = np.ones((10, 10), np.uint8)
        session = cv2.morphologyEx(session, cv2.MORPH_CLOSE, kernel)

        vpd.find_vps(session)
        # vps = vpd.vps_2D
        vl_session_img, vl_session_list = vpd.create_debug_VP_image(show_vl=True)

        #     Feed Binary inv Threshold     #
        gray = cv2.cvtColor(session, cv2.COLOR_RGB2GRAY)
        ret, thr_session = cv2.threshold(session, 127, 255, cv2.THRESH_BINARY)
        thr_session = cv2.morphologyEx(thr_session, cv2.MORPH_CLOSE, kernel)
        thr_session_copy = thr_session.copy()
        vpd.find_vps(thr_session_copy)
        # vps = vpd.vps_2D
        vl_thresh_img, vl_thresh_list = vpd.create_debug_VP_image(show_vl=True)

        # plt.imshow(vl_thresh_img)
        # plt.title('vl_thresh_img')
        # plt.show()

        #       Edge Session      #
        edge_session = cv2.Canny(session, 20, 60)

        #     Before Canny    #
        session_prob = session / 255.

        #     Multiply mask map with Edge => Erase Unnecessary vl line    #
        #   multiply one time   #
        session2 = session_prob * (session)
        edge_session2 = cv2.Canny(session2.astype(np.uint8), 20, 60)
        #       two time      #
        # session2 = session_prob**2 * (session)
        # edge_session2 = cv2.Canny(session2.astype(np.uint8), 20, 60)

        vpd.find_vps(edge_session)
        # vps = vpd.vps_2D
        vl_edge_img, vl_edge_list = vpd.create_debug_VP_image(show_vl=True)

        vpd.find_vps(edge_session2)
        # vps = vpd.vps_2D
        vl_edge_img2, vl_edge_list2 = vpd.create_debug_VP_image(show_vl=True)

      except Exception as e:
        print("Error in vpd Sessions Zone :", e)
        print()
          #     i != 0 인 경우 break     #
        if reg_index == 0 and np.sum(sorted_predict_cen) != 0 and not iter:
          iter = True
          print('iter :', iter)
          continue

        else:
          break

      # session = wrapper.dealAImage(session,True,True,True)

      # #             color             #
      # session = org_color_np[:, int(min_x):int(max_x)]

      #       Use made vl_img (get more vn_lines)     #
      # session = vl_org_img[:, int(min_x):int(max_x)]

      plt.figure(figsize=(15, 10))
      plt.subplot(151)
      plt.imshow(vl_img)
      plt.title('vl_session_img')

      plt.subplot(152)
      plt.imshow(vl_thresh_img)
      plt.title('vl_thresh_img')

      # plt.subplot(152)
      # plt.imshow(vl_session_img)
      # plt.title('vl_session_img morphed')

      plt.subplot(153)
      plt.imshow(edge_session2)
      plt.title('edge_session2')

      plt.subplot(154)
      plt.imshow(vl_edge_img)
      plt.title('vl_edge_img')

      plt.subplot(155)
      plt.imshow(vl_edge_img2)
      plt.title('vl_edge_img2')
      # plt.imshow(vl_img)
      # plt.title('vl_img')
      plt.show()

      h_lines = list()
      # print('vps :', vps)

      top_vl = list()
      bottom_vl = list()

      left_vl_list = list()
      right_vl_list = list()

      left_angle = list()
      right_angle = list()

      #         What is best vl_list      #
      vl_list = vl_edge_list2
      print('#            Used vl_list for hline work          #')
      print('vl_edge_list2')
      print()

      for vl in vl_list:
        x0, y0, x1, y1 = vl
        slope = (y1 - y0) / float(x1 - x0)
        angle = math.degrees(math.atan(slope))
        # print(angle)
        # if abs(angle) > 80:
        #   # cv2.line(skl_copy, (int(x1), int(y1)), (int(x0), int(y0)), (255, 0, 0), 3,
        #   #                  cv2.LINE_AA)
        #   v_lines.append(vl)

        if abs(angle) < 70:
          if (y0 + y1) / 2 < vl_img.shape[0] / 2:
            # print(y0, y1)
            top_vl.append(vl)
            if angle < 0:
              left_vl_list.append(vl)
              left_angle.append(abs(angle))
            else:
              right_vl_list.append(vl)
              right_angle.append(abs(angle))
          else:
            bottom_vl.append(vl)
            if angle > 0:
              left_vl_list.append(vl)
              left_angle.append(abs(angle))
            else:
              right_vl_list.append(vl)
              right_angle.append(abs(angle))
          # cv2.line(skl_copy, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 2,
          #                  cv2.LINE_AA)
          h_lines.append(vl)
        #   regression(vl_img, (x0, x1), (y0, y1), color=(0,255,255),axis=1)
      print('#        Original vl list        #')
      print('len(left_vl_list) :', len(left_vl_list))
      print('len(right_vl_list) :', len(right_vl_list))
      print()

      print("#            Angle Condition           #")
      limit_angle = 15
      print('limit_angle :', limit_angle)

      copy_left_vl_list = left_vl_list.copy()
      copy_right_vl_list = right_vl_list.copy()

      #             Delete over standarded angle vl             #
      if len(left_vl_list) != 0:
        copy_left_angle = left_angle.copy()
        mean_left_angle = np.array(left_angle).mean()
        # print('mean_left_angle :', mean_left_angle)
        # print('left_angle :', left_angle)
        print('max left_angle gap : ', max(left_angle) - mean_left_angle)

        for angle_index, angle in enumerate(copy_left_angle):
          if abs(angle - mean_left_angle) > limit_angle:
            left_vl_list.remove(copy_left_vl_list[angle_index])
            left_angle.remove(copy_left_angle[angle_index])
        # standardScaler = StandardScaler()
        # std_left_angle = standardScaler.fit_transform(np.array(left_angle).reshape(-1, 1))
        # print('standardScaler.fit_transform(left_angle) :', std_left_angle)
        # print('left_angle :', left_angle)

        # for std_index, std_angle in enumerate(std_left_angle):
        #   if abs(std_angle) > 2:
        #     left_vl_list.remove(copy_left_vl_list[std_index])
            # left_angle.remove(copy_left_angle[std_index])

        # print('left_angle :', left_angle)

      if len(right_vl_list) != 0:
        copy_right_angle = right_angle.copy()
        mean_right_angle = np.array(right_angle).mean()
        # print('mean_right_angle :', mean_right_angle)
        # print('right_angle :', right_angle)
        print('max right_angle gap : ', max(right_angle) - mean_right_angle)

        for angle_index, angle in enumerate(copy_right_angle):
          if abs(angle - mean_right_angle) > limit_angle:
            right_vl_list.remove(copy_right_vl_list[angle_index])
            right_angle.remove(copy_right_angle[angle_index])

      print()

      # top_condition = False
      # bot_condition = False
      # for vl in left_vl_list:
      #   if vl in top_vl:
      #     top_condition = True
      #   elif vl in bottom_vl:
      #     bot_condition = True
      # if top_condition and bot_condition:
      #   pass
      # else:
      #   left_vl_list = list()
      #   print('left top bot condition False')
      #   print('top_condition :', top_condition)
      #   print('bot_condition :', bot_condition)

      # top_condition = False
      # bot_condition = False
      # for vl in right_vl_list:
      #   # print('vl :', vl)
      #   if vl in top_vl:
      #     top_condition = True
      #   elif vl in bottom_vl:
      #     bot_condition = True
      # if top_condition and bot_condition:
      #   pass
      # else:
      #   right_vl_list = list()
      #   print('right top bot condition False')
      #   print('top_condition :', top_condition)
      #   print('bot_condition :', bot_condition)

      # print('-- after --')
      # print('len(left_vl_list) :', len(left_vl_list))
      # print('len(right_vl_list) :', len(right_vl_list))
      # print()

      # for vl in vl_list:
      #   x0, y0, x1, y1 = vl

      #   # if right_gap < 30:
      #   #   left_vl_list.append(vl)
      #   #   print('right_angle :', angle)
      #   if vl in left_vl_list:
      #     regression(vl_img, (x0, x1), (y0, y1), color=(0,0,255),axis=1)
      #   # elif left_gap < 30:
      #   elif vl in right_vl_list:
      #     # right_vl_list.append(vl)
      #     # print('left_angle :', angle)
      #     regression(vl_img, (x0, x1), (y0, y1), color=(255,0,0),axis=1)
      if len(left_vl_list) == 0 and len(right_vl_list) == 0:
          #     i != 0 인 경우 break     #
        if reg_index == 0 and not iter:
          iter = True
          print('iter :', iter)
          continue

        else:
          break

      #                       Find External vn_line                     #
      h_border_top = Line((0, 0), (vl_img.shape[1], 0))
      h_border_bot = Line((0, vl_img.shape[0]), (vl_img.shape[1], vl_img.shape[0]))
      top_down_ratio = 0.66
      left_margin = 0.33
      right_margin = 1 - left_margin

      ex_left_top_vl = list()
      ex_left_bot_vl = list()
      ex_right_top_vl = list()
      ex_right_bot_vl = list()

      l2 = Line((0, 0), (0, vl_img.shape[0]))
      l3 = Line((vl_img.shape[1], 0), (vl_img.shape[1], vl_img.shape[0]))

      for vl in left_vl_list:
        x0, y0, x1, y1 = vl
        l1 = Line((x0, y0), (x1, y1))
        #     v_border line     #
        [left_point] = l1.intersection(l2)
        # print('left_point :', left_point)
        [right_point] = l1.intersection(l3)

        top_bot_pass = False
        try:
          [top_point] = l1.intersection(h_border_top)
          [bot_point] = l1.intersection(h_border_bot)
        except:
          top_bot_pass = True
        # print('top_point :', int(top_point[0]), int(top_point[1]))
        # print('bot_point :', int(bot_point[0]), int(bot_point[1]))

        #      Figure out current vl is top / bottom vl     #
        extend = False
        if vl in top_vl:
          #      line should have same height (y)     #
          if (left_point[1] < vl_img.shape[0] * top_down_ratio and right_point[1] < vl_img.shape[0] * top_down_ratio):
            #     h_border 와의 교차점 조건    #
            # if not top_bot_pass:
            # if top_point[0] >= vl_img.shape[1] * right_margin and bot_point[0] <= vl_img.shape[1] * left_margin:
              # extend = True
              ex_left_top_vl.append((left_point[0], left_point[1], right_point[0], right_point[1]))

        elif vl in bottom_vl:
          # print('vl in bottom at left :', vl)
          #      line should have same height (y)     #
          if (left_point[1] > vl_img.shape[0] * (1 - top_down_ratio) and right_point[1] > vl_img.shape[0] * (1 - top_down_ratio)):
            #     h_border 와의 교차점 조건    #
            # if not top_bot_pass:
            # if top_point[0] <= vl_img.shape[1] * left_margin and bot_point[0] >= vl_img.shape[1] * right_margin:
              # extend = True
              ex_left_bot_vl.append((left_point[0], left_point[1], right_point[0], right_point[1]))

      for vl in right_vl_list:
        x0, y0, x1, y1 = vl
        l1 = Line((x0, y0), (x1, y1))
        #     v_border line     #

        [left_point] = l1.intersection(l2)
        # print('left_point :', left_point)
        [right_point] = l1.intersection(l3)

        top_bot_pass = False
        try:
          [top_point] = l1.intersection(h_border_top)
          [bot_point] = l1.intersection(h_border_bot)
        except:
          top_bot_pass = True
        # print('top_point :', int(top_point[0]), int(top_point[1]))
        # print('bot_point :', int(bot_point[0]), int(bot_point[1]))

        #      Figure out current vl is top / bottom vl     #
        extend = False
        if vl in top_vl:
          #      line should have same height (y)     #
          if (left_point[1] < vl_img.shape[0] * top_down_ratio and right_point[1] < vl_img.shape[0] * top_down_ratio):
            #     h_border 와의 교차점 조건    #
            # if top_point[0] <= vl_img.shape[1] * left_margin and bot_point[0] >= vl_img.shape[1] * right_margin:
              # extend = True
              ex_right_top_vl.append((left_point[0], left_point[1], right_point[0], right_point[1]))

        elif vl in bottom_vl:
          # print('vl in bottom at right :', vl)
          #      line should have same height (y)     #
          if left_point[1] > vl_img.shape[0] * (1 - top_down_ratio) and right_point[1] > vl_img.shape[0] * (1 - top_down_ratio):
            #     h_border 와의 교차점 조건    #
            # if top_point[0] >= vl_img.shape[1] * right_margin and bot_point[0] <= vl_img.shape[1] * left_margin:
              # extend = True
              ex_right_bot_vl.append((left_point[0], left_point[1], right_point[0], right_point[1]))

        # if extend:
        #   ex_right_vl.append((left_point[0], left_point[1], right_point[0], right_point[1]))
        # print('ex_left_vl[-1] :', ex_left_vl[-1])

      print()
      print('#          Extended vl list left / right  TB Condition        #')
      print('len(ex_left_top_vl) :', len(ex_left_top_vl))
      print('len(ex_left_bot_vl) :', len(ex_left_bot_vl))
      print('len(ex_right_top_vl) :', len(ex_right_top_vl))
      print('len(ex_right_bot_vl) :', len(ex_right_bot_vl))
      print()

      #       find min max Line     #
      max_y = 0
      right_bottest_vl = None
      min_y = vl_img.shape[0]
      right_toppest_vl = None

      for vl in ex_right_bot_vl:
        if vl[1] > max_y:
          right_bottest_vl = vl
          max_y = vl[1]

      for vl in ex_right_top_vl:
        if vl[1] < min_y:
          right_toppest_vl = vl
          min_y = vl[1]

      max_y = 0
      left_bottest_vl = None
      min_y = vl_img.shape[0]
      left_toppest_vl = None

      for vl in ex_left_bot_vl:
        if vl[3] > max_y:
          left_bottest_vl = vl
          max_y = vl[3]

      for vl in ex_left_top_vl:
        if vl[3] < min_y:
          left_toppest_vl = vl
          min_y = vl[3]

      #       Figure out min_y & max_y of thr_session       #
      # print('thr_session.shape :', thr_session.shape)

      #     바닥은 오히려 하면 안된다.    #
      thr_session_gray = cv2.cvtColor(thr_session, cv2.COLOR_RGB2GRAY)
      print('thr_session_gray.shape :', thr_session_gray.shape)

      top_white_cnt = 0
      bot_white_cnt = 0
      white_min_x = thr_session_gray.shape[1]
      white_max_x = 0
      white_min_y = thr_session.shape[0]
      white_max_y = 0
      temp_min_y = white_min_y
      temp_max_y = white_max_y
      top_parallel = False
      bot_parallel = False
      for i in range(thr_session_gray.shape[1]):

        #     Check Top Condition     #
        for j in range(int(thr_session_gray.shape[0] / 10)):
          # if thr_session_gray[j][i] == 255 and j < thr_min_y:
            # thr_min_y = j
            # thr_min_y_coord = (i, j)
          if thr_session_gray[j][i] == 255:
            top_white_cnt += 1
            if i > white_max_x:
              white_max_x = i
              max_x_coord = (i, j)
            if i < white_min_x:
              white_min_x = i
              min_x_coord = (i, j)
            if j < temp_min_y:
              temp_min_y = j
              temp_min_y_coord = (i, j)

        for j in range(int(thr_session_gray.shape[0] / 10), int(thr_session_gray.shape[0] * 9/10)):
          if thr_session_gray[j][i] == 255:
            if j > white_max_y:
              white_max_y = j
              max_y_coord = (i, j)
            if j < white_min_y:
              white_min_y = j
              min_y_coord = (i, j)

        #     Check Bot Condition   #
        for j in range(int(thr_session_gray.shape[0] * 9/10), thr_session_gray.shape[0]):
          # if thr_session_gray[j][i] == 255 and j < thr_min_y:
            # thr_min_y = j
            # thr_min_y_coord = (i, j)
          if thr_session_gray[j][i] == 255:
            bot_white_cnt += 1
            if j > temp_max_y:
              temp_max_y = j
              temp_max_y_coord = (i, j)
            # if i > white_max_x:
            #   white_max_x = i
            #   max_x_coord = (i, j)
            # if i < white_min_x:
            #   white_min_x = i
            #   min_x_coord = (i, j)

      if top_white_cnt / (thr_session_gray.shape[1] * int(thr_session_gray.shape[0] / 10)) > 0.5:
        top_parallel = True
        min_y_coord = temp_min_y_coord

      if bot_white_cnt / (thr_session_gray.shape[1] * int(thr_session_gray.shape[0] / 10)) > 0.5:
        bot_parallel = True
        max_y_coord = temp_max_y_coord

          # elif thr_session_gray[j][i] == 1 and j > thr_max_y:
          #   thr_max_y = j
          #   thr_max_y_coord = (j, i)
      # print('thr_min_y :', thr_min_y)
      # print('thr_min_y_coord :', thr_min_y_coord)
      try:
        max_x_coord, min_x_coord
        print('max_x_coord, min_x_coord :', max_x_coord, min_x_coord)
      except:
        pass

      #     None 이라면 대칭이동을 통해 만들어주어야한다.     #
      if len(ex_right_bot_vl) == 0:
        if right_toppest_vl is not None:
          x0, y0, x1, y1 = right_toppest_vl
          slope = (y1 - y0) / float(x1 - x0)
          angle = - math.degrees(math.atan(slope))

          #       white_max_y 를 - toppest_vl angle 로 지나는 직선      #
          print('max_y_coord :', max_y_coord)
          print('angle :', angle)
          endy = 100 * math.sin(math.radians(angle))
          endx = 100 * math.cos(math.radians(angle))
          temp_vl = Line(max_y_coord, (max_y_coord[0] + endx, max_y_coord[1] + endy))
          [left_point] = temp_vl.intersection(l2)
          [right_point] = temp_vl.intersection(l3)
          # print('left_point :', left_point)
          right_bottest_vl = (float(left_point[0]), float(left_point[1]), float(right_point[0]), float(right_point[1]))
          print('right_bottest_vl :', right_bottest_vl)

      elif len(ex_right_top_vl) == 0:
        if right_bottest_vl is not None:
          x0, y0, x1, y1 = right_bottest_vl
          slope = (y1 - y0) / float(x1 - x0)
          angle = - math.degrees(math.atan(slope))

          #       white_max_y 를 - toppest_vl angle 로 지나는 직선      #
          print('min_y_coord :', min_y_coord)
          print('angle :', angle)
          endy = 100 * math.sin(math.radians(angle))
          endx = 100 * math.cos(math.radians(angle))
          temp_vl = Line(min_y_coord, (min_y_coord[0] + endx, min_y_coord[1] + endy))
          [left_point] = temp_vl.intersection(l2)
          [right_point] = temp_vl.intersection(l3)
          # print('left_point :', left_point)
          right_toppest_vl = (float(left_point[0]), float(left_point[1]), float(right_point[0]), float(right_point[1]))
          print('right_toppest_vl :', right_toppest_vl)

      try:
        x1, y1, x0, y0 = right_toppest_vl
        cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)
        x1, y1, x0, y0 = right_bottest_vl
        cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)
      except:
        pass

      if len(ex_left_bot_vl) == 0:
        if left_toppest_vl is not None:
          x0, y0, x1, y1 = left_toppest_vl
          slope = (y1 - y0) / float(x1 - x0)
          angle = - math.degrees(math.atan(slope))

          #       white_max_y 를 - toppest_vl angle 로 지나는 직선      #
          print('max_y_coord :', max_y_coord)
          print('angle :', angle)
          endy = 100 * math.sin(math.radians(angle))
          endx = 100 * math.cos(math.radians(angle))
          temp_vl = Line(max_y_coord, (max_y_coord[0] + endx, max_y_coord[1] + endy))
          [left_point] = temp_vl.intersection(l2)
          [right_point] = temp_vl.intersection(l3)
          # print('left_point :', left_point)
          left_bottest_vl = (float(left_point[0]), float(left_point[1]), float(right_point[0]), float(right_point[1]))
          print('left_bottest_vl :', left_bottest_vl)

      elif len(ex_left_top_vl) == 0:
        if left_bottest_vl is not None:
          x0, y0, x1, y1 = left_bottest_vl
          slope = (y1 - y0) / float(x1 - x0)
          angle = - math.degrees(math.atan(slope))

          #       white_max_y 를 - toppest_vl angle 로 지나는 직선      #
          print('min_y_coord :', min_y_coord)
          print('angle :', angle)
          endy = 100 * math.sin(math.radians(angle))
          endx = 100 * math.cos(math.radians(angle))
          temp_vl = Line(min_y_coord, (min_y_coord[0] + endx, min_y_coord[1] + endy))
          [left_point] = temp_vl.intersection(l2)
          [right_point] = temp_vl.intersection(l3)
          # print('left_point :', left_point)
          left_toppest_vl = (float(left_point[0]), float(left_point[1]), float(right_point[0]), float(right_point[1]))
          print('left_toppest_vl :', left_toppest_vl)

      try:
        x1, y1, x0, y0 = left_bottest_vl
        cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (255, 255, 0), 3, cv2.LINE_AA)
        x1, y1, x0, y0 = left_toppest_vl
        cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (255, 255, 0), 3, cv2.LINE_AA)
      except:
        pass

      #       일단은 위아래 둘다 있는 조건 성립     #
      #   있는 쪽만 사용, 양방향 소실점 다 존재하면 소실선 개수가 더 많은 쪽 사용   #

      # left_condition = left_bottest_vl is not None and left_toppest_vl is not None
      # right_condition = right_bottest_vl is not None and right_toppest_vl is not None

      # if left_condition or right_condition:
        # print('left_bottest_vl', left_bottest_vl)

      #       original vl_list 는 remove 가 존재하기 때문에 copy_ version 사용한다.       #
      print('len(copy_left_vl_list) :', len(copy_left_vl_list))
      print('len(copy_right_vl_list) :', len(copy_right_vl_list))
      if len(copy_left_vl_list) >= len(copy_right_vl_list):
        #       toppest & bottest vl should be on the min_y & max_y of white mask       #
        # x1, y1, x0, y0 = left_bottest_vl
        # cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)
        x1, y1, x0, y0 = left_toppest_vl
        # cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)

        # if thr_min_y == 0:
        #   thr_min_x = thr_session_gray.shape[1]
        #   for i in range(thr_session_gray.shape[1]):
        #       if thr_session_gray[0][i] == 255 and i < thr_min_x:
        #         thr_min_x = i
        #         thr_min_y_coord = (i, 0)
        #         break
        #   print('thr_min_y_coord :', thr_min_y_coord)
        if top_parallel:
          top_line = Line((x1, y1), (x0, y0))
          print('top_line.slope :', float(top_line.slope))
          if abs(top_line.slope) < 0.05:
              max_x_coord = (0, 0)
          parallel_tl = top_line.parallel_line(min_x_coord)
          [tl] = parallel_tl.intersection(l2)
          [tr] = parallel_tl.intersection(l3)
          br, bl = left_bottest_vl[2:], left_bottest_vl[:2]
        else:
          tl, tr, br, bl = left_toppest_vl[:2], left_toppest_vl[2:], left_bottest_vl[2:], left_bottest_vl[:2]

      else:
        # x1, y1, x0, y0 = right_bottest_vl
        # cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)
        x1, y1, x0, y0 = right_toppest_vl
        # cv2.line(vl_img, (int(x1), int(y1)), (int(x0), int(y0)), (0, 0, 255), 3, cv2.LINE_AA)

        # if thr_min_y == 0:
        #   thr_max_x = 0
        #   for i in range(thr_session_gray.shape[1] - 1, 0, -1):
        #       print('i :', i)
        #       if thr_session_gray[0][i] == 255 and i > thr_max_x:
        #         thr_max_x = i
        #         thr_min_y_coord = (i, 0)
        #         break
        #   print('thr_min_y_coord :', thr_min_y_coord)
        if top_parallel:
          top_line = Line((x1, y1), (x0, y0))
          print('top_line.slope :', float(top_line.slope))
          if abs(top_line.slope) < 0.05:
              max_x_coord = (thr_session_gray.shape[1], 0)
          parallel_tl = top_line.parallel_line(max_x_coord)
          [tl] = parallel_tl.intersection(l2)
          [tr] = parallel_tl.intersection(l3)
          br, bl = right_bottest_vl[2:], right_bottest_vl[:2]
        else:
          tl, tr, br, bl = right_toppest_vl[:2], right_toppest_vl[2:], right_bottest_vl[2:], right_bottest_vl[:2]

      #       Refering      #
      refered = np.asarray(refer)[:vl_img.shape[0], :vl_img.shape[1]]

      # plt.imshow(refered)
      # plt.title('refered')
      # plt.show()

      #   tl, tr, br, bl    #
      #     refer를 위해 src_x => 0 으로 맞춰준다.    #
      src = np.array([
          [0, 0],
          [vl_img.shape[1], 0],
          [vl_img.shape[1], vl_img.shape[0]],
          [0, vl_img.shape[0]]], dtype = "float32")
      dst = np.array([list(tl),
                      list(tr),
                      list(br),
                      list(bl)], dtype = "float32")

      print()
      print("#          Warping Points        #")
      print('src :', src)
      print('dst :', dst)

      # compute the perspective transform matrix and then apply it
      matrix = cv2.getPerspectiveTransform(src, dst)
      refered = cv2.warpPerspective(refered, matrix, (refered.shape[1], refered.shape[0]))

      plt.subplot(121)
      plt.imshow(vl_img)
      plt.title('Session_vl_img')
      # plt.xlim(np.min(vps[:, [0]]), np.max(vps[:, [0]]))
      # plt.ylim(np.min(vps[:, [1]]), np.max(vps[:, [1]]))
      plt.subplot(122)
      plt.imshow(refered)
      plt.show()

      org_color_np2[:, int(min_x):int(max_x)] = refered

      #     i != 0 인 경우 break     #
      if reg_index == 0 and np.sum(sorted_predict_cen) != 0 and not iter:
        iter = True
        print('iter :', iter)

      else:
        break

  #         Refer의 검은 부분은 original image로 채운다.       #
  for i in range(org_color_np2.shape[1]):
    for j in range(org_color_np2.shape[0]):
      if sum(org_color_np2[j][i]) == 0:
        org_color_np2[j][i] = org_color_np[j][i]

  print('elapsed time :', time.time() - start)
  print()

  plt.figure(figsize=(15, 10))
  plt.subplot(131)
  plt.imshow(org_np3)

  plt.subplot(132)
  plt.imshow(org_color_np2)
  # plt.show()

  # print('np.max(org_np) :', np.max(org_np))
  org_np = org_np/ 255.
  refer = org_color_np2

  #     1.wall_mask * reference color image + (1 - wall_mask) * original_image)    #
  syn_ = org_np * (refer) + (1 - org_np) * org_color_np
  syn = org_np * syn_ + (1 - org_np) * org_color_np
  plt.subplot(133)
  plt.imshow(syn.astype(np.uint8))
  plt.show()

  if one_iter:
    break
  # break
```

    Output hidden; open in https://colab.research.google.com to view.

```

  # v_border = line((0, 0), (0, skl_copy.shape[0]))
  # v_border_lower = line((skl_copy.shape[1], 0), (skl_copy.shape[1], skl_copy.shape[0]))
  # for h_line in h_lines:

  #     #       Find Intersection between v_line, h_line      #
  #     vh_intersections_x = list()
  #     vh_intersections_y = list()

  #     hline = line(h_line[:2], h_line[2:])
  #     for reg_x, reg_y in zip(reg_xs, reg_ys):
  #       vline = line((reg_x[0], reg_y[0]), (reg_x[1], reg_y[1]))

  #       #     Extract only x - coordination     #
  #       vh_intersections_x.append(intersection2(vline, hline)[0])
  #       vh_intersections_y.append(intersection2(vline, hline)[1])

  #     # h_x = np.array([h_line[0], h_line[2]])
  #     # h_y = np.array([h_line[1], h_line[3]])
  #     # ex_h_x, ex_h_y = extended(h_x, h_y, 500)
  #     # ex_h_line = line((int(h_x[0]), int(h_y[0])), (int(h_x[1]), int(h_y[1])))
  #     r = intersection2(v_border, hline)
  #     r_lower = intersection2(v_border_lower, hline)

  #     vh_intersections_x.append(r[0])
  #     vh_intersections_y.append(r[1])
  #     vh_intersections_x.append(r_lower[0])
  #     vh_intersections_y.append(r_lower[1])

  #     sorted_vh_inter_x = sorted(vh_intersections_x)
  #     # print('vh_intersections_x :', vh_intersections_x)
  #     # print('sorted_vh_inter_x :', sorted_vh_inter_x)

  #     center_h_x = (h_line[0] + h_line[2]) / 2
  #     # print('center_h_x :', center_h_x)

  #     #       hline 상의 교차점을 찾아내서 범위 내에서 연결한다.      #
  #     for i in range(1, len(sorted_vh_inter_x)):

  #       if sorted_vh_inter_x[i - 1] <= center_h_x <= sorted_vh_inter_x[i]:
  #         lx, ly = sorted_vh_inter_x[i - 1], vh_intersections_y[vh_intersections_x.index(sorted_vh_inter_x[i - 1])]
  #         rx, ry = sorted_vh_inter_x[i], vh_intersections_y[vh_intersections_x.index(sorted_vh_inter_x[i])]
  #         # print('lx, ly, rx, ry :', lx, ly, rx, ry)
  #         #       이곳의 lx, ly, rx, ry 는 close_p 다. 좌우로 나뉘어진 lx, ly가 아니다.     #

  #           #       정제된 교차점을 만드는 hline만 사용해야한다.        #
  #         for inters in all_closest_inters:
  #           for inter in inters:
  #             if lx in inter or rx in inter:
  #               h_intersections.append((lx, ly, rx, ry))
  #               # cv2.line(black_plane, (int(lx), int(ly)), (int(rx), int(ry)), (0, 0, 255), 1, cv2.LINE_AA)

  #       vline 기준으로 구획해야한다. -> 가장 왼쪽 / 오른쪽 vline + 구역 먼저 찾아 작업하기     #
  #       vline 별로 구획 나누기      #
  # h_intersections = list(set(h_intersections))
  # print('np.array(all_closest_inters).shape :', np.array(all_closest_inters).shape)
  # all_closest_inters = np.array(all_closest_inters)
  # print('len(all_closest_inters) :', len(all_closest_inters))

  # center_xs = list()
  # for inters in all_closest_inters:
  #   #     all_closest_inter가 정렬되지 않았다면 inters type = list()      #
  #   sum_x = 0
  #   for inter in inters:
  #     sum_x += inter[0] + inter[2]
  #   center_xs.append(sum_x / len(inters) * 2)
  # sorted_center_xs = sorted(center_xs)

  # sorted_index = list()
  # for center_x in sorted_center_xs:
  #   # print('center_x :', center_x)
  #   sorted_index.append(center_xs.index(center_x))
  # # print('sorted_index :', sorted_index)
  # #       sorted_center_xs 순서로 all_closest_inter를 정렬한다.     #
  # #       index list를 추출해서 for 문으로 all_closest_inter의 inters를 추출해
  # #       sorted_all ... 에 append 시킨다.      #
  # sorted_all_closest_inters = list()
  # sorted_all_centroid_inters = list()
  # for s_index in sorted_index:
  #   sorted_all_closest_inters.append(all_closest_inters[s_index])
  #   sorted_all_centroid_inters.append(all_centroid_inters[s_index])
  # all_closest_inters = sorted_all_closest_inters
  # all_centroid_inters = sorted_all_centroid_inters

```

```
#                                   Refering                                 #
      #     Find Top left / right & Bottom left / right       #
      #     ->  tl, tr, bl, br     #
      four_inters = np.array(four_inters)
      [tl, tr, br, bl] = order_points(four_inters)
      # print(four_inters.shape)
      top_length = math.hypot(tl[0] - tr[0], tl[1] - tr[1])
      bottom_length = math.hypot(bl[0] - br[0], bl[1] - br[1])
      max_hlength = max(top_length, bottom_length)

      #     1. Compare left / right height     #
      l_height = bl[1] - tl[1]
      r_height = br[1] - tr[1]

      #     2. Extend Shorter Height      #
      if l_height <= r_height:
        shorter_points = [bl, tl]
        longer_points = [br, tr]
      else:
        shorter_points = [br, tr]
        longer_points = [bl, tl]

      ex_shorter_points = np.zeros_like(shorter_points)
      ex_longer_points = np.zeros_like(longer_points)

      print('shorter_points :', shorter_points)
      x = (shorter_points[0][0], shorter_points[1][0])
      y = (shorter_points[0][1], shorter_points[1][1])
      print('shorter x, y :', x, y)

      y_ext = np.array([org_np.shape[0], 0])
      p = np.polyfit(y, x , deg=1)
      x_ext = np.poly1d(p)(y_ext)

      ex_shorter_points[0] = [x_ext[0], y_ext[0]]
      ex_shorter_points[1] = [x_ext[1], y_ext[1]]

      #     Find intersection between (longer + shorter points)'s parallel line and longer line   #
      long_short_lower = Line(longer_points[0], shorter_points[0])
      long_short = Line(longer_points[1], shorter_points[1])
      p_line_lower = long_short_lower.parallel_line(ex_shorter_points[0])
      p_line = long_short.parallel_line(ex_shorter_points[1])
      longer_line = Line(longer_points[0], longer_points[1])

      # print('ex_shorter_points[1] in p_line ? :', ex_shorter_points[1] in p_line)
      # print('p_line_lower.intersection(longer_line) :', p_line_lower.intersection(longer_line)[0][0])
      ex_longer_points[0] = [p_line_lower.intersection(longer_line)[0][0], p_line_lower.intersection(longer_line)[0][1]]
      ex_longer_points[1] = [p_line.intersection(longer_line)[0][0], p_line.intersection(longer_line)[0][1]]

      if l_height <= r_height:
        ex_bl, ex_tl = ex_shorter_points
        ex_br, ex_tr = ex_longer_points
      else:
        ex_br, ex_tr = ex_shorter_points
        ex_bl, ex_tl = ex_longer_points

      for ex_p in [ex_bl, ex_br, ex_tl, ex_tr]:
        print(ex_p)
        # cv2.circle(black_plane2, (int(ex_p[0]), int(ex_p[1])), 10, (255, 0, 255), -1)

      min_x, max_x = min(ex_bl[0], ex_tl[0]), max(ex_br[0], ex_tr[0])
      min_y, max_y = min(ex_tl[1], ex_tr[1]), max(ex_bl[1], ex_br[1])
      src_x = max((max_x - min_x), max_hlength) * math.sqrt(2)
      print('src_x :',  src_x)

      refered = np.asarray(refer)[:int(max_y - min_y), :int(src_x)]

      # plt.imshow(refered)
      # plt.title('refered')
      # plt.show()

      #   tl, tr, br, bl    #
      #     refer를 위해 src_x => 0 으로 맞춰준다.    #
      src = np.array([
          [0, 0],
          [src_x, 0],
          [src_x, max_y - min_y],
          [0, max_y - min_y]], dtype = "float32")
      dst = np.array([
                      [ex_tl[0] - min_x, ex_tl[1] - min_y],
                      [ex_tr[0] - min_x, ex_tr[1] - min_y],
                      [ex_br[0] - min_x, ex_br[1] - min_y],
                      [ex_bl[0] - min_x, ex_bl[1] - min_y]], dtype = "float32")

      print('src :', src)
      print('dst :', dst)

      # compute the perspective transform matrix and then apply it
      matrix = cv2.getPerspectiveTransform(src, dst)
      refered = cv2.warpPerspective(refered, matrix, (refered.shape[1], refered.shape[0]))

      # print('refered.min() :', refered.min())
      # print(refered[[0], [0]])
      # print('refered.shape :', refered.shape)
      # plt.imshow(refered)
      # plt.title('warfed')
      # plt.xlim(min_x, max_x)
      # plt.xlim(0, max_x - min_x)
      # plt.ylim(max_y, min_y)
      # plt.xlim(-1000, 1000)
      # plt.ylim(1000, -1000)
      # plt.show()

      refer_crop = refered[int(abs(min_y)): int(abs(min_y)) + org_np.shape[0], :]
      # print('refer_crop.shape :', refer_crop.shape)

      # plt.imshow(refer_crop)
      # plt.title('cropped refer')
      # plt.show()

      #     Crop Section From Cropped Refer     #
      #         src_x => 0으로 맞춰줬던 것을 되돌린다.      #
      print('int(min_x) :', int(min_x))
      print('int(min_y) :', int(min_y))

      for i in range(refer_crop.shape[1]):
        for j in range(refer_crop.shape[0]):
          if sum(refer_crop[j][i]) != 0:
            if i + int(min_x) < black_plane2.shape[1]:
              black_plane2[j][i + int(min_x)] = refer_crop[j][i]
```

Layout Estimation without AI kmeans.ipynb


# Assignment 1
**CS283 Computer Vision, Harvard University, Fall 2019**

**Due Wednesday, Sep. 18, at 5:00pm**

Name: *(fill name here)*

---

The intended outcomes of this assignment are for you to be familiar with Python as a tool for manipulating images and to deepen your understanding of model-fitting and the two-dimensional projective plane. There is a **Hints and Information** section at the end of this document that is likely to help you a lot. This will be a common feature in future assignments.

Assignments in this course will use Python 3. The skeleton code we provide will only be tested on JupyterHub (via the course webpage) using a Python 3 notebook and certain pre-installed packages (inlcuding numpy, scipy, and opencv). Likewise, we expect any code that you submit to execute in the course's JupyterHub environment.

The input image files that are required to complete this assignment can be found in the <tt>pset1.zip</tt> archive that accompanies this notebook on the course website. Be sure to upload this ZIP archive tp JupterHub before running the notebook's first cell (which unpacks the archive into a <tt>./pset1</tt> folder). This will be another common feature in future assignments.

You will submit your work by editing this notebook and uploading it to the submission system on the course website. It is important that your submission executes and, as much as possible, maintains the notebook's original formatting. Submissions that do not execute or that deviate substantially in terms of formatting risk not being graded. Please read the **CS283 Assignment Submission Guidelines** for more detailed instructions.

Remember that the online submission system closes *exactly* at the stated deadline. If you find yourself in the uncomfortable position of working very close to the deadline, we advise that you upload early and often.

```python
# Extract required pset files

import os             # for file handling
import zipfile as zf  # For unpacking pset ZIP archives

# Extract required pset files
assert os.path.exists("./pset1.zip"), 'Upload the pset ZIP archive and then re-run the cell.'
files = zf.ZipFile("./pset1.zip", 'r')
files.extractall('.')
files.close()
```

```python
# Import other required libraries here
import cv2           # OpenCV
import numpy as np   # numpy

# Use this line to make matplotlib plot inline (only need to call it once when importing matplotlib)
%matplotlib inline

import matplotlib.pyplot as plt
# Modify this line to adjust the displayed plot size. You can also call
# it with different parameters before specific plots.
plt.rcParams['figure.figsize'] = [10, 10]
```

## Question 1 (10 points)

In the <tt>./pset1/data</tt> folder there is a color image called <tt>baboon.tif</tt>.  This image appears frequently in the image processing literature.

**a.** Write a sequence of Python commands that loads the image using OpenCV and reports its height and width in pixels.

*Hints: See OpenCV documentation for <tt>cv2.imread()</tt> and the <tt>shape</tt> attribute of numpy arrays. Use the built-in Python function <tt>print()</tt> for display text (and note that, unlike Python 2, Python 3 requires parentheses for this function). The <tt>format()</tt> method for string formatting may also be useful.*

```python
# TO DO: your code here
```

**b.**  Write a sequence of Python commands that converts this image to a grayscale image and displays it using the matplotlib package. In addition, display three other grayscale images that correspond to each of the three separate RGB color components. To do this you will need to understand the way OpenCV represents RGB images and how to decompose them.  Use the <tt>subplot</tt> command of the <tt>matplotlib.pyplot</tt> package to display the four results in a single row.

*Hints: See OpenCV documentation for <tt>cv2.cvtColor()</tt> and matplotlib documentation for <tt>matplotlib.pyplot.imshow()</tt>. To pretty up your plots, see the commands <tt>matplotlib.pyplot.axis()</tt> and <tt>matplotlib.pyplot.title()</tt>. Also note that by default, OpenCV loads images in BGR order and not RGB.*

```python
# TO DO: your code here
```

**c.** You can use the <tt>cv2.imwrite()</tt> command to write an image to a file in various formats with varying levels of compression. Write code that creates a new JPEG version of the original color image with a quality setting of 95 to the file <tt>baboon_compressed.jpg</tt>, and then reads and displays this new image next to the original image in a single row using <tt>subplot</tt>. Can you tell the difference between the compressed image and the original?

```python
# TO DO: your code here
```

### Answer:
*TO DO: Write your answer here.*

**d.** The compression ratio is the ratio between the size of the original file and the size of the compressed file (in bytes). The following cell will query the file sizes and report them. Based on the cell's output (you may need to modify the cell to point to the correct location of your compressed image file), what is the compression ratio for this quality setting of 0.95?

```python
filesize = os.path.getsize('./pset1/data/baboon.tif')
filesize_compressed = os.path.getsize('baboon_compressed.jpg')
print("The original file is {} bytes and the compressed one is {} bytes".format(filesize, filesize_compressed))
```

### Answer:
*TO DO: Write your answer here.*

**e.** Write code in the following cell that allows you to experiment with the JPEG quality settings, allowing you to visually compare the original and compressed images and also see the compression ratio for any quality setting you desire. Using this code, determine the smallest quality value for which the compressed image is indistinguishable from the original. What is this quality value and what is the associated compression ratio?

```python
# TO DO: your code here
```

### Answer:

*TO DO: Write your answer here.*

## Question 2 (10 points)

The <tt>./pset1/data</tt> folder contains another popular image, <tt>cameraman.tif</tt>.

**a.** Write code that loads the image, converts it to grayscale, uses a random number generator to select exactly 10% of the pixels, and then replaces their gray-level values with independent, random integers uniformly distributed between 0 and 255. Display the result. Next, use subplot to display two more results beside this one, where the percentage of randomly replaced pixels in the original image is 25% and 50%, respectively.

*Hints: The functions <tt>numpy.random.choice()</tt> and <tt>numpy.random.randint()</tt>, as well as the <tt>flat</tt> attribute of numpy arrays may be useful.*

```python
# TO DO: your code here
```

## Question 3 (30 points)

There is a set of points $\tilde{\mathbf{x}}_i=(x_i,y_i)$ in the image plane and we want to find the best line passing through them.

The next cell defines two functions that calculate the line in two different ways using two different measures of quality. In both functions, the input is two $N\times 1$ numpy arrays with inhomogeneous coordinates, $\{x_i\}$ and $\{y_i\}$, of $N$ points. The output is the coordinates of a line $\boldsymbol\ell=(a,b,c)$.

1. The function <tt>fit_line_vertical()</tt> solves the problem by finding $(a, c)$ (setting $b = 1$) that most closely satisfy the
equations $y_i=-ax_i-c$, in a least-squares sense. It minimizes the sum of squares of vertical distances between the points and the line by encoding the constraints in matrix form
\begin{equation*}
\left.
\begin{array}{c}
-a x_1 - c = y_1 \\
-a x_2 - c = y_2 \\
\vdots \\
-a x_N - c = y_N
\end{array} \right\} \Rightarrow \underbrace{\begin{bmatrix}
-x_1 & -1 \\
-x_2 & -1 \\
\vdots & \vdots \\
-x_N & -1
\end{bmatrix}}_{\bf A} \cdot \underbrace{\begin{bmatrix}
a \\
c
\end{bmatrix}}_{\bf v} = \underbrace{\begin{bmatrix}
y_1 \\
y_2 \\
\vdots \\
y_N
\end{bmatrix}}_{\bf b}
\end{equation*}
and solving
\begin{equation*}
\text{arg}\min_\mathbf{v}\|\mathbf{A}\mathbf{v}-\mathbf{b}\|^2.
\end{equation*}

2. The function <tt>fit_line_homogeneous()</tt> solves the problem by finding $\boldsymbol\ell=(a,\ b,\ c)$ that most closely satisfies the equations $ax_i+by_i+c=0$, in a least-squares sense. That is, it minimizes the sum of homogeneous algebraic errors, $\sum\left(\boldsymbol\ell^\top \mathbf{x}_i\right)^2$ with $\mathbf{x}_i\triangleq(x_i,y_i,1)$. It does this by encoding the constraints in matrix form
\begin{equation*}
\underbrace{\begin{bmatrix}
x_1 & y_1 & 1 \\
x_2 & y_2 & 1 \\
\vdots & \vdots \\
x_N & y_N & 1 \\
\end{bmatrix}}_{\bf A} \cdot \underbrace{\begin{bmatrix}
a \\
b \\
c
\end{bmatrix}}_{\boldsymbol\ell} = \underbrace{\begin{bmatrix}
0 \\
0 \\
\vdots \\
0
\end{bmatrix}}_{\bf 0}
\end{equation*}
and solving
\begin{equation*}
\text{arg}\min_\boldsymbol\ell\|\mathbf{A}\boldsymbol\ell\|^2\quad \text{ such that } \|\boldsymbol\ell\|=1.
\end{equation*}

```python
def fit_line_vertical(x, y):
    # Method A: linear regression (vertical distance)

    # Construct the Nx2 "A matrix"
    A = -np.concatenate([x[:, np.newaxis], np.ones((x.size, 1))], axis=1)

    # Least squares solution
    l = np.linalg.lstsq(A, y, rcond=None)[0]

    # Format line as (a,b,c)
    return l[0], 1.0, l[1]

def fit_line_homogeneous(x, y):
    # Method B: Naive homogeneous

    # Construct the "A matrix"
    A = np.concatenate([x[:, np.newaxis], y[:, np.newaxis], np.ones((x.size, 1))], axis=1)

    # SVD
    _, _, V = np.linalg.svd(A)

    # Extract last column of V matrix (note that np.linalg.svd() returns a transposed version of V)
    l = V[2, :]

    return l
```

**a.** Write code that loads the image <tt>dots.tif</tt> (from the <tt>./pset1/data</tt> folder, as usual) and: i) detects the red, green, and  blue points and obtain their $(x,y)$ image coordinates; ii) calls the functions <tt>fit_line_vertical()</tt> and <tt>fit_line_homogeneous()</tt> to fit two different lines to each set of points; and iii) plots these lines (two lines for each color) superimposed on the image.

*Hints: The functions <tt>numpy.nonzero()</tt>, <tt>numpy.concatenate()</tt>, and <tt>numpy.bitwise_and()</tt> may be useful. You may also want to use <tt>matplotlib.pyplot.xlim()</tt> and <tt>matplotlib.pyplot.ylim()</tt> for plotting. Note that flipping the limits in <tt>matplotlib.pyplot.ylim()</tt> flips the y axis, which may be desirable for images.*

```python
# TO DO: your code here
```

**b.** Neither of the two solutions from part (a) minimize the sum of squares of perpendicular distances between the points and the line, which for a single point is

\begin{equation*}
 \frac{|a x_i + b y_i + c|}{\sqrt{a^2 + b^2}}.
\end{equation*}

Write code for a new function <tt>fit_line_perpendicular()</tt> that has the same input and output as the previous functions but that provides a solution that minimizes the sum of squares of perpendicular distances.

Also, write code (adapted from (a)) that calls the function <tt>fit_line_perpendicular()</tt> and plots the fitted lines (one line for each color) superimposed on the image <tt>dots.tif</tt>.

```python
# TO DO: your code here
```

**c.** These lines do not intersect at a point, but we can find a point that comes "closest" to a three-way intersection by finding the point that minimizes the sum of squared perpendicular distances from the point to the three lines found in part (b). For this, you should create a procedure that is analogous to the method you implemented in part (b): formulate and solve an appropriate linear system of the form ${\bf A}{\bf x}={\bf 0}$ with constraints on the solution ${\bf x}$. Using words and equations, describe your construction of such a constrained linear system and explain why its solution is the minimum of the sum of squared perpendicular distances. Write code that implements your constructioon, computes the "best point", and displays this point superimposed on the image and with the three lines found in (b).

### Answer:

*TO DO: Write your answer here.*

```python
# TO DO: your code here
```

## Question 4 (20 points)

In the presence of outliers, we require more robust techniques for model fitting. RANSAC is one such method that is both useful and conceptually simple.

**a.** Write code that loads the image <tt>./pset1/data/dots_outliers.tif</tt>, detects the coordinates of the white pixels in the image and then calls your function <tt>fit_line_perpendicular()</tt> to fit a line to these inlying points. Also, write code that displays your result superimposed on the image. Note that, in this case, the image is black-and-white, with the points being white (you may need to zoom-in to see the points clearly).

```python
# TO DO: your code here
```

**b.** Write code that uses RANSAC to improve the fit and draw a better line, by first identifying a subset of inlying, nearly-collinear points, and then applying <tt>fit_line_perpendicular()</tt> to the inliers. We suggest that the number of iterations be 100 and the threshold used to determine the inlying set for each iteration be a distance of 20 pixels (as defined in the code below). Display your lines from both (a) and (b) together, superimposed on the original image.

```python
num_iter = 100         # number of RANSAC iterations
inlier_threshold = 20  # threshold for inlier set of each line

# TO DO: your code here
```

## Question 5 (10 points)

Following Hartley & Zisserman, the notation in this question is such that ${\bf x}$ and $\tilde{\bf x}$ indicate homogeneous and inhomogeneous vectors, respectively.

Consider a right triangle with vertices $\tilde{\bf x}_1=(0,\ 0)$, $\tilde{\bf x}_2=(m,\  0)$ and $\tilde{\bf x}_3=(0,\ m)$, and suppose this triangle is warped by an affine transformation such that
\begin{equation}\label{eq:affine}
\left(\begin{array}{c} x' \\ y' \\ 1\end{array}\right)=
\left[\begin{array}{ccc} a_{11} & a_{12} & t_x \\ a_{21} & a_{22} & t_y \\ 0 & 0 & 1\end{array}\right]
\left(\begin{array}{c} x \\ y \\ 1\end{array}\right),\nonumber
\end{equation}
or ${\bf x}'={\bf A}{\bf x}$.  Derive an expression for the area of the warped triangle defined by $\tilde{\bf x}'_1$, $\tilde{\bf x}'_2$ and $\tilde{\bf x}'_3$.  Use this expression to prove that if two right triangles (with $m=m_1$ and
$m=m_2$) are warped by the same affine transformation, the ratio of their areas is preserved.

### Answer:

*TO DO: Write your answer here.*

## Hints and Information

- For help with Python itself, use the Python 3.7 [documentation](https://docs.python.org/3.7/). Throughout the course we will be using popular libraries such as [OpenCV](https://opencv.org/), [numpy, scipy](https://docs.scipy.org/doc/numpy/reference/) and [matplotlib](https://matplotlib.org/), all of which have documentation available online. For refreshing your numpy knowledge, we highly recommend going through a basic numpy tutorial [here](https://docs.scipy.org/doc/numpy/user/quickstart.html). OpenCV is a package for image processing and computer vision which we will be using heavily throughout the course. It stores images and other data as numpy arrays, and therefore we will be working with numpy a lot.

-  A linear least-squares problem is one in which we want to determine the vector ${\bf x}$ that best satisfies a set of inconsistent linear constraints ${\bf A}{\bf x}={\bf b}$ in the sense of minimum square error. That is, we wish to solve:
\begin{equation}
{\bf x}^*=\textrm{arg}\min_{\bf x} ||{\bf A}{\bf x}-{\bf b}||^2 =\textrm{arg}\min_{\bf x} ({\bf A}{\bf x}-{\bf b})^\top({\bf A}{\bf x}-{\bf b}).
\end{equation}
The solution is found in closed-form by differentiating the objective function with respect to ${\bf x}$ and equating the result to zero. This yields
\begin{equation}
{\bf x}^*= ({\bf A}^\top{\bf A})^{-1}{\bf A}^\top{\bf b}.
\end{equation}
In the above expression, the inverse is used only for notational purpose. The explicit calculation of $({\bf
A}^\top{\bf A})^{-1}$ is *very bad practice*, because finding the inverse is both very
expensive and [numerically unstable](http://blogs.mathworks.com/loren/2007/05/16/purpose-of-inv/). Instead, it is better to use a method such as QR factorization to solve the (consistent) linear system $({\bf A}^\top{\bf A}){\bf x}={\bf A}^\top{\bf b}$, sometimes called the *normal equation* of the original linear system. In order to solve linear least squares problems in such a way we can use <tt>numpy.linalg.lstsq()</tt> or <tt>numpy.linalg.solve()</tt> (similar to doing <tt>x = A \ b</tt> in MATLAB).

- Based on the "CS283 Assignment Submission Guidelines", your submission should have the following file structure:

<tt>lastname_firstname_psetx.zip</tt>

  <tt>+-- lastname_firstname_psetx.ipynb.............</tt>Jupyter notebook with all code, $\LaTeX$ answers, and output.

  <tt>+-- lastname_firstname_psetx.html..............</tt>HTML version of notebook with all code, $\LaTeX$ answers, and output.

  <tt>+-- src/.......................................</tt>External python functions required by the notebook (none for pset1).

  <tt>+-- img/.......................................</tt>Images embedded in notebook (none expected for pset1).

  <tt>+-- data/......................................</tt>Image and other data files for the notebook, such as <tt>cameraman.tif</tt> and <tt>baboon.tif</tt>.


```python
! pip install qeds
import pandas as pd
%matplotlib inline
import qeds
qeds.themes.mpl_style();
```

    Requirement already satisfied: qeds in c:\users\asus\anaconda3\lib\site-packages (0.6.2)
    Requirement already satisfied: statsmodels in c:\users\asus\anaconda3\lib\site-packages (from qeds) (0.10.1)
    Requirement already satisfied: openpyxl in c:\users\asus\anaconda3\lib\site-packages (from qeds) (3.0.0)
    Requirement already satisfied: scipy in c:\users\asus\anaconda3\lib\site-packages (from qeds) (1.3.1)
    Requirement already satisfied: pyarrow in c:\users\asus\anaconda3\lib\site-packages (from qeds) (0.16.0)
    Requirement already satisfied: numpy in c:\users\asus\anaconda3\lib\site-packages (from qeds) (1.16.5)
    Requirement already satisfied: matplotlib in c:\users\asus\anaconda3\lib\site-packages (from qeds) (3.1.1)
    Requirement already satisfied: quantecon in c:\users\asus\anaconda3\lib\site-packages (from qeds) (0.4.6)
    Requirement already satisfied: seaborn in c:\users\asus\anaconda3\lib\site-packages (from qeds) (0.9.0)
    Requirement already satisfied: pandas in c:\users\asus\anaconda3\lib\site-packages (from qeds) (0.25.1)
    Requirement already satisfied: requests in c:\users\asus\anaconda3\lib\site-packages (from qeds) (2.22.0)
    Requirement already satisfied: plotly in c:\users\asus\anaconda3\lib\site-packages (from qeds) (4.5.4)
    Requirement already satisfied: pandas-datareader in c:\users\asus\anaconda3\lib\site-packages (from qeds) (0.8.1)
    Requirement already satisfied: scikit-learn in c:\users\asus\anaconda3\lib\site-packages (from qeds) (0.21.3)
    Requirement already satisfied: quandl in c:\users\asus\anaconda3\lib\site-packages (from qeds) (3.5.0)
    Requirement already satisfied: patsy>=0.4.0 in c:\users\asus\anaconda3\lib\site-packages (from statsmodels->qeds) (0.5.1)
    Requirement already satisfied: et-xmlfile in c:\users\asus\anaconda3\lib\site-packages (from openpyxl->qeds) (1.0.1)
    Requirement already satisfied: jdcal in c:\users\asus\anaconda3\lib\site-packages (from openpyxl->qeds) (1.4.1)
    Requirement already satisfied: six>=1.0.0 in c:\users\asus\anaconda3\lib\site-packages (from pyarrow->qeds) (1.12.0)
    Requirement already satisfied: cycler>=0.10 in c:\users\asus\anaconda3\lib\site-packages (from matplotlib->qeds) (0.10.0)
    Requirement already satisfied: kiwisolver>=1.0.1 in c:\users\asus\anaconda3\lib\site-packages (from matplotlib->qeds) (1.1.0)
    Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\users\asus\anaconda3\lib\site-packages (from matplotlib->qeds) (2.4.2)
    Requirement already satisfied: python-dateutil>=2.1 in c:\users\asus\anaconda3\lib\site-packages (from matplotlib->qeds) (2.8.0)
    Requirement already satisfied: sympy in c:\users\asus\anaconda3\lib\site-packages (from quantecon->qeds) (1.4)
    Requirement already satisfied: numba>=0.38 in c:\users\asus\anaconda3\lib\site-packages (from quantecon->qeds) (0.45.1)
    Requirement already satisfied: pytz>=2017.2 in c:\users\asus\anaconda3\lib\site-packages (from pandas->qeds) (2019.3)
    Requirement already satisfied: certifi>=2017.4.17 in c:\users\asus\anaconda3\lib\site-packages (from requests->qeds) (2019.9.11)
    Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\users\asus\anaconda3\lib\site-packages (from requests->qeds) (3.0.4)
    Requirement already satisfied: idna<2.9,>=2.5 in c:\users\asus\anaconda3\lib\site-packages (from requests->qeds) (2.8)
    Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\users\asus\anaconda3\lib\site-packages (from requests->qeds) (1.24.2)
    Requirement already satisfied: retrying>=1.3.3 in c:\users\asus\anaconda3\lib\site-packages (from plotly->qeds) (1.3.3)
    Requirement already satisfied: lxml in c:\users\asus\anaconda3\lib\site-packages (from pandas-datareader->qeds) (4.4.1)
    Requirement already satisfied: joblib>=0.11 in c:\users\asus\anaconda3\lib\site-packages (from scikit-learn->qeds) (0.13.2)
    Requirement already satisfied: inflection>=0.3.1 in c:\users\asus\anaconda3\lib\site-packages (from quandl->qeds) (0.3.1)
    Requirement already satisfied: more-itertools in c:\users\asus\anaconda3\lib\site-packages (from quandl->qeds) (7.2.0)
    Requirement already satisfied: setuptools in c:\users\asus\anaconda3\lib\site-packages (from kiwisolver>=1.0.1->matplotlib->qeds) (41.4.0)
    Requirement already satisfied: mpmath>=0.19 in c:\users\asus\anaconda3\lib\site-packages (from sympy->quantecon->qeds) (1.1.0)
    Requirement already satisfied: llvmlite>=0.29.0dev0 in c:\users\asus\anaconda3\lib\site-packages (from numba>=0.38->quantecon->qeds) (0.29.0)

# Introduction - Exercises

**For the purpose of these exercises, we create:**

- **a Series which contains the US unemployment rate every other year starting in 1995**

- **a DataFrame that contains the unemployment rate every other year by region of the US starting in 1995**

```python
#Series which contains the US unemployment rate every other year starting in 1995
values = [5.6, 5.3, 4.3, 4.2, 5.8, 5.3, 4.6, 7.8, 9.1, 8., 5.7]
years = list(range(1995, 2017, 2))

unemp = pd.Series(data=values, index=years, name="Unemployment")
unemp
```

    1995    5.6
    1997    5.3
    1999    4.3
    2001    4.2
    2003    5.8
    2005    5.3
    2007    4.6
    2009    7.8
    2011    9.1
    2013    8.0
    2015    5.7
    Name: Unemployment, dtype: float64

```python
#DataFrame that contains the unemployment rate every other year by region of the US starting in 1995
data = {
    "NorthEast": [5.9,  5.6,  4.4,  3.8,  5.8,  4.9,  4.3,  7.1,  8.3,  7.9,  5.7],
    "MidWest": [4.5,  4.3,  3.6,  4. ,  5.7,  5.7,  4.9,  8.1,  8.7,  7.4,  5.1],
    "South": [5.3,  5.2,  4.2,  4. ,  5.7,  5.2,  4.3,  7.6,  9.1,  7.4,  5.5],
    "West": [6.6, 6., 5.2, 4.6, 6.5, 5.5, 4.5, 8.6, 10.7, 8.5, 6.1],
    "National": [5.6, 5.3, 4.3, 4.2, 5.8, 5.3, 4.6, 7.8, 9.1, 8., 5.7]
}

unemp_region = pd.DataFrame(data, index=years)
unemp_region
```

      NorthEast
      MidWest
      South
      West
      National

      1995
      5.9
      4.5
      5.3
      6.6
      5.6

      1997
      5.6
      4.3
      5.2
      6.0
      5.3

      1999
      4.4
      3.6
      4.2
      5.2
      4.3

      2001
      3.8
      4.0
      4.0
      4.6
      4.2

      2003
      5.8
      5.7
      5.7
      6.5
      5.8

      2005
      4.9
      5.7
      5.2
      5.5
      5.3

      2007
      4.3
      4.9
      4.3
      4.5
      4.6

      2009
      7.1
      8.1
      7.6
      8.6
      7.8

      2011
      8.3
      8.7
      9.1
      10.7
      9.1

      2013
      7.9
      7.4
      7.4
      8.5
      8.0

      2015
      5.7
      5.1
      5.5
      6.1
      5.7

## Exercise 1

1. **Display only the first 2 elements of the Series using the `.head` method.**

```python
# To display only the two first lines, we use the .head function with "2" as an argument.
unemp.head(2)
```

    1995    5.6
    1997    5.3
    Name: Unemployment, dtype: float64

2. **Using the `plot` method, make a bar plot.**

```python
# The following makes a bar for each state and for each year.
unemp.plot.bar()
# Note that, by default, the index is taken as the x variable here.
```

3. **Use `.loc` to select the lowest/highest unemployment rate shown in the Series.**

```python
#lowest temperature
unemp.loc[unemp.idxmin()]
```

    4.2

```python
#highest temperature
unemp.loc[unemp.idxmax()]
```

    9.1

4. **Run the code `unemp.dtype` below. What does it give you? Where do you think it comes from?**

```python
unemp.dtype
#we get that the dtype is float64
#this tells us the type of data stored in the series
```

    dtype('float64')

## Exercise 2

**Now we create we create a DataFrame that contains the unemployment rate every other year by region of the US starting in 1995.**

1. **Use introspection (or google-fu) to find a way to obtain a list with all of the column names in `unemp_region`.**

```python
list(unemp_region)

#to get the index, we can do:
#list(unemp_region.index)
```

    ['NorthEast', 'MidWest', 'South', 'West', 'National']

2. **Using the `plot` method, make a bar plot. What does it look like now?**

```python
# Note that, by default, the index is taken as the x variable here.
unemp_region.plot.bar()

#We create a column entitled UnweightedMean to play around with other plotting characteristics
unemp_region["UnweightedMean"] = (unemp_region["NorthEast"] +
                                  unemp_region["MidWest"] +
                                  unemp_region["South"] +
                                  unemp_region["West"])/4

# The following makes a bar for each year, and this bar is the unweighted mean
unemp_region.plot.bar(y='UnweightedMean')
# The following makes a bar for each year, but plots the value "Nation"
unemp_region.plot.bar(y='National')
```

3. **Use `.loc` to select the the unemployment data for the `NorthEast` and `West` for the years 1995, 2005, 2011, and 2015.**

```python
# .loc "accesses a group of rows and columns by label(s)..." Arguments are labels
unemp_region.loc[[1995, 2005, 2011, 2015], ["NorthEast", "West"]]
```

      NorthEast
      West

      1995
      5.9
      6.6

      2005
      4.9
      5.5

      2011
      8.3
      10.7

      2015
      5.7
      6.1

4. **Run the code `unemp_region.dtypes` below. What does it give you? How does this compare with `unemp.dtype`?**

```python
unemp_region.dtypes
```

    NorthEast         float64
    MidWest           float64
    South             float64
    West              float64
    National          float64
    UnweightedMean    float64
    dtype: object


## Initial setup

```python
from google.colab import drive
drive.mount('/content/drive')
```

    Mounted at /content/drive

```python
import tensorflow as tf
print(tf.__version__)
# tensorflow version used is 2.8.0
import torch
print(torch.__version__)
# torch version used is 1.10+cu111
```

    2.8.0
    1.10.0+cu111

```python
!nvidia-smi
```

    Sun Mar 20 16:57:53 2022
    +-----------------------------------------------------------------------------+
    | NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |
    |-------------------------------+----------------------+----------------------+
    | GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
    | Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
    |                               |                      |               MIG M. |
    |===============================+======================+======================|
    |   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |
    | N/A   47C    P8    10W /  70W |      0MiB / 15109MiB |      0%      Default |
    |                               |                      |                  N/A |
    +-------------------------------+----------------------+----------------------+

    +-----------------------------------------------------------------------------+
    | Processes:                                                                  |
    |  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
    |        ID   ID                                                   Usage      |
    |=============================================================================|
    |  No running processes found                                                 |
    +-----------------------------------------------------------------------------+

```python
# Other imports
! pip install tensorflow_addons
! pip install tensorflow_io
! pip install lightgbm
! pip install xgboost
! pip install catboost

import os
from tensorflow.keras.layers import *
from tensorflow.keras.models import *
from tensorflow import keras as tfkeras
import matplotlib.pyplot as plt
from imutils import paths
from tqdm import tqdm
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow_datasets as tfds
import tensorflow_io as tfio
import tensorflow_hub as hub
import numpy as np
import cv2
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.cluster import KMeans
from sklearn.metrics.pairwise import cosine_similarity as cos
from sympy.utilities.iterables import multiset_permutations
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, mean_absolute_error, r2_score
from sklearn.model_selection import *
from IPython.display import Image, display
from scipy import ndimage

from xgboost import XGBRegressor as xgb
from sklearn.ensemble import ExtraTreesRegressor as extratree
from sklearn.ensemble import RandomForestRegressor as randomforest
from sklearn.svm import NuSVR as svr
from sklearn.gaussian_process import GaussianProcessRegressor as gpr
from sklearn.ensemble import AdaBoostRegressor as ada
from sklearn.tree import DecisionTreeRegressor as tree
from sklearn.ensemble import GradientBoostingRegressor as gdb
from sklearn.neural_network import MLPRegressor as mlp
from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingRegressor as hgb
from lightgbm import LGBMRegressor as lgb
from catboost import CatBoostRegressor as cgb
from sklearn.ensemble import StackingRegressor as stk
from sklearn.linear_model import RidgeCV
from sklearn.model_selection import cross_validate, cross_val_predict
from sklearn.ensemble import VotingRegressor as vot

import zipfile
import concurrent.futures

# Random seed fix
random_seed = 42
tf.random.set_seed(random_seed)
np.random.seed(random_seed)
```

    Collecting tensorflow_addons
      Downloading tensorflow_addons-0.16.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)
         |████████████████████████████████| 1.1 MB 4.7 MB/s
    Requirement already satisfied: typeguard>=2.7 in /usr/local/lib/python3.7/dist-packages (from tensorflow_addons) (2.7.1)
    Installing collected packages: tensorflow-addons
    Successfully installed tensorflow-addons-0.16.1
    Collecting tensorflow_io
      Downloading tensorflow_io-0.24.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (23.4 MB)
         |████████████████████████████████| 23.4 MB 1.7 MB/s
    Requirement already satisfied: tensorflow-io-gcs-filesystem==0.24.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_io) (0.24.0)
    Installing collected packages: tensorflow-io
    Successfully installed tensorflow-io-0.24.0
    Requirement already satisfied: lightgbm in /usr/local/lib/python3.7/dist-packages (2.2.3)
    Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from lightgbm) (1.0.2)
    Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from lightgbm) (1.21.5)
    Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from lightgbm) (1.4.1)
    Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->lightgbm) (3.1.0)
    Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->lightgbm) (1.1.0)
    Requirement already satisfied: xgboost in /usr/local/lib/python3.7/dist-packages (0.90)
    Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.21.5)
    Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.4.1)
    Collecting catboost
      Downloading catboost-1.0.4-cp37-none-manylinux1_x86_64.whl (76.1 MB)
         |████████████████████████████████| 76.1 MB 1.2 MB/s
    Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from catboost) (1.21.5)
    Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from catboost) (3.2.2)
    Requirement already satisfied: plotly in /usr/local/lib/python3.7/dist-packages (from catboost) (5.5.0)
    Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.7/dist-packages (from catboost) (1.3.5)
    Requirement already satisfied: graphviz in /usr/local/lib/python3.7/dist-packages (from catboost) (0.10.1)
    Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from catboost) (1.15.0)
    Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from catboost) (1.4.1)
    Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->catboost) (2.8.2)
    Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->catboost) (2018.9)
    Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->catboost) (3.0.7)
    Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->catboost) (1.4.0)
    Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->catboost) (0.11.0)
    Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->catboost) (3.10.0.2)
    Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from plotly->catboost) (8.0.1)
    Installing collected packages: catboost
    Successfully installed catboost-1.0.4

    /usr/local/lib/python3.7/dist-packages/sklearn/experimental/enable_hist_gradient_boosting.py:17: UserWarning: Since version 1.0, it is not needed to import enable_hist_gradient_boosting anymore. HistGradientBoostingClassifier and HistGradientBoostingRegressor are now stable and can be normally imported from sklearn.ensemble.
      "Since version 1.0, "

## Dataset gathering and preparation

```python
%cd /content/drive/MyDrive/ML\ course\ 2022_Team\ BNL
```

    /content/drive/MyDrive/ML course 2022_Team BNL

```python
edge_percentage_threshold = np.array([0.00, 0.01, 0.02, 0.05, 0.10])
cap = np.array([20, 50])
valid_images_size = np.array([[2653, 2231, 1811, 1013, 510], [5878, 4943, 3995, 2235, 1186]])
training_data_size = np.array([10, 20, 50, 80, 100, 150, 200])
data_percentage_legend = np.arange(len(training_data_size))

color_method = ['C3', 'C1', 'C2', 'C3', 'C4']
# boxplot and violin plot parameters for better aesthetics
box_width = 0.15
shift = 0.3
# plt.rcParams['axes.prop_cycle'] = plt.cycler(color=plt.cm.tab10.colors)
plt.rcParams['axes.prop_cycle'] = plt.cycler(color=plt.cm.tab10.colors[3:4] + plt.cm.tab10.colors[0:3] + plt.cm.tab10.colors[4:5])    # avoid having both green and red in the same boxplot format

holder = {}
for i in range(len(cap)):
  for j in range(len(edge_percentage_threshold)):
    tempo = np.load('performance_log/level1/cap%i_cutoff%1.2f_valid%i.npz' %(cap[i], edge_percentage_threshold[j], valid_images_size[i, j]), allow_pickle=True)
    holder.update({i * 10 + j: tempo})

for i in range(len(cap)):
  for j in range(len(edge_percentage_threshold)):
# plot R_squared.
    fig = plt.figure(figsize=(12, 5))
    ax = fig.add_subplot(111)

    ax.set_ylim(0.2, 1.0)

    for k in range(2, len(training_data_size)):
      box0 = plt.boxplot([np.reshape(holder[i * 10 + j]['ridge'][k, :, 0], -1) for k in range(2, len(training_data_size))], widths=box_width, positions=np.arange(3, 8) - shift * 1.2, notch=True, patch_artist=True, boxprops=dict(facecolor=color_method[0]), medianprops=dict(color="black",linewidth=1.5))
      box1 = plt.boxplot([np.reshape(holder[i * 10 + j]['rf'][k, :, 0], -1) for k in range(2, len(training_data_size))], widths=box_width, positions=np.arange(3, 8) , notch=True, patch_artist=True, boxprops=dict(facecolor=color_method[1]), medianprops=dict(color="black",linewidth=1.5))
      box2 = plt.boxplot([np.reshape(holder[i * 10 + j]['mlp'][k, :, 0], -1) for k in range(2, len(training_data_size))], widths=box_width, positions=np.arange(3, 8) + shift * 1.2, notch=True, patch_artist=True, boxprops=dict(facecolor=color_method[2]), medianprops=dict(color="black",linewidth=1.5))

    for p in range(6):
      ax.vlines(1.5 + p, 0.2, 1.0, colors='grey')
    ax.set_xticks((data_percentage_legend + 1))
    ax.set_xticklabels((training_data_size).astype(str))
    ax.set_xlim(2.5, 7.5)
    ax.tick_params(labelsize=16, length=4)
    ax.set_xlabel(r'number of training images', fontsize=20, fontweight='bold')
    ax.set_ylabel(r'$R^{2}$', fontsize=20, fontweight='bold')
    ax.legend([box0["boxes"][0], box1["boxes"][0], box2["boxes"][0]], ['ridge-linear', 'random_forest', 'multi-layer perceptron'], fontsize=16, loc='lower right', framealpha=1)

# plot mean average error
    fig = plt.figure(figsize=(12, 5))
    ax = fig.add_subplot(111)

    if i == 0:
      ax.set_ylim(0, 5)
    else:
      ax.set_ylim(0, 10)

    for k in range(2, len(training_data_size)):
      box0 = plt.boxplot([np.reshape(holder[i * 10 + j]['ridge'][k, :, 1], -1) for k in range(2, len(training_data_size))], widths=box_width, positions=np.arange(3, 8) - shift * 1.2, notch=True, patch_artist=True, boxprops=dict(facecolor=color_method[0]), medianprops=dict(color="black",linewidth=1.5))
      box1 = plt.boxplot([np.reshape(holder[i * 10 + j]['rf'][k, :, 1], -1) for k in range(2, len(training_data_size))], widths=box_width, positions=np.arange(3, 8) , notch=True, patch_artist=True, boxprops=dict(facecolor=color_method[1]), medianprops=dict(color="black",linewidth=1.5))
      box2 = plt.boxplot([np.reshape(holder[i * 10 + j]['mlp'][k, :, 1], -1) for k in range(2, len(training_data_size))], widths=box_width, positions=np.arange(3, 8) + shift * 1.2, notch=True, patch_artist=True, boxprops=dict(facecolor=color_method[2]), medianprops=dict(color="black",linewidth=1.5))

    for p in range(6):
      if i == 0:
        ax.vlines(1.5 + p, 0, 5.0, colors='grey')
      if i == 1:
        ax.vlines(1.5 + p, 0, 10.0, colors='grey')
    ax.set_xticks((data_percentage_legend + 1))
    ax.set_xticklabels((training_data_size).astype(str))
    ax.set_xlim(2.5, 7.5)
    ax.tick_params(labelsize=16, length=4)
    ax.set_xlabel(r'number of training images', fontsize=20, fontweight='bold')
    ax.set_ylabel(r'mean average error [$\mu m$]', fontsize=20, fontweight='bold')
    # ax.legend([box0["boxes"][0], box1["boxes"][0], box2["boxes"][0]], ['ridge-linear', 'random_forest', 'multi-layer perceptron'], fontsize=16, loc='lower right', framealpha=1)

```

```python
print(np.average(holder[i * 10 + j]['rf'], axis=1))
print(np.std(holder[i * 10 + j]['rf'], axis=1))
```

    [[-0.21639083  7.47460511]
     [ 0.69821329  5.12855177]
     [ 0.89179968  3.47963082]
     [ 0.9233557   2.90292342]
     [ 0.93089882  2.66149753]
     [ 0.94185578  2.43684   ]
     [ 0.9463053   2.31654862]]
    [[0.74898413 1.07219296]
     [0.12144482 0.70805477]
     [0.03172447 0.44216891]
     [0.0258458  0.40550791]
     [0.0282795  0.39263678]
     [0.02288682 0.32602473]
     [0.02268225 0.32143601]]