from rest_framework.decorators import api_view, APIView, permission_classes
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated
from rest_framework.parsers import JSONParser
from base.models import Room, Topic, User
from .serializers import RoomSerializer, TopicSerializer, SingleTopicSerializer, ProfileSerializer, SingleProfileSerializer
from django.db import IntegrityError

@api_view(['GET'])
def getRoutes(request):
    routes =[
        {"GET": "api/rooms"},
        {"GET": "api/rooms/id"}
    ]
    return Response(routes)

@api_view(['GET', 'POST'])
def getRooms(request):
    if request.method == 'GET':
        rooms = Room.objects.all()
        serializer = RoomSerializer(rooms, many=True)
        return Response(serializer.data)
    elif request.method == 'POST':
        topic_name = request.data['topic']
        topic, created = Topic.objects.get_or_create(name=topic_name)

        rooms = Room.objects.create(
            #host = request.user,

            host = User.objects.get(email=(request.data['host'])),
            topic = topic,
            name = request.data['name'],
            description = request.data['description']
        )
        serializer = serializer = RoomSerializer(rooms, many=False)
        return Response(serializer.data)

@permission_classes([IsAuthenticated])
class RoomDetails(APIView):
    def get_object(self, pk):
        try:
            return Room.objects.get(id=pk)
        except Room.DoesNotExist:
            return None

    def get(self, request, pk):
        room = self.get_object(pk)
        if not room :
            return Response("Room does not exist")
        serializer = RoomSerializer(room, many=False)
        return Response(serializer.data)

    def delete(self, request, pk):
        room = self.get_object(pk)
        if not room :
            return Response("Room does not exist")
        room.delete()
        return Response("Room was deleted")

    def patch(self, request, pk):
        room = self.get_object(pk)
        if not room :
            return Response("Room does not exist")
        topic_name = request.data['topic']
        topic, created = Topic.objects.get_or_create(name=topic_name)

        room.host = User.objects.get(email=request.data["email"])
        room.topic = topic
        room.name = request.data['name']
        room.description = request.data['description']
        room.save()
        serializer = RoomSerializer(room, many=False)
        return Response(serializer.data)

@api_view(['GET', 'POST'])
def getTopics(request):
    if request.method == 'GET':
        topics = Topic.objects.all()
        serializer = TopicSerializer(topics, many=True)
        return Response(serializer.data)
    elif request.method == 'POST':
        try:
            topic = Topic.objects.create(
                name = request.data['name']
            )
        except IntegrityError:
            return Response("Error this topic already exist")
        serializer = TopicSerializer(topic, many=False)
        return Response(serializer.data)

@permission_classes([IsAuthenticated])
class TopicDetails(APIView):
    def get_object(self, pk):
        try:
            return Topic.objects.get(id=pk)
        except:
            return None

    def get(self, request, pk):
        topic = self.get_object(pk)
        if not topic :
            return Response("Topic does not exist")
        serializer = SingleTopicSerializer(topic, many=False)
        return Response(serializer.data)

    def patch(self, request, pk):
        topic = self.get_object(pk)
        if not topic :
            return Response("Topic does not exist")
        topic.name = request.data["name"]
        topic.save()
        serializer = SingleTopicSerializer(topic, many=False)
        return Response(serializer.data)

    def delete(self, request, pk):
        topic = self.get_object(pk)
        if not topic :
            return Response("Topic does not exist")
        topic.delete()
        return Response("Topic was deleted")

@api_view(["GET", "PUT"])
@permission_classes([IsAuthenticated])
def addParticipents(request, pk):
    room = Room.objects.get(id=pk)
    if request.method == "PUT":
        user = User.objects.get(email=request.data["email"])
        room.participents.add(user)
        room.save()
    serializer = ProfileSerializer(room.participents, many=True)
    return Response(serializer.data)

@api_view(["GET", "POST"])
def getUsers(request):
    if request.method == "GET":
        users = User.objects.all()
        serializer = ProfileSerializer(users, many=True)
        return Response(serializer.data)
    elif request.method == "POST":
        try:
            user = User.objects.create(
                username = request.data['username'],
                email = request.data['email'],
            )
            user.save()
        except IntegrityError:
            return Response("User already exists")

        serializer = SingleProfileSerializer(user, many=False)
        return Response(serializer.data)

@permission_classes([IsAuthenticated])
class UserDetails(APIView):
    def get_object(self, email):
        try:
            user = User.objects.get(email=email)
        except User.DoesNotExist:
            user = None
        return user

    def get(self, request, email):
        user = self.get_object(email)
        if not user:
            return Response("User does not exist")
        serializer = SingleProfileSerializer(user, many=False)
        return Response(serializer.data)

    def delete(self, request, email):
        user = self.get_object(email)
        if user:
            user.delete()
            return Response("User was successfully deleted")
        else:
            return Response("Can not delete non-existed user")

    def put(self, request, email):
        user = self.get_object(email)
        if not user:
            return Response("User does not exist")

        user_details = JSONParser().parse(request)
        serializer = SingleProfileSerializer(user, data=user_details)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response("Incorrect input")


from django.urls import path
# from rest_framework.routers import DefaultRouter

from .views import MessageListView, MessageCreateView, UserLoginView,UserRegisterView,ConversationView

# router=DefaultRouter()

# router.register('register/',UserRegisterView,basename='register')

urlpatterns = [
    path('api/user/register/',UserRegisterView.as_view(),name='register'),
    path('api/user/login/',UserLoginView.as_view(),name='login'),
    path('get-all-users/',MessageListView.as_view(),name='get_all_users'),
    path('get-conversation/',ConversationView.as_view(),name="get_conversation"),
    path('save-message/',MessageCreateView.as_view(),name='create-message'),

    # path('api/user/',include(router.urls))
]

#http://127.0.0.1:8000/get-conversation/
#request parameters: -  sender=1, reciever=2
# {
#     "message": [
#         {'id': "1", "name": "rahul", "message": "hey", "date": ""},
#         {'id': "2", "name": "vishal", "message": "hi", "date": ""},

#     ]
# }

#/get-all-users
# {
#     "users": [
#         {"id": 1,"name": "rahul"},
#         {"id": 2, "name": "vishal"}
#     ],
# }

#/save-message
# payload:-
# {
#     "message": "hey",
#     "sender": 1,
#     "receiver": 2,
# }
# response:-
# {
#     "status": "message saved successfully",
#     "message": "hey",
#     "sender": 1,
#     "receiver": 2
# }


from __future__ import absolute_import, division, print_function

import warnings
import tensorflow as tf
import os
import sys
import keras.backend as K
import numpy as np
os.environ["CUDA_VISIBLE_DEVICES"] = sys.argv[1]
tf.enable_eager_execution()
from keras.datasets import cifar10
import keras.callbacks as callbacks
import keras.utils.np_utils as kutils
from keras import regularizers

_BATCH_NORM_DECAY = 0.997
_BATCH_NORM_EPSILON = 1e-5
# DEFAULT_DTYPE = tf.float32

class Resnet(tf.keras.Model):

    def build_blocks(self, out_filters, num_blocks, stride):
        blocks = []
        strides = [stride] + [1]*(num_blocks-1)
        for _stride in strides:
            blocks.append(self.make_basic_block(self.in_filters, out_filters, _stride))
            self.in_filters = out_filters
        return blocks

    def make_basic_block(self, in_filters, out_filters, stride=1):

        block = []

        layers = tf.keras.Sequential([
            tf.keras.layers.ZeroPadding2D(1),
            tf.keras.layers.Conv2D(filters = out_filters, kernel_size = 3, strides= stride, padding = "valid", use_bias = False, kernel_initializer='VarianceScaling', kernel_regularizer=regularizers.l2(self.wd)),
            tf.keras.layers.BatchNormalization(axis=self.channel_axis, momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON, center=True,
                                                                            scale=True, fused=True),
            tf.keras.layers.Activation('relu'),

            tf.keras.layers.ZeroPadding2D(1),
            tf.keras.layers.Conv2D(filters = out_filters, kernel_size = 3, strides= 1, padding = "valid", use_bias = False, kernel_initializer='VarianceScaling', kernel_regularizer=regularizers.l2(self.wd)),
            tf.keras.layers.BatchNormalization(axis=self.channel_axis, momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON, center=True,
                                                                            scale=True, fused=True),

            # tf.keras.layers.Activation('relu')
            ])

        if stride != 1 or in_filters != out_filters:
            shortcut = tf.keras.Sequential([
                tf.keras.layers.Conv2D(filters = out_filters, kernel_size = 1, strides= stride, padding = "valid", use_bias = False, kernel_initializer='VarianceScaling', kernel_regularizer=regularizers.l2(self.wd)),
                 tf.keras.layers.BatchNormalization(axis=self.channel_axis, momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON, center=True,
                                                                            scale=True, fused=True)])
        else:
            shortcut = tf.keras.Sequential([])

        block.append(layers)
        block.append(shortcut)

        return block

    def _create_ResnetModel(self, filters):
        # filters : Number of fliters used in 1st blk (after initial layer)
        # kernel_size : kerel size for all the basic building blocks.
        # blocklist : List of values, each value shows the number of basic building blocks in the ith blk.
        # strides : stride for the 1st conv layer of all 4 blks (each basic building blk made of 2 conv layers)
        # For resnet18, blocklist = [2, 2, 2, 2], eack blk made of 2 basic building blocks.

        self.intial_conv = tf.keras.Sequential([
            tf.keras.layers.ZeroPadding2D(1),
            tf.keras.layers.Conv2D(filters=64, kernel_size=3, strides=1, padding = "valid", data_format = self.data_format, use_bias = False, kernel_initializer='VarianceScaling', kernel_regularizer=regularizers.l2(self.wd)),
            tf.keras.layers.BatchNormalization(axis=self.channel_axis, momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON, center=True,
                                                                            scale=True, fused=True),
            tf.keras.layers.Activation('relu')
            ])

        self.blocks_1 = self.build_blocks(64, self.block_list[0], stride=1)
        self.blocks_2 = self.build_blocks(128, self.block_list[1], stride=2)
        self.blocks_3 = self.build_blocks(256, self.block_list[2], stride=2)
        self.blocks_4 = self.build_blocks(512, self.block_list[3], stride=2)

        self.final_dense = tf.keras.Sequential([
            tf.keras.layers.AveragePooling2D(pool_size=4),
            tf.keras.layers.Flatten(),
            tf.keras.layers.Dense(self.classes, activation='softmax', kernel_regularizer=regularizers.l2(self.wd))
            ])

    # resnet with basic building blocks
    def __init__(self, data_format, initial_filters=64, block_list=[2, 2, 2, 2], classes=10, wt_decay = 0.001):

        """training: Either True or False, whether we are currently training the
           model. Needed for batch norm.
          data_format: The input format ('channels_last' or 'channels_first')."""

        super(Resnet, self).__init__()

        self.in_filters = initial_filters
        self.block_list = block_list
        self.num_blocks = len(block_list)
        self.data_format = data_format
        self.wd = wt_decay
        self.classes = classes
        # self.training = training
        self.channel_axis = 1 if self.data_format == 'channels_first' else -1

        self._create_ResnetModel(filters = initial_filters)

        # self.avg_pool = tf.keras.layers.GlobalAveragePooling2D()
        # self.flatten = tf.keras.layers.Flatten(data_format = self.data_format)
        # self.fc = tf.keras.layers.Dense(self.classes, kernel_regularizer=regularizers.l2(self.wd))

    def call(self, inputs):
        """if self.data_format == 'channels_first':
            # Convert the inputs from channels_last (NHWC) to channels_first (NCHW).
            # This provides a large performance boost on GPU. See
            # https://www.tensorflow.org/performance/performance_guide#data_formats
            inputs = tf.transpose(inputs, [0, 3, 1, 2])"""

        inputs = self.intial_conv(inputs)

        for block in self.blocks_1:
            block_x = block[0](inputs)
            block_y = block[1](inputs)
            inputs = tf.keras.layers.Activation('relu')(block_x+ block_y)

        for block in self.blocks_2:
            block_x = block[0](inputs)
            block_y = block[1](inputs)
            inputs = tf.keras.layers.Activation('relu')(block_x+ block_y)

        for block in self.blocks_3:
            block_x = block[0](inputs)
            block_y = block[1](inputs)
            inputs = tf.keras.layers.Activation('relu')(block_x+ block_y)

        for block in self.blocks_4:
            block_x = block[0](inputs)
            block_y = block[1](inputs)
            inputs = tf.keras.layers.Activation('relu')(block_x+ block_y)

        return self.final_dense(inputs)

if __name__ == '__main__':

    batch_size = 128
    img_rows, img_cols = 32, 32
    epochs = 200

    (trainX, trainY), (testX, testY) = cifar10.load_data()

    trainX = trainX.astype('float32')
    trainX = (trainX - trainX.mean(axis=0)) / (trainX.std(axis=0))
    testX = testX.astype('float32')
    testX = (testX - testX.mean(axis=0)) / (testX.std(axis=0))

    trainY = kutils.to_categorical(trainY, num_classes = 10)
    testY = kutils.to_categorical(testY, num_classes = 10)

    #testY = testY.astype(np.int64)
    #trainY = trainY.astype(np.int64)

    print(K.image_data_format())
    model = Resnet(data_format='channels_last')

    model._set_inputs(tf.zeros((batch_size, 32, 32, 3)))

    model.compile(optimizer=tf.train.AdamOptimizer(0.001), loss='categorical_crossentropy',
                      metrics=['accuracy'])

    model.fit(trainX, trainY, batch_size=batch_size, epochs=epochs,validation_data=(testX, testY), verbose=1)
    scores = model.evaluate(testX, testY, batch_size, verbose=1)


import pandas as pd
import numpy as np
import laspy
import json
import pptk
from math import *
from matplotlib import pyplot as plt
from matplotlib import path
from mpl_toolkits.mplot3d import Axes3D
from sklearn.neighbors import KDTree
import alphashape
from shapely.geometry import Point
from scipy.spatial import ConvexHull
from pysolar.solar import *
import datetime
import csv
import pytz
import time
import logging
import re
import os
from pyproj import Transformer
from datetime import timedelta

# from ipywidgets import IntProgress
# from IPython.display import display

from multiprocessing import Pool

#HARDCODED VALUES

COLOR_DICT = {
    "WHITE" : [255,255,255],
    "RED" : [1,0,0],
    "GREEN" : [0,1,0],
    "BLUE" : [0,0,1],
    "YELLOW" : [255,255,0],
    "LIGHT_BLUE" : [0,255,255],
    "PINK" : [255,0,255]
}

# DATE_DICT = {
#     "YEAR" : 2021,
#     "MONTH" : 6,
#     "DAY" : 21
# }

def Create_Dataframe_fromLas(lasFilePath):
    las = laspy.read(lasFilePath)
    #point_format = las.point_format
    lidar_points = np.array((las.X,las.Y,las.Z,las.intensity,las.classification, las.return_number, las.number_of_returns)).transpose()
    lidar_df = pd.DataFrame(lidar_points)
    lidar_df[0] = lidar_df[0]/100
    lidar_df[1] = lidar_df[1]/100
    lidar_df[2] = lidar_df[2]/100
    lidar_df.columns = ['X', 'Y', 'Z', 'intens', 'class', 'return_number', 'number_of_returns']

    return lidar_df

def Extract_SRdata(SR_JSONarr):
    #Store all Raw Treepoints
    SR_RawTreePoints = []
    #Store all Tree points mapped to a Tree ID
    SR_TreeClusterDict = {}
    #Store Tree Heights
    TreeHeights_ClusterHashmap = {}
    #Store GroundZ Elevation
    GroundElevation_ClusterHashmap = {}
    #Store All Tree Ids
    JSONClusterID_SR = []

    for i in range(len(SR_JSONarr)):
        #get points and cluster ID
        points = SR_JSONarr[i]["SRpointsInfo"]["SpecificClusterSRpoints"]
        cluster_id = SR_JSONarr[i]['SRpointsInfo']['SRpointsTreeCluster']
        #Storing with Cluster ID hashmap prevents storing multiple points from the same tree, if Tree present
        if cluster_id not in SR_TreeClusterDict:
            SR_TreeClusterDict[cluster_id] = points
        else:
            for p in points:
                SR_TreeClusterDict[cluster_id].append(p)
        #Tree Heights
        if cluster_id not in TreeHeights_ClusterHashmap:
            TreeHeights_ClusterHashmap[cluster_id] = SR_JSONarr[i]['TreeFoliageHeight']
        #GroundZvalue
        if cluster_id not in GroundElevation_ClusterHashmap:
            GroundElevation_ClusterHashmap[cluster_id] = SR_JSONarr[i]['GroundZValue']

        #Raw Points
        for j in points:
            SR_RawTreePoints.append(j)

    return SR_RawTreePoints, SR_TreeClusterDict, TreeHeights_ClusterHashmap, GroundElevation_ClusterHashmap

def Extract_MRdata(MR_JSONarr):
    ##Store all Raw Treepoints
    MR_RawTreePoints = []
    #Store all Tree points mapped to a Tree ID
    MR_TreeClusterDict = {}
    #Store Cluster Location
    TreeLocation_ClusterHashmap = {}

    for i in range(len(MR_JSONarr)):
        #get points and cluster ID
        points = MR_JSONarr[i]["ConvexHullDict"]["ClusterPoints"]
        cluster_id = MR_JSONarr[i]['ClusterID']
        #Storing with Cluster ID hashmap prevents storing multiple points from the same tree, if Tree present
        if cluster_id not in MR_TreeClusterDict:
            MR_TreeClusterDict[cluster_id] = points
        else:
            for p_mr in points:
                MR_TreeClusterDict[cluster_id].append(p_mr)

        Tree_lat = MR_JSONarr[i]["PredictedTreeLocation"]["Latitude"]
        Tree_long = MR_JSONarr[i]["PredictedTreeLocation"]["Longitude"]

        if cluster_id not in TreeLocation_ClusterHashmap:
            TreeLocation_ClusterHashmap[cluster_id] = [Tree_lat,Tree_long]

        #Raw points
        for k in points:
            MR_RawTreePoints.append(k)

    return MR_RawTreePoints, MR_TreeClusterDict, TreeLocation_ClusterHashmap

def Get_SR_Treepoints_from_Hashmap(TreeClusterID, SR_TreeClusterDict, las_filename='25192'):
    return SR_TreeClusterDict[las_filename+"_"+str(TreeClusterID)]

def Get_MR_Treepoints_from_Hashmap(TreeClusterID, MR_TreeClusterDict, las_filename='25192'):
    return MR_TreeClusterDict[las_filename+"_"+str(TreeClusterID)]

def Get_FullTreeCentroid(Full_Tree_points):
    centroid = np.mean(Full_Tree_points,axis=0)
    centroid[2] = 0

    return centroid

def Get_Shadow(points, az, amp):
    projected_points = []
    for point in points:
        sinAz = sin( radians( az + 180.0 ) )
        cosAz = cos( radians( az + 180.0 ) )
        tanAmp = tan( radians(amp) )
        pointGroundX = point[0] + ( ( point[2] / tanAmp ) *sinAz )
        pointGroundY = point[1] + ( ( point[2] / tanAmp ) *cosAz )
        pointGroundZ =  point[2] * 0

        projected_points.append([pointGroundX,pointGroundY,pointGroundZ])

    return np.array(projected_points)

def Get_ShadowCharacteristics(Full_Tree, TreeShadow):

    F_distance, F_point = Get_FurthestPointFromTreeCentroid(Full_Tree, TreeShadow)
    centroid = Get_FullTreeCentroid(Full_Tree)
    hull = ConvexHull(TreeShadow[:,:2])

    Shadow_Area = hull.area
    Shadow_length = np.linalg.norm(centroid - F_point) # Tree Centroid - Furthest point from Tree centroid on Tree shadow
    shadow_breadth = Shadow_Area/Shadow_length

    return Shadow_length, shadow_breadth, Shadow_Area

def Get_FurthestPointFromTreeCentroid(Full_Tree, TreeShadow):

    Shadow_KDTree = KDTree(TreeShadow)
    centroid = Get_FullTreeCentroid(Full_Tree)
    dist,ind = Shadow_KDTree.query(centroid.reshape(1,3),len(TreeShadow))

    Furtherst_Point_Distance = dist[0][-1]
    Furthest_Point_Vector = TreeShadow[ind[0][-1]]

    return Furtherst_Point_Distance, Furthest_Point_Vector

def Get_ShadowAlphaShape(ShadowPoints):
    return alphashape.alphashape(ShadowPoints, alpha=0)

def Get_GroundElevation(BuildingLidarDict):
    arr = (np.concatenate(list(BuildingLidarDict.values())))
    #Get ground elevation from building points min(list(BuildingLidarDict.values())[0][:,2])
    return min(arr[:,2])

#Code to automate Obtaining Az and Amp
def Get_SunData(Tree_Latitude, Tree_Longitude, year, month, day, hour, minute):
    date = datetime.datetime(year, month, day, hour, minute, 0, 0, tzinfo=datetime.timezone.utc)
    date_est = date.astimezone(pytz.timezone('US/Eastern')).isoformat()
    az = get_azimuth(Tree_Latitude, Tree_Longitude, date)
    amp = get_altitude(Tree_Latitude, Tree_Longitude, date)

    return date_est, az, amp

def pptk_wrapperSimplePlot(PointsArr, Color):
    v = pptk.viewer(PointsArr, [Color]*len(PointsArr))
    v.set(show_grid=False)
    v.set(show_axis=False)
    v.set(bg_color = [0,0,0,0])
    v.set(point_size = 0.04)

def sample_polygon(V, eps=0.25):
    # samples polygon V s.t. consecutive samples are no greater than eps apart
    # assumes last vertex in V is a duplicate of the first
    M = np.ceil(np.sqrt(np.sum(np.diff(V, axis=0) ** 2, axis = 1)) / eps)
    Q = []
    for (m, v1, v2) in zip(M, V[: -1], V[1:]):
        Q.append(np.vstack([ \
            np.linspace(v1[0], v2[0], endpoint = False), \
            np.linspace(v1[1], v2[1], endpoint = False)]).T)
    Q = np.vstack(Q)

    return Q

def Read_GeoJSON(filepath):
    with open(filepath, 'rb') as fd:
        data = json.load(fd)

    return data

def Get_SampledBuildingFootprints(Buildingdata):
    BuildingFeature_Coords = [np.array(F['geometry']['coordinates'][0][0]) for F in Buildingdata['features']]
    SampledFootprint = np.vstack([sample_polygon(W) for W in BuildingFeature_Coords])
    SampledFootprint = np.c_[SampledFootprint, np.zeros(len(SampledFootprint))]

    return SampledFootprint

def Get_BuildingFootprint(GeoJSON_Filepath):
    BuidlingFootprintData = Read_GeoJSON(GeoJSON_Filepath)

    return Get_SampledBuildingFootprints(BuidlingFootprintData)

def Get_BuildingDataDict(BuildingFilePath, f, year=2017):
    with open(BuildingFilePath, 'rb') as fd:
        NYC_building_Footprint = json.load(fd)

    Vs_25192 = [np.array(F['geometry']['coordinates'][0][0]) for F in NYC_building_Footprint['features']]
    ### **Note** : NYC points already projected to state plane, No need for pyproj
    Ws_25192 = Vs_25192
    Total_Building_Count = len(NYC_building_Footprint['features'])
    #Building_Data_dict
    Building_coords_dict = {}
    B_ID = 0
    Building_Key = f[:-4] + "_" + str(year) + "_Building_" + str(B_ID) # initialized to 25192_2015_Building_0

    for building in range(Total_Building_Count):

        B_ID += 1
        Building_Key = f[:-4] + "_" + str(year) + "_Building_" + str(B_ID) # initialized to 25192_2015_Building_0
        building_coords = sample_polygon(Ws_25192[building]) #NYC_building_Footprint['features'][building]['geometry']['coordinates'][0][0]
        Building_coords_dict[Building_Key] = building_coords

    return Building_coords_dict

#NOTE: Function takes most time
def Get_BuildingsUnderEffectOfShadow(Building_coords_dict, TreeShadowPoints):
    ShadowAlphaShape = Get_ShadowAlphaShape(TreeShadowPoints)
    Affected_Building_IDarr = [] #Buildings Under the Direct Affect of the Shadow
    for B_ID, Sampled_B_coords in Building_coords_dict.items():
        bool_temp_list = []
        for point in Sampled_B_coords:
            tp = Point(point)
            bool_temp_list.append(tp.within(ShadowAlphaShape))
        #check if any points in the sampled footprint fall within the area of shadow
        if len(Sampled_B_coords[bool_temp_list]) > 0:
            Affected_Building_IDarr.append(B_ID)

    return Affected_Building_IDarr

def Get_BuildingsUnderEffectOfShadow_Rapid(Building_coords_dict, TreeShadowPoints):

    Affected_Building_IDarr = [] #Buildings Under the Direct Affect of the Shadow
    for B_ID, Sampled_B_coords in Building_coords_dict.items():
        bool_temp_list = []

        p = path.Path(TreeShadowPoints[:,:2])
        bool_temp_list = p.contains_points(Sampled_B_coords)
        # #check if any points in the sampled footprint fall within the area of shadow
        if len(Sampled_B_coords[bool_temp_list]) > 0:
            Affected_Building_IDarr.append(B_ID)

    return Affected_Building_IDarr

def Get_BuildingFootprintSampledCoords(BIDs_Arr, Building_coords_dict):

    labelled_BF_points = []
    for B_ID in BIDs_Arr:

        b_points = Building_coords_dict[B_ID]

        for p in b_points:
            labelled_BF_points.append(p)

    labelled_BF_points = np.array(labelled_BF_points) #shape : (Nx2)

    #reformat for plotting purposes
    LBF_arr = []
    for cord in range(len(labelled_BF_points)):
        x = labelled_BF_points[cord][0]
        y = labelled_BF_points[cord][1]
        z = 0
        LBF_arr.append([x,y,z])

    LBF_arr = np.array(LBF_arr)

    return LBF_arr

def Get_BuildingLidarPoints(BIDs_Arr, Building_coords_dict, lasdf):

    lidarPointsRaw = lasdf.iloc[:,:3].to_numpy()
    # Selecting Lidar Points within Building footprint

    X_max , X_min = lasdf.X.max(), lasdf.X.min()
    Y_max , Y_min = lasdf.Y.max(), lasdf.Y.min()

    X_plane_tile_divisor = 50 #in m - indicates the number of tiles you want to divide the tiles into
    Y_plane_tile_divisor = 50 #in m

    X_diff = X_max - X_min
    Y_diff = Y_max - Y_min

    X_div_len = X_diff/X_plane_tile_divisor
    Y_div_len = Y_diff/Y_plane_tile_divisor

    # print(X_max , X_min)
    # print(Y_max , Y_min)

    #NOTE : Iterating over all the points takes too long, need a more optimized way

    lidar_BpointsDict = {}

    #For every building under the effect of the shadow
    for B_ID in BIDs_Arr:

        #List to append lidar points into for a specifc building
        B_ID_LidarPointsTempList = []

        #get the sampled building footprint points
        bf_points = Building_coords_dict[B_ID]

        #create a 2D convex Hull
        bf_shape = ConvexHull(bf_points[:,:2])

        #Get an area of points to look at (No need to iterate through all points in the tile)
        #get bounding subset to look at
        bf_shape_X_max = bf_shape.max_bound[0] + X_div_len
        bf_shape_Y_max = bf_shape.max_bound[1] + Y_div_len

        bf_shape_X_min = bf_shape.min_bound[0] - X_div_len
        bf_shape_Y_min = bf_shape.min_bound[1] - Y_div_len

        # print(bf_shape_X_max,bf_shape_Y_max,bf_shape_X_min,bf_shape_Y_min)

        #Bounded_Vertices_BF = bf_points[bf_shape.vertices]

        lidar_subset_df = lasdf[
            (lasdf['X'].between(bf_shape_X_min, bf_shape_X_max, inclusive=False) &
        lasdf['Y'].between(bf_shape_Y_min, bf_shape_Y_max, inclusive=False))
        ]

        # print("subset shape : ",lidar_subset_df.shape)

        #Get only points from BF
        lidar_BFMasked_points = lidar_subset_df.iloc[:,:3].to_numpy()

        #Check if a lidar point is within a building Footprint

        #Add a z-coordinate to BF ,  needed for alphashape
        bf_points_with_Z = np.c_[bf_points, np.zeros(len(bf_points))]
        #Create a shape of the building footprint
        bf_shape_alpha = alphashape.alphashape(bf_points_with_Z, alpha=0)

        for p in lidar_BFMasked_points:

            tp = Point(p)
            if(tp.within(bf_shape_alpha)):
                B_ID_LidarPointsTempList.append(p)

        if B_ID not in lidar_BpointsDict:
            lidar_BpointsDict[B_ID] = np.array(B_ID_LidarPointsTempList)

    return lidar_BpointsDict

def Set_BuildingLidarHeightAdjustment(BuidlingDataDict, height_Adjustment):

    for B_ID, B_Lidarpoints in BuidlingDataDict.items():

        bpArr = B_Lidarpoints
        #adjust height of buildings
        lidar_Bpoints_Hadj = []
        for bp in bpArr:
            lidar_Bpoints_Hadj.append(bp - [0,0,height_Adjustment])

        BuidlingDataDict[B_ID] = np.array(lidar_Bpoints_Hadj)

    return BuidlingDataDict

#https://github.com/ulikoehler/UliEngineering/blob/master/UliEngineering/Math/Coordinates.py
# -*- coding: utf-8 -*-
"""
Estimating BOunding Box
"""
__all__ = ["BoundingBox"]

class BoundingBox(object):
    """
    A 2D bounding box
    """
    def __init__(self, points):
        """
        Compute the upright 2D bounding box for a set of
        2D coordinates in a (n,2) numpy array.

        You can access the bbox using the
        (minx, maxx, miny, maxy) members.
        """
        if len(points.shape) != 2 or points.shape[1] != 2:
            raise ValueError("Points must be a (n,2), array but it has shape {}".format(
                points.shape))
        if points.shape[0] < 1:
            raise ValueError("Can't compute bounding box for empty coordinates")
        self.minx, self.miny = np.min(points, axis=0)
        self.maxx, self.maxy = np.max(points, axis=0)

    @property
    def width(self):
        """X-axis extent of the bounding box"""
        return self.maxx - self.minx

    @property
    def height(self):
        """Y-axis extent of the bounding box"""
        return self.maxy - self.miny

    @property
    def area(self):
        """width * height"""
        return self.width * self.height

    @property
    def aspect_ratio(self):
        """width / height"""
        return self.width / self.height

    @property
    def center(self):
        """(x,y) center point of the bounding box"""
        return (self.minx + self.width / 2, self.miny + self.height / 2)

    @property
    def max_dim(self):
        """The larger dimension: max(width, height)"""
        return max(self.width, self.height)

    @property
    def min_dim(self):
        """The larger dimension: max(width, height)"""
        return min(self.width, self.height)

    def __repr__(self):
        return "BoundingBox({}, {}, {}, {})".format(
            self.minx, self.maxx, self.miny, self.maxy)
    #NOTE: Added by me
    def Get_Coords(self):
        return [self.minx, self.maxx, self.miny, self.maxy]

def Get_TreeShadowBoundingBoxLimits(TreeShadow):
    Box_obj = BoundingBox(TreeShadow[:,:2])
    Box_Bounds = np.array(Box_obj.Get_Coords())

    B_minx = Box_Bounds[0]
    B_maxx = Box_Bounds[1]
    B_miny = Box_Bounds[2]
    B_maxy = Box_Bounds[3]

    return B_minx, B_maxx, B_miny, B_maxy

def Get_TreeShadowBoundingBoxesCoords(TreeShadow, Distance):

    B_minx, B_maxx, B_miny, B_maxy = Get_TreeShadowBoundingBoxLimits(TreeShadow)

    Tree_ShadowBox_Coords = [
        [B_minx, B_miny, 0],
        [B_minx, B_maxy, 0],
        [B_maxx, B_maxy, 0],
        [B_maxx, B_miny, 0],
        [B_minx, B_miny, 0] # final one is to complete the box used for plotting purposes only
    ]

    Extend_dist = Distance #in m
    Tree_ShadowBoxExtend_Coords = [
        [B_minx, B_miny - Extend_dist, 0],
        [B_minx, B_maxy, 0],
        [B_maxx, B_maxy, 0],
        [B_maxx, B_miny - Extend_dist, 0],
        [B_minx, B_miny - Extend_dist, 0] # final one is to complete the box used for plotting purposes only
    ]

    return Tree_ShadowBox_Coords, Tree_ShadowBoxExtend_Coords

#Facade shade
def Get_FacadeShadePoints(TreeShadow, PrimaryBuilding_IdArr, Building_coords_dict):

    FacadeShade_Points = []

    RemainingShadow_Points = TreeShadow

    for B_ID in PrimaryBuilding_IdArr:

        #get the sampled building footprint points
        bf_points = Building_coords_dict[B_ID]

        #Get only points from BF
        BuildingFootprint_shape = alphashape.alphashape(bf_points, alpha=0)

        curr_BF_bool_list = []
        #find Tree Shadow Points in StreetShade
        for tsp in RemainingShadow_Points:
            tp = Point(tsp)
            curr_BF_bool_list.append(tp.within(BuildingFootprint_shape))

        for fp in RemainingShadow_Points[curr_BF_bool_list]: #only inside the BF shape
            FacadeShade_Points.append(fp)

        RemainingShadow_Points = RemainingShadow_Points[~np.array(curr_BF_bool_list)]

    return RemainingShadow_Points, FacadeShade_Points

def Get_InShadePoints(TreeShadow, All_Building_LidarPointsDict_HAdj, Az, Amp):

    InShade_Points = []
    RemainingShadow_Points = TreeShadow

    for B_ID, B_lidarpoints in All_Building_LidarPointsDict_HAdj.items():

        B_Shadow = Get_Shadow(B_lidarpoints, Az, Amp)

        #Get only points from BS
        BuildingShadow_shape = alphashape.alphashape(B_Shadow, alpha=0)

        curr_BS_bool_list = []
        #find Tree Shadow Points in InShade
        for tsp in RemainingShadow_Points:
            tp = Point(tsp)
            curr_BS_bool_list.append(tp.within(BuildingShadow_shape))

        for Ip in RemainingShadow_Points[curr_BS_bool_list]: #only inside the BS shape
            InShade_Points.append(Ip)

        if(len(curr_BS_bool_list) > 0):
            RemainingShadow_Points = RemainingShadow_Points[~np.array(curr_BS_bool_list)]

    return RemainingShadow_Points, InShade_Points

def InitiateShadingLogger(filename:str,year:int)-> None:

    #Where the logger file will be created
    LoggerPath = "Datasets/"+"Package_Generated/"+filename[:-4]+"/"+str(year)+"/Shading_Logs_"+filename[:-4]+"/"

    print("Logger Folder Path : ",LoggerPath)
    # Check whether the specified LoggerPath exists or not
    isExist = os.path.exists(LoggerPath)

    if not isExist:
    # Create a new directory because it does not exist
        os.makedirs(LoggerPath)

    logfilename = LoggerPath + 'ShadingScript_'+filename[:-4]+'.log'
    logger = logging.getLogger()
    fhandler = logging.FileHandler(filename=logfilename, mode='a')
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fhandler.setFormatter(formatter)
    logger.addHandler(fhandler)
    logger.setLevel(logging.INFO)

def parse_log_Completedfiles(log_file_path):
    """_summary_

    Args:
        log_file_path (_type_): _description_

    Returns:
        _type_: _description_

    Usage:
    log_file_path = 'TEST_log.log'
    filenames = parse_log_Completedfiles(log_file_path)
    print(filenames)
    """
    filenames = []
    with open(log_file_path, 'r') as log_file:
        for line in log_file:
            match = re.search(r'(?<=Completed file: )\S+', line)
            if match:
                filenames.append(match.group())
    return filenames

#Functions modified from v1
def Get_dirnames(parent_path:str)->list:
    dirnames = [d for d in os.listdir(parent_path) if os.path.isdir(os.path.join(parent_path, d))]
    return dirnames

# Reading JSON tree data
def Get_JSONfilenames(folder_path:str, year:int)->list:
    filenames = [f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))]
    return filenames

def Get_TreeLocation(json_dataBuffer):
    lat = json_dataBuffer['PredictedTreeLocation']['Latitude']
    lon = json_dataBuffer['PredictedTreeLocation']['Longitude']
    return lat,lon

def Get_TreeGroundElevation(json_dataBuffer):
    return json_dataBuffer['GroundZValue']

def Get_TreeHeight(json_dataBuffer):
    return json_dataBuffer['TreeFoliageHeight']

def Get_FullTree_points(json_dataBuffer, Ground_Elevation = 0, Tree_Height = 0):
    Full_Tree = []

    #TODO : might be doing repetitive conversion here
    All_SingleTree_points = np.array(json_dataBuffer['Tree_Points'])*3.28 # Convert m to ft
    for p in All_SingleTree_points:
        Full_Tree.append(p - [0,0,Ground_Elevation+Tree_Height]) #Height Adjusted, default = 0

    return np.array(Full_Tree)

def Get_TreesNotPark(json_filenames, JSON_TreeData_path):
    JsonFiles_notInPark = []
    for jf in json_filenames:
        json_filobj = open(JSON_TreeData_path + jf)
        json_SingleTreeDataBuffer = json.load(json_filobj)
        if not json_SingleTreeDataBuffer['InPark']:
            JsonFiles_notInPark.append(jf)

    return JsonFiles_notInPark

def Get_AllTreePoints_from_JSONfiles(json_filenames,JSON_TreeData_path):
    tree_p = []
    for jf in json_filenames:
        json_filobj = open(JSON_TreeData_path + jf)
        json_SingleTreeDataBuffer = json.load(json_filobj)

        singleTreePoints = json_SingleTreeDataBuffer['Tree_Points']

        for p in singleTreePoints:
            tree_p.append(p)

    return tree_p

def footprintPointsFromGeoJSON_v2(feature):
    points = []

    try:
        height = feature["properties"]["heightroof"]

        if type(height) == float:
            height = height
        else:
            height = 0
    except:
        height = 0

    if feature['geometry']['type'] == 'Point':
        point = [feature["geometry"]["coordinates"][0],feature["geometry"]["coordinates"][1],height]
        points.append(point)
        point = [feature["geometry"]["coordinates"][0],feature["geometry"]["coordinates"][1],0]
        points.append(point)
    elif feature['geometry']['type'] == 'LineString':
        for polygonPart in feature["geometry"]["coordinates"]:
            for polygonSubPart in polygonPart:
                point = [polygonSubPart[0],polygonSubPart[1],height]
                points.append(point)
                point = [polygonSubPart[0],polygonSubPart[1],0]
                points.append(point)
    # elif feature['geometry']['type'] == 'Polygon':
    #     for polygonPart in feature["geometry"]["coordinates"]:
    #         for polygonSubPart in polygonPart:
    #             for coordinates in polygonSubPart:
    #                 point = [coordinates[0],coordinates[1],height]
    #                 points.append(point)
    #                 point = [coordinates[0],coordinates[1],0]
    #                 points.append(point)
    elif feature['geometry']['type'] == 'MultiPolygon':
        for MpolygonPart in feature["geometry"]["coordinates"]:
            for polygonPart in MpolygonPart:
                for polygonSubPart in polygonPart:
                    #print(polygonSubPart)
                    #for coordinates in polygonSubPart:
                    point = [polygonSubPart[0],polygonSubPart[1],height]
                    points.append(point)
                    point = [polygonSubPart[0],polygonSubPart[1],0]
                    points.append(point)
    else:
        pass
        #print(feature['geometry']['type'])
        #print(feature["geometry"]["coordinates"])

    return points, height

#Functions to trim geojson
def findCentroid(buildingPoints):
    xs = []
    ys = []
    for buildingPoint in buildingPoints:
        xs.append(buildingPoint[0])
        ys.append(buildingPoint[1])
    xCenter = sum(xs)/len(xs)
    yCenter = sum(ys)/len(ys)
    return xCenter, yCenter

def convertLatLon(lat,lon):
    #translate from geojson CRS (NAD 1983) to .las CRS (UTM Zone 18N (meters))
    transformer = Transformer.from_crs( "epsg:4326", "epsg:2263" )
    x, y = transformer.transform(lat, lon)
    return x, y

def convertXY(x,y):
    #translate from .las CRS to geojson CRS (NAD 1983) (UTM Zone 18N (meters))
    transformer = Transformer.from_crs( "epsg:2263", "epsg:4326" )
    lat, lon = transformer.transform(x, y)
    return lat, lon

def lasdf_bounds(lasdf):

    xMin = min(lasdf.X)
    xMax = max(lasdf.X)
    yMin = min(lasdf.Y)
    yMax = max(lasdf.Y)

    return xMin,yMin,xMax,yMax

def trimGeoJSON_v2(features,lasdf):

    #print(len(features))

    xMin,yMin,xMax,yMax = lasdf_bounds(lasdf)

    Min_lat,Min_lon  = convertXY(xMin, yMin)
    Max_lat,Max_lon  = convertXY(xMax, yMax)

    features2 = []

    for feature in features: #NOTE: modified from - for feature in features[:]:
        buildingPoints,height = footprintPointsFromGeoJSON_v2(feature)

        xCenter, yCenter = findCentroid(buildingPoints)

        #lat varies in y direction
        #lon varies in x direction

        if yCenter > Min_lat and yCenter < Max_lat and xCenter > Min_lon and xCenter < Max_lon:
            features2.append(feature)

        else:
            continue

    return features2

def convert_dict_of_arrays(data):
    transformer = Transformer.from_crs("epsg:4326", "epsg:2263" )
    converted_data = {}
    for key, value in data.items():
        x, y = transformer.transform(value[:,1], value[:,0])
        converted_data[key] = np.column_stack((x, y))
    return converted_data

def ProcessShading(f,year,fpath):

    try:

        stime = time.time()

        InitiateShadingLogger(f,year)

        ShadeYear = 2021
        start = datetime.datetime(ShadeYear, 1, 8) #8 am
        end = datetime.datetime(ShadeYear, 6, 30, 17, 59) #6 pm
        delta = timedelta(minutes=15)

        logging.info("Year : %d",ShadeYear)
        logging.info("Start date : %s",start)
        logging.info("End date : %s",end)
        logging.info("Time Interval : 15")

        logging.info("TerraVide lidar Shading Metrics Estimation Initated")

        las_file_path = fpath+f[:-4]+'/'+str(year)+'/LasClassified_'+f[:-4]+'/lasFile_Reconstructed_'+f[:-4]+'.las'

        logging.info("Reading Reconstructed lasfile from : %s",las_file_path)

        #Create a dataframe from las file
        lasdf = Create_Dataframe_fromLas(las_file_path)

        #in ft
        lasdf.X = lasdf.X*3.28
        lasdf.Y = lasdf.Y*3.28
        lasdf.Z = lasdf.Z*3.28

        #Buidling footprint of all NYC buildings - 2015 data
        NYC_GeoJSON_Filepath = 'Datasets/BuildingTileSet/building_BL.geojson'
        BuidlingFootprintData = Read_GeoJSON(NYC_GeoJSON_Filepath)

        Filtered_FeatureList = trimGeoJSON_v2(BuidlingFootprintData['features'],lasdf)

        #Save to Geojson

        # Serialize features into a GeoJSON string
        features_geojson = {
            "type": "FeatureCollection",
            "features": Filtered_FeatureList
        }
        features_str = json.dumps(features_geojson)

        geojson_folderpath = 'Datasets/Package_Generated/'+f[:-4]+'/'+str(year)+'/GeoJSON_BuildingFootprint_'+f[:-4]+'/'
        geojson_filename = 'Filtered_buildingsTile_'+f[:-4]+'.geojson'

        # Check whether the specified geojson_folderpath exists or not
        isExist = os.path.exists(geojson_folderpath)

        if not isExist:
        # Create a new directory because it does not exist
            os.makedirs(geojson_folderpath)
        # Write GeoJSON string to a file
        with open(geojson_folderpath+geojson_filename, "w") as file:
            file.write(features_str)
            logging.info("Building Footprint for tileset created")

        Building_coords_dict_latlong_coords = Get_BuildingDataDict(geojson_folderpath+geojson_filename, f, year=2017)
        Building_coords_dict = convert_dict_of_arrays(Building_coords_dict_latlong_coords)

        logging.info("TerraVide lidar Shading Metrics Estimation Initated")

        #Path to where each lasfiles JSON tree data is located (generated by Process_lasFiles.py)
        JSON_TreeData_path = fpath+f[:-4]+'/'+str(year)+'/JSON_TreeData_'+f[:-4]+'/'
        #Get filenames(1 json file = 1 tree cluster) of each json file
        JSON_filenames = Get_JSONfilenames(JSON_TreeData_path,year)
        #Get trees not present in park areas
        JSON_filenamesNotPark = Get_TreesNotPark(JSON_filenames, JSON_TreeData_path)

        logging.info("Total number of Trees : %d",len(JSON_filenames))
        logging.info("Total number of Trees not in Park: %d",len(JSON_filenamesNotPark))

        #Used for debugging with pptk viewer
        Raw_TreePointsNotPark = Get_AllTreePoints_from_JSONfiles(JSON_filenamesNotPark,JSON_TreeData_path)

        # CSV header
        header = ['DateTime_ISO', 'Year', 'Month', 'Day', 'hour', 'minute', 'Sun_Azimuth', 'Sun_Amplitude',
            'Tree_Number', 'Tree_Latitude', 'Tree_Longitude',
            'Shadow_Length','Shadow_Breadth','Shadow_Area','TreeShadow_PointCount',
            'Perc_Canopy_StreetShade', 'Perc_Canopy_FacadeShade', 'Perc_Canopy_InShade',
            'ShadowArea_Ground', 'ShadowArea_OnBuilding', 'ShadowArea_InBuildingShadow']

        for jf in JSON_filenamesNotPark:
            #Reading a single file
            TreeId = jf.split('_')[3] #25192_2017_ID_1_TreeCluster.json

            logging.info("Shading Metrics Calculation started for tree : %s",TreeId)

            json_filobj = open(JSON_TreeData_path + jf)
            json_SingleTreeDataBuffer = json.load(json_filobj)

            shadingCSV_folderpath =fpath+f[:-4]+'/'+str(year)+'/ShadingMetrics'+f[:-4]+'/'
            shadingCSV_filename = 'ShadingMetric_'+f[:-4]+'_Tree_ID_'+TreeId+'.csv'

            # Check whether the specified geojson_folderpath exists or not
            isExist = os.path.exists(shadingCSV_folderpath)

            if not isExist:
            # Create a new directory because it does not exist
                os.makedirs(shadingCSV_folderpath)

            with open(shadingCSV_folderpath+shadingCSV_filename, 'w', encoding='UTF8') as csv_f:
                writer = csv.writer(csv_f)

                # write the header
                writer.writerow(header)

                current = start
                while current <= end:
                    day = current.day
                    hour = current.hour
                    minute = current.minute
                    month = current.month
                    # Use the values as needed
                    current += delta

                    #print(ShadeYear, month, day, hour, minute)

                    #Tree Data
                    Tree_Lat , Tree_Long = Get_TreeLocation(json_SingleTreeDataBuffer)
                    Tree_GroundZ = Get_TreeGroundElevation(json_SingleTreeDataBuffer) #Convert to ft, JSON file has m stored
                    Tree_Height = Get_TreeHeight(json_SingleTreeDataBuffer)

                    # GET AZ and Amp
                    date, Az, Amp = Get_SunData(Tree_Lat, Tree_Long, ShadeYear, month, day, hour, minute)

                    if Amp > 0:
                        #Get Tree Points
                        Full_Tree = Get_FullTree_points(json_SingleTreeDataBuffer,Tree_GroundZ)

                        #Proj_Tree = get_projection(Full_Tree, 45, True)
                        TreeShadow = Get_Shadow(Full_Tree, Az, Amp) #(points, az, amp)

                        L,B,A = Get_ShadowCharacteristics(Full_Tree, TreeShadow)

                        #Get Primary Building IDs
                        #PrimaryBuilding_IdArr = Get_BuildingsUnderEffectOfShadow(Building_coords_dict, TreeShadow)
                        PrimaryBuilding_IdArr = Get_BuildingsUnderEffectOfShadow_Rapid(Building_coords_dict, TreeShadow) #Function is 50% faster
                        #Get Primary Building Coords
                        PrimaryBuilding_FootprintCoords = Get_BuildingFootprintSampledCoords(PrimaryBuilding_IdArr, Building_coords_dict)

                        #Get Primary Building Lidar Points
                        PrimaryBuilding_LidarPointsDict = Get_BuildingLidarPoints(PrimaryBuilding_IdArr, Building_coords_dict, lasdf)

                        #Get ground elevation from building points
                        G_height = Tree_GroundZ #NOTE : Same as Tree_GroundZ
                        #Adjusting height of Buildings
                        PrimaryBuilding_LidarPointsDict_HAdj = Set_BuildingLidarHeightAdjustment(PrimaryBuilding_LidarPointsDict, G_height)

                        #Bounding Box
                        Distance = 200
                        Tree_ShadowBox_Coords, Tree_ShadowBoxExtend_Coords = Get_TreeShadowBoundingBoxesCoords(TreeShadow, Distance)
                        TSBox = sample_polygon(Tree_ShadowBox_Coords)
                        TSBox = np.c_[TSBox, np.zeros(len(TSBox))]
                        TSBoxExtend = sample_polygon(Tree_ShadowBoxExtend_Coords)
                        TSBoxExtend = np.c_[TSBoxExtend, np.zeros(len(TSBoxExtend))]

                        #Get All buildings Falling under Extended Tree ShadowBox
                        All_Building_IdArr = Get_BuildingsUnderEffectOfShadow_Rapid(Building_coords_dict, TSBoxExtend)
                        #Get Secondary Building Lidar Points
                        All_Building_LidarPointsDict = Get_BuildingLidarPoints(All_Building_IdArr, Building_coords_dict, lasdf)
                        #Get ground elevation from building points
                        G_height = Tree_GroundZ #NOTE : Same as Tree_GroundZ
                        #Adjusting height of Buildings
                        All_Building_LidarPointsDict_HAdj = Set_BuildingLidarHeightAdjustment(All_Building_LidarPointsDict, G_height)

                        #Get Secondary Buildings
                        SecondaryBuilding_IdArr = [b for b in All_Building_IdArr if b not in PrimaryBuilding_IdArr]
                        #Get Secondary Building Coords
                        SecondaryBuilding_FootprintCoords = Get_BuildingFootprintSampledCoords(SecondaryBuilding_IdArr, Building_coords_dict)
                        #Get Secondary Building Lidar Points
                        SecondaryBuilding_LidarPointsDict = Get_BuildingLidarPoints(SecondaryBuilding_IdArr, Building_coords_dict, lasdf)
                        #Get ground elevation from building points
                        G_height = Tree_GroundZ #NOTE : Same as Tree_GroundZ
                        #Adjusting height of Buildings
                        SecondaryBuilding_LidarPointsDict_HAdj = Set_BuildingLidarHeightAdjustment(SecondaryBuilding_LidarPointsDict, G_height)

                        # Shade Metrics

                        # 1. Facade Shade - Points falling on the building footprint
                        # 2. In Shade - Points Fall within the shadow of Buildings not in the building footprint
                        # 3. Street Shade - Tree Shadow Points not overcasted by Anything
                        RemainingShadow_Points, FacadeShade_Points = Get_FacadeShadePoints(TreeShadow, PrimaryBuilding_IdArr, Building_coords_dict)
                        RemainingShadow_Points, InShade_Points = Get_InShadePoints(RemainingShadow_Points, All_Building_LidarPointsDict_HAdj, Az, Amp)
                        StreetShade_Points = RemainingShadow_Points

                        #print(len(FacadeShade_Points)/len(TreeShadow) , len(InShade_Points)/len(TreeShadow) ,len(StreetShade_Points)/len(TreeShadow))

                        Perc_Canopy_StreetShade = round(len(StreetShade_Points)/len(TreeShadow),3)
                        Perc_Canopy_FacadeShade = round(len(FacadeShade_Points)/len(TreeShadow),3)
                        Perc_Canopy_InShade = round(len(InShade_Points)/len(TreeShadow),3)

                        data = [date,ShadeYear, month, day, hour+4, minute, Az, Amp,
                                TreeId, round(Tree_Lat,6), round(Tree_Long,6), round(L,3), round(B,3), round(A,3), len(TreeShadow),
                                Perc_Canopy_StreetShade*100, Perc_Canopy_FacadeShade*100, Perc_Canopy_InShade*100,
                                A*Perc_Canopy_StreetShade, A*Perc_Canopy_FacadeShade, A*Perc_Canopy_InShade]
                    else:
                        data = [date,ShadeYear, month, day, hour+4, minute, Az, Amp,
                                TreeId, round(Tree_Lat,6), round(Tree_Long,6), 0, 0, 0, 0,
                                0, 0, 0,
                                0, 0, 0]

                    print(data)
                    writer.writerow(data)

        etime = time.time()

        logging.info("Shade Estimation Completed (in min): %d", (etime - stime)/60)

    except Exception as e:
        logging.error("ERROR : %s",str(e))

if __name__ == "__main__":
    #TEST SCRIPT - LOGS in TEST Directory - modify to Dataset paths with original files for deployment

    script_start_time = time.time()

    #Get las directory names from parent root folder
    p_rootpath = 'Datasets/Package_Generated/' #'/Volumes/Elements/TerraVide/Datasets/FTP_files/LiDAR/'
    year = 2017

    LAS_dirnames = Get_dirnames(p_rootpath)

    print("FileCount : "+str(len(LAS_dirnames))+" .las files found in path = "+p_rootpath )

    args = [(i, year,p_rootpath) for i in LAS_dirnames[:2]]#testing for first 2 files

    for arg_item in args:
        print(arg_item)

    with Pool(2) as p:

         p.starmap(ProcessShading,args)

    script_end_time = time.time()

    print("SCRIPT TOTAL TIME (in min): ",(script_end_time - script_start_time)/60)


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.7.1
#   kernelspec:
#     display_name: Manifold 2
#     language: python
#     name: manifold2
# ---

# # Null Data Embeddings
#
# This notebook creates the swarm file to compute embeddings using null data.

import os
import os.path as osp
from utils.data_info import PRJDIR, conda_loc, conda_env, SBJ_list, wl_sec, tr, ws_trs

# +
# Choose parameters for each technique
LE_k   = 80 # Best k-NN value for LE
LE_metric = 'correlation' # Best metric for LE

p      = 50 # Best perplexity value for TSNE
TSNE_metric = 'cosine' # Best metric for TSNE

UMAP_k = 160 # Best k-NN value for UMAP
UMAP_metric = 'euclidean' # Best metric for UMAP

n = 3 # Number of dimesnions to reduce
# -

# Create logs directory if doesnt already exist
os.system('if [ ! -d ../logs ]; then mkdir ../logs; fi')
os.system('if [ ! -d ../logs/Null_embedding.logs ]; then mkdir ../logs/Null_embedding.logs; fi')

# Create data directory if doesnt already exist
os.system('if [ ! -d ../derivatives ]; then mkdir ../derivatives; fi')
os.system('if [ ! -d ../derivatives/Null_Data ]; then mkdir ../derivatives/Null_Data; fi')

# Create SWARM file
drop = 'FullData'
os.system('echo "#swarm -f ./Null_embedding.SWARM.sh -g 30 -t 30 --time 8:00:00 --logdir ../logs/Null_embedding.logs" > ./Null_embedding.SWARM.sh')
for SBJ in SBJ_list:
    for data in ['ROI', 'SWC']: # Null data type (phase randomize ROI or randomize SWC connections)
        os.system('echo "export PRJDIR={PRJDIR} conda_loc={conda_loc} conda_env={conda_env} SBJ={SBJ} wl_sec={wl_sec} tr={tr} ws_trs={ws_trs} LE_k={LE_k} LE_metric={LE_metric} p={p} TSNE_metric={TSNE_metric} UMAP_k={UMAP_k} UMAP_metric={UMAP_metric} n={n} data={data} drop={drop}; sh ./Null_embedding.sh" >> ./Null_embedding.SWARM.sh'.format(PRJDIR=PRJDIR, conda_loc=conda_loc, conda_env=conda_env, SBJ=SBJ, wl_sec=wl_sec, tr=tr, ws_trs=ws_trs, LE_k=LE_k, LE_metric=LE_metric, p=p, TSNE_metric=TSNE_metric, UMAP_k=UMAP_k, UMAP_metric=UMAP_metric, n=n, data=data, drop=drop))


import time
import json
import cherrypy
import markdown
import urllib.parse
import functools
import collections
import hashlib

def jsonify(function):
    @functools.wraps(function)
    def wrapper(*args, **kwargs):
        cherrypy.response.headers["Content-Type"] = "application/json"
        return json.dumps(function(*args, **kwargs)).encode()
    return wrapper

class ChatMixIn (object):

    def create_new_room(self, room):
        self.db.request("POST", "/%s_rooms/" %
            (self.couchdb_prefix), body=room)
        self.db.request("PUT", "/%s_room_%s_messages" % (
            self.couchdb_prefix, room["num_id"]))
        self.db.request("PUT", "/%s_room_%s_messages/_design/messages/" % (
            self.couchdb_prefix, room["num_id"]), body={
                '_id': "_design/by_date",
                'language': 'javascript',
                "views": {
                   "by_date": {
                        "map": "function (doc) { emit(doc[\"date\"], doc); }"
                   }
               }
            })
        self.db.request("PUT", "/%s_room_%s_messages/_design/highlight/" % (
            self.couchdb_prefix, room["num_id"]), body={
                '_id': "_design/user_by_message",
                'language': 'javascript',
                "views": {
                   "user_by_message": {
                        "map": "function (doc) { emit(doc[\"num_id\"], doc[\"user\"]); }"
                   }
               }
            })
        return room["num_id"]

    def find_room(self, num_id):
        rooms = self.db.request("GET", "/%s_rooms/_design/"
            "by_num_id/_view/by_num_id/" % self.couchdb_prefix)
        s = [i for i in rooms['rows'] if i["key"] == num_id]
        return self.db.request("GET", "/%s_rooms/%s" %
            (self.couchdb_prefix, s[0]['id'])) if s else None

    def get_n_rooms(self):
        try:
            return len(self.db.request("GET", ("/%s_rooms/_design/by_msg_count/"
                "_view/by_msg_count/") % self.couchdb_prefix)["rows"])
        except KeyError:
            return 0

    @cherrypy.expose
    def post_message(self, room, message):
        if cherrypy.request.method != "POST":
            raise cherrypy.HTTPError(400)
        user = self.get_user()
        if not user:
            raise cherrypy.HTTPError(402)
        reply, highlight, prepend = None, [], None
        if message.startswith("::") and " " in message:
            reply = message.split()[0][2:]
            r_msg = self.find_message(room, reply)
            if r_msg:
                message = message.replace("::" + reply, "")
                highlight = [r_msg]
                p = self.find_user(r_msg)
                if p:
                    if p["name"]:
                        p = ("@" + p["name"].split()[0])
                    else:
                        p = "@User" + p["num_id"]
                    prepend = p
            else:
                reply = None
        for i in [j[1:] for j in message.lower().split() if j.startswith("@")]:
            u = list(self.find_user_by_name(''.join(j for j in i if j in
                "abcdefghijklmnopqrstuvwxyz")))
            if u:
                highlight.extend(u)
        self.update_user(user["_id"], {'last_seen_date': time.time()})
        message = {
            'raw': self.safe(message),
            'markdown': self.onebox_html(self.markdown(self.onebox(message))),
            'date': time.time(),
            'user': user["num_id"],
            'num_id': self.get_num_id(),
            'stars': [],
            'reply_to': reply,
            'highlight': highlight,
            'prepend': prepend,
        }
        self.inc_message_count(room)
        self.do_get_messages.cache_clear()
        self.db.request("POST", "/%s_room_%s_messages/" %
            (self.couchdb_prefix, room), body=message)

    def find_user_by_name(self, name):
        users = self.db.request("GET", "/%s_users/_design/"
            "by_name/_view/by_name/" % self.couchdb_prefix)
        name = ''.join(i for i in name.lower() if i in
            "abcdefghijklmnopqrstuvwxyz0123456789")
        if "rows" in users:
            for i in users["rows"]:
                n, i = i["key"], i["value"]
                if n and name.strip() == n.split()[0].lower().strip():
                    yield i

    def find_message(self, room, num_id):
        r = self.db.request("POST", ("/%s_room_%s_messages/_design/" +
            "highlight/_view/user_by_message/") %
            (self.couchdb_prefix, room))
        if "rows" in r:
            r = [i for i in r["rows"] if i["key"] == num_id]
            return r[0]["value"] if r else None

    def markdown(self, content):
        return markdown.markdown(content, safe_mode="escape")

    @jsonify
    @cherrypy.expose
    def read_messages(self, room="", last_atime=""):
        if self.get_user():
            self.active_users[room][self.get_user()["num_id"]] = time.time()
            return self.do_get_messages(room, last_atime)
        raise cherrypy.HTTPError(404)

    @functools.lru_cache(100)
    def do_get_messages(self, room, last_atime):
        if float(last_atime) <= 1:
            last_atime = time.time() - (60 * 60 * 24 * 7)
        user = self.get_user()
        if not user:
            raise cherrypy.HTTPError(404)
        room_data = self.find_room(room)
        if room_data["type"] == "private" and user["num_id"] not in room_data["users"]:
            raise cherrypy.HTTPError(404)
        data = self.db.request("GET", ("/%s_room_%s_messages/"
            "_design/messages/_view/by_date/?startkey=%s") % (
            self.couchdb_prefix, room, json.dumps(float(last_atime))))
        if "rows" in data:
            data = [i['value'] for i in data["rows"]]
        else:
            data = []
        data = list(sorted(data, key=lambda x: x["date"]))
        if len(data) >= 50:
            data = data[-50:]
        for i in data:
            i['user'] = self.find_user(i["user"])
            del i["user"]["email"]
            del i["user"]["identity"]
            del i["user"]["website"]
            del i["user"]["creation_date"]
            del i["user"]["openid_provider"]
            del i["user"]["country"]
            del i["user"]["_rev"]
            del i["user"]["messages"]
            del i["user"]["last_login_date"]
            del i["user"]["last_login_ip"]
            del i["user"]["about_me"]
            del i["user"]["owns_rooms"]
            del i["user"]["birthday"]
            del i["user"]["last_seen_date"]
            i["user"]["name"] = self.safe(i["user"]["name"])
        if data:
            last_atime = max(i["date"] for i in data)
        for i in data:
            i["nice_date"] = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime(i["date"]))
        return {'last_atime': last_atime, 'data': data}

    def inc_message_count(self, num_id):
        room = self.find_room(num_id)
        room["n_messages"] += 1
        self.db.request("PUT", "/%s_rooms/%s" % (
            self.couchdb_prefix, room["_id"]), body=room)

    @cherrypy.expose
    def star(self, room, message_id):
        user = self.get_user()
        if not user:
            raise cherrypy.HTTPError(404)
        d = self.db.request("GET", "/%s_room_%s_messages/%s" % (
            self.couchdb_prefix, room, message_id))
        if user["num_id"] not in d["stars"] and user["num_id"] != d["user"]:
            d["stars"].append(user["num_id"])
            self.get_starred.cache_clear()
        self.db.request("PUT", "/%s_room_%s_messages/%s" % (
            self.couchdb_prefix, room, message_id), body=d)
        cherrypy.response.headers["Content-Type"] = "application/json"
        return json.dumps(True).encode()

    @cherrypy.expose
    def unstar(self, room, message_id):
        user = self.get_user()
        if not user:
            raise cherrypy.HTTPError(404)
        d = self.db.request("GET", "/%s_room_%s_messages/%s" % (
            self.couchdb_prefix, room, message_id))
        if user["num_id"] in d["stars"]  and user["num_id"] != d["user"]:
            d["stars"].remove(user["num_id"])
            self.get_starred.cache_clear()
        self.db.request("PUT", "/%s_room_%s_messages/%s" % (
            self.couchdb_prefix, room, message_id), body=d)
        cherrypy.response.headers["Content-Type"] = "application/json"
        return json.dumps(True).encode()

    @jsonify
    @cherrypy.expose
    @functools.lru_cache(100)
    def get_starred(self, room=""):
        if self.get_user():
            last_atime = time.time() - (60 * 60 * 24)
            data = self.db.request("GET", ("/%s_room_%s_messages/"
                "_design/messages/_view/by_date/?startkey=%s") % (
                self.couchdb_prefix, room, json.dumps(float(last_atime))))
            if "rows" in data:
                data = [i['value'] for i in data["rows"]]
            else:
                data = []
            for i in data:
                i['user'] = self.find_user(i["user"])
                del i["user"]["email"]
                del i["user"]["identity"]
                del i["user"]["website"]
                del i["user"]["creation_date"]
                del i["user"]["openid_provider"]
                del i["user"]["country"]
                del i["user"]["_rev"]
                del i["user"]["messages"]
                del i["user"]["last_login_date"]
                del i["user"]["last_login_ip"]
                del i["user"]["about_me"]
                del i["user"]["owns_rooms"]
                del i["user"]["birthday"]
                del i["user"]["last_seen_date"]
                i["user"]["name"] = self.safe(i["user"]["name"])
            data = sorted(data, key=lambda x: len(x["stars"]), reverse=True)
            data = list(i for i in data if i["stars"])[:10]
            data = sorted(data, key=lambda x: x["date"], reverse=True)
            cherrypy.response.headers["Content-Type"] = "application/json"
            h = hashlib.md5("".join(i["_id"] + str(len(i["stars"]))
                for i in data).encode()).hexdigest()
            for i in data:
                i["nice_date"] = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime(i["date"]))
            return {'last_atime': last_atime, 'data': data, 'hash': h}
        raise cherrypy.HTTPError(404)

    @jsonify
    @cherrypy.expose
    def get_user_list(self, room):
        self.active_users[room] = {k: v for k, v in
            self.active_users[room].items() if time.time() - v < 10}
        r = list(self.find_user(i) for i in self.active_users[room].keys())
        for i in r:
            del i["email"]
            del i["identity"]
            del i["website"]
            del i["creation_date"]
            del i["openid_provider"]
            del i["country"]
            del i["_rev"]
            del i["messages"]
            del i["last_login_date"]
            del i["last_login_ip"]
            del i["about_me"]
            del i["owns_rooms"]
            del i["birthday"]
            del i["last_seen_date"]
            i["name"] = self.safe(i["name"])
        data = sorted(r, key=lambda x: x["num_id"] or "")
        h = hashlib.md5("".join(i["_id"]
            for i in data).encode()).hexdigest()
        return {"data": data, "hash": h}

    @cherrypy.expose
    def remove_user_from_room(self, room, num_id):
        room_num_id = room
        room = self.find_room(room)
        user = self.get_user()
        if user and user["_id"] == room["owner"]:
            if num_id in room["users"]:
                room["users"].remove(num_id)
                self.db.request("PUT", "/%s_rooms/%s" % (
                    self.couchdb_prefix, room["_id"]), body=room)
                self.do_get_messages.cache_clear()
        raise cherrypy.HTTPRedirect("/rooms/%s" % room_num_id)

    @jsonify
    @cherrypy.expose
    def add_user_to_room(self, room, user_url=""):
        room_num_id = room
        room = self.find_room(room)
        user = self.get_user()
        if user and user["_id"] == room["owner"]:
            u = urllib.parse.urlparse(user_url).path.split("/")
            if len(u) > 2 and u[1] == "users" and self.find_user(u[2]):
                if u[2] not in room["users"]:
                    room["users"].append(u[2])
                self.db.request("PUT", "/%s_rooms/%s" % (
                    self.couchdb_prefix, room["_id"]), body=room)
        raise cherrypy.HTTPRedirect("/rooms/%s" % room_num_id)


import numpy as np

import matplotlib.pyplot as plt

from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score

import catboost as cb

class CrossValidCatBoost:

    estimators = []

    def __init__(self, X, y, cat_features='auto'):

        self.X = X
        self.y = y
        self.cat_features = cat_features

        self.oof_preds = np.zeros(len(X))

    def train(self, params, n_splits=5, plot=True):

        X, y = self.X, self.y
        oof_preds = self.oof_preds

        train_auc, valid_auc = 0, 0
        colors = ['r', 'orange', 'g', 'c', 'b']

        plt.figure(figsize=(14, 8))
        plt.title('AUC')

        for i, (train_idx, valid_idx) in enumerate(KFold(n_splits=n_splits).split(X, y)):

            Xtrain, Xvalid = X.loc[train_idx], X.loc[valid_idx]
            ytrain, yvalid = y[train_idx], y[valid_idx]

            model = cb.CatBoostClassifier(**params)
            model.fit(
                Xtrain, ytrain,
                eval_set=[(Xvalid, yvalid)],
                cat_features=self.cat_features
            )

            oof_preds[valid_idx] = model.predict_proba(Xvalid)[:, 1]

            evals_train = model.evals_result_['learn']['AUC']
            evals_valid = model.evals_result_['validation']['AUC']

            train_auc += evals_train[-1]
            valid_auc += evals_valid[-1]

            if plot:
                plt.plot(evals_train, color=colors[i], label=f'fold = {i + 1} train', linestyle='--')
                plt.plot(evals_valid, color=colors[i], label=f'fold = {i + 1} validation')

            self.estimators.append(model)

        train_auc /= n_splits
        valid_auc /= n_splits

        print(f'\ntrain: mean: {round(train_auc, 4)}')
        print(f'valid: mean: {round(valid_auc, 4)}')

        plt.grid()
        plt.legend()

        return self.estimators

    def predict_proba(self, test):

        dtest = cb.Pool(test, cat_features=self.cat_features)

        preds = 0
        for estimator in self.estimators:
            preds += estimator.predict_proba(dtest)

        preds /= len(self.estimators)

        return preds

    def get_estimators(self):
        return self.estimators

    def get_oof_preds(self):
        return self.oof_preds


import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.utils as vutils
from torch.autograd import Variable
import itertools, time, os
from glob import glob
import models.cyclegan as cyclegan

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)

    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

class Trainer(object):
    def __init__(self, config, dataloader):
        self.config = config
        self.a_data_loader = dataloader[0]
        self.b_data_loader = dataloader[1]

        self.ngpu = int(config.ngpu)
        self.input_nc = int(config.input_nc)
        self.output_nc = int(config.output_nc)

        self.ngf = int(config.ngf)
        self.ndf = int(config.ndf)
        self.cuda = config.cuda

        self.num_steps = len(self.a_data_loader)
        self.batch_size = config.batch_size
        self.image_size = config.image_size

        self.lr = config.lr
        self.beta1 = config.beta1
        self.beta2 = config.beta2

        self.niter = config.niter
        self.decay_epoch = config.decay_epoch
        self.cycle_lambda = config.cycle_lambda

        self.outf = config.outf
        self.sample_step = config.sample_step
        self.checkpoint_step = config.checkpoint_step

        self.build_model()

        if self.cuda:
            self.netG_AB.cuda()
            self.netD_A.cuda()
            self.netG_BA.cuda()
            self.netD_B.cuda()

    def load_model(self):
        print("[*] Load models from {}...".format(self.outf))

        paths = glob(os.path.join(self.outf, 'net*.pth'))
        paths.sort()

        if len(paths) == 0:
            print("[!] No checkpoint found in {}...".format(self.outf))
            return

        epochs = [int(os.path.basename(path.split('.')[0].split('_')[-2].split('-')[-1])) for path in paths]
        self.start_epoch = str(max(epochs))
        steps = [int(os.path.basename(path.split('.')[0].split('_')[-1].split('-')[-1])) for path in paths]
        self.start_step = str(max(steps))

        G_AB_filename = '{}/netG_A_epoch-{}_step-{}.pth'.format(self.outf, self.start_epoch, self.start_step)
        G_BA_filename = '{}/netG_B_epoch-{}_step-{}.pth'.format(self.outf, self.start_epoch, self.start_step)
        D_A_filename = '{}/netD_A_epoch-{}_step-{}.pth'.format(self.outf, self.start_epoch, self.start_step)
        D_B_filename = '{}/netD_B_epoch-{}_step-{}.pth'.format(self.outf, self.start_epoch, self.start_step)

        self.netG_AB.load_state_dict(torch.load(G_AB_filename))
        self.netG_BA.load_state_dict(torch.load(G_BA_filename))
        self.netD_A.load_state_dict(torch.load(D_A_filename))
        self.netD_B.load_state_dict(torch.load(D_B_filename))

        print("[*] Model loaded: {}".format(G_AB_filename))

    def build_model(self):
        self.netG_AB = cyclegan._netG(self.ngpu, self.ngf,
                                      self.input_nc, self.output_nc)
        self.netG_AB.apply(weights_init)

        self.netG_BA = cyclegan._netG(self.ngpu, self.ngf,
                                      self.input_nc, self.output_nc)
        self.netG_BA.apply(weights_init)

        self.netD_A = cyclegan._netD(self.ngpu, self.ndf, self.input_nc)
        self.netD_A.apply(weights_init)

        self.netD_B = cyclegan._netD(self.ngpu, self.ndf, self.input_nc)
        self.netD_B.apply(weights_init)

        if self.config.model_path != '':
            self.load_model()

    def train(self):
        MSELoss = nn.MSELoss()
        L1loss = nn.L1Loss()

        if self.cuda:
            MSELoss.cuda()
            L1loss.cuda()

        # setup optimizer
        optimizerD_A = optim.Adam(self.netD_A.parameters(), lr=self.lr, betas=(self.beta1, self.beta2))
        optimizerD_B = optim.Adam(self.netD_B.parameters(), lr=self.lr, betas=(self.beta1, self.beta2))
        optimizerG = optim.Adam(itertools.chain(self.netG_AB.parameters(), self.netG_BA.parameters()), lr=self.lr, betas=(self.beta1, self.beta2))

        A_loader, B_loader = iter(self.a_data_loader), iter(self.b_data_loader)
        valid_x_A, valid_x_B = self._get_variable(A_loader.next()), self._get_variable(B_loader.next())

        vutils.save_image(valid_x_A.data, '{}/valid_x_A.png'.format(self.outf), nrow=10)
        vutils.save_image(valid_x_B.data, '{}/valid_x_B.png'.format(self.outf), nrow=10)

        start_time = time.time()
        for epoch in range(self.niter):
            if (epoch+1) > self.decay_epoch:
                optimizerD_A.param_groups[0]['lr'] -= self.lr / (self.niter - self.decay_epoch)
                optimizerD_B.param_groups[0]['lr'] -= self.lr / (self.niter - self.decay_epoch)
                optimizerG.param_groups[0]['lr'] -= self.lr / (self.niter - self.decay_epoch)

            for step in range(self.num_steps):
                try:
                    realA, realB = A_loader.next(), B_loader.next()
                except StopIteration:
                    A_loader, B_loader = iter(self.a_data_loader), iter(self.b_data_loader)
                    realA, realB = A_loader.next(), B_loader.next()
                if realA.size(0) != realB.size(0):
                    print("[!] Sampled dataset from A and B have different # of data. Try resampling...")
                    continue

                batch_size = realA.size(0)
                realA, realB = Variable(realA.cuda()), Variable(realB.cuda())
                ############################
                # (1) Update G network: minimize Lgan(MSE) + Lcycle(L1)
                ###########################
                for p in self.netD_A.parameters():
                    p.requires_grad = False
                for p in self.netD_B.parameters():
                    p.requires_grad = False

                self.netG_AB.zero_grad()
                self.netG_BA.zero_grad()

                # GAN loss: D_B(G_A(A))
                fakeB = self.netG_AB(realA)
                output = self.netD_B(fakeB)
                loss_G_A = MSELoss(output, Variable(torch.ones(output.size()).cuda()))

                # GAN loss: D_A(G_B(B))
                fakeA = self.netG_BA(realB)
                output = self.netD_A(fakeA)
                loss_G_B = MSELoss(output, Variable(torch.ones(output.size()).cuda()))

                # Forward cycle loss: A <-> G_B(G_A(A))
                cycleA = self.netG_BA(fakeB)
                loss_cycle_A = L1loss(cycleA, realA) * self.cycle_lambda

                # Backward cycle loss: B <-> G_A(G_B(B))
                cycleB = self.netG_AB(fakeA)
                loss_cycle_B = L1loss(cycleB, realB) * self.cycle_lambda

                # Combined Generator loss
                loss_G = loss_G_A + loss_G_B + loss_cycle_A + loss_cycle_B
                loss_G.backward()
                optimizerG.step()

                ############################
                # (2) Update D network: minimize LSGAN loss
                ###########################
                for p in self.netD_A.parameters():
                    p.requires_grad = True
                for p in self.netD_B.parameters():
                    p.requires_grad = True

                ## train D_A ##
                # train with real
                self.netD_A.zero_grad()

                D_A_real = self.netD_A(realA)
                loss_D_A_real = MSELoss(D_A_real, Variable(torch.ones(D_A_real.size()).cuda()))

                # train with fake
                D_A_fake = self.netD_A(fakeA.detach())
                loss_D_A_fake = MSELoss(D_A_fake, Variable(torch.zeros(D_A_fake.size()).cuda()))

                loss_D_A = loss_D_A_real + loss_D_A_fake
                loss_D_A.backward()
                optimizerD_A.step()

                ## train D_B ##
                # train with real
                self.netD_B.zero_grad()

                D_B_real = self.netD_B(realB)
                loss_D_B_real = MSELoss(D_B_real, Variable(torch.ones(D_B_real.size()).cuda()))

                # train with fake
                D_B_fake = self.netD_B(fakeB.detach())
                loss_D_B_fake = MSELoss(D_B_fake, Variable(torch.zeros(D_B_fake.size()).cuda()))

                loss_D_B = loss_D_B_real + loss_D_B_fake
                loss_D_B.backward()
                optimizerD_B.step()

                step_end_time = time.time()

                print('[%d/%d][%d/%d] - time_passed: %.2f, loss_D_A: %.3f, loss_D_B: %.3f, '
                      'loss_G_A: %.3f, loss_G_B: %.3f, loss_A_cycle: %.3f, loss_B_cycle: %.3f'
                      % (epoch, self.niter, step, len(self.a_data_loader), step_end_time - start_time,
                         loss_D_A, loss_D_B, loss_G_A, loss_G_B, loss_cycle_A, loss_cycle_B))

                if step % self.sample_step == 0:
                    fake = self.netG_AB(valid_x_A)
                    vutils.save_image(fake.data, '%s/AB_%03d_%d.png' % (self.outf, epoch, step),
                                      nrow=10)
                    fake = self.netG_BA(valid_x_B)
                    vutils.save_image(fake.data, '%s/BA_%03d_%d.png' % (self.outf, epoch, step),
                                      nrow=10)

                if step% self.checkpoint_step == 0 and step != 0:
                    torch.save(self.netG_AB.state_dict(), '%s/netG_A_epoch-%d_step-%s.pth' % (self.outf, epoch, step))
                    torch.save(self.netD_A.state_dict(), '%s/netD_A_epoch-%d_step-%s.pth' % (self.outf, epoch, step))
                    torch.save(self.netG_BA.state_dict(), '%s/netG_B_epoch-%d_step-%s.pth' % (self.outf, epoch, step))
                    torch.save(self.netD_B.state_dict(), '%s/netD_B_epoch-%d_step-%s.pth' % (self.outf, epoch, step))
                    print("Saved checkpoint")

    def _get_variable(self, inputs):
        if self.ngpu > 0:
            out = Variable(inputs.cuda())
        else:
            out = Variable(inputs)
        return out


# v2.5-beta
# Feature Update: Alias History. Load and save alias history as a .txt file

import random
import string
import tkinter as tk
import threading
import datetime
import re
from tkinter import filedialog, messagebox

class RandomEmailAliasGenerator:
    def __init__(self, master):
        """Initializes the GUI"""
        self.master = master
        self.master.title("R.E.A.G. ʕ º ᴥ ºʔ")

        # create alias history array
        self.alias_history = []

        # Set window size for responsive window
        self.master.rowconfigure((0,1,2), weight=1, minsize=30)
        self.master.columnconfigure((0), weight=1, minsize=30)

        # Create new frame for base input
        base_email_frame = tk.Frame(self.master, relief="groove")
        base_email_frame.grid(row=0, column=0, columnspan=1, padx=5, pady=3)

        # Create new frame for all buttons
        buttons_frame = tk.Frame(self.master, relief="groove")
        buttons_frame.grid(row=1, column=0, columnspan=1, padx=5, pady=3)

        # Create new frame for feeling lucky output
        feeling_lucky_output_frame = tk.Frame(self.master,  relief="groove")
        feeling_lucky_output_frame.grid(row=2, column=0, columnspan=1, padx=2, pady=3)

        # Base email label and input field in base_email_frame
        tk.Label(base_email_frame, text="Base Email:").grid(row=1, column=0, padx=5, pady=5)
        self.base_email = tk.Entry(base_email_frame, width=25)
        self.base_email.grid(row=2, column=0, padx=10, pady=5)
        self.base_email.focus()

        # Base alias label and input field
        tk.Label(base_email_frame, text="Base Alias: IE: 'TEST'").grid(row=3, column=0, padx=5, pady=5)
        self.base_alias = tk.Entry(base_email_frame, width=25)
        self.base_alias.grid(row=4, column=0, padx=5, pady=5)

         # Toggle for timestamp alias
        ts_toggle = tk.BooleanVar()
        checkbutton = tk.Checkbutton(base_email_frame, text=f"Timestamp Alias Override", variable=ts_toggle, onvalue=True, offvalue=False)
        checkbutton.grid(row=6, column=0, padx=5, pady=2)

        # Generated email alias label and output field
        tk.Label(buttons_frame, text="Magic Output:").grid(row=4, column=0, padx=5, pady=5)
        self.email_alias = tk.Entry(buttons_frame, width=25)
        self.email_alias.grid(row=5, column=0, padx=5, pady=5)

        # Random email button info
        tk.Label(buttons_frame, text="ie: jake+abc123@gmail.com").grid(row=1, column=0, columnspan=1, padx=5, pady=1)

        # Generate random email button
        self.generate_button = tk.Button(buttons_frame, text="Generate Random Alias", command=self.generate_random_email_alias)
        self.generate_button.grid(row=0, column=0, columnspan=1, padx=5, pady=5)

        # Generate email base alias button
        self.generate_alias_button = tk.Button(buttons_frame, text="Generate Using Base Alias", command=lambda: self.generate_base_alias_email_alias(ts_toggle))
        self.generate_alias_button.grid(row=2, column=0, columnspan=1, padx=5, pady=5)

        # Base alias email button info
        tk.Label(buttons_frame, text="ie: jake+TEST.abc123@gmail.com").grid(row=3, column=0, columnspan=1, padx=5, pady=5)

        # Copy to Clipboard button
        self.copy_button = tk.Button(buttons_frame, text="Copy to Clipboard", command=self.copy_to_clipboard)
        self.copy_button.grid(row=6, column=0, columnspan=2, padx=5, pady=5)

        # Label to display confirmation message
        self.confirmation_label = tk.Label(buttons_frame, text="waiting..")
        self.confirmation_label.grid(row=7, column=0, padx=5, pady=5)

        # Generate 10 aliases button info
        tk.Label(feeling_lucky_output_frame, text="Generate 10 aliases using base alias").grid(row=0, column=0, columnspan=1, padx=5, pady=5)

        # Generate 10 aliases button
        self.lucky_button = tk.Button(feeling_lucky_output_frame, text="Feeling Lucky", command=self.feeling_lucky)
        self.lucky_button.grid(row=1, column=0, columnspan=1, padx=5, pady=5)

        # Feeling lucky output field
        self.feeling_lucky_output = tk.Text(feeling_lucky_output_frame, height=10, width=30)
        self.feeling_lucky_output.grid(row=2, column=0, padx=5, pady=10)

        # Show Alias History button
        self.alias_history_button = tk.Button(self.master, text=">>\n\n\n>>\n\n\n>>", font="bold", command=self.show_alias_history)
        self.alias_history_button.grid(row=1, column=1, padx=5, pady=10)

    def show_alias_history(self):

        # Check if history frame already exists
        if hasattr(self, 'history_frame'):
            self.toggle_history()
        else:
            # Update History Button to toggle the visibility of the history section
            self.alias_history_button.config(text="<<\n\n\n<<\n\n\n<<")

            # Create a Frame for the collapsible history section
            self.history_frame = tk.Frame(self.master, relief="groove")
            self.history_frame.grid(row=0, column=2, rowspan=4, padx=2, pady=3, sticky="nsew")

            # Configure the row and column to fill vertical space
            self.history_frame.grid_rowconfigure(1, weight=1)
            self.history_frame.grid_columnconfigure(0, weight=1)

             # Create new frame for alias history save/load buttons in history frame
            history_function_frame = tk.Frame(self.history_frame, relief="groove")
            history_function_frame.grid(row=0, column=0, columnspan=1, padx=2, pady=3)

            # Create a Text widget to display the history
            self.history_text = tk.Text(self.history_frame, height=15, width=75)
            self.history_text.grid(row=1, column=0, columnspan=1, padx=2, pady=3, sticky="nsew")

            # Save alias history button
            self.save_history_button = tk.Button(history_function_frame, text="Save Alias History", command=self.save_alias_history)
            self.save_history_button.grid(row=0, column=0, padx=5, pady=5)

            # Load alias history button
            self.load_history_button = tk.Button(history_function_frame, text="Load Alias History", command=self.load_alias_history)
            self.load_history_button.grid(row=0, column=1, padx=5, pady=5)

            # Insert the history into the Text widget
            for alias in self.alias_history:
                self.history_text.insert(tk.END, alias + "\n")

    def update_history_display(self):
        if hasattr(self, 'history_text'):  # Check if history_text attribute exists
            self.history_text.delete("1.0", tk.END)
            for alias in self.alias_history:
                self.history_text.insert(tk.END, alias + "\n")

    def toggle_history(self):
        # Toggle the visibility of the history section
        if self.history_frame.winfo_ismapped():
            self.history_frame.grid_forget()
            self.alias_history_button.config(text=">>\n\n\n>>\n\n\n>>")
        else:
            self.history_frame.grid(row=0, column=2, rowspan=3, padx=2, pady=3, sticky="nsew")
            self.alias_history_button.config(text="<<\n\n\n<<\n\n\n<<")

    def save_alias_history(self):
        # Prompt the user for a custom file name
        file_name = tk.filedialog.asksaveasfilename(defaultextension=".txt", filetypes=[("Text files", "*.txt")])

        if file_name:
            with open(file_name, "w") as file:
                # Write the alias history to the file
                for alias in self.alias_history:
                    file.write(alias + "\n")

            # Display a confirmation message
            self.confirmation_label.config(text="Alias History Saved", fg="White", bg="Green")
            # Reset label text after 2 seconds
            t = threading.Timer(2.0, self.reset_confirmation)
            t.start()

    def load_alias_history(self):
        if messagebox.askyesno("Warning", "Loading a file will overwrite the current alias history. Do you want to proceed?"):
            file_name = filedialog.askopenfilename(defaultextension=".txt", filetypes=[("Text files", "*.txt")])
            if file_name:
                self.alias_history.clear()  # Clear alias history array
                with open(file_name, 'r') as file:
                    content = file.read()
                    lines = content.splitlines()
                    self.confirmation_label.config(text="Alias History Loaded", fg="White", bg="Green")
                    if self.history_text.get("1.0", tk.END) != "\n": # check for blank history
                        self.history_text.delete("1.0", tk.END)
                    for line in lines:
                        self.alias_history.append(line)
                        self.history_text.insert(tk.END, line + "\n")

    def is_valid_base_email(self, email):
        # Regular expression for email validation
        if email != '':
            pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
            return bool(re.match(pattern, email))
        else: return False

    def is_base_alias_not_null(self, alias):
        # Check if the base alias is not null using regular expression
        if alias != '':
            pattern = r'^[a-zA-Z0-9._%-]*$'
            return bool(re.match(pattern, alias))
        else: return False

    def generate_random_email_alias(self, copy_to_clipboard=True):
        """Generates a random email alias based on a base email. 6 chars"""
        base_email = self.base_email.get()
        now = datetime.datetime.utcnow()
        timestamp = now.strftime("%y-%m-%d-%H.%M.%S")
        # check for valid base email
        if self.is_valid_base_email(base_email):
            username, domain = base_email.split('@')
            random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=6))
            self.email_alias.delete(0, tk.END)
            self.email_alias.insert(0, f"{username}+{random_string}@{domain}")

            if copy_to_clipboard:
                self.copy_to_clipboard()
                self.generate_click_confirmation(self.generate_button)
                # Add alias to history
                self.alias_history.append(self.email_alias.get() + " | Timestamp: " + timestamp)

                 # Check if the history window exists before updating the display
                if hasattr(self, 'history_text'):
                    if self.history_text.winfo_exists():  # Check if history_text widget exists and is open
                        self.update_history_display()
                    else:
                        delattr(self, 'history_text')  # Remove the attribute if the history window is closed
        else:
            self.email_alias.delete(0, tk.END)
            self.email_alias.insert(0, f"ENTER A VALID BASE EMAIL")
            self.error_confirmation()

    def generate_base_alias_email_alias(self, ts_toggle, copy_to_clipboard=True):
        """Generates a random email alias based on a base email and base alias. 6 chars"""
        base_email = self.base_email.get()
        base_alias = self.base_alias.get()
        now = datetime.datetime.utcnow()
        timestamp = now.strftime("%y-%m-%d-%H.%M.%S")
        # check for base alias not null
        if self.is_base_alias_not_null(base_alias):
            # Check for valid base email
            if self.is_valid_base_email(base_email):
                username, domain = base_email.split('@')

                if ts_toggle.get():
                    random_string = ''.join(timestamp)
                    self.email_alias.delete(0, tk.END)
                    self.email_alias.insert(0, f"{username}+{base_alias}.{random_string}@{domain}")
                else:
                    random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=6))
                    self.email_alias.delete(0, tk.END)
                    self.email_alias.insert(0, f"{username}+{base_alias}.{random_string}@{domain}")

                if copy_to_clipboard:
                    self.copy_to_clipboard()
                    self.generate_click_confirmation(self.generate_alias_button)
                    self.alias_history.append(self.email_alias.get() + " | Timestamp: " + timestamp)
                    # Update History Display
                    self.update_history_display()
            else:
                # Display error message
                self.email_alias.delete(0, tk.END)
                self.email_alias.insert(0, f"ENTER A VALID BASE EMAIL")
                self.error_confirmation()
        else:
            # Display error message
            self.email_alias.delete(0, tk.END)
            self.email_alias.insert(0, f"ENTER A VALID BASE ALIAS")
            self.error_confirmation()

    def feeling_lucky(self):
        """Generates 10 random email aliases at once using base_alias"""
        base_email = self.base_email.get()
        base_alias = self.base_alias.get()
        now = datetime.datetime.utcnow()
        timestamp = now.strftime("%y-%m-%d-%H.%M.%S")
        # check for base alias not null
        if self.is_base_alias_not_null(base_alias):
            # Check for valid base email
            if self.is_valid_base_email(base_email):
                username, domain = base_email.split('@')
                random_aliases = [f"{username}+{base_alias}.{''.join(random.choices(string.ascii_letters + string.digits, k=6))}@{domain}" for i in range(10)]
                self.feeling_lucky_output.delete('1.0', tk.END)
                self.feeling_lucky_output.insert('1.0', '\n'.join(random_aliases))
                # Click confirmation prompt
                self.generate_click_confirmation(self.lucky_button)
                index='1.0'
                for alias in random_aliases:
                    self.alias_history.append(alias + " | Timestamp: " + timestamp + " (FL)")
                    index = self.feeling_lucky_output.index(f"{index}+1c")  # Increment index to the next line
                # Update History Display after appending lucky output
                self.update_history_display()
            else:
                # Display error message
                self.feeling_lucky_output.delete('1.0', tk.END)
                self.feeling_lucky_output.insert('1.0', f"ENTER A VALID BASE EMAIL")
                self.error_confirmation()
        else:
            # Display error message
            self.feeling_lucky_output.delete('1.0', tk.END)
            self.feeling_lucky_output.insert('1.0', f"ENTER A VALID BASE ALIAS")
            self.error_confirmation()

    def copy_to_clipboard(self):
        """Copies the generated email alias to the clipboard"""
        if self.is_valid_base_email(self.email_alias.get()):
            self.email_alias.clipboard_clear()  # Clear the clipboard
            self.email_alias.clipboard_append(self.email_alias.get())  # Append the email alias to the clipboard
            self.email_alias.update()  # Update the clipboard
            self.confirmation_label.config(text="EMAIL COPIED", fg="White", bg="Green")
            # Reset label text after 2 seconds
            t = threading.Timer(2.0, self.reset_confirmation)
            t.start()

    def generate_click_confirmation(self, button):
        """When button clicked display confirmation"""
        button.config(text="ʕ º ᴥ ºʔ", fg="White", bg="Blue")
        # Reset label text after 0.1 second
        self.master.after(100, self.reset_button, button)

    def error_confirmation(self):
        """When invalid base email or null base alias display error"""
        self.confirmation_label.config(text="(ノಠ益ಠ)ノ彡┻━┻", fg="White", bg="Red")
        t = threading.Timer(1.0, self.reset_confirmation)
        t.start()

    def reset_button(self, button):
        """Reset generate email button text"""
        if button == self.generate_button:
            button.config(text="Generate Random Alias", fg= "Black", bg= "White")
        elif button == self.generate_alias_button:
            button.config(text="Generate Using Base Alias", fg= "Black", bg= "White")
        elif button == self.lucky_button:
            button.config(text="Feeling Lucky", fg= "Black", bg= "White")
        elif button == self.copy_button:
            button.config(text="Copy to Clipboard", fg= "Black", bg= "White")
        else:
            return

    def reset_confirmation(self):
        """Reset confirmation label text"""
        self.confirmation_label.config(text="waiting..")

def main():
    root = tk.Tk()
    app = RandomEmailAliasGenerator(root)
    root.mainloop()

if __name__ == '__main__':
    main()


# coding=utf-8
#

import unittest2
from utils.dframe import dictListToDataFrame

class TestDFrame(unittest2.TestCase):

    def __init__(self, *args, **kwargs):
        super(TestDFrame, self).__init__(*args, **kwargs)

    def testDictListToDataFrame(self):
        # just file name
        df = dictListToDataFrame([ {'x': 88, 'y': 99}
                                 , {'x': 101, 'y': 120}
                                 , {'x': 111, 'y': 132}
                                 ])

        self.assertEqual(3, len(df))
        self.assertEqual(99, df.iloc[0]['y'])
        self.assertEqual(101, df.iloc[1]['x'])
        self.assertEqual(132, df.iloc[2]['y'])