# encoding:utf8
import os
import codecs
import pickle
import itertools
import tensorflow as tf
import numpy as np
from model.model import Model
from model_helper.loader import load_sentences, update_tag_scheme
from model_helper.loader import char_mapping, tag_mapping
from model_helper.loader import augment_with_pretrained, prepare_dataset
from model_helper.utils import get_logger, make_path, clean, create_model, save_model, config_model, evaluate
from model_helper.utils import print_config, save_config, load_config, test_ner
from data_preprocess.data_utils import load_word2vec, create_input, input_from_line, BatchManager

flags = tf.app.flags
flags.DEFINE_boolean("clean",       False,      "clean train folder")
flags.DEFINE_boolean("train",       False,      "Whether train the model")

# configurations for the model
flags.DEFINE_integer("seg_dim",     20,         "Embedding size for segmentation, 0 if not used")
flags.DEFINE_integer("char_dim",    100,        "Embedding size for characters")
flags.DEFINE_integer("num_filter",    100,      "num of filters in IDCNN")
flags.DEFINE_string("tag_schema",   "iobes",    "tagging schema iobes or iob")

# configurations for training
flags.DEFINE_float("clip",          5,          "Gradient clip")
flags.DEFINE_float("dropout",       0.5,        "Dropout rate")
flags.DEFINE_integer("batch_size",    20,         "batch size")
flags.DEFINE_float("lr",            0.001,      "Initial learning rate")
flags.DEFINE_string("optimizer",    "adam",     "Optimizer for training")
flags.DEFINE_boolean("pre_emb",     True,       "Wither use pre-trained embedding")
flags.DEFINE_boolean("zeros",       False,      "Wither replace digits with zero")
flags.DEFINE_boolean("lower",       True,       "Wither lower case")

flags.DEFINE_integer("max_epoch",   100,        "maximum training epochs")
flags.DEFINE_integer("steps_check", 100,        "steps per checkpoint")
flags.DEFINE_string("script",       "conlleval",    "Evaluation script")
flags.DEFINE_string("ckpt_path",    "ckpt",      "Path to save model")
flags.DEFINE_string("summary_path", "summary",      "Path to store summaries")
flags.DEFINE_string("result_path",  "result",       "Path for results")
flags.DEFINE_string("log_file",     "train.log",    "File for log")
flags.DEFINE_string("map_file",     os.path.join("config", "maps.pkl"),     "File for maps")
flags.DEFINE_string("vocab_file",   os.path.join("config", "vocab.json"),   "File for vocab")
flags.DEFINE_string("config_file",  os.path.join("config", "config_file"),  "File for config")
flags.DEFINE_string("emb_file",     os.path.join("config", "vec.txt"),  "Path for pre_trained embedding")
flags.DEFINE_string("train_file",   os.path.join("data", "example.train"),  "Path for train data")
flags.DEFINE_string("dev_file",     os.path.join("data", "example.dev"),    "Path for dev data")
flags.DEFINE_string("test_file",    os.path.join("data", "example.test"),   "Path for test data")

FLAGS = tf.app.flags.FLAGS
assert FLAGS.clip < 5.1, "gradient clip should't be too much"
assert 0 <= FLAGS.dropout < 1, "dropout rate between 0 and 1"
assert FLAGS.lr > 0, "learning rate must larger than zero"
assert FLAGS.optimizer in ["adam", "sgd", "adagrad"]

def train():
    # load data sets
    # [[[出,O]], [[前,O]],...]
    train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
    dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
    test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)

    # Use selected tagging scheme (IOB / IOBES)
    update_tag_scheme(train_sentences, FLAGS.tag_schema)
    update_tag_scheme(test_sentences, FLAGS.tag_schema)

    # create maps if not exist
    if not os.path.isfile(FLAGS.map_file):
        # create dictionary for word
        if FLAGS.pre_emb:
            dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
            dico_chars, char_to_id, id_to_char = augment_with_pretrained(
                dico_chars_train.copy(),
                FLAGS.emb_file,
                list(itertools.chain.from_iterable(
                    [[w[0] for w in s] for s in test_sentences])
                )
            )
        else:
            _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)

        # Create a dictionary and a mapping for tags
        _t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
        with open(FLAGS.map_file, "wb") as f:
            pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
    else:
        with open(FLAGS.map_file, "rb") as f:
            char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)

    # prepare data, get a collection of list containing index
    train_data = prepare_dataset(
        train_sentences, char_to_id, tag_to_id, FLAGS.lower
    )
    dev_data = prepare_dataset(
        dev_sentences, char_to_id, tag_to_id, FLAGS.lower
    )
    test_data = prepare_dataset(
        test_sentences, char_to_id, tag_to_id, FLAGS.lower
    )
    print("%i / %i / %i sentences in train / dev / test." % (
        len(train_data), len(dev_data), len(test_data)))

    train_manager = BatchManager(train_data, FLAGS.batch_size)
    dev_manager = BatchManager(dev_data, 100)
    test_manager = BatchManager(test_data, 100)
    # make path for store log and model if not exist
    make_path(FLAGS)
    if os.path.isfile(FLAGS.config_file):
        config = load_config(FLAGS.config_file)
    else:
        config = config_model(char_to_id, tag_to_id, FLAGS)
        save_config(config, FLAGS.config_file)

    logger = get_logger(os.path.join('log', FLAGS.log_file))
    print_config(config, logger)

    # limit GPU memory
    tf_config = tf.ConfigProto()
    tf_config.gpu_options.allow_growth = True
    steps_per_epoch = train_manager.len_batch_data

    with tf.Session(config=tf_config) as sess:
        model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger)
        logger.info("start training")

        # 配置 tensorboard
        # 跟踪节点
        tf.summary.scalar('loss', model.loss)

        # summary节点合并
        merged_summary = tf.summary.merge_all()

        # 构造summary文件写入器
        # 接受一个log的目录作为保存文件的路径。log目录如果不存在，会被程序自动创建。
        # 通常训练集日志和验证集日志分开存放，分别构造各自的summary文件写入器即可
        # 添加一个参数sess.graph,绘制静态的graph（计算图），否则绘制动态数据
        writer = tf.summary.FileWriter(FLAGS.summary_path)

        loss = []
        for i in range(100):
            for batch in train_manager.iter_batch(shuffle=True):
                step, batch_loss, feed_dict = model.run_step(sess, True, batch)
                loss.append(batch_loss)

                if i % 5 == 0:  # 每5次将训练结果写入tensorboard scalar
                    summary = sess.run(merged_summary, feed_dict=feed_dict)  # 运行summary节点

                    # 向记录器添加，除了接受summary节点的运行输出值，还接受一个global_step参数来作为序列号
                    writer.add_summary(summary, i)

                if step % FLAGS.steps_check == 0:
                    iteration = step // steps_per_epoch + 1
                    logger.info("iteration:{} step:{}/{}, "
                                "NER loss:{:>9.6f}".format(
                        iteration, step%steps_per_epoch, steps_per_epoch, np.mean(loss)))
                    loss = []

            best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger, FLAGS)
            if best:
                save_model(sess, model, FLAGS.ckpt_path, logger)
            evaluate(sess, model, "test", test_manager, id_to_tag, logger, FLAGS)

def evaluate_line():
    config = load_config(FLAGS.config_file)
    logger = get_logger(FLAGS.log_file)
    # limit GPU memory
    tf_config = tf.ConfigProto()
    tf_config.gpu_options.allow_growth = True
    with open(FLAGS.map_file, "rb") as f:
        char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
    with tf.Session(config=tf_config) as sess:
        model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger)
        while True:
                line = input("请输入测试句子:")
                result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag)
                print(result)

def main(_):

    if FLAGS.train:
        if FLAGS.clean:
            clean(FLAGS)
        train()
    else:
        evaluate_line()

if __name__ == "__main__":
    tf.app.run(main)


from question_bank import Question
from data import question_data
from quiz_brain import QuizBrain

question_bank=[]
for question in question_data:
    text=question["text"]
    answer=question["answer"]
    current_question=Question(text,answer)
    question_bank.append(current_question)
quiz=QuizBrain(question_bank)
while quiz.any_more_ques():
    quiz.next_question()
print("the game is over")
print(f"your final score is {quiz.score}/{quiz.question_number}")


# import required files
import json
import os
import argparse
# import load_data/write_data from LoadWrite
from LoadWrite import load_data, write_data

# Parse the arguments given to DataPrep
parser = argparse.ArgumentParser()
parser.add_argument("path_to_json_files")
parser.add_argument("output_data_path")
args = parser.parse_args()

# USER: format the data such that it is in the form: List[articles: str]

# get all JSON file names as a list
json_file_names = [filename for filename in os.listdir(args.path_to_json_files) if filename.endswith('.json')]

data = []
# get all data into one big list
for json_file_name in json_file_names:
    with open(os.path.join(args.path_to_json_files, json_file_name)) as json_file:
        json_text = json.load(json_file)
        data.append(json_text['text'])

# write the data to file
write_data(args.output_data_path, data)


import cv2 as cv
import numpy as np
import matplotlib.pyplot as plt
import skimage as sk
from skimage.segmentation import clear_border
from customFunctions import erode

def resize_image_and_show(image):
    preRezised = image
    scale_percent = 20
    # width = int(preRezised.shape[1] * scale_percent / 100)
    # height = int(preRezised.shape[0] * scale_percent / 100)
    width = int(1000)
    height = int(1000)
    dim = (width, height)
    Resized = cv.resize(preRezised, dim, interpolation=cv.INTER_AREA)

    # Show scaled image
    # cv.imshow("Resized", Resized)
    return Resized
    # cv.waitKey(0)
    # cv.destroyAllWindows()

# Own implementation of matlab find function to find all white corner pixels in image
def find_coords_box(image):
    c1, c2, c3, c4 = 0, 0, 0, 0
    list_of_white_pixels = []

    # make multithreaded with image sections

    for y in range(0,image.shape[0],5):
        for x in range(0,image.shape[1],5):
            if image[y,x] == 255:
                list_of_white_pixels.append([x,y])

    # Find top left
    c1 = c2 = c3 = c4 = list_of_white_pixels[0]

    for i in range(len(list_of_white_pixels)):
        x = list_of_white_pixels[i][0]
        y = list_of_white_pixels[i][1]
        if (y + x) < (c1[1] + c1[0]):
            c1 = list_of_white_pixels[i]
        if (y - x) < (c2[1] - c2[0]):
            c2 = list_of_white_pixels[i]
        if (y + x) > (c3[1] + c3[0]):
            c3 = list_of_white_pixels[i]
        if (y - x) > (c4[1] - c4[0]):
            c4 = list_of_white_pixels[i]

    return c1, c2, c3, c4

def process_image(image):

    grayscale = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
    # Apply threshold
    # ret, thresh = cv.threshold(grayscale, 45, 255, 0)
    ret, thresh = cv.threshold(grayscale,0,255,cv.THRESH_BINARY+cv.THRESH_OTSU)
    thresh = cv.bitwise_not(thresh)

    # open thresh
    kernel = np.ones((3,3), np.uint8)
    dilated = cv.dilate(thresh, kernel, iterations=3)

    # dilate thresh
    eroded = erode(dilated, 3, iterations=3)

    eroded = clear_border(eroded)

    # create big kernel
    kernel = np.ones((11,11), np.uint8)

    # dilate eroded 10 times
    dilated = cv.dilate(eroded, kernel, iterations=6)

    nb_component, output, stats, centroids = cv.connectedComponentsWithStats(dilated, connectivity=8)

    sizes = stats[1:, -1]; nb_component = nb_component - 1

    min_size = 1000000

    img2 = np.zeros((output.shape))

    for i in range(0, nb_component):
        if sizes[i] >= min_size:
            img2[output == i + 1] = 255

    new = cv.bitwise_and(np.uint8(img2), eroded)

    originalPoints = find_coords_box(img2)

    Left = int((originalPoints[0][0] + originalPoints[3][0]) / 2 + 1000)
    Right = int((originalPoints[1][0] + originalPoints[2][0]) / 2 + 1000)
    Top = int((originalPoints[0][1] + originalPoints[1][1]) / 2 + 1000)
    Bottom = int((originalPoints[2][1] + originalPoints[3][1]) / 2 + 1000)

    pts1 = np.float32([originalPoints[0], originalPoints[1], originalPoints[2], originalPoints[3]])
    pts2 = np.float32([(Left, Top),(Right, Top),(Right, Bottom),(Left, Bottom)])

    M = cv.getPerspectiveTransform(pts1,pts2)

    warped = cv.warpPerspective(new,M,(Right, Bottom))
    warped2 = cv.warpPerspective(image,M,(image.shape[1] * 2, image.shape[0] * 2))
    ret, IM = cv.invert(M)

    # crop to only the box
    warped = warped[Top:Bottom, Left:Right]

    # cv.imshow("warped", resize_image_and_show(warped2))

    # make warped black and white
    ret, warped = cv.threshold(warped, 0, 255, cv.THRESH_BINARY + cv.THRESH_OTSU)

    mazeHeight, mazeWidth = warped.shape
    approxWallWidth = mazeWidth // 21
    approxWallHeight = mazeHeight // 21
    print(warped.shape)

    warped = ~warped[0+approxWallHeight:mazeHeight-approxWallHeight, 0+approxWallWidth:mazeWidth-approxWallWidth]
    print(warped.shape)
    # look for white pixel in top row
    # if not found, rotate image 90 degrees
    rotated = 0
    for x in range(warped.shape[1]):
        if warped[0,x] == 255:
            break
    else:
        warped = np.rot90(warped)
        print("Rotated 90 degrees")
        rotated = 1

    solved = solve_maze(warped)

    if rotated:
        solved = np.rot90(solved, 3)

    combined = cv.bitwise_xor(warped2[Top+approxWallHeight:Bottom-approxWallHeight, Left+approxWallWidth:Right-approxWallWidth], solved)
    warped2[Top+approxWallHeight:Bottom-approxWallHeight, Left+approxWallWidth:Right-approxWallWidth] = combined
    warped3 = cv.warpPerspective(warped2,IM,(image.shape[1], image.shape[0]))

    return warped3

def find_start(maze):
    for x in range(maze.shape[1]):
        if maze[0,x] == 255:
            return (x,0)

def find_end(maze):
    for x in range(maze.shape[1]):
        if maze[maze.shape[0]-1,x] == 255:
            return (x,maze.shape[0]-1)

def find_path(maze, start, end):
    UP      = 1
    RIGHT   = 2
    DOWN    = 3
    LEFT    = 4
    WALL    = 0
    PATH    = 255

    PENCIL_X = start[0]
    PENCIL_Y = start[1]

    PENCIL_DIRECTION = DOWN

    # create a new canvas
    canvas = np.zeros((maze.shape[0], maze.shape[1], 3), np.uint8)
    print("Canvas Dimentions: ", canvas.shape)
    print("Maze Dimentions: ", maze.shape)

    while (PENCIL_X,PENCIL_Y) != end:

        PixelLeft = maze[PENCIL_Y, PENCIL_X - 1]
        PixelRight = maze[PENCIL_Y, PENCIL_X + 1]
        PixelUp = maze[PENCIL_Y - 1, PENCIL_X]
        PixelDown = maze[PENCIL_Y + 1, PENCIL_X]

        # Switch case for direction
        if PENCIL_DIRECTION == UP:
            if PixelUp == WALL and PixelLeft == WALL and PixelRight == WALL:
                PENCIL_DIRECTION = DOWN
            elif PixelUp == PATH and PixelRight == WALL:
                PENCIL_DIRECTION = UP
            elif PixelUp == WALL and PixelRight == WALL:
                PENCIL_DIRECTION = LEFT
            elif PixelUp == PATH and PixelRight == PATH:
                PENCIL_DIRECTION = RIGHT
            elif PixelUp == WALL and PixelRight == PATH:
                PENCIL_DIRECTION = RIGHT

        elif PENCIL_DIRECTION == RIGHT:
            if PixelRight == WALL and PixelUp == WALL and PixelDown == WALL:
                PENCIL_DIRECTION = LEFT
            elif PixelRight == PATH and PixelDown == WALL:
                PENCIL_DIRECTION = RIGHT
            elif PixelRight == WALL and PixelDown == WALL:
                PENCIL_DIRECTION = UP
            elif PixelRight == PATH and PixelDown == PATH:
                PENCIL_DIRECTION = DOWN
            elif PixelRight == WALL and PixelDown == PATH:
                PENCIL_DIRECTION = DOWN

        elif PENCIL_DIRECTION == DOWN:
            if PixelDown == PATH and PixelLeft == WALL:
                PENCIL_DIRECTION = DOWN
            elif PixelDown == WALL and PixelLeft == WALL:
                if PixelRight == WALL:
                    PENCIL_DIRECTION = UP
                else:
                    PENCIL_DIRECTION = RIGHT
            elif PixelDown == PATH and PixelLeft == PATH:
                PENCIL_DIRECTION = LEFT
            elif PixelDown == WALL and PixelLeft == PATH:
                PENCIL_DIRECTION = LEFT

        elif PENCIL_DIRECTION == LEFT:
            if PixelLeft == WALL and PixelUp == WALL and PixelDown == WALL:
                PENCIL_DIRECTION = RIGHT
            elif PixelLeft == PATH and PixelUp == WALL:
                PENCIL_DIRECTION = LEFT
            elif PixelLeft == WALL and PixelUp == WALL:
                PENCIL_DIRECTION = DOWN
            elif PixelLeft == PATH and PixelUp == PATH:
                PENCIL_DIRECTION = UP
            elif PixelLeft == WALL and PixelUp == PATH:
                PENCIL_DIRECTION = UP
        if(maze[PENCIL_Y+1, PENCIL_X+1] == PATH):
            canvas[PENCIL_Y:PENCIL_Y + 10, PENCIL_X:PENCIL_X + 10, [0,1]] = PATH
            canvas[PENCIL_Y:PENCIL_Y + 10, PENCIL_X:PENCIL_X + 10, [2]] = WALL
        else:
            canvas[PENCIL_Y:PENCIL_Y + 10, PENCIL_X:PENCIL_X + 10, [0,1]] = WALL
            canvas[PENCIL_Y:PENCIL_Y + 10, PENCIL_X:PENCIL_X + 10, [2]] = PATH

        # Move the pencil
        if PENCIL_DIRECTION == UP:
            PENCIL_Y -= 1
        elif PENCIL_DIRECTION == RIGHT:
            PENCIL_X += 1
        elif PENCIL_DIRECTION == DOWN:
            PENCIL_Y += 1
        elif PENCIL_DIRECTION == LEFT:
            PENCIL_X -= 1

    # add two dimentions to maze
    maze = np.expand_dims(maze, axis=2)

    return canvas

def solve_maze(maze):
    # find start and end points
    start = find_start(maze)
    print("Start: ", start)
    end = find_end(maze)
    print("End: ", end)

    # find path
    path = find_path(maze, start, end)

    # return path
    return path

def run_maze_solver(image):
    # Load image

    processed_image = process_image(image)

    return cv.cvtColor(processed_image, cv.COLOR_BGR2RGB)

if __name__ == "__main__":
    image = cv.imread("maze2.jpg")
    run_maze_solver(image)
    cv.waitKey(0)
    cv.destroyAllWindows()


from datetime import datetime
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import generics
from .models import TrackInformation
from .models import ArtistInformation
from symphony.serializer import TrackInformationSerializer
from symphony.serializer import ArtistInformationSerializer

class ArtistTrackInfoDateFilter(generics.ListAPIView):
    """This class handels the GET request for date and artist filter"""
    # TODO: Try to integrate with TrackInformationViews
    serializer_class = TrackInformationSerializer

    def get_queryset(self):
        """This method is called with the URL for tracks is passed with ArtistId and Release date."""
        artistId = self.kwargs.get('artistId')
        releaseDate = self.kwargs.get('releaseDate')
        queryset = TrackInformation.objects.all()
        da = datetime.strptime(releaseDate, '%d-%m-%Y')
        filter_query = queryset.filter(artistId__artistId=artistId, releaseDate__lte=da)
        return filter_query

class ArtistInformationQuerySet(generics.ListAPIView):
    """This class is specifically designed to handel GET request for artist ID. For E.G if there is Request for artistID
    then this view class will be called."""
    serializer_class = TrackInformationSerializer

    def get_queryset(request):
        artist_id = request.kwargs.get('artistId')
        if not artist_id:
            return Response({"response": 'FAIL', 'message': 'Please specify artistID'})
        try:
            artist_obj = ArtistInformation.objects.get(artistId=artist_id)
        except ArtistInformation.DoesNotExist:
            return []
        track_queryset = TrackInformation.objects.filter(artistId=artist_obj)
        return track_queryset

class ArtistInformationView(APIView):
    """ This view is for handling operation for Artist Information. It will perform GET, POST, DELETE request
    for artist information."""

    def get(self, pk=None):
        """Performs GET operation. Returns a Response with all artist information"""
        """This will handel all the GET request."""
        track_info = ArtistInformation.objects.all()
        serializer = ArtistInformationSerializer(track_info, many=True)
        return Response({"artist_info": serializer.data})

    def post(self, request):
        """Performs POST operation. Creates new artist record.  Returns a Response with artist information"""

        artist_name = request.data.get('name')
        serializer = ArtistInformationSerializer(data={'artistName': artist_name})
        if serializer.is_valid(raise_exception=True):
            artist_save = serializer.save()
        return Response({"response": 'SUCCESS', 'artistId': '{}'.format(artist_save.artistId),
                         "message": "Artist'{}' created successfully".format(artist_save.artistName)})

    def delete(self, request):
        """Performs DELETE operation. Deleted the artisrt ID record from the DB.
        Returns a Response with all artist information"""
        artistId = request.data.get('artistId')
        try:
            del_obj = ArtistInformation.objects.get(artistId=artistId)
            del_obj.delete()
            return Response({'response': 'SUCCESS', 'message': "Artist '{}' Deleted Successfully".format(artistId)})
        except ArtistInformation.DoesNotExist:
            return Response({'response': 'FAIL'})

class TrackInformationView(APIView):
    """This class handels the GET, POST, PUT operations for Tracks"""

    def check_track_exsist(self, **kwargs):
        """This is a custom method for retriving trackId and artistId"""
        try:
            artist_obj = TrackInformation.objects.get(trackId=kwargs.get('trackId')[0], artistId__artistId=kwargs.get('artistId')[0])
        except TrackInformation.DoesNotExist:
            artist_obj = []
        return artist_obj

    def get(self, request):
        track_info = TrackInformation.objects.all()
        serializer = TrackInformationSerializer(track_info, many=True)
        return Response({"track_info": serializer.data})

    def post(self, request):
        track_info = request.data.copy()
        track_info['releaseDate'] = datetime.datetime.now()
        serializer = TrackInformationSerializer(data=track_info, many=True)
        if serializer.is_valid(raise_exception=True):
            track_save = serializer.save()
        return Response({"response": 'SUCCESS', 'trackId': '{}'.format(track_save.artistId),
                         "message": "Track'{}' created successfully".format(track_save.trackName)})

    def put(self, request):
        """This method handels the Updates on Track Information"""
        da = request.data.copy()
        obj_data = self.check_track_exsist(**da)
        print(da)
        if not self.check_track_exsist(**da):
            return Response({'response': 'FAIL', 'message': 'Please check TrackID/ArtistID'})
        serializer = TrackInformationSerializer(obj_data, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)


print("Importing libraries")
from keras import backend as K
import tensorflow as tf
from keras.utils.vis_utils import plot_model
import pandas as pd
from sklearn.metrics import mean_squared_error
from datetime import datetime as dt
import numpy as np
import keras
from tensorflow.python.ops import math_ops
from tensorflow.keras.layers import BatchNormalization
import scipy
from sklearn.decomposition import TruncatedSVD

error_1_weight=2
error_2_weight=1

def data_preprocessing(df,numeric_col_list,n_components,target):
    one_hot = pd.get_dummies(df.drop([numeric_col_list],axis = 1))
    df2 = pd.concat([df,one_hot],axis=1)
    csr_sparse_matrix = scipy.sparse.csr_matrix(df2.values)
    pca = TruncatedSVD(n_components=n_components)
    principalComponents = pca.fit_transform(csr_sparse_matrix)
    print(pca.explained_variance_ratio_.sum())
    principalDf = pd.DataFrame(data = principalComponents)
    df2 = pd.concat([principalDf, df2[[target]]], axis = 1)
    return df2

def customLossCreator(error_1_weight, error_2_weight):
    def custom_loss(y_true, y_pred):
        diff =  math_ops.sub(y_true , y_pred)
        diff_error_1 = tf.math.multiply(error_1_weight,tf.math.multiply(diff,diff))
        diff_error_2 = tf.math.multiply(error_2_weight,tf.math.multiply(diff,diff))
        diff4 = tf.where(tf.greater(diff, 0), diff_error_1, diff_error_2)
        loss = K.mean(diff4, axis=-1) #mean over last dimension
        return loss
    return custom_loss

def createModel(X_train):
    return keras.Sequential([
                   keras.layers.Dense(32, activation='relu', input_shape=(1,X_train.shape[1]), dtype='float32'),
                   BatchNormalization(),
                   keras.layers.Dense(16, activation='relu'),
                   BatchNormalization(),
                   keras.layers.Dense(8, activation='relu'),
                   BatchNormalization(),
                   keras.layers.Dense(1)])

def fetchNumbers(model, X_train,y_train,test_main_X,test_main_y,model_name):
    y_pred = model.predict(X_train)
    print('MSE:',mean_squared_error(y_train, y_pred))
    y_pred_round = [np.round(x) for x in y_pred]
    print('Rounded MSE:',mean_squared_error(y_train, y_pred_round))
    y_train_arr = np.array(y_train)
    accuracy = [1  for i in range(0,len(y_train)) if (y_train_arr[i] == y_pred_round[i])]
    error_1 = [1  for i in range(0,len(y_train)) if y_train_arr[i] > y_pred_round[i]]
    error_2 = [1  for i in range(0,len(y_train)) if y_train_arr[i] < y_pred_round[i] ]
    print('Train BAE Nos:',np.sum(accuracy)/len(y_train),np.sum(error_1)/len(y_train),np.sum(error_2)/len(y_train))

    y_pred_test = model.predict(test_main_X)
    print('MSE:',mean_squared_error(test_main_y, y_pred_test))
    y_pred_round_test = [np.round(x) for x in y_pred_test]
    print('Rounded MSE:',mean_squared_error(test_main_y, y_pred_round_test))
    y_test_arr = np.array(test_main_y)
    accuracy = [1  for i in range(0,len(test_main_y)) if (y_test_arr[i] == y_pred_round_test[i])]
    error_1 = [1  for i in range(0,len(test_main_y)) if y_test_arr[i] > y_pred_round_test[i]]
    error_2 = [1  for i in range(0,len(test_main_y)) if y_test_arr[i] < y_pred_round_test[i]]
    print('Test BAE Nos:',np.sum(accuracy)/len(test_main_y),np.sum(error_1)/len(test_main_y),np.sum(error_2)/len(test_main_y))

    model.save( 'reg_model_' + str(dt.today().date()) + model_name + '.h5')
    X_train['actual'] = y_train
    X_train['predicted'] = y_pred_round
    return X_train

def createAndSaveModel(main_X, main_y,test_main_X,test_main_y,model_name, breaches_weight, earliness_weight):

    tf.config.run_functions_eagerly(True)
    X_train = main_X.copy()
    y_train = main_y.copy()
    model = createModel(X_train)
    model.compile(loss=customLossCreator(error_1_weight,error_2_weight), optimizer='sgd')
    plot_model(model, to_file=model_name + '.png', show_shapes=True, show_layer_names=True)
    model.fit(X_train, y_train, batch_size=256, epochs=8)
    return fetchNumbers(model, X_train, y_train, test_main_X,test_main_y,model_name)

def data_split(df,time_interval,target):
    train_data = df[:time_interval].sample(frac=1)
    test_data=df[time_interval:]
    main_X=train_data.drop([target],axis = 1)
    main_y = train_data[target].astype('float')
    test_main_X=test_data.drop([target],axis = 1)
    test_main_y=test_data[target].astype('float')
    return main_X, main_y, test_main_X, test_main_y

choice_set=[(error_1_weight,error_2_weight)]

print('read your dataframe')
print('define numerical col, target and number of principle components')

'''
1.pass the dataframe through :
   a.data_preprocessing
   b.data_split

2.pass the main_X, main_y, test_main_X, test_main_y found in above through createAndSaveModel function

3.Run the below line of codes:
    result_set=[createAndSaveModel(main_X, main_y,test_main_X,test_main_y, 'Model-v0.1'+str(i)+str(j), i,j) for i,j in choice_set]
    res = result_set[0]
    res['predicted'] = res['predicted'].map(lambda x: x[0])
    df.join(res[['actual','predicted']]).to_csv('Results_'+ str(dt.today().date()) + '.csv.gz', compression = 'gzip')

'''
##result_set=[createAndSaveModel(main_X, main_y,test_main_X,test_main_y, 'Model-v0.1'+str(i)+str(j), i,j) for i,j in choice_set]


EXPOUT = "@variables{c:#0f0}a{color:var(c)}\na{color:#0f0}\n"
EXPERR = 'Property: Found valid "CSS Level 2.1" value: #0f0 [6:9: color]\n'

def main():
    import cssutils
    import cssutils.script

    css = '''
    @variables {
        c: #0f0;
    }
    a {
        color: var(c);
    }
    '''
    s = cssutils.parseString(css)

    cssutils.ser.prefs.useMinified()
    cssutils.ser.prefs.resolveVariables = False
    print(s.cssText)

    # reset
    cssutils.ser.prefs.resolveVariables = True
    print(s.cssText)

if __name__ == '__main__':
    main()


'''
Given a matrix of 1s and 0s, return the number of "islands" in the matrix. A 1 represents land and 0 represents water, so an island is a group of 1s that are neighboring whose perimeter is surrounded by water.

For example, this matrix has 4 islands.

1 0 0 0 0
0 0 1 1 0
0 1 1 0 0
0 0 0 0 0
1 1 0 0 1
1 1 0 0 1
'''

from typing import List, Tuple

def countIslands(matrix: List[List[int]]):
    # Recursive function to build out an island. Will mark an entire island as traversed and return 1 at the top level.
    def rCountIslands(matrix: List[List[int]], coords: Tuple[int, int], traversed: List[List[bool]]):
        y, x = coords

        traversed[y][x] = True

        # West
        if x > 0 and matrix[y][x-1] == 1 and not traversed[y][x-1]:
            rCountIslands(matrix, (y, x-1), traversed)
        # Southwest
        if x > 0 and y < len(matrix)-1 and matrix[y+1][x-1] == 1 and not traversed[y+1][x-1]:
            rCountIslands(matrix, (y+1, x-1), traversed)
        # South
        if y < len(matrix)-1 and matrix[y+1][x] == 1 and not traversed[y+1][x]:
            rCountIslands(matrix, (y+1, x), traversed)
        # Southeast
        if x < len(matrix[0])-1 and y < len(matrix)-1 and matrix[y+1][x+1] == 1 and not traversed[y+1][x+1]:
            rCountIslands(matrix, (y+1, x+1), traversed)
        # East
        if x < len(matrix[0])-1 and matrix[y][x+1] == 1 and not traversed[y][x+1]:
            rCountIslands(matrix, (y, x+1), traversed)
        # Northeast
        if x < len(matrix[0])-1 and y > 0 and matrix[y-1][x+1] == 1 and not traversed[y-1][x+1]:
            rCountIslands(matrix, (y-1, x+1), traversed)
        # North
        if y > 0 and matrix[y-1][x] == 1 and not traversed[y-1][x]:
            rCountIslands(matrix, (y-1, x), traversed)
        # Northwest
        if x > 0 and y > 0 and matrix[y-1][x-1] == 1 and not traversed[y-1][x-1]:
            rCountIslands(matrix, (y-1, x-1), traversed)

        return 1

    # Setup all False traversed matrix and a count of 0.
    traversed = [[False for x in range(0, len(matrix[0]))] for y in range(0, len(matrix))]
    count = 0
    # Traverse through the matrix.
    for y in range(0, len(matrix)):
        for x in range(0, len(matrix[0])):
            if not traversed[y][x]:
                if matrix[y][x] == 1:
                    count += rCountIslands(matrix, (y, x), traversed)
                traversed[y][x] = True

    return count

island_map = [
    [1, 0, 0, 0, 0],
    [0, 0, 1, 1, 0],
    [0, 1, 1, 0, 0],
    [0, 0, 0, 0, 0],
    [1, 1, 0, 0, 1],
    [1, 1, 0, 0, 1],
]

print(countIslands(island_map))

island_map_2 = [
    [0, 0, 0, 0, 1],
    [1, 1, 1, 0, 1],
    [1, 0, 1, 0, 1],
    [1, 0, 0, 0, 1],
    [1, 1, 1, 1, 1],
]

print(countIslands(island_map_2))


import json
import cv2, math

# json 생성
def make_json_file(pill_shape, pill_text, pill_line=False):
    # json 기본 구조 설정
    drugData = {
        "drug_name" : "",
        "drug_type" : "",
        "drug_shape" : "",
        "drug_color" : "",
        "drug_line" : "없음"
    }
    drugData['drug_shape'] = pill_shape # 알약 모양
    drugData["drug_name"] = pill_text # 알약 문자
    # drugData["drug_line"] = pill_line
    try:
        # 알약 특징 정보를 가진 json 파일 생성
        with open('pill_data.json', 'w') as pill_data:
            json.dump(drugData, pill_data, indent='\t')
    except:
        print("error")

# crop한 text영역 이미지 회전 (croped img, polys=영역 좌표, 중심좌표)
def rotate_img(img, polys, center):
    center = tuple(map(int, center))
    # 좌상단(polys[0]), 좌하단(polys[3])
    # 좌하단을 기준으로 절대각도 확인
    x1, y1 = polys[0]
    x3, y3 = polys[2]
    x4, y4 = polys[3]
    dx, dy = x4 - x3, y4 - y3
    dhx, dhy = x4 - x1, y4 - y1
    angle = math.degrees(math.atan2(dy, dx))
    # text영역 크기 지정
    imgW = int(math.sqrt((dx**2)+(dy**2)))
    imgH = int(math.sqrt((dhx**2)+(dhy**2)))

    if dx < 0:
        angle += 180

    height, width, channels = img.shape
    M = cv2.getRotationMatrix2D(center, angle, 1)
    # atan2로 구한 각도에 맞추어 회전
    img_rot = cv2.warpAffine(img, M, (width, height))
    # text 영역 중심을 기준으로 이미지 crop
    img_crop = cv2.getRectSubPix(img_rot, (imgW,imgH), center)
    if imgH > (imgW*1.3):
        img_crop = cv2.rotate(img_crop, cv2.ROTATE_90_CLOCKWISE)

    return img_crop


from django.shortcuts import render, redirect, reverse, get_object_or_404
from django.contrib import messages
from django.conf import settings
from django.core.mail import send_mail
from .forms import OrderForm
from cart.contexts import cart_contents
from sculptures.models import Sculpture
from .models import OrderLineItem, Order
import stripe

def checkout(request):
    """
    Checkout function, creates users order and calls stripe payments,
    Sends details through to the checkout_success view.
    """
    stripe_public_key = settings.STRIPE_PUBLIC_KEY
    stripe_secret_key = settings.STRIPE_SECRET_KEY
    try:
        if request.method == 'POST':
            cart = request.session.get('cart', {})
            form_data = {
                'full_name': request.POST['full_name'],
                'email': request.POST['email'],
                'phone_number': request.POST['phone_number'],
                'country': 'Ireland',
                'postcode': request.POST['postcode'],
                'town_or_city': request.POST['town_or_city'],
                'street_address1': request.POST['street_address1'],
                'street_address2': request.POST['street_address2'],
                'county': request.POST['county'],
            }
            order_form = OrderForm(form_data)
            if order_form.is_valid():
                order = order_form.save()
                for item_id, quantity in cart.items():
                    try:
                        product = get_object_or_404(Sculpture, id=item_id)
                        order_line_item = OrderLineItem(
                            order=order,
                            product=product,
                            quantity=quantity
                        )
                        order_line_item.save()
                    except Sculpture.DoesNotExist:
                        messages.error(request, (
                            'Something has gone wrong with your cart'
                            'Please call us for assistance')
                        )
                        order.delete()
                        return redirect(reverse('view_cart'))
                return redirect(
                    reverse('checkout_success', args=[order.order_number]))
            else:
                messages.error(
                    request,
                    'Something has gone wrong with your order,\
                        Double check your details.')
    except ValueError:
        message.error(request, 'Wait!')

    else:
        cart = request.session.get('cart', {})
        if not cart:
            messages.error(request, 'Your Cart is Empty!')
            return redirect(reverse('shop'))

        current_cart = cart_contents(request)
        total = current_cart['grand_total']
        stripe_total = round(total * 100)
        stripe.api_key = stripe_secret_key
        intent = stripe.PaymentIntent.create(
            amount=stripe_total,
            currency=settings.STRIPE_CURRENCY,
        )

    order_form = OrderForm()

    if not stripe_public_key:
        messages.warning(request, 'Stripe public key is missing!')

    context = {
        'form': order_form,
        'stripe_public_key': stripe_public_key,
        'client_secret': intent.client_secret,
    }
    return render(request, 'checkout/checkout.html', context)

def checkout_success(request, order_number):
    """
    Calls the order confirmation page,
    Updates stock quantity after successful purchase.
    Sends order confirmation email.
    """
    order = get_object_or_404(Order, order_number=order_number)
    messages.success(request, f'Payment successful! \
                                A confirmation email will been sent to \
                                {order.email}.')

    cart = request.session.get('cart', {})
    alter_quantity = []
    product_object = []
    number = 0
    email_intro = f"Hi {order.full_name},\n\nThank you for your order of: \n\n"
    email_outro = (
        f"\nWe will do our best to ship your items "
        + f"as soon as possible."
        + f"\nIf you have any questions about your order "
        + f"please feel free to get in touch. \n"
        + f"\nKind Regards, \n"
        + f"The Forged Nature Team")
    email_body = ""

    try:
        for item_id, quantity in cart.items():
            product_object.append(Sculpture.objects.get(id=item_id))
            alter_quantity.append(quantity)

            for item in product_object:
                if item.title not in email_body:
                    email_body += (
                        str(item.title) + f", Quantity : {quantity}, \n")

        email_intro = email_intro + email_body + email_outro

        send_mail(
            "Thank you for you order!",
            f"{email_intro}",
            settings.EMAIL_HOST_USER,
            [order.email],
            fail_silently=False,)

        for item in product_object:
            item.quantity = item.quantity - alter_quantity[number]
            item.save()
            number += 1
    except ValueError:
        messages.error(
            request, "Whoops, we've had a problem reaching our database.")

    del request.session['cart']
    context = {
        'order': order,
    }
    return render(request, 'checkout/checkout_success.html', context)


"""
VGG transfer learning Test
"""
import tensorflow as tf
import myVGG as myvgg19
import utils
import numpy as np
from sklearn.utils import shuffle
import cifar10.cifar10 as data
from scipy.misc import imread, imresize
import matplotlib.pyplot as plt

#----------------load CIFAR10-------------------#
class_names = data.load_class_names()
images_train, cls_train, labels_train = data.load_training_data()
images_test, cls_test, labels_test = data.load_test_data()

# print(len(images_test))

#-----------------------resize-------------------#

batch = imresize(images_test[0], (224, 224))

#see how resize effect
# plt.imshow(batch)
# plt.show()
# plt.imshow(batch)
# plt.show()

#----------------set up---------------------#
batch_size = 1
learning_rate = 0.0001
images = tf.placeholder(tf.float32, [batch_size, 224, 224, 3])
train_mode = tf.placeholder(tf.bool)
print(train_mode)

vgg = myvgg19.Vgg19(vgg19_npy_path='./vgg19.npy', retrain=['fc8'], outputclass=10)
vgg.build(images, train_mode)

# print number of variables used: 143667240 variables, i.e. ideal size = 548MB
print(vgg.get_var_count())

merged_summary_op = tf.summary.merge_all()

#-------------------------------start training----------------------------------------#
# a = False
a = True
if a:
    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    # test classification
    feature = sess.run(vgg.pool1, feed_dict={images: [batch], train_mode: False})[0]
    fm = np.transpose(feature, axes=[2, 0, 1])

    #show
    plt.show()
    for m in fm:
        plt.imshow(m)
        plt.pause(0.1)


# 324. Wiggle Sort II
import statistics
class Solution:
    def wiggleSort(self, nums: List[int]) -> None:
        """
        Do not return anything, modify nums in-place instead.
        """
        '''
        1/2 Simple solution using sorting with O(NlogN) time.
        Sort and find the mid point, and slice the list such that we
        assign numbers smaller than mid to odd slots, and bigger numbers
        to even slots.

        Note that we need to reversely sort to handle case like [4, 5, 5, 6]
        where the numbers around mid point are the same.

        Time: O(n log(n))
        Space: O(n)
        '''
        nums.sort(reverse=True)
        mid = len(nums) // 2
        nums[1::2], nums[0::2] = nums[:mid], nums[mid:]

        '''
        2/2 Complex soultion but using O(N) time and O(1) space

        Find the median using "statistics.median(list)", this should be O(N).

        Do a 3-way-partitioning, or color sort, we can sort the nums
        into 3 parts [biggers, medians, smallers] (note that this is reversed)
        When doing the color sort:
            For biggers, remap the index to odd slots.
            For smallers, remap the index to even slots;
            For medians, keep them unchanged.
        The remap logic relies on the relative position of the index,
        when the index falls into the first half of the list, it maps to
        odd slots, when it falls into the second half, it maps to even slots.

        In the classic color sort:
        [ :i] are values less than mid;
        [i:j] are values equal to mid;
        [j:k] are values not yet sorted;
        [k: ] are values greater than mid
        '''
        def remap(i):
            return (1 + 2 * i) % (len(nums) | 1)
        def swapRemap(a, b):
            nums[remap(a)], nums[remap(b)] = nums[remap(b)], nums[remap(a)]
        i = j = 0
        k = len(nums) - 1
        mid = statistics.median(nums)
        while j <= k:
            if nums[remap(j)] < mid:
                swapRemap(j, k)
                k -= 1
            elif nums[remap(j)] > mid:
                swapRemap(i, j)
                i += 1
                j += 1
            else:
                j += 1


from scipy.stats import norm
import h5py
import pandas as pd
import sys
import os
import yaml
import matplotlib.pyplot as plt
import numpy as np
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
import utils
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = True
plt.rcParams['font.size'] = 18

config_path = os.path.join(os.path.dirname(__file__), '../../config.yml')
cfg = yaml.safe_load(open(config_path, 'r'))
spike_dt = 0.001
if __name__=='__main__':
    example_filename = '../../data/intermediate/rockstar.p'
    example_trial = 234
    example_filename = '../../data/intermediate/mack.p'
    example_trial = 43
    example_filename = '../../data/intermediate/raju-M1-no-bad-trials.p'
    example_trial = 121

    run_info = yaml.safe_load(open(os.path.dirname(__file__) + '/../../lfads_file_locations.yml', 'r'))
    datasets = list(run_info.keys())
    example_filenames = ['../../data/intermediate/%s.p'%dataset for dataset in datasets]
    example_trials = [234, 43, 121]
    example_filenames = [example_filenames[0]]
    example_trials = [45]
    for idx, (example_filename, example_trial) in enumerate(zip(example_filenames, example_trials)):
        df = pd.read_pickle(example_filename)
        trial_df = df.loc[example_trial].loc[:4]
        x_vel, y_vel = trial_df.kinematic[['x_vel','y_vel']].values.T
        speed = utils.get_speed(x_vel, y_vel)
        sigma_bin = cfg['rate_sigma_example_plot']
        bin_cutoff = sigma_bin*5 #cutoff kernal at 5 sigmas
        neural_kernel = norm.pdf(np.arange(-bin_cutoff, bin_cutoff), scale=sigma_bin)
        neural_kernel /= np.sum(neural_kernel)*spike_dt
        fr = np.apply_along_axis(np.convolve, 0, trial_df.neural.values, neural_kernel, mode='same')
        trial_pop_rate = fr.sum(1)
        t = trial_df.index.values
        t_targets = trial_df.kinematic.query('hit_target').index.values
        lns = []
        plt.figure(figsize=(18,6))
        lns += plt.plot(t, speed,'g')
        plt.ylabel('Cursor Speed (mm/s)')
        plt.xlabel('Time (s)')
        # plt.twinx()
        # lns += plt.plot(t, trial_pop_rate)
        # plt.ylabel('Population Firing Rate (spikes/s)')
        ymin, ymax = plt.ylim()
        plt.vlines(t_targets[1:], ymin, ymax)
        plt.text(t_targets[1]+0.02, ymax*.915, "Target 1\nAcquired", fontsize=12)
        for i,t in enumerate(t_targets[2:-1]):
            plt.text(t+0.01, ymax*.95, "Target %d"%(i+2), fontsize=12)

        if idx == 0:
            plt.arrow(2.02, 45, .10, 40, head_width=.02, head_length=5)
            plt.text(1.65, 30, "Corrective Movement", fontsize=12)
        # plt.legend(handles=lns, labels=['Cursor Speed', 'Population Firing Rate'],
        #            bbox_to_anchor=(0, 1.02, 1, 0.2), loc='lowerleft', ncol=3)
            plt.savefig('../../figures/final_figures/example_trial.png')
        plt.savefig('../../figures/final_figures/numbered/6a%d.png'%(idx+1))
        plt.savefig('../../figures/final_figures/numbered/6a%d.png'%(idx+1))
        plt.savefig('../../figures/final_figures/numbered/6a%d.png'%(idx+1))
    plt.show()


import numpy as np
import pandas as pd

from xgboost import XGBRegressor

from sklearn.svm import SVR
from sklearn.linear_model import Ridge
from sklearn import svm

from DNN import *

# Not used
# import lightgbm as lgb
# from sklearn.ensemble import RandomForestRegressor
# from sklearn.linear_model import ElasticNet, Lasso

def train_test(data, val=True):
    """Drop the galaxies near the edges of the simulation.
    Split data into train test, validation subsets based on 'x' spatial position. The split is 70-20-10%.
    If val==False we return train-test split in the 80-20 ratio."""

    # Drop data near edges of simulation (units of kpc/h)
    data = data[(data["posX"] > 1_000) & (data["posX"] < 204_000) &
                (data["posY"] > 1_000) & (data["posY"] < 204_000) & (data["posZ"] > 1_000) & (data["posZ"] < 204_000)]

    # x 'legnth' of simulation now is 203k kpc/h -> make train, test, validation splits:
    test = data[data["posX"] < 40_600].copy()  # 20%
    if val == True:
        validation = data[(data["posX"] > 40_600) & (
            data["posX"] < 60_900)].copy()  # 10%
        train = data[data["posX"] > 60_900].copy()  # 70%
        return train, test, validation
    else:
        train = data[data["posX"] > 40_600].copy()  # 70%
        return train, test

def make_models(params):
    """Models to add to stacking (excluded DNN, which will be added manually).
    Params is a dict object containing hyperparams for the models."""
    svr_model = svm.SVR(**params["svr"])  # rbf, 5
    xgboost_moel = XGBRegressor(**params["xgb"])
    # Some other models such as RF or simple Lasso/Elascti-Net/Ridge,... were tried but underperformed
    # lgb_model = lgb.LGBMRegressor(random_state=42)   # not included, xgboost better
    # lgb_model.set_params(**params["lgb"]) # drop rate
    # rf_model = RandomForestRegressor(**params["rf"]) # n_estimators, max_depth, min_samples_leaf, min_samples_split
    # ridge_model = Ridge(**params["ridge"])  # alpha
    # lasso_model = Lasso(**params["lasso"])  # alpha
    # elastic_net_model = ElasticNet(**params["elastic_net"]) # alpha
    # can be arbitrary long list of models compatible with scikit-learn
    models = [svr_model, xgboost_moel]
    return models

def stacked_model(X_train, y_train, X_test, y_test, models, params, drop_features=["posY"], batch=10, N=13):
    """Produce predictions with all base learners, as defined in 'make_models' + DNN model as defined in 'DNN.py'. """
    X_train = X_train.copy().drop(drop_features, axis=1)
    X_test = X_test.copy().drop(drop_features, axis=1)

    model1, model2 = models  # Should iterate if there are multiple models
    model1.fit(X_train, y_train)
    model2.fit(X_train, y_train)
    predictions1 = model1.predict(X_test)
    predictions2 = model2.predict(X_test)

    # make predictions with DNN
    n_epoch = params["DNN"]["n_epoch"]  # 150
    LR, WD = 1.e-7, 1.e-7  # hardcoded, changing did not lead to improvements

    net = make_network(params["DNN"]["hidden_layers"], params["DNN"]
                       ["hidden_layer_size"], N, dropout=params["DNN"]["dropout"], bias=True)

    train_loader = convert_to_tensors(X_train, y_train, batch=batch)
    test_loader = convert_to_tensors(X_test, y_test, batch=batch)
    train_losses, valid_losses = main_training(
        net, train_loader, test_loader, LR, WD, n_epoch)
    # print("Training loss;", train_losses)
    # print("Test/validation loss;", valid_losses)
    pred_actual = validation(test_loader, net)

    pred_actual_test = []
    # Convert to appropriate type
    for batch in pred_actual:
        for i in range(len(batch[0])):
            pred_actual_test.append((float(batch[0][i]), float(batch[1][i])))

    predictions_dnn = np.array(pred_actual_test)[:, 0]
    return pd.DataFrame([predictions1, predictions2, predictions_dnn]).T

def stacked_predictions(X_train, y_train, X_test, y_test, params, drop_features=["posY"], batch=10, N=13):
    """Main function for stacking predictions, using cross validation. 'Test' can be used for both actual test set and validation set."""
    # Cross validation split is needed. Data should contain posY information! (which is dropped for training)
    models = make_models(params)

    # Hardcode this value based on the desired cross validation split
    x_min_split = 68300
    x_max_split = 136700

    X_train, y_train, X_test, y_test = X_train.copy(
    ), y_train.copy(), X_test.copy(), y_test.copy()
    # Also reset indexes to avoid any possible index errors
    X1, y1 = X_train[X_train["posY"] < x_min_split].reset_index(
        drop=True), y_train[X_train["posY"] < x_min_split].reset_index(drop=True)
    X2, y2 = X_train[X_train["posY"] <
                     x_max_split].reset_index(drop=True), y_train[X_train["posY"] < x_max_split].reset_index(drop=True)
    X2, y2 = X2[X2["posY"] > x_min_split].reset_index(
        drop=True), y2[X2["posY"] > x_min_split].reset_index(drop=True)
    X3, y3 = X_train[X_train["posY"] >
                     x_max_split].reset_index(drop=True), y_train[X_train["posY"] > x_max_split].reset_index(drop=True)

    X_merge1, y_merge1 = pd.concat(
        [X1, X2], axis=0), pd.concat([y1, y2], axis=0)
    X_merge2, y_merge2 = pd.concat(
        [X1, X3], axis=0), pd.concat([y1, y3], axis=0)
    X_merge3, y_merge3 = pd.concat(
        [X2, X3], axis=0), pd.concat([y2, y3], axis=0)
    X_merge_all, y_merge_all = pd.concat(
        [X1, X2, X3], axis=0), pd.concat([y1, y2, y3], axis=0)

    pred1 = stacked_model(X_merge1, y_merge1, X3, y3.values,
                          models, params, drop_features, batch=batch, N=N)
    pred2 = stacked_model(X_merge2, y_merge2, X2, y2.values,
                          models, params, drop_features, batch=batch, N=N)
    pred3 = stacked_model(X_merge3, y_merge3, X1, y1.values,
                          models, params, drop_features, batch=batch, N=N)
    pred_val = stacked_model(X_merge_all, y_merge_all, X_test,
                             y_test, models, params, drop_features, batch=batch, N=N)

    meta_X, meta_y = pd.concat(
        [pred1, pred2, pred3], axis=0), pd.concat([y3, y2, y1], axis=0)

    meta_learner = Ridge()
    meta_learner.fit(meta_X, meta_y)
    return meta_learner.predict(pred_val)


from django.contrib import admin

from .models import Message

class MessageAdmin(admin.ModelAdmin):
    fields = ('method', 'body', 'parsed')
    readonly_fields = ('date',)
    list_display = ('method', 'body', 'parsed', 'date')

admin.site.register(Message, MessageAdmin)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('mafia_app', '0007_death_game'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='player',
            name='redditID',
        ),
        migrations.AddField(
            model_name='player',
            name='alignment',
            field=models.CharField(default=b'U', max_length=1, choices=[(b'U', b'Unknown'), (b'M', b'Mafia'), (b'T', b'Town'), (b'O', b'Other')]),
        ),
        migrations.AddField(
            model_name='player',
            name='notes',
            field=models.TextField(blank=True),
        ),
        migrations.AddField(
            model_name='player',
            name='role',
            field=models.CharField(max_length=50, blank=True),
        ),
    ]


from pyllicalabs import *
import datetime
from datetime import date
from dateutil.relativedelta import relativedelta

relativePath = "/.../"

allPath   = relativePath + "/all_texts.txt"
getPath   = relativePath + "/get_text.txt"
outputDir = relativePath + "/DATABASE/"

def scrapping():
    # Récupération de l'état
    getFile = open(getPath, "r")
    getInfos = [int(getFile.readline().replace('\n', '')) for _ in range(8)]
    getFile.close()

    # Récupération de la base de données
    allFile = open(allPath, "r")
    paths = [allFile.readline().replace('\n', '') for _ in range(1500)]
    allFile.close()

    # Récupération du champ d'action
    startIndex = getInfos[6] - 1
    endIndex   = getInfos[7] - 1

    if startIndex > endIndex :
        print("Travail terminé.")
        return

    if getInfos[0] == 0 :
        print("Préparation pour le premier document")
        startDate  = datetime.datetime(int(paths[2::11][startIndex]),
                                       int(paths[3::11][startIndex]),
                                       int(paths[4::11][startIndex]))
        endDate    = datetime.datetime(int(paths[5::11][startIndex]),
                                       int(paths[6::11][startIndex]),
                                       int(paths[7::11][startIndex]))
    else:
        startDate  = datetime.datetime(getInfos[0], getInfos[1], getInfos[2])
        endDate    = datetime.datetime(getInfos[3], getInfos[4], getInfos[5])

    if startDate > endDate :
        print("Préparation pour le prochain document")
        startIndex += 1

        # Incrément du texte et réinitialisation de la date de début
        getFile = open(getPath, "w")
        getFile.writelines([paths[2::11][startIndex] + "\n",
                            paths[3::11][startIndex] + "\n",
                            paths[4::11][startIndex] + "\n",
                            paths[5::11][startIndex] + "\n",
                            paths[6::11][startIndex] + "\n",
                            paths[7::11][startIndex] + "\n",
                            str(startIndex + 1)      + "\n",
                            str(endIndex   + 1)])
        getFile.close()
        return

    # --- Traitement du texte actuel à la date actuelle ---
    # 1) Mémorisation des paramètres à utiliser
    text_url = paths[1::11][startIndex]
    text_year = startDate.year
    text_month = startDate.month
    text_day = startDate.day
    text_title = outputDir + str(text_year) + "/" + str(text_month) + "/" + paths[::11][startIndex]

    # 2) Incrémentation
    if paths[9::11][startIndex] == 'j':
        startDate += datetime.timedelta(days = int(paths[8::11][startIndex]))
    elif paths[9::11][startIndex] == 'm':
        startDate += relativedelta(months =+ int(paths[8::11][startIndex]))

    # 3) Préparation de la valeur suivante
    getFile = open(getPath, "w")
    getFile.writelines([str(startDate.year ) + "\n",
                        str(startDate.month) + "\n",
                        str(startDate.day  ) + "\n",
                        str(endDate.year   ) + "\n",
                        str(endDate.month  ) + "\n",
                        str(endDate.day    ) + "\n",
                        str(startIndex + 1 ) + "\n",
                        str(endIndex   + 1 ) + "\n"])

    # 4) Récupération de la donnée
    print("Appel de :")
    print("pdfpress(url=\""  + text_url        + "\",")
    print("    title=\""     + text_title      + "\",")
    print("    year="        + str(text_year)  + ",")
    print("    month="       + str(text_month) + ",")
    print("    day="         + str(text_day)   + ",")
    print("    item=1,\nrate=1)\n")

    pdfpress(url=      text_url,
             title=    text_title,
             year=     text_year,
             month=    text_month,
             day=      text_day,
             item=     1,
             rate=     1)

scrapping()


#!/usr/bin/env python3

# *****************************************
# irrigator - Initialize Crontab
# *****************************************
#
# Description: Add cron entries for items
#
# 1. Add reboot script to initialize relays
#  @reboot cd /usr/local/bin/irrigator && sudo python3 control.py -i &
#
# 2. Add monthly log cleanup
#  0 0 1 * * cd /usr/local/bin/irrigator/logs && sh backup.sh
#
# 3. Add periodic weather check
#  */15 * * * * cd /usr/local/bin/irrigator && sudo python3 openwx.py
#
# *****************************************
from crontab import CronTab
from common import *

def checkexists(comment_string, system_cron):
    count = 0 # Init count variable
    for count in system_cron.find_comment(comment_string):
        print(count) # Look for finding the CRON Job
    return(count)

# Main Start
system_cron = CronTab(user='root')

# Initialize Relays on Boot
command_string = "cd /usr/local/bin/irrigator && sudo python3 control.py -i &"
comment_string = "Irrigator-Init"

if (checkexists(comment_string, system_cron)==0):
    # If not found, create it.
    entry = system_cron.new(command=command_string,comment=comment_string)
    entry.every_reboot()
    entry.enable()
    system_cron.write()

# Monthly Log Cleanup
command_string = "cd /usr/local/bin/irrigator/logs && sh backup.sh"
comment_string = "Irrigator-Log"

if (checkexists(comment_string, system_cron)==0):
    # If not found, create it.
    entry = system_cron.new(command=command_string,comment=comment_string)
    entry.setall("0 0 1 * *")
    entry.enable()
    system_cron.write()

# Cache weather information every 15 minutes
command_string = "cd /usr/local/bin/irrigator && sudo python3 openwx.py"
comment_string = "Irrigator-WxCache"

if (checkexists(comment_string, system_cron)==0):
    # If not found, create it.
    entry = system_cron.new(command=command_string,comment=comment_string)
    entry.minute.every(15)
    entry.enable()
    system_cron.write()

# Add schedules to the crontab
json_data_dict = ReadJSON() # This will create default schedules if none exist

for item in json_data_dict['schedules']:
    command_string = "cd /usr/local/bin/irrigator && sudo python3 control.py -s " + item + " &"
    if (checkexists(item, system_cron)==0):
        # If not found, create it.
        entry = system_cron.new(command=command_string,comment=item)
        entry.setall(json_data_dict['schedules'][item]['start_time']['cron_string'])
        if(json_data_dict['schedules'][item]['start_time']['enabled']):
            entry.enable()
        else:
            entry.enable(False)
        system_cron.write()


# Given a set of n segments {[a_0,b_0],[a_1,b_1],...,[a_{n−1},b_{n−1}]} with integer coordinates on a line, find the minimum number m of points such that each segment contains at least one point. That is, find a set of integers X of the minimum size such that for any segment [a_i,b_i] there is a point x∈X such that a_i≤x≤b_i.

import sys
from collections import namedtuple

Segment=namedtuple('Segment','start end')

def optimal_points(segments):
    points=[]
    while segments!=[]:
          y=[s.end for s in segments]
          i=y.index(min(y))
          b=y[i]
          points.append(b)
          segments.pop(i)
          nseg=[s for s in segments]
          for s in nseg:
              if s.start<=b: segments.remove(s)
    return points

if __name__ == '__main__':
   input = sys.stdin.read()
   n,*data=map(int,input.split())
   segments=list(map(lambda x: Segment(x[0],x[1]),zip(data[::2],data[1::2])))
   points=optimal_points(segments)
   print(len(points))
   for p in points: print(p, end=' ')


'''
Convergents of e
https://projecteuler.net/problem=65
'''
from fractions import Fraction
from numpy import e

def returnPeriods(num, limit):
    if num == e:
        return returnEuler(limit)
    periods = []
    r = num
    i = int(num)
    fractional = r - i
    for _ in range(0, limit):
        periods.append(i)
        r = 1 / fractional
        print(r)
        i = int(r)
        fractional = r - i
    return periods

def returnEuler(limit):
    periods = [2 * (i+1) // 3 if i % 3 == 2 else 1 for i in range(limit)]
    periods[0] += 1
    return periods

def returnConvergent(num, convergent):
    periods = returnPeriods(num, convergent)
    twoago_convergent = Fraction(periods[0], 1)
    if convergent == 1:
        return twoago_convergent
    previous_convergent = Fraction(periods[1] * periods[0] + 1, periods[1])
    if convergent == 2:
        return previous_convergent
    else:
        for n in range(2, convergent):
            current_convergent = Fraction(periods[n]
                                          * previous_convergent.numerator
                                          + twoago_convergent.numerator,
                                          periods[n]
                                          * previous_convergent.denominator
                                          + twoago_convergent.denominator)
            twoago_convergent = previous_convergent
            previous_convergent = current_convergent
    return current_convergent

print(sum([int(n) for n in str(returnConvergent(e, 100).numerator)]))


'https://arxiv.org/pdf/2206.07293.pdf'

import torch
import torch.nn as nn
import torch.nn.functional as F

from conv_stft import ConvSTFT, ConviSTFT
from modules.complex_nn import ComplexConv2d, ComplexConvTranspose2d, ComplexLSTM, ComplexBatchNorm2d, complex_concat
from modules import CCBAM

class ConvBlock(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, padding=(0, 0), norm=True, act=True, causal=True, is_complex=True, **kwargs) -> None:
        super().__init__()

        self.causal = causal
        self.padding = padding
        negative_slope = kwargs.get('negative_slope', 0.2)

        if is_complex:
            self.conv = ComplexConv2d(in_channels, out_channels, kernel_size, padding=(self.padding[0], 0), bias=not norm, **kwargs)
            self.norm = ComplexBatchNorm2d(out_channels) if norm else nn.Identity()
            self.act = nn.LeakyReLU(negative_slope) if act else nn.Identity()
        else:
            self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, padding=(self.padding[0], 0), bias=not norm, **kwargs)
            self.norm = nn.BatchNorm2d(out_channels) if norm else nn.Identity()
            self.act = nn.LeakyReLU(negative_slope) if act else nn.Identity()

    def forward(self, x):
        if self.causal:
            x = F.pad(x, (self.padding[1], 0, 0, 0))
        else:
            x = F.pad(x, (self.padding[1], self.padding[1], 0, 0))

        return self.act(self.norm(self.conv(x)))

class ConvTransposeBlock(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, padding=(0, 0), norm=True, act=True, causal=True, is_complex=True, **kwargs) -> None:
        super().__init__()

        self.causal = causal
        self.padding = padding
        negative_slope = kwargs.get('negative_slope', 0.2)

        if is_complex:
            self.conv_transposed = ComplexConvTranspose2d(in_channels, out_channels, kernel_size, padding=(self.padding[0], 0), bias=not norm, **kwargs)
            self.norm = ComplexBatchNorm2d(out_channels) if norm else nn.Identity()
            self.act = nn.LeakyReLU(negative_slope) if act else nn.Identity()
        else:
            self.conv_transposed = nn.ConvTranspose2d(in_channels, out_channels, kernel_size, padding=(self.padding[0], 0), bias=not norm, **kwargs)
            self.norm = nn.BatchNorm2d(out_channels) if norm else nn.Identity()
            self.act = nn.LeakyReLU(negative_slope) if act else nn.Identity()

    def forward(self, x):
        if self.causal:
            x = F.pad(x, (self.padding[1], 0, 0, 0))
        else:
            x = F.pad(x, (self.padding[1], self.padding[1], 0, 0))

        return self.act(self.norm(self.conv_transposed(x)))

class Encoder(nn.Module):
    def __init__(self, in_channels=1, out_channels=128, num_repeats=6, is_complex=True) -> None:
        super().__init__()

        self.layers = nn.ModuleList()
        for _ in range(num_repeats):
            self.layers += [ConvBlock(in_channels, out_channels, kernel_size=(5, 2), stride=(2, 1), padding=(0, 1), causal=True, is_complex=is_complex)]
            in_channels = out_channels

    def forward(self, x):
        outputs = []
        for layer in self.layers:
            x = layer(x)
            outputs.append(x)
        return x, outputs

class Decoder(nn.Module):
    def __init__(self, in_channels=128, out_channels=128, num_repeats=6, reduction_ratio=16, is_complex=True) -> None:
        super().__init__()

        self.skip_connection_attention_layers = nn.ModuleList()
        self.layers = nn.ModuleList()

        for i in range(num_repeats):
            self.skip_connection_attention_layers += [CCBAM(in_channels, reduction_ratio)]
            self.layers += [ConvTransposeBlock(in_channels * 2, out_channels, kernel_size=(5, 2), stride=(2, 1), padding=(0, 0), causal=True, is_complex=is_complex)]
            in_channels = out_channels

    def forward(self, x, encoder_outputs):
        for i, layer in enumerate(self.layers):
            encoder_output = encoder_outputs.pop()
            encoder_output = self.skip_connection_attention_layers[i](encoder_output)

            if x.shape[-1] > encoder_output.shape[-1]:
                x = x[:, :, :, :-1]
            if x.shape[-2] < encoder_output.shape[-2]:
                x = F.pad(x, (0, 0, 0, 1))

            x = complex_concat([x, encoder_output], dim=1)
            x = layer(x)
        return x

class FRCRN(nn.Module):
    def __init__(self, window_size=320, hop_size=160, fft_size=640, lstm_channels=256, reduction_ratio=16, is_complex=True) -> None:
        super().__init__()

        self.stft = ConvSTFT(window_size, hop_size, fft_size)
        self.istft = ConviSTFT(window_size, hop_size, fft_size)

        self.encoder = Encoder(in_channels=2, out_channels=128, is_complex=is_complex)
        self.decoder = Decoder(in_channels=128, out_channels=128, is_complex=is_complex, reduction_ratio=reduction_ratio)
        self.lstm = ComplexLSTM(256, lstm_channels, num_layers=2, bidirectional=False, batch_first=True)

        self.final_conv = nn.Conv2d(128, 2, kernel_size=(1, 2), bias=False)

        self.fft_size = fft_size

    def forward(self, x):
        ### stft
        complex_specs = self.stft(x)

        noisy_real = complex_specs[:, :self.fft_size // 2 + 1, :]
        noisy_imag = complex_specs[:, self.fft_size // 2 + 1:, :]

        complex_specs = torch.stack([noisy_real, noisy_imag], dim=1)
        complex_specs = complex_specs[:, :, 1:, :]

        ### process
        x = complex_specs
        x, encoder_outputs = self.encoder(x)

        batch, channels, freq, time = x.shape
        x = x.view(batch, channels * freq, time).permute(0, 2, 1)

        x = self.lstm(x)
        x = x.permute(0, 2, 1).view(batch, channels, freq, time)

        x = self.decoder(x, encoder_outputs)
        x = self.final_conv(x)
        x = F.pad(x, (0, 0, 1, 0))

        ### masking
        mask = torch.tanh(x)
        clean_estimate_spec = mask * complex_specs
        clean_estimate_spec = F.pad(clean_estimate_spec, (0, 0, 1, 0))

        ### istft
        real = clean_estimate_spec[:, 0, :, :]
        imag = clean_estimate_spec[:, 1, :, :]

        clean_estimate_spec = torch.cat([real, imag], dim=1)
        clean_estimate_wav = self.istft(clean_estimate_spec)
        clean_estimate_wav = torch.clamp_(clean_estimate_wav, -1, 1)
        return clean_estimate_spec, clean_estimate_wav

if __name__ == '__main__':
    # TODO 1: FSMN, CFSMN

    win_size = 320  # = 16000Hz * 0.02s (20ms) in paper
    hop_size = 160  # = 16000Hz * 0.01s (10ms) in paper
    fft_size = 640

    signal = torch.randn(1, 16000)
    model = FRCRN(win_size, hop_size, fft_size, is_complex=True)

    clean_estimate_spec, clean_estimate_wav = model(signal)
    print(clean_estimate_wav.shape)


# 显示控件的提示信息

# 需要用到 QToolTip

import sys
from PyQt5.QtWidgets import  QHBoxLayout,QMainWindow,QApplication,QToolTip,QPushButton,QWidget
# 提示信息需要设置字体
from PyQt5.QtGui import QFont

class TooltipForm(QMainWindow):
    def __init__(self):
        super().__init__()
        # 调用初始化ui的一个方法
        self.initUI()

    # 编写初始化UI的方法
    def initUI(self):
        # 设置字体和字号
        QToolTip.setFont(QFont('SansSerif',12))
        # 给窗口设置提示，这个方法支持富文本
        self.setToolTip('今天是个<b>好日子</b>')
        # 设置窗口的位置和尺寸
        self.setGeometry(300,300,200,200)
        # 设置窗口的标题
        self.setWindowTitle('设置控件提示消息')

    # 添加butoon按钮并设置提示信息

        # 添加Button
        # 创建全局对象self.button1
        self.button1 = QPushButton('我的按钮')
        # 设置按钮提示
        self.button1.setToolTip('这是按钮的提示信息')
        # 发送单击信号，执行对应的方法  (将信息与槽关联)
        self.button1.clicked.connect(self.onClick_Button)

        # 创建水平布局
        layout = QHBoxLayout()
        # 将组件加到水平局部里面
        layout.addWidget(self.button1)

        # 放置一个主框架
        mainFrame = QWidget()
        # 在主框架内添加水平布局
        mainFrame.setLayout(layout)
        # 把主框架放在窗口上
        self.setCentralWidget(mainFrame)

    # 按钮的单击事件的方法(自定义的槽)
    def onClick_Button(self):
        sender = self.sender()
        print(sender.text() + '按钮被按下')
        # 得到实例
        app = QApplication.instance()
        # 退出应用程序
        app.quit()

# 防止别的脚本调用，只有自己单独运行时，才会调用下面代码
if __name__ == '__main__':
    # 创建app实例，并传入参数
    app= QApplication(sys.argv)
    # 创建对象
    main = TooltipForm()
    # 创建窗口
    main.show()
    # 进入程序的主循环，并通过exit函数，确保主循环安全结束(该释放资源的释放资源)
    sys.exit(app.exec_())


# US Industrial PDF Extraction Web-App

#############
# LIBRARIES #
#############

import os
import glob
import tabula
import streamlit as st
from st_aggrid import GridOptionsBuilder, AgGrid, GridUpdateMode, DataReturnMode
import random
import PyPDF2
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
import pdfplumber
from io import StringIO
import re
# import camelot
import fitz
from PIL import Image
from streamlit_drawable_canvas import st_canvas
import numpy as np
from PyPDF2 import PdfFileReader
import pandas as pd
from PIL import Image
import time

######################
# COLUMN HEADER DICT #
######################

# Determines mapping for "structure_data" function

lease_comps_dict = {
"ADDRESS": ["ADDRESS", "BUILDING", "PROPERTY"],
"MARKET": ["MARKET", "CITY", "TOWN", "SUBMARKET"],
"YEAR BUILT": ["YEAR BUILT"],
"BLDG. CONFIG": ["CONFIG"],
"CLEAR HEIGHT (FT)": ["CLEAR"],
"BLDG. DEPTH (FT)": ["DEPTH"],
"TRAILER PARKING": ["TRAILER", "PARKING"],
"LANDLORD": ["LANDLORD"],
"TENANT NAME": ["TENANT"],
"BUILDING SQUARE FOOTAGE": ["BLDG SF"],
"LEASED SQUARE FOOTAGE": ["LEASED SF", "SQUARE", "FEET", "FOOTAGE", "SIZE"],
"LEASE START DATE": ["LCD", "DATE", "LEASE START", "COMMENCEMENT"],
"TERM (MONTHS)": ["TERM"],
"FREE RENT (MONTHS)": ["FREE RENT"],
"STARTING NNN RATE (PSF/YR)": ["STARTING", "NNN", "RENTAL"],
"ESCALATIONS": ["ESCALATIONS", "ESC", "INCREASES"],
"TENANT IMPROVEMENTS (PSF)": ["TI", "TENANT IMPROVEMENTS", "IMPROVEMENTS"],
"LEASE TYPE": ["LEASE TYPE"]
}

sales_comps_dict = {
"ADDRESS": ["ADDRESS", "BUILDING", "PROPERTY"],
"MARKET": ["MARKET", "CITY", "TOWN", "SUBMARKET"],
"YEAR BUILT": ["YEAR BUILT", "YR BUILT"],
"BLDG. CONFIG": ["CONFIG"],
"CLEAR HEIGHT (FT)": ["CLEAR"],
"SELLER": ["SELLER"],
"BUYER": ["BUYER"],
"SALE DATE": ["SALES DATE", "SALE DATE", "DATE"],
"OCCUPANCY": ["OCCUPANCY"],
"WALT (YRS)": ["WALT"],
"SQUARE FOOTAGE": ["(?<!P)SF", "SQUARE", "FOOTAGE", ""],
"SALE PRICE": ["SALES PRICE((?<! PSF)|(?<! PLSF)|(?<! (PSF))|(?<! (PLSF)))", "SALE PRICE"],
"SALE PRICE PSF": ["SPSF", "PSF"],
"CAP RATE WITH NO VACANCY DEDUCT": ["CAP RATE", "CAP RATE W/ NO"],
"CAP RATE WITH 5% VACANCY DEDUCT": ["CAP RATE W/ 5%"],
"BROKER": ["BROKER"]
}

land_sales_comps_dict = {
"ADDRESS": ["ADDRESS", "BUILDING", "PROPERTY"],
"MARKET": ["MARKET", "CITY", "TOWN", "SUBMARKET"],
"SELLER": ["SELLER"],
"BUYER": ["BUYER"],
"SALE DATE": ["SALES DATE", "SALE DATE"],
"LAND AREA (ACRES)": ["ACRES", "AREA"],
"SALE PRICE": ["SALES PRICE((?<! PSF)|(?<! PLSF)|(?<! (PSF))|(?<! (PLSF)))", "SALE PRICE"],
"SALE PRICE PSF": ["PLSF", "PRICE PER", "PSF"],
"BUILDABLE SQUARE FOOTAGE": ["BUILDABLE", "LAND SF", "SQUARE"],
"RAW LAND (BLDG $/SF)": ["RAW"],
"LAND IMPROVEMENTS TO PAD READY (BLDG $/SF)": ["IMPROVEMENTS", "PAD"],
"TOTAL LAND (BLDG $/SF)": ["TOTAL"]
}
#SALES PRICE(?! \(PSF\))(?! PSF)(?! PLSF)(?! \(PLSF\))
##################
# HELPER CLASSES #
##################

class Page:

    def __init__(self, pdf_path, page_number, library_option, comp_type):

        # PDF and page info
        self.PDF_Path = pdf_path
        self.Page_Number = page_number
        self.Comp_Type = comp_type

        # Library option is given to start
        self.Library_Option = library_option

        # Extraction parameter names and details for sliders
        # {"Param Name": [Min, Max, Step]}
        self.Extraction_Parameters = []
        self.Determine_Extraction_Params()

        # Extraction parameter values
        self.Extraction_Parameter_Values = {}

        # Table coordinates
        self.Table_Coordinates = []

        # Table list
        self.Tables = []

    ## Extraction parameter input values
    def Determine_Extraction_Params(self):

        # if self.Library_Option == "Camelot":
        #     table_settings = {
        #     "Edge Tolerance": [0, 1000, 10],
        #     "Row Tolerance": [0, 100, 1],
        #     # "Line Scale": [0, 100, 1],
        #     }

        if self.Library_Option == "PDF Plumber":
            table_settings = {
            "Snap Tolerance": [0.0, 10.0, 0.1],
            "Join Tolerance": [0.0, 10.0, 0.1],
            "Edge Minimum Length": [0.0, 10.0, 0.1],
            "Text Tolerance": [0.0, 10.0, 0.1],
            "Intersection Tolerance": [0.0, 10.0, 0.1],
            "Vertical Strategy": ["Lines", "Text"],
            "Horizontal Strategy": ["Lines", "Text"]
            }

        elif self.Library_Option == "Tabula":
            table_settings = {}

        self.Extraction_Parameters = table_settings

        return True

    ## Collect extraction parameter values input by user
    def Get_Extraction_Param_Vals(self, vals):

        # if self.Library_Option == "Camelot":
        #     table_settings_vals = {
        #     "edge_tol": vals["Edge Tolerance"],
        #     "row_tol": vals["Row Tolerance"]
        #     # "line_scale": vals["Line Scale"],
        #     }

        if self.Library_Option == "PDF Plumber":

            table_settings_vals = {
            "vertical_strategy": vals["Vertical Strategy"],
            "horizontal_strategy": vals["Horizontal Strategy"],
            "snap_tolerance": vals["Snap Tolerance"],
            "join_tolerance": vals["Join Tolerance"],
            "edge_min_length": vals["Edge Minimum Length"],
            "min_words_vertical": 1,
            "min_words_horizontal": 1,
            "keep_blank_chars": True,
            "text_tolerance": vals["Text Tolerance"],
            "intersection_tolerance": vals["Intersection Tolerance"],
            }

        elif self.Library_Option == "Tabula":
            table_settings_vals = {}

        self.Extraction_Parameter_Values = table_settings_vals

        return True

    ## Get coordinate data from bounding box and scale to PDF
    def Populate_Table_Coordinates(self, image_data):
        pdf = pdfplumber.open(self.PDF_Path)
        p = pdf.pages[self.Page_Number]

        if np.nonzero(image_data)[0].size != 0:

            min_x = np.amin(np.nonzero(image_data)[1])
            max_x = np.amax(np.nonzero(image_data)[1])
            min_y = np.amin(np.nonzero(image_data)[0])
            max_y = np.amax(np.nonzero(image_data)[0])

            x_ratio = p.width / image_data.shape[1]
            y_ratio = p.height / image_data.shape[0]

            new_min_x = round(min_x * x_ratio)
            new_max_x = round(max_x * x_ratio)
            new_min_y = round(min_y * y_ratio)
            new_max_y = round(max_y * y_ratio)

            # Return accurate coordinates
            coords = [new_min_x, new_min_y, new_max_x, new_max_y]

        else:
            coords = []

        self.Table_Coordinates = coords

        return True

    ## Get image of PDF page cropped according to bounding box
    def Get_Cropped_Image(self):
        PDF = fitz.open(self.PDF_Path)
        PDF_page = PDF.loadPage(self.Page_Number)  # number of page
        PDF_page.set_cropbox(fitz.Rect(self.Table_Coordinates[0], self.Table_Coordinates[1], self.Table_Coordinates[2],
        self.Table_Coordinates[3]))
        pix = PDF_page.get_pixmap()
        st.image(Image.frombytes("RGB", [pix.width, pix.height], pix.samples))
        return True

    ## Extraction functions
    def Extract_Tables_From_PDF_Page_PDF_Plumber(self):

        pdf = pdfplumber.open(self.PDF_Path)
        p = pdf.pages[self.Page_Number]
        p = p.crop(self.Table_Coordinates)

        tables = p.extract_tables(self.Extraction_Parameter_Values)
        pd_tables = []

        for table in tables:
            pd_tables.append(pd.DataFrame(table[1::],columns=table[0]))

        self.Tables = pd_tables

        return True

    def Extract_Tables_From_PDF_Page_Tabula(self):
        tables = tabula.read_pdf(self.PDF_Path, stream = True, pages = self.Page_Number)
        pd_tables = []

        for table in tables:
            pd_tables.append(table)

        self.Tables = pd_tables

        return True

    # def Extract_Tables_From_PDF_Page_Camelot(self):
    #     tables = camelot.read_pdf(self.PDF_Path, flavor='stream', table_areas = [','.join([str(i) for i in self.Table_Coordinates])],
    #     edge_tol = self.Extraction_Parameter_Values['edge_tol'], row_tol = self.Extraction_Parameter_Values['row_tol'] # ,
    #     # line_scale = self.Extraction_Parameter_Values['line_scale']
    #     )
    #     pd_tables = []

    #     for table in tables:
    #         pd_tables.append(table.df)

    #     self.Tables = pd_tables

    #     return True

    def Extract_Tables_From_PDF_Page(self):

        # if self.Library_Option == "Camelot":
        #     self.Extract_Tables_From_PDF_Page_Camelot()

        if self.Library_Option == "PDF Plumber":
            self.Extract_Tables_From_PDF_Page_PDF_Plumber()

        elif self.Library_Option == "Tabula":
            self.Extract_Tables_From_PDF_Page_Tabula()

        return True

####################
# HELPER FUNCTIONS #
####################

## Creates page title
def p_title(title):
    st.markdown(f'
{title}
', unsafe_allow_html=True)

## Translates image data to byte data to display with markdown
def img_to_bytes(img_path):
    img_bytes = Path(img_path).read_bytes()
    encoded = base64.b64encode(img_bytes).decode()
    return encoded

## Saves uploaded PDF files to 'tempDir' and returns the pathname
def save_uploadedfile(uploadedfile):
    try:
        os.mkdir("tempDir")
    except:
        pass
    #tempDir_contents = str(glob.glob("tempDir/*"))
    with open(os.path.join("tempDir",uploadedfile.name),"wb") as f:
        f.write(uploadedfile.getbuffer())
    return os.path.join("tempDir", uploadedfile.name)

## Function for displaying editable canvas
def display_page_as_canvas(pdf_path, page):
    # Get original page coordinates
    input1 = PdfFileReader(open(pdf_path, 'rb'))
    pdf_coordinates = input1.getPage(0).mediaBox

    # Get image from PDF
    PDF = fitz.open(pdf_path)
    PDF_page = PDF.loadPage(page)  # number of page
    pix = PDF_page.get_pixmap()

    # Get canvas drawing
    coordinates = st_canvas(
    fill_color="rgba(255, 165, 0, 0)",
    stroke_width=5,
    stroke_color="rgba(255, 0, 0, 1)",
    background_image=Image.frombytes("RGB", [pix.width, pix.height], pix.samples),
    update_streamlit=True,
    drawing_mode="rect",
    point_display_radius=0,
    key="canvas",
    height = round(float(pdf_coordinates[3])),
    width = round(float(pdf_coordinates[2]))
    )

    # Return the coordinates of the bounding box
    return coordinates.image_data

def get_extraction_param_values(page):

    return_settings = {}

    for param in page.Extraction_Parameters.keys():
        if param == "Text Tolerance":
            return_settings[param] = 9.0
        elif param == "Snap Tolerance":
            return_settings[param] = 5.0
        else:
            return_settings[param] = 0

    for param in page.Extraction_Parameters:
        if param == "Vertical Strategy":
            val = st.radio(param, page.Extraction_Parameters[param], horizontal = True)
            st.markdown("**Vertical Strategy:** Whether columns in the table are separated by lines, or simply aligned by text with blank space in between.")
            return_settings[param] = val.lower()
        elif param == "Horizontal Strategy":
            val = st.radio(param, page.Extraction_Parameters[param], horizontal = True)
            st.markdown("**Horizontal Strategy:** Whether rows in the table are separated by lines, or simply aligned by text with blank space in between.")
            return_settings[param] = val.lower()

    with st.expander(f"See advanced extraction parameters"):
        for param in page.Extraction_Parameters:
            if param == "Text Tolerance":
                val = st.slider(param, min_value = page.Extraction_Parameters[param][0],
                max_value = page.Extraction_Parameters[param][1], step = page.Extraction_Parameters[param][2], value = 9.0)
                st.markdown("**Text Tolerance:** When the text strategy searches for words, it will expect the individual letters in each word to be no more than text_tolerance pixels apart.")
                return_settings[param] = val
            elif param == "Snap Tolerance":
                val = st.slider(param, min_value = page.Extraction_Parameters[param][0],
                max_value = page.Extraction_Parameters[param][1], step = page.Extraction_Parameters[param][2], value = 5.0)
                st.markdown("**Snap Tolerance:** Parallel lines within snap_tolerance pixels will be 'snapped' to the same horizontal or vertical position.")
                return_settings[param] = val
            elif param == "Join Tolerance":
                val = st.slider(param, min_value = page.Extraction_Parameters[param][0],
                max_value = page.Extraction_Parameters[param][1], step = page.Extraction_Parameters[param][2])
                st.markdown("**Join Tolerance:** Line segments on the same infinite line, and whose ends are within join_tolerance of one another, will be 'joined' into a single line segment.")
                return_settings[param] = val
            elif param == "Edge Minimum Length":
                val = st.slider(param, min_value = page.Extraction_Parameters[param][0],
                max_value = page.Extraction_Parameters[param][1], step = page.Extraction_Parameters[param][2])
                st.markdown("**Edge Minimum Length:** Edges shorter than edge_min_length will be discarded before attempting to reconstruct the table.")
                return_settings[param] = val
            elif param == "Intersection Tolerance":
                val = st.slider(param, min_value = page.Extraction_Parameters[param][0],
                max_value = page.Extraction_Parameters[param][1], step = page.Extraction_Parameters[param][2])
                st.markdown("**Intersection Tolerance:** When combining edges into cells, orthogonal edges must be within intersection_tolerance pixels to be considered intersecting.")
                return_settings[param] = val

    return return_settings

## Displays data as an editable AgGrid table
def display_tables(tables):

    data = tables[0]

    gb = GridOptionsBuilder.from_dataframe(data)
    gb.configure_default_column(editable=True)
    # gb.configure_pagination(paginationAutoPageSize=True) #Add pagination
    gb.configure_side_bar() #Add a sidebar
    gb.configure_selection('multiple', use_checkbox=True, groupSelectsChildren="Group checkbox select children")
    gridOptions = gb.build()

    grid_response = AgGrid(
        data,
        gridOptions=gridOptions,
        data_return_mode='FILTERED_AND_SORTED',
        update_mode='MODEL_CHANGED',
        fit_columns_on_grid_load=True,
        theme='blue', #Add theme color to the table
        enable_enterprise_modules=True,
        height=500,
        width='100%',
        reload_data=False
    )

    data = pd.DataFrame(grid_response['selected_rows']) #Pass the selected rows to a new dataframe df

    return data

## Returns a random key (when multiple instances of one widget is required)
def key_generator():
    return random.randint(1, 10000000)

## Function for structuring data into standard format
def structure_data(df, comps_type):
    # Get rid of "Unnamed" columns:
    for col in df.columns:
        if "UNNAMED" in col.upper():
            df = df.drop(columns=[col])

    # Uppercase column names
    df.columns = df.columns.str.upper()

    # Process land sales comps
    if comps_type == "Land Sales Comps":
        for col in df.columns:
            for key in land_sales_comps_dict.keys():
                if any(term in col for term in land_sales_comps_dict[key]):
                    df = df.rename(columns={col: key})

        for key in land_sales_comps_dict.keys():
            if key not in df.columns:
                df[key] = None

    # Process sales comps
    elif comps_type == "Sales Comps":
        for col in df.columns:
            for key in sales_comps_dict.keys():
                if any(term in col for term in sales_comps_dict[key]):
                    df = df.rename(columns={col: key})

        for key in sales_comps_dict.keys():
            if key not in df.columns:
                df[key] = None

    # Process lease comps
    elif comps_type == "Lease Comps":
        for col in df.columns:
            for key in lease_comps_dict.keys():
                if any(re.search(term, col) for term in lease_comps_dict[key]):
                    df = df.rename(columns={col: key})

        for key in lease_comps_dict.keys():
            if key not in df.columns:
                df[key] = None

    # Alphebetize columns
    df = df.reindex(sorted(df.columns), axis=1)

    return df


import pandas as pd
import numpy as np
import seaborn as sns
from sklearn import ensemble
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from math import sqrt

trainingData = pd.read_csv('C:/Users/.../train.csv')
testingData = pd.read_csv('C:/Users/.../test.csv')

##missing data
#total = trainingData.isnull().sum().sort_values(ascending=False)
#percent = (trainingData.isnull().sum()/trainingData.shape[0]).sort_values(ascending=False)
#missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
#print(missing_data.head(20))

columnsToDrop=['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'LotFrontage']
trainingData.drop(columnsToDrop, axis=1, inplace=True)
testingData.drop(columnsToDrop, axis=1, inplace=True)

##missing data
#total = trainingData.isnull().sum().sort_values(ascending=False)
#percent = (trainingData.isnull().sum()/trainingData.shape[0]).sort_values(ascending=False)
#missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
#print(missing_data.head(15))

##missing data
#total = testingData.isnull().sum().sort_values(ascending=False)
#percent = (testingData.isnull().sum()/testingData.shape[0]).sort_values(ascending=False)
#missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
#print(missing_data.head(30))

# fillna cause by no garage
trainingData['GarageType'].fillna('NA', inplace=True)
trainingData['GarageYrBlt'].fillna(trainingData['YearBuilt'], inplace=True)
trainingData['GarageFinish'].fillna('NA', inplace=True)
trainingData['GarageCond'].fillna('NA', inplace=True)
trainingData['GarageQual'].fillna('NA', inplace=True)

testingData['GarageType'].fillna('NA', inplace=True)
testingData['GarageYrBlt'].fillna(testingData['YearBuilt'], inplace=True)
testingData['GarageFinish'].fillna('NA', inplace=True)
testingData['GarageCond'].fillna('NA', inplace=True)
testingData['GarageQual'].fillna('NA', inplace=True)
testingData['GarageArea'].fillna(0.0, inplace=True)
testingData['GarageCars'].fillna(0.0, inplace=True)

# fillna cause by no basement
trainingData['BsmtFinType2'].fillna('NA', inplace=True)
trainingData['BsmtExposure'].fillna('NA', inplace=True)
trainingData['BsmtQual'].fillna('NA', inplace=True)
trainingData['BsmtFinType1'].fillna('NA', inplace=True)
trainingData['BsmtCond'].fillna('NA', inplace=True)

testingData['BsmtFinType2'].fillna('NA', inplace=True)
testingData['BsmtExposure'].fillna('NA', inplace=True)
testingData['BsmtQual'].fillna('NA', inplace=True)
testingData['BsmtFinType1'].fillna('NA', inplace=True)
testingData['BsmtCond'].fillna('NA', inplace=True)
testingData['BsmtFullBath'].fillna(0.0, inplace=True)
testingData['BsmtHalfBath'].fillna(0.0, inplace=True)
testingData['TotalBsmtSF'].fillna(0.0, inplace=True)
testingData['BsmtFinSF2'].fillna(0.0, inplace=True)
testingData['BsmtFinSF1'].fillna(0.0, inplace=True)
testingData['BsmtUnfSF'].fillna(0.0, inplace=True)

# fillna cause by no Masonry veneer
trainingData['MasVnrType'].fillna('None', inplace=True)
trainingData['MasVnrArea'].fillna(0.0, inplace=True)

testingData['MasVnrType'].fillna('None', inplace=True)
testingData['MasVnrArea'].fillna(0.0, inplace=True)

##missing data
#total = trainingData.isnull().sum().sort_values(ascending=False)
#percent = (trainingData.isnull().sum()/trainingData.shape[0]).sort_values(ascending=False)
#missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
#print(missing_data.head(5))
#
##missing data
#total = testingData.isnull().sum().sort_values(ascending=False)
#percent = (testingData.isnull().sum()/testingData.shape[0]).sort_values(ascending=False)
#missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
#print(missing_data.head(10))

#fill other missing data
trainingData['Electrical'].fillna(trainingData['Electrical'].mode()[0], inplace=True)

testingData['MSZoning'].fillna(trainingData['MSZoning'].mode()[0], inplace=True)
testingData['Utilities'].fillna(trainingData['Utilities'].mode()[0], inplace=True)
testingData['MSZoning'].fillna(trainingData['MSZoning'].mode()[0], inplace=True)
testingData['Exterior1st'].fillna(trainingData['Exterior1st'].mode()[0], inplace=True)
testingData['Exterior2nd'].fillna(trainingData['Exterior2nd'].mode()[0], inplace=True)
testingData['Functional'].fillna(trainingData['Functional'].mode()[0], inplace=True)
testingData['SaleType'].fillna(trainingData['SaleType'].mode()[0], inplace=True)
testingData['KitchenQual'].fillna(trainingData['KitchenQual'].mode()[0], inplace=True)

##missing data
#total = trainingData.isnull().sum().sort_values(ascending=False)
#percent = (trainingData.isnull().sum()/trainingData.shape[0]).sort_values(ascending=False)
#missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
#print(missing_data.head(5))
#
##missing data
#total = testingData.isnull().sum().sort_values(ascending=False)
#percent = (testingData.isnull().sum()/testingData.shape[0]).sort_values(ascending=False)
#missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
#print(missing_data.head(6))

#Add some combine attributes
trainingData['TotalSF'] = trainingData['TotalBsmtSF'] + trainingData['1stFlrSF'] + trainingData['2ndFlrSF']
trainingData['TotalBath'] = trainingData['BsmtFullBath'] + trainingData['BsmtHalfBath'] + trainingData['FullBath'] + trainingData['HalfBath']

testingData['TotalSF'] = testingData['TotalBsmtSF'] + testingData['1stFlrSF'] + testingData['2ndFlrSF']
testingData['TotalBath'] = testingData['BsmtFullBath'] + testingData['BsmtHalfBath'] + testingData['FullBath'] + testingData['HalfBath']
##drop some attributes
#trainingData.drop(['TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath'], axis=1, inplace=True)
#testingData.drop(['TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath'], axis=1, inplace=True)

#change some attributes to numerical
trainingData["ExterQual"] = trainingData["ExterQual"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})
trainingData["ExterCond"] = trainingData["ExterCond"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})
trainingData["HeatingQC"] = trainingData["HeatingQC"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})
trainingData["KitchenQual"] = trainingData["KitchenQual"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})

testingData["ExterQual"] = testingData["ExterQual"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})
testingData["ExterCond"] = testingData["ExterCond"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})
testingData["HeatingQC"] = testingData["HeatingQC"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})
testingData["KitchenQual"] = testingData["KitchenQual"].map({"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1})

#change dtypes
trainingData['MSSubClass'] = trainingData['MSSubClass'].astype(str)
testingData['MSSubClass'] = testingData['MSSubClass'].astype(str)

##check for outliers
#col=['GrLivArea', 'TotalSF']
#for c in col:
#    sns.relplot(x=c, y='SalePrice', data=trainingData)
#
#print(trainingData.shape)
trainingData.drop(trainingData[(trainingData['GrLivArea']>4000) & (trainingData['SalePrice']<300000)].index, inplace = True)
#
#for c in col:
#    sns.relplot(x=c, y='SalePrice', data=trainingData)
#
#print(trainingData.shape)

train_Y=trainingData['SalePrice']
train_X=trainingData.drop(['Id', 'SalePrice'], axis=1)

test_X=testingData.drop(['Id'], axis=1)

#print(train_X.dtypes)
#print(train_Y.shape)

#prosess categorical features
len_train_x=train_X.shape[0]
tmp_all=pd.concat(objs=[train_X, test_X], axis=0, sort=False).reset_index(drop=True)
tmp_all=pd.get_dummies(tmp_all)
tmp_all= tmp_all.astype(float)

new_train_X=tmp_all[:len_train_x]
new_test_X=tmp_all[len_train_x:]

#Set model
GBR = ensemble.GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=3, max_features='sqrt',
                                         min_samples_leaf=10, loss='huber')
#training error
GBR.fit(new_train_X, train_Y)
y_pred = GBR.predict(new_train_X)
y_pred=np.log(y_pred)
log_Y=np.log(train_Y)
rms = sqrt(mean_squared_error(y_pred, log_Y))
print('TE: ', rms)

# Splitting
x_train, x_test, y_train, y_test = train_test_split(new_train_X, train_Y, test_size=0.5)

GBR.fit(x_train, y_train)
y_pred = GBR.predict(x_test)
y_pred=np.log(y_pred)
y_test=np.log(y_test)

rms = sqrt(mean_squared_error(y_test, y_pred))
print('CV: ', rms)

#retraining with whole training data
numberOfModel=30
box=[]
result=np.zeros(new_test_X.shape[0])
for i in range(numberOfModel):
    GBR = ensemble.GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=3, max_features='sqrt',
                                             min_samples_leaf=10, loss='huber').fit(new_train_X, train_Y)
    box.append(GBR)
    # Getting our SalePrice estimation
    tmp = GBR.predict(new_test_X)
    result=result+tmp
    print('Model number: ', i, ' completes.')

result=result/numberOfModel

## Saving to CSV
pd.DataFrame({'Id': testingData.Id, 'SalePrice': result}).to_csv('C:/Users/.../output.csv', index =False)