import argparse
import time
import pickle
import os

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow_probability as tfp

from utils import Make_Video_batch, make_checkpoint_folder, pandas_res_saver, \
                  build_video_batch_graph, plot_latents, MSE_rotation
from utils_circles_grid import Make_circles, Make_squares, plot_circle, plot_square
from SVGPVAE_model import SVGP, build_SVGPVAE_elbo_graph
from GPVAE_Pearce_model import build_pearce_elbo_graphs

tfd = tfp.distributions
tfk = tfp.math.psd_kernels

def run_experiment(args):
    """
    Moving ball experiment.

    :param args:
    :return:
    """
    if args.save:
        # Make a folder to save everything
        extra = args.elbo + "_" + str(args.beta0)

        chkpnt_dir = make_checkpoint_folder(args.base_dir, args.expid, extra)
        pic_folder = chkpnt_dir + "pics/"
        res_file = chkpnt_dir + "res/ELBO_pandas"
        print("\nCheckpoint Directory:\n"+str(chkpnt_dir)+"\n")

    # Data synthesis settings
    batch = 35
    tmax = args.tmax
    px = 32
    py = 32
    r = 3
    vid_lt = args.vidlt
    m = args.m

    if args.elbo == 'VAE':
        # A GP prior with a RBF kernel and a very small length scale reduces to the standard Gaussian prior
        model_lt = 0.001
    else:
        model_lt = args.modellt

    assert model_lt == vid_lt or args.GP_joint or args.elbo == 'VAE', \
        "GP params of data and model should match. Except when \
         doing a joint optimization of GP parameters or when fitting normal VAE."

    # Load/create batches of reproducible videos
    if os.path.isfile(args.base_dir + "/Test_Batches_{}_{}.pkl".format(vid_lt, tmax)):
        with open(args.base_dir + "/Test_Batches_{}_{}.pkl".format(vid_lt, tmax), "rb") as f:
            Test_Batches = pickle.load(f)
    else:
        make_batch = lambda s: Make_Video_batch(tmax=tmax, px=px, py=py, lt=vid_lt, batch=batch, seed=s, r=r)
        Test_Batches = [make_batch(s) for s in range(10)]
        with open(args.base_dir + "/Test_Batches_{}_{}.pkl".format(vid_lt, tmax), "wb") as f:
            pickle.dump(Test_Batches, f)

    # Initialise a plots
    # this plot displays a  batch of videos + latents + reconstructions
    if args.save or args.show_pics:
        fig, ax = plt.subplots(4, 4, figsize=(8, 8), constrained_layout=True)
        plt.ion()

    if args.squares_circles:
        truth_c, V_c = Make_circles(tmax=tmax); batch_V_c = np.tile(V_c, (batch,1,1,1))
        truth_sq, V_sq = Make_squares(tmax=tmax); batch_V_sq = np.tile(V_sq, (batch,1,1,1))

    # make sure everything is created in the same graph!
    graph = tf.Graph()
    with graph.as_default():

        # Make all the graphs
        beta = tf.compat.v1.placeholder(dtype=tf.float32, shape=())

        vid_batch = build_video_batch_graph(batch=batch, tmax=tmax, px=px, py=py, r=r, lt=vid_lt)

        if args.elbo in ['GPVAE_Pearce', 'VAE', 'NP']:
            elbo, rec, pkl, p_m, \
                p_v, q_m, q_v, pred_vid, \
                l_GP_x, l_GP_y, _ = build_pearce_elbo_graphs(vid_batch, beta, type_elbo=args.elbo, lt=model_lt,
                                                             GP_joint=args.GP_joint, GP_init=args.GP_init)

        else:  # SVGPVAE_Titsias, SVGPVAE_Hensman

            titsias = 'Titsias' in args.elbo
            fixed_gp_params = not args.GP_joint
            fixed_inducing_points = not args.ip_joint
            svgp_x_ = SVGP(titsias=titsias, num_inducing_points=m,
                           fixed_inducing_points=fixed_inducing_points,
                           tmin=1, tmax=tmax, vidlt=vid_lt, fixed_gp_params=fixed_gp_params, name='x',
                           jitter=args.jitter, ip_min=args.ip_min, ip_max=args.ip_max, GP_init=args.GP_init)
            svgp_y_ = SVGP(titsias=titsias, num_inducing_points=m, fixed_inducing_points=fixed_inducing_points,
                           tmin=1, tmax=tmax, vidlt=vid_lt, fixed_gp_params=fixed_gp_params, name='y',
                           jitter=args.jitter, ip_min=args.ip_min, ip_max=args.ip_max, GP_init=args.GP_init)

            elbo, rec, pkl, l3_elbo, ce_term,\
            p_m, p_v, q_m, q_v, pred_vid, l_GP_x, l_GP_y,\
            l3_elbo_recon, l3_elbo_kl, inducing_points_x, inducing_points_y, \
            gp_cov_full_mean_x, gp_cov_full_mean_y, _ = build_SVGPVAE_elbo_graph(vid_batch, beta,
                                                                                svgp_x=svgp_x_, svgp_y=svgp_y_,
                                                                                clipping_qs=args.clip_qs)

        # The actual loss functions
        loss = -tf.reduce_mean(elbo)
        e_elb = tf.reduce_mean(elbo)
        e_pkl = tf.reduce_mean(pkl)
        e_rec = tf.reduce_mean(rec)
        if 'SVGPVAE' in args.elbo:
            e_l3_elbo = tf.reduce_mean(l3_elbo)
            e_ce_term = tf.reduce_mean(ce_term)
            e_l3_elbo_recon = tf.reduce_mean(l3_elbo_recon)
            e_l3_elbo_kl = tf.reduce_mean(l3_elbo_kl)

        # Add optimizer ops to graph (minimizing neg elbo!), print out trainable vars
        global_step = tf.Variable(0, name='global_step', trainable=False)
        optimizer = tf.compat.v1.train.AdamOptimizer()
        train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)

        if args.clip_grad:
            gradients = tf.gradients(loss, train_vars)
            gradients = [tf.clip_by_value(grad, -100000.0, 100000.0) for grad in gradients]
            optim_step = optimizer.apply_gradients(grads_and_vars=zip(gradients, train_vars),
                                                   global_step=global_step)

        else:
            optim_step = optimizer.minimize(loss=loss,
                                            var_list=train_vars,
                                            global_step=global_step)

        print("\n\nTrainable variables:")
        for v in train_vars:
            print(v)

        # Initializer ops for the graph and saver
        init_op = tf.global_variables_initializer()
        saver = tf.compat.v1.train.Saver()

        if args.save:
            # Results to be tracked and Pandas saver
            res_vars = [global_step,
                        loss,
                        e_elb,
                        e_rec,
                        e_pkl,
                        tf.math.reduce_min(q_v),
                        tf.math.reduce_max(q_v),
                        tf.math.reduce_min(p_v),
                        tf.math.reduce_max(p_v),
                        tf.math.reduce_min(q_m),
                        tf.math.reduce_max(q_m),
                        tf.math.reduce_min(p_m),
                        tf.math.reduce_max(p_m),
                        l_GP_x,
                        l_GP_y]
            if 'SVGPVAE' in args.elbo:
                res_vars += [e_l3_elbo,
                             e_ce_term,
                             e_l3_elbo_recon,
                             e_l3_elbo_kl,
                             inducing_points_x,
                             inducing_points_y,
                             gp_cov_full_mean_x,
                             gp_cov_full_mean_y]
            res_names= ["Step",
                        "Loss",
                        "Train ELBO",
                        "Train Reconstruction",
                        "Train Prior KL",
                        "min qs_var",
                        "max qs_var",
                        "min q_var",
                        "max q_var",
                        'min qs_mean',
                        'max qs_mean',
                        'min q_mean',
                        'max q_mean',
                        "l_GP_x",
                        "l_GP_y"]
            if 'SVGPVAE' in args.elbo:
                res_names += ['SVGP elbo',
                              'ce term',
                              'SVGP elbo recon',
                              'SVGP elbo KL',
                               'inducing_points_x',
                              'inducing_points_y',
                              'gp_cov_full_mean_x',
                              'gp_cov_full_mean_y']
            res_names += ["MSE",  "Beta",  "Time"]
            res_saver = pandas_res_saver(res_file, res_names)

        # Now let's start doing some computation!
        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.ram)
        with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:

            # Attempt to restore weights
            try:
                saver.restore(sess, tf.train.latest_checkpoint(chkpnt_dir))
                print("\n\nRestored Model Weights")
            except:
                sess.run(init_op)
                print("\n\nInitialised Model Weights")

            # Start training that elbo!

            for t in range(args.steps):

                # Train: do an optim step
                _, g_s = sess.run([optim_step, global_step], {beta: args.beta0})

                # Print out diagnostics/tracking
                if g_s % 1000 == 0:
                    TD = Test_Batches[0][1]
                    if 'SVGPVAE' in args.elbo:
                        test_elbo, e_rec_i, e_pkl_i, e_l3_elbo_i, e_ce_term_i, e_l3_elbo_recon_i, e_l3_elbo_kl_i = \
                            sess.run([e_elb, e_rec, e_pkl, e_l3_elbo, e_ce_term, e_l3_elbo_recon, e_l3_elbo_kl],
                                     {vid_batch: TD, beta: 1.0})

                    else:
                        test_elbo, e_rec_i, e_pkl_i = sess.run([e_elb, e_rec, e_pkl], {vid_batch: TD, beta: 1.0})

                    test_qv, test_pv, test_pm, test_qm = sess.run([q_v, p_v, p_m, q_m], {vid_batch: TD, beta: 1.0})

                    print(str(g_s)+": elbo "+str(test_elbo))
                    print("Recon term: {}. KL term: {}.".format(e_rec_i, e_pkl_i))
                    if 'SVGPVAE' in args.elbo:
                        print("L{} elbo term: {}. CE term: {}.".format(2 if titsias else 3, e_l3_elbo_i, e_ce_term_i))
                        if not titsias:
                            print("L3 elbo recon term: {}. L3 elbo KL term: {}.".format(e_l3_elbo_recon_i,
                                                                                e_l3_elbo_kl_i))
                    print("VAE posterior variance range: min {}, max  {}".format(np.min(test_qv), np.max(test_qv)))
                    print("VAE posterior mean range: min {}, max {}".format(np.min(test_qm), np.max(test_qm)))
                    print("GP approx posterior variance range: min {}, max {}".format(np.min(test_pv), np.max(test_pv)))
                    print("GP approx posterior mean range: min {}, max {}".format(np.min(test_pm), np.max(test_pm)))

                # Save elbo, recon, priorKL....
                if g_s % 1000 == 0:
                    TT, TD = Test_Batches[0]
                    p_m_i, p_v_i = sess.run([p_m, p_v], {vid_batch: TD, beta: 1})

                    _, _, MSE, _ = MSE_rotation(p_m_i, TT, p_v_i)
                    print('MSE : {}'.format(MSE))

                    if args.save:
                        new_res = sess.run(res_vars, {vid_batch: TD, beta: 1})
                        new_res += [MSE, args.beta0, time.time()]
                        res_saver(new_res)

                # show plot and occasionally save
                if g_s % 10000 == 0 and (args.save or args.show_pics):
                    # [[ax_ij.clear() for ax_ij in ax_i] for ax_i in ax]
                    TT, TD = Test_Batches[0]
                    reconpath, reconvar, reconvid = sess.run([p_m, p_v, pred_vid], {vid_batch:TD, beta:1})

                    rp, W, MSE, rv = MSE_rotation(reconpath, TT, reconvar)
                    _ = plot_latents(TD, TT, reconvid, rp, rv, ax=ax, nplots=4)
                    # plt.tight_layout()
                    plt.draw()
                    fig.suptitle(str(g_s)+' ELBO: ' + str(test_elbo))

                    if args.squares_circles:
                        q_m_c = sess.run(q_m, {vid_batch: batch_V_c})
                        q_m_sq = sess.run(q_m, {vid_batch: batch_V_sq})

                        # import pdb; pdb.set_trace()

                        q_m_c = np.hstack([ q_m_c[0,:,:], np.ones((tmax, 1))])
                        rot_qnet_c = np.matmul(q_m_c, W)
                        plot_circle(ax[3][0], ax[3][1], rot_qnet_c)

                        q_m_sq = np.hstack([ q_m_sq[0,:,:], np.ones((tmax, 1))])
                        rot_qnet_sq = np.matmul(q_m_sq, W)
                        plot_square(ax[3][2], ax[3][3], rot_qnet_sq)

                    if args.show_pics:
                        plt.show()
                        plt.pause(0.01)
                    if args.save:
                        plt.savefig(pic_folder + str(g_s).zfill(6)+".png")

                # Save NN weights
                if args.save and g_s % 50000 == 0:
                    saver.save(sess, chkpnt_dir+"model", global_step=g_s)
                    print("\n\nModel Saved: " + chkpnt_dir +"\n\n")

                # Save predicted trajectories for plotting purposes
                if args.save and g_s % 50000 == 0:
                    preds = (TD, TT, reconvid, rp, rv)
                    pickle.dump(preds, open(chkpnt_dir + "preds/preds_{}.p".format(g_s), "wb"))

if __name__=="__main__":

    default_base_dir = os.getcwd()

    parser = argparse.ArgumentParser(description='Moving ball experiment')
    parser.add_argument('--steps', type=int, default=25000, help='Number of steps of Adam')
    parser.add_argument('--beta0', type=float, default=1, help='initial beta annealing value')
    parser.add_argument('--elbo', type=str, choices=['GPVAE_Pearce', 'VAE', 'NP', 'SVGPVAE_Hensman', 'SVGPVAE_Titsias'],
                        default='GPVAE_Pearce',
                        help='Structured Inf Nets ELBO or Neural Processes ELBO')
    parser.add_argument('--modellt', type=float, default=2, help='time scale of model to fit to data')
    parser.add_argument('--base_dir', type=str, default=default_base_dir, help='folder within a new dir is made for each run')
    parser.add_argument('--expid', type=str, default="debug", help='give this experiment a name')
    parser.add_argument('--ram', type=float, default=0.5, help='fraction of GPU ram to use')
    parser.add_argument('--seed', type=int, default=None, help='seed for rng')
    parser.add_argument('--tmax', type=int, default=30, help='length of videos')
    parser.add_argument('--m', type=int, default=15, help='number of inducing points')
    parser.add_argument('--GP_joint', action="store_true", help='GP hyperparams joint optimization.')
    parser.add_argument('--ip_joint', action="store_true", help='Inducing points joint optimization.')
    parser.add_argument('--clip_qs', action="store_true", help='Clip variance of inference network.')

    parser.add_argument('--show_pics', action="store_true", help='Show images during training.')
    parser.add_argument('--save', action="store_true", help='Save model metrics in Pandas df as well as images.')
    parser.add_argument('--squares_circles', action="store_true", help='Whether or not to plot squares and circles.')

    parser.add_argument('--ip_min', type=int, default=1, help='ip start')
    parser.add_argument('--ip_max', type=int, default=30, help='ip end')
    parser.add_argument('--jitter', type=float, default=1e-9, help='noise for GP operations (inverse, cholesky)')
    parser.add_argument('--clip_grad', action="store_true", help='Whether or not to clip gradients.')
    parser.add_argument('--vidlt', type=float, default=2, help='time scale for data generation')
    parser.add_argument('--GP_init', type=float, default=2,
                        help='Initial value for GP kernel length scale. Used when running --GP_joint .')

    args = parser.parse_args()

    run_experiment(args)


# -*- coding: cp1252 -*-
from finefun import *
import subprocess
import os, sys, re, time, gzip
import csv
from tempfile import NamedTemporaryFile
import shutil
import logging
import subprocess
from io import StringIO
#import material_file
#from db import *

from datetime import datetime
import unidecode
import urllib
import urllib.parse

# /// HISTORY/CHANGE LOG -----------------------------------------------------
# DATE            AUTHER                                    ACTION
# 2018-06-19      Saiswarup Sahu <ssahu@artinfo.com>        New version program          Ticket #
# Modified: Supriyo, July 20, 2022.
emailIdPattern = re.compile(r"\W(\w+\.?\w{0,}@\w+\.\w+\.?\w*)\W", re.MULTILINE | re.DOTALL)
absUrlPattern = re.compile(r"^https?:\/\/", re.IGNORECASE)
anchorTagPattern = re.compile(r"]{0,}href=([^\s\>]+)\s?.*?>\s*\w+", re.IGNORECASE | re.MULTILINE | re.DOTALL)
doubleQuotePattern = re.compile('"', re.MULTILINE | re.DOTALL)
htmlTagPattern = re.compile(r"<[^>]+>", re.MULTILINE | re.DOTALL)
newlinePattern = re.compile(r"\n")
multipleWhitespacePattern = re.compile(r"\s+")
pathEndingWithSlashPattern = re.compile(r"\/$")
javascriptUrlPattern = re.compile("^javascript:")
startsWithSlashPattern = re.compile("^/")
htmlEntitiesDict = {' ': ' ', ' ': ' ', '&': '&', '&': '&', '<': '<', '<': '<', '>': '>',
                    '>': '>', ''': '\'', ''': '\'', '"': '"', '"': '"', '–': '-',
                    '€': 'Euro', '…': '...'}

def quoteText(content):
    content = str(stripHtmlEntities(content))
    content = content.replace('"', '\"')
    content = '"' + content + '"'
    return content

def stripHtmlEntities(content):
    for entityKey in htmlEntitiesDict.keys():
        entityKeyPattern = re.compile(entityKey)
        content = re.sub(entityKeyPattern, htmlEntitiesDict[entityKey], content)
    return content

def getDimaision(dim):
    d = dim.replace(".","99999999").replace(",","").replace("x"," ").replace("/","05123450").split(" ")
    dimList = []
    height = "0"
    width = "0"
    depth = "0"
    size = ""
    for rrr in d:
        s = ''.join([i for i in rrr if i.isdigit()])
        if s.__len__() != 0:
            dimList.append(s.replace("99999999",".").replace("05123450","/"))
    for line in dimList:
        if "/" in line:
            index = dimList.index(line)
            dimList[dimList.index(dimList[index-1])] = getDimansionPart(dimList[index-1] +" "+ line)
            dimList[index] = ""
    #print(dimList)
    if dimList.__len__() != 0:
        dimaisionList = dimList
        if dimaisionList.__len__() == 3 or dimaisionList.__len__() > 3:
            height = dimaisionList[0]
            width = dimaisionList[1]
            depth = dimaisionList[2]
            size = height + " x " + width + " x " + depth
        if dimaisionList.__len__() == 2:
            height = dimaisionList[0]
            width = dimaisionList[1]
            depth = "0"
            size = height + " x " + width
        if dimaisionList.__len__() == 1:
            height = dimaisionList[0]
            width = "0"
            depth = "0"
            size = height
        return height,width,depth,size
    else:
        return "0","0","0","",None

def getMaterial(descriptionList):
    mediumPattern = re.compile("(gold)|(silver)|(brass)|(bronze)|(wood)|(porcelain)|(stone)|(marble)|(metal)|(steel)|(iron)|(glass)|(plexiglas)|(chromogenic)|(paper)|(canvas)|(masonite)|(newsprint)|(silkscreen)|(parchment)|(linen)|(inkjet)|(ink\s+jet)|(pigment)|(\stin\s)|(photogravure)|(^oil\s+)|(\s+panel\s+)|(terracotta)|(ivory)|(\s+oak\s+)|(^oak\s+)|(\s+oak$)|(alabaster)|(\s+ink\s+)|(pencil)|(albumen)|(oil\s+)|(\s+oil)|(panel)|(acrylic)|(print)|(Huile)|(toile)|(plume)|(encre)|(lavis)|(Gravure)|(Pierre)|(pastel)|(panneau)|(Sanguine)|(Crayon)|(aquarelle)|(papier)|(stampa\s+offset)|(tappeto)", re.DOTALL|re.IGNORECASE)
    try:
        materialList = [x for x in descriptionList if re.search(
                            materialDict["material"],
                        x.lower()) and yearExtract(
                        x).__len__() == 0 and "numbered" not in x.lower() and "date" not in x.lower()
                                    and ":" not in x and "framed" not in x.lower()
                        and "cm" not in x.lower() and "marked" not in x.lower() and "sign" not in x.lower() and "label" not in x.lower()]
        material = materialList[0].strip()
        if "bronze" in material:
            material = "bronze"
        mps = re.search(mediumPattern, material)
        if mps:
            material = mps.groups()[0]
        if "(" in material:
            material=material.split("(")[0].strip()
        return material
    except:
        return ""

def stripHTML(dataitem):
    dataitem = re.sub(htmlTagPattern, "", dataitem)  # stripped off all HTML tags...
    # Handle HTML entities...
    for entity in htmlEntitiesDict.keys():
        dataitem = dataitem.replace(entity, htmlEntitiesDict[entity])
    return (dataitem)

class Scrape:
    def __init__(self, auctionId, auctionUrl, downloadImages, fp):
        self.auctionId = auctionId
        self.mainUrl = auctionUrl
        self.domainUrl = "https://www.farsettiarte.it"
        self.downloadImages = downloadImages
        self.scrapperName = "Farsettiarte"
        self.fp = fp
        self.run()

    def run(self):
        nextPage = True
        soup = get_soup(self.mainUrl)
        self.writeHeaders(soup)  # Write the header part of csv
        try:
            total_page_nos = 24
        except:
            total_page_nos = ""
            pass
        httpHeaders = { 'User-Agent' : r'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36',  'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'Accept-Language' : 'en-us,en;q=0.5', 'Accept-Encoding' : 'gzip,deflate', 'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Connection' : 'keep-alive', }
        httpHeaders['cache-control'] = "max-age=0"
        httpHeaders['sec-ch-ua'] = "\".Not/A)Brand\";v=\"99\", \"Google Chrome\";v=\"103\", \"Chromium\";v=\"103\""
        httpHeaders['sec-ch-ua-mobile'] = "?0"
        httpHeaders['sec-ch-ua-platform'] = "Linux"
        httpHeaders['upgrade-insecure-requests'] = "1"
        httpHeaders['sec-fetch-dest'] = "document"
        httpHeaders['sec-fetch-mode'] = "navigate"
        httpHeaders['sec-fetch-site'] = "same-origin"
        httpHeaders['sec-fetch-user'] = "?1"
        httpHeaders['referer'] = self.mainUrl
        auctionurlparts = auctionUrl.split("?")
        aucurl = auctionurlparts[0]
        next_page_urls = [aucurl +"?pag="+ str(x) + "#catalogue" for x in range(1, total_page_nos + 1)]
        pagectr = 1
        for page_url in next_page_urls:
            print(page_url)
            #d = datetime.now()
            #dd = d.strftime("%Y-%m-%dT%H:%M:%SZ")
            #dd = dd.replace(":", "%3A")
            #httpHeaders['cookie'] = "registrazione%5FCasa+d%27aste+Farsettiarte=pagineVis=" + str(pagectr) + "&attivato=0;_jsuid=1397622566; _heatmaps_g2g_100718721=no; _iub_cs-97000345=%7B%22timestamp%22%3A%22" + str(dd) + "%22%2C%22version%22%3A%221.40.0%22%2C%22purposes%22%3A%7B%221%22%3Atrue%2C%224%22%3Atrue%7D%2C%22id%22%3A97000345%2C%22cons%22%3A%7B%22rand%22%3A%22869ac2%22%7D%7D;"
            #httpHeaders['cookie'] = "registrazione%5FCasa+d%27aste+Farsettiarte=pagineVis=1&attivato=0; ASPSESSIONIDQGCSARAS=NMLHBBPCKEEMIBBNDKDLLKLP; _fbp=fb.1.1658238082444.1245240520; _ga=GA1.2.1260475549.1658238083; _gid=GA1.2.1409088931.1658238083; _jsuid=1397622566; _heatmaps_g2g_100718721=no; _iub_cs-97000345=%7B%22timestamp%22%3A%222022-07-19T13%3A41%3A43.245Z%22%2C%22version%22%3A%221.40.0%22%2C%22purposes%22%3A%7B%221%22%3Atrue%2C%224%22%3Atrue%7D%2C%22id%22%3A97000345%2C%22cons%22%3A%7B%22rand%22%3A%22869ac2%22%7D%7D; _ga_GTGLLXD3VR=GS1.1.1658238082.1.1.1658238108.0"
            pagectr += 1
            soup = get_soup(page_url, httpHeaders=httpHeaders)
            productDetails = soup.findAll('div', {'class': 'image-holder-list'})#?pag=2
            for product in productDetails:
                auction_house_name = "Farsettiarte"
                auction_location = "Toronto"
                auction_num = ""
                auction_start_date = ""
                auction_end_date = ""
                auction_name = ''
                lot_num = ""
                sublot_num = ""
                price_kind = ""
                price_estimate_min = ""
                price_estimate_max = ""
                price_sold = "0"
                artist_name = ""
                artist_birth = ""
                artish_death = ""
                artist_nationallity = ""
                artwork_name = ""
                artwork_year_identifier = ""
                artwork_start_year = ""
                artwork_end_year = ""
                artwork_materials = ""
                artwork_category = ""
                artwork_markings = ""
                artwork_edition = ""
                artwork_description = ""
                artwork_measurements_height = "0"
                artwork_measurements_width = "0"
                artwork_measurements_depth = "0"
                artwork_size_notes = ""
                auction_measureunit = ""
                artwork_condition_in = ""
                artwork_provenance = ""
                artwork_exhibited = ""
                artwork_literature = ""
                artwork_images1 = ""
                artwork_images2 = ""
                artwork_images3 = ""
                artwork_images4 = ""
                artwork_images5 = ""
                image1_name = ""
                image2_name = ""
                image3_name = ""
                image4_name = ""
                image5_name = ""
                lot_origin_url = ""

                aTag = product.find("a")['href']
                try:
                    lot_origin_url = aTag
                except:
                    pass
                try:
                    detailPageSoup = get_soup(aTag)
                except:
                    pass
                try:
                    lot_num = ''.join(re.findall('\d+',detailPageSoup.find('div',{'class':'number'}).getText().strip()))
                    print(lot_num)
                except:
                    pass
                try:
                    artist_name_data = detailPageSoup.find('div',{'id': 'autore'})
                except:
                    pass
                try:
                    artist_name = re.sub("['('].*",'',artist_name_data.getText())
                except:
                    pass
                try:
                    artist_nationallity = detailPageSoup.find('div',{'class': 'dateAuthor'}).getText().replace("['(']","").replace("[')']","")
                    artist_nationallity=''.join(re.findall("['('](.*?\s)",artist_nationallity))
                    artist_nationallity=re.sub("[')']","",artist_nationallity)
                    #print(artist_nationallity)
                except:
                    pass
                try:
                    years = re.findall("\d{4}", artist_name_data.getText())
                    artist_birth, artish_death = ['', '']
                    if len(years) > 1:
                        artist_birth = years[0]
                        artish_death = years[1]
                    elif len(years) == 1:
                        artist_birth = years[0]
                except:
                    pass
                try:
                    artwork_name = detailPageSoup.find('div',{'id': 'titleLotto'}).getText()
                except:
                    pass
                try:
                    artwork_start_year, artwork_end_year = getStartEndYear(artwork_name)
                except:
                    pass
                try:
                    estimation = detailPageSoup.find('b',{'class':'didascalia'}).getText()
                    priceList = re.findall('\d+',estimation)
                    if priceList.__len__() == 2:
                        price_estimate_min = ''.join(re.findall("\d+", X(priceList[0].strip())))
                        price_estimate_max = ''.join(re.findall("\d+", X(priceList[1].strip())))
                    if priceList.__len__() == 1:
                        price_estimate_min = ''.join(re.findall("\d+", X(priceList[0].strip())))
                        price_estimate_max = "0"
                except:
                    pass

                if price_estimate_min == '':
                    price_estimate_min = '0'
                if price_estimate_max == '':
                    price_estimate_max = '0'
                if price_sold == '':
                    price_sold = '0'
                if price_sold == "0" and price_estimate_min != "0":
                    price_kind = "estimate"
                elif price_sold != "0":
                    price_kind = "price realized"
                else:
                    price_kind = "unknown"
                try:
                    All_Data = detailPageSoup.find('div', {'class': 'OperaText'}).getText(separator=u'@').strip().replace('@@@', '@').replace('@@', '@').split("@")
                except:
                    pass
                try:
                    artwork_materials = getMaterial(All_Data)
                    print(artwork_materials)
                except:
                    pass
                try:
                    artwork_markings = getSignature(All_Data)
                except:
                    pass
                try:
                    dimension = [x for x in All_Data if "cm." in x.strip().lower()]
                    dimension = dimension[0].replace(',', '.').replace("cm.","").strip()
                    auction_measureunit = 'cm'
                    dimaisionList = getDimaision(dimension)
                    artwork_measurements_height = dimaisionList[0].replace("1.9", "19.").replace(".9", "9.")
                    artwork_measurements_width = dimaisionList[1].replace("1.9", "19.").replace(".9", "9.")
                    artwork_measurements_depth = dimaisionList[2].replace("1.9", "19.").replace(".9", "9.")
                except:
                    pass
                try:
                    artwork_size_notes = artwork_measurements_height + 'x' + artwork_measurements_width + 'x' + artwork_measurements_depth + " " + 'cm'
                    endxPattern = re.compile("x\.?\s+(cm)|(in)")
                    artwork_size_notes = endxPattern.sub(" cm", artwork_size_notes)
                    print(artwork_size_notes)
                except:
                    pass

                try:
                    auction_name = detailPageSoup.find('h2',{'class': 'mb-0'}).getText()
                except:
                    pass
                try:
                    auction_num=''.join(re.findall('\d+',auction_name))
                except:
                    pass
                try:
                    artwork_images1 = 'https://www.farsettiarte.it' + detailPageSoup.find("div", {"class": "stageImg"}).find("img")['src']
                    #image1_name = scrapperName + '-' + auction_location + '-'+auction_num+ '-'+auction_name+'-'+auction_start_date + '-' + lot_num.strip()+ '-a' + '.jpg'
                    processedArtistName = artist_name.replace(" ", "_")
                    processedArtistName = unidecode.unidecode(processedArtistName)
                    newname1 = str(auction_num) + "__" + processedArtistName + "__" + str(lot_num).strip() + "_a.jpg"
                    image1_name = newname1
                    image1_name=re.sub(r"\s+", "", image1_name, flags=re.UNICODE)
                except:
                    pass
                try:
                    date_time_auction_details = detailPageSoup.find('div',{'class': 'dataAsta'}).getText().strip().split(' ')
                    auction_start_date = ''.join(re.findall('\d+',date_time_auction_details[-3])) + '-' + date_time_auction_details[-2][:3] + '-' + date_time_auction_details[-1][-2:]
                except:
                    pass
                try:
                    artwork_description = "
Description:
" + lot_num+' '+detailPageSoup.find('div',{'id':'autore'}).getText(separator=u'  ')+' '+detailPageSoup.find('div',{'id':'titleLotto'}).getText(separator=u'  ')+' '+detailPageSoup.find('div',{'id':'descLotto'}).getText(separator=u'  ')
                except:
                    pass
                try:
                    fun=translatToEnglish(artwork_materials)
                except:
                    pass
                try:
                    if artwork_materials.__len__() != 0 or artwork_materials != "":
                        artwork_materials_data = fun.replace(',', '').split(' ')[0]
                        if artwork_materials_data.lower() == "hanging":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "mounted":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "oil":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "acrylic":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "tempera":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "enamel paint":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "watercolor":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "pencil":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "crayon":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "pastel":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "gouache":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "oil pastel":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "grease pencil":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "ink":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "pen":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "lithograph":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "screenprint":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "etching":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "engraving":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "woodcut":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "poster":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "linocut":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "monotype":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "c-print":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "gelatin silver print":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "platinum":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "daguerreotype":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "photogravure":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "dye transfer print":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "polaroid":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "ink-jet":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "video":
                            artwork_category = "Sculpture"
                        if artwork_materials_data.lower() == "chromogenic":
                            artwork_category = "print"
                        if artwork_materials_data.lower() == "mixed":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "gelatin":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "bronze":
                            artwork_category = "Sculptures"
                        if artwork_materials_data.lower() == "sketch":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "colored":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "silkscreen":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "serigraph":
                            artwork_category = "Prints"
                except:
                    pass

                try:
                    if artwork_category == '':
                        artwork_materials_data = artwork_materials.strip().replace('\t', '').split(' ')[0]
                        # print(artwork_materials_data), 'saisidididhdihdihd'
                        if artwork_materials_data.lower() == "hanging":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "mounted":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "oil":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "acrylic":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "tempera":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "enamel paint":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "watercolor":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "pencil":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "crayon":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "pastel":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "gouache":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "oil pastel":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "grease pencil":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "ink":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "pen":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "lithograph":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "screenprint":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "etching":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "engraving":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "woodcut":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "poster":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "linocut":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "monotype":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "c-print":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "gelatin silver print":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "platinum":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "daguerreotype":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "photogravure":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "dye transfer print":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "polaroid":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "ink-jet":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "video":
                            artwork_category = "Sculpture"
                        if artwork_materials_data.lower() == "chromogenic":
                            artwork_category = "print"
                        if artwork_materials_data.lower() == "mixed":
                            artwork_category = "Paintings"
                        if artwork_materials_data.lower() == "gelatin":
                            artwork_category = "Photographs"
                        if artwork_materials_data.lower() == "bronze":
                            artwork_category = "Sculptures"
                        if artwork_materials_data.lower() == "sketch":
                            artwork_category = "Works on paper"
                        if artwork_materials_data.lower() == "colored":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "silkscreen":
                            artwork_category = "Prints"
                        if artwork_materials_data.lower() == "serigraph":
                            artwork_category = "Prints"
                        if artwork_materials_data == "Olio":
                            artwork_category = "Prints"
                        if artwork_materials_data == "Tempera":
                            artwork_category = "Prints"
                        if artwork_materials_data == "Scultura":
                            artwork_category = "Sculptures"
                except:
                    pass

                rX = lambda x: " ".join(x.replace(",", "").replace("\n", "").replace("\t", "").replace('"', "").splitlines())
                artwork_markings = artwork_markings.replace('"', "'").replace(";", " ")
                artwork_description = artwork_description.replace('"', "'").replace(";", " ")
                artwork_materials = artwork_materials.replace('"', "'").replace(";", " ")
                fp.write(
                        rX(auction_house_name) + ',' + rX(auction_location) + ',' + rX(auction_num) + ',' + rX(
                            auction_start_date) + ',' + rX(auction_end_date) + ',' + rX(auction_name) + ','
                        + rX(lot_num) + ',' + rX(sublot_num) + ',' + rX(price_kind) + ',' + rX(
                            price_estimate_min) + ',' + rX(price_estimate_max) + ',' + rX(price_sold) + ',' + rX(
                            artist_name) + ','
                        + rX(artist_birth) + ',' + rX(artish_death) + ',' + rX(artist_nationallity) + ',' + rX(
                            artwork_name) + ',' +
                        rX(artwork_year_identifier) + ',' + rX(artwork_start_year) + ',' + rX(
                            artwork_end_year) + ',' + rX(
                            artwork_materials) + ',' + rX(artwork_category) + ',' + rX(artwork_markings.strip()) + ','
                        + rX(artwork_edition) + ',' + rX(artwork_description) + ',' + rX(
                            artwork_measurements_height) + ',' + rX(artwork_measurements_width) + ',' + rX(
                            artwork_measurements_depth) + ',' +
                        rX(artwork_size_notes) + ',' + rX(auction_measureunit) + ',' + rX(
                            artwork_condition_in) + ',' + rX(
                            artwork_provenance) + ',' + rX(artwork_exhibited) + ',' + rX(artwork_literature) + ',' +
                        rX(artwork_images1) + ',' + rX(artwork_images2) + ',' + rX(artwork_images3) + ',' + rX(
                            artwork_images4) + ',' + rX(artwork_images5) + ',' +
                        rX(image1_name) + ',' + rX(image2_name) + ',' + rX(image3_name) + ',' + rX(
                            image4_name) + ',' + rX(
                            image5_name) + ',' + rX(lot_origin_url) + '\n')

    def writeHeaders(self, soup):
        auction_name, auction_date, auction_title, auction_location, lotCount, lot_sold_in = "", "", "", "", "", ""
        lot_sold_in = "USD"

        writeHeader(self.fp, auction_name, auction_location, auction_date, auction_title, auctionId, lotCount,
                    lot_sold_in)

    def getTextData(self, All_Data, textName):
        try:
            textData = [item for index, item in enumerate(All_Data) if textName in item.lower()][0]
        except:
            return ""

    def getIndexData(self, All_Data, textName):
        try:
            indexNo = [index for index, item in enumerate(All_Data) if textName in item][0]
        except:
            return ""

def updatestatus(auctionno, auctionurl):
    auctionurl = auctionurl.replace("%3A", ":")
    auctionurl = auctionurl.replace("%2F", "/")
    pageurl = "http://216.137.189.57:8080/scrapers/finish/?scrapername=Farsettiarte&auctionnumber=%s&auctionurl=%s&scrapertype=2"%(auctionno, urllib.parse.quote(auctionurl))
    pageResponse = None
    try:
        pageResponse = urllib.request.urlopen(pageurl)
    except:
        print ("Error: %s"%sys.exc_info()[1].__str__())

if __name__ == "__main__":
    if sys.argv.__len__() < 5:
        print("Insufficient parameters")
        sys.exit()
    auctionUrl = sys.argv[1]
    auctionId = sys.argv[2]
    csvpath = sys.argv[3]
    imagepath = sys.argv[4]
    downloadImages = 0
    convertfractions = 0
    if sys.argv.__len__() > 5:
        downloadImages = sys.argv[5]
    if sys.argv.__len__() > 6:
        convertfractions = sys.argv[6]
    fp = open(csvpath, "w")
    Scrape(auctionId, auctionUrl, downloadImages, fp)
    fp.close()
    updatestatus(auctionId, auctionUrl)

# Sample command to run: python farsettiarte.py https://www.farsettiarte.it/uk/auction-0215-1/parade.asp?action=reset 0215 /home/supmit/work/art2/farsettiarte_0215.csv /home/supmit/work/art2/images/farsettiarte/0215 0 0


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import uuid

from openstack import exceptions
from openstack.tests.functional import base
from openstack import utils

class TestZoneShare(base.BaseFunctionalTest):
    def setUp(self):
        super(TestZoneShare, self).setUp()
        self.require_service('dns')
        if not self.user_cloud:
            self.skipTest("The demo cloud is required for this test")

        # Note: zone deletion is not an immediate operation, so each time
        # chose a new zone name for a test
        # getUniqueString is not guaranteed to return unique string between
        # different tests of the same class.
        self.ZONE_NAME = 'example-{0}.org.'.format(uuid.uuid4().hex)

        # Make sure the API under test has shared zones support
        if not utils.supports_version(self.conn.dns, '2.1'):
            self.skipTest(
                'Designate API version does not support shared zones.'
            )

        self.zone = self.operator_cloud.dns.create_zone(
            name=self.ZONE_NAME,
            email='joe@example.org',
            type='PRIMARY',
            ttl=7200,
            description='example zone for sdk zone share tests',
        )
        self.addCleanup(
            self.operator_cloud.dns.delete_zone,
            self.zone,
            delete_shares=True,
        )

        self.project_id = self.operator_cloud.session.get_project_id()
        self.demo_project_id = self.user_cloud.session.get_project_id()

    def test_create_delete_zone_share(self):
        zone_share = self.operator_cloud.dns.create_zone_share(
            self.zone, target_project_id=self.demo_project_id
        )
        self.addCleanup(
            self.operator_cloud.dns.delete_zone_share,
            self.zone,
            zone_share,
        )

        self.assertEqual(self.zone.id, zone_share.zone_id)
        self.assertEqual(self.project_id, zone_share.project_id)
        self.assertEqual(self.demo_project_id, zone_share.target_project_id)
        self.assertIsNotNone(zone_share.id)
        self.assertIsNotNone(zone_share.created_at)
        self.assertIsNone(zone_share.updated_at)

    def test_get_zone_share(self):
        orig_zone_share = self.operator_cloud.dns.create_zone_share(
            self.zone,
            target_project_id=self.demo_project_id,
        )
        self.addCleanup(
            self.operator_cloud.dns.delete_zone_share,
            self.zone,
            orig_zone_share,
        )

        zone_share = self.operator_cloud.dns.get_zone_share(
            self.zone,
            orig_zone_share,
        )

        self.assertEqual(self.zone.id, zone_share.zone_id)
        self.assertEqual(self.project_id, zone_share.project_id)
        self.assertEqual(self.demo_project_id, zone_share.target_project_id)
        self.assertEqual(orig_zone_share.id, zone_share.id)
        self.assertEqual(orig_zone_share.created_at, zone_share.created_at)
        self.assertEqual(orig_zone_share.updated_at, zone_share.updated_at)

    def test_find_zone_share(self):
        orig_zone_share = self.operator_cloud.dns.create_zone_share(
            self.zone, target_project_id=self.demo_project_id
        )
        self.addCleanup(
            self.operator_cloud.dns.delete_zone_share,
            self.zone,
            orig_zone_share,
        )

        zone_share = self.operator_cloud.dns.find_zone_share(
            self.zone,
            orig_zone_share.id,
        )

        self.assertEqual(self.zone.id, zone_share.zone_id)
        self.assertEqual(self.project_id, zone_share.project_id)
        self.assertEqual(self.demo_project_id, zone_share.target_project_id)
        self.assertEqual(orig_zone_share.id, zone_share.id)
        self.assertEqual(orig_zone_share.created_at, zone_share.created_at)
        self.assertEqual(orig_zone_share.updated_at, zone_share.updated_at)

    def test_find_zone_share_ignore_missing(self):
        zone_share = self.operator_cloud.dns.find_zone_share(
            self.zone,
            'bogus_id',
        )
        self.assertIsNone(zone_share)

    def test_find_zone_share_ignore_missing_false(self):
        self.assertRaises(
            exceptions.ResourceNotFound,
            self.operator_cloud.dns.find_zone_share,
            self.zone,
            'bogus_id',
            ignore_missing=False,
        )

    def test_list_zone_shares(self):
        zone_share = self.operator_cloud.dns.create_zone_share(
            self.zone,
            target_project_id=self.demo_project_id,
        )
        self.addCleanup(
            self.operator_cloud.dns.delete_zone_share,
            self.zone,
            zone_share,
        )

        target_ids = [
            o.target_project_id
            for o in self.operator_cloud.dns.zone_shares(self.zone)
        ]
        self.assertIn(self.demo_project_id, target_ids)

    def test_list_zone_shares_with_target_id(self):
        zone_share = self.operator_cloud.dns.create_zone_share(
            self.zone,
            target_project_id=self.demo_project_id,
        )
        self.addCleanup(
            self.operator_cloud.dns.delete_zone_share,
            self.zone,
            zone_share,
        )

        target_ids = [
            o.target_project_id
            for o in self.operator_cloud.dns.zone_shares(
                self.zone, target_project_id=self.demo_project_id
            )
        ]
        self.assertIn(self.demo_project_id, target_ids)


from typing import List

class Solution:
    def merge(self, intervals: List[List[int]]) -> List[List[int]]:
        if not intervals:
            return []

        result = []
        sorted_intervals = sorted(intervals)
        low, high = sorted_intervals[0]
        for interval_low, interval_high in sorted_intervals[1:]:
            if (
                (interval_low <= low and interval_high >= low and interval_high <= high)
                or (
                    interval_low >= low
                    and interval_low <= high
                    and interval_high >= high
                )
                or (interval_low <= low and interval_high >= high)
                or (interval_low >= low and interval_high <= high)
            ):
                high = max(high, interval_high)
                low = min(low, interval_low)
            else:
                result.append([low, high])
                low = interval_low
                high = interval_high
        result.append([low, high])
        return result


#!/usr/bin/python
# -*- coding:utf-8 -*-

###lib required:pdfminer3k,python_docx
import sys,io,re,os
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from io import StringIO
from io import open
from docx import Document
from configparser import ConfigParser
from concurrent.futures import ProcessPoolExecutor
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')

def read_from_pdf(file_path):
    with open(file_path,'rb') as file:
        resource_manager=PDFResourceManager()
        return_str=StringIO()
        lap_params=LAParams()
        device=TextConverter(resource_manager,return_str,laparams=lap_params)
        process_pdf(resource_manager,device,file)
        device.close()
        content=return_str.getvalue()
        return_str.close()
        return content

def readPDF(pdfFile):
    rsrcmgr = PDFResourceManager()
    retstr = StringIO()
    laparams = LAParams()
    device = TextConverter(rsrcmgr, retstr, laparams=laparams)
    process_pdf(rsrcmgr, device, pdfFile)
    device.close()
    content = retstr.getvalue()
    retstr.close()
    return content

def save2txt(out_file,txt_content):
    with open(out_file,"w",encoding='utf-8') as f:
        f.write(txt_content)

def save_text_to_word(content,file_path):
    doc = Document()
    for line in content.split('\n'):
        paragraph=doc.add_paragraph()
        paragraph.add_run(remove_control_characters(line))
    doc.save(file_path)

def remove_control_characters(content):
    mpa=dict.fromkeys(range(32))
    return content.translate(mpa)

def pdf_to_word(pdf_file_path,word_file_path):
    content=read_from_pdf(pdf_file_path)
    save_text_to_word(content,word_file_path)

def main():
    if len(sys.argv)<2:
        print (f"\n\tpython {sys.argv[0]} <input_pdf>\n")
        sys.exit(0)
    else:
        txt_out_file=sys.argv[1].replace("pdf","txt")
        docx_out_file=sys.argv[1].replace("pdf","docx")

        ####pdf转成txt
        pdf_content = readPDF(open(sys.argv[1], 'rb'))
        save2txt(txt_out_file,pdf_content)

        ####pdf转成word
        with ProcessPoolExecutor(max_workers=1) as executor:
            result=executor.submit(pdf_to_word,sys.argv[1],docx_out_file)

if __name__=='__main__':
    main()


"""empty message

Revision ID: 338df913cbf6
Revises: c23954660d11
Create Date: 2021-08-20 13:29:42.521603

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '338df913cbf6'
down_revision = 'c23954660d11'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('pet_book', schema=None) as batch_op:
        batch_op.add_column(sa.Column('food', sa.String(length=50), nullable=True))
        batch_op.add_column(sa.Column('descript', sa.String(length=200), nullable=True))
        batch_op.add_column(sa.Column('p_type', sa.String(length=20), nullable=True))

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('pet_book', schema=None) as batch_op:
        batch_op.drop_column('p_type')
        batch_op.drop_column('descript')
        batch_op.drop_column('food')

    # ### end Alembic commands ###


from typing import Optional, Any
import argparse
import logging
import lsprotocol.types as lsp_types
from pygls.server import LanguageServer
import logging
from . import commands, features
from .colorful_log_handler import ColorfulHandler

LOGGER = logging.getLogger(__name__)

def init_language_server() -> LanguageServer:
    language_server = LanguageServer("tentiris", "v0.1")

    def register_feature(
        tentiris_server, feature_name: str, f, options: Optional[Any] = None
    ):
        @tentiris_server.feature(feature_name, options)
        def wrapper(*args, **keys):
            return f(tentiris_server, *args, **keys)

        return wrapper

    def register_command(tentiris_server, command_name: str, f):
        @tentiris_server.command(command_name)
        def wrapper(*args, **keys):
            return f(tentiris_server, *args, **keys)

        return wrapper

    def register_thread_command(tentiris_server, command_name: str, f):
        @tentiris_server.thread()
        @tentiris_server.command(command_name)
        def wrapper(*args, **keys):
            return f(tentiris_server, *args, **keys)

        return wrapper

    register_feature(
        language_server,
        lsp_types.TEXT_DOCUMENT_COMPLETION,
        features.completions,
        lsp_types.CompletionOptions(trigger_characters=[","]),
    )
    register_command(
        language_server,
        "countDownBlocking",
        commands.count_down_10_seconds_blocking,
    )
    register_command(
        language_server,
        "countDownNonBlocking",
        commands.count_down_10_seconds_non_blocking,
    )
    register_feature(
        language_server, lsp_types.TEXT_DOCUMENT_DID_CHANGE, features.did_change
    )
    register_feature(
        language_server, lsp_types.TEXT_DOCUMENT_DID_CLOSE, features.did_close
    )
    register_feature(
        language_server, lsp_types.TEXT_DOCUMENT_DID_OPEN, features.did_open
    )
    register_feature(
        language_server,
        lsp_types.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
        features.semantic_tokens,
        lsp_types.SemanticTokensLegend(token_types=["operator"], token_modifiers=[]),
    )
    register_command(language_server, "progress", commands.progress)
    register_command(
        language_server,
        "registerCompletions",
        commands.register_completions,
    )
    register_command(
        language_server,
        "showConfigurationAsync",
        commands.show_configuration_async,
    )
    register_command(
        language_server,
        "showConfigurationCallback",
        commands.show_configuration_callback,
    )
    register_thread_command(
        language_server, "showConfigurationThread", commands.show_configuration_thread
    )
    register_command(
        language_server,
        "unregisterCompletions",
        commands.unregister_completions,
    )

    return language_server

def main():
    logging.basicConfig(level=logging.WARN, handlers=[ColorfulHandler()])

    # args
    parser = argparse.ArgumentParser()
    parser.description = "simple json server example"
    parser.add_argument("--tcp", action="store_true", help="Use TCP server")
    parser.add_argument("--ws", action="store_true", help="Use WebSocket server")
    parser.add_argument("--host", default="127.0.0.1", help="Bind to this address")
    parser.add_argument("--port", type=int, default=32123, help="Bind to this port")
    args = parser.parse_args()

    # language server
    language_server = init_language_server()
    if args.tcp:
        language_server.start_tcp(args.host, args.port)
    elif args.ws:
        language_server.start_ws(args.host, args.port)
    else:
        language_server.start_io()

if __name__ == "__main__":
    main()


import sys
import argparse

from dotenv import load_dotenv

import chromadb

from chromadb.config import Settings

from langchain.vectorstores import Chroma
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.llms import OpenAI
from langchain.chains import RetrievalQA
from langchain.document_loaders import TextLoader

# Parse Arguments

parser = argparse.ArgumentParser(
    prog=sys.argv[0],
    description="Loads and/or queries data into/from ChromaDB using OpenAI",
    allow_abbrev=True
)

parser.add_argument("-l", "--load", help="load file", type=str, action="append")
parser.add_argument("-q", "--query", help="ask question", type=str, action="append")
parser.add_argument("-v", "--verbose", help="enable verbose mode", action="store_true")
args = parser.parse_args()

if (not args.load or len(args.load) == 0) and (not args.query or len(args.query) == 0):
    print(f"no file(s) to load and no queries specified")
    parser.print_usage()
    exit(1)

def print_verbose(message):
    if args.verbose:
        print(message)

# Load Configuration

load_dotenv()

# Setup Database Access

chroma_settings = Settings(chroma_api_impl="rest", chroma_server_host="localhost", chroma_server_http_port="8000")
chroma_client = chromadb.Client(chroma_settings)

# Load New Data

documents_to_load = []
if args.load:
    for fn in args.load:
        print_verbose(f"reading({fn})")
        loader = TextLoader(fn)
        for doc in loader.load():
            documents_to_load.append(doc)

# Access vector store and create new embeddings if needed

embeddings = OpenAIEmbeddings()
if len(documents_to_load) > 0:
    # see https://github.com/hwchase17/langchain/issues/1310 for using chunk size to help with Rate Limit errors
    print_verbose("RecursiveCharacterTextSplitter")
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
    print_verbose("text_splitter.split_documents")
    texts = text_splitter.split_documents(documents_to_load)
    print_verbose("Chroma.from_documents")
    vectordb = Chroma.from_documents(texts, embeddings, client=chroma_client)
else:
    print_verbose("Chroma")
    vectordb = Chroma(embedding_function=embeddings, client=chroma_client)

# Ask questions

if args.query:
    qa = RetrievalQA.from_chain_type(
        llm=OpenAI(),
        chain_type="stuff",
        retriever=vectordb.as_retriever()
    )
    for q in args.query:
        print_verbose(f"running query: {q}")
        answer = qa.run(q)
        print_verbose("the answer is:")
        print(answer)


"""QGIS Unit tests for QgsFieldValidator.

.. note:: This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
"""
__author__ = 'Alessandro Pasotti'
__date__ = '31/01/2018'
__copyright__ = 'Copyright 2018, The QGIS Project'

import os
import shutil
import tempfile

from qgis.PyQt.QtCore import QLocale, QVariant
from qgis.PyQt.QtGui import QValidator
from qgis.core import QgsVectorLayer
from qgis.gui import QgsFieldValidator
import unittest
from qgis.testing import start_app, QgisTestCase

from utilities import unitTestDataPath

TEST_DATA_DIR = unitTestDataPath()

start_app()

class TestQgsFieldValidator(QgisTestCase):

    @classmethod
    def setUpClass(cls):
        """Run before all tests."""
        super().setUpClass()
        testPath = TEST_DATA_DIR + '/' + 'bug_17878.gpkg'
        # Copy it
        tempdir = tempfile.mkdtemp()
        testPathCopy = os.path.join(tempdir, 'bug_17878.gpkg')
        shutil.copy(testPath, testPathCopy)
        cls.vl = QgsVectorLayer(testPathCopy + '|layername=bug_17878', "test_data", "ogr")
        assert cls.vl.isValid()

    @classmethod
    def tearDownClass(cls):
        """Run after all tests."""
        cls.vl = None
        super().tearDownClass()

    def _fld_checker(self, field):
        """
        Expected results from validate
        QValidator::Invalid 0 The string is clearly invalid.
        QValidator::Intermediate 1 The string is a plausible intermediate value.
        QValidator::Acceptable 2 The string is acceptable as a final result; i.e. it is valid.
        """
        DECIMAL_SEPARATOR = QLocale().decimalPoint()
        OTHER_SEPARATOR = ',' if DECIMAL_SEPARATOR == '.' else '.'

        validator = QgsFieldValidator(None, field, '0.0', '')

        def _test(value, expected):
            ret = validator.validate(value, 0)
            self.assertEqual(ret[0], expected)
            if value:
                self.assertEqual(validator.validate('-' + value, 0)[0], expected, '-' + value)

        # NOTE!!! This should ALWAYS be valid, but the behavior changed in Qt > 5.12.
        # accordingly here we can only run the test if in a locale with decimal separator as dot.
        # The previous tests were moved to test_disabled_tests.py for now, until the QgsFieldValidator
        # class can be reworked to fix this regression.

        if DECIMAL_SEPARATOR != ',':
            _test('0.1234', QValidator.Acceptable)

        # Apparently we accept comma only when locale say so
        if DECIMAL_SEPARATOR != '.':
            _test('0,1234', QValidator.Acceptable)

        # If precision is > 0, regexp validator is used (and it does not support sci notation)
        if field.precision() == 0:
            # NOTE!!! This should ALWAYS be valid, but the behavior changed in Qt > 5.12.
            # accordingly here we can only run the test if in a locale with decimal separator as dot.
            # The previous tests were moved to test_disabled_tests.py for now, until the QgsFieldValidator
            # class can be reworked to fix this regression.
            if DECIMAL_SEPARATOR != ',':
                _test('12345.1234e+123', QValidator.Acceptable)
                _test('12345.1234e-123', QValidator.Acceptable)

            if DECIMAL_SEPARATOR != '.':
                _test('12345,1234e+123', QValidator.Acceptable)
                _test('12345,1234e-123', QValidator.Acceptable)
            _test('', QValidator.Acceptable)

            # Out of range
            _test('12345.1234e+823', QValidator.Intermediate)
            _test('12345.1234e-823', QValidator.Intermediate)
            if DECIMAL_SEPARATOR != '.':
                _test('12345,1234e+823', QValidator.Intermediate)
                _test('12345,1234e-823', QValidator.Intermediate)

        # Invalid
        _test('12345-1234', QValidator.Invalid)
        _test('onetwothree', QValidator.Invalid)

        int_field = self.vl.fields()[self.vl.fields().indexFromName('int_field')]
        self.assertEqual(int_field.precision(), 0)  # this is what the provider reports :(
        self.assertEqual(int_field.length(), 0)  # not set
        self.assertEqual(int_field.type(), QVariant.Int)

        validator = QgsFieldValidator(None, int_field, '0', '')

        # Valid
        _test('0', QValidator.Acceptable)
        _test('1234', QValidator.Acceptable)
        _test('', QValidator.Acceptable)

        # Invalid
        _test('12345-1234', QValidator.Invalid)
        _test(f'12345{DECIMAL_SEPARATOR}1234', QValidator.Invalid)
        _test('onetwothree', QValidator.Invalid)

    def test_doubleValidator(self):
        """Test the double with default (system) locale"""
        field = self.vl.fields()[self.vl.fields().indexFromName('double_field')]
        self.assertEqual(field.precision(), 0)  # this is what the provider reports :(
        self.assertEqual(field.length(), 0)  # not set
        self.assertEqual(field.type(), QVariant.Double)
        self._fld_checker(field)

    def test_doubleValidatorCommaLocale(self):
        """Test the double with german locale"""
        QLocale.setDefault(QLocale(QLocale.German, QLocale.Germany))
        self.assertEqual(QLocale().decimalPoint(), ',')
        field = self.vl.fields()[self.vl.fields().indexFromName('double_field')]
        self._fld_checker(field)

    def test_doubleValidatorDotLocale(self):
        """Test the double with english locale"""
        QLocale.setDefault(QLocale(QLocale.English))
        self.assertEqual(QLocale().decimalPoint(), '.')
        field = self.vl.fields()[self.vl.fields().indexFromName('double_field')]
        self._fld_checker(field)

    def test_precision(self):
        """Test different precision"""
        QLocale.setDefault(QLocale(QLocale.English))
        self.assertEqual(QLocale().decimalPoint(), '.')
        field = self.vl.fields()[self.vl.fields().indexFromName('double_field')]
        field.setPrecision(4)
        self._fld_checker(field)

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/python
import xml.etree.ElementTree as ET
import argparse
import os
import HTMLParser

from pprint import pprint
ns = '{http://checklists.nist.gov/xccdf/1.1}'
dc = '{http://purl.org/dc/elements/1.1/}'
h = HTMLParser.HTMLParser()

objRef = {}
# def show_root_info(self):
	# for item in self.root.items():
		# for n, field in enumerate(item):
			# if n == 0:
				# p = field.index('}')
				# print(f'{field[p+1:] :20}: ', end='')
			# else:
				# print(field)

def parseRule(rule):
	rules = {}
	rules.update( rule.attrib.items())
	for rItem in rule:
		tag = rItem.tag.replace(ns,'')
		if 'reference' in tag or 'check' in tag:
			for sub in rItem:
				if 'reference' in tag:
					subtag = sub.tag.replace(dc,'')
				else:
					subtag = sub.tag.replace(ns,'')
				if sub.text is not None:
					rules[subtag] = sub.text
		else:
			itemDic = {}
			if len(rItem.attrib) > 0:
				itemDic.update(rule.attrib.items())
			if rItem.text is not None:
				itemDic['text'] = rItem.text
		rules[tag] = itemDic
	return rules

def parseStig(xmlfile):
	# create element tree object
	tree = ET.parse(xmlfile.name)
	root = tree.getroot()
	groupitems = []

	# iterate news items
	for item in root.findall('./{http://checklists.nist.gov/xccdf/1.1}Group'):
		#pprint(item.attrib['id'])
		group = {'fileName' : xmlfile.name, 'id': item.attrib['id']}
		for child in item:
			if('Rule' in child.tag):
				group['Rule'] = parseRule(child)
			else:
				tag = child.tag.replace(ns,'')
				group[tag] = child.text
			#group[child.tag] = child.attrib
		# append news dictionary to news items list
		objRef[item.attrib['id']] = group
		groupitems.append(group)
	# return news items list
	return groupitems

def buildAudit(stigData, auditFile):
	f = open(str(auditFile),"w+")
	f.write('<check_type:"Cisco"> \n')

	for item in stigData :
		print('<?xml version="1.0"?><collection>'+ h.unescape(item['Rule']['description']['text']).encode('utf8') +'</collection>')
		textXml = ET.fromstring('<?xml version="1.0"?><collection>'+ h.unescape(item['Rule']['description']['text']).replace('&','').encode('utf8') +'</collection>')
		ruleDict = {}
		for elem in textXml.iter():
			pprint({elem.tag: 'nothing' if elem.text is None else elem.text})
			ruleDict.update({elem.tag: 'nothing' if elem.text is None else elem.text})

		refDic = {'Rule-ID': item['Rule']['check']['id'],"Vuln-ID":item['id'],"STIG-ID":item['Rule']['version']['text']}

		f.write('\n<item>\n')
		f.write('  type \t: CONFIG_CHECK \n')
		f.write('  description \t: "' + item['Rule']['version']['text'] +' - ' + item['title'] +'"\n')
		f.write('  info \t: "' + item['Rule']['title'] +'\n\n'+ ruleDict['VulnDiscussion'].encode('utf8') +'"\n')
		f.write('  solution \t: "' + item['Rule']['fixtext']['text'].encode('utf8') +'"\n')
		f.write('  reference \t: "'+ ','.join("{0}|{1}".format(key,val.replace("'",'')) for (key,val) in refDic.items()) +'"\n')
		f.write('  see_also \t: "https://iasecontent.disa.mil/stigs/zip/'+ item['fileName'] +'"\n')
		f.write('  context \t: \n')
		f.write('  item \t: \n')
		f.write('\n</item>\n\n')

	f.close()
##
##AUDIT Fields
## Title, Description, Rule: id, severity, weight, version, title, description
def main():
	parser = argparse.ArgumentParser(description='Process a STIG xml into a nessus audit file')
	parser.add_argument('stig', metavar='N', type=argparse.FileType('r'), help='the stig file you want converted')
	parser.add_argument('auditFile', metavar='N', type=str, help='the output auditFile')
	args = parser.parse_args()
	stig = parseStig(args.stig)
	buildAudit(stig, args.auditFile)
	#pprint(stig)

if __name__ == "__main__":

	# calling main function
	main()


import torch
import torch.nn.functional as F

def bl_matmul(mat_a, mat_b):
    return torch.einsum('mij,jk->mik', mat_a, mat_b)

def compute_permu_matrix(s: torch.FloatTensor, tau=1):

    device = s.get_device()
    mat_as = s - s.permute(0, 2, 1)
    mat_as = torch.abs(mat_as)
    n = s.shape[1]
    one = torch.ones(n, 1).to(device)

    b = bl_matmul(mat_as, one @ one.transpose(0, 1))
    k = torch.arange(n) + 1
    d = (n + 1 - 2 * k).float().detach().requires_grad_(True).unsqueeze(0).to(device)

    c = bl_matmul(s, d)
    mat_p = (c - b).permute(0, 2, 1)
    mat_p = F.softmax(mat_p / tau, -1)

    return mat_p

def prop_any_correct(p1, p2):
    z1 = torch.argmax(p1, axis=-1)
    z2 = torch.argmax(p2, axis=-1)
    eq = torch.eq(z1, z2).float()
    correct = torch.mean(eq, axis=-1)
    return torch.mean(correct)

def prop_correct(p1, p2):
    z1 = torch.argmax(p1, axis=-1)
    z2 = torch.argmax(p2, axis=-1)
    eq = torch.eq(z1, z2)
    correct = torch.all(eq, axis=-1).float()
    return torch.sum(correct)

def top_k_correct(ranking, ranking_true, k):
    return sum(ranking[i] in ranking_true[:k] for i in range(k))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Command line tool to merge possibly overlapping flux.dat files from hotstart runs
"""
import argparse
import numpy as np

def combine_flux(infiles,outfile,prefer_last=False):
    """ Merge possibly overlapping flux.dat files from hostart runs
    """
    filedata = []
    for f in infiles:
        try:
            filedata.append(np.loadtxt(f))
        except:
            print("Problem reading file: {}".format(f))
            raise
    nfile = len(filedata)
    starts = np.array([fd[ 0,0] for fd in filedata])
    ends   = np.array([fd[-1,0] for fd in filedata])
    sorder = np.argsort(starts)
    eorder = np.argsort(ends)
    if np.any(np.not_equal(sorder,eorder)):
        raise ValueError("input files are ambiguously ordered (one lies inside another)")
    if np.any(sorder[1:]<sorder[:-1]):
        raise ValueError("Files out of order. This is the apparent order of inputs: {}".format(sorder))

    nstation = filedata[0].shape[1] -1
    writeformat = ["%16.6f"] + nstation*["%14.4e"]

    ndxlast = filedata[-1].shape[0]
    if prefer_last:
        sblock = np.full(len(filedata), 0, dtype=np.int64)
        eblock = [np.searchsorted(filedata[i][:,0],starts[i+1]) for i in range(nfile-1)] + [ndxlast]
    else:
        sblock = [0] + [np.searchsorted(filedata[i][:,0],ends[i-1],side="right") for i in range(1,nfile)]
        eblock = [filedata[i].shape[0] for i in range(len(filedata))]

    blocks2merge = [d[s:e] for d,s,e in zip(filedata,sblock,eblock)]
    for m in blocks2merge: print(m.shape)
    merged = np.vstack(tuple(blocks2merge))

    ddt = np.diff(merged[:,0],n=2)
    if np.max(ddt) > 1.e-4:
        raise ValueError("The continuity of time in the files is wrong. Successive time steps should differ by less than 1e-4")

    np.savetxt(outfile,merged,delimiter=" ",fmt=writeformat)

def test_combine_flux():
    full = open("flux_small.dat","r")
    breaks = [60,120,200]
    b = 0
    fnames=[]
    lines = full.readlines()
    for i,e in enumerate(breaks):
        fname = "flux{}.dat.".format(i)
        fnames.append(fname)
        out = open(fname,"w")
        for line in lines[b:(e+10)]:
            out.write(line+"\n")
        b=e
        out.close()
    fout = "flux_comb.dat"
    combine_flux(fnames,fout,prefer_last=False)

def create_arg_parser():
    parser = argparse.ArgumentParser(description="Merge possibly-overlapping flux.dat files from hotstarted runs. ")
    parser.add_argument('--prefer_last', action='store_true')
    parser.add_argument('--output',
                        help="output file")
    parser.add_argument('files',type=argparse.FileType('r'),nargs="+",
                        help="List of input files in chronological order")

    return parser

def main():
    """ Driver to parse arguments
    """
    parser = create_arg_parser()
    args = parser.parse_args()
    filelist = args.files
    inputs = [x.name for x in filelist]
    prefer_last = args.prefer_last
    output = args.output
    if output in inputs:
        raise ValueError("Cannot overwrite input file names")
    combine_flux(filelist,output,prefer_last)

if __name__ == '__main__':
    #test_combine_flux()
    main()


import rtree

class PathIndex:
    def __init__(self, path_graph):
        self.idx = rtree.index.Index()
        self.path_graph = path_graph
        for index, coordinate in path_graph.iter_starts_with_index():
            self.idx.add(index, coordinate + coordinate)

    def get_nearest(self, coordinate):
        return next(self.idx.nearest(coordinate))

    def delete(self, index):
        coordinate = self.path_graph.get_coordinates(index)
        self.idx.delete(index, coordinate + coordinate)

    def delete_pair(self, index):
        self.delete(index)
        self.delete(self.path_graph.get_disjoint(index))

def greedy_walk(path_graph):
    path_index = PathIndex(path_graph)
    location = path_graph.get_coordinates(path_graph.ORIGIN)
    while True:
        try:
            next_point = path_index.get_nearest(location)
        except StopIteration:
            break
        location = path_graph.get_coordinates(next_point, True)
        path_index.delete_pair(next_point)
        yield next_point

from collections import Counter

def check_valid_solution(solution, graph):
    """Check that the solution is valid: every path is visited exactly once."""
    expected = Counter(
        i for (i, _) in graph.iter_starts_with_index()
        if i < graph.get_disjoint(i)
    )
    actual = Counter(
        min(i, graph.get_disjoint(i))
        for i in solution
    )

    difference = Counter(expected)
    difference.subtract(actual)
    difference = {k: v for k, v in difference.items() if v != 0}
    if difference:
        print('Solution is not valid!'
              'Difference in node counts (expected - actual): {}'.format(difference))
        return False
    return True

def get_route_from_solution(solution, graph):
    """Converts a solution (a list of node indices) into a list
    of paths suitable for rendering."""

    # As a guard against comparing invalid "solutions",
    # ensure that this solution is valid.
    assert check_valid_solution(solution, graph)

    return [graph.get_path(i) for i in solution]