import csv
from logging import Logger
import os
import sys
from typing import List
import re

import numpy as np
from tensorboardX import SummaryWriter
import torch
from tqdm import trange
import pickle
from torch.optim.lr_scheduler import ExponentialLR

from .evaluate import evaluate, evaluate_predictions,judge
from .predict import predict
from .train import train
from chemprop.args import TrainArgs
from chemprop.data import StandardScaler, MoleculeDataLoader
from chemprop.data.utils import get_class_sizes, get_data, get_task_names, split_data
from chemprop.models import MoleculeModel
from chemprop.nn_utils import param_count
from chemprop.utils import build_optimizer, build_lr_scheduler, get_loss_func, get_metric_func, load_checkpoint,\
    makedirs, save_checkpoint, save_smiles_splits
import matplotlib.pyplot as plt
import pylab as pl

def predict_feature(args,logger,model,external_test_path):
    if logger is not None:
        debug, info = logger.debug, logger.info
    else:
        debug = info = print

    # Print command line
    debug('Command line')
    debug(f'python {" ".join(sys.argv)}')

    # Print args
    debug('Args')
    debug(args)

    # Save args
    args.save(os.path.join(args.save_dir, 'args.json'))

    # Set pytorch seed for random initial weights
    torch.manual_seed(args.pytorch_seed)

    # Get data
    debug('Loading data')
    args.task_names = args.target_columns or get_task_names(args.data_path)
    data = get_data(path=args.data_path, args=args, logger=logger)
    args.num_tasks = data.num_tasks()
    args.features_size = data.features_size()
    debug(f'Number of tasks = {args.num_tasks}')

    external_test_data = get_data(path=external_test_path, args=args, logger=logger)

    # Split data
    debug(f'Splitting data with seed {args.seed}')
    if args.separate_test_path:
        test_data = get_data(path=args.separate_test_path, args=args, features_path=args.separate_test_features_path,
                             logger=logger)
    if args.separate_val_path:
        val_data = get_data(path=args.separate_val_path, args=args, features_path=args.separate_val_features_path,
                            logger=logger)

    if args.separate_val_path and args.separate_test_path:
        train_data = data
    elif args.separate_val_path:
        train_data, _, test_data = split_data(data=data, split_type=args.split_type, sizes=(0.8, 0.0, 0.2),
                                              seed=args.seed, args=args, logger=logger)
    elif args.separate_test_path:
        train_data, val_data, _ = split_data(data=data, split_type=args.split_type, sizes=(0.8, 0.2, 0.0),
                                             seed=args.seed, args=args, logger=logger)
    else:
        train_data, val_data, test_data = split_data(data=data, split_type=args.split_type, sizes=args.split_sizes,
                                                     seed=args.seed, args=args, logger=logger)

    if args.dataset_type == 'classification':
        class_sizes = get_class_sizes(data)
        debug('Class sizes')
        for i, task_class_sizes in enumerate(class_sizes):
            debug(f'{args.task_names[i]} '
                  f'{", ".join(f"{cls}: {size * 100:.2f}%" for cls, size in enumerate(task_class_sizes))}')

    if args.save_smiles_splits:
        save_smiles_splits(
            train_data=train_data,
            val_data=val_data,
            test_data=test_data,
            data_path=args.data_path,
            save_dir=args.save_dir
        )

    if args.features_scaling:
        features_scaler = train_data.normalize_features(replace_nan_token=0)
        external_test_data.normalize_features(features_scaler)
    else:
        features_scaler = None

    args.train_data_size = len(train_data)

    debug(f'Total size = {len(data):,} | '
          f'train size = {len(train_data):,} | val size = {len(val_data):,} | test size = {len(test_data):,}')

    # Initialize scaler and scale training targets by subtracting mean and dividing standard deviation (regression only)
    if args.dataset_type == 'regression':
        debug('Fitting scaler')
        train_smiles, train_targets = train_data.smiles(), train_data.targets()
        scaler = StandardScaler().fit(train_targets)
        scaled_targets = scaler.transform(train_targets).tolist()
        train_data.set_targets(scaled_targets)
    else:
        scaler = None

    # Automatically determine whether to cache
    if len(data) <= args.cache_cutoff:
        cache = True
        num_workers = 0
    else:
        cache = False
        num_workers = args.num_workers

    # Create data loaders
    train_data_loader = MoleculeDataLoader(
        dataset=train_data,
        batch_size=args.batch_size,
        num_workers=num_workers,
        cache=cache,
        class_balance=args.class_balance,
        shuffle=False,
        seed=args.seed
    )

    external_test_loader = MoleculeDataLoader(
        dataset=external_test_data,
        batch_size=args.batch_size,
        num_workers=num_workers,
        cache=cache,
        class_balance=args.class_balance,
        shuffle=False,
        seed=args.seed
    )
    external_test_preds, external_test_feature = predict(
        model=model,
        data_loader=external_test_loader,
        scaler=scaler
    )
    external_test_smiles, external_test_targets = external_test_data.smiles(), external_test_data.targets()
    return external_test_smiles,external_test_feature,external_test_preds,external_test_targets


from __future__ import print_function

import os
import numpy as np
from skimage.io import imread

image_rows = int(256)
image_cols = int(256)
image_depth = 16

def create_train_data(options, labels):
    train_data_path = options.outputdir+"/train/"
    mask_data_path = options.outputdir+'/masks/'
    dirs = os.listdir(train_data_path)
    total = int(len(dirs)*16*2)

    imgs = np.ndarray((total, image_depth, image_rows, image_cols), dtype=np.uint8)
    imgs_mask = np.ndarray((total, image_depth, image_rows, image_cols), dtype=np.uint8)

    imgs_temp = np.ndarray((total, image_depth//2, image_rows, image_cols), dtype=np.uint8)
    imgs_mask_temp = np.ndarray((total, image_depth//2, image_rows, image_cols), dtype=np.uint8)

    i = 0
    print('-'*30)
    print('Creating training images...')
    print('-'*30)
    for dirr in sorted(os.listdir(train_data_path)):
        j = 0
        dirr = train_data_path+"/"+dirr
        if not os.path.isdir(dirr):
            continue
        images = sorted(os.listdir(dirr))
        count = total
        for image_name in images:
            if not image_name.endswith('.png'):
                print('Ignored invalid file: ', image_name)
                continue
            img = imread(os.path.join(dirr, image_name), as_gray=True)
            img = img.astype(np.uint8)
            img = np.array([img])
            imgs_temp[i, j] = img
            j += 1
            if j % (image_depth/2) == 0:
                j = 0
                i += 1
                print('Done: {0}/{1} 3d images'.format(i, count), end='\r')

    for x in range(0, imgs_temp.shape[0]-1):
        imgs[x] = np.append(imgs_temp[x], imgs_temp[x+1], axis=0)

    print('Loading of train data done.')
    imgs = preprocess(imgs)
    np.save(options.outputdir+'/imgs_train.npy', imgs)
    print('Training NPY saved at: ' + options.outputdir + '/imgs_train.npy')

    print('-' * 30)
    print('Creating labeled images...')
    print('-' * 30)

    for label in labels:
        create_mask_data(options, label)
    print('Loading all labels done.')

    # Convert the whole volume to .npy
    print('Processing the whole mask...')
    i = 0
    for dirr in sorted(os.listdir(train_data_path)):
        j = 0
        dirr = mask_data_path + dirr + '/whole'
        if not os.path.exists(dirr):
            return
        images = sorted(file for file in os.listdir(dirr) if file.endswith('.png'))
        count = total
        for mask_name in images:
            img_mask = imread(dirr + '/' + mask_name, as_gray=True)
            img_mask = img_mask.astype(np.uint16)
            img_mask = np.array([img_mask])
            imgs_mask_temp[i, j] = img_mask

            j += 1
            if j % (image_depth / 2) == 0:
                j = 0
                i += 1
                print('Done: {0}/{1} 3d images'.format(i, count), end='\r')

    for x in range(0, imgs_mask_temp.shape[0] - 1):
        imgs_mask[x] = np.append(imgs_mask_temp[x], imgs_mask_temp[x + 1], axis=0)

    imgs_mask = preprocess(imgs_mask)
    np.save(options.outputdir + '/imgs_mask_train.npy', imgs_mask)
    print('Saving to .npy files: done.')

def create_mask_data(options, label):
    train_data_path = options.outputdir+"/train/"
    mask_data_path = options.outputdir+'/masks/'
    dirs = os.listdir(train_data_path)
    total = int(len(dirs)*16*2)

    imgs_mask = np.ndarray((total, image_depth, image_rows, image_cols), dtype=np.uint8)
    imgs_mask_temp = np.ndarray((total, image_depth//2, image_rows, image_cols), dtype=np.uint8)

    i = 0
    print('-' * 30)
    print('Processing label ' + str(label))
    for dirr in sorted(os.listdir(train_data_path)):
        if not os.path.isdir(train_data_path + dirr):
            continue
        j = 0
        dirr = mask_data_path + dirr + '/label-' + "{:0>5}".format(str(label))

        # Label did not exist
        if not os.path.exists(dirr):
            return
        images = sorted(file for file in os.listdir(dirr) if file.endswith('.png'))
        count = total
        for mask_name in images:
            img_mask = imread(dirr + '/' + mask_name, as_gray=True)
            img_mask = img_mask.astype(np.uint16)
            img_mask = np.array([img_mask])
            imgs_mask_temp[i, j] = img_mask

            j += 1
            if j % (image_depth / 2) == 0:
                j = 0
                i += 1
                print('Done: {0}/{1} 3d images'.format(i, count), end='\r')

    for x in range(0, imgs_mask_temp.shape[0] - 1):
        imgs_mask[x] = np.append(imgs_mask_temp[x], imgs_mask_temp[x + 1], axis=0)

    imgs_mask = preprocess(imgs_mask)
    np.save(options.outputdir + '/label-' + "{:0>5}".format(str(label)) + '_mask_train.npy', imgs_mask)
    print('Size of .npy file: ' + str(imgs_mask.shape))
    print('NPY for label ' + "{:0>5}".format(str(label)) + ' saved.')

def preprocess(imgs):
    imgs = np.expand_dims(imgs, axis=4)
    return imgs

def preprocess_squeeze(imgs):
    imgs = np.squeeze(imgs, axis=4)
    return imgs

if __name__ == '__main__':
    create_train_data()


from flask_restful import Resource, reqparse
from flask_jwt import jwt_required
import sqlite3

class Event(Resource):
    TABLE_NAME = 'events'

    parser = reqparse.RequestParser()
    # Add argument date
    parser.add_argument('name',
        type=str,
        required=True,
        help="This field cannot be left blank!"
    )
    parser.add_argument('date',
        type=str,
        required=True,
        help="This field cannot be left blank!"
    )

    parser.add_argument('lieu',
        type=str,
        required=True,
        help="This field cannot be left blank!"
    )

    parser.add_argument('objet',
        type=str,
        required=True,
        help="This field cannot be left blank!"
    )

    @jwt_required()
    def get(self, name):
        event = self.find_by_name(name)
        if event:
            return event
        return {'message': 'Event not found'}, 404

    @classmethod
    def find_by_name(cls, name):
        #Find an event by its name
        connection = sqlite3.connect('data.db')
        cursor = connection.cursor()

        query = "SELECT * FROM {table} WHERE name=?".format(table=cls.TABLE_NAME)
        result = cursor.execute(query, (name,))
        row = result.fetchone()
        connection.close()

        if row:
            return {'event': {'name': row[0], 'date': row[1], 'lieu': row[2]}}

    def post(self, name):
        if self.find_by_name(name):
            return {'message': "An event with name '{}' already exists.".format(name)}

        data = Event.parser.parse_args()

        event = {'name': name, 'date': data['date'], 'lieu': data['lieu'], 'objet': data['objet'] }

        try:
            Event.insert(event)
        except:
            return {"message": "An error occurred inserting the event."}

        return event

    @classmethod
    def insert(cls, event):
        connection = sqlite3.connect('data.db')
        cursor = connection.cursor()

        query = "INSERT INTO {table} VALUES(?, ?, ?, ?)".format(table=cls.TABLE_NAME)
        cursor.execute(query, (event['name'], event['date'], event['lieu'], event['objet']))

        connection.commit()
        connection.close()

    @jwt_required()
    def delete(self, name):
        connection = sqlite3.connect('data.db')
        cursor = connection.cursor()

        query = "DELETE FROM {table} WHERE name=?".format(table=self.TABLE_NAME)
        cursor.execute(query, (name,))

        connection.commit()
        connection.close()

        return {'message': 'Event deleted'}

    @jwt_required()
    def put(self, name):
        data = Event.parser.parse_args()
        event = self.find_by_name(name)
        updated_event = {'name': name, 'date': data['date'], 'lieu': ['lieu'], 'objet': ['objet']}
        if event is None:
            try:
                Event.insert(updated_event)
            except:
                return {"message": "An error occurred inserting the event."}
        else:
            try:
                Event.update(updated_event)
            except:
                raise
                return {"message": "An error occurred updating the event."}
        return updated_event

    @classmethod
    def update(cls, event):
        connection = sqlite3.connect('data.db')
        cursor = connection.cursor()

        query = "UPDATE {table} SET date=? WHERE name=?".format(table=cls.TABLE_NAME)
        cursor.execute(query, (event['date'], event['name']))

        connection.commit()
        connection.close()

class EventList(Resource):
    TABLE_NAME = 'events'

    def get(self):
        connection = sqlite3.connect('data.db')
        cursor = connection.cursor()

        query = "SELECT * FROM {table}".format(table=self.TABLE_NAME)
        result = cursor.execute(query)
        events = []
        for row in result:
            events.append({'name': row[0], 'date': row[1], 'lieu': row[2], 'objet': row[3]})
        connection.close()

        return {'events': events, 'status': 'SUCCESS'}


# GaussianDecisionBoundaries.py
#
# Draws the decision boundary between two Gaussian distributions according to the
# Maximum a Posteriori criterium. You can change the a priori probabilities, the
# mean vectors and covariance matrices. You can also show the difference between
# the two Probability Densidty Functions (PDFs) and display contours of the original
# PDFs.
#
# TODO
# - adjust axis limits depending on the Gaussian parameters
# - clean up and simplify the code
#
# (C) 2017 Giampiero Salvi <giampi@kth.se>
import tkinter as tk
import tkinter.ttk as ttk
from tkinter import font
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import multivariate_normal
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.colors import LogNorm
from tkinter import messagebox
#import matplotlib.backends.backend_tkagg as tkagg

def about():
   aboutText = "GaussianDecisionBoundaries.py\n\n(C) 2017 Giampiero Salvi\n\nDraws the decision boundary between two Gaussian distributions according to the Maximum a Posteriori criterion. You can change the a priori probabilities, the mean vectors and covariance matrices. You can also show the difference between the two Probability Densidty Functions (PDFs) and display contours of the original PDFs.\n\nSource code at: https://github.com/giampierosalvi/GaussianDecisionBoundaries"
   messagebox.showinfo("About", aboutText)

def redraw(fig):
   # acquie Gaussian parameters
   p = np.array([float(p1.get()), 1.0-float(p1.get())])
   mu1 = np.array([float(mu1x.get()), float(mu1y.get())])
   mu2 = np.array([float(mu2x.get()), float(mu2y.get())])
   s1 = np.array([[float(s1x.get()), float(s1xy.get())],
                  [float(s1xy.get()), float(s1y.get())]])
   s2 = np.array([[float(s2x.get()), float(s2xy.get())],
                  [float(s2xy.get()), float(s2y.get())]])
   # greate Multivariate Gaussian objects
   try:
      rv1 = multivariate_normal(mu1, s1)
   except ValueError:
      messagebox.showerror("Error!", "Covariance matrix must be positive semidefinite (Gaussian 1)")
   try:
      rv2 = multivariate_normal(mu2, s2)
   except ValueError:
      messagebox.showerror("Error!", "Covariance matrix must be positive semidefinite (Gaussian 2)")
   # Compute PDF for a certain range of x and y
   xlim = [float(xmin.get()), float(xmax.get())]
   ylim = [float(ymin.get()), float(ymax.get())]
   x, y = np.mgrid[xlim[0]:xlim[1]:(xlim[1]-xlim[0])/500.0, ylim[0]:ylim[1]:(ylim[1]-ylim[0])/500.0]
   pos = np.dstack((x, y))
   rv1g = p[0]*rv1.pdf(pos)
   rv2g = p[1]*rv2.pdf(pos)
   sum12 = rv1g+rv2g
   post1 = np.divide(rv1g, sum12)
   post2 = np.divide(rv2g, sum12)
   fig.clf()
   #plt.set_cmap('seismic')
   ax = fig.add_subplot(111)
   # plot Decision Boundary or Difference of PDFs
   plotType = drawType.get()
   if plotType == 'Decision Boundary':
      ax.imshow((post1>post2).T, origin='lower', extent=[xlim[0], xlim[1], ylim[0], ylim[1]], cmap='bwr')
      fig.suptitle(plotType)
   elif plotType == 'PDF difference':
      maxdata = np.max(np.abs(rv1g-rv2g))
      cax = ax.imshow((rv1g-rv2g).T, origin='lower', extent=[xlim[0], xlim[1], ylim[0], ylim[1]], cmap='Spectral_r', vmin=-maxdata, vmax=maxdata)
      fig.colorbar(cax)
      fig.suptitle('P(1)p(x|1) - P(2)p(x|2)')
   elif plotType == 'Posterior difference':
      maxdata = np.max(np.abs(post1-post2))
      cax = ax.imshow((post1-post2).T, origin='lower', extent=[xlim[0], xlim[1], ylim[0], ylim[1]], cmap='Spectral_r', vmin=-maxdata, vmax=maxdata)
      fig.colorbar(cax)
      fig.suptitle('P(1|x) - P(2|x)')
   else:
      messagebox.showerror("Error!", "Plot type not supported")
   ax.text(mu1[0], mu1[1], '+', color='white', horizontalalignment='center', verticalalignment='center')
   ax.text(mu2[0], mu2[1], 'o', color='white', horizontalalignment='center', verticalalignment='center')
   ax.set_xlabel('x')
   ax.set_ylabel('y')
   # plot contours for each PDF
   if drawPDFContour.get():
      ax.contour(x, y, rv1g, colors='w')
      ax.contour(x, y, rv2g, colors='w')
      #plt.contour(x, y, rv1g.reshape(x.shape), norm=LogNorm(vmin=1.0, vmax=40.0),levels=np.logspace(0, 3, 10))
      #plt.contour(x, y, rv2g.reshape(x.shape), norm=LogNorm(vmin=1.0, vmax=40.0),levels=np.logspace(0, 3, 10))
   canvas.draw()

root = tk.Tk()
root.title("Gaussian Decision Boundaries")
# set default font size
defaultFont = font.nametofont("TkDefaultFont")
defaultFont.configure(family="helvetica", size=14)

# Gaussian distribution parameters
p1 = tk.StringVar()
mu1x = tk.StringVar()
mu1y = tk.StringVar()
s1x = tk.StringVar()
s1y = tk.StringVar()
s1xy = tk.StringVar()
p2 = tk.StringVar()
mu2x = tk.StringVar()
mu2y = tk.StringVar()
s2x = tk.StringVar()
s2y = tk.StringVar()
s2xy = tk.StringVar()

# drawing parameters
drawType = tk.StringVar()
drawPDFContour = tk.BooleanVar()
xmin = tk.StringVar()
xmax = tk.StringVar()
ymin = tk.StringVar()
ymax = tk.StringVar()

# set default values
def reset():
   p1.set('0.5')
   mu1x.set('-1.0')
   mu1y.set('-1.0')
   s1x.set('1.0')
   s1y.set('1.0')
   s1xy.set('0.0')
   p2.set('0.5')
   mu2x.set('1.0')
   mu2y.set('1.0')
   s2x.set('1.0')
   s2y.set('1.0')
   s2xy.set('0.0')
   drawType.set('Decision Boundary')
   drawPDFContour.set(True)
   xmin.set("-1.5")
   xmax.set("1.5")
   ymin.set("-1.5")
   ymax.set("1.5")

reset()

# create control widgets
entryWidth=5
controlFrame = ttk.Frame(root)
gaussianFrame = ttk.Frame(controlFrame)
gaussian1Frame = ttk.LabelFrame(gaussianFrame, text='Gaussian 1')
p1W = ttk.Entry(gaussian1Frame, textvariable=p1, width=entryWidth, font=defaultFont)
mu1xW = ttk.Entry(gaussian1Frame, textvariable=mu1x, width=entryWidth, font=defaultFont)
mu1yW = ttk.Entry(gaussian1Frame, textvariable=mu1y, width=entryWidth, font=defaultFont)
s1xW = ttk.Entry(gaussian1Frame, textvariable=s1x, width=entryWidth, font=defaultFont)
s1yW = ttk.Entry(gaussian1Frame, textvariable=s1y, width=entryWidth, font=defaultFont)
s1xyW = ttk.Entry(gaussian1Frame, textvariable=s1xy, width=entryWidth, font=defaultFont)
s1yxW = ttk.Entry(gaussian1Frame, textvariable=s1xy, width=entryWidth, font=defaultFont)
gaussian2Frame = ttk.LabelFrame(gaussianFrame, text='Gaussian 2')
p2W = ttk.Label(gaussian2Frame, text='1-p1')
mu2xW = ttk.Entry(gaussian2Frame, textvariable=mu2x, width=entryWidth, font=defaultFont)
mu2yW = ttk.Entry(gaussian2Frame, textvariable=mu2y, width=entryWidth, font=defaultFont)
s2xW = ttk.Entry(gaussian2Frame, textvariable=s2x, width=entryWidth, font=defaultFont)
s2yW = ttk.Entry(gaussian2Frame, textvariable=s2y, width=entryWidth, font=defaultFont)
s2xyW = ttk.Entry(gaussian2Frame, textvariable=s2xy, width=entryWidth, font=defaultFont)
s2yxW = ttk.Entry(gaussian2Frame, textvariable=s2xy, width=entryWidth, font=defaultFont)
drawingFrame = ttk.LabelFrame(controlFrame, text='Drawing')
drawTypeW = ttk.Combobox(drawingFrame, textvariable=drawType, font=defaultFont)
drawTypeW['values'] = ('Decision Boundary', 'PDF difference', 'Posterior difference')
drawPDFContourW = ttk.Checkbutton(drawingFrame, text="Draw PDF Contours", variable=drawPDFContour)
xlimFrame = ttk.Frame(drawingFrame)
xlimL = ttk.Label(xlimFrame, text='xlim')
xminW = ttk.Entry(xlimFrame, textvariable=xmin, width=entryWidth, font=defaultFont)
xmaxW = ttk.Entry(xlimFrame, textvariable=xmax, width=entryWidth, font=defaultFont)
ylimFrame = ttk.Frame(drawingFrame)
ylimL = ttk.Label(ylimFrame, text='ylim')
yminW = ttk.Entry(ylimFrame, textvariable=ymin, width=entryWidth, font=defaultFont)
ymaxW = ttk.Entry(ylimFrame, textvariable=ymax, width=entryWidth, font=defaultFont)
redrawButton = ttk.Button(drawingFrame, text="Redraw", command=lambda: redraw(fig))
aboutButton = ttk.Button(controlFrame, text="About...", command=about)
resetButton = ttk.Button(controlFrame, text="Reset", command=reset)

# place widgets within gaussian1Frame
gaussian1Frame.grid(column=0, row=0, columnspan=2, rowspan=6)
p1L = ttk.Label(gaussian1Frame, text='p1')
p1L.grid(row=0, column=0)
p1W.grid(row=0, column=1)
mu1L = ttk.Label(gaussian1Frame, text='mean1')
mu1L.grid(row=1, column=0, columnspan=2)
mu1xW.grid(row=2, column=0)
mu1yW.grid(row=2, column=1)
s1L = ttk.Label(gaussian1Frame, text='cov1')
s1L.grid(row=3, column=0, columnspan=2)
s1xW.grid(row=4, column=0)
s1xyW.grid(row=4, column=1)
s1yxW.grid(row=5, column=0)
s1yW.grid(row=5, column=1)

# place widgets within gaussian2Frame
gaussian2Frame.grid(column=0, row=0, columnspan=2, rowspan=6)
p2L = ttk.Label(gaussian2Frame, text='p2 = 1-p1')
p2L.grid(row=0, column=0, columnspan=2)
mu2L = ttk.Label(gaussian2Frame, text='mean2')
mu2L.grid(row=1, column=0, columnspan=2)
mu2xW.grid(row=2, column=0)
mu2yW.grid(row=2, column=1)
s2L = ttk.Label(gaussian2Frame, text='cov2')
s2L.grid(row=3, column=0, columnspan=2)
s2xW.grid(row=4, column=0)
s2xyW.grid(row=4, column=1)
s2yxW.grid(row=5, column=0)
s2yW.grid(row=5, column=1)

# place widgets within drawing frame
drawTypeW.pack(side="top")
drawPDFContourW.pack(side="top")
xlimL.pack(side="left")
xminW.pack(side="left")
xmaxW.pack(side="left")
xlimFrame.pack(side="top")
ylimL.pack(side="left")
yminW.pack(side="left")
ymaxW.pack(side="left")
ylimFrame.pack(side="top")
redrawButton.pack(side="top")

# place Gaussian frames within gaussianFrame
gaussian1Frame.pack(side="left")
gaussian2Frame.pack(side="left")

# place gfame and drawing frame within controlFrame
aboutButton.pack(side="top")
resetButton.pack(side="top")
gaussianFrame.pack(side="top")
drawingFrame.pack(side="top")

controlFrame.pack(side="left")

figureFrame = tk.Frame(root)
fig = plt.Figure()
canvas = FigureCanvasTkAgg(fig, master=root)
#tkagg.NavigationToolbar2TkAgg(canvas, root)
canvas.show()
canvas.get_tk_widget().pack(side='top', fill='both', expand=1)
figureFrame.pack()
redraw(fig)
root.mainloop()


# Otto-Grabber - Grabbs Products from Otto.de
#
# Creation:    30.09.2019
# Last Update: 04.10.2019
#
#
# MIT License
#
# Copyright (c) 2019 by PiereLucas
# https://github.com/pierelucas
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

#
# Module
#
import requests, os, subprocess, re
from bs4 import BeautifulSoup

#
# Output
#
def out():

    subprocess.call("clear", shell=True)

    print("""
    This tool is for scarping new Base_URLS and save to csv.
    After that, just copy the new URLS in the grabby.py

    [1] Multimedia
    [2] Haushalt
    [3] Kueche
    [0] Exit
    """)

#
# Grab Function
#
def grab(url, sec):
    # Open file
    if os.path.exists("new_urls_" + sec):
        subprocess.call("rm -rf new_urls_" + sec, shell=True)
        os.mkdir("new_urls" + sec)
        new_urls = open("new_urls_" + sec + "/new_urls.csv", 'wt')
    else:
        os.mkdir("new_urls" + sec)
        new_urls = open("new_urls_" + sec + "/new_urls.csv", 'wt')

    # Parse URL to get new_urls
    url = requests.get(url)
    urltext = url.text
    urlsoup = BeautifulSoup(urltext, 'html.parser')

    url_list = urlsoup.find_all('a', 'href', class_="ts-link")
    for item in url_list:
        item = str(item)
        url_list_item = re.findall("href=\".*\"", item)

        for item in url_list_item:
            # Regex
            link = re.sub("href=\"", "", item)
            link = re.sub("><span.*", "", link)
            link = re.sub("\"", "", link)
            link = re.sub("\?selektion", "", link)
            link = re.sub("target=.*", "", link)
            link = re.sub("/shoppages.*", "", link)
            link = re.sub("/user.*", "", link)
            link = re.sub("/reblog.*", "", link)
            link = re.sub("/twoforfashion.*", "", link)
            link = re.sub("/roombeez.*", "", link)
            link = re.sub("/soulfully.*", "", link)
            link = re.sub("/supermarkt.*", "", link)
            link = re.sub("/datenschutz.*", "", link)
            link = re.sub("https.*", "", link)
            link = re.sub("#SVGID.*", "", link)
            link = re.sub(" ", "", link)
            link = re.sub("/=\(.*", "", link)
            link = "https://otto.de" + link

            print(link)
            # Write to file
            new_urls.write(link + "\n")
    # Close to file
    new_urls.close()

#
# Main Function
#
def run():

    out()
    choice = str(input("Which option number : "))

    if choice == '1':

        sec = "multimedia"
        url = "https://www.otto.de/technik/multimedia/"
        grab(url, sec)

    elif choice == '2':
        sec = "haushalt"
        url = "https://www.otto.de/haushalt/"
        grab(url, sec)

    elif choice == '3':
        sec = kueche
        url = "https://www.otto.de/moebel/?ansicht=einstieg&thema=kueche"
        grab(url, sec)

# TO BE CONTINUED ...

out()
run()


import unittest
from itertools import zip_longest

class TestRecursive(unittest.TestCase):
    def assertEqualRecursive(self, first, second):
        """
        Asserts that two nested JSON-like structures consisting of dictionaries and lists are fully equal.

        >>> TestRecursive().assertEqualRecursive({1: [{'a': 2}, 'b']}, {1: [{'a': 3}, 'b']})
        Traceback (most recent call last):
        ...
        AssertionError: 1 error:
        Values differ: 2 != 3 at [1][0][a]

        """
        paths = self._check_equal_recursive(first, second)
        if paths:
            msg = '{} error{}:\n'.format(len(paths), 's' if len(paths) > 1 else '') + '\n'.join(paths)
            raise self.failureException(msg)

    def _check_equal_recursive(self, first, second, *path_so_far):
        diffs = []
        if isinstance(first, list) and isinstance(second, list):
            for i, (f, s) in enumerate(zip_longest(first, second)):
                diffs.extend(
                    self._check_equal_recursive(f, s, str(i), *path_so_far)
                )
            return diffs

        joined_path = ']['.join(str(i) for i in reversed(path_so_far))
        if isinstance(first, dict) and isinstance(second, dict):
            keys1 = first.keys()
            keys2 = second.keys()
            for k in keys1 - keys2:
                diffs.append('`{k}: {v}` not found in second object at [{path}] '.format(
                    path=joined_path,
                    k=k,
                    v=first[k]
                ))
            for k in keys2 - keys1:
                diffs.append('`{k}: {v}` not found in first object at [{path}] '.format(
                    path=joined_path,
                    k=k,
                    v=second[k]
                ))
            for k in keys1 & keys2:
                diffs.extend(
                    self._check_equal_recursive(first[k], second[k], k, *path_so_far)
                )
        elif not first == second:
            # If one is still a dict or list, don't display (since it might be large),
            # and don't str() since it's a json type
            if {type(first), type(second)} & {dict, list}:
                diffs.append('Types differ: {first} != {second} at [{path}]'.format(
                    path=joined_path,
                    first=type(first),
                    second=type(second)
                ))
            # Also don't compare other json types as strings
            elif {type(first), type(second)} & {int, float, bool, type(None)} or \
                    not str(first) == str(second):
                diffs.append('Values differ: {first} != {second} at [{path}]'.format(
                    path=joined_path,
                    first=first,
                    second=second
                ))

        return diffs


import pytest

from mirakuru import TCPExecutor

@pytest.fixture
def use_local_matrix_server():
    """
    If True, try to test with a real client and start a fresh local
    matrix server for each test. This should be used for integration
    tests only.
    If False, use the mock matrix client.
    """
    return False

# the empty fixtures are overwritten in integration/conftest.py
@pytest.fixture
def local_matrix():
    return None

@pytest.fixture
def matrix_host():
    return None

@pytest.fixture
def matrix_port():
    return None

@pytest.fixture
def local_matrix_server(
        use_matrix,
        use_local_matrix_server,
        local_matrix,
        matrix_host,
        matrix_port
):

    if not (use_matrix and use_local_matrix_server):
        yield None
        return

    assert local_matrix is not None, \
        "No command to start the local matrix server given. (--local-matrix option)"

    server = TCPExecutor(
        local_matrix,
        host=matrix_host,
        port=matrix_port,
        timeout=120,
        sleep=0.1,
        shell=True
    )

    server.start()
    yield 'http://{}:{}'.format(matrix_host, matrix_port)
    server.stop()


import sys
import logging

APP_LOGGER_NAME = 'snappi_ixnetwork'

def setup_ixnet_logger(log_level, file_name=None, module_name=None):
    logger = logging.getLogger(APP_LOGGER_NAME)
    logger.setLevel(log_level)
    formatter = logging.Formatter(fmt="%(asctime)s [%(name)s] [%(levelname)s] %(message)s",
                                  datefmt="%Y-%m-%d %H:%M:%S")
    sh = logging.StreamHandler(sys.stdout)
    sh.setFormatter(formatter)
    if len(logger.handlers) > 0:
        del logger.handlers[:]
    logger.addHandler(sh)
    if file_name:
        fh = logging.FileHandler(file_name)
        fh.setFormatter(formatter)
        logger.addHandler(fh)
    if module_name is not None:
        logger = get_ixnet_logger(module_name)
    return logger

def get_ixnet_logger(module_name):
    module_name = ".".join(str(module_name).split(".")[1:])
    return logging.getLogger(APP_LOGGER_NAME).getChild(module_name)


# -*- coding: UTF-8 -*-
import networkx as nx
import random
import copy
import os
import BigPoint
import time

# 3条最优路径,且同时产生
# 在slack允许的情况下找等待时间最长的点进行插入
# 对待插入的点，随机选择一天插入
# 对每一条数据进行测试
# Super-POI的值如果小于1000的话，增大10倍。如果原始值为0，那么设置为5000

def createGraph(myGraph, fileName):
    categoryMap = {1:70, 2:40, 3:20, 4:10, 5:5}

    file = open(fileName)
    # 略过两行注释
    file.readline()
    file.readline()
    lists = file.readline().strip().split(';')
    # print(lists)
    G.graph['nb_nodes'] = int(lists[0])
    G.graph['RouteMaxDuration'] = int(lists[1])
    G.graph['TotalMaxDuration'] = int(lists[2])
    print(G.graph)

    # 略过两行注释
    file.readline()
    file.readline()
    for i in range(G.graph['nb_nodes']):
        lists = file.readline().strip().split(';')
        node = int(lists[0])
        G.add_node(node)
        G.nodes[node]['ID'] = node
        G.nodes[node]['ServiceTime'] = categoryMap[int(lists[1])]
        G.nodes[node]['Priority'] = int(lists[2])
        G.nodes[node]['Profit'] = int(lists[3])
        G.nodes[node]['Probability'] = float(lists[4])
        if int(lists[1] == 1):
            if tmpP <= 0:
                tmpP = 5000
            if tmpP < 1000:
                tmpP = tmpP * 10
            G.nodes[node]['Profit'] = tmpP
        TWS = lists[5].split(',')
        # for TW in TWS:
        timeWindows = [{} for _ in range(4)]
        for TW in TWS:
            window = {}
            params = TW.split(':')
            window['day'] = int(params[0])
            window['opentime'] = int(params[1].split('-')[0])
            window['closetime'] = int(params[1].split('-')[1])
            index = int(params[0])
            if timeWindows[index].get('day', False) == False:
                timeWindows[index] = window
            else:
                timeWindows[index]['closetime'] = window['opentime']
        for index, TW in enumerate(timeWindows):
            if TW == {}:
                opentime = random.randrange(0, 600)
                closetime = random.randrange(0, 600)
                if opentime > closetime:
                    opentime, closetime = closetime, opentime
                while closetime - opentime < 200:
                    opentime = random.randrange(0, 600)
                    closetime = random.randrange(0, 600)
                    if opentime > closetime:
                        opentime, closetime = closetime, opentime
                TW['day'] = index
                TW['opentime'] = opentime
                TW['closetime'] = closetime
        G.nodes[node]['TimeWindows'] = tuple(timeWindows[:])
    # print(G.nodes.data())

    # 略过两行注释
    file.readline()
    file.readline()
    arcs = file.readlines()
    for line in arcs:
        lists = line.strip().split(';')
        s = int(lists[0]); t = int(lists[1]); duration = int(lists[2])
        G.add_edge(s, t)
        G.edges[s, t]['duration'] = duration
    # print(G.edges.data())

    file.close()

def isCompleteGraph(G):
    nodeNum = len(G.nodes)
    edgeNum = len(G.edges)
    print(nodeNum * (nodeNum - 1) / 2)
    print(edgeNum)

def calcuSlack(myGraph, travelPath, node, insertLocation, timeParamDict):
    # 插入到insertLocation之前
    travelPath.insert(insertLocation, {'ID':node['ID']})
    travelPath[insertLocation]['aTime'] = timeParamDict['arriveTimeToCandiNode']
    travelPath[insertLocation]['dTime'] = timeParamDict['deparTimeOnCandiNode']
    travelPath[insertLocation]['closetime'] = node['TimeWindows'][timeParamDict['day']]['closetime']
    preComponentID = node['ID']
    nextComponentID = -1

    # 对插入点之后的点进行时间更新
    for index, pathComponent in enumerate(travelPath[insertLocation+1:]):
        # !!!注意index的值!!!
        index = index + insertLocation + 1

        nextComponentID = pathComponent['ID']
        duration = myGraph.edges[preComponentID, nextComponentID]['duration']
        arriveToNextComponent = travelPath[index-1]['dTime'] + duration
        window = myGraph.nodes[nextComponentID]['TimeWindows'][timeParamDict['day']]
        deparTimeOnNextComponent = max(arriveToNextComponent, window['opentime']) + myGraph.nodes[nextComponentID]['ServiceTime']
        travelPath[index]['aTime'] = arriveToNextComponent
        travelPath[index]['dTime'] = deparTimeOnNextComponent
        travelPath[index]['closetime'] = window['closetime']
        preComponentID = nextComponentID

    # 计算终点的slack值
    tmpSlack = travelPath[-1]['slack'] = myGraph.graph['RouteMaxDuration'] - travelPath[-1]['aTime']
    totalSlack = tmpSlack
    # 全体slack更新
    travelPath[0]['closetime'] = 10000000
    for pathComponent in travelPath[-2::-1]:
        pathComponent['slack'] = min(pathComponent['closetime']-pathComponent['dTime'], tmpSlack)
        tmpSlack = pathComponent['slack']
        totalSlack = totalSlack + tmpSlack

    return totalSlack, travelPath

# 从头重新计算各自的时间
def calcuSlack2(myGraph, travelPath, node, insertLocation, timeParamDict):
    # 插入到insertLocation之前
    travelPath.insert(insertLocation, {'ID':node['ID']})
    # travelPath[insertLocation]['aTime'] = timeParamDict['arriveTimeToCandiNode']
    # travelPath[insertLocation]['dTime'] = timeParamDict['deparTimeOnCandiNode']
    # travelPath[insertLocation]['closetime'] = node['TimeWindows'][timeParamDict['day']]['closetime']
    preComponentID = travelPath[0]['ID']
    nextComponentID = -1

    # 对插入点之后的点进行时间更新
    for index, pathComponent in enumerate(travelPath[1:]):
        # !!!注意index的值!!!
        index = index + 1

        nextComponentID = pathComponent['ID']
        duration = myGraph.edges[preComponentID, nextComponentID]['duration']
        # if index == 1:
        #     print(travelPath[index-1]['dTime'], duration)
        arriveToNextComponent = travelPath[index-1]['dTime'] + duration
        window = myGraph.nodes[nextComponentID]['TimeWindows'][timeParamDict['day']]
        deparTimeOnNextComponent = max(arriveToNextComponent, window['opentime']) + myGraph.nodes[nextComponentID]['ServiceTime']
        travelPath[index]['aTime'] = arriveToNextComponent
        travelPath[index]['dTime'] = deparTimeOnNextComponent
        travelPath[index]['closetime'] = window['closetime']
        travelPath[index]['waitTime'] = window['opentime'] - arriveToNextComponent
        if travelPath[index]['waitTime'] < 0:
            travelPath[index]['waitTime'] = 0
        preComponentID = nextComponentID

    # 计算终点的slack值
    tmpSlack = travelPath[-1]['slack'] = myGraph.graph['RouteMaxDuration'] - travelPath[-1]['aTime']
    totalSlack = tmpSlack
    # 全体slack更新
    travelPath[0]['closetime'] = 10000000
    for pathComponent in travelPath[-2::-1]:
        # print(pathComponent)
        pathComponent['slack'] = min(pathComponent['closetime'] - pathComponent['dTime'], tmpSlack)
        tmpSlack = pathComponent['slack']
        totalSlack = totalSlack + tmpSlack

    return totalSlack, travelPath

def toptw(startDestList, myGraph, bigPointDir):

    # 用来记录加入到路径中的点
    existList = []

    routeMaxDuration = myGraph.graph['RouteMaxDuration']
    threeDayPath = []
    threeDayProfitList = [0, 0, 0]
    threeDayBestRatioList = [-1, -1, -1]
    for i, val in enumerate(startDestList):
        s = val['s']
        t = val['t']
        existList.append(s)
        existList.append(t)
        tP = [{'ID':s}, {'ID':t}]
        duration = myGraph.edges[s, t]['duration']
        startSlack = 0
        destSlack = routeMaxDuration - duration
        tP[0]['slack'] = startSlack
        tP[0]['aTime'] = 0
        tP[0]['dTime'] = 0
        tP[0]['waitTime'] = 0
        tP[1]['slack'] = destSlack
        tP[1]['aTime'] = duration
        tP[1]['dTime'] = duration
        # 终点的等待时间，如何确定呢
        tP[1]['waitTime'] = myGraph.nodes[t]['TimeWindows'][i+1]['opentime'] - duration
        if tP[1]['waitTime'] < 0:
            tP[1]['waitTime'] = 40
        threeDayPath.append(tP)

    finish = False
    while finish == False:
        finish = True
        # top-k最优路径
        # k = 5
        # bestKPath = []
        # bestKRatio = []
        # bestKProfit = []
        # bestKNodeId = []
        # 寻找一个合适点进行插入
        for data in myGraph.nodes.data():
            node = data[1]
            if node['ID'] in existList:
                continue

            threeDayRatioList = [-1, -1, -1]
            threeDayTmpPathList = [[] for i in range(3)]
            threeDayTmpProfitList = [x for x in threeDayProfitList]

            # 遍历三条路径，选择一条最适合的插入这个点
            for day, travelPath in enumerate(threeDayPath):
                day = day + 1
                # 找一个最匹配的slack插入到它前边
                bestMatch = -1
                tmpWaitTime = -1
                gapTime = 10000000
                for index, _ in enumerate(travelPath[1:]):
                    # index值是阶段后的列表中的索引值
                    index = index + 1
                    preComponent = travelPath[index-1]
                    preToCandiNodeDuration = myGraph.edges[preComponent['ID'], node['ID']]['duration']
                    arriveTimeToCandiNode = preComponent['dTime'] + preToCandiNodeDuration
                    if (arriveTimeToCandiNode > routeMaxDuration
                    or max(arriveTimeToCandiNode, node['TimeWindows'][day]['opentime']) + node['ServiceTime'] > routeMaxDuration
                    or node['TimeWindows'][day]['closetime'] - arriveTimeToCandiNode < node['ServiceTime']):
                        break

                    deparTimeOnCandiNode = max(arriveTimeToCandiNode, node['TimeWindows'][day]['opentime']) + node['ServiceTime']
                    curComponent = travelPath[index]
                    candiNodeToCurNodeDuration = myGraph.edges[node['ID'], curComponent['ID']]['duration']
                    arriveTimeToCurNode = deparTimeOnCandiNode + candiNodeToCurNodeDuration
                    # 由于插入这个点增加的时间
                    # t1为加入的点带来的时间duraion，t2为原先的pre到cur的duration
                    t1 = arriveTimeToCurNode - preComponent['dTime']
                    t2 = curComponent['aTime'] - preComponent['dTime']
                    deltaTime = t1 - t2
                    # 在slack允许的情况下找等待时间最长的点进行插入
                    curGapTime = curComponent['slack'] - deltaTime
                    if curGapTime < 0:
                        continue
                    if travelPath[index]['waitTime'] > tmpWaitTime:
                        bestMatch = index
                        tmpWaitTime = travelPath[index]['waitTime']

                    # # 更新当前的间隔值
                    # if curGapTime < gapTime:
                    #     gapTime = curGapTime
                    #     bestMatch = index

                # 没有合适的插入点
                if bestMatch == -1:
                    continue

                # 计算插入后的总收益与slack的比值
                tmpTotalProfit = threeDayProfitList[day-1] + node['Profit']
                tmpTravelPath = [copy.copy(x) for x in travelPath]
                timeParamDict = {
                                'arriveTimeToCandiNode':arriveTimeToCandiNode,
                                'deparTimeOnCandiNode':deparTimeOnCandiNode,
                                'arriveTimeToCurNode':arriveTimeToCurNode,
                                'day':day
                                }
                tmpTotalSlack, tmpTravelPath = calcuSlack2(myGraph, tmpTravelPath, node, bestMatch, timeParamDict)
                if float(tmpTotalSlack) == 0:
                    continue
                ratio = float(tmpTotalProfit) / float(tmpTotalSlack)
                threeDayRatioList[day-1] = ratio
                threeDayTmpPathList[day-1] = [copy.copy(x) for x in tmpTravelPath]
                threeDayTmpProfitList[day-1] = tmpTotalProfit

            # 随机选择一天进行插入
            accept = False

            dieTime = 0
            while accept == False:
                randomDay = random.randrange(0, 3)
                if threeDayRatioList[randomDay] > threeDayBestRatioList[randomDay]:
                    threeDayBestRatioList[randomDay] = threeDayRatioList[randomDay]
                    threeDayPath[randomDay] = [copy.copy(x) for x in threeDayTmpPathList[randomDay]]
                    threeDayProfitList[randomDay] = threeDayTmpProfitList[randomDay]
                    accept = True
                dieTime = dieTime + 1
                if dieTime > 15:
                    break

            # 取threeDayRatioList中最大值的那条路就是要插入的路径
            # accept = False
            # for i in range(3):
            #     if accept == False:
            #         bestRatio = max(threeDayRatioList)
            #         whichDay = threeDayRatioList.index(bestRatio)
            #         if bestRatio > threeDayBestRatioList[whichDay]:
            #             threeDayBestRatioList[whichDay] = bestRatio
            #             threeDayPath[whichDay] = [copy.copy(x) for x in threeDayTmpPathList[whichDay]]
            #             threeDayProfitList[whichDay] = threeDayTmpProfitList[whichDay]
            #             accept = True
            #         threeDayRatioList[whichDay] = -5

            if accept == True:
                existList.append(node['ID'])
                finish = False

    profitSum = sum(threeDayProfitList)
    bigPointPaths = os.listdir(bigPointDir)
    for day, path in enumerate(threeDayPath):
        print(path)
        for component in path:
            if str(component['ID']) in bigPointPaths:
                smallG = nx.read_gml(bigPointDir + '\\' + str(component['ID']))
                smallG = nx.convert_node_labels_to_integers(smallG)
                tmpP = BigPoint.dfsTraverse(smallG, component['aTime'] + component['waitTime'], component['dTime'], day+1)
                profitSum = profitSum + tmpP
                profitSum = profitSum - myGraph.nodes[component['ID']]['Profit']
        print()
    print('total profit:', profitSum)

    #         # 检查当前点是否可以插入到bestKPath中
    #         # 当前的ratio比存储的k条路径中的最差路径好
    #         if len(bestKRatio) < k:
    #             bestKPath.append([copy.copy(x) for x in tmpTravelPath])
    #             bestKRatio.append(ratio)
    #             bestKProfit.append(tmpTotalProfit)
    #             bestKNodeId.append(node['ID'])
    #             # 代表还有比较好的点
    #             finish = False
    #         elif ratio > min(bestKRatio):
    #             index = bestKRatio.index(min(bestKRatio))
    #             del bestKPath[index]
    #             del bestKRatio[index]
    #             del bestKNodeId[index]
    #             del bestKProfit[index]
    #             bestKPath.append([copy.copy(x) for x in tmpTravelPath])
    #             bestKRatio.append(ratio)
    #             bestKProfit.append(tmpTotalProfit)
    #             bestKNodeId.append(node['ID'])
    #             # 代表还有比较好的点
    #             finish = False

    #     # 从k个候选者中随机挑出一个插入,
    #     # ！！！！！注意k个候选者没有选满的情况啊啊啊啊啊啊啊啊啊啊
    #     # 更新路径travelPath
    #     length = min(k, len(bestKPath))
    #     if length == 0:
    #         continue
    #     selectNode = random.randrange(0, length)
    #     travelPath = [copy.copy(x) for x in bestKPath[selectNode]]
    #     bestRatio = bestKRatio[selectNode]
    #     totalProfit = bestKProfit[selectNode]
    #     existList.append(bestKNodeId[selectNode])
    # # print(travelPath)
    # return travelPath;

# os.chdir('instances')
# fileList = os.listdir('.')
# for f in fileList:
#     if os.path.isfile(f):
#         G = nx.Graph()
#         createGraph(G, f)
#         ll = random.sample(G.nodes, 6)
#         # print(ll)
#         startDestList = [{'s':ll[0],'t':ll[1]}, {'s':ll[2],'t':ll[3]}, {'s':ll[4],'t':ll[5]}]
#         # print(startDestList)
#         toptw(startDestList, G, f.split('.')[0])

f = 'd:\PythonCode\TOPTW\instances\TPA_6_40-3.txt'
G = nx.Graph()
createGraph(G, f)
ll = random.sample(G.nodes, 6)
# print(ll)
startDestList = [{'s':ll[0],'t':ll[1]}, {'s':ll[2],'t':ll[3]}, {'s':ll[4],'t':ll[5]}]
# print(startDestList)
t1 = time.time()
toptw(startDestList, G, f.split('.')[0])
t2 = time.time()
print('time:', t2 - t1)


import os
import requests
from fake_useragent import UserAgent
from lxml import etree
from urllib import request
# tarenacode
# code_2013
# http://code.tarena.com.cn/AIDCode/aid1907/11-Ajax/

FILEPATH = './mynote'

class TarenaCode:
    def __init__(self):
        self.url = 'http://code.tarena.com.cn/AIDCode/aid1907/11-Ajax/'
        self.auth = ('tarenacode','code_2013')
        self.headers = {
            'User-Agent': UserAgent().random
        }

    def parse_html(self):
        """解析网页"""
        html = self.get_html()
        text = etree.HTML(html)
        a_list = text.xpath('//a/@href')
        # ['day01/','day02/','mytest.zip']
        print(a_list)
        myfile = [ i for i in a_list if '/' not in i]
        print(myfile)
        self.save_file(myfile)

    def get_html(self):
        """下载网页"""
        try:
            response = requests.get(
                url=self.url,
                auth=self.auth,
                headers = self.headers
            )
        except Exception as e:
            print('[error]:',e)
        else:
            return response.text

    def save_file(self,files):
        """保存文件"""
        if not os.path.exists(FILEPATH):
            os.makedirs('mynote')
        for each_file in files:
            file_url = self.url + each_file
            binary_file = requests.get(file_url,auth=self.auth,headers=self.headers).content
            with open(FILEPATH+'/'+each_file,'wb') as f:
                f.write(binary_file)
            print(each_file,'下载成功！')

    def main(self):
        """主方法，用来启动爬虫"""
        self.parse_html()

if __name__ == "__main__":
    tc = TarenaCode()
    tc.main()

"""
day01/                                             28-Oct-2019 21:27       -
day02/                                             28-Oct-2019 21:27       -
day03/                                             29-Oct-2019 17:26       -
practice/                                          30-Oct-2019 18:17       -
day01-note.zip                                     24-Oct-2019 23:57    499K
day03-note.zip                                     28-Oct-2019 22:38    343K
day03_all.zip

"""
# (.*?)


#!/usr/bin/env python3

import socket # Needed for socket creation
import ntplib # Needed for NCP time (syncronize the time between source and destination)
import sys	  # Needed library
from datetime import datetime # Needed for NCP time

# GENERAL DESCRIPTION OF THE SCRIPT
# According to our design, since there is a TCP connection between source and broker,
# reliable data transfer exists. So from source to broker, TCP is used.
# This script first creates and connects to broker with a socket, then starts to read
# the related 5 MB file (input.txt) 1024 bytes per segment and sends to the broker.
# At the end of the file, file is closed and "Closed" is printed for tracking purposes.
# When this script is run and finished, there suppose to be an inout.txt file at the
# broker, sent by source. As soon as broker receives the file, it will start to transfer
# the data with reliable UDP implemented by us. (more detailed explanation in broker.py)

# GLOBAL variables
HOST = '10.10.1.2'  # Broker's IP adress (used to send the file)
PORT = 31336        # PORT used for file transfer from source to broker (TCP)
input_file = ''		# Initialize the input file name (to read input.txt)
input_message = []	# Initialize the message, (segment by segment, this will be updated)

if __name__ == "__main__":
    input_file = sys.argv[1] # Get the input file as argument.

    # Below three lines are used to get time from time.google.com, for tracking purposes
    c = ntplib.NTPClient()
    response = c.request('time.google.com', version=3)
    time = datetime.fromtimestamp(response.orig_time)

    print(time) #Print current time (Used to compare with destination time)

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Create socket (which will be used to connect to the broker)
    s.connect((HOST, PORT)) # Connect to the broker

    # sys.stdout.flush() # calculate input file size

    inp = open('input.txt','rb') 	# Open the input.txt file to start reading and sending.
    input_message = inp.read(1024)	# Send the message as 1024 bytes segments.
    print("Sending...")				# Print for tracking purposes

    # This while loop sends the file as 1024 bytes segment untill reaching the end of file.
    while (input_message):				# Will be over when input message is NULL (eof)
        s.sendall(input_message) 		# Send the segment
        input_message = inp.read(1024)	# Read next 1024 from input.txt

    inp.close() 	# Close the file


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

###
###This file pulls from yahoo finance minute data for last 30 days
###and puts into MarketData folder in daily increments
###
from pandas_datareader import data as pdr
import datetime as dt
import yfinance as yf
import os

yf.pdr_override()
today = dt.date.today()

symbols =['AAPL', 'SPY', 'NVDA', 'SPCE', 'CPRX', 'BABA', 'AMZN',
         'GOOGL', 'MSFT', 'PTON','NKE','PYPL','LULU','TSM']

###makes path for every symbol if it does not exist

for i in symbols:
    if not os.path.exists('MarketData/'+i):
        os.makedirs('MarketData/'+i)

###pulls data from yahoo finance from last 30 days that does not exist
for i in symbols:
    day = dt.date.today()
    yesterday = day - dt.timedelta(days=1)
    for j in range(29):
        if not os.path.exists('MarketData/'+i+'/'+yesterday.strftime("%Y-%m-%d")+'.csv'):
            if not yesterday.weekday() > 4 :
                pdr.DataReader(i,yesterday.strftime("%Y-%m-%d"),day.strftime("%Y-%m-%d"), interval="1m").to_csv('MarketData/'+i+'/'+yesterday.strftime("%Y-%m-%d")+'.csv')
        day = yesterday
        yesterday = day - dt.timedelta(days=1)

print ("done")


"""multiple_apps URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.10/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib import admin

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^',include('apps.testmodels.urls',namespace='testmodels')),
    url(r'^timedisplay/',include('apps.timedisplay.urls',namespace='timedisplay')),
    url(r'^surveyform/',include('apps.surveyform.urls',namespace='surveyform')),
    url(r'^random_word/',include('apps.random_word.urls',namespace='random_word')),
    url(r'^login_reg/',include('apps.login_reg.urls',namespace='login_reg')),
    url(r'^course_app/',include('apps.course_app.urls',namespace='course_app')),
]


# -*- coding: utf-8 -*-
'''
		   @file: input.py
		   @date:
		 @author: Carlos Adir (carlos.adir.leite@gmail.com)
	@description: This code is to be the inputs of the codes, only for don't repeat the same part of code
					in all the files

'''

import sympy as sp
import numpy as np
import aux

limites = [	[1, 3],\
			[1, 5],\
			[1, 3],\
			[1, 3],\
			[1, 3]]

def get(algorithm, number):
	limite = limites[algorithm-1]
	if number < limite[0]:
		number = limite[0]
	if number > limite[1]:
		number = limite[1]

	if algorithm == 1:
		return lambda : in1(number)
	elif algorithm == 2:
		return lambda : in2(number)
	elif algorithm == 3:
		return lambda : in3(number)
	elif algorithm == 4:
		return lambda : in4(number)
	elif algorithm == 5:
		return lambda : in5(number)

def in1(number):
	t		= sp.symbols('t')
	if number == 1:
		a, b    = 1, 2							# The interval, it's suppose that a < b
		f       = t**2 - 3						# The function
		tol     = 1e-5 							# The max error acceptable
		nmax    = 10 							# Max number iteractions
	elif number == 2:
		a, b    = 2, 4							# The interval, it's suppose that a < b
		f       = t**2-2*t-4	 				# The function
		tol     = 1e-5 							# The max error acceptable
		nmax    = 10 							# Max number iteractions
	elif number == 3:
		a, b    = 1, 2							# The interval, it's suppose that a < b
		f       = t**3+4*t**2-10 				# The function
		tol     = 1e-5							# The max error acceptable
		nmax    = 10							# Max number iteractions

	# The begin to start the calculations
	feval 	= sp.lambdify(t, f, "numpy")
	flatex	= "$f(t) = " + aux.toLaTeX(f)
	f		= aux.Funcao(feval, flatex)
	return a, b, f, tol, nmax

def in2(number):
	t		= sp.symbols('t')
	if 1 <= number <= 5:
		p0 		= 1.5 							# The initial aproximation
		f       = t**3+4*t**2-10 				# The function that we want to calculate the roots
		tol     = 1e-5 							# The max error acceptable
		nmax    = 10 							# Max number iteractions
		if number == 1:
			g 		= t - t**3 - 4*t**2 + 10				# It doesn't converg
		elif number == 2:
			g 		= sp.sqrt(4*t - 10/t)						# It doesn't converg
		elif number == 3:
			g 		= sp.sqrt(10-t**3)/2					# The fixed point function, using f(x) = 0 we can get x = sqrt(10-x**3)/2
		elif number == 4:
			g 		= sp.sqrt(10/(4+t))						# It converges quite well
		elif number == 5:
			g 		= t - (t**3+4*t**2-10)/(3*t**2 + 8*t)	# It converges very well, we will see this funcion in the Newton's method

	g_ 		= sp.diff(g, t)						# The function g' derivative of g

	feval 	= sp.lambdify(t, f, "numpy")
	flatex	= "$f(t) = " + aux.toLaTeX(f)
	f		= aux.Funcao(feval, flatex)

	geval 	= sp.lambdify(t, g, "numpy")
	glatex	= "$g(t) = " + aux.toLaTeX(g)
	g		= aux.Funcao(geval, glatex)

	g_eval 	= sp.lambdify(t, g_, "numpy")
	g_latex	= "$g'(t) = " + aux.toLaTeX(g_)
	g_		= aux.Funcao(g_eval, g_latex)

	return p0, f, g, g_, tol, nmax

def in3(number):
	t		= sp.symbols('t')
	if number == 1:
		p0		= 1.5							# The interval, it's suppose that a < b
		f       = t**2 - 3						# The function
		tol     = 1e-5 							# The max error acceptable
		nmax    = 10 							# Max number iteractions
	elif number == 2:
		p0 		= 3								# The interval, it's suppose that a < b
		f       = t**2-2*t-4	 				# The function
		tol     = 1e-5 							# The max error acceptable
		nmax    = 10 							# Max number iteractions
	elif number == 3:
		p0 		= 1.5							# The interval, it's suppose that a < b
		f       = t**3+4*t**2-10 				# The function
		tol     = 1e-5							# The max error acceptable
		nmax    = 10							# Max number iteractions

	f_ 		= sp.diff(f, t)						# The function f' derivative of f

	# The begin to start the calculations
	feval 	= sp.lambdify(t, f, "numpy")
	flatex	= "$f(t) = " + aux.toLaTeX(f)
	f		= aux.Funcao(feval, flatex)

	f_eval 	= sp.lambdify(t, f_, "numpy")
	f_latex	= "$f'(t) = " + aux.toLaTeX(f_)
	f_		= aux.Funcao(f_eval, f_latex)

	return p0, f, f_, tol, nmax

def in4(number):
	t		= sp.symbols('t')
	if number == 1:
		p0, p1	= 1, 2							# The interval, it's suppose that a < b
		f		= t**2 - 3						# The function
		tol		= 1e-5 							# The max error acceptable
		nmax	= 10 							# Max number iteractions
	elif number == 2:
		p0, p1  = 2, 4							# The interval, it's suppose that a < b
		f       = t**2-2*t-4	 				# The function
		tol     = 1e-5 							# The max error acceptable
		nmax    = 10 							# Max number iteractions
	elif number == 3:
		p0, p1	= 1, 2							# The interval, it's suppose that a < b
		f       = t**3+4*t**2-10 				# The function
		tol     = 1e-5							# The max error acceptable
		nmax    = 10							# Max number iteractions

	# The begin to start the calculations
	feval 	= sp.lambdify(t, f, "numpy")
	flatex	= "$f(t) = " + aux.toLaTeX(f)
	f		= aux.Funcao(feval, flatex)
	return p0, p1, f, tol, nmax

def in5(number):
	return in4(number)


import gym
import gym_zmeyka

env = gym.make('zmeyka-v0')
last_reward = 0

env.reset()
for _ in range(1000):
    env.render()
    last_obs, rewards, done, info = env.step(env.action_space.sample())

    if done: break


import configparser
import requests
from wxpy import *
import itchat

# 获取每日精句
def get_message():
    message = requests.get('http://open.iciba.com/dsapi/')
    note = message.json()['note']
    content = message.json()['content']
    return note,content
    #print(message.json())

def send_message(message):
    bot = Bot()
    friends = bot.friends()
    for friend in friends:
        print(friend)

if __name__ == '__main__':
    send_message(2)
    get_message()
    # cf = configparser.ConfigParser()
    # cf.read('configure.ini', "utf-8")
    # print(cf.options('configure'))
    # print(cf.get('configure','weixin_name'))

爬虫学习/每天不同时段通过微信给好友发消息/每天不同时段通过微信给好友发消息.py

每天不同时段通过微信给好友发消息.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

'''

A command line utility for the Pydlock package.

Software:      Pydlock
Author:        Erick Edward Shepherd
E-mail:        Contact@ErickShepherd.com
GitHub:        https://www.github.com/ErickShepherd/pydlock
PyPI:          https://pypi.org/project/pydlock/
Date created:  2020-04-30
Last modified: 2020-04-30

Description:

    A command line utility for the Pydlock package, which allows users to lock
    and unlock files with a password, or run Python scripts locked by Pydlock.

Usage:

    This module may be executed from the command line as a Python script:

        python -m pydlock

    Running the script without any arguments will display the usage:

        usage: pydlock.py [-h] [--arguments ARGUMENTS] [--encoding ENCODING]
                          {lock,unlock,python,run} file

    Supported operations include:

        lock:   Encrypts a file in-place.
        unlock: Decrypts a file in-place.
        python: Decrypts and runs the contents of a Python file.
        run:    Temporarily decrypts, runs, and re-encrypts an arbitrary file.

    Example:

        python -m pydlock lock example.txt --encoding=utf-8

Copyright:

    Pydlock - A Python file encryption tool.

    Copyright (c) 2020 of Erick Edward Shepherd, all rights reserved.

License:

    This file is part of "Pydlock" (the "Software").

    MIT License

    Copyright (c) 2020 Erick Edward Shepherd

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the right to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.

Notes:

    Issues with use on Windows executables:

        Because the files are modified, locking and unlocking executables on
        Windows does not preserve their checksum. Consequently, after locking
        and unlocking an executable on Windows, when an execution is attempted,
        the system raises an error for security purposes:

            "This version of <file> is not compatible with the version of
            Windows you're running. Check your computer's system information
            and then contact the software publisher."

        There does not appear to be a simple resolution for this issue, and the
        files effectively become corrupted.

'''

# Standard library imports.
import os
from argparse import ArgumentParser

# Local application imports.
import pydlock
from pydlock.constants import DEFAULT_ENCODING

# Dunder definitions.
__author__  = pydlock.__author__
__version__ = pydlock.__version__

if __name__ == "__main__":

    # Maps function names to the respective function.
    function_map = {
        "lock"    : pydlock.lock,
        "unlock"  : pydlock.unlock,
        "python"  : pydlock.python,
        "run"     : pydlock.run
    }

    # Parses command-line arguments from the user.
    parser = ArgumentParser()
    parser.add_argument("operation",   choices = function_map.keys())
    parser.add_argument("file",        type = os.path.abspath)
    parser.add_argument("--arguments", type = str, default = "")
    parser.add_argument("--encoding",  type = str, default = DEFAULT_ENCODING)
    kwargv = vars(parser.parse_args())

    # Aliases parsed command-line arguments for brevity.
    task      = function_map[kwargv["operation"]]
    path      = kwargv["file"]
    arguments = kwargv["arguments"]
    encoding  = kwargv["encoding"]

    # Performs the indicated task.
    task(path, arguments, encoding)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 31 12:41:46 2018

@author: owen
"""

# https://leetcode.com/problems/network-delay-time/solution/
#import collections
#class Solution:
#    def networkDelayTime(self, times, N, K):
#        """
#        :type times: List[List[int]]
#        :type N: int
#        :type K: int
#        :rtype: int
#        """
#        graph=collections.defaultdict(list)
#        for u,v,w in times:
#            graph[u].append([w,v])
#
#        dist={nd:float('inf') for nd in range(1,N+1)}
#
#        def dfs(node,elapsed):
#            if elapsed>=dist[node]: # some signal arrived earlier, do not need broadcast this node
#                return
#            dist[node]=elapsed
#            for time, nei in sorted(graph[node]): # signal the faster neighbors first
#                dfs(nei, elapsed + time)
#
#        dfs(K,0)
#        res=max(dist.values())
#        return  res if res<float('inf') else -1

# http://www.cnblogs.com/grandyang/p/8278115.html
#import collections
#class Solution:
#    def networkDelayTime(self, times, N, K):
#        """
#        :type times: List[List[int]]
#        :type N: int
#        :type K: int
#        :rtype: int
#        """
#        # Dijkstra finds the shortest path from source to all targets. BFS implement. time O(N^2)
#        graph=[[-1]*(N+1) for _ in range(N+1)]
#        for u,v,w in times:
#            graph[u][v]=w
#
#        dist=[float('inf') for _ in range(N+1)]
#        dist[K]=0
#        dq=collections.deque([K])
#
#        while dq:
#            visited=set()
#            dq_len=len(dq)
#            for i in range(dq_len):
#                u=dq.popleft()
#                for v in range(1,N+1):
#                    if graph[u][v]!=-1 and dist[u]+graph[u][v]<dist[v]:
#                        dist[v]=dist[u]+graph[u][v]
#                        if v not in visited:
#                            visited.add(v)
#                            dq.append(v)
#
#        res=max(dist[1:]) # dist[0] is alway inf
#        return  res if res<float('inf') else -1

import heapq
import collections
class Solution:
    def networkDelayTime(self, times, N, K):
        """
        :type times: List[List[int]]
        :type N: int
        :type K: int
        :rtype: int
        """
        graph=collections.defaultdict(list)
        for u,v,w in times:
            graph[u].append((v,w))

        h=[(0,K)] # put distance first
        heapq.heapify(h)
        dist={}
        while h:
            d,node=heapq.heappop(h) # the poped d must be the smallest distance arrived node, cc we use heap here
            if node in dist:
                continue
            dist[node]=d
            for nei,nd in graph[node]:
                if nei not in dist:
                    heapq.heappush(h,(d+nd,nei))

        return max(dist.values()) if len(dist)==N else -1

if __name__=="__main__":
    print(Solution().networkDelayTime([[1,2,10],[2,4,11],[1,3,12],[3,4,8]],4,1))


from django.shortcuts import render
from .models import Credit, Bank
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.http import HttpResponse
# Create your views here.
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views import generic

def homepage(request):
    num_clients = Credit.objects.all().count()
    return render(
        request,
        'homepage.html',
        context={'num_clients': num_clients}
    )

@login_required
def my_view(request):
    return render(request, 'profile.html', {"limit": request.user.clientinformation.salary_month*12*0.5})

@login_required
def take_credit(request):
    salary = request.user.clientinformation.salary_month*12*0.5
    bank = Bank.objects.get(procent=30)
    if (request.user.credit.status==True):
        return HttpResponse("
Firstly pay for current credit!
", status=404)
    if 570000*0.1 > bank.budjet:
        return HttpResponse("
Wrong operation. Lending is temporarily unavailable!
", status=404)
    else:
        if request.method == 'POST':
            data = request.POST.copy()
            if data.get("YES") == None:
                return render(request, 'credit_operation/decline.html')
            if int(data.get("size")) > bank.budjet * 0.5 or int(data.get("size")) < bank.budjet * 0.01 or int(data.get("size")) > salary:
                return HttpResponse('
Wrong operation. You put wrong credit size
', status=403)
            bank.budjet = bank.budjet - int(data.get("size"))
            request.user.credit.status = True
            request.user.credit.size = int(data.get("size"))
            bank.save()
            request.user.credit.save()
            return render(request, 'credit_operation/success.html')
        if salary < bank.budjet * 0.01:
            return HttpResponse("
Wrong operation. Your credit limit is too low!
", status=404)
        if salary < bank.budjet * 0.5:
            context = {
                 "min": str(bank.budjet * 0.01),
                 "max": str(salary)
             }
        else:
            context = {
                "min": str(bank.budjet * 0.01),
                "max": str(bank.budjet * 0.5)
            }
        return render(request, 'credit_operation/take_credit.html', context)

@login_required
def pay_credit(request):
    bank = Bank.objects.get(procent=30)
    if (request.user.credit.status==False):
        return HttpResponse("
Firstly take some credit!
", status=404)
    else:
        if request.method == 'POST':
            data = request.POST.copy()
            if data.get("NO") == None:
               request.user.credit.status = False
               bank.budjet+=request.user.credit.size
               request.user.credit.size = 0
               bank.save()
               request.user.credit.save()
               return render(request, 'credit_operation/success.html')
            else:
                return render(request, 'credit_operation/decline.html')
        return render(request, 'credit_operation/pay_credit.html')


#!/usr/bin/env python
# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
# SPDX-License-Identifier: BSD-2 License
# The full license information can be found in LICENSE.txt
# in the root directory of this project.
"""
App for running iperf server or client.
"""
import logging
import random
import time

import axon.apps.console as console

log = logging.getLogger(__name__)

class Iperf(console.Console):

    IPERF_BIN = 'iperf3'
    _START_RAND_PORT = 5600
    _END_RAND_PORT = 6000
    _STARTING_JOB_ID = 1000

    def __init__(self):
        super(Iperf, self).__init__()
        # {<port>: <popen obj>}
        self._server_handles = {}

        # {<job_id>: {'popen_obj': <popen obj>,
        #             'state': 'running' | 'done',
        #             'result': < None | JSON output>,
        #             'cmd': <command>
        #             }
        self._client_handles = {}
        self._job_id = None
        self._gen_rand_port = self._gen_rand_port(self._START_RAND_PORT,
                                                  self._END_RAND_PORT)

    def start_iperf_server(self, port=None, args=''):
        """
        Start iperf server on given port. If port is not
        provided, random port will be used.

        :param port: (int) TCP port number (default: None)
        :param args: (str) Additional iperf supported args
        :return:
        port: (int) TCP port number being used for iperf server
        """
        if not port:
            port = next(self._gen_rand_port)
        if self.is_running(port):
            log.info("Server is already running on port: %d"
                     "Skip starting server." % port)
            return port
        cmd = self.IPERF_BIN + ' ' + '-s -p %s' % str(port)
        if args:
            cmd += ' ' + args
        p = self._start_subprocess(cmd)
        # Check server is running
        for _ in range(3):
            if self.is_running(port):
                self._server_handles[port] = p
                log.info("Server is running on port:%d " % port)
                break
            time.sleep(1)
        else:
            msg = "unable to start iperf3 server. check port:%d is available." % port
            log.error(msg)
            raise RuntimeError(msg)
        return port

    def _gen_rand_port(self, start_port, end_port):

        for _ in range(start_port, end_port + 1):
            port = random.randint(start_port, end_port)
            if port not in self._server_handles:
                yield port
        else:
            msg = "Running out of random port between starting port: %d - "\
                             "ending port: %d" % (start_port, end_port)
            log.error(msg)
            raise ValueError(msg)

    def get_server_ports(self):
        """
        Get running server ports.
        :return:
        list of currently running server ports
        """
        return [p for p in self._server_handles.keys()]

    def stop_iperf_server(self, port):
        """
        Stop iperf server for given port.
        """
        proc = self._server_handles.get(port)
        if proc:
            self._kill_subprocess(proc)
            self._server_handles.pop(port)

    def stop_iperf_client(self, job_id):
        """
        Stop iperf client for given job_id.
        """
        job = self._client_handles.get(job_id)
        if job:
            self._kill_subprocess(job.get('popen_obj'))
            self._client_handles.pop(job_id)

    def start_iperf_client(self, dst_ip, dst_port, duration=10, udp=False,
                           bandwidth=None, args=''):
        """
        Start iperf client to given dst_ip and port.

        :param dst_ip: (str) IP address of iperf server
        :param dst_port: (int) iperf server port to connect to
        :param dst_port: (int) iperf server port to connect to
        :param duration: (int) test duration
        :param udp: (bool) enable udp
        :param bandwidth: (int)  limit traffic to as much Mbits/second
        :return:
        job_id: (int)
        """

        cmd = self.IPERF_BIN + ' ' + '-c %s -p %s -t %s' % (str(dst_ip),
                                                            str(dst_port),
                                                            str(duration))
        cmd += ' ' + '--json'
        if udp:
            cmd += " --udp"
            # iperf3 udp default is 1 Mbit/s, this sets it to unlimited / 10Gb/s
            if not bandwidth:
                bandwidth = 10240
        if bandwidth:
            cmd += " --bandwidth %dM" % bandwidth
        if args:
            cmd += ' ' + args
        p = self._start_subprocess(cmd)
        self._job_id = self._STARTING_JOB_ID if not self._job_id \
            else self._job_id + 1

        self._client_handles[self._job_id] = {'popen_obj': p,
                                              'state': 'running',
                                              'result': None,
                                              'cmd': cmd}
        log.info("cmd: %s running with job_id: %s" % (cmd, self._job_id))
        return self._job_id

    def get_client_jobs(self):
        return [j for j in self._client_handles.keys()]

    def get_client_job_info(self, job_id):
        """
        Get client job info on given job id.

        :param job_id: (int)
        :return:
        job_details: (dict)
            # {'popen_obj': <popen obj>,
            #  'state': 'running' | 'done',
            #  'result': < None | JSON output>,
            #  'cmd': <command>
            # }
        """
        if job_id not in self._client_handles:
            log.error("Job ID:%d not found" % job_id)
            return None
        p = self._client_handles[job_id].get('popen_obj')
        if not self._is_alive(p):
            if self._client_handles[job_id].get('result') is None:
                result_json = p.communicate()[0].decode('utf-8')
                result_json = result_json.replace('\n', '').replace('\t', '')
                self._client_handles[job_id]['result'] = result_json
            self._client_handles[job_id]['state'] = 'done'

        return self._client_handles[job_id]

    def stop(self):
        """
        Stop iperf app.
        """
        server_ports = self.get_server_ports()
        client_jobs = self.get_client_jobs()

        for port in server_ports:
            log.info("Stopping iperf server on port %s", port)
            proc = self._server_handles.get(port)
            if proc:
                self._kill_subprocess(proc)

        for job_id in client_jobs:
            log.info("Stopping iperf client process for job: %s" % job_id)
            proc = self._client_handles[job_id].get('popen_obj')
            if proc:
                self._kill_subprocess(proc)

    def is_running(self, port):
        """
        Check iperf server is running on given port
        """
        netstat = self._start_subprocess('netstat -lnp')
        grep = self._start_subprocess('grep :%s' % port, stdin=netstat.stdout)
        netstat.stdout.close()  # Allow netstat to receive a SIGPIPE if grep exits.
        output = grep.communicate()[0].decode('utf-8')
        return self.IPERF_BIN in output


import time
import os
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
import pickle
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from keras.utils import to_categorical
from keras.models import (
                          Model,
                          )
from keras.layers import (
                          Activation,
                          Conv2D,
                          Dense,
                          Flatten,
                          Input,
                          MaxPooling2D,
                          Dropout,
                          )
import keras
from keras.models import Sequential

# Required magic to display matplotlib plots in notebooks
#%matplotlib inline

# Set up a figure of an appropriate size
# create empty list
images_dataset = []
class_labels = []
batch_size = 128
epochs = 50

# Helper function to resize image proportionally. size is a tuple (height,width)
def resize_image(img, size):
    from PIL import Image, ImageOps

    # resize the image so the longest dimension matches our target size
    img.thumbnail(size, Image.ANTIALIAS)

    # Create a new square white background image
    newimg = Image.new("RGB", size, (255, 255, 255))

    # Paste the resized image into the center of the square background
    if np.array(img).shape[2] == 4:
        # If the source is in RGBA format, use a mask to eliminate the transparency
        newimg.paste(img, (int((size[0] - img.size[0]) / 2), int((size[1] - img.size[1]) / 2)), mask=img.split()[3])
    else:
        newimg.paste(img, (int((size[0] - img.size[0]) / 2), int((size[1] - img.size[1]) / 2)))

    # return the resized image
    return newimg

# loop through the subfolders in the input directory
image_base_dir = './train'
n = 0
for root, folders, filenames in os.walk(image_base_dir):
    for folder in folders:
        class_labels.append(folder)
        print('processing folder:{}'.format(folder))
        n = n+1
        files = os.listdir(os.path.join(root,folder))
        for file in files:
            # construct the fully image filename(including path)
            imgFile = os.path.join(image_base_dir,folder, file)
            # read image data
            img = Image.open(imgFile)
            # resize image
            img = resize_image(img,(128,128))
            # images_dataset is list of numpy array and label
            images_dataset.append([np.array(np.array(img)),folder])

print('n={}'.format(n))
# shuffle the dataset
np.random.shuffle(images_dataset)

# fetch image data from gear_dataset
images_data = list(map(
                       lambda item: item[0],
                       images_dataset
                       ))

# fetch label data from gear_dataset
images_labels = list(map(
                       lambda item: item[1],
                       images_dataset,
                       ))
# images_data and images_labels are list now, should be converted to np.array
images_data = np.array(images_data,dtype=np.float)/255.
images_labels = np.array(images_labels)

# one hot encoding labels
# encode the labels into one hot encoding
encoder = preprocessing.LabelEncoder()
encoder.fit(images_labels)
print('class={}'.format(encoder.classes_))   # ['cats' 'dogs']
le_classes = encoder.transform(encoder.classes_)  # [0,1]
# zip pack the two object's element into a tuple: [0:'cats,1:'dogs']
# create a dictionary from a list of tuple
le_name_mapping = dict(zip(le_classes,encoder.classes_))
print(le_name_mapping,le_name_mapping[0])
output2 = open('pzsmodel2_labdic.pkl', 'wb')
pickle.dump(le_name_mapping, output2)
output2.close()

# save the digits and its corresponding labels
transfomed_label = encoder.fit_transform(class_labels)
print(transfomed_label)

# string to number. cats->0 dogs->1
labels_id = encoder.transform(images_labels)
# one hot encoding. Converts a class vector (integers) to binary class matrix. A binary matrix representation of the input. The classes axis is placed last.
labels_encoded = to_categorical(labels_id)
# split training and testing dataset
train_x,test_x,train_y,test_y = train_test_split(images_data,labels_encoded,test_size=0.2)
#print('shape={}'.format(np.array(images_dataset).shape))

#
model = Sequential()
# strides=(1,1) padding='valid'
model.add(Conv2D(32, (3, 3), activation='relu',input_shape=(128,128,3)))
print('output shape1={}'.format(model.output_shape))
model.add(Dropout(0.2))
print('output shape2={}'.format(model.output_shape))
# pool_size: integer or tuple of 2 integers, factors by which to downscale (vertical, horizontal).
# strides: Integer, tuple of 2 integers, or None. Strides values. If None, it will default to pool_size.
model.add(MaxPooling2D(pool_size=(3, 3)))
print('output shape3={}'.format(model.output_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
print('output shape4={}'.format(model.output_shape))
model.add(MaxPooling2D(pool_size=(3, 3)))
print('output shape5={}'.format(model.output_shape))
model.add(Dropout(0.2))
model.add(Conv2D(128, (3, 3), activation='relu'))
print('output shape6={}'.format(model.output_shape))
model.add(MaxPooling2D(pool_size=(3, 3)))
print('output shape7={}'.format(model.output_shape))
model.add(Flatten())
print('output shape8={}'.format(model.output_shape))
model.add(Dense(128, activation='relu'))
# Softmax makes the output sum up to 1 so the output can be interpreted as probabilities.
model.add(Dense(n, activation='softmax'))
print('output shape9={}'.format(model.output_shape))
# Configures the model for training.
model.compile(loss=keras.losses.categorical_crossentropy,
              optimizer=keras.optimizers.Adam(),
              metrics=['accuracy'])

# Trains the model for a given number of epochs (iterations on a dataset).
history= model.fit(train_x, train_y,
          batch_size=batch_size,
          epochs=epochs,
          verbose=1,
          validation_data=(test_x, test_y))

# Returns the loss value & metrics values for the model in test mode.
score = model.evaluate(test_x, test_y, verbose=0)
print('score={}'.format(score))
#print('history={}'.format(history.accuracy))

# load predict images
predict_data = []
predict_images = []
size = (128,128)
files = os.listdir('./predict')
for f in files:
    if not 'jpg' in f:
        continue
    img = Image.open(os.path.join('./predict',f))
    # resize images
    img = resize_image(img,size)
    predict_data.append(np.array(img))

#img = Image.open('resized_images/hardshell_jackets_test/resized_10269570x1012905_zm.jpeg')
predict_data = np.array(predict_data,dtype='float')/255.

predicted_labels_encoded = model.predict(
                                 predict_data,
                                 # batch_size=128,
                                 )
# predicted_labels_encoded is an array ,so np.argmax must specify the axis, so the result is also an arrar
predicted_labels = np.argmax(predicted_labels_encoded,axis=1)
print('predicted digits={}'.format(predicted_labels))
predicted_labels = encoder.inverse_transform(predicted_labels)
print('predicted labels=',predicted_labels)

# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

i = 0
#fig = plt.figure(figsize=(10, 10))
for data in predict_data:
    ax = plt.subplot(1,4,i+1)
    #img = Image.open(test_img_url)
    #img = np.array(img)
    data = (data * 255.).astype(np.uint8)
    imgplot = plt.imshow(data)
    ax.set_title(predicted_labels[i])
    i = i+1
plt.show()
# save the model

model.save('catsdogs.h5')


import praw
import configparser
import datetime

DATE_FILE = 'meta_last_posted'

c = configparser.ConfigParser()
c.read('config.ini')

reddit = praw.Reddit(**c['Auth'])
subreddit = reddit.subreddit(c['Options']['subreddit'])

# should only be posted every 4 weeks
with open(DATE_FILE, 'r') as f:
    try:
        last_posted_str = f.read().strip()
        last_posted = datetime.date.fromisoformat(last_posted_str)

        if datetime.date.today() - last_posted < datetime.timedelta(days=28):
            print(f'Not submitted (last post: {last_posted_str})')
            exit()
    except FileNotFoundError:
        print(f"Couldn't find {DATE_FILE}, continuing")

title = datetime.date.today().strftime('Meta Thread - Month of %B %d, %Y')
content = """A monthly thread to talk about meta topics. Keep it friendly and relevant to the subreddit.

Posts here must, of course, still abide by all subreddit rules other than the no meta requirement. Keep it friendly and be respectful. Occasionally the moderators will have specific topics that they want to get feedback on, so be on the lookout for distinguished posts.

Comments that are detrimental to discussion (aka circlejerks/shitposting) are subject to removal."""

post = subreddit.submit(title, selftext=content)
post.disable_inbox_replies()
post.mod.suggested_sort(sort='new')
post.mod.distinguish()
post.mod.sticky()

print(f'Submitted {post.title}')

with open(DATE_FILE, 'w') as f:
    f.write(datetime.date.today().isoformat())
    f.write('\n')


import sys
# Find jVMC package
sys.path.append(sys.path[0] + "/..")

import jax
from jax.config import config
config.update("jax_enable_x64", True)

import jax.numpy as jnp
import jax.random as random

import time
import numpy as np

import jVMC
import jVMC.util.stepper as jVMCstepper
import jVMC.mpi_wrapper as mpi
import jVMC.nets.activation_functions as act_funs
import jVMC.util.symmetries as sym

import collections

def get_iterable(x):
    if isinstance(x, collections.abc.Iterable):
        return x
    else:
        return (x,)

def init_net(descr, dims, seed=0):

    def get_activation_functions(actFuns):

        if type(actFuns) is list:
            return [act_funs.activationFunctions[fn] for fn in actFuns]

        return act_funs.activationFunctions[actFuns]

    netTypes = {
        "RBM": jVMC.nets.RBM,
        "FFN": jVMC.nets.FFN,
        "CNN": jVMC.nets.CNN,
        "LSTM": jVMC.nets.LSTM,
        "LSTMsym": jVMC.nets.LSTMsym,
        "PhaseRNN": jVMC.nets.RNN,
        "PhaseRNNsym": jVMC.nets.RNNsym,
        "CpxRNN": jVMC.nets.CpxRNN,
        "RNN": jVMC.nets.RNN,
        "RNN2D": jVMC.nets.RNN2D,
        "RNNsym": jVMC.nets.RNNsym,
        "RNN2Dsym": jVMC.nets.RNN2Dsym,
        "CpxRBM": jVMC.nets.CpxRBM,
        "CpxCNN": jVMC.nets.CpxCNN
    }

    def get_net(descr, dims, seed):

        return netTypes[descr["type"]](**descr["parameters"])

    if "actFun" in descr["net1"]["parameters"]:
        descr["net1"]["parameters"]["actFun"] = get_activation_functions(descr["net1"]["parameters"]["actFun"])

    if descr["net1"]["type"][-3:] == "sym":
        L = dims[0]

        # set symmetries ON - turn each one off manually
        kwargs_sym = {"translation": True, "reflection": True, "rotation": True}
        for key in kwargs_sym.keys():
            if key in descr["net1"]:
                kwargs_sym[key] = descr["net1"][key]

        if descr["net1"]["type"][-5:-3] == "2D":
            descr["net1"]["parameters"]["orbit"] = sym.get_orbit_2d_square(L, **kwargs_sym)
        else:
            descr["net1"]["parameters"]["orbit"] = sym.get_orbit_1d(L, **kwargs_sym)

    if "net2" in descr:
        if descr["net2"]["type"][-3:] == "sym":

            # set symmetries ON - turn each one off manually
            kwargs_sym = {"translation": True, "reflection": True, "rotation": True}
            for key in kwargs_sym.keys():
                if key in descr["net2"]:
                    kwargs_sym[key] = descr["net2"][key]

            L = dims[0]
            if descr["net2"]["type"][-5:-3] == "2D":
                descr["net2"]["parameters"]["orbit"] = sym.get_orbit_2d_square(L, **kwargs_sym)
            else:
                descr["net2"]["parameters"]["orbit"] = sym.get_orbit_1d(L, **kwargs_sym)

    if not "net2" in descr:

        model = get_net(descr["net1"], dims, seed)

        psi = jVMC.vqs.NQS(model, batchSize=descr["gradient_batch_size"], seed=seed)

    else:

        if "actFun" in descr["net2"]["parameters"]:

            descr["net2"]["parameters"]["actFun"] = get_activation_functions(descr["net2"]["parameters"]["actFun"])

        model1 = get_net(descr["net1"], dims, seed)
        model2 = get_net(descr["net2"], dims, seed)

        psi = jVMC.vqs.NQS((model1, model2), batchSize=descr["gradient_batch_size"], seed=seed)

    psi(jnp.zeros((1, 1) + dims, dtype=np.int32))

    return psi

def measure(observables, psi, sampler, numSamples=None):
    ''' This function measures expectation values of a given set of operators given a pure state.

    Arguments:
        * ``observables``: Dictionary of the form with operator names as keys and (lists of) operators as values, e.g.:

            .. code-block:: python

                { "op_name_1": [operator1, operator2], "op_name_2": operator3 }

        * ``psi``: Variational wave function (instance of ``jVMC.vqs.NQS``)
        * ``sampler``: Instance of ``jVMC.sampler`` used for sampling.
        * ``numSamples``: Number of samples (optional)

    Returns:
        A dictionary holding expectation values, variances, and MC error estimates for each operator. E.g. for the
        exemplary operator input given in `Arguments`:

        .. code-block:: python

            {
                "op_name_1": { "mean": [mean1, mean2],
                               "variance": [var1, var2],
                               "MC_error": [err1, err2] },
                "op_name_2": { "mean": [mean3],
                               "variance": [var3],
                               "MC_error": [err3] }
            }

    '''
    # Get sample
    sampleConfigs, sampleLogPsi, p = sampler.sample(numSamples=numSamples)

    result = {}

    for name, ops in observables.items():

        tmpMeans = []
        tmpVariances = []
        tmpErrors = []

        for op in get_iterable(ops):
            sampleOffdConfigs, matEls = op.get_s_primes(sampleConfigs)
            sampleLogPsiOffd = psi(sampleOffdConfigs)
            Oloc = op.get_O_loc(sampleLogPsi, sampleLogPsiOffd)

            if p is not None:
                tmpMeans.append(mpi.global_mean(Oloc, p))
                tmpVariances.append(mpi.global_variance(Oloc, p))
                tmpErrors.append(0.)
            else:
                tmpMeans.append(mpi.global_mean(Oloc))
                tmpVariances.append(mpi.global_variance(Oloc))
                tmpErrors.append(jnp.sqrt(tmpVariances[-1]) / jnp.sqrt(sampler.get_last_number_of_samples()))

        result[name] = {}
        result[name]["mean"] = jnp.real(jnp.array(tmpMeans))
        result[name]["variance"] = jnp.real(jnp.array(tmpVariances))
        result[name]["MC_error"] = jnp.real(jnp.array(tmpErrors))

    return result

def ground_state_search(psi, ham, tdvpEquation, sampler, numSteps=200, varianceTol=1e-10, stepSize=1e-2, observables=None, outp=None):
    ''' This function performs a ground state search by Stochastic Reconfiguration.

    Arguments:
        * ``psi``: Variational wave function (``jVMC.vqs.NQS``)
        * ``ham``: Hamiltonian operator
        * ``tdvpEquation``: An instance of ``jVMC.util.TDVP``
        * ``numSteps``: Maximal number of steps
        * ``varianceTol``: Stopping criterion
        * ``stepSize``: Update step size (learning rate)
        * ``observables``: Observables to be measured during ground state search
        * ``outp``: ``None`` or instance of ``jVMC.util.OutputManager``.

    '''

    delta = tdvpEquation.diagonalShift

    stepper = jVMCstepper.Euler(timeStep=stepSize)

    n = 0
    if outp is not None:
        if observables is not None:
            obs = measure(observables, psi, sampler)
            outp.write_observables(n, **obs)

    varE = 1.0

    while n < numSteps and varE > varianceTol:

        tic = time.perf_counter()

        dp, _ = stepper.step(0, tdvpEquation, psi.get_parameters(), hamiltonian=ham, psi=psi, numSamples=None, outp=outp)
        psi.set_parameters(dp)
        n += 1

        varE = tdvpEquation.get_energy_variance()

        if outp is not None:
            if observables is not None:
                obs = measure(observables, psi, sampler)
                outp.write_observables(n, **obs)

        delta = 0.95 * delta
        tdvpEquation.set_diagonal_shift(delta)

        if outp is not None:
            outp.print(" STEP %d" % (n))
            outp.print("   Energy mean: %f" % (tdvpEquation.get_energy_mean()))
            outp.print("   Energy variance: %f" % (varE))
            outp.print_timings(indent="   ")
            outp.print("   == Time for step: %fs" % (time.perf_counter() - tic))


import logging

import tensorflow as tf

LOGGER = logging.getLogger('cve-score')

def _parse_encoder(params):
    '''Unpacks encoding parameters into at data specifying `tf.Tensor`.
    Returns a triple of the form (<name>, <shape>, <dtype>).
    '''
    if 'vocabulary' in params:
        return (params['key'], [len(params['vocabulary'])], tf.float32)
    if params.get('isLabel', False):
        return (params['key'], [1], tf.int64)
    if params['key'] == 'weight':
        return ('weight', [1], tf.float32)

    raise ValueError('invalid encoding params %s' % params)

def get_parser(config):
    '''Converts a config object into protobuf parser.

    Arguments:
        config  [list] list of dictionaries, each defining an encoder.

    Returns a function with the signature `<protobuf> => (<dict>, <tensor>)`.
    The input is a serialized `tf.train.Example` object, and the output
    is a pair of features/label tensors.
    '''
    decoders = {
        name: tf.FixedLenFeature(shape, dtype)
        for (name, shape, dtype) in map(_parse_encoder, config)
    }
    LOGGER.debug('decoders => %s', decoders)

    label_keys = [
        item['key'] for item in config if item.get('isLabel', False)
    ]
    if len(label_keys) != 1:
        raise ValueError('missing or multiple labels: %s' % label_keys)

    def parser(serialized):
        example = tf.parse_single_example(serialized, decoders)
        label = example.pop(label_keys[0])
        return (example, label)

    return parser

def get_feature_columns(config):
    '''Converts a config object into list of `tf.feature_column` objects
    for use with `tf.estiimator` module.

    Arguments:
        config  [list] list of dictionaries, each defining an encoder.
    '''
    features = [item for item in config if not item.get('isLabel')]
    return [
        tf.feature_column.numeric_column(key, shape, dtype=dtype)
        for (key, shape, dtype) in map(_parse_encoder, features)
    ]