import datetime
import itertools
import math
import os
import random
import glob
import json
from typing import *

import scipy
import imageio
import scipy.misc
import scipy.spatial
import scipy.ndimage
import numpy as np
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import umap
from torch.utils.data import Dataset, DataLoader, Sampler
from torch.utils.tensorboard import SummaryWriter

from PIL import Image
import skimage.transform
import matplotlib.pyplot as plt

import elpips
from PythonExtras.distance_matrix import render_distance_matrix, DistanceMatrixConfig

import models
from dcgan import Discriminator, Generator

class CatDataset(Dataset):

    def __init__(self, imageSubdirPath: str, transform: Callable):
        self.rootPath = imageSubdirPath
        self.pathList = glob.glob(os.path.join(self.rootPath, 'cat*.jpg'))

        self.transform = transform

    def __getitem__(self, index):
        path = self.pathList[index]
        image = Image.open(path)

        if self.transform is not None:
            image = self.transform(image)

        return image, os.path.basename(path)  # Pass image name as metadata. (Useful for export.)

    def __len__(self):
        return len(self.pathList)

class AuthorDataset(Dataset):

    def __init__(self, jsonPath: str, normalize=True):
        self.jsonPath = jsonPath

        with open(self.jsonPath, 'r') as file:
            self.data = json.load(file)

        self.names = self.data['names']
        self.vectors = np.asarray(self.data['vectors'])
        # Normalize the vectors.
        self.vectors = self.vectors / np.linalg.norm(self.vectors, axis=-1, keepdims=True)

    def __getitem__(self, index):
        return self.vectors[index], self.names[index]

    def __len__(self):
        return len(self.names)

class InfiniteSampler(Sampler):
    """
    From https://discuss.pytorch.org/t/implementing-an-infinite-loop-dataset-dataloader-combo/35567/3
    """
    def __init__(self, data_source: Sized):
        super().__init__(data_source)
        self.dataset_size = len(data_source)

    def __iter__(self):
        # This wrapper is only needed if we want to use a sample size other than one.
        # Otherwise, iteration over the random permutation tensors is already yielding single indices.
        yield from itertools.islice(self._infinite(), 0, None, 1)  # Infinite iterator

    def _infinite(self):
        g = torch.Generator()
        while True:
            yield from torch.randperm(self.dataset_size, generator=g)

def plot_image_scatter(ax, data, images, downscaleRatio: Optional[int] = None):
    from matplotlib.offsetbox import AnnotationBbox, OffsetImage

    if downscaleRatio:
        ratioInv = 1 / downscaleRatio
        images = [scipy.ndimage.interpolation.zoom(i, [ratioInv, ratioInv, 1], order=1) for i in images]

    ax.scatter(data[:, 0], data[:, 1])

    for i in range(len(images)):
        x, y = tuple(data[i])
        ab = AnnotationBbox(OffsetImage(images[i]), (x, y), frameon=False)
        ax.add_artist(ab)

def l2_sqr_dist_matrix(x: torch.Tensor) -> torch.Tensor:
    gram = torch.mm(x, x.T)
    diag = gram.diagonal()

    return diag.unsqueeze(1) + diag.unsqueeze(0) - 2 * gram

def main():

    dataSize = 32
    batchSize = 8
    elpipsBatchSize = 1
    # imageSize = 32
    imageSize = 64
    nz = 100

    # discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth'
    discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth'
    genCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netG_epoch_24.pth'

    gpu = torch.device('cuda')

    # For now we normalize the vectors to have norm 1, but don't make sure
    # that the data has certain mean/std.
    pointDataset = AuthorDataset(
        jsonPath=r'E:\out\scripts\metaphor-vis\authors-all.json'
    )

    # Take top N points.
    points = np.asarray([pointDataset[i][0] for i in range(dataSize)])
    distPointsCpu = l2_sqr_dist_matrix(torch.tensor(points)).numpy()

    latents = torch.tensor(np.random.normal(0.0, 1.0, (dataSize, nz)),
                           requires_grad=True, dtype=torch.float32, device=gpu)

    scale = torch.tensor(2.7, requires_grad=True, dtype=torch.float32, device=gpu)  # todo Re-check!
    bias = torch.tensor(0.0, requires_grad=True, dtype=torch.float32, device=gpu)  # todo Re-check!

    lpips = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True).to(gpu)
    # lossModel = lpips
    config = elpips.Config()
    config.batch_size = elpipsBatchSize  # Ensemble size for ELPIPS.
    config.set_scale_levels_by_image_size(imageSize, imageSize)
    lossModel = elpips.ElpipsMetric(config, lpips).to(gpu)

    discriminator = Discriminator(3, 64, 1)
    if discCheckpointPath:
        discriminator.load_state_dict(torch.load(discCheckpointPath))
    else:
        discriminator.init_params()
    discriminator = discriminator.to(gpu)

    generator = Generator(nz=nz, ngf=64)
    if genCheckpointPath:
        generator.load_state_dict(torch.load(genCheckpointPath))
    else:
        generator.init_params()
    generator = generator.to(gpu)

    # optimizerImages = torch.optim.Adam([images, scale], lr=1e-2, betas=(0.9, 0.999))
    optimizerScale = torch.optim.Adam([scale, bias], lr=0.001)
    # optimizerGen = torch.optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999))
    # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999))
    # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999))
    optimizerLatents = torch.optim.Adam([latents], lr=5e-3, betas=(0.9, 0.999))

    fig, axes = plt.subplots(nrows=2, ncols=batchSize // 2)

    fig2 = plt.figure()
    ax2 = fig2.add_subplot(1, 1, 1)

    outPath = os.path.join('runs', datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S'))
    os.makedirs(outPath)

    summaryWriter = SummaryWriter(outPath)

    for batchIndex in range(10000):

        # noinspection PyTypeChecker
        randomIndices = np.random.randint(0, dataSize, batchSize).tolist()  # type: List[int]
        # # randomIndices = list(range(dataSize))  # type: List[int]
        distTarget = torch.tensor(distPointsCpu[randomIndices, :][:, randomIndices], dtype=torch.float32, device=gpu)
        latentsBatch = latents[randomIndices]

        imageBatchFake = generator(latentsBatch[:, :, None, None].float())

        # todo It's possible to compute this more efficiently, but would require re-implementing lpips.
        # For now, compute the full BSxBS matrix row-by-row to avoid memory issues.
        lossDistTotal = torch.tensor(0.0, device=gpu)
        distanceRows = []
        for iRow in range(batchSize):
            distPredFlat = lossModel(imageBatchFake[iRow].repeat(repeats=(batchSize, 1, 1, 1)).contiguous(),
                                     imageBatchFake, normalize=True)
            distPred = distPredFlat.reshape((1, batchSize))
            distanceRows.append(distPred)
            lossDist = torch.sum((distTarget[iRow] - (distPred * scale + bias)) ** 2)  # MSE
            lossDistTotal += lossDist

        lossDistTotal /= batchSize * batchSize  # Compute the mean.

        distPredFull = torch.cat(distanceRows, dim=0)

        # print('{} - {} || {} - {}'.format(
        #     torch.min(distPred).item(),
        #     torch.max(distPred).item(),
        #     torch.min(distTarget).item(),
        #     torch.max(distTarget).item()
        # ))

        # discPred = discriminator(imageBatchFake)
        # lossRealness = bceLoss(discPred, torch.ones(imageBatchFake.shape[0], device=gpu))
        # lossGen = lossDist + 1.0 * lossRealness
        lossLatents = lossDistTotal

        # optimizerGen.zero_grad()
        # optimizerScale.zero_grad()
        # lossGen.backward()
        # optimizerGen.step()
        # optimizerScale.step()

        optimizerLatents.zero_grad()
        # optimizerScale.zero_grad()
        lossLatents.backward()
        optimizerLatents.step()
        # optimizerScale.step()

        # with torch.no_grad():
        #     # todo  We're clamping all the images every batch, can we clamp only the ones updated?
        #     # images = torch.clamp(images, 0, 1)  # For some reason this was making the training worse.
        #     images.data = torch.clamp(images.data, 0, 1)

        if batchIndex % 100 == 0:
            msg = 'iter {} loss dist {:.3f} scale: {:.3f} bias: {:.3f}'.format(batchIndex, lossDistTotal.item(), scale.item(), bias.item())
            print(msg)

            summaryWriter.add_scalar('loss-dist', lossDistTotal.item(), global_step=batchIndex)

            def gpu_images_to_numpy(images):
                imagesNumpy = images.cpu().data.numpy().transpose(0, 2, 3, 1)
                imagesNumpy = (imagesNumpy + 1) / 2

                return imagesNumpy

            # print(discPred.tolist())
            imageBatchFakeCpu = gpu_images_to_numpy(imageBatchFake)
            # imageBatchRealCpu = gpu_images_to_numpy(imageBatchReal)
            for iCol, ax in enumerate(axes.flatten()[:batchSize]):
                ax.imshow(imageBatchFakeCpu[iCol])
            fig.suptitle(msg)

            with torch.no_grad():
                images = gpu_images_to_numpy(generator(latents[..., None, None]))

                authorVectorsProj = umap.UMAP(n_neighbors=min(5, dataSize), random_state=1337).fit_transform(points)
                plot_image_scatter(ax2, authorVectorsProj, images, downscaleRatio=2)

            fig.savefig(os.path.join(outPath, f'batch_{batchIndex}.png'))
            fig2.savefig(os.path.join(outPath, f'scatter_{batchIndex}.png'))
            plt.close(fig)
            plt.close(fig2)

            with torch.no_grad():
                imagesGpu = generator(latents[..., None, None])
                imageNumber = imagesGpu.shape[0]

                # Compute LPIPS distances, batch to avoid memory issues.
                bs = min(imageNumber, 8)
                assert imageNumber % bs == 0
                distPredEval = np.zeros((imagesGpu.shape[0], imagesGpu.shape[0]))
                for iCol in range(imageNumber // bs):
                    startA, endA = iCol * bs, (iCol + 1) * bs
                    imagesA = imagesGpu[startA:endA]
                    for j in range(imageNumber // bs):
                        startB, endB = j * bs, (j + 1) * bs
                        imagesB = imagesGpu[startB:endB]

                        distBatchEval = lossModel(imagesA.repeat(repeats=(bs, 1, 1, 1)).contiguous(),
                                                  imagesB.repeat_interleave(repeats=bs, dim=0).contiguous(),
                                                  normalize=True).cpu().numpy()

                        distPredEval[startA:endA, startB:endB] = distBatchEval.reshape((bs, bs))

                distPredEval = (distPredEval * scale.item() + bias.item())

                # Move to the CPU and append an alpha channel for rendering.
                images = gpu_images_to_numpy(imagesGpu)
                images = [np.concatenate([im, np.ones(im.shape[:-1] + (1,))], axis=-1) for im in images]

                distPoints = distPointsCpu
                assert np.abs(distPoints - distPoints.T).max() < 1e-5
                distPoints = np.minimum(distPoints, distPoints.T)  # Remove rounding errors, guarantee symmetry.
                config = DistanceMatrixConfig()
                config.dataRange = (0., 4.)
                _, pointIndicesSorted = render_distance_matrix(
                    os.path.join(outPath, f'dist_point_{batchIndex}.png'),
                    distPoints,
                    images,
                    config=config
                )

                # print(np.abs(distPredFlat - distPredFlat.T).max())
                # assert np.abs(distPredFlat - distPredFlat.T).max() < 1e-5
                # todo The symmetry doesn't hold for E-LPIPS, since it's stochastic.
                distPredEval = np.minimum(distPredEval, distPredEval.T)  # Remove rounding errors, guarantee symmetry.
                config = DistanceMatrixConfig()
                config.dataRange = (0., 4.)
                render_distance_matrix(
                    os.path.join(outPath, f'dist_images_{batchIndex}.png'),
                    distPredEval,
                    images,
                    config=config
                )

                config = DistanceMatrixConfig()
                config.dataRange = (0., 4.)
                render_distance_matrix(
                    os.path.join(outPath, f'dist_images_aligned_{batchIndex}.png'),
                    distPredEval,
                    images,
                    predefinedOrder=pointIndicesSorted,
                    config=config
                )

                fig, axes = plt.subplots(ncols=2)
                axes[0].matshow(distTarget.cpu().numpy(), vmin=0, vmax=4)
                axes[1].matshow(distPredFull.cpu().numpy() * scale.item(), vmin=0, vmax=4)
                fig.savefig(os.path.join(outPath, f'batch_dist_{batchIndex}.png'))
                plt.close(fig)

                surveySize = 30
                fig, axes = plt.subplots(nrows=3, ncols=surveySize, figsize=(surveySize, 3))
                assert len(images) == dataSize
                allIndices = list(range(dataSize))
                with open(os.path.join(outPath, f'survey_{batchIndex}.txt'), 'w') as file:
                    for iCol in range(surveySize):
                        randomIndices = random.sample(allIndices, k=3)
                        leftToMid = distPointsCpu[randomIndices[0], randomIndices[1]]
                        rightToMid = distPointsCpu[randomIndices[2], randomIndices[1]]

                        correctAnswer = 'left' if leftToMid < rightToMid else 'right'
                        file.write("{}\t{}\t{}\t{}\t{}\n".format(iCol, correctAnswer, leftToMid, rightToMid,
                                                                 str(tuple(randomIndices))))

                        for iRow in (0, 1, 2):
                            axes[iRow][iCol].imshow(images[randomIndices[iRow]])

                fig.savefig(os.path.join(outPath, f'survey_{batchIndex}.png'))
                plt.close(fig)

            torch.save(generator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex)))
            torch.save(discriminator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex)))

    summaryWriter.close()

if __name__ == '__main__':
    main()


"""
Synthetic and Endogenous RSS Dwell Time Distributions
--------------------------------------------------------------------------------
Authors: Soichi Hirokawa and Griffin Chure
Last Modified: September 25, 2019
License: MIT

Description
--------------------------------------------------------------------------------
This script generates a figure of representative dwell time distributions for
a variety of synthetic and endogenous RSS sequences, along with the credible
regions from fitting of a single exponential to the distribution.

Notes
--------------------------------------------------------------------------------
This script is designed to be run from the `code/figures` directory and uses a relative path to the `data` folder to access the appropriate CSV files.
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import scipy.stats
import vdj.io
import vdj.viz
vdj.viz.plotting_style()

#%%
# load the dwell time data
data = pd.read_csv('../../data/compiled_dwell_times.csv', comment='#')
data = data[(data['salt']=='Mg') & (data['hmgb1']==80)]

# Load the fitting statistics.
samps = pd.read_csv('../../data/exponential_fitting_samples.csv',
                    comment='#')
samps = samps[samps['salt']=='Mg']
stats = pd.read_csv('../../data/exponential_fitting_summary.csv',
                    comment='#')
stats = stats[stats['salt']=='Mg']

# select only the mutants of interest
endogenous = ['WT12rss', 'V19-93', 'V5-43']
heptamer = ['12HeptC3T', '12HeptA4T', '12HeptG7A']
spacer = ['12SpacC4T', '12SpacG11T', '12SpacA12C']
nonamer = ['12NonA1G', '12NonA3C', '12NonC8G']

rows = [endogenous, heptamer, spacer, nonamer]
colors = ['gray', 'tomato', 'dodgerblue', 'rebeccapurple']
DEADFILTER = 21 / 60 # Deadfilter of 21 seconds for shortest PC
time = np.linspace(0, 60, 500) -  DEADFILTER

#%%
# Set up the figure canvas
fig, ax = plt.subplots(4, 3, figsize=(4.5, 5), sharex=True, sharey=True)

# Format the axes
for a in ax.ravel():
    a.xaxis.set_tick_params(labelsize=8)
    a.yaxis.set_tick_params(labelsize=8)
    a.set_xscale('log')
    a.set_ylim([0, 1.1])
    a.set_xlim([DEADFILTER, 80])

# Add axis labels
for i in range(3):
    ax[-1,i].set_xlabel('dwell time [min]', fontsize=8)
for i in range(4):
    ax[i, 0].set_ylabel('cumulative\ndistribution', fontsize=8)

# Add row labels.
fig.text(-0.05, 0.85, 'Endogenous', rotation='vertical', color='slategrey',
         fontsize=9)
fig.text(-0.05, 0.62, 'Heptamer', rotation='vertical', color='tomato',
         fontsize=9)
fig.text(-0.05, 0.4, 'Spacer', rotation='vertical', color='dodgerblue',
         fontsize=9)
fig.text(-0.05, 0.22, 'Nonamer', rotation='vertical', color='rebeccapurple',
         fontsize=9)
fig.text(-0.12, 0.87, '(A)', fontsize=9)
fig.text(-0.12, 0.67, '(B)', fontsize=9)
fig.text(-0.12, 0.47, '(C)', fontsize=9)
fig.text(-0.12, 0.27, '(D)', fontsize=9)

# Plot the observed dwell times.
for i, row in enumerate(rows):
    for j, mut in enumerate(row):
        # Isolate the mutant
        mut_data = data[data['mutant']==mut]
        mut_stats = stats[stats['mutant']==mut]
        low = mut_stats[mut_stats['parameter']=='tau']['hpd_min'].values[0]
        high = mut_stats[mut_stats['parameter']=='tau']['hpd_max'].values[0]

        # Compute the empirical CDF
        x = list(np.sort(mut_data['dwell_time_min'] - DEADFILTER))
        y = list(np.arange(0, len(x), 1) / len(x))
        x.append(x[-1])
        y.append(1)
        # Plot the steps.
        ax[i, j].step(x, y, color='k', lw=1)

        # Compute the fit
        low= 1 - np.exp(-(time - DEADFILTER)/low)
        high = 1 - np.exp(-(time - DEADFILTER)/high)
        ax[i, j].fill_between(time, low, high, color=colors[i], alpha=0.5)

        # Add the titles
        if mut == 'WT12rss':
            title = 'V4-57-1'
        if 'Hept' in mut:
            title = f"Heptamer {mut.split('Hept')[-1]}"
        elif 'Spac' in mut:
            title = f'Spacer {mut.split("Spac")[-1]}'
        elif 'Non' in mut:
            title = f'Nonamer {mut.split("Non")[-1]}'
        else:
            title = mut
        ax[i, j].set_title(title, fontsize=8, y=0.95)

ax[0,0].set_title('V4-57-1', fontsize=8, y=0.95)
plt.savefig('../../figures/Fig_nonexponential_distributions.pdf',
            facecolor='white', bbox_inches='tight')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('setor', '0001_initial'),
        ('enderecos', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='InfoAbst',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nome', models.CharField(max_length=100)),
                ('telefone', models.PositiveIntegerField()),
                ('cpf', models.PositiveIntegerField()),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Funcionario',
            fields=[
                ('infoabst_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='funcionario.InfoAbst')),
                ('matricula', models.PositiveIntegerField()),
                ('salario', models.DecimalField(default=0, max_digits=7, decimal_places=2)),
                ('funcao', models.CharField(max_length=20)),
                ('endereco', models.ManyToManyField(to='enderecos.Endereco')),
                ('setor', models.ForeignKey(to='setor.Setor')),
            ],
            options={
            },
            bases=('funcionario.infoabst',),
        ),
    ]


import pandas as pd
import numpy as np
import time
import datetime as dt
import os
from tqdm import tqdm
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from sklearn.cluster import DBSCAN
from dtw import dtw
from sklearn.metrics import pairwise_distances
PRE_NAME = "onemin_ohlc_"
BEGIN_TIME = "09:00:00"
END_TIME = "11:00:00"
def load_data():
    X = []
    Y = []
    # df = pd.read_csv(os.path.join('dataset', '2327', PRE_NAME+"20180612.csv"))
    # mask = (df.loc[:, "time"] >= BEGIN_TIME) & (df.loc[:, "time"] <= END_TIME)
    # front_df = df[mask].loc[:, "return"]
    # end_df = df[~mask].loc[:, "return"]
    # X.append(np.array(front_df))
    # Y.append(np.array(end_df))
    # """我是分隔線^^~"""
    # df = pd.read_csv(os.path.join('dataset', '2327', PRE_NAME+"20180613.csv"))
    # mask = (df.loc[:, "time"] >= BEGIN_TIME) & (df.loc[:, "time"] <= END_TIME)
    # front_df = df[mask].loc[:, "return"]
    # end_df = df[~mask].loc[:, "return"]
    # X.append(np.array(front_df))
    # Y.append(np.array(end_df))

    sid = '2327'
    for file in os.listdir(os.path.join('dataset', sid)):
        # print(file)
        df = pd.read_csv(os.path.join('dataset', sid, file))
        mask = (df.loc[:, "time"] >= BEGIN_TIME) & (df.loc[:, "time"] <= END_TIME)
        front_df = df[mask].loc[:, "return"]
        # print(front_df)
        # exit()
        end_df = df[~mask].loc[:, "return"]
        if len(front_df) == 121:
        #     print(df.to_string())
        #     exit()
        # print(np.array(front_df).shape)
            X.append(np.array(front_df))
            Y.append(np.array(end_df))
    # print(end_df)
    # df = pd.read_csv(os.path.join('dataset', '2327', PRE_NAME+"20180612.csv"))
    # print(len(X))
    # print(len(X[0]))
    # X = np.array(X)
    # print(X.shape)

    # exit()
    return np.array(X), np.array(Y)

def dtw_d(X, Y):
    manhattan_distance = lambda x, y: np.abs(x - y)
    d, cost_matrix, acc_cost_matrix, path = dtw(X, Y, dist=manhattan_distance)
    return d

def dtw_affinity(X):
    return pairwise_distances(X, metric=dtw_d)

if __name__ == "__main__":
    X, Y = load_data()
    # print(X.shape)
    # print(X.shape)
    # print(np.array)
    # print(dtw_d(X[0], X[1]))
    # exit()
    # print(d)
    db = DBSCAN(metric=dtw_d, n_jobs=5)
    X_label = db.fit_predict(X)
    print(X_label)
    print(X.shape)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
  doitlive
  ~~~~~~~~

  :copyright: (c) 2014 by Steven Loria.
  :license: MIT, see LICENSE for more details.
"""

from __future__ import unicode_literals
import os
import sys
import re
import getpass
import subprocess
from tempfile import NamedTemporaryFile

import click
from click import echo, style, getchar

__version__ = '0.2.0'
__author__ = 'Steven Loria'
__license__ = 'MIT'

env = os.environ
PY2 = int(sys.version[0]) == 2
if PY2:
    basestring = basestring
else:
    basestring = (str, bytes)

HERE = os.path.abspath(os.path.dirname(__file__))
ESC = u'\x1b'
RETURNS = {'\r', '\n'}
OPTION_RE = re.compile(r'^#\s?doitlive\s+'
            '(?P<option>prompt|shell|alias|env|speed):'
            '\s*(?P<arg>.+)$')
TESTING = False

class PromptState(object):
    user = ''
    cwd = ''
    display_dir = ''

    def update(self):
        self.user = style(getpass.getuser(), fg='cyan', bold=True)
        self.cwd = os.getcwd()
        display_cwd_raw = '~' if self.cwd == env['HOME'] else os.path.split(self.cwd)[-1]
        self.display_cwd = style(display_cwd_raw, fg='green')

_prompt_state = PromptState()

def echof(text, *args, **kwargs):
    echo(style(text, *args, **kwargs))

def get_default_prompt():
    if env.get('DOITLIVE_PROMPT'):
        return env['DOITLIVE_PROMPT']
    _prompt_state.update()
    return '{user}@{cwd}: $'.format(
        user=_prompt_state.user,
        cwd=_prompt_state.display_cwd
    )

def ensure_utf(string):
    return string.encode('utf-8') if PY2 else string

def run_command(cmd, shell=None, aliases=None, envvars=None, test_mode=True):
    shell = shell or env.get('DOITLIVE_INTERPRETER') or '/bin/bash'
    if cmd.startswith("cd "):
        directory = cmd.split()[1]
        os.chdir(os.path.expanduser(directory))
        _prompt_state.update()
    else:
        # Need to make a temporary command file so that $ENV are used correctly
        # and that shell built-ins, e.g. "source" work
        with NamedTemporaryFile('w') as fp:
            fp.write('#!{0}\n'.format(shell))
            fp.write('# -*- coding: utf-8 -*-\n')
            # Make aliases work in bash:
            if 'bash' in shell:
                fp.write("shopt -s expand_aliases\n")

            if envvars:
                # Write envvars
                for envvar in envvars:
                    envvar_line = 'export {0}\n'.format(envvar)
                    fp.write(ensure_utf(envvar_line))

            if aliases:
                # Write aliases
                for alias in aliases:
                    alias_line = 'alias {0}\n'.format(alias)
                    fp.write(ensure_utf(alias_line))
            cmd_line = cmd + '\n'
            fp.write(ensure_utf(cmd_line))
            fp.flush()
            if test_mode:
                return subprocess.check_output([shell, fp.name])
            else:
                return subprocess.call([shell, fp.name])

def wait_for(chars):
    while True:
        in_char = getchar()
        if in_char == ESC:
            echo()
            raise click.Abort()
        if in_char in chars:
            echo()
            return in_char

def magictype(text, shell, prompt_func=get_default_prompt, aliases=None,
        envvars=None, speed=1, test_mode=False):
    prompt = prompt_func()
    echo(prompt + ' ', nl=False)
    i = 0
    while i < len(text):
        char = text[i:i + speed]
        in_char = getchar()
        if in_char == ESC:
            echo()
            raise click.Abort()
        echo(char, nl=False)
        i += speed
    wait_for(RETURNS)
    output = run_command(text, shell,aliases=aliases, envvars=envvars,
        test_mode=test_mode)
    if isinstance(output, basestring):
        echo(output)

def format_prompt(prompt):
    _prompt_state.update()
    return prompt.format(
        user=_prompt_state.user,
        cwd=_prompt_state.display_cwd,
        full_cwd=_prompt_state.cwd
    )

def make_prompt_formatter(template):
    return lambda: format_prompt(template)

def run(commands, shell='/bin/bash', prompt_func=get_default_prompt, speed=1,
        test_mode=False):
    echof("We'll do it live!", fg='red', bold=True)
    echof('STARTING SESSION: Press ESC at any time to exit.', fg='yellow', bold=True)

    click.pause()
    click.clear()
    aliases, envvars = [], []
    for line in commands:
        command = line.strip()
        if not command:
            continue
        if command.startswith('#'):
            # Parse comment magic
            match = OPTION_RE.match(command)
            if match:
                option, arg = match.group('option'), match.group('arg')
                if option == 'prompt':
                    prompt_func = make_prompt_formatter(arg)
                elif option == 'shell':
                    shell = arg
                elif option == 'alias':
                    aliases.append(arg)
                elif option == 'env':
                    envvars.append(arg)
                elif option == 'speed':
                    speed = int(arg)
            continue
        magictype(command, shell, prompt_func=prompt_func, aliases=aliases,
            envvars=envvars, speed=speed, test_mode=test_mode)
    prompt = prompt_func()
    echo(prompt + ' ', nl=False)
    wait_for(RETURNS)
    echof("FINISHED SESSION", fg='yellow', bold=True)

@click.option('--speed', '-S', metavar='<int>', default=1, help='Typing speed.')
@click.option('--shell', '-s', metavar='<shell>',
    default='/bin/bash', help='The shell to use.')
@click.argument('session_file', type=click.File('r', encoding='utf-8'))
@click.version_option(__version__, '--version', '-v')
@click.command(context_settings={'help_option_names': ('-h', '--help')})
def cli(session_file, shell, speed):
    """The doitlive CLI.

    \b
    How to use:
        1. Create a file called session.sh. Fill it with bash commands.
        2. Run "doitlive session.sh"
        3. Type like a madman.

    Press ESC or ^C at any time to exit the session.
    To see a demo session, run "doitlive-demo".
    """
    run(session_file.readlines(),
        shell=shell,
        speed=speed,
        test_mode=TESTING)

DEMO = [
    'echo "Greetings"',
    'echo "This is just a demo session"',
    'echo "For more info, check out the home page..."',
    'echo "http://doitlive.rtfd.org"'
]

@click.option('--speed', '-S', default=1, help='Typing speed.')
@click.option('--shell', '-i', default='/bin/bash')
@click.command()
def demo(shell, speed):
    """Run a demo doitlive session."""
    run(DEMO, shell=shell, speed=speed, test_mode=TESTING)

if __name__ == '__main__':
    cli()


import numpy as np
import matplotlib.pyplot as plt

class Image:
    """Class describing an n-dimensional object and its Fourier transform.

    For initialization, you may only provide one of the 'density', 'density_fct',
    'ft', 'ft_fct' arguments.

    Attributes:
        density : array_like (ndarray)
            d-D array representing density of an object.
        density_fct : function
            takes a d-dimensional point, evaluates density at this point.
        ft : array_like (ndarray)
            d-D array representing density Fourier transform.
        ft_fct : function
            takes a d-dimensional point, evaluates density Fourier transform.
        x_axes : list of ndarrays
            list of 1-D arrays, list contains d elements, representing
            coordinate axis in the object space.
        k_axes : list of ndarrays
            list of 1-D arrays, contains d elements, representing
            coordinate axis in the Fourier space.

    Examples:
    >>> im = Image();
    >>> im = Image(density_fct=sin, x_axes=np.linspace(0.,4*pi,200));
    >>> im = Image(ft_fct=abs, k_axes=np.linspace(-5.,5.,100));
    """
    def __init__(self,
                 density=None, density_fct=None,
                 ft=None, ft_fct=None,
                 x_axes=None, k_axes=None):

        self.set_default(x_axes=x_axes, k_axes=k_axes)

        # Check input args
        if [density, density_fct, ft, ft_fct].count(None) != 3:
            raise InputError('You need to provide one (and only one) of the following '
                             + "instantiation arguments: 'density', 'density_fct', "
                             + "'ft', 'ft_fct'. Provided, respectively ('None' is chosen "
                             + "by default): " + str(density) + ", " + str(density_fct) + ", "
                             + str(ft) + ", " + str(ft_fct) + ".")

        # Initialization from array
        if density != None:     # Density in the object space
            self.density = density
            self.ft = np.fft.fftn(self.density) # TODO: optimize to rfftn

        # Initialization from density function
        if density_fct!=None:
            # Fast evaluation of the density. For explanation, see, e.g.,
            # http://stackoverflow.com/questions/22774726/numpy-evaluate-function-on-a-grid-of-points
            if (self.d == 1):
                self.density = np.asarray(density_fct(self.x_axes[0]))
            elif (self.d == 2):
                self.density = np.asarray(density_fct(self.x_axes[0][:,None], self.x_axes[1][None,:]))
            elif (self.d == 3):
                self.density = np.asarray(density_fct(self.x_axes[0][:,None,None],
                                                      self.x_axes[1][None,:,None],
                                                      self.x_axes[2][None,None,:]))
            else:
                raise Error("Initialization from function is only realized for dimensions" +
                            " d=1,2,3. Input dimension: self.d=" + str(self.d) + ".");
            self.ft = np.fft.fftn(self.density)

        # Initialization from array
        if ft != None:     # Density in the object space
            self.ft = ft
            self.density = np.fft.ifftn(self.density)

        # Initialization from density function
        if ft_fct!=None:
            if (self.d == 1):
                self.ft = np.asarray(ft_fct(self.k_axes[0]))
            elif (self.d == 2):
                self.ft = np.asarray(ft_fct(self.k_axes[0][:,None], self.k_axes[1][None,:]))
            elif (self.d == 3):
                self.ft = np.asarray(ft_fct(self.k_axes[0][:,None,None],
                                                      self.k_axes[1][None,:,None],
                                                      self.k_axes[2][None,None,:]))
            else:
                raise Error("Initialization from function is only realized for dimensions" +
                            " d=1,2,3. Input dimension: self.d=" + str(self.d) + ".");
            self.density = np.fft.ifftn(self.ft)

    def set_default(self, d=2, N=(200,200),
                    x_min=(0.0,0.0), x_max=(2.*np.pi,2.*np.pi), x_axes=None, k_axes=None):
        """Creates a default axes setting for an image.

        Creates either a box set by default values or a box defined by my_axes.

        d : integer
            Dimension of the image.
        N : (N1, N2,..., Nd)
            Vector of integers. Describes number of points on the grid
            in the corresponding dimension.
        x_min : (x_min1, x_min2,..., x_mind)
            Vector of floats. Describes minimal coordinate value on the
            grid in the corresponding dimension.
        x_max : (x_max1, x_max2,..., x_maxd)
            Vector of floats. Describes maximal coordinate value on the
            grid in the corresponding dimension.
        x_axes : list of ndarrays
            list of 1-D arrays, contains d elements, representing
            coordinate axis in the object space.
        k_axes : list of ndarrays
            list of 1-D arrays, contains d elements, representing
            coordinate axis in the Fourier space.
        """
        if (x_axes==None and k_axes==None):
            self.d = d
            self.N = N
            self.x_min = x_min
            self.x_max = x_max
            self.k_min = [0 for i in range(self.d)]
            self.k_max = [self.N[i] - 1 for i in range(self.d)]
            # Create linear grid from x_min to x_max
            self.x_axes = [np.linspace(self.x_min[i], self.x_max[i], self.N[i]) for i in range(self.d)]
            # Create k-grid from 0 to N
            self.k_axes = [np.linspace(self.k_min[i], self.k_max[i], self.N[i]) for i in range(self.d)]

        elif (x_axes!=None and k_axes==None):
            self.d = len(x_axes)
            self.N = [len(x_axes[i]) for i in range(self.d)]
            self.x_min = [x_axes[i].min() for i in range(self.d)]
            self.x_max = [x_axes[i].max() for i in range(self.d)]
            self.k_min = [0 for i in range(self.d)]
            self.k_max = [self.N[i] - 1 for i in range(self.d)]
            self.x_axes = x_axes
            self.k_axes = [np.linspace(self.k_min[i], self.k_max[i], self.N[i]) for i in range(self.d)]

        elif (x_axes==None and k_axes!=None):
            self.d = len(k_axes)
            self.N = len(k_axes[0])
            self.x_min = [x_min[0] for i in range(self.d)]
            self.x_max = [x_max[0] for i in range(self.d)]
            self.k_min = [k_axes[i].min() for i in range(self.d)]
            self.k_max = [k_axes[i].max() for i in range(self.d)]
            self.x_axes = [np.linspace(self.x_min[i], self.x_max[i], self.N[i]) for i in range(self.d)]
            self.k_axes = k_axes

        elif (x_axes!=None and k_axes!=None):
            if len(k_axes) != len(x_axes):
                raise IOError("Argument size mismatch: grid in x-space ('x_axes') is "
                              + str(len(x_axes)) + "-dimensional, grid in k-space ('k_axes') is "
                              + str(len(k_axes)) + "-dimensional.")
            if len(k_axes[0]) != len(x_axes[0]):
                raise IOError("Argument size mismatch: grid in x-space ('x_axes') has "
                              + str(len(x_axes[0])) + " points in each dimension,"
                              + "grid in k-space ('k_axes') has "
                              + str(len(k_axes)) + " points in each dimension.")
            self.d = len(k_axes)
            self.N = [len(k_axes[i])  for i in range(self.d)]
            self.x_min = [x_axes[i].min() for i in range(self.d)]
            self.x_max = [x_axes[i].max() for i in range(self.d)]
            self.k_min = [k_axes[i].min() for i in range(self.d)]
            self.k_max = [k_axes[i].max() for i in range(self.d)]
            self.x_axes = x_axes
            self.k_axes = k_axes

        else:
            raise Error("Something went wrong during the initialization. Debug 'set_default()' in the class Image.")

    def plot(self):

        ft_ = np.fft.fftshift(self.ft)
        #k_axes_ = np.fft.fftshift(self.k_axes)

        if (self.d == 1):
            f, ax = plt.subplots(2, figsize = (12,9))
            f.subplots_adjust(hspace=0.35)
            ax2 = ax[1].twinx() # Phase will be plotted at the same place as modulus

            # Fonts for title and labels
            title_font = {'fontname':'TUM Neue Helvetica'}
            label_font = {'fontname':'TUM Neue Helvetica'}

            # Remove the plot frame lines. They are unnecessary chartjunk.
            for ax_i in ax:
                ax_i.spines["top"].set_visible(False)
                ax_i.spines["bottom"].set_visible(False)
                ax_i.spines["right"].set_visible(False)
                ax_i.spines["left"].set_visible(False)

                # Ensure that the axis ticks only show up on the bottom and left of the plot.
                # Ticks on the right and top of the plot are generally unnecessary chartjunk.
                ax_i.get_xaxis().tick_bottom()
                ax_i.get_yaxis().tick_left()

            # Limit the range of the plot to only where the data is.
            # Avoid unnecessary whitespace.
            wf = 1.02 # Whitespace factor
            # Density
            ax[0].set_xlim(min(self.x_min), min(self.x_max))
            ax[0].set_ylim(wf * min(self.density), wf * max(self.density))

            # FT modulus
            ax[1].set_xlim(min(self.k_min), max(self.k_max))
            ax[1].set_ylim(wf * min(abs(ft_)), wf * max(abs(ft_)))

            # FT phase
            ax2.set_xlim(min(self.k_min), max(self.k_max))
            ax2.set_ylim( - wf * np.pi, wf * np.pi)

            # Set ticks
            num_ticks = 7

            # Density
            # number of digits to round ticks
            n_rd = 2 - int(round(np.log10(max(self.density) - min(self.density))))
            ax[0].set_yticks(np.linspace(min(self.density),max(self.density),num_ticks))
            ax[0].set_yticklabels([str(round(x, n_rd)) for x in np.linspace(min(self.density),max(self.density),num_ticks)], color='b')

            # FT modulus
            n_rd = 2 - int(round(np.log10(max(np.abs(ft_)) - min(np.abs(ft_)))))
            if n_rd >= 1:
                ax[1].set_yticks(np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks))
                ax[1].set_yticklabels([str(round(x, n_rd)) for x in np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks)], **label_font)
            else:
                ax[1].set_yticks(np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks))
                ax[1].set_yticklabels([str(int(round(x, n_rd))) for x in np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks)], **label_font)

            # FT phase
            pi_str = r'$\pi$'
            ax2.set_yticks(np.linspace(-np.pi,np.pi,num_ticks))
            ax2.set_yticklabels(['-' + pi_str,
                                 '-2' + pi_str + '/3',
                                 #'-' + pi_str + '/2',
                                 '-' + pi_str + '/3',
                                 0,
                                 pi_str + '/3',
                                 #pi_str + '/2',
                                 '2' + pi_str + '/3',
                                 pi_str], **label_font)

            # Make sure your axis ticks are large enough to be easily read.
            # You don't want your viewers squinting to read your plot.
            my_fontsize = 14
            my_titlesize = 22
            for i in range(2):
                for tick in ax[i].xaxis.get_major_ticks():
                    tick.label.set_fontsize(my_fontsize)

                for tick in ax[i].yaxis.get_major_ticks():
                    tick.label.set_fontsize(my_fontsize)

            for tick in ax2.xaxis.get_major_ticks():
                tick.label.set_fontsize(my_fontsize)

            for tick in ax2.get_yticklabels():
                tick.set_color('g')
                tick.set_fontsize(my_fontsize)

            # Provide tick lines across the plot to help your viewers trace along
            # the axis ticks. Make sure that the lines are light and small so they
            # don't obscure the primary data lines.
            # Density
            for y in np.linspace(min(self.density), max(self.density), num_ticks):
                ax[0].plot(self.x_axes[0],
                           [y] * len(self.x_axes[0]),
                           "--", lw=0.5, color="black", alpha=0.3)

            # FT modulus
            for y in np.linspace(min(abs(ft_)), max(abs(ft_)), num_ticks):
                ax[1].plot(self.k_axes[0],
                           [y] * len(self.k_axes[0]),
                           "--", lw=0.5, color="black", alpha=0.3)

            # Remove the tick marks; they are unnecessary with the tick lines we just plotted.
            for i in range(2):
                ax[i].tick_params(axis="both", which="both", bottom="off", top="off",
                                  labelbottom="on", left="off", right="off", labelleft="on")

            title_shift = 1.07
            ax[0].plot(self.x_axes[0], self.density)
            ax[0].set_title('Object density', y=title_shift, fontsize=my_titlesize, **title_font)

            ax[1].plot(self.k_axes[0], np.abs(ft_))
            ax[1].set_title('Fourier transform', y=title_shift, fontsize=my_titlesize, **title_font)
            ax[1].set_ylabel('FT modulus', color='b', fontsize=my_fontsize+3, **label_font)
            for t1 in ax[1].get_yticklabels():
                t1.set_color('b')

            ax2.plot(self.k_axes[0], np.angle(ft_), color='g')
            ax2.set_ylabel('FT phase', color='g', fontsize=my_fontsize+3, **label_font)

class Approximation:
    """Class describing an approximation to an n-dimensional object. """

if __name__ == "__main__":
    import doctest
    doctest.testmod(extraglobs={'my_image': Image()})


import csv
from labMTsimple import storyLab
import re
from collections import defaultdict
import numpy as np
import matplotlib.pyplot as plt
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

def normalize(val):
    return 1.096*val + 5.27 #because vader scores range from -3.9 to 3.4

vaderAnalyzer = SentimentIntensityAnalyzer()

wsudata  = []
with open('meanscores.csv', 'r') as csvfile:
    wsudata = list(csv.reader(csvfile, delimiter=','))

wsudict = defaultdict(float)

for row in wsudata:
    wsudict[row[0].lower()] = float(row[1])

vaderDict = vaderAnalyzer.lexicon

commonWords = list(set(wsudict.keys()).intersection(vaderDict.keys()))
print("commonWords length", len(commonWords))

moverDict = defaultdict(lambda: [])

for word in commonWords:
    val = normalize(vaderDict[word]) - wsudict[word]
    moverDict[word] = [abs(val), val]

print("top 20 movers")
[print(x) for x in [(key, moverDict[key])for key in sorted(moverDict, key=moverDict.get, reverse=True)][0:20]]


import pandas as pd
import plotly.figure_factory as ff
import plotly.graph_objects as go
import statistics
import csv
import random

df = pd.read_csv("StudentsPerformance.csv")
data = df["reading score"].tolist()

mean = sum(data) / len(data)
std = statistics.stdev(data)
median = statistics.median(data)
mode = statistics.mode(data)

first_std_deviation_start , first_std_deviation_end = mean - std , mean+std
second_std_deviation_start , second_std_deviation_end = mean-(2*std) , mean+(2*std)
third_std_deviation_start , third_std_deviation_end = mean-(3*std) , mean+(3*std)

fig = ff.create_distplot([data], ["reading scores"], show_hist=False)
fig.add_trace(go.Scatter(x=[mean,mean] , y=[0,0.17] , mode ='lines' , name ="MEAN"))
fig.add_trace(go.Scatter(x=[first_std_deviation_start, first_std_deviation_start], y=[0,0.17] ,mode="lines" , name="SD 1"))
fig.add_trace(go.Scatter(x=[first_std_deviation_end, first_std_deviation_end], y=[0,0.17] ,mode="lines" , name="SD 1"))
fig.add_trace(go.Scatter(x=[second_std_deviation_start, second_std_deviation_start], y=[0,0.17] ,mode="lines" , name="SD 2"))
fig.add_trace(go.Scatter(x=[second_std_deviation_end, second_std_deviation_end], y=[0,0.17] ,mode="lines" , name="SD 2"))

list_of_data_within_1_std = [result for result in data if result > first_std_deviation_start and result < first_std_deviation_end]
list_of_data_within_2_std = [result for result in data if result > second_std_deviation_start and result < second_std_deviation_end]
list_of_data_within_3_std = [result for result in data if result > third_std_deviation_start and result < third_std_deviation_end]

print("Mean of this data is {}".format(mean))
print("Median of this data is {}".format(median))
print("Mode of this data is {}".format(mode))
print("SD of this data is {}".format(std))
print("\n")
print("{}% of data lies within 1 Std".format(len(list_of_data_within_1_std)*100.0/len(data)))
print("{}% of data lies within 2 Std".format(len(list_of_data_within_2_std)*100.0/len(data)))
print("{}% of data lies within 3 Std".format(len(list_of_data_within_3_std)*100.0/len(data)))


import requests
from lxml import etree
import sqlite3

header = {
    'Referer': 'http://xt.hnnjgzbt.com/gongshi.aspx',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',
    'Upgrade-Insecure-Requests': '1',
    'Origin': 'http://xt.hnnjgzbt.com',
    'Host': 'xt.hnnjgzbt.com',
}

s = requests.session()
URL = 'http://xt.hnnjgzbt.com/gongshi.aspx'

def get_value():
    response = s.get('http://xt.hnnjgzbt.com/gongshi.aspx').text
    html = etree.HTML(response)
    value = html.xpath('//*[@id="__VIEWSTATE"]/@value')
    return value

def get_data(value):
    for i in range(1, 1005):
        form_data = {
            '__VIEWSTATE': value,
            '__EVENTTARGET': 'AspNetPager1',
            '__EVENTARGUMENT': str(i),
            #'txtname': '',
            #'txtpm': '',
            #有些参数无关紧要，下面这个参数是跳转到另一页时的页码
            #'AspNetPager1_input': i - 1
        }
        response = s.post(URL,data=form_data).text
        print('正在打印第{}'.format(i))

        html = etree.HTML(response)
        div = html.xpath('//*[@id="form1"]//div[@class="contert_table"]')[0]
        row_tags = div.xpath('//table[@align="center"]//tr')

        for tr in row_tags[1:]:
            # 注意加上点，在此标签下
            column_tags = tr.xpath('.//td')
            item = []
            for td in column_tags:
                #print(td.text.strip())
                item.append(td.text.strip())

            c.execute(sql, item)

if __name__ == "__main__":

    conn = sqlite3.connect('test.db')
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE LEIWO
             (NAME          TEXT,
              ADDRESS       TEXT,
              PINMU         TEXT,
              FENDANG       TEXT,
              XINGHAO       TEXT,
              CHANGJIA      TEXT,
              JINGXIAO      TEXT,
              SHULIANG      INT,
              BUTIE1        INT,
              BUTIE2        INT,
              BUTIE3        INT,
              JIAGE         INT,
              ZHUANGTAI     INT);''')

        sql = "INSERT INTO LEIWO VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)"
        get_data(get_value())
        conn.commit()
    finally:
        c.close()
        conn.close()


from azure.identity import ManagedIdentityCredential
from azure.keyvault.secrets import SecretClient
import os
import json

multiplier: int = None

def load_secrets():
    """
    Replaces the values of environment variables with names containing "KV_SECRET" and values of the form "<SECRET_NAME>@<VAULT_URL>" with the actual secret values

    Uses the ManagedIdentityCredential to create a SecretClient for each <VAULT_URL>. The endpoint's Managed Identity should have the get permission for secrets.

    Example:
        KV_SECRET_FOO: foo@https://keyvault123.vault.azure.net

        Will be replaced with the actual vaule of the secret named foo in keyvault123.
    """
    secret_clients = {}
    credential = ManagedIdentityCredential()

    for k, v in os.environ.items():
        if "KV_SECRET" in k:
            try:
                secret_name, vault_url = v.split("@")
            except ValueError:
                raise ValueError(
                    f"Wrong value format for env var {k} with value {v}. Should be of the form <SECRET_NAME>@<VAULT_URL>"
                )

            if vault_url in secret_clients:
                secret_client = secret_clients[vault_url]
            else:
                secret_client = SecretClient(vault_url=vault_url, credential=credential)
                secret_clients[vault_url] = secret_client

            secret_value = secret_client.get_secret(secret_name).value
            os.environ[k] = secret_value

def init():
    load_secrets()

    global multiplier
    multiplier = int(os.getenv("KV_SECRET_MULTIPLIER"))

def run(data):
    data = json.loads(data)
    input = data["input"]
    output = input * multiplier

    return {"output": output}


"""
Command handling for the bot.
"""
import asyncio
import sys
import traceback

import discord
from discord.ext import commands

from discordbot.main import DiscordBot
from discordbot.utils import exceptions

class CommandHandler:
    """
    Command handling for the bot.
    """

    def __init__(self, bot: DiscordBot):
        self.bot = bot

    async def __global_check(self, ctx):

        author = ctx.author
        if await self.bot.is_owner(ctx.author):
            return True

        # user is a bot
        if author.bot:
            return False

        # user is blacklisted
        if author.id in self.bot.ignored.get("users"):
            return False

        perms = ctx.channel.permissions_for(author)
        perm_list = [perms.administrator, perms.manage_messages, perms.manage_guild]
        un_ignore = any(x for x in perm_list)

        # now we can finally realise if we can actually bypass the ignore
        if not un_ignore and ctx.channel.id in self.bot.ignored.get("channels"):
            raise exceptions.Ignored

        return True

    async def on_message_edit(self, before, after):
        """
        Checks message edit to see if I screwed up a command...
        """
        await self.bot.process_commands(after)

    async def on_command_error(self, ctx, error):
        """
        Catch command errors.
        """
        if isinstance(error.__cause__, discord.errors.NotFound):
            return
        elif isinstance(error, exceptions.Ignored):
            await ctx.channel.send("\N{CROSS MARK} This channel is currently being ignored.", delete_after=5)
        elif isinstance(error, commands.errors.NotOwner):
            await ctx.channel.send(f"\N{CROSS MARK} {error}", delete_after=5)
        elif isinstance(error, discord.errors.Forbidden):
            await ctx.channel.send("\N{NO ENTRY} I don't have permission to perform the action", delete_after=5)
        elif isinstance(error, exceptions.ClearanceError):
            await ctx.channel.send(f"\N{NO ENTRY} {error}", delete_after=5)
        elif isinstance(error, commands.errors.CommandNotFound):
            return
        elif isinstance(error, exceptions.EmbedError):
            await ctx.channel.send("\N{NO ENTRY} This command requires the `Embed Links` "
                                   "permission to execute!", delete_after=5)
        elif isinstance(error, commands.errors.NoPrivateMessage):
            await ctx.channel.send("\N{NO ENTRY} That command can not be run in PMs!",
                                   delete_after=5)
            return
        elif isinstance(error, commands.errors.DisabledCommand):
            await ctx.channel.send("\N{NO ENTRY} Sorry, but that command is currently disabled!",
                                   delete_after=5)
        elif isinstance(error, commands.errors.CheckFailure):
            await ctx.channel.send("\N{CROSS MARK} Check failed. You probably don't have "
                                   "permission to do this.", delete_after=5)
        elif isinstance(error, commands.errors.CommandOnCooldown):
            await ctx.channel.send(f"\N{NO ENTRY} {error}", delete_after=5)
        elif isinstance(error, (commands.errors.BadArgument, commands.errors.MissingRequiredArgument)):
            await ctx.channel.send(f"\N{CROSS MARK} Bad argument: {' '.join(error.args)}", delete_after=5)
            formatted_help = await ctx.bot.formatter.format_help_for(ctx, ctx.command)
            for page in formatted_help:
                await ctx.channel.send(page, delete_after=20)
        else:
            await ctx.channel.send("\N{NO ENTRY} An error happened. This has been logged and reported.",
                                   delete_after=5)
            if isinstance(error, commands.errors.CommandInvokeError):
                traceback.print_exception(type(error), error.__cause__, error.__cause__.__traceback__, file=sys.stderr)
            else:
                traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr)

    async def on_command(self, ctx):
        author = ctx.author
        if ctx.guild is not None:
            self.bot.command_logger.info(f"[Shard {ctx.guild.shard_id}] {ctx.guild.name} (ID: {ctx.guild.id}) > "
                                         f"{author} (ID: {author.id}): {ctx.message.clean_content}")
        else:
            self.bot.command_logger.info(f"[Shard 0] Private Messages > {author} (ID: {author.id}):"
                                         f" {ctx.message.clean_content}")

    async def on_command_completion(self, ctx):
        self.bot.commands_used[ctx.command.name] += 1
        if not ctx.guild:
            ctx.guild = "PMs"
        self.bot.commands_used_in[str(ctx.guild)] += 1
        await asyncio.sleep(5)
        try:
            await ctx.message.delete()
        except discord.DiscordException:
            pass

def setup(bot: DiscordBot):
    bot.add_cog(CommandHandler(bot))


"""Test suite for the main flashfocus module."""
try:
    from unittest.mock import call, MagicMock
except ImportError:
    from mock import call, MagicMock
from time import sleep

from pytest import approx, mark
from xpybutil.ewmh import get_wm_window_opacity

from flashfocus.client import client_request_flash
from flashfocus.xutil import count_windows, destroy_window, get_current_desktop
from test.helpers import change_focus, server_running, WindowSession

@mark.parametrize(
    "focus_indices,flash_indices",
    [
        # Test normal usage
        ([1, 0, 1], [1, 0, 1])
    ],
)
def test_event_loop(
    flash_server, windows, focus_indices, flash_indices, monkeypatch
):
    focus_shifts = [windows[i] for i in focus_indices]
    expected_calls = [
        call(windows[i], "focus_shift") for i in flash_indices
    ] + [call(focus_shifts[-1], "client_request")]
    flash_server.matcher.route_request = MagicMock()
    with server_running(flash_server):
        for window in focus_shifts:
            change_focus(window)
        client_request_flash()
    assert flash_server.matcher.route_request.call_args_list == expected_calls

def test_second_consecutive_focus_requests_ignored(flash_server, windows):
    flash_server.matcher.route_request = MagicMock()
    with server_running(flash_server):
        change_focus(windows[1])
        change_focus(windows[1])
    assert flash_server.matcher.route_request.call_args_list == [
        call(windows[1], "focus_shift")
    ]

def test_window_opacity_set_to_default_on_startup(
    mult_opacity_server, list_only_test_windows, windows
):
    with server_running(mult_opacity_server):
        assert get_wm_window_opacity(windows[0]).reply() == approx(0.2)
        assert get_wm_window_opacity(windows[1]).reply() == approx(0.5)

def test_window_opacity_unset_on_shutdown(
    mult_opacity_server, list_only_test_windows, windows
):
    with server_running(mult_opacity_server):
        pass
    assert get_wm_window_opacity(windows[0]).reply() == approx(1)
    assert get_wm_window_opacity(windows[1]).reply() == approx(1)

def test_new_window_opacity_set_to_default(
    transparent_flash_server, list_only_test_windows
):
    with server_running(transparent_flash_server):
        window_session = WindowSession()
        assert get_wm_window_opacity(
            window_session.windows[0]
        ).reply() == approx(0.4)
    window_session.destroy()

def test_server_handles_nonexistant_window(flash_server):
    with server_running(flash_server):
        flash_server.flash_requests.put((0, "client_request"))
        sleep(0.01)

def test_server_handles_none_window(flash_server):
    with server_running(flash_server):
        flash_server.flash_requests.put((None, "client_request"))
        sleep(0.1)

def test_server_handles_none_matches(monkeypatch, flash_server, windows):
    monkeypatch.setattr(
        "flashfocus.rule.Rule.match", lambda *args, **kwargs: None
    )
    with server_running(flash_server):
        change_focus(windows[1])


""" REST API Module """

import pytest

import nltk
import nltk.data

from lxml import etree
from lxml.html import tostring

import requests

import ml_api.utils as utils

@pytest.fixture(scope="module")
def parser(request):
    parser = utils.Parser()
    yield parser
    print ("Finalizing %s" % parser)

# content of test_class.py
class TestUtilsModule(object):
    """ Class used to test ML REST API """

    def test_tokenizer(self, parser):
        assert parser.tokenizer is not None

    def test_html_reading(self, parser):
        text = "<title>test
page title
"
        html = parser.read_html(text)
        html = etree.tostring(html)
        assert b"" in html

    def test_html_reading_from_url(self, parser):
        url = "http://microformats.org/"
        html = parser.read_html("", url=url)
        html = etree.tostring(html)
        assert b"html" in html

    def test_find_paragraphs(self, parser):
        url = "http://microformats.org/"
        html = parser.read_html("", url=url)
        par_list = parser.find_paragraphs(html)
        assert len(par_list) > 0

    def test_convert_pdf2html(self, parser):
        tmp_file = "C:\\Users\\pedro.castanha\\Downloads\\iniciais\\0000668-02.2015.8.19.0208.pdf"
        text = parser.convert_pdf_to_html(tmp_file)
        html = parser.read_html(text)
        assert b"html" in etree.tostring(html) and len(parser.find_paragraphs(html)) > 0

    def test_extarct_sentence(self, parser):
        text = "A confirmação da tutela antecipada, tornando-a definitiva também para declarar o desfazimento das relações jurídicas existentes entre as partes;"
        sentences = parser.sentence_extractor(text)
        assert len(sentences) > 0

    def test_extract_sentences(self, parser):
        texts = [
                    "A confirmação da tutela antecipada, tornando-a definitiva também para declarar o desfazimento das relações jurídicas existentes entre as partes;",
                    "A rescisão do contrato de compra e venda firmado com a segunda ré, bem como a determinação para que esta restitua o valor dado como entrada"
               ]
        sentences = parser.sentence_extractor(texts, list_=True)
        assert len(sentences[0]) > 0


import os
import torch
from torch.utils.data import Dataset, DataLoader
from skimage import io, transform
import skimage.morphology
from skimage.morphology import square
import numpy as np
import random
import pandas as pd
random.seed(0)
from scipy.ndimage.measurements import label as fun_label
import cv2

'''
    load data for fine-tuning on real data
'''

class loadReal(Dataset):
    '''
        load real data
    '''
    def __init__(self, SUNCG_data, IIW_data, NYU_data, DA_type, transform=None):
        '''
            data contains three dataset
            SUNCG synthetic data, contains ground truth albedo, shading, and normal
            IIW_data: intrinsic image in the wild, contains ground truth WHDR, shaing information
            NYU_data: NYU dataset, contains groundtruth normal, shading information
        '''
        self.SUNCG_list = []
        self.IIW_list = []
        self.NYU_list = []
        with open(SUNCG_data['fileNameList']) as f:
            for line in f:
                self.SUNCG_list.append(line.strip())
        with open(IIW_data['fileNameList']) as f:
            for line in f:
                self.IIW_list.append(line.strip())
        with open(NYU_data['fileNameList']) as f:
            for line in f:
                # NOTE: we have two file names for each NYU data,
                # One in original NYU dataset, used to load image and normal
                # The other in SAW dataset, used to load shading
                self.NYU_list.append(line.strip().split())
        self.SUNCG_dic = SUNCG_data
        self.IIW_dic = IIW_data
        self.NYU_dic = NYU_data
        self.transform = transform
        self.DA_type = DA_type
        self.lenNYU = len(self.NYU_list)
        self.numCompare = 100

    def __len__(self):
        # since the three datasets have different number of samples
        # we use number of IIW images as the length of the dataset
        return len(self.IIW_list)

    def __getitem__(self, idx):
        '''
            load one sample for each of the data set
        '''
        # load data for synthetic images
        label_SUNCG = {}
        image_SUNCG = np.array([])
        if self.SUNCG_dic:
            image_SUNCG, label_SUNCG['albedo'], label_SUNCG['shading'], \
                    label_SUNCG['normal'], label_SUNCG['mask'], label_SUNCG['label'] \
                    = self.load_synthetic(idx)
            if self.transform:
                image_SUNCG, label_SUNCG = self.transform(image_SUNCG, label_SUNCG, self.DA_type)

        # load data for IIW
        label_IIW = {}
        image_IIW = np.array([])
        if self.IIW_dic:
            label_IIW = {}
            image_IIW, label_IIW['saw_mask_1'], label_IIW['saw_mask_2'], \
                    label_IIW['label'] = self.load_IIW(idx)
            if self.transform:
                image_IIW, label_IIW = self.transform(image_IIW, label_IIW, self.DA_type)

        # load data for NYU
        label_NYU = {}
        image_NYU = np.array([])
        if self.NYU_dic:
            label_NYU = {}
            image_NYU, label_NYU['normal'], label_NYU['mask'],\
                    label_NYU['saw_mask_1'], label_NYU['saw_mask_2'] = self.load_NYU(idx)
            if self.transform:
                image_NYU, label_NYU = self.transform(image_NYU, label_NYU, self.DA_type)

        return image_SUNCG, image_IIW, image_NYU, label_SUNCG, label_IIW, label_NYU, self.IIW_list[idx]

    def loadLabel(self, labelName, dataName):
        '''
            load csv file
        '''
        data_csv = pd.read_csv(labelName).values
        data_csv = data_csv[:,2:]

        if dataName == 'SUNCG':
            # for suncg dataset, select 3000 comparisons
            tmpLabel = data_csv[:,-2]
            ind_equal = np.where(tmpLabel == 0)[0]
            ind_darker = np.where(tmpLabel == 1)[0]
            ind_lighter = np.where(tmpLabel == 2)[0]
            if ind_equal.shape[0] > self.numCompare:
                np.random.shuffle(ind_equal)
                ind_equal = ind_equal[0:self.numCompare]
            if ind_darker.shape[0] > self.numCompare:
                np.random.shuffle(ind_darker)
                ind_darker = ind_darker[0:self.numCompare]
            if ind_lighter.shape[0] > self.numCompare:
                np.random.shuffle(ind_lighter)
                ind_lighter = ind_lighter[0:self.numCompare]
            data_csv = np.concatenate((data_csv[ind_equal], data_csv[ind_darker],
                data_csv[ind_lighter]), axis=0)

        return data_csv

    def load_synthetic(self, idx):
        '''
            load one data for SUNCG
            load data for our synthesized data
            including: image, albedo, shading, normal, mask, comparison
        '''
        fileName = self.SUNCG_list[idx]
        # load image, albedo, shading, normal and mask
        imgName = os.path.join(self.SUNCG_dic['dataFolder'], fileName)
        image = io.imread(imgName)
        if len(image.shape)==2:
            image = np.tile(image[...,None], (1, 3))
        albedoName = os.path.join(self.SUNCG_dic['albedoFolder'], fileName)
        albedo = io.imread(albedoName)
        if len(albedo.shape)==2:
            albedo = np.tile(albedo[...,None], (1, 3))
        shadingName = os.path.join(self.SUNCG_dic['shadingFolder'], fileName)
        shading = io.imread(shadingName)
        if len(shading.shape)==2:
            shading = np.tile(shading[...,None], (1, 3))
        normalName = os.path.join(self.SUNCG_dic['normalFolder'], fileName[0:-8] + '_norm_camera.png')
        normal = io.imread(normalName)

        normalMaskName = os.path.join(self.SUNCG_dic['normalFolder'], fileName[0:-8] + '_valid.png')
        normalMask = io.imread(normalMaskName)
        normalMask = np.expand_dims(normalMask, axis=-1)

        maskName = os.path.join(self.SUNCG_dic['maskFolder'], fileName)
        mask = io.imread(maskName)
        mask = cv2.resize(mask, tuple(normalMask.shape[1::-1]), interpolation=cv2.INTER_CUBIC)
        mask = np.expand_dims(mask, axis=-1)

        labelName = os.path.join(self.SUNCG_dic['labelFolder'], fileName[0:-4] + '.csv')

        # pre-process data
        image = 1.0*image/255.0
        albedo = 1.0*albedo/255.0
        shading = 1.0*shading/255.0
        normal = normal.astype(np.float)
        normal = (normal/255.0-0.5)*2
        normal = normal/(np.linalg.norm(normal, axis=-1, keepdims=True) + 1e-6)
        mask = 1.0*mask/255.0
        normalMask = 1.0*normalMask/255.0
        mask = mask*normalMask
        return image, albedo, shading, normal, mask, labelName
        #return image, albedo, shading, normal, mask, label

    def load_IIW(self, idx):
        '''
            load images and labels for IIW  (and labels for SAW)
        '''
        fileName = self.IIW_list[idx]
        imgName = os.path.join(self.IIW_dic['dataFolder'], fileName + '.png')
        image = io.imread(imgName)
        if len(image.shape)==2:
            image = np.tile(image[...,None], (1, 3))

        # prepare shading mask
        shadingName = os.path.join(self.IIW_dic['SAW'], fileName + '.npy')
        sawShading = np.load(shadingName)
        saw_mask_1, saw_mask_2 = self.processShadingMask(sawShading)

        labelName = os.path.join(self.IIW_dic['labelFolder'], fileName + '.csv')
        #label = self.loadLabel(labelName, 'IIW')

        image = 1.0*image/255.0
        return image, saw_mask_1, saw_mask_2, labelName
        #return image, saw_mask_1, saw_mask_2, label

    def load_NYU(self, idx):
        '''
            load images, normals and mask for NYU dataset
        '''
        idx = idx % self.lenNYU
        fileName_SAW = self.NYU_list[idx][0]
        fileName_NYU = self.NYU_list[idx][1]
        imgName = os.path.join(self.NYU_dic['dataFolder'], fileName_NYU + '.png')
        image = io.imread(imgName)
        if len(image.shape)==2:
            image = np.tile(image[...,None], (1,3))
        normalName = os.path.join(self.NYU_dic['normalFolder'], fileName_NYU + '_normal.png')
        normal = io.imread(normalName)
        maskName = os.path.join(self.NYU_dic['normalMask'], fileName_NYU + '_mask.png')
        mask = io.imread(maskName)
        mask = np.expand_dims(mask, axis=-1)
        shadingName = os.path.join(self.NYU_dic['SAW'], fileName_SAW + '.npy')
        shading = np.load(shadingName)
        saw_mask_1, saw_mask_2 = self.processShadingMask(shading)

        image = 1.0*image/255.0
        mask = 1.0*mask/255.0
        normal = normal.astype(np.float)
        normal = (normal/255.0-0.5)*2
        normal = normal/(np.linalg.norm(normal, axis=-1, keepdims=True) + 1e-6)

        return image, normal, mask, saw_mask_1, saw_mask_2

    def processShadingMask(self, sawShading):
        '''
            filtering sawShading
        '''
        # NOTE: saw_mask_1 is the boundary of the shadow
        #       saw_mask_2 is the smooth region of the shadow
        saw_mask_1 = (sawShading == 1)
        saw_mask_1 = skimage.morphology.binary_dilation(saw_mask_1, square(9)).astype(np.float32)

        saw_mask_2 = (sawShading == 2).astype(np.float32)
        return saw_mask_1, saw_mask_2

class dataAugmentation(object):
    '''
        define data augmentation for non-comparison purpose
    '''
    def __init__(self, output_size, expand=10, ind_rotate=True):
        self.size = output_size
        self.expand = expand
        self.ind_rotate = ind_rotate

    def processImage(self, image, sH, sW, H, W, rotate_M):
        # rotate
        imgHeight, imgWidth = image.shape[0:2]
        image = cv2.warpAffine(image, rotate_M, (imgWidth, imgHeight))
        # resize and random crop
        image = cv2.resize(image, (W, H), interpolation=cv2.INTER_CUBIC)
        if len(image.shape) < 3:
            image = image[sH:sH+self.size, sW:sW+self.size]
        else:
            image = image[sH:sH+self.size, sW:sW+self.size, :]
        return image

    def __call__(self, image, label, DA_type):
        '''
            random crop, in-plane rotaion for images
            and apply same data augmentation for the labels
        '''
        imgHeight, imgWidth,_ = image.shape
        if not DA_type == 'compare':
            # first rotate
            if self.ind_rotate:
                if random.random() > 0.5:
                    # random rotate -10 to 10 degrees
                    angle = -10*random.random()
                    rotate_M = cv2.getRotationMatrix2D((imgHeight/2.0, imgWidth/2.0), angle, 1)
                else:
                    rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)
            # random crop
            H = self.size + self.expand
            W = self.size + self.expand
            maxH = H - self.size
            maxW = W - self.size
            sH = random.randint(0, maxH)
            sW = random.randint(0, maxW)
        else:
            # rotate 0 degree
            rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)

            # no resize and crop
            H = self.size
            W = self.size
            sH = 0
            sW = 0

        # preprocess image
        image = self.processImage(image, sH, sW, H, W, rotate_M)
        image = np.transpose(image, (2,0,1))
        ind = image < 0
        image[ind] = 0
        if 'albedo' in label.keys():
            tmp = self.processImage(label['albedo'], sH, sW, H, W, rotate_M)
            ind = tmp < 0
            tmp[ind] = 0
            label['albedo'] = np.transpose(tmp, (2,0,1))
        if 'shading' in label.keys():
            tmp = self.processImage(label['shading'], sH, sW, H, W, rotate_M)
            ind = tmp < 0
            tmp[ind] = 0
            label['shading'] = np.transpose(tmp, (2,0,1))
        if 'saw_mask_1' in label.keys():
            tmp_mask = self.processImage(label['saw_mask_1'], sH, sW, H, W, rotate_M)
            tmp_mask, numLabel = fun_label(tmp_mask)
            label['saw_mask_1'] = np.expand_dims(tmp_mask, axis=0)
            label['num_saw_mask_1'] = np.array([numLabel])
        if 'saw_mask_2' in label.keys():
            tmp_mask = self.processImage(label['saw_mask_2'], sH, sW, H, W, rotate_M)
            tmp_mask, numLabel = fun_label(tmp_mask)
            label['saw_mask_2'] = np.expand_dims(tmp_mask, axis=0)
            label['num_saw_mask_2'] = np.array([numLabel])
        if 'mask' in label.keys():
            tmp_mask = self.processImage(label['mask'], sH, sW, H, W, rotate_M)
            tmp_mask = (tmp_mask==1.0).astype(np.float)
            label['mask'] = np.expand_dims(tmp_mask, axis=0)
        if 'normal' in label.keys():
            # in-plane rotation
            #rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)
            tmpNormal = self.processImage(label['normal'], sH, sW, H, W, rotate_M)
            maskInd = np.sum(np.abs(tmpNormal), axis=2) == 0
            maskInd = np.tile(np.expand_dims(maskInd, -1), (1,1,3))
            # we need to rotate normal
            # NOTE: 0 and 2 coordinate represent normal along x and y plane
            #if DA_type=='compare':
            #    rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)
            #else:
            #    rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 90, 1)
            tmpNormal_xy = np.reshape(tmpNormal[:,:,0:3:2], (-1,2))
            tmpNormal_xy = np.dot(tmpNormal_xy, np.transpose(rotate_M[0:2, 0:2], (1,0)))
            tmpNormal_xy = np.reshape(tmpNormal_xy, (self.size, self.size, 2))
            tmpNormal[:,:,0:3:2] = tmpNormal_xy
            tmpNormal[maskInd] = 0
            tmpNormal = tmpNormal / (np.tile(np.sqrt(np.sum(tmpNormal**2, axis=2, keepdims=True)), (1,1,3)) + 1e-6)
            label['normal'] = np.transpose(tmpNormal, (2,0,1))
        return image, label


import json
import os
from math import sqrt

os.chdir('../log_input')
batchfileDir = os.getcwd()
batch = []
with open(os.path.join(batchfileDir, 'batch_log.json')) as f:
    for line in f:
        batch.append(json.loads(line))

D = int(batch[0]["D"])
T = int(batch[0]["T"])

network = []
eventbatch = batch[1:len(batch)]

for event in eventbatch:
    if event['event_type'] == 'befriend':
        network.append({event['id1']: event['id2']})
        network.append({event['id2']: event['id1']})

    if event['event_type'] == 'unfriend':
        network.remove({event['id1']: event['id2']})
        network.remove({event['id2']: event['id1']})

os.chdir('../log_input')
streamfileDir = os.getcwd()
stream = []
with open(os.path.join(streamfileDir, 'stream_log.json')) as g:
    for line in g:
        stream.append(json.loads(line))

flagged_purchases = []
for activity in stream:
    if activity['event_type'] == 'befriend':
        network.append({activity['id1']: activity['id2']})
        network.append({activity['id2']: activity['id1']})
    if activity['event_type'] == 'unfriend':
        network.remove({activity['id1']: activity['id2']})
        network.remove({activity['id2']: activity['id1']})
    if activity['event_type'] == 'purchase':
        person = activity['id']
        personlist = [person]
        friendlist = []
        for i in range(D):
            for friends in network:
                for eachperson in personlist:
                    if eachperson in friends:
                        if friends[eachperson] != person and friends[eachperson] not in personlist:
                            friendlist.append(friends[eachperson])
                            personlist.append(friends[eachperson])

        friendpurchase = []
        for everyfriend in friendlist:
            for event in eventbatch:
                if event['event_type'] != 'purchase': continue
                if event['id'] == everyfriend:
                    friendpurchase.append(event)

        if len(friendpurchase) < 2:
            flagged_purchases = flagged_purchases

        if len(friendpurchase) >= 2:
            if len(friendpurchase) < T:
                amountlist = []
                for everypurchase in friendpurchase:
                    amountlist.append(eval(everypurchase['amount']))
                mean = sum(amountlist) / len(amountlist)
                y = 0.0
                for xn in amountlist:
                    y = y + (xn - mean) ** 2
                sd = sqrt(y / len(amountlist))

                if eval(activity['amount']) > mean + (3 * sd):
                    showmean = round(mean, 2)
                    showsd = round(sd, 2)
                    activity['mean'] = '%.2f' % showmean
                    activity['sd'] = str(showsd)
                    flagged_purchases.append(activity)

            if len(friendpurchase) >= T:
                callist = friendpurchase[len(friendpurchase)-T:len(friendpurchase)]
                amountlist = []
                for everypurchase in callist:
                    amountlist.append(eval(callist['amount']))
                mean = sum(amountlist) / len(amountlist)
                y = 0.0
                for xn in amountlist:
                    y = y + (xn - mean) ** 2
                sd = sqrt(y / len(amountlist))
                if eval(activity['amount']) > mean + (3 * sd):
                    showmean = round(mean, 2)
                    showsd = round(sd, 2)
                    activity['mean'] = '%.2f' % showmean
                    activity['sd'] = str(showsd)
                    flagged_purchases.append(activity)

    eventbatch.append(activity)

print(flagged_purchases)


import unittest
from POM.page import Wechat
from appium import webdriver
from HTMLTestRunner import HTMLTestRunner
from POM.logging_page import *
testcase_dir = 'E:\pycharm\Demo\First_Po\Testcase.py'
class Touch(unittest.TestCase):
    @classmethod
    def setUp(self):
        desired_caps = {
        'platformName' :'Android',
        'platformVersion': '4.4',
        'deviceName' : 'DU2SSE15C3115777',
        'appPackage' :'com.tencent.mm' ,# 'com.android.contacts'
        'appActivity': '.ui.LauncherUI',  # '.activities.DialtactsActivity'
        'unicodekeybord' : 'True',
        "resetKeyboard":"True"
        }
        self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)
        # self.driver.implicitly_wait(30)
        self.driver.implicitly_wait(30)
    @classmethod
    def tearDown(self):
        self.driver.quit()
    def test000(self):
        x=loging(self.driver)
        #为登录页面断言
        self.assertEqual(self.driver.find_element_by_name('2971966792').text, '2971966792')
        #输入密码
        x.input_passw('lixiang123456')
        x.loging_enter()
        sleep(4)
        self.assertEqual(self.driver.find_element_by_name('Duang').text, 'Duang')
    def test001(self):
        print('正在测试test001')
        li=Wechat(self.driver)
        li.contacts('qichuangla')
        self.assertEqual(self.driver.find_element_by_name('Duang').text,'Duang')
    def test002(self):
        print('正在测试test001')
        li=Wechat(self.driver)
        li.discover()
        li.me()


import pytest
import os

from trade_journal.trade_journal import TradeJournal
from pathlib import Path

@pytest.fixture
def tjO():
    '''Returns a trade_journal object for a Counter trade'''

    td = TradeJournal(url="../../data/testCounter.xlsx",
                      worksheet="trading_journal",
                      settingf="../../data/settings.ini")
    return td

@pytest.fixture
def clean_tmp():
    yield
    print("\nCleanup .xlsx file")
    os.remove("../../data/testCounter1.xlsx")

def test_tj_nonexists(clean_tmp):
    '''Instantiates a trade_journal object when the .xlsx pointed by url does not exist'''

    fname = Path("../../data/testCounter1.xlsx")

    td = TradeJournal(url=fname,
                      worksheet="trading_journal",
                      settingf="../../data/settings.ini")

    # check that fname has been initialized

    assert fname.exists() is True

def test_fetch_tradelist(tjO):
    trade_list = tjO.fetch_tradelist()

    assert len(trade_list.tlist) == 4

def test_write_tradelist(tjO):
    trade_list = tjO.fetch_tradelist()
    trade_list.analyze()

    tjO.write_tradelist(trade_list)


# Copyright (C) 2014,2015 VA Linux Systems Japan K.K.
# Copyright (C) 2014,2015 YAMAMOTO Takashi <yamamoto at valinux co jp>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron.plugins.ml2.drivers.openvswitch.agent.common import constants
from neutron.plugins.ml2.drivers.openvswitch.agent.openflow.ovs_ofctl \
    import br_dvr_process
from neutron.plugins.ml2.drivers.openvswitch.agent.openflow.ovs_ofctl \
    import ovs_bridge

class OVSPhysicalBridge(ovs_bridge.OVSAgentBridge,
                        br_dvr_process.OVSDVRProcessMixin):
    """openvswitch agent physical bridge specific logic."""

    # Used by OVSDVRProcessMixin
    dvr_process_table_id = constants.DVR_PROCESS_VLAN
    dvr_process_next_table_id = constants.LOCAL_VLAN_TRANSLATION

    def setup_default_table(self):
        self.install_normal()

    def provision_local_vlan(self, port, lvid, segmentation_id, distributed):
        table_id = constants.LOCAL_VLAN_TRANSLATION if distributed else 0
        if segmentation_id is None:
            self.add_flow(table=table_id,
                          priority=4,
                          in_port=port,
                          dl_vlan=lvid,
                          actions="strip_vlan,normal")
        else:
            self.add_flow(table=table_id,
                          priority=4,
                          in_port=port,
                          dl_vlan=lvid,
                          actions="mod_vlan_vid:%s,normal" % segmentation_id)

    def reclaim_local_vlan(self, port, lvid):
        self.delete_flows(in_port=port, dl_vlan=lvid)

    def add_dvr_mac_vlan(self, mac, port):
        self.install_output(table_id=constants.DVR_NOT_LEARN_VLAN,
            priority=2, eth_src=mac, port=port)

    def remove_dvr_mac_vlan(self, mac):
        # REVISIT(yamamoto): match in_port as well?
        self.delete_flows(table=constants.DVR_NOT_LEARN_VLAN,
                          dl_src=mac)


import numpy as np
import pandas as pd
import sqlalchemy
import psycopg2
import os

from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func, inspect
from flask_sqlalchemy import SQLAlchemy

from flask import Flask, jsonify, render_template

engine = create_engine("postgresql+psycopg2://postgres:lovelife2@localhost:5432/Youtube_project")

# Reflect database into  a new model
Base = automap_base()
# Reflect the tables

Base.prepare(engine, reflect=True)

Trending = Base.Classes.youtube
session = Session(engine)

# Flask Setup
#################################################
app = Flask(__name__)

# dialect+driver://username:password@host:port/database
#app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql+psycopg2://postgres:postgres@localhost:5432/Youtube_project"
#db = SQLAlchemy(app)

#################################################
# Flask Routes
#################################################
@app.route("/")
def home():
    return (
        render_template("index.html")
    )

@app.route("/data")
def data():
   sel = [
   Trending.video_id,
   Trending.trending_date,
   Trending.category_id,
   Trending.views,
   Trending.category,
   Trending.likes,
   Trending.dislikes,
   Trending.comments_count
   ]
   results = session.query(*sel).all()
   season_data = []
   for video_id, trending_date, category_id, views, category, likes, dislikes, comments_count  in results:
       season_dict = {}
       season_dict["video_id"] = video_id
       season_dict["trending_date"] = trending_date
       season_dict["category_id"] = category_id
       season_dict["views"] = views
       season_dict["category"] = category
       season_dict["likes"] = likes
       season_dict["dislikes"] = dislikes
       season_dict["comments_count"] = comments_count
       season_data.append(season_dict)
   return jsonify(season_data)