import threading,time,sys,os,signal,subprocess,json
from kazoo.client import KazooClient
from kazoo.retry import RetryFailedError
from kazoo.exceptions import KazooException
import rate_processing_interval_combinations3
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import metadata,send_mail,util
import zmq,json

class Hawk(object):
  def __init__(self,config,log_dir,run_id,\
    zk_connector,fe_address,mq_connector,liveliness_mins):
    self.config=config
    self.log_dir=log_dir
    self.run_id=run_id
    self.zk=KazooClient(hosts=zk_connector)
    self.event=threading.Event()
    self.execution_attempt=0
    self.zk_connector=zk_connector
    self.fe_address=fe_address

    self.mq_connector=mq_connector
    self.context=zmq.Context(1)
    self.sockets=[]
    for connector in mq_connector.split(','):
      socket=self.context.socket(zmq.PUSH)
      socket.connect(connector)
      self.sockets.append(socket)

    self.periodic_check_timer_interval_sec=10
    self.grace_period_mins=3
    self.execution_failure_attempts=((liveliness_mins+\
      self.grace_period_mins)*60)/self.periodic_check_timer_interval_sec

  def reset_variables(self):
    self.attempt=0
    self.execution_failed=False
    self.event.clear()

  def start_periodic_check_timer(self):
    self.periodic_check=threading.Timer(self.periodic_check_timer_interval_sec,
      self.check)
    self.periodic_check.start()

  def zk_query(self):
    if (self.zk.exists('/experiment/test/barriers')):
      children=self.zk.get_children('/experiment/test/barriers')
    else:
      children=None

    if self.zk.exists('/experiment/test/finished'):
      print('Test Execution for run_id:%d was successful'%(self.run_id))
      self.event.set()
      return

    if ((children is not None) and (len(children)>0) and (self.attempt>self.execution_failure_attempts)):
      print('Test Execution for run_id:%d has failed'%(self.run_id))
      self.execution_failed=True
      self.event.set()
      return

  def check(self):
    self.attempt+=1
    print('Attempt number:%d'%(self.attempt))
    if (self.attempt>self.execution_failure_attempts):
      self.execution_failed=True
      self.event.set()
      return
    try:
      self.zk.retry(self.zk_query)
    except (KazooException,RetryFailedError) as e:
      print('Caught KazooException')

    #install timer again
    self.start_periodic_check_timer()

  def run(self):
    self.execution_attempt+=1
    #start zk
    self.zk.start()
    #delete finished znode if it exists
    if self.zk.exists('/experiment/test/finished'):
      self.zk.delete('/experiment/test/finished')

    #reset variables
    self.reset_variables()
    #start timer
    self.start_periodic_check_timer()
    #start test
    print('Starting test for run_id:%d and config:%s\n'%(self.run_id,self.config))
    args=['python','src/tests/rate_processing_interval_combinations3.py',\
      '-config',json.dumps(self.config),\
      '-log_dir',self.log_dir,\
      '-run_id','%d'%(self.run_id),\
      '-zk_connector',self.zk_connector,\
      '-fe_address',self.fe_address,\
      '-mq_connector',self.mq_connector]
    p= subprocess.Popen(args)

    #wait for test to finish
    self.event.wait()

    #check execution status
    if self.execution_failed:
      print('Execution Attempt:%d has failed.'%(self.execution_attempt))
      #kill the experiment run
      p.kill()
      #cancel periodic check timer
      self.periodic_check.cancel()
      #delete barriers
      try:
        self.zk.delete('/experiment/test/barriers',recursive=True)
      except KazooException as e:
        print('Caught KazooException')

      self.zk.stop()
      #kill existing endpoints
      sub_placement=rate_processing_interval_combinations3.place('sub',\
        ['%s'%(tdesc.split(',')[4]) for tdesc in self.config])
      pub_placement=rate_processing_interval_combinations3.place('pub',\
        ['%s'%(tdesc.split(',')[3]) for tdesc in self.config])
      endpoint_machines=list(sub_placement.keys())+ list(pub_placement.keys())

      #command_string='cd %s && ansible-playbook playbooks/util/kill.yml \
      #  --extra-vars="pattern=edgent.endpoints" --limit %s'%\
      #  (metadata.ansible,','.join(endpoint_machines))
      #subprocess.check_call(['bash','-c', command_string])

      util.kill2(self.sockets,','.join(endpoint_machines))
      #check if endpoints are still running. If so, restart those nodes
      #wait for sometime
      #restart test
      if (self.execution_attempt< 3):
        time.sleep(5)
        self.run()
      else:
        print('All execution attempts for test:%d failed.'%(self.run_id))
        self.cleanup()
    else:
      print('Execution attempt:%d was successful.Exiting'%(self.execution_attempt))
      self.periodic_check.cancel()
      self.cleanup()

  def cleanup(self):
    self.zk.stop()
    for socket in self.sockets:
      socket.setsockopt(zmq.LINGER,0)
      socket.close()
    self.context.destroy()

if __name__=="__main__":
  zk_connector=metadata.public_zk3
  fe_address='10.20.30.5'
  runs=1

  k_colocation=3
  mq_connector='tcp://129.59.104.151:3025'
  root_log_dir='/home/kharesp/workspace/java/edgent/model_learning_urd/training'

  for runid in range(1,runs+1,1):
    config_file='%s/configurations/%d_colocation'%(root_log_dir,k_colocation)
    with open(config_file,'r') as f:
      for idx,line in enumerate(f):
        #if (idx+1) in [186,747,827,297,481,525]:
        if (idx+1) in [687, 747, 748, 761, 810, 909, 924, 945, 965]:
          log_dir= '%s/test_interval_impact/%d_colocation/between_.04_and_.09/run%d/%d'%(root_log_dir,k_colocation,runid,idx+1)
          if not os.path.exists(log_dir):
            os.makedirs(log_dir)
          for l in [2,5,10,20,40]:
            if (idx+1==687) and (l<=20):
              continue
            config=rate_processing_interval_combinations3.create_configuration(line.rstrip(),liveliness=l*60)

            print('Starting test runid:%d iteration:%d liveliness:%d mins with config:%s\n\n\n'%(runid,idx+1,l,config))
            start_time=time.time()
            h=Hawk(config,log_dir,l,\
              zk_connector,fe_address,mq_connector,l)
            h.run()
            end_time=time.time()

            elapsed_time_min=(end_time-start_time)/60.0
            print('\n\nRunid:%d iteration:%d liveliness:%d mins took %f mins to finish\n\n'%(runid,idx+1,l,elapsed_time_min))


import streamlit as st
import streamlit.components.v1 as components
import pandas as pd
import utils.database as database
import base64

def download_button(object_to_download, download_filename):
    try:
        # some strings <-> bytes conversions necessary here
        b64 = base64.b64encode(object_to_download.encode()).decode()

    except AttributeError as e:
        b64 = base64.b64encode(object_to_download).decode()

    dl_link = f"""
    <html>
    <head>
    <script src="http://code.jquery.com/jquery-3.2.1.min.js"></script>
    <script>
    $('<a href="data:text/csv;base64,{b64}" download="{download_filename}">')[0].click()
    </script>
    </head>
    </html>
    """
    return dl_link

def create_page():
    with st.form(key="dont_refresh", clear_on_submit=False):
        event_to_download = st.number_input("Enter event ID", value=1)
        submitted = st.form_submit_button("Download")

        if submitted:
            create_download(event_to_download)

def create_download(event_id = None):
    if event_id == None:
        event_id = st.experimental_get_query_params()["event"][0]
        if not event_id.isnumeric():
            return

    conn = database.get_connection()

    # Search for the specific event.
    df = pd.read_sql(f"SELECT * FROM event WHERE event_id={event_id} AND pdf IS NOT NULL", conn)

    if df.size == 0:
        st.write(f"A summary of the event {event_id} does not yet exist.")
        return

    pdf_bytes = df.iloc[0]["pdf"]

    components.html(
        download_button(pdf_bytes, f"Summary - event {event_id}.pdf"),
        height=0,
    )

    st.write("Download is finished.")


from vcl3datlantis.models.PanoDR.layer import init_weights
import numpy as np
import cv2
import torch
import torch.nn as nn
import torch.nn.functional as F
import functools
from vcl3datlantis.models.GatedConv.network_module import *
from vcl3datlantis.models.PanoDR.SEAN.normalization import *
from vcl3datlantis.models.PanoDR.SEAN.spade_arch import *
from vcl3datlantis.models.semantic_segmentation_models.unet.model import UNet

class GatedGenerator(nn.Module):
    def __init__(self, opt, device):
        super(GatedGenerator, self).__init__()
        self.opt = opt
        self.sem_layout = None
        self.soft_sem_layout = None
        if opt.structure_model == "unet":
           self.structure_model = UNet(n_channels = opt.in_layout_channels, n_classes = opt.num_classes)

        if opt.type_sp == 'SEAN':
            self.Zencoder = Zencoder(opt.in_SEAN_channels, opt.style_code_dim)
            self.spade_block_1 = SPADEResnetBlock(opt.latent_channels*4, opt.latent_channels*4, opt.in_spade_channels, device=device, Block_Name='up_0')
            self.spade_block_2 = SPADEResnetBlock(opt.latent_channels*2, opt.latent_channels*2, opt.in_spade_channels, device=device, Block_Name='up_1')

        self.refinement = nn.Sequential(
            # Surrounding Context Encoder
            GatedConv2d(opt.in_channels, opt.latent_channels, 5, 2, 2, pad_type = opt.pad_type, activation = opt.activation, norm='none'),
            GatedConv2d(opt.latent_channels, opt.latent_channels * 2, 3, 1, 1, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 2, opt.latent_channels * 4, 3, 2, 1, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 1, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 1, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 1, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 2, dilation = 2, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 4, dilation = 4, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 8, dilation = 8, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 16, dilation = 16, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 1, pad_type = opt.pad_type, activation = opt.activation, norm = opt.norm),
            GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 4, 3, 1, 1, pad_type = opt.pad_type, activation = opt.activation),
        )
        #Structure-Aware Decoder
        self.refine_dec_1 = nn.Sequential(nn.Upsample(scale_factor=2),
        GatedConv2d(opt.latent_channels * 4, opt.latent_channels * 2, 3, 1, 1, activation = opt.activation, pad_type='zero', norm=opt.norm),
        )
        self.refine_dec_2 =  GatedConv2d(opt.latent_channels * 2, opt.latent_channels * 2, 3, 1, 1, pad_type = opt.pad_type, activation = opt.activation)
        self.refine_dec_3 = nn.Sequential(nn.Upsample(scale_factor=2),

        GatedConv2d(opt.latent_channels * 2, opt.latent_channels, 3, 1, 1, pad_type ='zero', activation = opt.activation),
        )
        self.refine_dec_4 = GatedConv2d(opt.latent_channels, opt.out_channels, 3, 1, 1, pad_type = opt.pad_type, norm='none', activation = 'tanh')

    def forward(self, img, inverse_mask, masked_input, device, use_sean):
        self.sem_layout = torch.zeros((self.opt.batch_size, self.opt.in_layout_channels, self.opt.height, self.opt.width)).float().to(device)
        first_out = masked_input
        structure_model_output = self.structure_model(masked_input).clone()

        if self.opt.use_argmax:
            self.soft_sem_layout = torch.softmax(structure_model_output, dim = 1)
            self.soft_sem_layout = torch.argmax(self.soft_sem_layout, dim=1, keepdim=True)
            self.soft_sem_layout = torch.clamp(self.soft_sem_layout, min=0, max=2)
            self.sem_layout.scatter_(1, self.soft_sem_layout, 1)

        else:
            self.sem_layout = torch.softmax(structure_model_output, dim = 1)

        second_out = self.refinement(torch.cat((masked_input, inverse_mask), 1))  #Enconder + bottleneck

        if self.opt.type_sp == 'SEAN':
            style_codes = self.Zencoder(input=img, segmap=self.sem_layout)
            z = self.spade_block_1(second_out, self.sem_layout, style_codes)
            second_out = self.refine_dec_1(z)
            second_out = self.refine_dec_2(second_out)
            z = self.spade_block_2(second_out, self.sem_layout, style_codes)
            second_out = self.refine_dec_3(z)
            second_out = self.refine_dec_4(second_out)
        else:
            second_out = self.refine_dec_1(second_out)
            second_out = self.refine_dec_2(second_out)
            second_out = self.refine_dec_3(second_out)
            second_out = self.refine_dec_4(second_out)
        second_out = torch.clamp(second_out, 0, 1)

        return first_out, second_out, structure_model_output, self.sem_layout

class Discriminator(nn.Module):
    def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.InstanceNorm2d, use_sigmoid=False, getIntermFeatures=True):
        super(Discriminator, self).__init__()
        self.n_layers = n_layers
        self.getIntermFeatures = getIntermFeatures

        if type(norm_layer) == functools.partial:
            use_bias = norm_layer.func == nn.InstanceNorm2d
        else:
            use_bias = norm_layer == nn.InstanceNorm2d

        kw = 4
        padw = 1
        sequence = [
            spectral_norm(nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw),True),
            nn.LeakyReLU(0.2, True)
        ]

        nf_mult = 1
        nf_mult_prev = 1
        for n in range(1, n_layers):
            nf_mult_prev = nf_mult
            nf_mult = min(2**n, 8)
            sequence += [
                spectral_norm(nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult,
                          kernel_size=kw, stride=2, padding=padw, bias=use_bias),True),

                nn.LeakyReLU(0.2, True),
            ]

        nf_mult_prev = nf_mult
        nf_mult = min(2**n_layers, 8)
        sequence += [
            spectral_norm(nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult,
                                    kernel_size=kw, stride=2, padding=padw, bias=use_bias), True),

            nn.LeakyReLU(0.2, True)
        ]

        sequence += [spectral_norm(nn.Conv2d(ndf * nf_mult, 1,
                          kernel_size=kw, stride=2, padding=padw, bias=use_bias),True)]

        if getIntermFeatures:
            for n in range(len(sequence)):
                setattr(self, 'Dmodel'+str(n), nn.Sequential(sequence[n]))

        if use_sigmoid:
            sequence += [nn.Sigmoid()]

        self.model = nn.Sequential(*sequence)

    def forward(self, img, mask, getIntermFeatures):
        self.getIntermFeatures = getIntermFeatures
        input = torch.cat((img,mask), 1)

        if self.getIntermFeatures:
            _out = [input]
            for n in range(self.n_layers+6):
                model = getattr(self, 'Dmodel'+str(n)) #Dmodel
                _out.append(model(_out[-1]))
            #return _out[1:]
            return _out[9], _out[1:6]
        else:
            out = self.model(input)
            return out

def spectral_norm(module, mode=True):
    if mode:
        return nn.utils.spectral_norm(module)

    return module

def get_norm_layer(norm_type='instance'):
    """Return a normalization layer
    Parameters:
        norm_type (str) -- the name of the normalization layer: batch | instance | none
    For BatchNorm, we use learnable affine parameters and track running statistics (mean/stddev).
    For InstanceNorm, we do not use learnable affine parameters. We do not track running statistics.
    """
    if norm_type == 'batch':
        norm_layer = functools.partial(nn.BatchNorm2d, affine=True, track_running_stats=True)
    elif norm_type == 'instance':
        norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False)
    elif norm_type == 'none':
        def norm_layer(x): return Identity()
    else:
        raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
    return norm_layer


import torch
from torch.nn import functional as F
from torch import nn, autograd
from torch.autograd import Variable
import numpy as np
import random
from tqdm import tqdm
from torch.utils.data import Subset, ConcatDataset, DataLoader, TensorDataset
from torch import optim
import sys
from torchvision import transforms

def craft_adv(net, X, label, X_target, label_target, epsilon, num_steps, decay=1.0, rule='adaptive', random_init=False):
    if rule == 'adaptive':
        step_size = 2.*epsilon/num_steps
    elif rule == 'fixed':
        step_size = 0.1*epsilon

    net.eval()

    X_pgd = Variable(X.data, requires_grad=True)
    momentum = torch.zeros_like(X_pgd).detach().cuda()
    if random_init:
        random_noise = torch.FloatTensor(*X_pgd.shape).uniform_(-epsilon, epsilon).cuda()
        X_pgd = Variable(X_pgd.data + random_noise, requires_grad=True)

    for idx in range(num_steps):
        with torch.enable_grad():
            loss_target = F.cross_entropy(net(X_target), label_target)
            target_grads = torch.autograd.grad(loss_target, net.parameters(), retain_graph=True, create_graph=True)
            net.zero_grad()

            loss_posion = F.cross_entropy(net(X_pgd), label)
            poison_grads = torch.autograd.grad(loss_posion, net.parameters(), retain_graph=True, create_graph=True)
            net.zero_grad()

            grads = 0
            for poison_grad, target_grad in zip(poison_grads, target_grads):
                grads -= F.cosine_similarity(target_grad.flatten(), poison_grad.flatten(), dim=0)

            X_grad = torch.autograd.grad(grads, X_pgd)[0]

            X_grad = X_grad / torch.mean(torch.abs(X_grad), dim=(1,2,3), keepdim=True)

            X_grad = X_grad + momentum*decay
            momentum = X_grad
            eta = step_size * X_grad.data.sign()

            X_pgd = Variable(X_pgd.data - eta, requires_grad=True)
            eta = torch.clamp(X_pgd.data - X.data, -epsilon, epsilon)
            X_pgd = Variable(X.data + eta, requires_grad=True)
            X_pgd = Variable(torch.clamp(X_pgd, 0, 1.0), requires_grad=True)

    return X_pgd

def dataset_split(dataset, sample_size):
    index = list(range(len(dataset)))
    random.shuffle(index)
    rind = index[:sample_size]

    keep = list(set(index)-set(rind))

    dataset_k = Subset(dataset, keep)
    dataset_r = Subset(dataset, rind)

    return dataset_k, dataset_r

def attack_dataset(net, train_set, target_set, epsilon, num_steps, percentage, decay=1., rule='adaptive', device='cuda:0', random_init=True):
    sample_size = int(len(train_set)*(percentage*0.01))

    clean_set, poison_set = dataset_split(train_set, sample_size)

    poison_loader = DataLoader(poison_set, shuffle=True, num_workers=0, batch_size=500)
    target_loader = DataLoader(target_set, shuffle=True, num_workers=0, batch_size=500)

    poison_data = []
    poison_targets = []

    for _, (X, label) in tqdm(enumerate(poison_loader)):
        X_target, label_target = next(iter(target_loader))
        X_target = X_target.cuda()
        label_target = (label_target+torch.randint_like(label_target,1,10))%10
        label_target = label_target.cuda()

        X = X.cuda()
        label = label.cuda()

        X_adv = craft_adv(net, X, label, X_target, label_target, epsilon, num_steps, decay=decay, rule=rule, random_init=random_init)

        poison_data.append(X_adv.detach().cpu())
        poison_targets.append(label.detach().cpu())

    poison_data = torch.cat(poison_data, 0)

    poison_targets = torch.cat(poison_targets, 0)

    P = TensorDataset(poison_data, poison_targets)

    return clean_set+P


import scrapy

class UueduudisedSpider(scrapy.Spider):
    name = 'uueduudised'
    #allowed_domains = ['uueduudised.ee/arhiiv']
    start_urls = ['https://uueduudised.ee/rubriik/arvamus/','https://uueduudised.ee/rubriik/uudis/eesti/', 'https://uueduudised.ee/rubriik/uudis/maailm/', 'https://uueduudised.ee/rubriik/majandus/']

    def parse(self, response):
        HL_SELECTOR = '.news_box_item_content'
        for news in response.css(HL_SELECTOR):
            LINK_SELECTOR = 'h3 a ::attr(href)'
            yield {
                'link': news.css(LINK_SELECTOR).extract_first(),
            }
        #next_page = response.css('.next page-numbers a ::attr(href)').extract_first()
        next_page = response.xpath('.//a[@class="next page-numbers"]/@href').extract_first()
        print("Järgmine lehekülg on: ", next_page)
        #print("URL " + response.urljoin(next_page))

        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)


#!/usr/bin/env python

# This script looks into a directory and checks if
# some files have changed since last execution

from stat import S_ISREG, ST_CTIME, ST_MODE
import os, time
import pickle
import subprocess
from datetime import datetime
import json

dirPath = "/dir_monitoring/to_monitor"
savePath = "/dir_monitoring/tmp/files.pkl"
indexFile = "/dir_monitoring/htdocs/index.json"

time_to_sleep = 60 # Time in seconds between two executions

def save_object(obj, filename):
    with open(filename, 'wb') as output:
        pickle.dump(obj, output)

# Make a system call
def system_call(command, from_docker=True):
	if from_docker:
		command = "docker exec ipfs-container " + command
	print("Executing: ", command)
	p = subprocess.Popen([command], stdout=subprocess.PIPE, shell=True)
	return p.stdout.read()

print("--------------------------")
print("Welcome to IPFS monitoring")
print("--------------------------")

# Wait for other containers to start
time.sleep(10)

##############################
# Load current IPFS state
##############################
try:
	with open(savePath, 'rb') as input:
	    currentIPFSState = pickle.load(input)
except FileNotFoundError:
	# First launch: initialize currentIPFSState to empty dict
	currentIPFSState = {}

# Main loop
while True:

	print("--------------------------------------")
	print("IPFS monitoring")
	print("Looking for modifications in directory")

	##############################
	# Scan monitored directory
	##############################

	# get all entries in the directory w/ stats
	entries = ((os.path.join(dirPath, fn), fn) for fn in os.listdir(dirPath))
	entries = ((os.stat(path), path, fn) for path, fn in entries)

	# leave only regular files, insert creation date
	entries = [(stat[ST_CTIME], path, fn)
	           for stat, path, fn in entries if S_ISREG(stat[ST_MODE])]

	###############################
	# Compare curent IPFS state
	# with directory state
	###############################

	# Select IPFS files that have been modified or removed
	filesToRemove = [mHash for mHash in currentIPFSState if currentIPFSState[mHash] not in entries]

	# Select IPFS files that have been modified or removed
	filesToAdd = [f for f in entries if f not in currentIPFSState.values()]

	################################
	# Record changes in IPFS
	################################

	# Remove old files from IPFS
	print ("Removing files from IPFS...")
	count = 0
	for mHash in filesToRemove:
		# This test protects again empty hash which may cause a freeze in system_call
		if mHash != "":
			count += 1
			# Remove file from ipfs
			print("Removing file from IPFS: ", mHash, currentIPFSState[mHash][1])
			response = system_call("ipfs pin rm " + mHash)
			print (response.strip().decode('UTF-8'))
		# Remove file from dictionary
		currentIPFSState.pop(mHash, None)

	print("%i files removed." % count)

	# Add new files to IPFS
	print ("Adding files to IPFS...")
	count = 0
	for f in filesToAdd:
		count += 1
		# Add file to ipfs
		print("Adding file: ", f[1])
		mHash = system_call("ipfs add -q \"" + f[1] + "\"")
		mHash = mHash.strip().decode('UTF-8')
		print (mHash, f)
		currentIPFSState[mHash] = f

	print ("%i files added." % count)

	#################################
	# Save new IPFS state in file
	#################################

	print ("Saving new state...")
	save_object(currentIPFSState, savePath)

	######################################
	# Create index file with new state
	######################################

	print ("Saving index file: %s" % indexFile)
	index = [{	'hash': mHash,
				'file': currentIPFSState[mHash][2],
				'uploaded_at': datetime.utcfromtimestamp(currentIPFSState[mHash][0]).strftime('%Y-%m-%dT%H:%M:%SZ')}
				for mHash in currentIPFSState]

	with open(indexFile, 'w') as outfile:
	    json.dump(index, outfile)

	print ("IPFS monitoring going to sleep for %d seconds" % time_to_sleep)
	print ("-----------------------------------------------------")

	#####################################
	# Sleep for some time
	#####################################
	time.sleep(time_to_sleep)


# 几种简单的边缘提取方法
import cv2
import matplotlib.pyplot as plt
import numpy as np
from general import extract_edge, fuse_edge_x_y, thr_process

threshold = None

img_path = "./images/lena.bmp"
img = cv2.imread(img_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# 简单梯度算子
kernel_h = np.array([[-1,1]], dtype=np.float32)
kernel_v = np.array([[-1],
                     [1]], dtype=np.float32)

edge_1 = extract_edge(img, kernel_h, kernel_v, threshold=threshold)

# Roberts梯度算子
kernel_h = np.array([
    [1, 0],
    [0, -1],
], dtype=np.float32)
kernel_v = np.array([
    [0, 1],
    [-1, 0],
], dtype=np.float32)
edge_roberts = extract_edge(img, kernel_h, kernel_v, threshold=threshold)

# Prewitt梯度算子
kernel_h = np.array([
    [-1, 0, 1],
    [-1, 0, 1],
    [-1, 0, 1],
], dtype=np.float32)
kernel_v = np.array([
    [1, 1, 1],
    [0, 0, 0],
    [-1, -1, -1],
], dtype=np.float32)
edge_prewitt = extract_edge(img, kernel_h, kernel_v, threshold=threshold)

# Sobel梯度算子
kernel_h = np.array([
    [-1, 0, 1],
    [-2, 0, 2],
    [-1, 0, 1],
], dtype=np.float32)
kernel_v = np.array([
    [1, 2, 1],
    [0, 0, 0],
    [-1, -2, -1],
], dtype=np.float32)
edge_sobel = extract_edge(img, kernel_h, kernel_v, threshold=threshold)

# # # 沿着x/y两个方向的梯度
# edge_sobel = cv2.Sobel(img, cv2.CV_32F, 1, 1)
# # # 沿x方向的边缘检测
# # sobelx = cv2.Sobel(img, cv2.CV_32F, 1, 0)
# # # 沿y方向的边缘检测
# # sobely = cv2.Sobel(img, cv2.CV_32F, 0, 1)
# # edge_sobel = fuse_edge_x_y(sobelx, sobely)
# edge_sobel = thr_process(edge_sobel, threshold=threshold)

fig, ax = plt.subplots(2,2)
ax = ax.ravel()

ax[0].imshow(edge_1, cmap='gray')
ax[0].axis("off")
ax[0].set_title("简单梯度算子")

ax[1].imshow(edge_roberts, cmap='gray')
ax[1].axis("off")
ax[1].set_title("roberts")

ax[2].imshow(edge_prewitt, cmap='gray')
ax[2].axis("off")
ax[2].set_title("prewitt")

ax[3].imshow(edge_sobel, cmap='gray')
ax[3].axis("off")
ax[3].set_title("Sobel")

#解决中文显示问题
# plt.rcParams['font.sans-serif']=['SimHei'] # 设置字体
# plt.rcParams['axes.unicode_minus'] = False # 使坐标轴刻度表签正常显示正负号
plt.rcParams['font.sans-serif'] = ['Arial Unicode MS']

plt.show()


import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore

def init(key="../keys/cheqout-57ee7-firebase-adminsdk-8b1oa-8dd14d0e11.json"):
    """
    Initializes firestore
    @return: returns a Collection Object that references the top level inventory collection in Firestore
    """
    cred = credentials.Certificate(
        key)
    firebase_admin.initialize_app(cred)

    # App is an app object of sorts
    App = firebase_admin.initialize_app(cred, None, 'Cheqout Inventory')
    # Client is a firestore client, the docs is https://googlecloudplatform.github.io/google-cloud-python/latest/firestore/client.html
    Client = firebase_admin.firestore.client()

    # get a reference to the top level collection
    return Client.collection('inventory')

def get_item_data(inventory, item_id):
    """
    Get the data associated with an item from an inventory
    @param inventory: The Collection object that represents the inventory
    @param item_id: The id of the item data being requested from
    @return: a dict containing data about the requested item
    """
    for document in inventory.get():
        if document.id == item_id:
            return_data = document.to_dict()
            return_data['id'] = item_id
            return return_data
    return None

def add_item_data(inventory, item_id, item_data):
    """
    Stores the item data and item id to the firestore database
    @param inventory: The Collection object that represents the inventory
    @param item_id: The id of the item data being stored in
    @param item_data: The data of the item to be stored
    @return: A boolean flag representing whether the operation succeeded or not

    Assumes the input item_data is valid
    """
    # if the item already exists, just update its' information
    for document in inventory.get():
        if document.id == item_id:
            doc_ref = document.reference
            doc_ref.set(item_data)
            return True
    # otherwise, create a new entry and add it to the collection
    inventory.add(item_data, item_id)
    return True


# -*- coding: utf-8 -*-
__author__ = 'dkold_000'

from django.db import models
from django.contrib import admin

from basis.widgets import ImageWidget
from contacts.models import Contact

class ContactAdmin(admin.ModelAdmin):

    fieldsets = [
        (None,          {'fields':  ['image', 'text']}),
        (u'Координаты', {'fields':  ['coord', 'coord_popup']}),
        ('Admin',       {'fields':  ['_created', '_updated', '_active', '_comment'],
                         'classes': ['collapse']}),
    ]

    formfield_overrides = {
        models.ImageField:  {'widget':  ImageWidget()}
    }

    readonly_fields = ['_created', '_updated']

admin.site.register(Contact, ContactAdmin)


import asyncio
import base64
import io
import math
import random
import re
import string
import urllib.request
from os import remove

import cloudscraper
import emoji as catemoji
from bs4 import BeautifulSoup as bs
from PIL import Image
from telethon import events
from telethon.errors.rpcerrorlist import YouBlockedUserError
from telethon.tl import functions, types
from telethon.tl.functions.messages import GetStickerSetRequest
from telethon.tl.functions.messages import ImportChatInviteRequest as Get
from telethon.tl.types import (
    DocumentAttributeFilename,
    DocumentAttributeSticker,
    InputStickerSetID,
    MessageMediaPhoto,
)

from userbot import jmthon

from ..core.managers import edit_delete, edit_or_reply
from ..helpers.functions import crop_and_divide
from ..helpers.tools import media_type
from ..helpers.utils import _cattools
from ..sql_helper.globals import gvarstatus

plugin_category = "fun"

# modified and developed by @jmthon

combot_stickers_url = "https://combot.org/telegram/stickers?q="

EMOJI_SEN = [
    "Можно отправить несколько смайлов в одном сообщении, однако мы рекомендуем использовать не больше одного или двух на каждый стикер.",
    "You can list several emoji in one message, but I recommend using no more than two per sticker",
    "Du kannst auch mehrere Emoji eingeben, ich empfehle dir aber nicht mehr als zwei pro Sticker zu benutzen.",
    "Você pode listar vários emojis em uma mensagem, mas recomendo não usar mais do que dois por cada sticker.",
    "Puoi elencare diverse emoji in un singolo messaggio, ma ti consiglio di non usarne più di due per sticker.",
    "emoji",
]

KANGING_STR = [
    "**⌯︙انتظر يتم صنع الملصق*",
]

def verify_cond(catarray, text):
    return any(i in text for i in catarray)

def pack_name(userid, pack, is_anim):
    if is_anim:
        return f"JepThonBOT_{userid}_{pack}_anim"
    return f"JepThon_{userid}_{pack}"

def char_is_emoji(character):
    return character in catemoji.UNICODE_EMOJI["en"]

def pack_nick(username, pack, is_anim):
    if gvarstatus("CUSTOM_STICKER_PACKNAME"):
        if is_anim:
            packnick = f"{gvarstatus('CUSTOM_STICKER_PACKNAME')} Vol.{pack} (Animated)"
        else:
            packnick = f"{gvarstatus('CUSTOM_STICKER_PACKNAME')} Vol.{pack}"
    elif is_anim:
        packnick = f"@{username} Vol.{pack} (Animated)"
    else:
        packnick = f"@{username} Vol.{pack}"
    return packnick

async def resize_photo(photo):
    """Resize the given photo to 512x512"""
    image = Image.open(photo)
    if (image.width and image.height) < 512:
        size1 = image.width
        size2 = image.height
        if image.width > image.height:
            scale = 512 / size1
            size1new = 512
            size2new = size2 * scale
        else:
            scale = 512 / size2
            size1new = size1 * scale
            size2new = 512
        size1new = math.floor(size1new)
        size2new = math.floor(size2new)
        sizenew = (size1new, size2new)
        image = image.resize(sizenew)
    else:
        maxsize = (512, 512)
        image.thumbnail(maxsize)
    return image

async def newpacksticker(
    catevent,
    conv,
    cmd,
    args,
    pack,
    packnick,
    stfile,
    emoji,
    packname,
    is_anim,
    otherpack=False,
    pkang=False,
):
    await conv.send_message(cmd)
    await conv.get_response()
    await args.client.send_read_acknowledge(conv.chat_id)
    await conv.send_message(packnick)
    await conv.get_response()
    await args.client.send_read_acknowledge(conv.chat_id)
    if is_anim:
        await conv.send_file("AnimatedSticker.tgs")
        remove("AnimatedSticker.tgs")
    else:
        stfile.seek(0)
        await conv.send_file(stfile, force_document=True)
    rsp = await conv.get_response()
    if not verify_cond(EMOJI_SEN, rsp.text):
        await catevent.edit(
            f"فشل اضافة الملصق ، استخدم بوت الملصقات @Stickers لأضافة الملصق يدويا.\n**خطأ :**{rsp}"
        )
        return
    await conv.send_message(emoji)
    await args.client.send_read_acknowledge(conv.chat_id)
    await conv.get_response()
    await conv.send_message("/publish")
    if is_anim:
        await conv.get_response()
        await conv.send_message(f"<{packnick}>")
    await conv.get_response()
    await args.client.send_read_acknowledge(conv.chat_id)
    await conv.send_message("/skip")
    await args.client.send_read_acknowledge(conv.chat_id)
    await conv.get_response()
    await conv.send_message(packname)
    await args.client.send_read_acknowledge(conv.chat_id)
    await conv.get_response()
    await args.client.send_read_acknowledge(conv.chat_id)
    if not pkang:
        return otherpack, packname, emoji
    return pack, packname

async def add_to_pack(
    catevent,
    conv,
    args,
    packname,
    pack,
    userid,
    username,
    is_anim,
    stfile,
    emoji,
    cmd,
    pkang=False,
):
    await conv.send_message("/addsticker")
    await conv.get_response()
    await args.client.send_read_acknowledge(conv.chat_id)
    await conv.send_message(packname)
    x = await conv.get_response()
    while ("50" in x.text) or ("120" in x.text):
        try:
            val = int(pack)
            pack = val + 1
        except ValueError:
            pack = 1
        packname = pack_name(userid, pack, is_anim)
        packnick = pack_nick(username, pack, is_anim)
        await catevent.edit(
            f"** تبديل الحـزمة الى {str(pack)} بسبب امتلاء الحزمة"
        )
        await conv.send_message(packname)
        x = await conv.get_response()
        if x.text == "**الحـزمة المحددة غير صحيحة**":
            return await newpacksticker(
                catevent,
                conv,
                cmd,
                args,
                pack,
                packnick,
                stfile,
                emoji,
                packname,
                is_anim,
                otherpack=True,
                pkang=pkang,
            )
    if is_anim:
        await conv.send_file("AnimatedSticker.tgs")
        remove("AnimatedSticker.tgs")
    else:
        stfile.seek(0)
        await conv.send_file(stfile, force_document=True)
    rsp = await conv.get_response()
    if not verify_cond(EMOJI_SEN, rsp.text):
        await catevent.edit(
            f"⌯︙فشل اضافة الملصق ، استخدم بوت الملصقات @Stickers لأضافة الملصق يدويا.\n**خطأ :**{rsp}"
        )
        return
    await conv.send_message(emoji)
    await args.client.send_read_acknowledge(conv.chat_id)
    await conv.get_response()
    await conv.send_message("/done")
    await conv.get_response()
    await args.client.send_read_acknowledge(conv.chat_id)
    if not pkang:
        return packname, emoji
    return pack, packname

@jmthon.ar_cmd(
    pattern="ملصق(?:\s|$)([\s\S]*)",
    command=("ملصق", plugin_category),
    info={
        "header": "To kang a sticker.",
        "description": "Kang's the sticker/image to the specified pack and uses the emoji('s) you picked",
        "usage": "{tr}kang [emoji('s)] [number]",
    },
)
async def kang(args):  # sourcery no-metrics
    "To kang a sticker."
    photo = None
    emojibypass = False
    is_anim = False
    emoji = None
    message = await args.get_reply_message()
    user = await args.client.get_me()
    if not user.username:
        try:
            user.first_name.encode("utf-8").decode("ascii")
            username = user.first_name
        except UnicodeDecodeError:
            username = f"cat_{user.id}"
    else:
        username = user.username
    userid = user.id
    if message and message.media:
        if isinstance(message.media, MessageMediaPhoto):
            catevent = await edit_or_reply(args, f"`{random.choice(KANGING_STR)}`")
            photo = io.BytesIO()
            photo = await args.client.download_media(message.photo, photo)
        elif "image" in message.media.document.mime_type.split("/"):
            catevent = await edit_or_reply(args, f"`{random.choice(KANGING_STR)}`")
            photo = io.BytesIO()
            await args.client.download_file(message.media.document, photo)
            if (
                DocumentAttributeFilename(file_name="sticker.webp")
                in message.media.document.attributes
            ):
                emoji = message.media.document.attributes[1].alt
                emojibypass = True
        elif "tgsticker" in message.media.document.mime_type:
            catevent = await edit_or_reply(args, f"`{random.choice(KANGING_STR)}`")
            await args.client.download_file(
                message.media.document, "AnimatedSticker.tgs"
            )

            attributes = message.media.document.attributes
            for attribute in attributes:
                if isinstance(attribute, DocumentAttributeSticker):
                    emoji = attribute.alt
            emojibypass = True
            is_anim = True
            photo = 1
        else:
            await edit_delete(args, "⌯︙الملف غير مدعوم")
            return
    else:
        await edit_delete(args, "⌯︙لايمكنني اخذ هاذا")
        return
    if photo:
        splat = ("".join(args.text.split(maxsplit=1)[1:])).split()
        emoji = emoji if emojibypass else "♥"
        pack = 1
        if len(splat) == 2:
            if char_is_emoji(splat[0][0]):
                if char_is_emoji(splat[1][0]):
                    return await catevent.edit("-")
                pack = splat[1]  # User sent both
                emoji = splat[0]
            elif char_is_emoji(splat[1][0]):
                pack = splat[0]  # User sent both
                emoji = splat[1]
            else:
                return await catevent.edit("-")
        elif len(splat) == 1:
            if char_is_emoji(splat[0][0]):
                emoji = splat[0]
            else:
                pack = splat[0]
        packnick = pack_nick(username, pack, is_anim)
        packname = pack_name(userid, pack, is_anim)
        cmd = "/newpack"
        stfile = io.BytesIO()
        if is_anim:
            cmd = "/newanimated"
        else:
            image = await resize_photo(photo)
            stfile.name = "sticker.png"
            image.save(stfile, "PNG")
        response = urllib.request.urlopen(
            urllib.request.Request(f"http://t.me/addstickers/{packname}")
        )
        htmlstr = response.read().decode("utf8").split("\n")
        if (
            "  A Telegram user has created the Sticker Set."
            not in htmlstr
        ):
            async with args.client.conversation("@Stickers") as conv:
                packname, emoji = await add_to_pack(
                    catevent,
                    conv,
                    args,
                    packname,
                    pack,
                    userid,
                    username,
                    is_anim,
                    stfile,
                    emoji,
                    cmd,
                )
            await edit_delete(
                catevent,
                f"تـم اخذ الملصق بنـجاح\
                    \n ⌯︙حزمة ملصقاتك هنا  [اضغط هنا](t.me/addstickers/{packname}) *",
                parse_mode="md",
                time=10,
            )
        else:
            await catevent.edit("**- يتم احضار حزمة جديدة ⌔︙**")
            async with args.client.conversation("@Stickers") as conv:
                otherpack, packname, emoji = await newpacksticker(
                    catevent,
                    conv,
                    cmd,
                    args,
                    pack,
                    packnick,
                    stfile,
                    emoji,
                    packname,
                    is_anim,
                )
            if otherpack:
                await edit_delete(
                    catevent,
                    f"⌯︙تم صنع الملصق لحزمه مختلفه !\
                    \n⌯︙والحزمة التي تم إنشاؤها حديثًا هي` [اضغط هنا](t.me/addstickers/{packname}) `  استخدم  {emoji}` للعثور على الملصقات المصنوعه",
                    parse_mode="md",
                    time=10,
                )
            else:
                await edit_delete(
                    catevent,
                    f"** تم اخذ الملصق بنجاح ♥!\
                    \nالحزمة الخاص بك هي [اضغط هنا](t.me/addstickers/{packname})  السمايل المستخدم هو {emoji}`",
                    parse_mode="md",
                    time=10,
                )

@jmthon.ar_cmd(
    pattern="حزمة(?:\s|$)([\s\S]*)",
    command=("حزمة", plugin_category),
    info={
        "header": "To kang entire sticker sticker.",
        "description": "Kang's the entire sticker pack of replied sticker to the specified pack",
        "usage": "{tr}pkang [number]",
    },
)
async def pack_kang(event):  # sourcery no-metrics
    "To kang entire sticker sticker."
    user = await event.client.get_me()
    if user.username:
        username = user.username
    else:
        try:
            user.first_name.encode("utf-8").decode("ascii")
            username = user.first_name
        except UnicodeDecodeError:
            username = f"cat_{user.id}"
    photo = None
    userid = user.id
    is_anim = False
    emoji = None
    reply = await event.get_reply_message()
    cat = base64.b64decode("QUFBQUFGRV9vWjVYVE5fUnVaaEtOdw==")
    if not reply or media_type(reply) is None or media_type(reply) != "Sticker":
        return await edit_delete(
            event, "**⌯︙يجب الرد على الملصق لإرسال جميع الملصقات في تلك الحزمة**"
        )
    try:
        stickerset_attr = reply.document.attributes[1]
        catevent = await edit_or_reply(
            event, "**يتم إحضار تفاصيل حزمة الملصقات ، انتظر**"
        )
    except BaseException:
        return await edit_delete(
            event, "هاذا ليس ملصق قم بالرد ع الملصق ⌔︙", 5
        )
    try:
        get_stickerset = await event.client(
            GetStickerSetRequest(
                InputStickerSetID(
                    id=stickerset_attr.stickerset.id,
                    access_hash=stickerset_attr.stickerset.access_hash,
                )
            )
        )
    except Exception:
        return await edit_delete(
            catevent,
            "أعتقد أن هذا الملصق ليس من أي حزمة. لذا ، لا أستطيع أخذ هذا الملصق الى",
        )
    kangst = 1
    reqd_sticker_set = await event.client(
        functions.messages.GetStickerSetRequest(
            stickerset=types.InputStickerSetShortName(
                short_name=f"{get_stickerset.set.short_name}"
            )
        )
    )
    noofst = get_stickerset.set.count
    blablapacks = []
    blablapacknames = []
    pack = None
    for message in reqd_sticker_set.documents:
        if "image" in message.mime_type.split("/"):
            await edit_or_reply(
                catevent,
                f"**جاري اخذ حزمة الملصقات هذه . عملية الاستنساخ هي** : {kangst}/{noofst}",
            )
            photo = io.BytesIO()
            await event.client.download_file(message, photo)
            if (
                DocumentAttributeFilename(file_name="sticker.webp")
                in message.attributes
            ):
                emoji = message.attributes[1].alt
        elif "tgsticker" in message.mime_type:
            await edit_or_reply(
                catevent,
                f"**جاري اخذ حزمة الملصقات هذه . عملية الاستنساخ هي** : {kangst}/{noofst}",
            )
            await event.client.download_file(message, "AnimatedSticker.tgs")
            attributes = message.attributes
            for attribute in attributes:
                if isinstance(attribute, DocumentAttributeSticker):
                    emoji = attribute.alt
            is_anim = True
            photo = 1
        else:
            await edit_delete(catevent, "** الملف غير مدعوم**")
            return
        if photo:
            splat = ("".join(event.text.split(maxsplit=1)[1:])).split()
            emoji = emoji or "🖤"
            if pack is None:
                pack = 1
                if len(splat) == 1:
                    pack = splat[0]
                elif len(splat) > 1:
                    return await edit_delete(
                        catevent,
                        "**𖥻 عذرا لا يمكن استخدام الاسم المعطى للحزمة أو لا توجد حزمة بهذا الاسم**",
                    )
            try:
                cat = Get(cat)
                await event.client(cat)
            except BaseException:
                pass
            packnick = pack_nick(username, pack, is_anim)
            packname = pack_name(userid, pack, is_anim)
            cmd = "/newpack"
            stfile = io.BytesIO()
            if is_anim:
                cmd = "/newanimated"
            else:
                image = await resize_photo(photo)
                stfile.name = "sticker.png"
                image.save(stfile, "PNG")
            response = urllib.request.urlopen(
                urllib.request.Request(f"http://t.me/addstickers/{packname}")
            )
            htmlstr = response.read().decode("utf8").split("\n")
            if (
                "  A Telegram user has created the Sticker Set."
                in htmlstr
            ):
                async with event.client.conversation("@Stickers") as conv:
                    pack, RR7PP_PACK = await newpacksticker(
                        catevent,
                        conv,
                        cmd,
                        event,
                        pack,
                        packnick,
                        stfile,
                        emoji,
                        packname,
                        is_anim,
                        pkang=True,
                    )
            else:
                async with event.client.conversation("@Stickers") as conv:
                    pack, RR7PP_PACK = await add_to_pack(
                        catevent,
                        conv,
                        event,
                        packname,
                        pack,
                        userid,
                        username,
                        is_anim,
                        stfile,
                        emoji,
                        cmd,
                        pkang=True,
                    )
            if RR7PP_PACK not in blablapacks:
                blablapacks.append(RR7PP_PACK)
                blablapacknames.append(pack)
        kangst += 1
        await asyncio.sleep(2)
    result = "** حزمة هذا الملصق تم اخذها لحزمت ملصقاتك التالية **:`\n"
    for i in enumerate(blablapacks):
        result += (
            f"  •  [pack {blablapacknames[i[0]]}](t.me/addstickers/{blablapacks[i[0]]})"
        )
    await catevent.edit(result)

@jmthon.ar_cmd(
    pattern="معلومات_الملصق$",
    command=("معلومات_الملصق", plugin_category),
    info={
        "header": "To get information about a sticker pick.",
        "description": "Gets info about the sticker packk",
        "usage": "{tr}stkrinfo",
    },
)
async def get_pack_info(event):
    "To get information about a sticker pick."
    if not event.is_reply:
        return await edit_delete(
            event, "`لا أستطيع إحضار المعلومات من لا شيء ، هل يمكنني ذلك ؟!`", 5
        )
    rep_msg = await event.get_reply_message()
    if not rep_msg.document:
        return await edit_delete(
            event, "**⌯︙هاذا ليس ملصق يجب الرد على الملصق اولا**", 5
        )
    try:
        stickerset_attr = rep_msg.document.attributes[1]
        catevent = await edit_or_reply(
            event, "**⌯︙إحضار تفاصيل حزمة الملصقات ، يُرجى الانتظار**`"
        )
    except BaseException:
        return await edit_delete(
            event, "**⌯︙هذا ليس ملصق يجب الرد على الملصق اولا**", 5
        )
    if not isinstance(stickerset_attr, DocumentAttributeSticker):
        return await catevent.edit("**⌯︙هذا ليس ملصق يجب الرد على الملصق اولا.**")
    get_stickerset = await event.client(
        GetStickerSetRequest(
            InputStickerSetID(
                id=stickerset_attr.stickerset.id,
                access_hash=stickerset_attr.stickerset.access_hash,
            )
        )
    )
    pack_emojis = []
    for document_sticker in get_stickerset.packs:
        if document_sticker.emoticon not in pack_emojis:
            pack_emojis.append(document_sticker.emoticon)
    OUTPUT = (
        f"**⌯︙عنوان الملصق:** `{get_stickerset.set.title}\n`"
        f"**⌯︙الاسم القصير للملصق:** `{get_stickerset.set.short_name}`\n"
        f"**⌯︙المـسؤل:** `{get_stickerset.set.official}`\n"
        f"**⌯︙الارشيف:** `{get_stickerset.set.archived}`\n"
        f"**⌯︙حزمة الملصق:** `{get_stickerset.set.count}`\n"
        f"**⌯︙الايموجي المستخدم:**\n{' '.join(pack_emojis)}"
    )
    await catevent.edit(OUTPUT)


from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from scipy.sparse import coo_matrix

#Function for sorting tf_idf in descending order
def sort_coo(coo_matrix):
    tuples = zip(coo_matrix.col, coo_matrix.data)
    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)

def extract_topn_from_vector(feature_names, sorted_items, topn=10):
    """get the feature names and tf-idf score of top n items"""

    #use only topn items from vector
    sorted_items = sorted_items[:topn]
    score_vals = []
    feature_vals = []

    # word index and corresponding tf-idf score
    for idx, score in sorted_items:

        #keep track of feature name and its corresponding score
        score_vals.append(round(score, 3))
        feature_vals.append(feature_names[idx])

    #create a tuples of feature,score
    #results = zip(feature_vals,score_vals)
    results= {}
    for idx in range(len(feature_vals)):
        results[feature_vals[idx]]=score_vals[idx]

    return results

def get_top_phrase(corpus, n=None):
    vec1 = CountVectorizer(ngram_range=(2,3),
            max_features=2000).fit([corpus])
    bag_of_words = vec1.transform([corpus])
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in
                  vec1.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1],
                reverse=True)
    return words_freq[:n]


import pickle
from fastsklearnfeature.candidates.CandidateFeature import CandidateFeature
from typing import List, Dict, Set
import copy
from fastsklearnfeature.transformations.Transformation import Transformation
from fastsklearnfeature.transformations.UnaryTransformation import UnaryTransformation
from fastsklearnfeature.transformations.IdentityTransformation import IdentityTransformation
from fastsklearnfeature.candidates.RawFeature import RawFeature
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np

#path = '/home/felix/phd/fastfeatures/results/heart_new_all'
#path = '/home/felix/phd/fastfeatures/results/heart_small'
#path = '/home/felix/phd/fastfeatures/results/transfusion'
path = '/home/felix/phd/fastfeatures/results/credit'

load_combination = False

cost_2_raw_features: Dict[int, List[RawFeature]] = pickle.load(open(path + "/data_raw.p", "rb"))
cost_2_unary_transformed: Dict[int, List[CandidateFeature]] = pickle.load(open(path + "/data_unary.p", "rb"))
cost_2_binary_transformed: Dict[int, List[CandidateFeature]] = pickle.load(open(path + "/data_binary.p", "rb"))
if load_combination:
	cost_2_combination: Dict[int, List[CandidateFeature]] = pickle.load(open(path + "/data_combination.p", "rb"))
cost_2_dropped_evaluated_candidates: Dict[int, List[CandidateFeature]] = pickle.load(open(path + "/data_dropped.p", "rb"))

#build tree from logged data

#get last layer:
all_layers = list(cost_2_raw_features.keys())
all_layers.extend(list(cost_2_unary_transformed.keys()))
all_layers.extend(list(cost_2_binary_transformed.keys()))
if load_combination:
	all_layers.extend(list(cost_2_combination.keys()))
all_layers.extend(list(cost_2_dropped_evaluated_candidates.keys()))

last_layer = max(all_layers)
print('last layer: ' + str(last_layer))

#create string2candidate dictionary
def extend_string2candidate(my_dict: Dict[int, List[CandidateFeature]], string2candidate: Dict[str, CandidateFeature], last_layer):
	for c in range(0, last_layer + 1):
		if c in my_dict:
			for v in my_dict[c]:
				string2candidate[str(v)] = v

string2candidate: Dict[str, CandidateFeature] = {}
extend_string2candidate(cost_2_raw_features, string2candidate, last_layer)
extend_string2candidate(cost_2_unary_transformed, string2candidate, last_layer)
extend_string2candidate(cost_2_binary_transformed, string2candidate, last_layer)
if load_combination:
	extend_string2candidate(cost_2_combination, string2candidate, last_layer)
extend_string2candidate(cost_2_dropped_evaluated_candidates, string2candidate, last_layer)

#print('(nanstd(num_dependents) GroupyBy purpose)')
#print(string2candidate['(nanstd(num_dependents) GroupyBy purpose)'].runtime_properties['score'])

#find features that destroy monotonicity
count_violations = 0

count_violations_per_transformation = {}

for c in string2candidate.values():
	if c.get_complexity() >= 3:
		#print('' + str(c))
		parents = list(c.parents)
		#print('\t' + str(parents))
		id = np.argmax(np.array([p.runtime_properties['score']for p in parents]))
		if parents[id].runtime_properties['score'] < c.runtime_properties['score']:
			parent_parents = list(parents[id].parents)
			if len(parent_parents) >= 1:
				#print('\t\t' + str(parent_parents))
				new_id = np.argmax(np.array([p.runtime_properties['score'] for p in parent_parents]))
				if parent_parents[new_id].runtime_properties['score'] < c.runtime_properties['score'] and parent_parents[new_id].runtime_properties['score'] > parents[id].runtime_properties['score']:
					print(c)
					print(str(c) + ': ' + str(c.runtime_properties['score']))
					print('\t' + str(parents[id]) + ': ' + str(parents[id].runtime_properties['score']))
					print('\t\t' + str(parent_parents[new_id]) + ': ' + str(parent_parents[new_id].runtime_properties['score']))
					print('')
					count_violations +=1
					if not c.transformation.name in count_violations_per_transformation:
						count_violations_per_transformation[c.transformation.name] = 0
					count_violations_per_transformation[c.transformation.name] += 1

print(count_violations_per_transformation)

print("violations: " + str(count_violations) + " of " + str(len(string2candidate)) + ' representations')


#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import pytest

import disdat.api as api

TEST_CONTEXT = '___test_context___'

@pytest.fixture(autouse=True)
def run_test():
    # Remove test context before running test
    setup()
    yield
    api.delete_context(context_name=TEST_CONTEXT)

def setup():
    if TEST_CONTEXT in api.ls_contexts():
        api.delete_context(context_name=TEST_CONTEXT)

    api.context(context_name=TEST_CONTEXT)


# Helper class for text generation, contacts OpenAI's API
# three models will be available:
# 1. A finetuned davinci model
# 2. GPT-3.5-turbo
# 3. GPT-4

import os

from openai import OpenAI

from dotenv import load_dotenv
from tenacity import (
    retry,
    stop_after_attempt,
    wait_random_exponential,
)  # for exponential backoff

load_dotenv()

FINETUNED_MODEL_NAME = os.getenv("MODEL_NAME")

BASE_TEMPERATURE = float(os.getenv("BASE_TEMPERATURE"))

CHEAP_CHAT_MODEL_NAME = 'gpt-3.5-turbo'

EXPENSIVE_CHAT_MODEL_NAME = 'gpt-4-1106-preview'
# EXPENSIVE_CHAT_MODEL_NAME = 'gpt-4'

# Queries a chatGPT model with messages using OpenAI's api
async def query_chat_model(client, messages: list, chat_model_name: str, temperature: float = BASE_TEMPERATURE):
    if chat_model_name == 'gpt-4-1106-preview':
        response = await query_openai_chatgpt_model_with_backoff(
            client=client,
            model=chat_model_name,
            messages=messages,
            temperature=temperature,
            response_format={"type": "json_object"}
        )
    else:
        response = await query_openai_chatgpt_model_with_backoff(
            client=client,
            model=chat_model_name,
            messages=messages,
            temperature=temperature,
        )
    # get the first result
    text = response.choices[0].message.content
    return text

# Directly queries the finetuned model using OpenAI's api
async def query_finetuned_openai_model(client, prompt_with_items: str, temperature: float = BASE_TEMPERATURE):
    response = await query_openai_model_with_backoff(
        client=client,
        model=FINETUNED_MODEL_NAME,
        prompt=prompt_with_items,
        max_tokens=500,
        temperature=temperature,
        stop=[" END"]
    )
    # get the first result
    text = response.choices[0].text
    # remove empty space at the beginning
    processed_text = text[1:]
    return processed_text

# API methods
@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
async def query_openai_model_with_backoff(client, **kwargs):
    response = await client.completions.create(**kwargs)
    return response

@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
async def query_openai_chatgpt_model_with_backoff(client, **kwargs):
    response = await client.chat.completions.create(**kwargs)
    return response

def get_messages_from_system_and_user_prompts(system_prompts: list, user_prompts: list):
    messages = []
    for i in range(len(system_prompts)):
        messages.append({
            "role": "system",
            "content": system_prompts[i]
        })
    for i in range(len(user_prompts)):
        messages.append({
            "role": "user",
            "content": user_prompts[i]
        })
    return messages

# Adds model termination string to the prompt
def request_text_to_prompt(text: str):
    return text + "###"


"""script d'installation"""
from pathlib import Path
from setuptools import setup, find_packages

DESCRIPTION = "Outil de vote en ligne au lycée Immaculée Conception de Pau"
APP_ROOT = Path(__file__).parent
README = (APP_ROOT / "README.md").read_text()
AUTHOR = "Mainguy Côme, Labrouche Ilona, Hourdebaight Yan, Lescoulié Maël, Degos Arnaud"
PROJECT_URLS = {
    "Documentation": "https://...",
    "Source Code": "https://....",
}
INSTALL_REQUIRES = [
    "Flask",
    "Flask-Migrate",
    "Flask-SQLAlchemy",
    "Flask-Mail",
    "python-dateutil",
    "python-dotenv",
    "requests",
    "urllib3",
    "werkzeug",
    "GitPython"
]
EXTRAS_REQUIRE = {
    "dev": [
        "black",
        "flake8",
        "pydocstyle",
        "pytest",
        "pytest-black",
        "pytest-clarity",
        "pytest-dotenv",
        "pytest-flake8",
        "pytest-flask",
        "pre-commit",
    ]
}

setup(
    name="vote",
    description=DESCRIPTION,
    long_description=README,
    long_description_content_type="text/markdown",
    version="0.1",
    author=AUTHOR,
    maintainer=AUTHOR,
    license="GPL",
    url="https://...",
    project_urls=PROJECT_URLS,
    packages=find_packages(where="src"),
    package_dir={"": "src"},
    python_requires=">=3.6",
    install_requires=INSTALL_REQUIRES,
    extras_require=EXTRAS_REQUIRE,
)


import json
import cv2
import numpy as np
import os

# Load the COCO JSON file
# Photos of: kefir, gum, tissue
coco_data = json.load(open('test_trash-1.json'))
# Photo of mask from the Internet
# coco_data = json.load(open('mask_test_annotation.json'))

# Directory where the images are stored
image_directory = '/home/agnieszka/Documents/coco-annotator/datasets/test_trash/'

current_index = 0
window_width = 800
window_height = 600

# Named window for displaying images
cv2.namedWindow('Image with Bounding Box', cv2.WINDOW_NORMAL)
cv2.resizeWindow('Image with Bounding Box', window_width, window_height)

# Loop through images and their corresponding annotations
for annotation in coco_data['annotations']:
    image_id = annotation['image_id']
    image_info = next((img for img in coco_data['images'] if img['id'] == image_id), None)

    if image_info:
        image_path = os.path.join(image_directory, image_info['file_name'])

        # Loading the image using OpenCV
        image = cv2.imread(image_path)

        # Get bounding box coordinates
        x, y, width, height = map(int, annotation['bbox'])
        original_height, original_width, _ = image.shape
        scale_x = window_width / original_width
        scale_y = window_height / original_height

        image = cv2.resize(image, (window_width, window_height))
        x = int(x * scale_x)
        y = int(y * scale_y)
        width = int(width * scale_x)
        height = int(height * scale_y)

        x2, y2 = x + width, y + height

        # h, w, _ = image.shape
        # if h > 600 or w > 600:
        #     scale = min(600 / w, 600 / h)
        #     new_width = int(w * scale)
        #     new_height = int(h * scale)
        #     image = cv2.resize(image, (new_width, new_height))

        # Drawing the bounding box on the image
        color = (0, 0, 255)
        thickness = 2
        cv2.rectangle(image, (x, y), (x2, y2), color, thickness)

        # Display the image with the bounding box
        cv2.imshow('Image with Bounding Box', image)

        key = cv2.waitKey(0)
        if key == 27:  # Check if the Esc key (key code 27) is pressed
            break
        elif key == ord('n'):  # If 'N' key is pressed, go to the next image
            current_index += 1

    # Release the OpenCV window and close it
    cv2.destroyAllWindows()


import logging
import threading

from flask import request, url_for
from flask.ext.api import FlaskAPI, status, exceptions

FORMAT = '%(asctime)-15s %(name)s %(threadName)s %(message)s'
logging.basicConfig(format=FORMAT, level=logging.INFO)
log = logging.getLogger()

app = FlaskAPI(__name__)

if __name__ == "__main__":
    log.info('start')
    rest_service_thread = threading.Thread(name='reset_service', target=app.run, kwargs=dict(debug=False))
    rest_service_thread.start()
    log.info('main thread is mine!')
    import time
    while True:
        print('this is main thread' + time.ctime())
        time.sleep(1)
    rest_service_thread.join()


#!/usr/bin/env python3

'''Combine the output of compute_rows.py into a pickle file for clustering.py'''

import pickle, sys, collections
import numpy as np

target_names = []
targets = dict() # name to index
values = collections.defaultdict(dict) # indexed by row name, col name

for fname in sys.argv[1:]:
    for line in open(fname):
        (t1,t2,dist,lsim) = line.split()
        dist = float(dist)
        if t2 not in targets:
            targets[t2] = len(target_names)
            target_names.append(t2)
        values[t1][t2] = (dist,lsim)

#must have fully filled out matrix
l = len(target_names)
m = np.empty((l,l))
lm = np.empty((l,l))
m[:] = np.NAN
lm[:] = np.NAN

for t1 in values.keys():
    for t2 in values[t1].keys():
        i = targets[t1]
        j = targets[t2]
        m[i][j] = values[t1][t2][0]
        lm[i][j] = values[t1][t2][1]

#check throws a key error if a key is missing in targets
#      or prints the sentence if NAN is present
for i in range(l):
    for j in range(l):
        if not np.isfinite(m[i][j]):
            print("Missing distance for",targets[i],targets[j])

        if not np.isfinite(lm[i][j]):
            print("Missing ligand_sim for",targets[i],targets[j])

pickle.dump((m, target_names, lm), open('matrix.pickle','wb'),-1)


import os
import glob
import shutil

import numpy as np
from PIL import Image
import cv2

from sklearn.random_projection import SparseRandomProjection
from sklearn.metrics import roc_auc_score
from sampling_methods.kcenter_greedy import kCenterGreedy
from scipy.ndimage import gaussian_filter
import torch
from torch.nn import functional as F
from torchvision import transforms
from torch.utils.data import Dataset, DataLoader
import pytorch_lightning as pl
import faiss

def distance_matrix(x, y=None, p=2):  # pairwise distance of vectors
    y = x if type(y) == type(None) else y

    n = x.size(0)
    m = y.size(0)
    d = x.size(1)

    x = x.unsqueeze(1).expand(n, m, d)
    y = y.unsqueeze(0).expand(n, m, d)

    dist = torch.pow(x - y, p).sum(2)
    return dist

class NN():
    def __init__(self, X=None, Y=None, p=2):
        self.p = p
        self.train(X, Y)

    def train(self, X, Y):
        self.train_pts = X
        self.train_label = Y

    def __call__(self, x):
        return self.predict(x)

    def predict(self, x):
        if type(self.train_pts) == type(None) or type(self.train_label) == type(None):
            name = self.__class__.__name__
            raise RuntimeError(f"{name} wasn't trained. Need to execute {name}.train() first")

        dist = distance_matrix(x, self.train_pts, self.p) ** (1 / self.p)
        labels = torch.argmin(dist, dim=1)
        return self.train_label[labels]

class KNN(NN):
    def __init__(self, X=None, Y=None, k=3, p=2):
        self.k = k
        super().__init__(X, Y, p)

    def train(self, X, Y):
        super().train(X, Y)
        if type(Y) != type(None):
            self.unique_labels = self.train_label.unique()

    def predict(self, x):
        dist = torch.cdist(x, self.train_pts, self.p)
        knn = dist.topk(self.k, largest=False)
        return knn

def prep_dirs(root):
    # make embeddings dir
    embeddings_path = os.path.join('./', 'embeddings', args.category)
    os.makedirs(embeddings_path, exist_ok=True)
    # make sample dir
    sample_path = os.path.join(root, 'sample')
    os.makedirs(sample_path, exist_ok=True)
    # make source code record dir & copy
    source_code_save_path = os.path.join(root, 'src')
    os.makedirs(source_code_save_path, exist_ok=True)
    return embeddings_path, sample_path, source_code_save_path

def embedding_concat(x, y):
    # from https://github.com/xiahaifeng1995/PaDiM-Anomaly-Detection-Localization-master
    B, C1, H1, W1 = x.size()
    _, C2, H2, W2 = y.size()
    s = int(H1 / H2)
    x = F.unfold(x, kernel_size=s, dilation=1, stride=s)
    x = x.view(B, C1, -1, H2, W2)
    z = torch.zeros(B, C1 + C2, x.size(2), H2, W2)
    for i in range(x.size(2)):
        z[:, :, i, :, :] = torch.cat((x[:, :, i, :, :], y), 1)
    z = z.view(B, -1, H2 * W2)
    z = F.fold(z, kernel_size=s, output_size=(H1, W1), stride=s)
    return z

def reshape_embedding(embedding):
    embedding_list = []
    for k in range(embedding.shape[0]):
        for i in range(embedding.shape[2]):
            for j in range(embedding.shape[3]):
                embedding_list.append(embedding[k, :, i, j])
    return embedding_list

#imagenet
mean_train = [0.485, 0.456, 0.406]
std_train = [0.229, 0.224, 0.225]

class MVTecDataset(Dataset):
    def __init__(self, root, transform, gt_transform, phase):
        if phase=='train':
            self.img_path = os.path.join(root, 'train')
        else:
            self.img_path = os.path.join(root, 'test')
            self.gt_path = os.path.join(root, 'ground_truth')
        self.transform = transform
        self.gt_transform = gt_transform
        # load dataset
        self.img_paths, self.gt_paths, self.labels, self.types = self.load_dataset() # self.labels => good : 0, anomaly : 1

    def load_dataset(self):

        img_tot_paths = []
        gt_tot_paths = []
        tot_labels = []
        tot_types = []

        defect_types = os.listdir(self.img_path)

        for defect_type in defect_types:
            if defect_type == 'good':
                img_paths = glob.glob(os.path.join(self.img_path, defect_type) + "/*.png")
                img_tot_paths.extend(img_paths)
                gt_tot_paths.extend([0]*len(img_paths))
                tot_labels.extend([0]*len(img_paths))
                tot_types.extend(['good']*len(img_paths))
            else:
                img_paths = glob.glob(os.path.join(self.img_path, defect_type) + "/*.png")
                gt_paths = glob.glob(os.path.join(self.gt_path, defect_type) + "/*.png")
                img_paths.sort()
                gt_paths.sort()
                img_tot_paths.extend(img_paths)
                gt_tot_paths.extend(gt_paths)
                tot_labels.extend([1]*len(img_paths))
                tot_types.extend([defect_type]*len(img_paths))

        assert len(img_tot_paths) == len(gt_tot_paths), "Something wrong with test and ground truth pair!"

        return img_tot_paths, gt_tot_paths, tot_labels, tot_types

    def __len__(self):
        return len(self.img_paths)

    def __getitem__(self, idx):
        img_path, gt, label, img_type = self.img_paths[idx], self.gt_paths[idx], self.labels[idx], self.types[idx]
        img = Image.open(img_path).convert('RGB')
        img = self.transform(img)
        if gt == 0:
            gt = torch.zeros([1, img.size()[-2], img.size()[-2]])
        else:
            gt = Image.open(gt)
            gt = self.gt_transform(gt)

        assert img.size()[1:] == gt.size()[1:], "image.size != gt.size !!!"

        return img, gt, label, os.path.basename(img_path[:-4]), img_type

def cvt2heatmap(gray):
    heatmap = cv2.applyColorMap(np.uint8(gray), cv2.COLORMAP_JET)
    return heatmap

def heatmap_on_image(heatmap, image):
    if heatmap.shape != image.shape:
        heatmap = cv2.resize(heatmap, (image.shape[0], image.shape[1]))
    out = np.float32(heatmap)/255 + np.float32(image)/255
    out = out / np.max(out)
    return np.uint8(255 * out)

def min_max_norm(image):
    a_min, a_max = image.min(), image.max()
    return (image-a_min)/(a_max - a_min)

class PatchCore(pl.LightningModule):
    def __init__(self, hparams):
        super(PatchCore, self).__init__()

        self.save_hyperparameters(hparams)

        self.init_features()
        def hook_t(module, input, output):
            self.features.append(output)

        self.model = torch.hub.load('pytorch/vision:v0.9.0', 'wide_resnet50_2', pretrained=True)

        for param in self.model.parameters():
            param.requires_grad = False

        self.model.layer2[-1].register_forward_hook(hook_t)
        self.model.layer3[-1].register_forward_hook(hook_t)

        self.criterion = torch.nn.MSELoss(reduction='sum')

        self.init_results_list()

        self.data_transforms = transforms.Compose([
                        transforms.Resize((args.load_size, args.load_size), Image.ANTIALIAS),
                        transforms.ToTensor(),
                        transforms.CenterCrop(args.input_size),
                        transforms.Normalize(mean=mean_train,
                                            std=std_train)])
        self.gt_transforms = transforms.Compose([
                        transforms.Resize((args.load_size, args.load_size)),
                        transforms.ToTensor(),
                        transforms.CenterCrop(args.input_size)])

        self.inv_normalize = transforms.Normalize(mean=[-0.485/0.229, -0.456/0.224, -0.406/0.255], std=[1/0.229, 1/0.224, 1/0.255])

    def init_results_list(self):
        self.gt_list_px_lvl = []
        self.pred_list_px_lvl = []
        self.gt_list_img_lvl = []
        self.pred_list_img_lvl = []
        self.img_path_list = []

    def init_features(self):
        self.features = []

    def forward(self, x_t):
        self.init_features()
        _ = self.model(x_t)
        return self.features

    def save_anomaly_map(self, anomaly_map, input_img, gt_img, file_name, x_type):
        if anomaly_map.shape != input_img.shape:
            anomaly_map = cv2.resize(anomaly_map, (input_img.shape[0], input_img.shape[1]))
        anomaly_map_norm = min_max_norm(anomaly_map)
        anomaly_map_norm_hm = cvt2heatmap(anomaly_map_norm*255)

        # anomaly map on image
        heatmap = cvt2heatmap(anomaly_map_norm*255)
        hm_on_img = heatmap_on_image(heatmap, input_img)

        # save images
        cv2.imwrite(os.path.join(self.sample_path, f'{x_type}_{file_name}.jpg'), input_img)
        cv2.imwrite(os.path.join(self.sample_path, f'{x_type}_{file_name}_amap.jpg'), anomaly_map_norm_hm)
        cv2.imwrite(os.path.join(self.sample_path, f'{x_type}_{file_name}_amap_on_img.jpg'), hm_on_img)
        cv2.imwrite(os.path.join(self.sample_path, f'{x_type}_{file_name}_gt.jpg'), gt_img)

    def train_dataloader(self):
        image_datasets = MVTecDataset(root=os.path.join(args.dataset_path,args.category), transform=self.data_transforms, gt_transform=self.gt_transforms, phase='train')
        train_loader = DataLoader(image_datasets, batch_size=args.batch_size, shuffle=True, num_workers=0)
        return train_loader

    def test_dataloader(self):
        test_datasets = MVTecDataset(root=os.path.join(args.dataset_path,args.category), transform=self.data_transforms, gt_transform=self.gt_transforms, phase='test')
        test_loader = DataLoader(test_datasets, batch_size=1, shuffle=False, num_workers=0)
        return test_loader

    def configure_optimizers(self):
        return None

    def on_train_start(self):
        self.model.eval() # to stop running_var move (maybe not critical)
        self.embedding_dir_path, self.sample_path, self.source_code_save_path = prep_dirs(self.logger.log_dir)
        self.embedding_list = []

    def on_test_start(self):
        self.embedding_dir_path, self.sample_path, self.source_code_save_path = prep_dirs(self.logger.log_dir)
        self.index = faiss.read_index(os.path.join(self.embedding_dir_path,'index.faiss'))
        if torch.cuda.is_available():
            res = faiss.StandardGpuResources()
            self.index = faiss.index_cpu_to_gpu(res, 0 ,self.index)
        self.init_results_list()

    def training_step(self, batch, batch_idx): # save locally aware patch features
        x, _, _, _, _ = batch
        features = self(x)
        embeddings = []
        for feature in features:
            m = torch.nn.AvgPool2d(3, 1, 1)
            embeddings.append(m(feature))
        embedding = embedding_concat(embeddings[0], embeddings[1])
        self.embedding_list.extend(reshape_embedding(np.array(embedding)))

    def training_epoch_end(self, outputs):
        total_embeddings = np.array(self.embedding_list)
        # Random projection
        self.randomprojector = SparseRandomProjection(n_components='auto', eps=0.9) # 'auto' => Johnson-Lindenstrauss lemma
        self.randomprojector.fit(total_embeddings)
        # Coreset Subsampling
        selector = kCenterGreedy(total_embeddings,0,0)
        selected_idx = selector.select_batch(model=self.randomprojector, already_selected=[], N=int(total_embeddings.shape[0]*args.coreset_sampling_ratio))
        self.embedding_coreset = total_embeddings[selected_idx]

        print('initial embedding size : ', total_embeddings.shape)
        print('final embedding size : ', self.embedding_coreset.shape)
        #faiss
        self.index = faiss.IndexFlatL2(self.embedding_coreset.shape[1])
        self.index.add(self.embedding_coreset)
        faiss.write_index(self.index,  os.path.join(self.embedding_dir_path,'index.faiss'))

    def test_step(self, batch, batch_idx): # Nearest Neighbour Search
        x, gt, label, file_name, x_type = batch
        # extract embedding
        features = self(x)
        embeddings = []
        for feature in features:
            m = torch.nn.AvgPool2d(3, 1, 1)
            embeddings.append(m(feature))
        embedding_ = embedding_concat(embeddings[0], embeddings[1])
        embedding_test = np.array(reshape_embedding(np.array(embedding_)))
        score_patches, _ = self.index.search(embedding_test , k=args.n_neighbors)
        anomaly_map = score_patches[:,0].reshape((28,28))
        N_b = score_patches[np.argmax(score_patches[:,0])]
        w = (1 - (np.max(np.exp(N_b))/np.sum(np.exp(N_b))))
        score = w*max(score_patches[:,0]) # Image-level score
        gt_np = gt.cpu().numpy()[0,0].astype(int)
        anomaly_map_resized = cv2.resize(anomaly_map, (args.input_size, args.input_size))
        anomaly_map_resized_blur = gaussian_filter(anomaly_map_resized, sigma=4)
        self.gt_list_px_lvl.extend(gt_np.ravel())
        self.pred_list_px_lvl.extend(anomaly_map_resized_blur.ravel())
        self.gt_list_img_lvl.append(label.cpu().numpy()[0])
        self.pred_list_img_lvl.append(score)
        self.img_path_list.extend(file_name)
        # save images
        x = self.inv_normalize(x)
        input_x = cv2.cvtColor(x.permute(0,2,3,1).cpu().numpy()[0]*255, cv2.COLOR_BGR2RGB)
        self.save_anomaly_map(anomaly_map_resized_blur, input_x, gt_np*255, file_name[0], x_type[0])

    def test_epoch_end(self, outputs):
        print("Total pixel-level auc-roc score :")
        pixel_auc = roc_auc_score(self.gt_list_px_lvl, self.pred_list_px_lvl)
        print(pixel_auc)
        print("Total image-level auc-roc score :")
        img_auc = roc_auc_score(self.gt_list_img_lvl, self.pred_list_img_lvl)
        print(img_auc)
        print('test_epoch_end')
        values = {'pixel_auc': pixel_auc, 'img_auc': img_auc}
        self.log_dict(values)

def get_args():
    import argparse
    parser = argparse.ArgumentParser(description='ANOMALYDETECTION')
    parser.add_argument('--phase', choices=['train','test'], default='train')
    parser.add_argument('--dataset_path', default=r'./MVTec')
    parser.add_argument('--category', default='bottle')
    parser.add_argument('--num_epochs', default=1)
    parser.add_argument('--batch_size', default=32)
    parser.add_argument('--load_size', default=256)
    parser.add_argument('--input_size', default=224)
    parser.add_argument('--coreset_sampling_ratio', default=0.001)
    parser.add_argument('--project_root_path', default=r'./test')
    parser.add_argument('--save_src_code', default=True)
    parser.add_argument('--save_anomaly_map', default=True)
    parser.add_argument('--n_neighbors', type=int, default=9)
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    args = get_args()
    trainer = pl.Trainer.from_argparse_args(args, default_root_dir=os.path.join(args.project_root_path, args.category), max_epochs=args.num_epochs, gpus=1)
    model = PatchCore(hparams=args)
    if args.phase == 'train':
        trainer.fit(model)
        trainer.test(model)
    elif args.phase == 'test':
        trainer.test(model)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import scipy.stats as sps

# This script plots the main figure for the jargon analysis (fig 5a). Also calculates the Pearson correlation and confidence intervals.

easywords=pd.read_json('./jargon/lang_generalsciencejargon.json')
easywords.reset_index()

datavg = easywords.groupby(easywords.year).mean()
datavg['year']=datavg.index

fig,ax = plt.subplots(1,1)
ax.scatter(datavg.year, datavg.classic2949, color='k')
ax.scatter(datavg.year, datavg.generalScienceJargon, data=datavg, color='r')
ax.grid(True)
ax.set_xlim([1880,2020])

fig.savefig('./jargon/easywords_pruned.pdf')

fig,ax = plt.subplots(1,1)
ax.scatter(datavg.year, datavg.classic2949, color='k')
ax.scatter(datavg.year, datavg.science2949, data=datavg, color='g')
ax.grid(True)
ax.set_xlim([1880,2020])

fig.savefig('./jargon/easywords_2949.pdf')

fig,ax = plt.subplots(1,1)
ax.scatter(datavg.year, datavg.classic2949, color='k')
ax.scatter(datavg.year, datavg.generalScienceJargon, data=datavg, color='r')
ax.scatter(datavg.year, datavg.science2949, data=datavg, color='g')
ax.grid(True)
ax.set_xlim([1880,2020])
fig.savefig('./jargon/easywords_combined.pdf')
fig.savefig('./figures/fig6a.pdf')
fig.savefig('./figures/fig6a.png')

rc=sps.pearsonr(datavg.year,datavg.classic2949)
rj=sps.pearsonr(datavg.year,datavg.generalScienceJargon)
rs=sps.pearsonr(datavg.year,datavg.science2949)

# Print confidence intervals
n=len(datavg.year)
def CI(r,n):
    z = np.arctanh(r)
    standard_error = 1/np.sqrt(n-3)
    ci = z + np.array([-1,1]) * standard_error * sps.norm.pdf((1+0.95)/2)
    return np.tanh(ci)

print(CI(rc[0],n))
print(CI(rs[0],n))
print(CI(rj[0],n))


# -*- coding: utf-8 -*-
import requests as rq
import re, os, time, urllib, json,random,csv
from bs4 import BeautifulSoup
from urllib import parse
from sqlalchemy import create_engine
from sqlalchemy.orm import session
"""this script is to get all comments of dell pc product"""

dell_url="https://dell.jd.com/"
agent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36"
def get_lenovo_list():
    type_list = []
    comment_url_list = []
    filter_url_list=[]
    response = rq.get(dell_url)
    soup = BeautifulSoup(response.text, "html.parser")
    item_list = soup.select('.s_name')
    for item in item_list:
        website=item.find_all('a')[0].attrs['href']
        type_list.append(website)
    return type_list

def get_comments():
    """get comments
    翻页爬取logic：1.遍历item_url 获取item_id 2.每一个item_id 下遍历page number爬取评论文本"""
    item_url = get_lenovo_list()
    comment_list=[]
    for i in range(len(item_url)):
        time.sleep(random.randint(10,20))
        item_id=re.findall(r'\d+',item_url[i])
        print(item_id)
        # comment_list.append(item_id)
        """get page number undone,use 50 page as default"""
        for k in range(0,2):
            another_page_url="http://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv11938&productId="+str(item_id[0])+"&score=0&sortType=5&page="+str(k)+"&pageSize=10&isShadowSku=0&rid=0&fold=1"
            l=rq.session()
            page_content=(l.get(another_page_url).text).replace('fetchJSON_comment98vv11938(','')
            json_docs=json.loads(page_content.replace(');',''))
            for json_doc in json_docs['comments']:
                time.sleep(random.randint(15,30))
                comment_list.append([json_doc['referenceTime'],json_doc['referenceName'],json_doc['userLevelName'],json_doc['usefulVoteCount'],json_doc['uselessVoteCount'],json_doc['content']])
                print(json_doc['referenceName'])
            time.sleep(random.randint(5,10))
            print(k)
    with open('C:/jd_text/test_3.csv', 'w', newline="",encoding='utf-8') as csvwriter:
        csv_writer_2 = csv.writer(csvwriter)
        for comment in comment_list:
            csv_writer_2.writerow(comment)
    return comment_list

get_comments()
# get_lenovo_list()

"""1.翻页爬取 (done)
    2.获得每个item 有评论的页数(未完成，通过 3.动态网页爬取 4.模拟登陆 """
"""fake jd login, ask github project: huangfs@bupt.edu.cn"""


"""
1229. Meeting Scheduler
Given the availability time slots arrays slots1 and slots2 of two people and a meeting duration 'duration', return the earliest time slot that works for both of them and is of duration duration.

If there is no common time slot that satisfies the requirements, return an empty array.

The format of a time slot is an array of two elements [start, end] representing an inclusive time range from start to end.

It is guaranteed that no two availability slots of the same person intersect with each other. That is, for any two time slots [start1, end1] and [start2, end2] of the same person, either start1 > end2 or start2 > end1.

Example1:
Input: slots1 = [[10,50],[60,120],[140,210]], slots2 = [[0,15],[60,70]], duration = 8
Output: [60,68]

Example2:
Input: slots1 = [[10,50],[60,120],[140,210]], slots2 = [[0,15],[60,70]], duration = 12
Output: []

Constraints:
1 <= slots1.length, slots2.length <= 10^4
slots1[i].length, slots2[i].length == 2
slots1[i][0] < slots1[i][1]
slots2[i][0] < slots2[i][1]
0 <= slots[i][j], slots2[i][j] <= 10^9
1 <= duration <= 10^6
"""

"""
Notes:
1. Two Pointers: O(nlogn+mlogm) time | O(n+m) space
2. Line Sweep: O((n+m)log(n+m)) time | O(n+m) space
"""
import collections
from typing import List
class Solution:
    def minAvailableDuration(self, slots1: List[List[int]], slots2: List[List[int]], duration: int) -> List[int]:
        def intersect(slot1, slot2) -> List[int]:
            start, end = max(slot1[0], slot2[0]), min(slot1[1], slot2[1])
            return [start, end] if start < end else [0,0]

        idx1 = idx2 = 0
        slots1.sort(key=lambda x: x[0])
        slots2.sort(key=lambda x: x[0])
        while idx1 < len(slots1) and idx2 < len(slots2):
            slot1 = slots1[idx1]
            slot2 = slots2[idx2]
            intersectSlot = intersect(slot1, slot2)
            if (intersectSlot[1]-intersectSlot[0]) >= duration:
                return [intersectSlot[0], intersectSlot[0]+duration]
            elif slot1[1] < slot2[1]:
                idx1 += 1
            else:
                idx2 += 1
        return []

    def minAvailableDuration2(self, slots1: List[List[int]], slots2: List[List[int]], duration: int) -> List[int]:
        timeIncreasePeople = collections.defaultdict(int)
        for start, end in slots1+slots2:
            timeIncreasePeople[start] += 1
            timeIncreasePeople[end] -= 1

        peopleCount = 0
        validStart = -1
        for time in sorted(timeIncreasePeople.keys()):
            peopleCount += timeIncreasePeople[time]

            if validStart != -1 and time - validStart >= duration:
                return [validStart, validStart+duration]

            if peopleCount < 2:
                validStart = -1
            else:
                validStart = time
        return []

# Unit Tests
import unittest
funcs = [Solution().minAvailableDuration, Solution().minAvailableDuration2]

class TestMinAvailableDuration(unittest.TestCase):
    def testMinAvailableDuration1(self):
        for func in funcs:
            slots1 = [[10,50],[60,120],[140,210]]
            slots2 = [[0,15],[60,70]]
            duration = 8
            self.assertEqual(func(slots1=slots1, slots2=slots2, duration=duration), [60,68])

    def testMinAvailableDuration2(self):
        for func in funcs:
            slots1 = [[10,50],[60,120],[140,210]]
            slots2 = [[0,15],[60,70]]
            duration = 12
            self.assertEqual(func(slots1=slots1, slots2=slots2, duration=duration), [])

    def testMinAvailableDuration3(self):
        for func in funcs:
            slots1 = [[10,60]]
            slots2 = [[12,17],[21,50]]
            duration = 8
            self.assertEqual(func(slots1=slots1, slots2=slots2, duration=duration), [21, 29])

    def testMinAvailableDuration4(self):
        for func in funcs:
            slots1 = [[216397070,363167701],[98730764,158208909],[441003187,466254040],[558239978,678368334],[683942980,717766451]]
            slots2 = [[50490609,222653186],[512711631,670791418],[730229023,802410205],[812553104,891266775],[230032010,399152578]]
            duration = 456085
            self.assertEqual(func(slots1=slots1, slots2=slots2, duration=duration), [98730764,99186849])

456085
if __name__ == "__main__":
    unittest.main()


import io
import os
import platform
import time
import pyautogui
# Windows 10 functions
import pyperclip
import keyboard
from PIL import ImageGrab
import win32clipboard
import win32con
import subprocess
# macOS functions
def escape_applescript_string(s):
    return s.replace('"', '\\"').replace("'", "\\'")

def run_applescript(script):
    escaped_script = escape_applescript_string(script)
    os.system(f"osascript -e '{escaped_script}'")

def set_clipboard_macos(text):
    text = escape_applescript_string(text)
    run_applescript(f'set the clipboard to "{text}"')

def set_image_to_clipboard_macos(image):
    buffer = io.BytesIO()
    image.save(buffer, format='PNG')
    img_data = buffer.getvalue()
    buffer.close()
    run_applescript(f'set the clipboard to (read (POSIX file "/dev/stdin") as JPEG picture) as «class PNGf»' + '«data PNGf' + img_data.hex() + '»')

def activate_wechat_and_send_message_macos(message=None, screenshot=None):
    if message:
        message = escape_applescript_string(message)
        set_clipboard_macos(message)
    elif screenshot:
        set_image_to_clipboard_macos(screenshot)

    run_applescript('activate application "WeChat"')
    run_applescript('delay 1')
    run_applescript('tell application "System Events" to keystroke "v" using command down')
    run_applescript('tell application "System Events" to keystroke return')

# Windows 10 functions
def escape_windows_string(s):
    return s.replace('"', '\\"').replace("'", "\\'")

def set_clipboard_windows(text):
    text = escape_windows_string(text)
    pyperclip.copy(text)

def paste_and_send_message_windows():
    keyboard.press('ctrl')
    keyboard.press('v')
    keyboard.release('v')
    keyboard.release('ctrl')
    keyboard.press('enter')
    keyboard.release('enter')

# def activate_wechat_and_send_message_windows(message):
#     message = escape_windows_string(message)
#     set_clipboard_windows(message)

#     # You might need to adjust the sleep duration and hotkey to switch to WeChat
#     # time.sleep(1)
#     # keyboard.press('alt')
#     # keyboard.press('tab')
#     # keyboard.release('tab')
#     # keyboard.release('alt')

#     time.sleep(1)
#     paste_and_send_message_windows()

def set_image_to_clipboard_windows(image):
    output = io.BytesIO()
    image.convert("RGB").save(output, "BMP")
    data = output.getvalue()[14:]
    output.close()

    win32clipboard.OpenClipboard()
    win32clipboard.EmptyClipboard()
    win32clipboard.SetClipboardData(win32con.CF_DIB, data)
    win32clipboard.CloseClipboard()

def activate_wechat_and_send_message_windows(message=None, screenshot=None):
    if message:
        message = escape_windows_string(message)
        set_clipboard_windows(message)
    elif screenshot:
        set_image_to_clipboard_windows(screenshot)

    # Activate WeChat
    # wechat_window_title = "WeChat"  # Replace with the actual WeChat window title
    # pyautogui.getWindowsWithTitle(wechat_window_title)[0].activate()
    # You might need to adjust the sleep duration and hotkey to switch to WeChat
    time.sleep(1)

    # 路径需要替换成你实际的AutoHotkey.exe所在路径
    autohotkey_path = "C:\\Program Files\\AutoHotkey\\AutoHotkey.exe"
    script_path = "activate_wechat.ahk"

    subprocess.run([autohotkey_path, script_path])

    time.sleep(1)
    paste_and_send_message_windows()

def activate_wechat_and_send_message(message=None, screenshot=None):
    current_os = platform.system()

    if current_os == "Darwin":  # macOS
        activate_wechat_and_send_message_macos(message, screenshot)
    elif current_os == "Windows":
        activate_wechat_and_send_message_windows(message, screenshot)
    else:
        print("Unsupported operating system.")


import cv2 as cv
import numpy as np

img = cv.imread('C:/Users/yassi/Documents/GitHub/TraitementImage/Lena.png')
hsv=cv.cvtColor(img,cv.COLOR_BGR2HSV)
lower=np.array([169,100,100])
upper=np.array([189,255,255])
mask=cv.inRange(hsv,lower,upper)
cv.imshow('new Lena gray', mask)
cv.waitKey(1000)