import json

class Question:
    def __init__(self, text: str, answers: list, right_answer: int):
        self.right_answer = right_answer
        self.answers = answers
        self.text = text

    def to_json(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

    def from_json(json_str: str):
        loaded_json = json.loads(json_str)
        text = loaded_json['text']
        answers = loaded_json['answers']
        right_answer = loaded_json['right_answer']

        question = Question(text, answers, right_answer)
        return question

if __name__ == '__main__':
    json_str = '{"text": "asd", "answers": [1, 2, 3], "right_answer": 1}'
    a = Question.from_json(json_str)


from typing import List

class Solution:
    def containsDuplicate(self, numbers: List[int]) -> bool:
        return self.__method1(numbers)

    # 基于哈希表的解法
    def __method1(self, numbers: List[int]) -> bool:
        d = {}
        for n in numbers:
            if n in d:
                return True
            else:
                d[n] = 1
        return False

    # 基于排序和双指针的解法
    def __method2(self, numbers: List[int]) -> bool:
        length = len(numbers)
        if length == 1:
            return False

        numbers = sorted(numbers)
        i, j = 0, 1
        while j < length:
            if numbers[i] == numbers[j]:
                return True
            i += 1
            j += 1
        return False


#!/usr/bin/env python3

# This code jam solution is powered by Nathan West's LibCodeJam; see
# https://github.com/Lucretiel/LibCodeJam for source code and (ostensibly) some
# documentation.

from code_jam import *
from functools import lru_cache

# code_jam.INSERT_NEWLINE = True

# quick reference:
#   @autosolve, @collects, @cases(n)
#   tokens.token(t), tokens.many(n, t)
#   debug(expr), @unroll(t)gen
#   solve(int_token: int, list_token: ('int_token', str), other_list: (None, float)):

G = '+'
B = '-'

@lru_cache()
@unroll(''.join)
def flip(stack):
    for i in reversed(stack):
        yield B if i == G else G

@lru_cache()
@unroll(''.join)
def flip_head(stack):
    stack = iter(stack)

    while next(stack) == G:
        yield B

    yield B
    yield from stack

def num_flips(stack):
    stack = stack.rstrip(G)

    if not stack:
        return 0

    flips = 0

    if stack[0] == G:
        stack = flip_head(stack)
        flips += 1

    stack = flip(stack)
    flips += 1

    return flips + num_flips(stack)

@autosolve
@collects
def solve(stack: str):
    return num_flips(stack)


import matplotlib.pyplot as plt
from mydataset import load_data, DataSetType
from torch.utils.data import DataLoader
import math
# Ignore warnings
import warnings
warnings.filterwarnings("ignore")

dataset = load_data("CLEVR", DataSetType.Train)

print("Number of images = %r" % len(dataset))

print("Shape = ", dataset[-1].shape)

def multiplot(figures, cols: int=3):
    rows = math.ceil(len(figures)/cols)
    fig, ax = plt.subplots(rows, cols)
    c = 0
    r = 0
    for f in figures:
        ax[r,c].imshow(f.permute(1,2,0))
        c += 1
        if c==cols:
            r += 1
            c = 0
    plt.show()

figs = [dataset[i] for i in range(0,9)]
multiplot(figs)


from sklearn.cluster import KMeans
from sklearn.mixture import GaussianMixture
import pandas as pd
import numpy as np
import matplotlib.pyplot as a
data = pd.read_csv('ds8.csv')
f1 =data['Distance_Feature']
f2=data['Speeding_Feature']
x =np.array(list(zip(f1,f2)))
a.scatter(f1,f2,color='black')
a.title('Real')
a.ylabel('speeding_feature')
a.xlabel('distance_feature')
a.show()
kmeans = KMeans(3).fit(x)
labels = kmeans.predict(x)
a.scatter(f1,f2,c=labels)
a.title('KMeans')
a.ylabel('speeding_feature')
a.xlabel('distance_feature')
a.show()
gm = GaussianMixture(3).fit(x)
labels = gm.predict(x)
a.scatter(f1,f2,c=labels)
a.title('GMM Classification')
a.ylabel('speeding_feature')
a.xlabel('distance_feature')
a.show()

#Dataset
Driver_ID,Distance_Feature,Speeding_Feature
3423311935,71.24,28.0
3423313212,52.53,25.0
3423313724,64.54,78.0
3423311373,55.69,22.0
3423310999,54.58,25.0
3423319935,90.24,28.0
3423318212,82.53,25.0
3423317724,6.54,26.0
3423311363,65.69,32.0
3423310929,44.58,25.0


"""
OpenAI (Tensorflow): https://github.com/openai/gpt-2/blob/master/src/model.py
Huggingface (Pytorch): https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/modeling_gpt2.py
NanoGPT (Pytorch): https://github.com/karpathy/nanoGPT/blob/master/model.py
"""

import math
import torch
import torch.nn as nn
import torch.nn.functional as F

from dataclasses import dataclass
from from_scratch.optim.sophia import SophiaG

optimizer_dict = {'adamw': torch.optim.AdamW,
                  'sophiag': SophiaG
                 }

@dataclass
class GPTConfig:
    block_size: int = 1024
    vocab_size: int = 50304 # GPT has 50257 vocab
    n_layer: int = 12
    n_head: int = 12
    n_embd: int = 768
    dropout: float = 0.0
    bias: bool = True

class LayerNorm(nn.Module):
    def __init__(self, ndim: int, bias: bool) -> None:
        super().__init__()
        self.weight =nn.Parameter(torch.ones(ndim))
        self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None

    def forward(self, input):
        return F.layer_norm(input, self.weight.size(), self.weight, self.bias, eps=1e-5)

class CausalSelfAttention(nn.Module):
    def __init__(self, config: GPTConfig) -> None:
        super().__init__()
        assert config.n_embd % config.n_head == 0
        # MultiHead
        self.c_attn = nn.Linear(config.n_embd, 3*config.n_embd, bias=config.bias)
        # Output Projection
        self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
        # Regularization
        self.attn_dropout = nn.Dropout(config.dropout)
        self.proj_dropout = nn.Dropout(config.dropout)

        self.n_head = config.n_head
        self.n_embd = config.n_embd
        self.dropout = config.dropout

        self.flash = hasattr(F, 'scaled_dot_product_attention')
        if not self.flash:
            print("Fash attention require Pytorch >= 2.0. Using slow attention")
            self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size))
                                              .view(1, 1, config.block_size, config.block_size))

    def forward(self, x):
        B, T, C = x.size() # batch_size, seq_len, emb_dim

        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)

        if self.flash:
            y = F.scaled_dot_product_attention(q, k, v,
                                           attn_mask=None,
                                           dropout_p=self.dropout if self.training else 0.0,
                                           is_causal=True)
        else:
            attn = (q @ k.transpose(-2, -1) * (1.0 / math.sqrt(k.size(-1))))
            attn = attn.masked_fill(self.bias[:,:,:T,:T]==0, float('-inf'))
            attn = F.softmax(attn, dim=-1)
            attn = self.attn_dropout(attn)
            y = attn @ v

        y = y.transpose(1, 2).contiguous().view(B, T, C)
        y = self.proj_dropout(self.c_proj(y))
        return y

class GPT2MLP(nn.Module):
    def __init__(self, config: GPTConfig) -> None:
        super().__init__()
        self.c_fc = nn.Linear(config.n_embd, 4*config.n_embd, bias=config.bias)
        self.gelu = nn.GELU()
        self.c_proj = nn.Linear(4*config.n_embd, config.n_embd, bias=config.bias)
        self.dropout = nn.Dropout(config.dropout)

    def forward(self, x):
        x = self.c_fc(x)
        x = self.gelu(x)
        x = self.c_proj(x)
        x = self.dropout(x)
        return x

class GPT2Block(nn.Module):
    def __init__(self,
                 config: GPTConfig
                 ) -> None:
        super().__init__()
        self.ln_1 = LayerNorm(config.n_embd, bias=config.bias)
        self.attn = CausalSelfAttention(config)
        self.ln_2 = LayerNorm(config.n_embd, bias=config.bias)
        self.mlp = GPT2MLP(config)

    def forward(self, x):
        x = x + self.attn(self.ln_1(x))
        x = x + self.mlp(self.ln_2(x))
        return x

class GPT2Model(nn.Module):
    def __init__(self, config: GPTConfig) -> None:
        super().__init__()
        self.config = config
        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(config.vocab_size, config.n_embd),
            wpe = nn.Embedding(config.block_size, config.n_embd),
            emb_drop = nn.Dropout(config.dropout),
            h = nn.ModuleList([GPT2Block(config) for _ in range(config.n_layer)]),
            ln_f = LayerNorm(config.n_embd, bias=config.bias)
        ))
        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
        self.transformer.wte.weight = self.lm_head.weight

        self.apply(self._init_weight)
        for pn, p in self.named_parameters():
            if pn.endswith('c_proj.weight'):
                nn.init.normal_(p, mean=0.0, std=0.02/math.sqrt(2*config.n_layer))

        print("number of parameters: %.2fM" % (self.get_num_params()/1e6))

    def get_num_params(self, non_embedding=True):
        n_params = sum(p.numel() for p in self.parameters())
        if non_embedding:
            n_params -= self.transformer.wpe.weight.numel()
        return n_params

    def _init_weight(self, module):
        if isinstance(module, nn.Linear):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)
            if module.bias is not None:
                nn.init.constant_(module.bias, 0.0)
        elif isinstance(module, nn.Embedding):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)

    def forward(self, input_ids, targets=None):
        device = input_ids.device
        b, t = input_ids.size()
        assert t <= self.config.block_size, f"Cannot forward sequence of length {t}, block size is only {self.config.block_size}"

        pos =  torch.arange(0, t, dtype=torch.long, device=device)

        token_embd = self.transformer.wte(input_ids)
        pos_embd = self.transformer.wpe(pos)
        x = self.transformer.emb_drop(token_embd + pos_embd)

        for block in self.transformer.h:
            x = block(x)
        x = self.transformer.ln_f(x)

        loss = None
        if targets is not None:
            logits =self.lm_head(x)
            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
        else:
            # only forward the lm_head on the very last position
            logits = self.lm_head(x[:, [-1], :])

        return logits, loss

    def crop_block_size(self, block_size):
        assert block_size <= self.config.block_size
        self.config.block_size = block_size
        self.transformer.wpe.weight = nn.Parameter(self.transformer.wpe.weight[:block_size])
        for block in self.transformer.h:
            if hasattr(block.attn, 'bias'):
                block.attn.bias = block.attn.bias[:,:,:block_size,:block_size]

    @classmethod
    def from_pretrained(cls, model_type, override_args=None):
        assert model_type in {'gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'}
        override_args = override_args or {} # default to empty dict
        # only dropout can be overridden see more notes below
        assert all(k == 'dropout' for k in override_args)
        from transformers import GPT2LMHeadModel
        print("loading weights from pretrained gpt: %s" % model_type)

        # n_layer, n_head and n_embd are determined from model_type
        config_args = {
            'gpt2':         dict(n_layer=12, n_head=12, n_embd=768),  # 124M params
            'gpt2-medium':  dict(n_layer=24, n_head=16, n_embd=1024), # 350M params
            'gpt2-large':   dict(n_layer=36, n_head=20, n_embd=1280), # 774M params
            'gpt2-xl':      dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params
        }[model_type]
        print("forcing vocab_size=50257, block_size=1024, bias=True")
        config_args['vocab_size'] = 50257 # always 50257 for GPT model checkpoints
        config_args['block_size'] = 1024 # always 1024 for GPT model checkpoints
        config_args['bias'] = True # always True for GPT model checkpoints
        # we can override the dropout rate, if desired
        if 'dropout' in override_args:
            print(f"overriding dropout rate to {override_args['dropout']}")
            config_args['dropout'] = override_args['dropout']
        # create a from-scratch initialized minGPT model
        config = GPTConfig(**config_args)
        model = GPT2Model(config)
        sd = model.state_dict()
        sd_keys = sd.keys()
        sd_keys = [k for k in sd_keys if not k.endswith('.attn.bias')] # discard this mask / buffer, not a param

        # init a huggingface/transformers model
        model_hf = GPT2LMHeadModel.from_pretrained(model_type)
        sd_hf = model_hf.state_dict()

        # copy while ensuring all of the parameters are aligned and match in names and shapes
        sd_keys_hf = sd_hf.keys()
        sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.masked_bias')] # ignore these, just a buffer
        sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.bias')] # same, just the mask (buffer)
        transposed = ['attn.c_attn.weight', 'attn.c_proj.weight', 'mlp.c_fc.weight', 'mlp.c_proj.weight']
        # basically the openai checkpoints use a "Conv1D" module, but we only want to use a vanilla Linear
        # this means that we have to transpose these weights when we import them
        assert len(sd_keys_hf) == len(sd_keys), f"mismatched keys: {len(sd_keys_hf)} != {len(sd_keys)}"
        for k in sd_keys_hf:
            if any(k.endswith(w) for w in transposed):
                # special treatment for the Conv1D weights we need to transpose
                assert sd_hf[k].shape[::-1] == sd[k].shape
                with torch.no_grad():
                    sd[k].copy_(sd_hf[k].t())
            else:
                # vanilla copy over the other parameters
                assert sd_hf[k].shape == sd[k].shape
                with torch.no_grad():
                    sd[k].copy_(sd_hf[k])

        return model

    def configure_optimizers(self, weight_decay, learning_rate, betas, device_type):
        # start with all of the candidate parameters
        param_dict = {pn: p for pn, p in self.named_parameters()}
        # filter out those that do not require grad
        param_dict = {pn: p for pn, p in param_dict.items() if p.requires_grad}
        # create optim groups. Any parameters that is 2D will be weight decayed, otherwise no.
        # i.e. all weight tensors in matmuls + embeddings decay, all biases and layernorms don't.
        decay_params = [p for n, p in param_dict.items() if p.dim() >= 2]
        nodecay_params = [p for n, p in param_dict.items() if p.dim() < 2]
        optim_groups = [
            {'params': decay_params, 'weight_decay': weight_decay},
            {'params': nodecay_params, 'weight_decay': 0.0}
        ]
        num_decay_params = sum(p.numel() for p in decay_params)
        num_nodecay_params = sum(p.numel() for p in nodecay_params)
        print(f"num decayed parameter tensors: {len(decay_params)}, with {num_decay_params:,} parameters")
        print(f"num non-decayed parameter tensors: {len(nodecay_params)}, with {num_nodecay_params:,} parameters")
        # Create AdamW optimizer and use the fused version if it is available
        fused_available = 'fused' in inspect.signature(torch.optim.AdamW).parameters
        use_fused = fused_available and device_type == 'cuda'
        extra_args = dict(fused=True) if use_fused else dict()
        optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas, **extra_args)
        print(f"using fused AdamW: {use_fused}")

        return optimizer

    def configure_optimizers_sophia(self, optimizer_name, weight_decay, learning_rate, betas, rho, device_type):
        """
        This long function is unfortunately doing something very simple and is being very defensive:
        We are separating out all parameters of the model into two buckets: those that will experience
        weight decay for regularization and those that won't (biases, and layernorm/embedding weights).
        We are then returning the PyTorch optimizer object.
        """

        # separate out all parameters to those that will and won't experience regularizing weight decay
        decay = set()
        no_decay = set()
        whitelist_weight_modules = (torch.nn.Linear, )
        blacklist_weight_modules = (torch.nn.LayerNorm, LayerNorm, torch.nn.Embedding)
        for mn, m in self.named_modules():
            for pn, p in m.named_parameters():
                fpn = '%s.%s' % (mn, pn) if mn else pn # full param name
                # random note: because named_modules and named_parameters are recursive
                # we will see the same tensors p many many times. but doing it this way
                # allows us to know which parent module any tensor p belongs to...
                if pn.endswith('bias'):
                    # all biases will not be decayed
                    no_decay.add(fpn)
                elif pn.endswith('weight') and isinstance(m, whitelist_weight_modules):
                    # weights of whitelist modules will be weight decayed
                    decay.add(fpn)
                elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules):
                    # weights of blacklist modules will NOT be weight decayed
                    no_decay.add(fpn)

        # subtle: 'transformer.wte.weight' and 'lm_head.weight' are tied, so they
        # will appear in the no_decay and decay sets respectively after the above.
        # In addition, because named_parameters() doesn't return duplicates, it
        # will only return the first occurrence, key'd by 'transformer.wte.weight', below.
        # so let's manually remove 'lm_head.weight' from decay set. This will include
        # this tensor into optimization via transformer.wte.weight only, and not decayed.
        decay.remove('lm_head.weight')

        # validate that we considered every parameter
        param_dict = {pn: p for pn, p in self.named_parameters()}
        inter_params = decay & no_decay
        union_params = decay | no_decay
        assert len(inter_params) == 0, "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), )
        assert len(param_dict.keys() - union_params) == 0, "parameters %s were not separated into either decay/no_decay set!" \
                                                    % (str(param_dict.keys() - union_params), )

        # create the pytorch optimizer object
        optim_groups = [
            {"params": [param_dict[pn] for pn in sorted(list(decay))], "weight_decay": weight_decay},
            {"params": [param_dict[pn] for pn in sorted(list(no_decay))], "weight_decay": 0.0},
        ]

        opt_func = optimizer_dict[optimizer_name]
        if optimizer_name == 'adamw':
        # new PyTorch nightly has a new 'fused' option for AdamW that is much faster
            use_fused = (device_type == 'cuda') and ('fused' in inspect.signature(torch.optim.AdamW).parameters)
            print(f"using fused AdamW: {use_fused}")
            extra_args = dict(fused=True) if use_fused else dict()
            optimizer = opt_func(optim_groups, lr=learning_rate, betas=betas, **extra_args)
        elif optimizer_name == 'sophiag':
            optimizer = opt_func(optim_groups, lr=learning_rate, betas=betas, rho=rho)
        else:
            raise ValueError('Invalid optimizer.')
        return optimizer

    def estimate_mfu(self, fwdbwd_per_iter, dt):
        """ estimate model flops utilization (MFU) in units of A100 bfloat16 peak FLOPS """
        # first estimate the number of flops we do per iteration.
        # see PaLM paper Appendix B as ref: https://arxiv.org/abs/2204.02311
        N = self.get_num_params()
        cfg = self.config
        L, H, Q, T = cfg.n_layer, cfg.n_head, cfg.n_embd//cfg.n_head, cfg.block_size
        flops_per_token = 6*N + 12*L*H*Q*T
        flops_per_fwdbwd = flops_per_token * T
        flops_per_iter = flops_per_fwdbwd * fwdbwd_per_iter
        # express our flops throughput as ratio of A100 bfloat16 peak flops
        flops_achieved = flops_per_iter * (1.0/dt) # per second
        flops_promised = 312e12 # A100 GPU bfloat16 peak flops is 312 TFLOPS
        mfu = flops_achieved / flops_promised
        return mfu

    @torch.no_grad()
    def generate(self, input_ids, max_new_tokens, temperature=1.0, top_k=None):
        for _ in range(max_new_tokens):
            idx_cond = input_ids if input_ids.size(1) <= self.config.block_size else input_ids[:, -self.config.block_size:]
            logits, _ = self(idx_cond)
            logits = logits[:, -1, :] / temperature
            if top_k is not None:
                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
                logits[logits<v[:, [-1]]] = -float('inf')
            probs = F.softmax(logits, dim=-1)
            idx_next = torch.multinomial(probs, num_samples=1)
            idx = torch.cat((input_ids, idx_next), dim=1)
        return idx

if __name__ == "__main__":
    config = GPTConfig()
    model = GPT2Model(config)

    batch_size = 16
    batch_data = torch.randint(low=0, high=config.vocab_size,
                               size=(batch_size, config.block_size),
                               dtype=torch.long)

    logits, loss = model(batch_data)
    print(logits.size())


# importing dependencies
import requests # request html file of link
from bs4 import BeautifulSoup # html parser to get the data that we need
import numpy as np # vector operations
import pandas as pd # label data
import matplotlib.pyplot as plt # visualize data
from sklearn.linear_model import LinearRegression # process data
import pickle # to use save and reuse model

# NOTE: KIJIJI often changes their html attribute values to avoid parsers, these have
# to be manually scanned and updated for the script to work

# the search result link that will be scraped
# split into 2 as an interger will indicate the page in between
baseSearch = "https://www.kijiji.ca/b-cars-vehicles/canada/equinox/page-"
baseSearch2 = "/k0c27l0?ad=offering&price=3500__"

# get the links of a search page
def get_ads_link(searchLink, numPages):
	# searchLink - the link that will be scraped
	#			   comes in a list of two parts
	# numPages - the number of pages you wish to scrape
	#			 each page has around 20 - 25 unique ads

	ads = []

	#loop through pages
	for i in range(1, numPages+1):
	#request the link, then parse it with beautiful soup
		r = requests.get(searchLink[0] + str(i) + searchLink[1])
		soup = BeautifulSoup(r.content)
		listOfDivs = soup.find_all("div")

	#search for divs with correct attributes then save the link
		for div in listOfDivs:
		# data-vip-url is the attribute for the link of the ad that pops up on search
			if div.has_attr("data-vip-url"):

				# retriving the link
				link = div["data-vip-url"]
				# concat strings to result in a proper link in english
				ads.append("https://www.kijiji.ca" + link + "?null&siteLocale=en_CA")

		print("%d of %d" %(i, numPages), end="\r")
	#return unique ads
	return list(set(ads))

# get relevant data from data
def get_ads_data(ads):
	raw_data = []
	errorLog = []
	numAds = len(ads)

	# retrieve html of each ad
	for i, ad in enumerate(ads):
		r = requests.get(ad)
		soup = BeautifulSoup(r.content)

		# getting the price tag of each ad
		try:
			price = soup.find_all("span", {"class": "currentPrice-3369960085"})[0].text
		except:
			try:
				price = soup.find_all("span", {"data-reactid": "39"})[0]["value"]
			except:
				try:
					price = soup.find_all("span", {"data-reactid": "40"})[0]["value"]
				except:
					try:
						price = soup.find_all("span", {"data-reactid": "42"})[0]["value"]
					except:
						errorLog.append(i)
						print("Price Error")

		# retrieve attributes of car
		try:
			# retrieve the label of data and data
			s_data_label = soup.find_all("p", {"class": "attributeLabel-2129198691"}, text=True)
			s_data = soup.find_all("h4", {"class": "attributeValue-4132240161"})
			# check if data matches
			if len(s_data) != len(s_data_label):
				errorLog.append(i)
				print("Data Error")
			else:
				for j in range(len(s_data)):
					s_data[j] = s_data[j].text
					s_data_label[j] = s_data_label[j].text
		except:
			errorLog.append(i)

	    # retrieve title description of the ad
		try:
			title = soup.find_all("h1", {"class": "title-1093844370"})[0].text
		except:
			errorLog.append(i)
			print("Title Error")
		raw_data.append([price, ad, title, [s_data_label, s_data]])
		print("%d of %d" %(i, numAds))

	# remove all ads in the error log
	for index in sorted(errorLog, reverse=True):
		del raw_data[index]

	return raw_data

ads = get_ads_link([baseSearch, baseSearch2], 1)
raw_data = get_ads_data(ads)

with open("raw_data.pkl", "wb") as output_file:
    pickle.dump(raw_data, output_file)


#pip install emoji
import emoji
def encoding_emoji(df):
	for i in df.index:
		string=''
		for w in df.loc[i,'tweet'].split():
			W=emoji.demojize(w)
			string=string+W+' '
		df.loc[i,'tweet']=string
	return df


__author__ = 'Scott Businge'

from django.conf.urls import url
from . import views

app_name = 'Appraisal'

urlpatterns = [
    url(r'^$', views.IndexView.as_view(), name='index'),
    url(r'^register/$', views.UserFormView.as_view(), name='register'),
    url(r'^(?P<pk>[0-9]+)/$', views.DetailView.as_view(), name='detail'),
    url(r'^department/add/$', views.DepartmentCreate.as_view(), name='department-add'),
    url(r'^department/(?P<pk>[0-9]+)/$', views.DepartmentUpdate.as_view(), name='department-update'),
    url(r'^department/(?P<pk>[0-9]+)/delete/$', views.DepartmentDelete.as_view(), name='department-delete'),

]


from importlib import import_module

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

_cached_class = None

def get_behaviour():
    global _cached_class
    if _cached_class:
        return _cached_class
    mod_name = settings.BEHAVIOUR_MODULE
    mod = import_module(mod_name)
    try:
        _cached_class = getattr(mod, 'Behaviour')
    except AttributeError:
        raise ImproperlyConfigured(
            'Behaviour module "{0}" does not define a '
            '"Behaviour" class'.format(mod_name))
    return _cached_class


from bson.objectid import ObjectId

class TicketController():
    def __init__(self, db):
        self.db = db

    def get_all_tickets(self):
        ticket_cursor = self.db.tickets.find({})
        tickets = []
        for document in ticket_cursor:
            document['_id'] = str(document['_id']) # this is a shitty hack for the json serializer
            tickets.append(document)

        response = {
            "ticket": tickets
        }

        return response
    #
    # def search_tickets(self, search_str):
    #     pass

    def get_ticket(self, ticket_id):
        ticket = self.db.tickets.find_one({"_id": ObjectId(ticket_id)})
        ticket['_id'] = str(ticket['_id']) # this is a shitty hack for the json serializer

        response = {
            "ticket": ticket
        }

        return response

    def buy_ticket(self, user_id):
        pass


from django.contrib import admin
from .models import Cart, CartItem

# Register your models here.

class CartAdmin(admin.ModelAdmin):

    """Ajout de la gestion des paniers des utilisateurs dans l'espace administrateur

    Args:
        admin.ModelAdmin (UserAdmin): Class hérité permettant de modifier l'espace administrateur du site
        par rapport a un model de base de donnée
    """

    list_display = ('cart_id', 'date_added')

class CartItemAdmin(admin.ModelAdmin):

    """Ajout de la gestion des lignes de paniers des utilisateurs dans l'espace administrateur

    Args:
        admin.ModelAdmin (UserAdmin): Class hérité permettant de modifier l'espace administrateur du site
        par rapport a un model de base de donnée
    """

    list_display = ('product', 'cart', 'quantity', 'is_active')

admin.site.register(Cart, CartAdmin)
admin.site.register(CartItem, CartItemAdmin)


import json
import os
import pickle

def load_synth_data(dataset_name, base_folder='data'):
    """
    This function loads the synthesized data provided in a picke file in the
    /data directory.
    """

    data_path = os.path.join(base_folder, dataset_name)

    with open(data_path, 'rb') as handle:
        data = pickle.load(handle)

    trainX = data['trainX']
    trainY = data['trainY']

    return trainX, trainY

def load_data(path):
    if 'cache' not in load_data.__dict__:
        load_data.cache = dict()
    if path not in load_data.cache:
        load_data.cache[path] = dict()
        with open(os.path.join(path, 'train.jsonl')) as f:
            train_exs = [json.loads(line) for line in f]
        with open(os.path.join(path, 'dev.jsonl')) as f:
            dev_exs = [json.loads(line) for line in f]
        with open(os.path.join(path, 'test.jsonl')) as f:
            test_exs = [json.loads(line) for line in f]
        load_data.cache[path] = {
            'train_exs': train_exs,
            'dev_exs': dev_exs,
            'test_exs': test_exs,
            }
    return load_data.cache[path]['train_exs'], load_data.cache[path]['dev_exs'], load_data.cache[path]['test_exs']


import sys
from collections import deque
input = sys.stdin.readline

drow = [1, -1, 0, 0]
dcol = [0, 0, 1, -1]

problem_num = 0
while True:
    N = int(input())
    if N == 0:
        break
    visited = [[float('inf')]*N for _ in range(N)]
    graph = [list(map(int, input().split())) for _ in range(N)]
    q = deque([(0, 0, graph[0][0])])
    visited[0][0] = graph[0][0]
    while q:
        row, col, rupee = q.popleft()
        for i in range(4):
            nrow = row + drow[i]
            ncol = col + dcol[i]
            if 0 <= nrow < N and 0 <= ncol < N:
                min_rupee = rupee + graph[nrow][ncol]
                if min_rupee < visited[nrow][ncol]:
                    visited[nrow][ncol] = min_rupee
                    q.append((nrow, ncol, min_rupee))
    problem_num += 1
    print("Problem {}: {}".format(problem_num, visited[N-1][N-1]))

백준/Gold/4485. 녹색 옷 입은 애가 젤다지？/녹색 옷 입은 애가 젤다지？.py

녹색 옷 입은 애가 젤다지？.py


import os
import json
import time
import threading
import torch
import cv2
import numpy as np
from matplotlib import pyplot as plt

from build_utils import img_utils, torch_utils, utils
from model import Darknet
from draw_box_utils import draw_box
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"

def main():
    img_size = 512  # 必须是32的整数倍 [416, 512, 608]
    cfg = "cfg/my_yolov3.cfg"  # 改成生成的.cfg文件
    weights = "weights/yolov3spp-voc-512.pt"  # 改成自己训练好的权重文件
    json_path = "./data/pascal_voc_classes.json"  # json标签文件
    img_path = "test.jpg"  # 改成当前目录下需要测试的文件名
    assert os.path.exists(cfg), "cfg file {} dose not exist.".format(cfg)
    assert os.path.exists(weights), "weights file {} dose not exist.".format(weights)
    assert os.path.exists(json_path), "json file {} dose not exist.".format(json_path)
    assert os.path.exists(img_path), "image file {} dose not exist.".format(img_path)

    json_file = open(json_path, 'r')
    class_dict = json.load(json_file)
    json_file.close()
    category_index = {v: k for k, v in class_dict.items()}  # key和value对换，转换成索引：类别

    input_size = (img_size, img_size)

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    model = Darknet(cfg, img_size)
    model.load_state_dict(torch.load(weights, map_location=device)["model"])
    model.to(device)

    model.eval()
    with torch.no_grad():
        # init
        img = torch.zeros((1, 3, img_size, img_size), device=device)  # 传入空的图片进行网络初始化，能提升预测速度
        model(img)

        # img_o = cv2.imread(img_path)  # BGR，图像应该当前路径下，或者给出完整的图像路径，绝对路径调用方式，要双反斜杠
        cap = cv2.VideoCapture(0)  # 调用自带的摄像头

        while True:
            # assert img_o is not None, "Image Not Found " + img_path
            ret, img_o = cap.read()  # 读取一帧

            img = img_utils.letterbox(img_o, new_shape=input_size, auto=True, color=(0, 0, 0))[0]  # 将图片缩放到指定大小
            # Convert
            img = img[:, :, ::-1].transpose(2, 0, 1)  # （H,W,C）中的C由BGR to RGB, to 3x416x416,注意np中的transpose可操作多维，tensor中的不行
            img = np.ascontiguousarray(img)  # 判断img在内存中是否连续，不是则转成连续

            img = torch.from_numpy(img).to(device).float()
            img /= 255.0  # scale (0, 255) to (0, 1)
            img = img.unsqueeze(0)  # add batch dimension # 在img中dim=0的位置加上一个维数为1的维度，因输入模型的数据格式要求(B,C,H,W)

            t1 = torch_utils.time_synchronized()
            pred = model(img)[0]  # only get inference result
            t2 = torch_utils.time_synchronized()
            print(t2 - t1)
            # 非极大值抑制NMS处理
            pred = utils.non_max_suppression(pred, conf_thres=0.1, iou_thres=0.6, multi_label=True)[0]
            t3 = time.time()
            print(t3 - t2)

            if pred is None:
                print("No target detected.")
                exit(0)

            # process detections
            pred[:, :4] = utils.scale_coords(img.shape[2:], pred[:, :4], img_o.shape).round()  # 将预测结果映射回原尺度上
            print(pred.shape)

            bboxes = pred[:, :4].detach().cpu().numpy()
            scores = pred[:, 4].detach().cpu().numpy()
            classes = pred[:, 5].detach().cpu().numpy().astype(np.int) + 1

            img_o = draw_box(img_o[:, :, ::-1], bboxes, classes, scores, category_index)
            # time.sleep(5)
            # plt.imshow(img_o)
            # plt.show()
            img_o = np.array(img_o)  # 将img_o从PIL的image数据类型用np.array()转成ndarray类型
            img_o = img_o[:, :, ::-1]  # 调换为opencv所需的bgr格式
            cv2.imshow('img_o', img_o)
            # cv2.waitKey(5)  # 用于设置在显示完一帧图像后程序等待5ms 再显示下一帧视频
            # img_o.save("test_result.jpg")
            if cv2.waitKey(1) == 27:  # 用于设置在显示完一帧图像后程序等待1ms， 接受按键esc信息， 再显示下一帧视频
                break
        cap.release()
        cv2.destroyWindow('img_o')

if __name__ == "__main__":
    main()


import os
import pydload
import tensorflow as tf

url = "https://github.com/bedapudi6788/NudeNet/releases/download/v0/classifier_model"
home = os.path.expanduser("~")
model_folder = os.path.join(home, ".NudeNet/")
if not os.path.exists(model_folder):
    os.mkdir(model_folder)

model_path = os.path.join(model_folder, "classifier")

if not os.path.exists(model_path):
                print("Downloading the checkpoint to", model_path)
                pydload.dload(url, save_to_path=model_path, max_time=None)

converter = tf.lite.TFLiteConverter.from_keras_model_file(model_path)
tflite_model = converter.convert()
open("converted_model.tflite", "wb").write(tflite_model)


from tokens import Kind, Token
import io

class Buffer:

  def set(self, line: str):
    self.current: str = line[0]
    self.pos: int = 0
    self.done: bool = False
    self.line: str = line

  def next(self) -> str:
    self.pos += 1
    if (self.pos < len(self.line)):
      self.current = self.line[self.pos]
    else:
      self.done = True
      self.current = '\0'
    return self.current

def lex(filename: str) -> "list[Token]":
  with open(filename) as file:
    return lex_file(file)

def lex_file(file: io.TextIOWrapper) -> "list[Token]":
  tokens: "list[Token]"= []
  keywords = {
    "uint": Kind.VAR_TYPE,
    "char": Kind.VAR_TYPE,
    "if": Kind.IF,
    "else": Kind.ELSE,
    "elif": Kind.ELIF,
    "while": Kind.WHILE,
    "macro": Kind.MACRO,
  }
  symbols = {
    '=': Kind.EQUAL,
    ';': Kind.SEMICOLON,
    '{': Kind.OPEN_BRACE,
    '}': Kind.CLOSE_BRACE,
    '>': Kind.COND_G,
    '<': Kind.COND_L,
    '+': Kind.PLUS,
    '-': Kind.MINUS,
    '*': Kind.MULT,
    '(': Kind.OPEN_PARAN,
    ')': Kind.CLOSE_PARAN,
    ',': Kind.COMMA,
    '/': Kind.DIV,
  }
  double_symbols = {
    "==": Kind.COND_E,
    ">=": Kind.COND_GE,
    "<=": Kind.COND_LE,
    "!=" : Kind.COND_NE,
    "//": Kind.COMMENT,
  }

  buf = Buffer()
  lineno = 1

  for line in file:
    buf.set(line)
    while not buf.done:
      #NUMBER TOKEN GENERATION
      if buf.current.isnumeric():
        start: int = buf.pos
        num: str = buf.current
        while buf.next().isnumeric():
          num += buf.current
        tokens.append(Token(Kind.NUMBER, num, line, lineno, start))

      #IDENTIFIER/KEYWORD TOKEN GENERATION
      elif buf.current.isalpha():
        start: int = buf.pos
        word: str = buf.current
        while buf.next().isalpha() or buf.current.isnumeric():
          word += buf.current
        kind: Kind = Kind.IDENTIFIER
        if word in keywords:
          kind = keywords[word]
        if buf.current == "!":
          buf.next()
          kind = Kind.MACROCALL

        tokens.append(Token(kind, word, line, lineno, start))

      else:
        if buf.current in symbols:
          current = buf.current
          symbol_kind: Kind = symbols[current]
          #Going to check for a double symbol here
          buf.next()
          double_symbol = buf.current + current
          if double_symbol in double_symbols:
            symbol_kind = double_symbols[double_symbol]
            current = double_symbol
            buf.next()

          if symbol_kind == Kind.COMMENT:
            buf.done = True
          else:
            tokens.append(Token(symbol_kind, current, line, lineno, buf.pos))
        else:
          buf.next()

    lineno += 1
  return tokens


from typing import List

class Solution:
    def largestSubmatrix(self, matrix: List[List[int]]) -> int:
        row, col = len(matrix), len(matrix[0])

        # Calculate heights for each column
        for i in range(1, row):
            for j in range(col):
                if matrix[i][j] == 1:
                    matrix[i][j] += matrix[i - 1][j]

        res = 0
        for i in range(row):
            # Sort the heights in ascending order
            matrix[i].sort(reverse=True)

            # Iterate through the sorted heights
            for j in range(col):
                height = matrix[i][j]
                width = j + 1
                res = max(res, height * width)

        return res

def test():
    sol = Solution()

    # Test Case 1
    matrix1 = [[0, 0, 1], [1, 1, 1], [1, 0, 1]]
    result1 = sol.largestSubmatrix(matrix1)
    print(f'Test Case 1: Input: {matrix1}, Output: {result1}')

    # Test Case 2
    matrix2 = [[1, 0, 1, 0, 1]]
    result2 = sol.largestSubmatrix(matrix2)
    print(f'Test Case 2: Input: {matrix2}, Output: {result2}')

    # Test Case 3
    matrix3 = [[1, 1, 0], [1, 0, 1]]
    result3 = sol.largestSubmatrix(matrix3)
    print(f'Test Case 3: Input: {matrix3}, Output: {result3}')

if __name__ == "__main__":
    test()


import argparse
from argparse import RawTextHelpFormatter
import base64
from copy import deepcopy
import inspect
import time
from typing import Dict, Tuple, List, Any, Callable, Optional, Union
import re
import pprint
import logging
import gc
import os
import socket

import git
import git.exc
import joblib
from joblib import Parallel, delayed
from mkdir_p import mkdir_p
from sklearn.model_selection import ParameterGrid

from .base import default_fn_dict, default_val_dict, \
        ParameterAlreadyRanError, VariableValueNotSetError, \
        VariableNotRegisteredError

logging.basicConfig(
    format='%(asctime)s %(levelname)-8s %(message)s',
    level=logging.WARNING,
    datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger(__name__)

def add_all_commit(repo, commit_msg="update"):
    repo.git.commit('-m', commit_msg)
    files = repo.git.diff(None, name_only=True)
    for f in files.split('\n'):
        repo.git.add(f)

class AutoVar(object):

    def __init__(self, before_experiment_hooks=None, after_experiment_hooks=None,
                 settings: Dict=None, logging_level: int=logging.WARNING) -> None:
        """
        settings : {
            'server_url': 'http://127.0.0.1:8080/nn_attack/',
            'result_file_dir': './results/'
            'file_format': 'json' or 'pickle'
        }
        """
        logger.setLevel(logging_level)

        self.experiments: Dict[str, Any] = {}
        self.variables: Dict[str, dict] = {}
        self.var_shown_name: Dict[str, Dict[str, str]] = {}
        self.var_description: Dict[str, str] = {}
        self.var_class: Dict[str, Any] = {}
        self.var_value: Dict[str, Any] = {}

        self.inter_var: Dict[str, Any] = {}
        self.result_fields: List[str] = []
        self.settings: Dict
        self.settings = {'file_format': 'json'}
        if isinstance(settings, dict):
            self.settings.update(settings)
        if ('result_file_dir' in self.settings) and self.settings['result_file_dir']:
            mkdir_p(self.settings['result_file_dir'])

        try:
            self.repo = git.Repo(search_parent_directories=True)
            self.var_value['git_hash'] = self.repo.head.object.hexsha
        except git.exc.InvalidGitRepositoryError:
            logger.warning("Git repository not found.")

        try:
            self.var_value['hostname'] = socket.gethostname()
        except:
            logger.warning("Unable to get hostname.")

        self.after_experiment_hooks = after_experiment_hooks
        self.before_experiment_hooks = before_experiment_hooks

        self._read_only: bool = False
        self._no_hooks: bool = False

    def set_logging_level(self, level: int):
        logger.setLevel(level)

    def add_variable_class(self, variable_class, var_name: str = None):
        if var_name is None:
            var_name = variable_class.__class__.var_name
            if var_name is None:
                var_name = type(variable_class).__name__

        if var_name in self.var_class:
            raise ValueError('Don\'t register class "{}" twice '.format(var_name))
        self.var_class[var_name] = variable_class
        #self.variables.setdefault(var_name, deepcopy(default_fn_dict))
        self.variables.update(variable_class.__class__.variables)
        self.var_shown_name.update(
                variable_class.__class__.variable_shown_name)

        if variable_class.default is not None:
            self._check_var_argument(var_name=var_name, argument=variable_class.default)

    def add_variable(self, var_name: str, dtype, default=None) -> None:
        d = deepcopy(default_val_dict)
        d['dtype'] = dtype
        self.variables.setdefault(var_name, d)
        if default is not None:
            self._check_var_argument(var_name=var_name, argument=default)
            d['default'] = default

    def get_var_shown_name(self, var_name: str,
                           argument: Union[str, None] = None) -> str:
        if argument is None:
            argument = self.get_variable_name(var_name)
        for key, val in self.var_shown_name[var_name].items():
            if re.fullmatch(key, argument):
                argument = re.sub(key, val, argument)
        return argument

    def get_var(self, var_name: str, *args, **kwargs):
        """[summary]

        Arguments:
            var_name {[type]} -- [description]
            argument {[type]} -- [description]

        Raises:
            ValueError -- [description]
            ValueError -- [description]

        Returns:
            [type] -- [description]
        """
        if var_name not in self.variables:
            raise VariableNotRegisteredError('Variable "%s" not registered.' % var_name)
        if var_name not in self.var_value:
            if self.variables[var_name]['type'] == 'choice':
                if self.var_class[var_name].default is not None:
                    argument = self.var_class[var_name].default
                else:
                    raise VariableValueNotSetError('Value for variable "%s" is not assigned.' % var_name)
            elif self.variables[var_name]['default'] is not None:
                argument = self.variables[var_name]['default']
            else:
                raise VariableValueNotSetError('Value for variable "%s" is not assigned.' % var_name)
        else:
            argument = self.var_value[var_name]

        return self.get_var_with_argument(
                var_name=var_name, argument=argument, *args, **kwargs)

    def match_variable(self, var_name: str, argument):
        if argument is None:
            return False

        if self.variables[var_name]["type"] == "val":
            return True
        else:
            if argument in self.variables[var_name]["argument_fn"]:
                return True
            else:
                for arg_template in self.variables[var_name]["argument_fn"]:
                    m = re.fullmatch(arg_template, argument)
                    if m is not None:
                        return True
        return None

    def get_variable_value(self, var_name: str):
        logger.warn("Use get_variable_name instead.")
        if var_name not in self.var_value:
            raise ValueError(f"{var_name} not in var_value")
        return self.var_value[var_name]

    def get_variable_name(self, var_name: str):
        if var_name not in self.var_value:
            raise ValueError(f"{var_name} not in variable name")
        return self.var_value[var_name]

    def get_var_with_argument(self, var_name: str, argument: str, *args, **kwargs):
        if self.variables[var_name]["type"] == "val":
            return argument
        else:
            cache_dir = None
            if argument in self.variables[var_name]["argument_fn"]:
                func = self.variables[var_name]["argument_fn"][argument]
                cache_dir = self.variables[var_name]['cache_dirs'][argument]
                required_vars = self.variables[var_name]['required_vars'][argument]
            else:
                m = None
                for arg_template, func in self.variables[var_name]["argument_fn"].items():
                    m = re.fullmatch(arg_template, argument)
                    if m is not None:
                        kwargs.update(m.groupdict())
                        cache_dir = self.variables[var_name]['cache_dirs'][arg_template]
                        required_vars = self.variables[var_name]['required_vars'][arg_template]
                        break
                if m is None:
                    raise ValueError('Argument "%s" not matched in Variable '
                                     '"%s".' % (argument, var_name))

            kwargs['auto_var'] = self
            named_args = inspect.getfullargspec(func)[0]
            if 'var_value' in named_args:
                kwargs['var_value'] = self.var_value
            if 'inter_var' in named_args:
                kwargs['inter_var'] = self.inter_var

            if cache_dir is not None:
                mkdir_p(cache_dir)
                #var_used = {var_name: self.var_value[var_name]}
                var_used = {var_name: argument}
                if required_vars is not None:
                    for var in required_vars:
                        if var not in self.var_value:
                            raise ValueError('Variable "%s" required by Variable '
                                             '"%s" is not set.' % (var, argument))

                        var_used[var] = self.var_value[var]
                required_variables = [str(v) for k, v in sorted(var_used.items())]
                cache_filename = os.path.join(cache_dir, '-'.join(required_variables) + '.pkl')

                if os.path.exists(cache_filename):
                    try:
                        func_outputs = joblib.load(cache_filename)
                        logger.info(f"using result from cache file {cache_filename} ...")
                    except:
                        os.unlink(cache_filename)
                        func_outputs = func(*args, **kwargs)
                        logger.info(f"dumping cache file to {cache_filename} ...")
                        joblib.dump(func_outputs, cache_filename)
                else:
                    func_outputs = func(*args, **kwargs)
                    logger.info(f"dumping cache file to {cache_filename} ...")
                    joblib.dump(func_outputs, cache_filename)
            else:
                func_outputs = func(*args, **kwargs)

            return func_outputs

    def get_intermidiate_variable(self, var_name: str):
        return self.inter_var[var_name]

    def set_variable_value(self, var_name: str, value) -> None:
        if self._read_only:
            raise ValueError("should not set variable value while read_only")
        #if var_name not in self.var_value:
        #    raise ValueError(f"{var_name} not in var_value")
        self.var_value[var_name] = value
        self._check_var_argument(var_name=var_name, argument=value)

    def set_intermidiate_variable(self, var_name: str, value) -> None:
        self.inter_var[var_name] = value

    def set_variable_value_by_dict(self, var_value: Dict[str, Any]) -> None:
        if self._read_only:
            raise ValueError("should not set variable value while read_only")
        for k, v in var_value.items():
            self.set_variable_value(k, v)

    def _run_before_hooks(self):
        if not self._no_hooks and self.before_experiment_hooks is not None:
            for hook_fn in self.before_experiment_hooks:
                try:
                    hook_fn(self)
                except ParameterAlreadyRanError:
                    logger.warning("The result for this parameter is "
                                        "already ran.")
                    return False
                except:
                    print("Error occur during running before hooks.")
                    raise
        return True

    def _run_after_hooks(self, ret):
        if not self._no_hooks and self.after_experiment_hooks is not None:
            for hook_fn in self.after_experiment_hooks:
                hook_fn(self, ret)

    def run_single_experiment(self, experiment_fn: Union[Callable[..., Any], str],
                              with_hook: bool=True,
                              verbose: int=0) -> Optional[bool]:
        self._read_only = True
        if isinstance(experiment_fn, str):
            if experiment_fn not in self.experiments:
                raise ValueError(f"experiment_fn {experiment_fn} is not a registered experiment")
            original_settings = deepcopy(self.settings)
            self.settings.update(self.experiments[experiment_fn]['settings'])

        ret = None
        ret_hook = True
        if with_hook:
            ret_hook = self._run_before_hooks()

        try:
            if ret_hook:
                start_time = time.time()
                if isinstance(experiment_fn, str):
                    ret = self.experiments[experiment_fn]['fn'](self)
                else:
                    ret = experiment_fn(self)
                end_time = time.time()
                logger.info("Running time: %f", end_time - start_time)
                if isinstance(ret, dict):
                    ret['running_time'] = end_time - start_time
                else:
                    # return value is not dict
                    pass
                ret['var_value'] = deepcopy(self.var_value)
        finally:
            if with_hook and ret_hook:
                self._run_after_hooks(ret)
            self.inter_var.clear()
            self._read_only = False
            gc.collect()

        if isinstance(experiment_fn, str):
            self.settings = original_settings
        return ret

    def _check_var_argument(self, var_name: str, argument):
        if self.variables[var_name]["type"] != "val" \
            and not self.match_variable(var_name, argument):
                raise ValueError('Argument "%s" not in Variable "%s".' % (argument, var_name))
        elif self.variables[var_name]["type"] == "val" \
            and not isinstance(argument, self.variables[var_name]['dtype']):
            raise TypeError('Argument "%s" not in the correct type "%s" for Variable "%s".' % (
                            type(argument), self.variables[var_name]['dtype'], var_name))

    def _check_grid_params(self, grid_params: Dict[str, List]) -> bool:
        # TODO check duplicated params
        # TODO duplicated check var argument since set_param_value will check
        for k, v in grid_params.items():
            for i in v:
                self._check_var_argument(k, i)
        return True

    def run_grid_params(self,
                        experiment_fn: Union[Callable[..., Any], str],
                        grid_params: Union[Dict[str, List], List[Dict[str, List]]],
                        with_hook: bool=True,
                        max_params: int=-1,
                        commit_before_run: bool=False,
                        verbose: int=0,
                        allow_failure: bool=True,
                        n_jobs: int=-1,
                        backend: Optional[str]=None,
                        pre_dispatch: str='2 * n_jobs') -> Tuple[List, List]:

        if commit_before_run:
            if self.repo is None:
                raise ValueError("Not currently in git repo.")
            add_all_commit(self.repo)

        if isinstance(grid_params, list):
            ret_params: List = []
            for grid_param in grid_params:
                self._check_grid_params(grid_param)

                grid = ParameterGrid(grid_param)
                ret_params += list(grid)
        else:
            ret_params = list(ParameterGrid(grid_params))

        if max_params != -1:
            ret_params = ret_params[:max_params]

        parallel = Parallel(n_jobs=n_jobs, verbose=verbose,
                            backend=backend, pre_dispatch=pre_dispatch)

        def _helper(auto_var, params):
            auto_var.set_variable_value_by_dict(params)
            if verbose:
                logger.info("Running parameter:" + str(params))
            try:
                results = auto_var.run_single_experiment(
                    experiment_fn,
                    with_hook=(with_hook and (not self._no_hooks)),
                    verbose=verbose
                )
            except Exception as e:
                if allow_failure:
                    logger.error("Error with " + str(params))
                else:
                    raise e
                results = None

            return results

        with parallel:
            ret_results = parallel(delayed(_helper)(deepcopy(self), params)
                                   for params in ret_params)

        return ret_params, ret_results

    def summary(self) -> None:
        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(self.variables)

    def generate_name(self) -> str:
        _ = hash(tuple(sorted(self.var_value.items())))
        return base64.b64encode(str(_).encode()).decode()

    def register_experiment(self, experiment_name: str,
                               experiment_fn: Callable[..., Any],
                               settings: Dict = None) -> None:
        if experiment_name in self.experiments:
            logger.info(f"Overriding experiment {experiment_name}")
        if settings is None:
            settings = dict()
        self.experiments[experiment_name] = {
            "fn": experiment_fn,
            "settings": settings,
        }

    def get_argparser(self) -> argparse.ArgumentParser:
        parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter)
        parser.add_argument('--no-hooks', required=False, action='store_true',
                            help="run without the hooks")

        if len(self.experiments) > 0:
            parser.add_argument('--experiment', type=str, required=True,
                                choices=[k for k in self.experiments.keys()],
                                help="The experiment to run")

        for var_name, v in self.variables.items():
            if v['type'] == 'choice':
                default = self.var_class[var_name].default
                required = False if default is not None else True
                help_str = ''
                if self.var_class[var_name].__doc__ is not None:
                    help_str += self.var_class[var_name].__doc__ + "\n"
                help_str += f"Options: (default: {default})\n"
                help_str += "\n".join(
                    ["  %s: %s" % (kk, vv.__doc__) for kk, vv in v['argument_fn'].items()])
                parser.add_argument(f'--{var_name}', type=str, required=required,
                                    action=make_action(self, var_name), default=default,
                                    help=help_str)
            else:
                default = v['default']
                required = False if default is not None else True
                parser.add_argument(f'--{var_name}', type=v['dtype'], required=required, default=default)

        return parser

    def parse_argparse(self, args: Optional[List[str]]=None):
        parser = self.get_argparser()
        parsed_args = parser.parse_args(args=args)
        self._no_hooks = parsed_args.no_hooks

        variables = vars(parsed_args)
        del variables['no_hooks']
        experiment_name = variables.pop('experiment', None)
        self.set_variable_value_by_dict(variables)

        if experiment_name:
            ret = self.run_single_experiment(experiment_name, with_hook=(not self._no_hooks))
            return ret

        return None

def make_action(auto_var, var_name):
    class CheckArgParseAction(argparse.Action):
        def __call__(self, parser, args, values, option_string=None):
            argument = values
            if not auto_var.match_variable(var_name, argument):
                raise argparse.ArgumentTypeError(
                    'Argument "%s" not in Variable "%s".' % (argument, var_name))
            setattr(args, self.dest, values)
    return CheckArgParseAction


import numpy as np
import cv2

def nothing(x):
    pass

face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
# eye_cascade = cv2.CascadeClassifier('haarcascade_eye.xml')
cap = cv2.VideoCapture(0)
# filters = ['Filters/cat_filter.png','Filters/citybg_filter.png','Filters/dog1_filter.png','Filters/dog_filter.png','Filters/hat_filter.png','Filters/rabbit1_filter.png','Filters/rabbit_filter.png','Filters/rainbow_filter.png','Filters/unicorn_filter.png']
filters = ['Filters/cat_filter.png','Filters/citybg_filter.png','Filters/dog1_filter.png','Filters/dog_filter.png','Filters/hat1_filter.png','Filters/rabbit1_filter.png','Filters/rabbit_filter.png','Filters/rainbow_filter.png','Filters/unicorn_filter.png','Filters/spectacles2.png']
cv2.namedWindow('Snapchat Filters')
cv2.createTrackbar('Filter','Snapchat Filters',0,9,nothing)

while 1:
    pos = cv2.getTrackbarPos('Filter','Snapchat Filters')
    img2 = cv2.imread(filters[pos])
    ret, img = cap.read()
    row1,col1,p1 = img.shape
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)
    roi_gray =  cv2.cvtColor(img, cv2.COLOR_BGR2GRAY);
    for (x,y,w,h) in faces:
        if pos == 4 or pos == 8:
            if y > h :
                img2 = cv2.resize(img2, (w,h))
            else :
                img2 = cv2.resize(img2, (w,y-2))
        elif pos == 1:
            img2 = cv2.resize(img2, (col1,row1))
        elif pos == 7:
            if y + h + h//2 > row1 :
                img2 = cv2.resize(img2, (w,row1 - y - h//2))
            else:
                img2 = cv2.resize(img2, (w,h))
        elif pos == 9:
            img2 = cv2.resize(img2,(w,h//2 + h//4))
        else :
            img2 = cv2.resize(img2, (w,h))

        # cv2.rectangle(img,(x,y),(x+w,y+h),(0,255,0),2)

        rows,cols,channels = img2.shape

        if pos == 4 or pos == 8:
            roi = img[y-rows:y ,x:x+cols]
        elif pos == 1:
            roi = img[0:row1,0:col1]
        elif pos == 7:
            if y + rows + rows//2 >  row1 :
                roi = img[y + rows//2: row1 ,x:x+cols]
            else :
                roi = img[y + rows//2:y + rows + rows//2 ,x:x+cols]
        elif pos == 9:
            roi = img[y:y + rows,x: x + cols]
        else:
            roi = img[y:y+rows ,x:x+cols]

        img2gray = cv2.cvtColor(img2,cv2.COLOR_BGR2GRAY)
        ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
        mask_inv = cv2.bitwise_not(mask)

        img1_bg = cv2.bitwise_and(roi,roi,mask = mask_inv)
        img2_fg = cv2.bitwise_and(img2,img2,mask = mask)

        dst = cv2.add(img1_bg,img2_fg)
        if pos == 4 or pos == 8:
            img[y-rows:y, x:x+cols] = dst
        elif pos == 1:
            img[0:rows,0:cols] = dst
        elif pos == 7:
            if y + rows + rows//2 >  row1 :
                img[ y+ rows//2:row1, x:x+cols] = dst
            else :
                img[ y+ rows//2:y+rows + rows//2, x:x+cols] = dst
        elif pos == 9:
            img[y:y + rows,x: x + cols] = dst
        else:
            img[ y:y+rows, x:x+cols] = dst

        # roi_color = img[y:y+h, x:x+w]
        # eyes = eye_cascade.detectMultiScale(roi_gray)
        # for (ex,ey,ew,eh) in eyes:
        #     cv2.rectangle(roi_color,(ex,ey),(ex+ew,ey+eh),(0,255,0),2)

    cv2.imshow('Snapchat Filters',img)
    k = cv2.waitKey(30) & 0xff
    if k == 27:
        break

cap.release()
cv2.destroyAllWindows()


import torch
import torch.nn.functional as F

class Net_log_softmax(torch.nn.Module):
    def __init__(self,num,inputSize,Neuron):
        super(Net_log_softmax,self).__init__()
        self.iSize = inputSize
        self.fc1 = torch.nn.Linear(self.iSize*self.iSize,Neuron)
        self.fc2 = torch.nn.Linear(Neuron,num)

    def forward(self,x):
        x = x.view(-1,self.iSize*self.iSize)
        x = self.fc1(x)
        x = torch.relu(x)
        x = self.fc2(x)
        return F.log_softmax(x,dim=1)

class CNN(torch.nn.Module):
    def __init__(self,num,inputSize,hidden1,hidden2):
        super(CNN,self).__init__()
        self.iSize  = inputSize
        self.conv1  = torch.nn.Conv2d(1,4,3)
        self.bn1    = torch.nn.BatchNorm2d(4)
        self.pool   = torch.nn.MaxPool2d(2,2)
        self.conv2  = torch.nn.Conv2d(4,16,3)
        self.bn2    = torch.nn.BatchNorm2d(16)
        self.fc1    = torch.nn.Linear(16*38*38,hidden1)
        self.fc2    = torch.nn.Linear(hidden1,hidden2)
        self.fc3    = torch.nn.Linear(hidden2,num)

    def forward(self,x):
        x = self.conv1(x)
        # print(x.size())
        x = torch.relu(x)
        x = self.bn1(x)
        x = self.pool(x)
        # print(x.size())
        x = self.conv2(x)
        # print(x.size())
        x = torch.relu(x)
        x = self.bn2(x)
        x = self.pool(x)
        # print(x.size())
        x = x.view(-1,16*38*38)
        x = self.fc1(x)
        x = torch.relu(x)
        x = self.fc2(x)
        x = torch.relu(x)
        x = self.fc3(x)
        return F.log_softmax(x,dim=1)


from selenium import webdriver
from time import sleep
from math import log, sin

def calc(el):
    return str(log(abs(12 * sin(int(el)))))

try:
    browser = webdriver.Chrome()
    browser.get('http://suninjuly.github.io/redirect_accept.html')
    browser.find_element_by_css_selector('button.trollface').click()        # нажимаем на плавающую кнопку

    new_window = browser.window_handles[1]                                  # узнаем имя новой вкладки в браузере
    browser.switch_to.window(new_window)                                    # переходим на новую вкладку в браузере

    x = int(browser.find_element_by_id('input_value').text)                 # получаем число X
    x = calc(x)                                                             # вычисляем по формуле

    input_res = browser.find_element_by_css_selector('input#answer')        # получаем поля для ввода
    input_res.send_keys(x)                                                  # вводим результат в поле для ввода

    button = browser.find_element_by_css_selector('button.btn').click()     # подтверждаем и отправляем овтет

finally:
    sleep(5)
    browser.quit()


import sys
import time
import copy
from enum import Enum

# so, everything that has .core, refers to the api1 level (i.e. swig
# wrapped cpp code)
from valkka import core
# api2 versions of the thread classes
from valkka.api2.threads import LiveThread, USBDeviceThread, OpenGLThread
from valkka.fs import ValkkaFSManager, ValkkaMultiFS, ValkkaSingleFS
from valkka.api2.tools import parameterInitCheck, typeCheck, generateGetters
from valkka.api2.chains.port import ViewPort

from valkka.live.chain.base import BaseFilterchain
from valkka.live import singleton

class ContextType(Enum):
    none = 0
    live = 1
    usb = 2

class RecordType(Enum):
    never = 0
    movement = 1
    always = 2

class MultiForkFilterchain(BaseFilterchain):
    """This class implements the following filterchain:

    ::

        *** main branch ***

        ANYSOURCE --> {ForkFrameFilterN:fork_filter_main} --+
                                                            |
                                       filesystem branch <--+
                                                            |
                                         decoding branch <--+
                                                            |
                                                 [other] <--+   [for tcp server, rtsp server, etc.]

        *** filesystem branch ***

        --> {GateFrameFilter:fs_gate} --> {ForkFrameFilterN:fork_filter_file} -->> ValkkaFSWriterThread
                     |
          [controlled by callback (**)]               [can be recorded to 0-N filesystems on-demand]

        *** decode branch ***
                                                (OpenGLThread: glthread) <<--+
                                                                             |
        -->> (AVThread:avthread) --> {ForkFrameFilterN:fork_filter_decode} --+--> analysis branch
                           [feeds AVBitmapFrames]                            |
                                                                             +--> [other]

        *** analysis branch ***

        --> {MovementFrameFilter: movement_filter}
             |             |
             |             +~~ callback (**)
             |
             +-- {GateFrameFilter: sws_gate} --> {SwScaleFrameFilter: sws_filter} --> {ForkFrameFilterN: sws_fork_M}
                                                                                |
                   +------------+------------+---------------------------------------------+
                   |            |            |
                 on-demand terminals for RGB images, for example

                 - RGBShmemFrameFilter(s)
                 - A common framefilter for all threads:

                 --> {ThreadSafeFrameFilter: common_sws_filter} --> {RGBShmemFrameFilter: common_rgb_shmem_filter}
                 [this could feed a common yolo detector for N streams]

        *** qt bitmap branch *** (in the case you need to pass bitmaps to the qt subsystem)

        --> {GateFrameFilter: qt_gate} --> {TimeIntervalFrameFilter: qt_interval} -->
        --> {SwScaleFrameFilter: qt_sws_filter} --> {ForkFrameFilterN: qt_fork_N}
                                                              |
                   +------------+------------+----------------+
                   |            |            |
                 on-demand terminals for RGB images, for example

    """

    parameter_defs = {
        "context_type"      : ContextType,
        "openglthreads"     : list,
        "livethread"        : LiveThread,
        "usbdevicethread"   : USBDeviceThread,

        #"valkkafsmanager"   : None, # ValkkaFSManager .. this is connected on-demand & can be detached & changed
        #"record_type"       : RecordType,

        # common to LiveThread & USBDeviceThread
        "address"      : str,  # string identifying the stream
        "slot"         : int,
        # Timestamp correction type: TimeCorrectionType_none,
        # TimeCorrectionType_dummy, or TimeCorrectionType_smart (default)
        "time_correction"
                       : (int, core.TimeCorrectionType_smart),

        # identify this device / stream
        "_id"          : int,

        # LiveThread specific
        "recv_buffer_size"  : (int, 0),       # Operating system socket ringbuffer size in bytes # 0 means default
        "reordering_mstime" : (int, 0),       # Reordering buffer time for Live555 packets in MILLIseconds # 0 means default
        # "request_tcp"       : (bool, False),  # Request video over tcp (instead of udp)
        ## that's RTSP specific, so let's set it elsewhere

        # these are for the AVThread instance:
        "n_basic"      : (int, 20),  # number of payload frames in the stack
        "n_setup"      : (int, 20),  # number of setup frames in the stack
        "n_signal"     : (int, 20),  # number of signal frames in the stack
        "flush_when_full"
                       : (bool, False),  # clear fifo at overflow
        "affinity"     : (int, -1),
        "number_of_threads" : (int, 2),  # let's set this to > 1.  Otherwise people start whining when their 4K 60 fps cameras don't work..
        "verbose"      : (bool, False),
        "msreconnect"  : (int, 0),

        "shmem_image_dimensions" : None, # (tuple, (1920//4, 1080//4)),
        "shmem_n_buffer"         : None, # (int, 10),
        "shmem_image_interval"   : None, # (int, 1000),

        "vaapi": (bool, False),

        # "movement_interval" : (int, 100), # pass frames at 10 fps # USE shmem_image_interval
        # "movement_treshold" : (float, 0.01),
        "movement_treshold" : (float, 0.0), # pass always
        "movement_duration" : (int, 30000) # when movement starts, pass through frames during 30 secs
    }

    def __init__(self, **kwargs):
        # auxiliary string for debugging output
        self.pre = self.__class__.__name__ + " : "
        # check for input parameters, attach them to this instance as
        # attributes
        parameterInitCheck(MultiForkFilterchain.parameter_defs, kwargs, self)
        generateGetters(self.parameter_defs, self)

        # check some types
        for openglthread in self.openglthreads:
            assert(issubclass(openglthread.__class__, OpenGLThread))
        #if self.valkkafsmanager is not None:
        #    assert(isinstance(self.valkkafsmanager, ValkkaFSManager))
        #if record_type != RecordType.never:
        #    assert(self.id_rec > -1)

        self.idst = str(id(self))

        self.initVars() # must be called before any clients are requested

        self.make_main_branch()
        self.make_filesystem_branch() # calls by default self.fs_gate.unSet()
        self.make_decode_branch()
        self.make_analysis_branch()
        self.make_qt_branch()

        self.createContext() # creates & registers contexes to LiveThread & USBDeviceThread

        self.start() # starts threads corresponding to this filterchain

    def setTCPStreaming(self, val):
        self.request_tcp = val

    def __del__(self):
        self.requestClose()

    def initVars(self):
        # client counters
        self.decoding_client_count = 0
        self.movement_client_count = 0

        self.sws_client_count = 0
        self.x_screen_count = {}

        self.sws_client_count_qt = 0
        self.x_screen_count_qt = {}

        for i in range(len(self.openglthreads)):
            self.x_screen_count[i] = 0

        # view port related
        self.ports = []
        self.tokens_by_port = {}

        # shmem related
        self.shmem_terminals = {}
        self.width      = self.shmem_image_dimensions[0]
        self.height     = self.shmem_image_dimensions[1]

        self.shmem_terminals_qt = {}

        self.record_type = None
        self.id_rec = None
        # self.valkkafsmanager = None

        self.closed = False

        # RTPS related
        self.request_tcp = False

    def start(self):
        """Starts threads required by the filter chain
        """
        self.avthread.startCall()

    def requestClose(self):
        if not self.closed:
            self.avthread.requestStopCall()
            self.disconnectRec()
            self.releaseAllShmem()
            self.clearAllViewPorts()
        self.closed = True

    def waitClose(self):
        self.avthread.waitStopCall()

    # *** Filesystem branch related ***

    def movement_cb(self, tup: tuple):
        """callback from the cpp side

        tup = (bool, SlotNumber, mstimestamp)

        bool = True means new movement even, bool = False means movement event ended stopped

        Controls the passthrough on self.fs_gate
        """
        try:
            if tup[0]:
                self.fs_gate.set()
            else:
                self.fs_gate.unSet()
        except Exception as e:
            print("MultiFork: movement_cb failed with", e)

    # (De)activate ValkkaFSWriterThread for this slot

    """
    def setRecording(self, record_type: RecordType, manager: ValkkaFSManager, id_rec: int = None, ):
        # for the moment, only one ValkkaFSManager can be set
        if self.record_type == RecordType.never:
            print("setRecording: never")
            return

        if id_rec is None:
            self.id_rec = self._id
        else:
            self.id_rec = id_rec

        if self.valkkafsmanager is not None:
            self.clearRecording()

        self.record_type = record_type
        self.valkkafsmanager = manager

        self.fork_filter_file.connect("recorder_" + str(self.slot) , self.valkkafsmanager.getFrameFilter())
        if self.record_type== RecordType.always:
            self.fs_gate.set()
        elif self.record_type == RecordType.movement:
            print("setRecording: movement")
            self.movement_client(inc = 1)
            self.movement_filter.setCallback(self.movement_cb)
        self.valkkafsmanager.setInput(self.id_rec, self.slot)

    def clearRecording(self):
        if self.record_type == RecordType.never:
            return
        if self.valkkafsmanager is None:
            return

        if self.record_type == RecordType.always:
            self.fs_gate.unSet()
        elif self.record_type == RecordType.movement:
            self.movement_client(inc = -1)
        self.valkkafsmanager.clearInput(self.slot)
        self.fork_filter_file.disconnect("recorder_" + str(self.slot))

        self.record_type = None
        self.id_rec = None
        self.valkkafsmanager = None
    """

    def connectRecTo(self, framefilter, record_type: RecordType):
        self.record_type=record_type
        self.fork_filter_file.connect("recorder_" + str(self.slot) , framefilter)
        if self.record_type== RecordType.always:
            self.fs_gate.set()
        elif self.record_type == RecordType.movement:
            print("setRecording: movement")
            self.movement_client(inc = 1)
            self.movement_filter.setCallback(self.movement_cb)

    def disconnectRec(self):
        if self.record_type is None:
            return
        if self.record_type == RecordType.always:
            self.fs_gate.unSet()
        elif self.record_type == RecordType.movement:
            self.movement_client(inc = -1)
        self.fork_filter_file.disconnect("recorder_" + str(self.slot))
        self.record_type=None

    # *** Context creation for the relevant thread (LiveThread or USBDeviceThread) ***

    def createContext(self):
        if self.context_type == ContextType.live:
            self.createLiveContext()
        elif self.context_type == ContextType.usb:
            self.createUSBContext()

    def closeContext(self):
        if self.context_type == ContextType.live:
            self.livethread.stopStreamCall(self.ctx)
            self.livethread.deRegisterStreamCall(self.ctx)
        elif self.context_type == ContextType.usb:
            self.usbdevicethread.stopStreamCall(self.ctx)

    def createLiveContext(self):
        """Context for LiveThread

        Creates & registers a context to LiveThread and connects it to the main filterchain

        Parameters required:

        ::

            self.slot
            self.address
            self.msreconnect

            self.time_correction
            self.recv_buffer_size
            self.reordering_mstime
        """

        self.ctx = core.LiveConnectionContext()
        self.ctx.slot = self.slot

        if (self.address.find("rtsp://") == 0):
            self.ctx.connection_type = core.LiveConnectionType_rtsp # rtsp connection
        else:
            self.ctx.connection_type = core.LiveConnectionType_sdp  # an sdp file

        if (self.request_tcp):
            self.ctx.request_tcp = self.request_tcp

        self.ctx.address = self.address
        # stream address, i.e. "rtsp://.."
        print("RTSP reconnection time:", self.msreconnect)
        self.ctx.msreconnect = self.msreconnect

        self.ctx.recv_buffer_size = self.recv_buffer_size
        self.ctx.reordering_time = self.reordering_mstime * 1000  # from millisecs to microsecs

        self.ctx.time_correction = self.time_correction
        if self.ctx.time_correction == core.TimeCorrectionType_smart:
            print("createLiveContext: smart timestamps")
        else:
            print("createLiveContext: dummy timestamps")

        # connect to the main filterchain
        self.ctx.framefilter = self.fork_filter_main

        # send the information about the stream to LiveThread
        self.livethread.registerStream(self.ctx) # NOTE: api level 2
        self.livethread.playStream(self.ctx)

    def createTCPContext(self): # TODO
        pass

    def createUSBContext(self):
        """Context for USBDeviceThread

        Creates & registers a context to USBDeviceThread and connects it to the main filterchain

        Required parameters:

        ::

            self.slot
            self.address
            self.msreconnect

            self.time_correction

        """
        self.ctx = core.USBCameraConnectionContext()
        self.ctx.slot        = self.slot
        self.ctx.device      = self.address
        self.ctx.width       = 1280
        self.ctx.height      = 720
        if (self.time_correction is not None):
            self.ctx.time_correction = self.time_correction

        # connect to the main filterchain
        self.ctx.framefilter = self.fork_filter_main

        # start playing
        print("createUSBContext: play")
        self.usbdevicethread.playStream(self.ctx)

    # *** Create filtergraph branches ***

    def make_main_branch(self):
        self.fork_filter_main = core.ForkFrameFilterN("fork_filter_main_" + str(self.slot))

    def make_filesystem_branch(self):
        self.fork_filter_file = core.ForkFrameFilterN("fork_filter_file_" + str(self.slot))
        self.fs_gate = core.GateFrameFilter("fs_gate_" + str(self.slot), self.fork_filter_file)
        # connect to main:
        self.fork_filter_main.connect("fs_gate_" + str(self.slot), self.fs_gate)
        self.fs_gate.unSet()

    def make_decode_branch(self):
        self.fork_filter_decode = core.ForkFrameFilterN("fork_filter_decode_" + str(self.slot))

        self.framefifo_ctx = core.FrameFifoContext()
        self.framefifo_ctx.n_basic = self.n_basic
        self.framefifo_ctx.n_setup = self.n_setup
        self.framefifo_ctx.n_signal = self.n_signal
        self.framefifo_ctx.flush_when_full = self.flush_when_full

        if self.vaapi:
            print("using VAAPIThread")
            self.avthread = core.VAAPIThread(
                "avthread_" + str(self.slot),
                self.fork_filter_decode,
                self.framefifo_ctx)

        else:
            self.avthread = core.AVThread(
                "avthread_" + str(self.slot),
                self.fork_filter_decode,
                self.framefifo_ctx)

        if self.affinity > -1: # affinity overwrites number of threads
            self.avthread.setAffinity(self.affinity)
        elif (self.number_of_threads > 1) and (not self.vaapi):
            # ffmpeg vaapi api doesn't like multithreading!
            self.avthread.setNumberOfThreads(self.number_of_threads) # two by default
            # that's been fixed in the core libValkka: only one decoding thread for vaapi

        # get input FrameFilter from AVThread
        self.av_in_filter = self.avthread.getFrameFilter()

        # connect to main:
        self.fork_filter_main.connect("decoding_" + str(self.slot), self.av_in_filter)

    def make_analysis_branch(self):
        """Connect only if movement detector is required:

        - Recording on movement
        - Analysis on movement
        """
        self.sws_fork_filter = core.ForkFrameFilterN("sws_fork_" + str(self.slot))
        self.sws_filter = core.SwScaleFrameFilter("sws_scale_" + str(self.slot), self.width, self.height, self.sws_fork_filter)
        self.sws_gate = core.GateFrameFilter("sws_gate_" + str(self.slot), self.sws_filter)
        self.movement_filter = core.MovementFrameFilter("movement_" + str(self.slot),
                # self.movement_interval,
                self.shmem_image_interval,
                self.movement_treshold,
                self.movement_duration,
                self.sws_gate
                )
        # MovementFrameFilter(const char* name, long int interval, float treshold, long int duration, FrameFilter* next=NULL);

    def make_qt_branch(self):
        """Connect only if bitmaps needed at the Qt side
        """
        self.qt_fork_filter = core.ForkFrameFilterN("qt_fork_" + str(self.slot))

        self.qt_sws_filter =\
        core.SwScaleFrameFilter("qt_sws_scale_" + str(self.slot), self.width,
            self.height, self.qt_fork_filter)

        self.qt_interval = core.TimeIntervalFrameFilter(
            "qt_interval_" + str(self.slot),
            # self.shmem_image_interval,
            500,
            self.qt_sws_filter
            )
        self.qt_gate = core.GateFrameFilter("qt_gate_" + str(self.slot), self.qt_interval)

        # connect to main:
        self.fork_filter_decode.connect("qt_branch_" + str(self.slot), self.qt_gate)

    # *** Client calculators ***

    def decoding_client(self, inc = 0):
        """Count instances that need decoding

        Start decoding if the number goes from 0 => 1
        Stop decoding if the number goes from 1 => 0
        """
        if self.decoding_client_count < 1 and inc > 0:
            # connect the analysis branch
            print(self.__class__.__name__, "start decoding for slot", self.slot)
            self.avthread.decodingOnCall()
        elif self.decoding_client_count == 1 and inc < 0:
            print(self.__class__.__name__, "stop decoding for slot", self.slot)
            self.avthread.decodingOffCall()
        self.decoding_client_count += inc

    def movement_client(self, inc = 0):
        """Count instances that need the movement detector

        Connect if the number goes from 0 => 1
        Disconnect if the number goes from 1 => 0
        """
        print("movement_client: count, inc:", self.movement_client_count, inc)
        if self.movement_client_count < 1 and inc > 0:
            # connect the analysis branch
            print("connecting analysis branch for slot", self.slot)
            self.fork_filter_decode.connect("analysis_" + str(self.slot), self.movement_filter)
        elif self.movement_client_count == 1 and inc < 0:
            print("disconnecting analysis branch for slot", self.slot)
            self.fork_filter_decode.disconnect("analysis_" + str(self.slot))
        self.movement_client_count += inc
        self.decoding_client(inc = inc)

    def sws_client(self, inc = 0):
        """Count instances that need the sw scaled images

        Enable sws_gate if number goes from 0 => 1
        Disable sws_gate if number goes from 1 => 0
        """
        if self.sws_client_count < 1 and inc > 0:
            # connect the analysis branch
            print("connecting sws_gate for slot", self.slot)
            self.sws_gate.set()
        elif self.sws_client_count == 1 and inc < 0:
            print("disconnecting sws_gate for slot", self.slot)
            self.sws_gate.unSet()
        self.sws_client_count += inc
        self.movement_client(inc = inc)

    def sws_client_qt(self, inc = 0):
        """Count instances that need the sw scaled images for Qt

        Enable qt_gate if number goes from 0 => 1
        Disable qt_gate if number goes from 1 => 0
        """
        if self.sws_client_count_qt < 1 and inc > 0:
            # connect the analysis branch
            print("connecting qt_gate for slot", self.slot)
            self.qt_gate.set()
        elif self.sws_client_count_qt == 1 and inc < 0:
            print("disconnecting qt_gate for slot", self.slot)
            self.qt_gate.unSet()
        self.sws_client_count_qt += inc

    def x_screen_client(self, index, inc = 0):
        if self.x_screen_count[index] < 1 and inc > 0:
            openglthread = self.openglthreads[index]
            self.fork_filter_decode.connect("openglthread_" + str(index), openglthread.getInput())
        elif self.x_screen_count[index] == 1 and inc < 0:
            self.fork_filter_decode.disconnect("openglthread_" + str(index))
        self.x_screen_count[index] += inc
        self.decoding_client(inc = inc)

    # *** Shmem hooks ***

    def getShmem(self):
        """Returns the unique name identifying the shared mem and semaphores.  The name can be passed to the machine vision routines.
        """
        shmem_name = singleton.sema_uuid + "_" +\
            self.idst + "_" + str(len(self.shmem_terminals))
        print("getShmem : reserving", shmem_name)
        shmem_filter = core.RGBShmemFrameFilter(shmem_name, self.shmem_n_buffer, self.width, self.height)
        # shmem_filter = core.BriefInfoFrameFilter(shmem_name) # DEBUG: see if you are actually getting any frames here ..
        self.shmem_terminals[shmem_name] = shmem_filter
        self.sws_fork_filter.connect(shmem_name, shmem_filter)
        # if first time, connect main branch to swscale_branch
        self.sws_client(inc = 1)
        return shmem_name

    def releaseShmem(self, shmem_name):
        try:
            self.shmem_terminals.pop(shmem_name)
        except KeyError:
            return False
        print("releaseShmem : releasing", shmem_name)
        self.sws_fork_filter.disconnect(shmem_name)
        self.sws_client(inc = -1)
        return True

    def releaseAllShmem(self):
        for key in list(self.shmem_terminals.keys()):
            self.releaseShmem(key)

    # *** Shmem hooks for qt bitmaps ***

    def getShmemQt(self):
        """For passing bitmaps to the Qt side
        """
        shmem_name = singleton.sema_uuid + "_" +\
            self.idst + "_qt_" + str(len(self.shmem_terminals_qt))
        print("getShmemQt : reserving", shmem_name)
        shmem_filter = core.RGBShmemFrameFilter(shmem_name, self.shmem_n_buffer, self.width, self.height)
        # shmem_filter = core.BriefInfoFrameFilter(shmem_name) # DEBUG: see if you are actually getting any frames here ..
        self.shmem_terminals_qt[shmem_name] = shmem_filter
        self.qt_fork_filter.connect(shmem_name, shmem_filter)
        # if first time, connect main branch to swscale_branch
        self.sws_client_qt(inc = 1)
        return shmem_name, self.shmem_n_buffer, self.width, self.height

    def releaseShmemQt(self, shmem_name):
        try:
            self.shmem_terminals_qt.pop(shmem_name)
        except KeyError:
            return False
        print("releaseShmemQt : releasing", shmem_name)
        self.qt_fork_filter.disconnect(shmem_name)
        self.sws_client_qt(inc = -1)
        return True

    def releaseAllShmemQt(self):
        for key in list(self.shmem_terminals_qt.keys()):
            self.releaseShmemQt(key)

    # *** Sending video to OpenGLThreads ***
    # now in the mother class

def createTestThreads():

    """
    valkkafsmanager = ValkkaFSManager(
        valkkafs,
        # read = False,   # debugging
        # cache = False,  # debugging
        # write = False   # debugging
        )
    """

    livethread = LiveThread(         # starts live stream services (using live555)
        name="live_thread",
        # verbose=True,
        verbose=False
        )

    usbdevicethread = USBDeviceThread(
        name="usb_thread",
        #verbose=True,
        verbose=False
        )

    openglthread = OpenGLThread(     # starts frame presenting services
        name="opengl_thread",
        # reserve stacks of YUV video frames for various resolutions
        n_720p  = 20,
        n_1080p = 20,
        n_1440p = 20,
        n_4K    = 0,
        verbose = True,
        # verbose=False,
        msbuftime = 300
        )

    return livethread, usbdevicethread, openglthread

def test1():
    """Test basic functionality of the MultiForkFilterchain
    """
    singleton.sema_uuid = "test1"

    valkkafs = ValkkaSingleFS.newFromDirectory(
        dirname = "/home/sampsa/tmp/testvalkkafs",
        blocksize = 512*1024,
        n_blocks = 10,
        verbose = True
        )

    valkkafsmanager = ValkkaFSManager([valkkafs])
    valkkafsmanager.start() # TODO: remember to add this elsewhere

    # getattr(valkkafsmanager,"iterateFsInput")

    livethread, usbdevicethread, openglthread = createTestThreads()

    address = sys.argv[2]
    context_type = ContextType.live

    #address = "/dev/video0"
    #context_type = ContextType.usb

    # record_type = RecordType.never
    #record_type = RecordType.always
    #record_type = RecordType.movement

    fc = MultiForkFilterchain(
        context_type     = context_type,
        openglthreads    = [openglthread],
        livethread       = livethread,
        usbdevicethread  = usbdevicethread,
        address          = address,
        slot             = 2,
        _id              = 123,
        shmem_image_dimensions = (1920//4, 1080//4),
        shmem_n_buffer   = 10,
        shmem_image_interval = 1000
        )

    window_id = openglthread.createWindow()
    view_port = ViewPort(window_id = window_id, x_screen_num = 0)

    n = 2

    print("\nadd view port\n")
    fc.addViewPort(view_port)
    print("\nsleep\n")
    time.sleep(n)

    print("\ndel view port\n")
    fc.delViewPort(view_port)
    print("\nsleep\n")
    time.sleep(n)

    print("\nsetRecording (always)\n")
    # fc.setRecording(12345, RecordType.always, valkkafsmanager)
    for valkkafs, inputfilter in valkkafsmanager.iterateFsInput():
        valkkafsmanager.map_(
            valkkafs=valkkafs,
            # read & cached stream is sent/output'd here:
            # (in this test it is not used)
            # framefilter=playback_fc.getInputFilter(),
            framefilter=None,
            write_slot=fc.slot,
            read_slot=None,
            _id=fc.slot
        )
        fc.connectRecTo(
            inputfilter,
            RecordType.always
        )
    print("\nsleep\n")
    time.sleep(n)

    print("\nclearRecording\n")
    for valkkafs, inputfilter in valkkafsmanager.iterateFsInput():
        valkkafsmanager.unmap(valkkafs, _id=fc.slot)
        fc.disconnectRec()

    print("\nsleep\n")
    time.sleep(n)

    """
    print("\nsetRecording (movement)\n")
    for valkkafs, inputfilter in valkkafsmanager.iterateFsInput():
        fc.connectRecTo(
            inputfilter,
            RecordType.movement
        )
    print("\nsleep\n")
    time.sleep(n)
    """

    again = True
    record_type = RecordType.always
    # record_type = RecordType.movement # ssssegfauuult!

    if again:
        print("\nsetRecording (always) again\n")
        for valkkafs, inputfilter in valkkafsmanager.iterateFsInput():
            valkkafsmanager.map_(
                valkkafs=valkkafs,
                # read & cached stream is sent/output'd here:
                # (in this test it is not used)
                # framefilter=playback_fc.getInputFilter(),
                framefilter=None,
                write_slot=fc.slot,
                read_slot=None,
                _id=fc.slot
            )
            fc.connectRecTo(
                inputfilter,
                record_type
            )
        print("\nsleep\n")
        time.sleep(n)

    print("\nsetRecording getShmem\n")
    name = fc.getShmem()
    print("\nsleep\n")
    time.sleep(n)

    print("\nsetRecording releaseShmem\n")
    fc.releaseShmem(name)
    print("\nsleep\n")
    time.sleep(n)

    if again:
        print("\nclearRecording\n")
        for valkkafs, inputfilter in valkkafsmanager.iterateFsInput():
            valkkafsmanager.unmap(valkkafs, _id=fc.slot)
            fc.disconnectRec()

    print("req close")

    fc.             requestClose()
    valkkafsmanager.requestClose()
    livethread.     requestClose()
    usbdevicethread.requestClose()
    openglthread.   requestClose()

    print("wait close fc")
    fc.             waitClose()
    print("wait close manager")
    valkkafsmanager.waitClose()
    print("wait close live")
    livethread.     waitClose()
    print("wait close usb")
    usbdevicethread.waitClose()
    print("wait close gl")
    openglthread.   waitClose()
    print("bye")

def main():
    pre = __name__ + "main :"
    print(pre, "main: arguments: ", sys.argv)
    if (len(sys.argv) < 2):
        print(pre, "main: needs test number")
    else:
        st = "test" + str(sys.argv[1]) + "()"
        exec(st)

if (__name__ == "__main__"):
    main()


from celery import shared_task
import time
import datetime
from django.conf import settings
from django.template.loader import render_to_string
from django.core.mail import EmailMultiAlternatives
from .models import *

@shared_task
def send_notifications(preview, pk, title, subscribers):
    mailing_list = list(PostCategory.objects.filter(postThrough_id=pk).values_list(
        'categoryThrough__subscribers__username',
        'categoryThrough__subscribers__first_name',
        'categoryThrough__subscribers__email',
        'categoryThrough__name',
        )
    )
    for user, first_name, email, category in mailing_list:
        if not first_name:
            first_name = user
    html_content= render_to_string(
        'post_created_email.html',
        {
            'name': first_name,
            'category': category,
            'title': title,
            'text': preview,
            'link': f'{settings.SITE_URL}/news/{pk}',

        }
    )
    msg = EmailMultiAlternatives(
        subject=title,
        body='',
        from_email=settings.DEFAULT_FROM_EMAIL,
        to=subscribers,
    )

    msg.attach_alternative(html_content, 'text/html')
    msg.send()

@shared_task
def every_monday_notify():
    today = datetime.datetime.now()
    last_week = today - datetime.timedelta(days = 7)
    posts = Post.objects.filter(time_creation__gte=last_week)
    categories = set(posts.values_list('category__theme', flat=True))
    subscribers = set(Category.objects.filter(theme__in=categories).values_list('subscribers__email', flat=True))
    html_content = render_to_string(
        'daily_post.html',
        {
            'link': settings.SITE_URL,
            'posts': posts,
        }
    )
    msg = EmailMultiAlternatives(
        subject='Статьи за неделю',
        body='',
        from_email=settings.DEFAULT_FROM_EMAIL,
        to=subscribers
    )
    msg.attach_alternative(html_content, 'text/html')
    msg.send()