# %%

import os
import sys
import numpy as np
import einops
from typing import Union, Optional, Tuple
import torch as t
from torch import Tensor
import torch.nn as nn
import torch.nn.functional as F
from jaxtyping import Float
from pathlib import Path
from torchvision import datasets, models
from torch.utils.data import Subset, DataLoader
from tqdm.notebook import tqdm
from dataclasses import dataclass

# Make sure exercises are in the path
section_dir = Path(__file__).parent
exercises_dir = section_dir.parent
assert exercises_dir.name == "exercises", f"This file should be run inside 'exercises/part2_cnns', not '{section_dir}'"
if str(exercises_dir) not in sys.path: sys.path.append(str(exercises_dir))

from plotly_utils import imshow, line, bar
import part2_cnns.tests as tests
from part2_cnns.solutions import ResNet34, copy_weights, Linear, IMAGENET_TRANSFORM

device = t.device("cuda" if t.cuda.is_available() else "cpu")

MAIN = __name__ == "__main__"

# %% 1⃣ CONVOLUTIONS

import torch as t
from collections import namedtuple

test_input = t.tensor(
	[[0, 1, 2, 3, 4],
	[5, 6, 7, 8, 9],
	[10, 11, 12, 13, 14],
	[15, 16, 17, 18, 19]], dtype=t.float
)

if MAIN:
	TestCase = namedtuple("TestCase", ["output", "size", "stride"])

	test_cases = [
		TestCase(
			output=t.tensor([0, 1, 2, 3]),
			size=(4,),
			stride=(1,),
		),
		TestCase(
			output=t.tensor([[0, 2], [5, 7]]),
			size=(2, 2),
			stride=(5, 2),
		),

		TestCase(
			output=t.tensor([0, 1, 2, 3, 4]),
			size=None,
			stride=None,
		),

		TestCase(
			output=t.tensor([0, 5, 10, 15]),
			size=None,
			stride=None,
		),

		TestCase(
			output=t.tensor([
				[0, 1, 2],
				[5, 6, 7]
			]),
			size=None,
			stride=None,
		),

		TestCase(
			output=t.tensor([
				[0, 1, 2],
				[10, 11, 12]
			]),
			size=None,
			stride=None,
		),

		TestCase(
			output=t.tensor([
				[0, 0, 0],
				[11, 11, 11]
			]),
			size=None,
			stride=None,
		),

		TestCase(
			output=t.tensor([0, 6, 12, 18]),
			size=None,
			stride=None,
		),
	]

	for (i, test_case) in enumerate(test_cases):
		if (test_case.size is None) or (test_case.stride is None):
			print(f"Test {i} failed: attempt missing.")
		else:
			actual = test_input.as_strided(size=test_case.size, stride=test_case.stride)
			if (test_case.output != actual).any():
				print(f"Test {i} failed:")
				print(f"Expected: {test_case.output}")
				print(f"Actual: {actual}\n")
			else:
				print(f"Test {i} passed!\n")

# %%

def as_strided_trace(mat: Float[Tensor, "i j"]) -> Float[Tensor, ""]:
	'''
	Returns the same as `torch.trace`, using only `as_strided` and `sum` methods.
	'''

	stride = mat.stride()

	assert len(stride) == 2, f"matrix should have size 2"
	assert mat.size(0) == mat.size(1), "matrix should be square"

	diag = mat.as_strided((mat.size(0),), (stride[0] + stride[1],))

	return diag.sum()

if MAIN:
	tests.test_trace(as_strided_trace)

# %%

def as_strided_mv(mat: Float[Tensor, "i j"], vec: Float[Tensor, "j"]) -> Float[Tensor, "i"]:
	'''
	Returns the same as `torch.matmul`, using only `as_strided` and `sum` methods.
	'''

	sizeM = mat.shape
	sizeV = vec.shape

	strideV = vec.stride()

	assert len(sizeM) == 2, f"mat1 should have size 2"
	assert sizeM[1] == sizeV[0], f"mat{list(sizeM)}, vec{list(sizeV)} not compatible for multiplication"

	vec_expanded = vec.as_strided(mat.shape, (0, strideV[0]))

	product_expanded = mat * vec_expanded

	return product_expanded.sum(dim=1)

if MAIN:
	tests.test_mv(as_strided_mv)
	tests.test_mv2(as_strided_mv)

# %%

def as_strided_mm(matA: Float[Tensor, "i j"], matB: Float[Tensor, "j k"]) -> Float[Tensor, "i k"]:
	'''
	Returns the same as `torch.matmul`, using only `as_strided` and `sum` methods.
	'''

	assert len(matA.shape) == 2, f"mat1 should have size 2"
	assert len(matB.shape) == 2, f"mat2 should have size 2"
	assert matA.shape[1] == matB.shape[0], f"mat1{list(matA.shape)}, mat2{list(matB.shape)} not compatible for multiplication"

	# Get the matrix strides, and matrix dims
	sA0, sA1 = matA.stride()
	dA0, dA1 = matA.shape
	sB0, sB1 = matB.stride()
	dB0, dB1 = matB.shape

	expanded_size = (dA0, dA1, dB1)

	matA_expanded_stride = (sA0, sA1, 0)
	matA_expanded = matA.as_strided(expanded_size, matA_expanded_stride)

	matB_expanded_stride = (0, sB0, sB1)
	matB_expanded = matB.as_strided(expanded_size, matB_expanded_stride)

	product_expanded = matA_expanded * matB_expanded

	return product_expanded.sum(dim=1)

if MAIN:
	tests.test_mm(as_strided_mm)
	tests.test_mm2(as_strided_mm)

# %% 3⃣ CONVOLUTIONS

def conv1d_minimal_simple(x: Float[Tensor, "w"], weights: Float[Tensor, "kw"]) -> Float[Tensor, "ow"]:
	'''
	Like torch's conv1d using bias=False and all other keyword arguments left at their default values.

	Simplifications: batch = input channels = output channels = 1.

	x: shape (width,)
	weights: shape (kernel_width,)

	Returns: shape (output_width,)
	'''

	w = x.shape[0]
	kw = weights.shape[0]
	# Get output width, using formula
	ow = w - kw + 1

	# Get strides for x
	s_w = x.stride(0)

	# Get strided x (the new dimension has same stride as the original stride of x)
	x_new_shape = (ow, kw)
	x_new_stride = (s_w, s_w)
	# Common error: s_w is always 1 if the tensor `x` wasn't itself created via striding, so if you put 1 here you won't spot your mistake until you try this with conv2d!
	x_strided = x.as_strided(size=x_new_shape, stride=x_new_stride)

	return einops.einsum(x_strided, weights, "ow kw, kw -> ow")

if MAIN:
	tests.test_conv1d_minimal_simple(conv1d_minimal_simple)

# %%

def conv1d_minimal(x: Float[Tensor, "b ic w"], weights: Float[Tensor, "oc ic kw"]) -> Float[Tensor, "b oc ow"]:
	'''
	Like torch's conv1d using bias=False and all other keyword arguments left at their default values.

	x: shape (batch, in_channels, width)
	weights: shape (out_channels, in_channels, kernel_width)

	Returns: shape (batch, out_channels, output_width)
	'''

	b, ic, w = x.shape
	oc, ic2, kw = weights.shape
	assert ic == ic2, "in_channels for x and weights don't match up"
	# Get output width, using formula
	ow = w - kw + 1

	# Get strides for x
	s_b, s_ic, s_w = x.stride()

	# Get strided x (the new dimension has the same stride as the original width-stride of x)
	x_new_shape = (b, ic, ow, kw)
	x_new_stride = (s_b, s_ic, s_w, s_w)
	# Common error: xsWi is always 1, so if you put 1 here you won't spot your mistake until you try this with conv2d!
	x_strided = x.as_strided(size=x_new_shape, stride=x_new_stride)

	return einops.einsum(x_strided, weights, "b ic ow kw, oc ic kw -> b oc ow",)

if MAIN:
	tests.test_conv1d_minimal(conv1d_minimal)

# %%

def conv2d_minimal(x: Float[Tensor, "b ic h w"], weights: Float[Tensor, "oc ic kh kw"]) -> Float[Tensor, "b oc oh ow"]:
	'''
	Like torch's conv2d using bias=False and all other keyword arguments left at their default values.

	x: shape (batch, in_channels, height, width)
	weights: shape (out_channels, in_channels, kernel_height, kernel_width)

	Returns: shape (batch, out_channels, output_height, output_width)
	'''

	b, ic, h, w = x.shape
	oc, ic2, kh, kw = weights.shape
	assert ic == ic2, "in_channels for x and weights don't match up"
	ow = w - kw + 1
	oh = h - kh + 1

	s_b, s_ic, s_h, s_w = x.stride()

	# Get strided x (the new height/width dims have the same stride as the original height/width-strides of x)
	x_new_shape = (b, ic, oh, ow, kh, kw)
	x_new_stride = (s_b, s_ic, s_h, s_w, s_h, s_w)

	x_strided = x.as_strided(size=x_new_shape, stride=x_new_stride)

	return einops.einsum(x_strided, weights, "b ic oh ow kh kw, oc ic kh kw -> b oc oh ow")

if MAIN:
	tests.test_conv2d_minimal(conv2d_minimal)

# %%

def pad1d(x: t.Tensor, left: int, right: int, pad_value: float) -> t.Tensor:
	'''Return a new tensor with padding applied to the edges.

	x: shape (batch, in_channels, width), dtype float32

	Return: shape (batch, in_channels, left + right + width)
	'''

	B, C, W = x.shape
	output = x.new_full(size=(B, C, left + W + right), fill_value=pad_value)
	output[..., left : left + W] = x
	# Note - you can't use `left:-right`, because `right` might be zero.
	return output

if MAIN:
	tests.test_pad1d(pad1d)
	tests.test_pad1d_multi_channel(pad1d)

# %%

def pad2d(x: t.Tensor, left: int, right: int, top: int, bottom: int, pad_value: float) -> t.Tensor:
	'''Return a new tensor with padding applied to the edges.

	x: shape (batch, in_channels, height, width), dtype float32

	Return: shape (batch, in_channels, top + height + bottom, left + width + right)
	'''

	B, C, H, W = x.shape
	output = x.new_full(size=(B, C, top + H + bottom, left + W + right), fill_value=pad_value)
	output[..., top : top + H, left : left + W] = x
	return output

if MAIN:
	tests.test_pad2d(pad2d)
	tests.test_pad2d_multi_channel(pad2d)

# %%

def conv1d(
	x: Float[Tensor, "b ic w"],
	weights: Float[Tensor, "oc ic kw"],
	stride: int = 1,
	padding: int = 0
) -> Float[Tensor, "b oc ow"]:
	'''
	Like torch's conv1d using bias=False.

	x: shape (batch, in_channels, width)
	weights: shape (out_channels, in_channels, kernel_width)

	Returns: shape (batch, out_channels, output_width)
	'''

	x_padded = pad1d(x, left=padding, right=padding, pad_value=0)

	b, ic, w = x_padded.shape
	oc, ic2, kw = weights.shape
	assert ic == ic2, "in_channels for x and weights don't match up"
	ow = 1 + (w - kw) // stride
	# note, we assume padding is zero in the formula here, because we're working with input which has already been padded

	s_b, s_ic, s_w = x_padded.stride()

	# Get strided x (the new height/width dims have the same stride as the original height/width-strides of x,
	# scaled by the stride (because we're "skipping over" x as we slide the kernel over it))
	# See diagram in hints for more explanation.
	x_new_shape = (b, ic, ow, kw)
	x_new_stride = (s_b, s_ic, s_w * stride, s_w)
	x_strided = x_padded.as_strided(size=x_new_shape, stride=x_new_stride)

	return einops.einsum(x_strided, weights, "b ic ow kw, oc ic kw -> b oc ow")

if MAIN:
	tests.test_conv1d(conv1d)

# %%

Pair = Tuple[int, int]
IntOrPair = Union[int, Tuple[int, int]]

def force_pair(v: IntOrPair) -> Pair:
	'''Convert v to a pair of int, if it isn't already.'''
	if isinstance(v, tuple):
		if len(v) != 2:
			raise ValueError(v)
		return (int(v[0]), int(v[1]))
	elif isinstance(v, int):
		return (v, v)
	raise ValueError(v)

# Examples of how this function can be used:

if MAIN:
	for v in [(1, 2), 2, (1, 2, 3)]:
		try:
			print(f"{v!r:9} -> {force_pair(v)!r}")
		except ValueError:
			print(f"{v!r:9} -> ValueError")

# %%

def conv2d(
	x: Float[Tensor, "b ic h w"],
	weights: Float[Tensor, "oc ic kh kw"],
	stride: IntOrPair = 1,
	padding: IntOrPair = 0
) -> Float[Tensor, "b oc oh ow"]:
	'''
	Like torch's conv2d using bias=False

	x: shape (batch, in_channels, height, width)
	weights: shape (out_channels, in_channels, kernel_height, kernel_width)

	Returns: shape (batch, out_channels, output_height, output_width)
	'''

	stride_h, stride_w = force_pair(stride)
	padding_h, padding_w = force_pair(padding)

	x_padded = pad2d(x, left=padding_w, right=padding_w, top=padding_h, bottom=padding_h, pad_value=0)

	b, ic, h, w = x_padded.shape
	oc, ic2, kh, kw = weights.shape
	assert ic == ic2, "in_channels for x and weights don't match up"
	ow = 1 + (w - kw) // stride_w
	oh = 1 + (h - kh) // stride_h

	s_b, s_ic, s_h, s_w = x_padded.stride()

	# Get strided x (new height/width dims have same stride as original height/width-strides of x, scaled by stride)
	x_new_shape = (b, ic, oh, ow, kh, kw)
	x_new_stride = (s_b, s_ic, s_h * stride_h, s_w * stride_w, s_h, s_w)
	x_strided = x_padded.as_strided(size=x_new_shape, stride=x_new_stride)

	return einops.einsum(x_strided, weights, "b ic oh ow kh kw, oc ic kh kw -> b oc oh ow")

if MAIN:
	tests.test_conv2d(conv2d)

# %%

def maxpool2d(
	x: Float[Tensor, "b ic h w"],
	kernel_size: IntOrPair,
	stride: Optional[IntOrPair] = None,
	padding: IntOrPair = 0
) -> Float[Tensor, "b ic oh ow"]:
	'''
	Like PyTorch's maxpool2d.

	x: shape (batch, channels, height, width)
	stride: if None, should be equal to the kernel size

	Return: (batch, channels, output_height, output_width)
	'''

	# Set actual values for stride and padding, using force_pair function
	if stride is None:
		stride = kernel_size
	stride_h, stride_w = force_pair(stride)
	padding_h, padding_w = force_pair(padding)
	kh, kw = force_pair(kernel_size)

	# Get padded version of x
	x_padded = pad2d(x, left=padding_w, right=padding_w, top=padding_h, bottom=padding_h, pad_value=-t.inf)

	# Calculate output height and width for x
	b, ic, h, w = x_padded.shape
	ow = 1 + (w - kw) // stride_w
	oh = 1 + (h - kh) // stride_h

	# Get strided x
	s_b, s_c, s_h, s_w = x_padded.stride()

	x_new_shape = (b, ic, oh, ow, kh, kw)
	x_new_stride = (s_b, s_c, s_h * stride_h, s_w * stride_w, s_h, s_w)
	x_strided = x_padded.as_strided(size=x_new_shape, stride=x_new_stride)

	# Argmax over dimensions of the maxpool kernel
	# (note these are the same dims that we multiply over in 2D convolutions)
	output = t.amax(x_strided, dim=(-1, -2))
	return output

if MAIN:
	tests.test_maxpool2d(maxpool2d)

# %% 4⃣ MAKING YOUR OWN MODULES

class MaxPool2d(nn.Module):
	def __init__(self, kernel_size: IntOrPair, stride: Optional[IntOrPair] = None, padding: IntOrPair = 1):
		super().__init__()
		self.kernel_size = kernel_size
		self.stride = stride
		self.padding = padding

	def forward(self, x: t.Tensor) -> t.Tensor:
		'''Call the functional version of maxpool2d.'''
		return maxpool2d(x, kernel_size=self.kernel_size, stride=self.stride, padding=self.padding)

	def extra_repr(self) -> str:
		'''Add additional information to the string representation of this class.'''
		return ", ".join([f"{key}={getattr(self, key)}" for key in ["kernel_size", "stride", "padding"]])

if MAIN:
	tests.test_maxpool2d_module(MaxPool2d)
	m = MaxPool2d(kernel_size=3, stride=2, padding=1)
	print(f"Manually verify that this is an informative repr: {m}")

# %%

class Conv2d(nn.Module):
	def __init__(
		self, in_channels: int, out_channels: int, kernel_size: IntOrPair, stride: IntOrPair = 1, padding: IntOrPair = 0
	):
		'''
		Same as torch.nn.Conv2d with bias=False.

		Name your weight field `self.weight` for compatibility with the PyTorch version.
		'''
		super().__init__()
		self.in_channels = in_channels
		self.out_channels = out_channels
		self.kernel_size = kernel_size
		self.stride = stride
		self.padding = padding

		kernel_height, kernel_width = force_pair(kernel_size)
		sf = 1 / np.sqrt(in_channels * kernel_width * kernel_height)
		weight = sf * (2 * t.rand(out_channels, in_channels, kernel_height, kernel_width) - 1)
		self.weight = nn.Parameter(weight)

	def forward(self, x: t.Tensor) -> t.Tensor:
		'''Apply the functional conv2d you wrote earlier.'''
		return conv2d(x, self.weight, self.stride, self.padding)

	def extra_repr(self) -> str:
		keys = ["in_channels", "out_channels", "kernel_size", "stride", "padding"]
		return ", ".join([f"{key}={getattr(self, key)}" for key in keys])

if MAIN:
	tests.test_conv2d_module(Conv2d)

# %% 2⃣ RESNET FEATURE EXTRACTION

if MAIN:
	layer0, layer1 = nn.Linear(3, 4), nn.Linear(4, 5)

	layer0.requires_grad_(False) # generic code to set `param.requires_grad = False` recursively for a module (or entire model)

	x = t.randn(3)
	out = layer1(layer0(x)).sum()
	out.backward()

	assert layer0.weight.grad is None
	assert layer1.weight.grad is not None

# %%

def get_resnet_for_feature_extraction(n_classes: int) -> ResNet34:
	'''
	Creates a ResNet34 instance, replaces its final linear layer with a classifier
	for `n_classes` classes, and freezes all weights except the ones in this layer.

	Returns the ResNet model.
	'''
	# Create a ResNet34 with the default number of classes
	my_resnet = ResNet34()

	# Load the pretrained weights
	pretrained_resnet = models.resnet34(weights=models.ResNet34_Weights.IMAGENET1K_V1)

	# Copy the weights over
	my_resnet = copy_weights(my_resnet, pretrained_resnet)

	# Freeze gradients for all layers (note that when we redefine the last layer, it will be unfrozen)
	my_resnet.requires_grad_(False)

	# Redefine last layer
	my_resnet.out_layers[-1] = Linear(
		my_resnet.out_features_per_group[-1],
		n_classes
	)

	return my_resnet

if MAIN:
	tests.test_get_resnet_for_feature_extraction(get_resnet_for_feature_extraction)

# %%

def get_cifar(subset: int):
	cifar_trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=IMAGENET_TRANSFORM)
	cifar_testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=IMAGENET_TRANSFORM)

	if subset > 1:
		cifar_trainset = Subset(cifar_trainset, indices=range(0, len(cifar_trainset), subset))
		cifar_testset = Subset(cifar_testset, indices=range(0, len(cifar_testset), subset))

	return cifar_trainset, cifar_testset

@dataclass
class ResNetTrainingArgs():
	batch_size: int = 64
	epochs: int = 3
	learning_rate: float = 1e-3
	n_classes: int = 10
	subset: int = 10

# %%

def train(args: ResNetTrainingArgs) -> ResNet34:
	'''
	Performs feature extraction on ResNet.
	'''
	model = get_resnet_for_feature_extraction(args.n_classes).to(device)

	trainset, testset = get_cifar(subset=args.subset)
	trainloader = DataLoader(trainset, batch_size=args.batch_size, shuffle=True)
	testloader = DataLoader(testset, batch_size=args.batch_size, shuffle=False)

	optimizer = t.optim.Adam(model.out_layers[-1].parameters(), lr=args.learning_rate)

	loss_list = []
	accuracy_list = []

	for epoch in tqdm(range(args.epochs)):

		# Training loop
		model.train()
		for imgs, labels in tqdm(trainloader, desc=f"Epoch {epoch+1}"):
			imgs = imgs.to(device)
			labels = labels.to(device)
			logits = model(imgs)
			loss = F.cross_entropy(logits, labels)
			loss.backward()
			optimizer.step()
			optimizer.zero_grad()
			loss_list.append(loss.item())

		# Validation loop
		model.eval()
		num_correct_classifications = 0
		for imgs, labels in testloader:
			imgs = imgs.to(device)
			labels = labels.to(device)
			with t.inference_mode():
				logits = model(imgs)
			predictions = t.argmax(logits, dim=1)
			num_correct_classifications += (predictions == labels).sum().item()
		accuracy = num_correct_classifications / len(testset)
		accuracy_list.append(accuracy)

	line(
		loss_list,
		yaxis_range=[0, max(loss_list) + 0.1],
		labels={"x": "Num batches seen", "y": "Cross entropy loss"},
		title="SimpleMLP training on MNIST",
		width=700,
	)
	line(
		accuracy_list,
		yaxis_range=[0, 1],
		labels={"x": "Num epochs", "y": "Accuracy"},
		title="SimpleMLP test accuracy on MNIST",
		width=700,
	)
	return model

if MAIN:
	args = ResNetTrainingArgs()
	model = train(args)

# %%


import sys

sys.path.append('./data/')
sys.path.append('./common/')
sys.path.append('./modules/')
sys.path.append('./model/')
# print(sys.path)
import os
import shutil
import time
from datetime import datetime
import numpy as np
import random
import torch
from torch import nn, optim

from common.options import parse_args
from data.sdp_data_loader import DataLoader
from model.sdp_biaffine_trainer import Trainer
from model import sdp_simple_scorer
from common import utils
from data.pretrain import Pretrain

from common.timer import Timer
from tensorboardX import SummaryWriter
import logging

def get_logger(logger_name, log_file, is_debug=False):
    logger = logging.getLogger(logger_name)
    if is_debug:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)
    # handlers:
    c_handler = logging.StreamHandler()
    f_handler = logging.FileHandler(log_file)

    # level:
    c_handler.setLevel(logging.WARNING)

    c_format = logging.Formatter("%(asctime)s-%(levelname)s-%(message)s")
    f_format = logging.Formatter("%(asctime)s-%(levelname)s-%(message)s")
    c_handler.setFormatter(c_format)
    f_handler.setFormatter(f_format)

    # add
    logger.addHandler(c_handler)
    logger.addHandler(f_handler)
    return logger

def main(args):
    torch.manual_seed(args.seed)
    np.random.seed(args.seed)
    random.seed(args.seed)
    if args.cpu:
        logger.critical('use CPU !')
        args.cuda = False
    elif args.cuda:
        logger.critical("use CUDA")
        torch.cuda.manual_seed(args.seed)

    args = vars(args)
    logger.info('model args:')
    for k, v in args.items():
        logger.info(f'--{k}: {v}')
    logger.critical(f"Running parser in {args['mode']} mode")

    if args['mode'] == 'train':
        with Timer('train time:'):
            train(args)
    else:
        with Timer('predict time:'):
            evaluate(args)

def train(args):
    utils.ensure_dir(args['save_dir'])
    model_text_file = os.path.join(args['save_dir'], 'TEXT_' + args['save_name_suffix'] + '.pt')
    model_news_file = os.path.join(args['save_dir'], 'NEWS_' + args['save_name_suffix'] + '.pt')

    # load pretrained vectors
    pretrain = Pretrain(args['vec_file'], args['logger_name'])
    # TensorboardX:
    summary_writer = SummaryWriter()

    # load data
    logger.critical(f"Loading data with batch size {args['batch_size']}...")
    train_batch = DataLoader(args['train_merge_file'], args['batch_size'], args, pretrain, evaluation=False)
    vocab = train_batch.vocab
    dev_text_batch = DataLoader(args['dev_text_file'], 1000, args, pretrain, vocab=vocab, evaluation=True)
    dev_news_batch = DataLoader(args['dev_news_file'], 1000, args, pretrain, vocab=vocab, evaluation=True)

    # pred and gold path
    output_text_file = os.path.join(args['output_file_path'], 'DEV_Text_' + args['save_name_suffix'] + '.conllu')
    output_news_file = os.path.join(args['output_file_path'], 'DEV_News_' + args['save_name_suffix'] + '.conllu')

    # skip training if the language does not have training or dev data
    if len(train_batch) == 0 or len(dev_text_batch) == 0 or len(dev_news_batch) == 0:
        logger.info("Skip training because no data available...")
        exit()

    logger.info("Training parser...")
    trainer = Trainer(args=args, vocab=vocab, pretrain=pretrain, use_cuda=args['cuda'])

    global_step = 0

    using_amsgrad = False
    task2idx = {'text': 0, 'news': 1}
    last_best_step = [0] * 2
    # start training
    train_loss = 0
    last_best_LAS = [0] * 2
    last_best_UAS = [0] * 2
    last_best_LAS_step = [0] * 2
    last_best_UAS_step = [0] * 2

    def _dev(dev_name):
        nonlocal last_best_LAS, last_best_UAS, last_best_LAS_step, last_best_UAS_step
        nonlocal train_loss, last_best_step, dev_text_batch, dev_news_batch
        # nonlocal output_news_file, output_text_file
        if dev_name == 'text':
            dev_batch = dev_text_batch
            output_file = output_text_file
            gold_file = args['dev_text_file']
            task_id = task2idx['text']
            model_file = model_text_file
        elif dev_name == 'news':
            dev_batch = dev_news_batch
            output_file = output_news_file
            gold_file = args['dev_news_file']
            task_id = task2idx['news']
            model_file = model_news_file
        else:
            raise ValueError('bad dev name')
        dev_preds = []
        for batch in dev_batch:
            preds = trainer.predict(batch, cuda_data=False)
            dev_preds += preds
            if args['cuda']:
                torch.cuda.empty_cache()
        dev_batch.conll.set(['deps'], [y for x in dev_preds for y in x])
        dev_batch.conll.write_conll(output_file)
        dev_uas, dev_score = sdp_simple_scorer.score(output_file, gold_file)
        logger.info(
            f"step:{global_step}; train_loss:{train_loss:0.5f}; dev_UAS:{dev_uas:.4f}; dev_LAS:{dev_score:.4f}")
        summary_writer.add_scalars(f'data/eval_{dev_name}',
                                   {'ave_loss': train_loss,
                                    'dev_UAS': dev_uas,
                                    'dev_LAS': dev_score},
                                   global_step)
        # train_loss = 0
        if dev_uas > last_best_UAS[task_id]:
            last_best_UAS[task_id] = dev_uas
            last_best_UAS_step[task_id] = global_step
        # save best model
        if dev_score > last_best_LAS[task_id]:
            last_best_LAS[task_id] = dev_score
            last_best_LAS_step[task_id] = global_step
            last_best_step[task_id] = global_step
            trainer.save(model_file)
            logger.info(f"{dev_name}: last_best_UAS:{last_best_UAS[task_id]:.4f} in step:{last_best_UAS_step[task_id]}")
            logger.critical(
                f"{dev_name}: last_best_LAS:{last_best_LAS[task_id]:.4f} in step:{last_best_LAS_step[task_id]}")
            logger.info(f"{dev_name}: new best model saved in {model_file}")

    while True:
        do_break = False
        for i, batch in enumerate(train_batch):
            # pprint(batch[0].size())
            # continue
            global_step += 1
            print('batch:')
            print(batch)
            loss = trainer.update(batch, global_step, cuda_data=False, eval=False)  # update step
            train_loss += loss
            if args['cuda']:
                torch.cuda.empty_cache()
            if global_step % args['log_step'] == 0:
                summary_writer.add_scalar('data/loss', loss, global_step)

            if global_step % args['eval_interval'] == 0:
                # eval on dev
                # logger.info("Evaluating on dev set...")
                train_loss = train_loss / args['eval_interval']  # avg loss per batch
                _dev('text')
                _dev('news')
                train_loss = 0
            if global_step - max(last_best_step) >= args['max_steps_before_stop']:
                if not using_amsgrad:
                    logger.critical(f"--->>> Optim:Switching to AMSGrad in step:{global_step}")
                    last_best_step = [global_step] * 2
                    using_amsgrad = True
                    trainer.optimizer = optim.Adam(trainer.model.parameters(), amsgrad=True, lr=args['lr'],
                                                   betas=(args['beta1'], args['beta2']), eps=args['eps'])
                else:
                    do_break = True
                    break

            if global_step >= args['max_steps']:
                do_break = True
                break

        if do_break:
            break

        train_batch.reshuffle()

    logger.critical(f"Training ended with {global_step} steps")
    logger.critical(f'Text: best dev LAS:{last_best_LAS[0]} in step:{last_best_LAS_step[0]}')
    logger.critical(f'News: best dev LAS:{last_best_LAS[1]} in step:{last_best_LAS_step[1]}')
    summary_writer.close()

def evaluate(args):
    utils.ensure_dir(args['save_dir'])
    model_text_file = os.path.join(args['save_dir'], 'TEXT_' + args['save_name_suffix'] + '.pt')
    model_news_file = os.path.join(args['save_dir'], 'NEWS_' + args['save_name_suffix'] + '.pt')
    pretrain = Pretrain(args['vec_file'], logger_name='evaluate')
    use_cuda = args['cuda'] and not args['cpu']

    def _eval(eval_name):
        if eval_name == 'text':
            model_file = model_text_file
            test_file = args['test_text_file']
            output_file = os.path.join(args['output_file_path'],
                                       'TEST_Text_' + args['save_name_suffix'] + '.conllu')
            gold_file = args['test_text_file']
        elif eval_name == 'news':
            model_file = model_news_file
            test_file = args['test_news_file']
            output_file = os.path.join(args['output_file_path'],
                                       'TEST_News_' + args['save_name_suffix'] + '.conllu')
            gold_file = args['test_news_file']
        else:
            raise ValueError('bad eval name')
        trainer = Trainer(pretrain=pretrain, model_file=model_file, use_cuda=use_cuda)
        loaded_args, vocab = trainer.args, trainer.vocab
        test_batch = DataLoader(test_file, 1000, args, pretrain, vocab=vocab, evaluation=True)
        if len(test_batch) > 0:
            print(f"{eval_name} Start evaluation...")
            preds = []
            for i, b in enumerate(test_batch):
                preds += trainer.predict(b)
        else:
            preds = []
        test_batch.conll.set(['deps'], [y for x in preds for y in x])
        test_batch.conll.write_conll(output_file)
        if gold_file is not None:
            UAS, LAS = sdp_simple_scorer.score(output_file, gold_file)
            print(f"{eval_name}: Test Dataset Parser score:")
            print(f'{eval_name} LAS:{LAS * 100:.3f}\t{eval_name} UAS:{UAS * 100:.3f}')

    _eval('text')
    _eval('news')

if __name__ == '__main__':
    from pprint import pprint

    args = parse_args()
    args.cpu = True
    # args.self_att = True
    # args.vec_file = 'Embeds/sem16_tencent.pkl'
    args.batch_size = 30
    args.rec_dropout = 0
    args.char_rec_dropout = 0
    args.num_layers = 1

    logger = get_logger(args.logger_name, args.log_file, is_debug=False)
    logger.critical('######----------->>>>main<<<<-----------######')
    # logger.info(f"args:{args}")
    main(args)


import os
import sys
import click

from app import create_app

app = create_app(os.getenv('FLASK_CONFIG') or 'default')

COV = None
if os.environ.get('FLASK_COVERAGE'):
    import coverage

    COV = coverage.coverage(branch=True, include='app/*')
    COV.start()

@app.cli.command()
@click.option('--coverage/--no-coverage', default=False, help='Run tests with code coverage.')
def test(coverage):
    """Run unit tests."""
    if coverage and not os.environ.get('FLASK_COVERAGE'):
        os.environ['FLASK_COVERAGE'] = '1'
        os.execvp(sys.executable, [sys.executable] + sys.argv)

    import unittest
    tests = unittest.TestLoader().discover('tests')
    unittest.TextTestRunner(verbosity=2).run(tests)

    if COV:
        COV.stop()
        COV.save()
        COV.report()
        basedir = os.path.abspath(os.path.dirname(__file__))
        cov_dir = os.path.join(basedir, 'tmp/coverage')
        COV.html_report(directory=cov_dir)
        print('Coverage HTML: file://%s/index.html' % cov_dir)
        COV.erase()


import pytest

import show

class TestGetFailedTemplate(object):
    @pytest.mark.parametrize(
        "expected",
        [({"text": open("src/bookmarklet_result/templates/failed.html.j2").read()})],
    )
    def test_normal(self, expected):
        actual = show.get_failed_template()
        assert actual == expected["text"]

class TestGetSuccessTemplate(object):
    @pytest.mark.parametrize(
        "expected",
        [({"text": open("src/bookmarklet_result/templates/success.html.j2").read()})],
    )
    def test_normal(self, expected):
        actual = show.get_success_template()
        assert actual == expected["text"]


#! python3
# -*- coding: utf-8 -*-

# -------------------- IMPORTANT LAUNCH INFO --------------------------
# Spyder (Anaconda v.2019-07) does not support 'QtWidgets.qApp.quit()'
# If you use Anaconda3 v.2019-07 comment out the line 95
# If you use PyCharm v2019.2.2 use raw code (line 95 is not commented)
# Data line edit is used for data file (*.*)
# Sensor info line edit is used for sensor information (*.*)
# The requirements for sensor_info.xlsx see in the Handler header
# IDE versions're given taking into account the time of writing the code
# ----------------------------------------------------------------------

# Return file result.pdf to code launch directory
# E.S.Tsyrendorzhiev E1-92 2019

from PyQt5 import QtWidgets
from PyQt5.QtGui import QIcon
import rawsHandler as hdl

class MyWindow(QtWidgets.QWidget):
    def __init__(self):
        QtWidgets.QWidget.__init__(self)
        self.setWindowIcon(QIcon(r'F:\Загрузки\ejector_project\ejector_raw\data.ico'))

        self.dataLabel = QtWidgets.QLabel('Data: ')
        self.sensorLabel = QtWidgets.QLabel('Sensor info: ')
        self.parametersLabel = QtWidgets.QLabel('Parameters info: ')

        self.dataEdit = QtWidgets.QLineEdit(r"F:\ejector_raw_files\2")
        self.sensorEdit = QtWidgets.QLineEdit(r"F:\ejector_raw_files\sensor_status")
        self.parametersEdit = QtWidgets.QLineEdit(r"F:\Загрузки\ejector_project\ejector_raw_files\parameters")

        self.dataBrowse = QtWidgets.QPushButton('Browse', self)
        self.dataBrowse.clicked.connect(self.btnDataClicked)

        self.sensorBrowse = QtWidgets.QPushButton('Browse', self)
        self.sensorBrowse.clicked.connect(self.btnSensorClicked)

        self.parametersBrowse = QtWidgets.QPushButton('Browse', self)
        self.parametersBrowse.clicked.connect(self.btnParametersClicked)

        self.save = QtWidgets.QPushButton('Расходная', self)
        self.save.clicked.connect(self.btnSaveClicked)

        self.saveThrottle = QtWidgets.QPushButton('Дроссельная', self)
        self.saveThrottle.clicked.connect(self.btnThrottleSaveClicked)
        # self.throttle_btn = QtWidgets.QPushButton('Throttle')
        # self.massflow_btn = QtWidgets.QPushButton('Mass Flow')
        # self.massflow_btn.setCheckable(True)
        # self.throttle_btn.setCheckable(True)
        # self.throttle_btn.clicked[bool].connect(self.setType)
        # self.massflow_btn.clicked[bool].connect(self.setType)

        self.vbox_main = QtWidgets.QVBoxLayout()
        self.hbox_main = QtWidgets.QHBoxLayout()

        self.vbox_mid = QtWidgets.QVBoxLayout()
        self.vbox_mid.addWidget(self.dataLabel)
        self.vbox_mid.addWidget(self.sensorLabel)
        self.vbox_mid.addWidget(self.parametersLabel)
        # self.vbox_mid.addWidget(self.checkboxLabel)

        self.vbox_left = QtWidgets.QVBoxLayout()
        self.vbox_left.addWidget(self.dataEdit)
        self.vbox_left.addWidget(self.sensorEdit)
        self.vbox_left.addWidget(self.parametersEdit)

        self.vbox_right = QtWidgets.QVBoxLayout()
        self.vbox_right.addWidget(self.dataBrowse)
        self.vbox_right.addWidget(self.sensorBrowse)
        self.vbox_right.addWidget(self.parametersBrowse)

        self.hbox_main.addLayout(self.vbox_mid)
        self.hbox_main.addLayout(self.vbox_left)
        self.hbox_main.addLayout(self.vbox_right)
        self.vbox_main.addLayout(self.hbox_main)
        self.vbox_main.addWidget(self.save)
        self.vbox_main.addWidget(self.saveThrottle)

        self.setLayout(self.vbox_main)

    def btnDataClicked(self):
        fname = QtWidgets.QFileDialog.getOpenFileName(self)[0]
        self.dataEdit.setText(fname)

    def btnSensorClicked(self):
        fname = QtWidgets.QFileDialog.getOpenFileName(self)[0]
        self.sensorEdit.setText(fname)

    def btnParametersClicked(self):
        fname = QtWidgets.QFileDialog.getOpenFileName(self)[0]
        self.dataEdit.setText(fname)

    def setType(self, pressed):
        source = self.sender()

        if pressed:
            val = 255
        else:
            val = 0

        if source.text() == 'Throttle':
            self.col.setThrottle(val)
        elif source.text() == 'Mass Flow':
            self.col.setMassFlow(val)

    def btnSaveClicked(self):
        fname = QtWidgets.QFileDialog.getSaveFileName(self)[0]
        data = hdl.opener(self.dataEdit.text())
        sensor_info = hdl.opener(self.sensorEdit.text())
        parameters_file = self.parametersEdit.text()

        params = hdl.set_experiment_parameters(parameters_file)
        ver_data = hdl.verification(sensor_info.T, data)
        data_dict = hdl.av(ver_data)
        p_or, p_01, p_04, p_02, ej_coeff_list, comp_ratio_list, eff_list, m1_list, m2_list = hdl.solver(data_dict,
                                                                                params)
        trans = hdl.transient(ver_data)
        p_or_tns, p_01_tns, p_04_tns, p_02_tns, ej_coeff_list_tns, comp_ratio_list_tns, eff_list_tns, m1_list_tns, m2_list_tns = hdl.solver(
            trans, params)

        self.graph = {
            (r'Коэффициент эжекции, k', r'$p_{02}$, кПа', 'Title_1'): (
                ej_coeff_list, p_02, ej_coeff_list_tns, p_02_tns),
            (r'$p_{04}$, кПа', r'$p_{02}$, кПа', 'Title_2'): (p_04, p_02, p_04_tns, p_02_tns),
            (r'$p_{04}$, кПа', r'$\varepsilon$', 'Title_3'): (p_04, comp_ratio_list, p_04_tns, comp_ratio_list_tns),
            (r'$p_{04}$, кПа', r'$\eta$', 'Title_4'): (p_04, eff_list, p_04_tns, eff_list_tns)
        }  # Оси и легенды графиков 1-4

        # maximize = ((r'$p_{04}$, МПа', r'$\varepsilon$', 'Title_3'), (r'$p_{04}$, МПа', r'$\eta$', 'Title_4'))
        # fig0, ax0 = plt.subplots()
        # ax0.scatter(0, 1)
        # plt.show()
        hdl.experiment_parameters(m1_list, m1_list_tns, m2_list, m2_list_tns, p_01, p_01_tns, ej_coeff_list, params)
        hdl.mult_plot(self.graph, sensor_info, data_dict)
        hdl.pdf_saver(fname)
        QtWidgets.qApp.quit()

    def btnThrottleSaveClicked(self):
        fname = QtWidgets.QFileDialog.getSaveFileName(self)[0]
        data = hdl.opener(self.dataEdit.text())
        sensor_info = hdl.opener(self.sensorEdit.text())
        parameters_file = self.parametersEdit.text()

        params = hdl.set_experiment_parameters(parameters_file)
        ver_data = hdl.verification(sensor_info.T, data)
        data_dict = hdl.av(ver_data)
        p_or, p_01, p_04, p_02, ej_coeff_list, comp_ratio_list, eff_list, m1_list, m2_list = hdl.solver(data_dict,
                                                                                params)
        trans = hdl.transient(ver_data)
        p_or_tns, p_01_tns, p_04_tns, p_02_tns, ej_coeff_list_tns, comp_ratio_list_tns, eff_list_tns, m1_list_tns, m2_list_tns = hdl.solver(
            trans, params)

        self.graph = {
            (r'$p_{04}$, кПа', r'Коэффициент эжекции, k', 'Зависимость коэффициента эжекции от давления'): (
                ej_coeff_list, p_02, ej_coeff_list_tns, p_02_tns),
            (r'$p_{04}$, кПа', r'$p_{02}$, кПа', 'Title_2'): (p_04, p_02, p_04_tns, p_02_tns),
            (r'$p_{04}$, кПа', r'$\varepsilon$', 'Title_3'): (p_04, comp_ratio_list, p_04_tns, comp_ratio_list_tns),
            (r'$p_{04}$, кПа', r'$\eta$', 'Title_4'): (p_04, eff_list, p_04_tns, eff_list_tns)
        }  # Оси и легенды графиков 1-4

        # maximize = ((r'$p_{04}$, МПа', r'$\varepsilon$', 'Title_3'), (r'$p_{04}$, МПа', r'$\eta$', 'Title_4'))
        # fig0, ax0 = plt.subplots()
        # ax0.scatter(0, 1)
        # plt.show()
        hdl.experiment_parameters(m1_list, m1_list_tns, m2_list, m2_list_tns, p_01, p_01_tns, ej_coeff_list, params)
        hdl.throttle_plot(self.graph, sensor_info, data_dict)
        hdl.pdf_saver(fname)
        QtWidgets.qApp.quit()

if __name__ == '__main__':
    import sys

    app = QtWidgets.QApplication(sys.argv)
    window = MyWindow()
    window.setWindowTitle("Program")
    window.setFixedSize(350, 200)
    window.show()
    sys.exit(app.exec_())


import os
import logging
from telegram import Update
from telegram.ext import ApplicationBuilder, ContextTypes, MessageHandler, CommandHandler, filters
from dotenv import load_dotenv
from aiogram import Bot, Dispatcher, executor, types
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton

# Load environment variables from .env file
load_dotenv()

# Get the Telegram Bot token from .env file
TELEGRAM_BOT_TOKEN = os.getenv("ENV_TELEGRAM_TOKEN")

logging.basicConfig(
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    level=logging.INFO
)

# Buttons
enter_the_world = InlineKeyboardButton(text="Enter the World! ", callback_data="In_First_button")

# STARTING THE GAME
async def start(update: Update, context: ContextTypes.DEFAULT_TYPE):
    await context.bot.send_message(chat_id=update.effective_chat.id,
                                   text="Hello, welcome to the game OMEGA! Type /rules to read the instructions.")

# LIST GAME RULES
async def rules(update: Update, context: ContextTypes.DEFAULT_TYPE):
    await context.bot.send_message(chat_id=update.effective_chat.id,
                                   text="Welcome to the OMEGA, my friends. Here are some rules of the game... To start the game, use /start_game")

# INTRO TO THE WORLD
async def start_game(update: Update, context: ContextTypes.DEFAULT_TYPE):
    # Photo caption
    caption = "In a realm where the past meets the future, welcome to the Omega, a world where the charm of steampunk entwines with the relentless advance of cybernetic technology. Here, towering brass and iron structures coexist harmoniously with sleek digital computers, creating a vibrant symphony of innovation. You, a daring adventurer, have been chosen to shape the Empire's fate. Legends speak of powerful ancient artifacts scattered across the land, each capable of either bolstering the regime's dominion or leading to its downfall. Embark on a quest of wonder, mystery, and danger, where your decisions will shape the destiny of this unique steampunk-cyberpunk universe, unraveling the enigma as your adventure begins."

    await context.bot.send_photo(chat_id=update.effective_chat.id,
                                   photo="./img/cityscape.png",
                                   caption=caption)

if __name__ == '__main__':
    application = ApplicationBuilder().token(TELEGRAM_BOT_TOKEN).build()

    start_handler = CommandHandler('start', start)
    rules_handler = CommandHandler('rules', rules)
    start_game_handler = CommandHandler('start_game', start_game)

    application.add_handler(start_handler)
    application.add_handler(rules_handler)
    application.add_handler(start_game_handler)

    application.run_polling()


__version__ = '0.5.5'

from docutils.parsers.rst import directives
from docutils import nodes
from sphinx.directives.code import CodeBlock
from sphinx.util.docutils import SphinxDirective
from sphinx.util.fileutil import copy_asset_file

import os

try:
    from sphinx.util.texescape import escape as latex_escape
except ImportError:     # ancient sphinx:
    from sphinx.util.texescape import tex_escape_map

    def latex_escape(s, latex_engine):
        return s.translate(tex_escape_map)

CSS_FILE = "code-tabs.css"
JS_FILE = "code-tabs.js"
STY_FILE = "tabenv.sty"

_html_builders = [
    "html",
    "singlehtml",
    "dirhtml",
    "readthedocs",
    "readthedocsdirhtml",
    "readthedocssinglehtml",
    "readthedocssinglehtmllocalmedia",
    "spelling",
]

_latex_builders = [
    "latex",
]

class TabsNode(nodes.container): pass
class TabNode(nodes.container): pass
class TabBarNode(nodes.Part, nodes.Element): pass
class TabButtonNode(nodes.Part, nodes.Element): pass

class TabsDirective(SphinxDirective):

    """
    This directive is used to contain a group of code blocks which can be
    selected as tabs of a single notebook.
    """

    final_argument_whitespace = True
    required_arguments = 0
    optional_arguments = 1
    has_content = True

    def run(self):
        self.assert_has_content()

        node = TabsNode()
        node["classes"].append("tabs")
        node["tabgroup"] = self.arguments[0] if self.arguments else None

        self.add_name(node)
        self.state.nested_parse(self.content, self.content_offset, node)

        # Generate navbar:
        if self.env.app.builder.name in _html_builders:
            tabbar = TabBarNode()
            tabbar["classes"].append("tabbar")

            selected = 0
            for i, tab in enumerate(node.children):
                button = TabButtonNode()
                button['classes'].append('tabbutton')
                button['tabid'] = i
                button.append(nodes.Text(tab["tabname"]))
                tabbar.append(button)
                if tab.get('selected'):
                    selected = i

            tabbar.children[selected]['classes'].append('selected')
            node.children[selected]['classes'].append('selected')
            node.insert(0, tabbar)

        return [node]

class TabDirective(SphinxDirective):

    option_spec = {
        'selected': directives.flag,
    }

    final_argument_whitespace = True
    required_arguments = 1
    optional_arguments = 0
    has_content = True

    def run(self):
        index = len(self.state.parent.children)

        title = self.options.get('title')
        if not title:
            title = self.options.get('caption')
        if not title and self.arguments:
            title = self.arguments[0]
        if not title:
            title = "Tab {}".format(index + 1)

        node = TabNode()
        node['tabid'] = index
        node['tabname'] = title
        node['selected'] = 'selected' in self.options
        node['classes'].append('tab')
        node += self.make_page(node)

        return [node]

    def make_page(self, node):
        node['classes'].append("texttab")
        page = nodes.container()
        self.state.nested_parse(self.content, self.content_offset, page)
        return page

class CodeTabDirective(CodeBlock):

    """Single code-block tab inside .. code-tabs."""

    option_spec = CodeBlock.option_spec.copy()
    option_spec.update({
        'title': directives.unchanged,
        'selected': directives.flag,
    })

    run = TabDirective.run

    def make_page(self, node):
        node['classes'].append("codetab")
        if self.env.app.builder.name in _html_builders:
            self.options.pop('caption', None)
        else:
            self.options.setdefault('caption', node['tabname'])
        return super().run()

def visit_tabgroup_html(self, node):
    self.body.append(self.starttag(node, 'div', **{
        'data-tabgroup': node['tabgroup'] or '',
        'class': 'docutils container',
    }))

def depart_tabgroup_html(self, node):
    self.body.append('
')

def visit_tabbar_html(self, node):
    self.body.append(self.starttag(node, 'ul'))

def depart_tabbar_html(self, node):
    self.body.append('
')

def visit_tabbutton_html(self, node):
    self.body.append(self.starttag(node, 'li', **{
        'data-id': node['tabid'],
        'onclick': "sphinx_code_tabs_onclick(this)",
    }))

def depart_tabbutton_html(self, node):
    self.body.append('
')

def visit_tab_html(self, node):
    self.body.append(self.starttag(node, 'div', **{
        'data-id': node['tabid'],
    }))

def depart_tab_html(self, node):
    self.body.append('
')

def visit_tab_latex(self, node):
    if 'texttab' in node['classes']:
        self.body.append(r'\sphinxSetupCaptionForVerbatim{{{}}}'.format(
            latex_escape(node['tabname'], self.config.latex_engine),
        ))
        self.body.append(r'\begin{tab}')

def depart_tab_latex(self, node):
    if 'texttab' in node['classes']:
        self.body.append(r'\end{tab}')

def add_assets(app):
    package_dir = os.path.dirname(__file__)
    app.config.html_static_path.append(package_dir)
    app.add_css_file(CSS_FILE)
    app.add_js_file(JS_FILE)
    if app.builder.name in _latex_builders:
        copy_asset_file(
            os.path.join(package_dir, STY_FILE),
            app.builder.outdir)
        app.add_latex_package('tabenv')

def setup(app):
    app.add_node(TabsNode, html=(visit_tabgroup_html, depart_tabgroup_html))
    app.add_node(TabBarNode, html=(visit_tabbar_html, depart_tabbar_html))
    app.add_node(
        TabButtonNode,
        html=(visit_tabbutton_html, depart_tabbutton_html))
    app.add_node(
        TabNode,
        html=(visit_tab_html, depart_tab_html),
        latex=(visit_tab_latex, depart_tab_latex))
    app.add_directive("tabs", TabsDirective)
    app.add_directive("tab", TabDirective)
    app.add_directive("code-tabs", TabsDirective)
    app.add_directive("code-tab", CodeTabDirective)
    app.connect("builder-inited", add_assets)

    return {
        "version": __version__,
        "parallel_read_safe": True,
    }


'''
Extract interesting spectra from the full HI cube.
'''

import os
from spectral_cube import SpectralCube
import astropy.units as u
import matplotlib.pyplot as p
import seaborn as sn
sn.set_context('poster')
sn.set_style("ticks")

# path = "/srv/astro/erickoch/M33/"
path = "/media/eric/MyRAID/M33/14B-088/HI/full_imaging/"

cube = SpectralCube.read(os.path.join(path, "M33_14B-088_HI.clean.image.fits"),
                         mode='denywrite')

arec_cube = "/media/eric/Data_3/M33/Arecibo/14B-088_items/M33_14B-088_HI_model.fits"
# arec_cube = "/srv/astro/surveys/m33/hi/M33only.fits"

arecibo = SpectralCube.read(arec_cube, mode='denywrite')

# arec_spec = arecibo[:, 170, 132]

# I've imaged a portion of the Southern Arm w/o Arecibo.
vla_path = "/media/eric/MyRAID/M33/14B-088/HI/imaging/south_arm_800_1200.image.fits"
sarm_vla = SpectralCube.read(vla_path, mode='denywrite')

# Interesting positions by-eye
posns = [["South Arm 1", 1383, 1129, -210, -75],
         ["South Arm 2", 1427, 1099, -210, -75],
         ["South Arm 3", 1429, 1109, -210, -75],
         ["South Arm 4", 1413, 1106, -210, -75],
         ["South Arm 5", 1370, 1186, -210, -75],
         ["NGC 604 1", 1106, 1419, -280, -150],
         ["NGC 604 2", 1104, 1432, -280, -150],
         ["NGC 604 3", 1080, 1439, -280, -150],
         ["NGC 604 4", 1078, 1444, -280, -150]]

# p.ioff()

for posn in posns:
    # p.title(posn[0]+" at "+str(posn[1])+","+str(posn[2]))
    filename = \
        os.path.join("/home/eric/Dropbox/M33/",
                     "M33_14B-088_HI.clean.image." +
                     posn[0].replace(" ", "_").lower()+"_" +
                     str(posn[1])+"_"+str(posn[2])+".pdf")
    spec = cube[:, posn[2], posn[1]]
    spec_conv = spec.to(u.K, equivalencies=cube.beam.jtok_equiv(1420.40575177*u.MHz))
    # Temp fix for WCS not being kept after unit conversion
    # spec_conv._wcs = spec.wcs
    spec_conv.with_spectral_unit(u.km/u.s).quicklook(label="VLA + Arecibo", c='b',
                                                     drawstyle='steps-mid', linestyle='-')
    arec_spec = arecibo[:, -posn[2], -posn[1]]
    arec_spec_conv = arec_spec.to(u.K, equivalencies=arecibo.beam.jtok_equiv(1420.40575177*u.MHz))
    arec_spec_conv._wcs = arec_spec.wcs
    arec_spec_conv.with_spectral_unit(u.km/u.s).quicklook(label="Arecibo", c='c',
                                                          drawstyle='steps-mid', linestyle='--')
    # p.plot(arec_spec_conv.spectral_axis[::-1], arec_spec_conv.value,
    #        drawstyle='steps-mid')
    if "South" in posn[0]:
        vla_spec = sarm_vla[:, posn[2], posn[1]]
        vla_spec_conv = vla_spec.to(u.K, equivalencies=sarm_vla.beam.jtok_equiv(1420.40575177*u.MHz))
        # vla_spec_conv._wcs = vla_spec.wcs
        vla_spec_conv.with_spectral_unit(u.km/u.s).quicklook(label="VLA", c='r',
                                                             drawstyle='steps-mid',
                                                             linestyle='-')
    # p.plot([posn[3], posn[4]], [0.0]*2, 'k--')
    p.xlim([posn[3], posn[4]])
    p.xlabel("Velocity (km/s)")
    p.ylabel("Intensity (K)")
    p.legend(loc='best')
    # p.savefig(filename)
    p.draw()
    raw_input("Next plot?")
    # p.show()
    p.clf()


from django.db import models
import hashlib
from pathlib import Path

class Target(models.Model):
    id = models.BigAutoField(primary_key=True)
    pqa_id = models.BigIntegerField('Permanent ID in engine', unique=True, null=True, blank=True, db_index=True)
    retain = models.BooleanField(default=True)
    title = models.CharField(max_length=255)
    link = models.TextField(max_length=65535)
    # The limit of 255 characters in the path is not because of Windows path length, but because of
    #   MySQL limitation: https://stackoverflow.com/questions/36953538/mysql-column-size-limit
    image = models.ImageField(max_length=255,upload_to='Uploads/TargetImages/%Y/%m/%d/')
    thumbnail = models.ImageField(max_length=255,upload_to='Uploads/TargetThumbnails/%Y/%m/%d/', null=True, blank=True)
    description = models.TextField(max_length=65535)
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    def __str__(self):
        return 'Title=[%s], Link=[%s], ImageMD5=[%s], Created=[%s], Modified=[%s]' % (
            self.title, self.link,
            hashlib.md5(Path(self.image.storage.path(self.image.name)).read_bytes()).hexdigest(),
            str(self.created), str(self.modified))

class Question(models.Model):
    id = models.BigAutoField(primary_key=True)
    pqa_id = models.BigIntegerField('Permanent ID in engine', unique=True, null=True, blank=True, db_index=True)
    retain = models.BooleanField(default=True)
    message = models.CharField(max_length=255)
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    def __str__(self):
        return 'PqaID=[%s], Message=[%s], Created=[%s], Modified=[%s]' % (
            str(self.pqa_id), self.message, str(self.created), str(self.modified))

class Answer(models.Model):
    id = models.BigAutoField(primary_key=True)
    question = models.ForeignKey(Question, on_delete=models.CASCADE)
    # For quiz resuming to work, answer options must not change their positions once created.
    option_pos = models.BigIntegerField('Option position number')
    message = models.CharField(max_length=255)
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    def __str__(self):
        return 'QuestionPqaID=[%s], OptionPos=[%d], Message=[%s], Created=[%s], Modified=[%s]' % (
            str(self.question.pqa_id), self.option_pos, self.message, str(self.created), str(self.modified))

class Quiz(models.Model):
    id = models.BigAutoField(primary_key=True)
    pqa_id = models.BigIntegerField('Permanent ID in engine', unique=True, db_index=True)
    active_question = models.ForeignKey(Question, on_delete=models.SET_NULL, null=True, blank=True)
    # https://github.com/un33k/django-ipware
    # https://stackoverflow.com/a/16203978/1915854
    user_ip = models.CharField(max_length=255, null=True, blank=True)
    username = models.CharField(max_length=255)
    created = models.DateTimeField(auto_now_add=True)

class QuizChoice(models.Model):
    id = models.BigAutoField(primary_key=True)
    quiz = models.ForeignKey(Quiz, on_delete=models.CASCADE)
    question_pqa_id = models.BigIntegerField()
    i_answer = models.BigIntegerField()

class QuizTarget(models.Model):
    id = models.BigAutoField(primary_key=True)
    quiz = models.ForeignKey(Quiz, on_delete=models.CASCADE)
    target_pqa_id = models.BigIntegerField()
    # It's not entirely a foreign key because it marks the end for a range of choices
    last_choice_id = models.BigIntegerField()

class UserTraining(models.Model):
    id = models.BigAutoField(primary_key=True)
    user_ip = models.CharField(max_length=255, null=True, blank=True)
    username = models.CharField(max_length=255)
    created = models.DateTimeField(auto_now_add=True)
    target_pqa_id = models.BigIntegerField()
    query_url = models.TextField(max_length=16777211)  # A little less than 2**24

class KnowledgeBase(models.Model):
    id = models.BigAutoField(primary_key=True)
    path = models.CharField(max_length=255)
    timestamp = models.DateTimeField(auto_now_add=True, db_index=True)

    def __str__(self):
        return 'path=[%s], timestamp=[%s]' % (self.path, str(self.timestamp))


import math
import numpy as np
import matplotlib.pyplot as plt

class Diabatic_Model:
    def __init__(self, num_states):
        self.num_states = num_states

    def get_adiabatic(self, x):
        v, ev = np.linalg.eig(self.V(x))
        d = {}
        for i in range(len(v)):
            d[v[i]] = ev[:, i]

        v_sorted = np.sort(v)
        ev_sorted = np.zeros((self.num_states, self.num_states))
        for i in range(len(v_sorted)):
            ev_sorted[:, i] = d[v[i]]

        return v_sorted, ev_sorted

    def get_adiabatic_energy(self, x):
        return self.get_adiabatic(x)[0]

    def get_wave_function(self, x):
        return self.get_adiabatic(x)[1]

    def get_d_adiabatic_energy(self, x, step=0.00001):
        v1 = self.get_adiabatic_energy(x + step)
        v0 = self.get_adiabatic_energy(x - step)
        return (v1 - v0)/(2*step)

    def get_d_wave_functions(self, x, step=0.00001):
        phi1 = self.get_wave_function(x + step)
        phi0 = self.get_wave_function(x - step)
        return (phi1 - phi0)/(2*step)

class Simple_Avoided_Crossing(Diabatic_Model):
    def __init__(self, A=0.01, B=1.6, C=0.005, D=1.0, discont=0):
        self.A = A
        self.B = B
        self.C = C
        self.D = D
        self.discont = discont
        self.num_states = 2

        super().__init__(self.num_states)

    def V(self, x):
        if x > self.discont:
            V11 = self.A*(1-(math.exp(-self.B*x)))
        else:
            V11 = -self.A*(1-(math.exp(self.B*x)))

        V22 = -V11
        V12 = V21 = self.C*math.exp(-self.D*(x**2))

        return np.asarray([[V11, V12], [V21, V22]])

    def dV(self, x):
        if x > self.discont:
            dV11 = self.A*self.B*x*math.exp(-self.B*x)
        else:
            dV11 = -self.A*self.B*x*math.exp(self.B*x)

        dV22 = -dV11
        dV12 = dV21 = -2*self.C*self.D*x*math.exp(-self.D*(x**2))

        return np.asarray([[dV11, dV12], [dV21, dV22]])

class Double_Avoided_Crossing(Diabatic_Model):
    def __init__(self, A=.1, B=.28, E0=.05, C=.015, D=.06):
        self.A = A
        self.B = B
        self.E0 = E0
        self.C = C
        self.D = D
        self.num_states = 2

        super().__init__(self.num_states)

    def V(self, x):
        V11 = 0.0
        V22 = (-self.A*math.exp(-self.B*(x**2))) + self.E0
        V12 = V21 = self.C*math.exp(-self.D*(x**2))

        return np.asarray([[V11, V12], [V21, V22]])

    def dV(self, x):
        dV11 = 0
        dV22 = 2*self.A*self.B*x*math.exp(-self.B*(x**2))
        dV12 = dV21 = -2*self.C*self.D*x*math.exp(-self.D*(x**2))

        return np.asarray([[dV11, dV12], [dV21, dV22]])

class Extended_Coupling_With_Reflection(Diabatic_Model):
    def __init__(self, A=6e-4, B=.1, C=.9, discont=0):
        self.A = A
        self.B = B
        self.C = C
        self.discont = discont
        self.num_states = 2

        super().__init__(self.num_states)

    def V(self, x):
        V11 = self.A
        V22 = -self.A
        if x > self.discont:
            V12 = V21 = self.B*(2-math.exp(-self.C*x))
        else:
            V12 = V21 = self.B*math.exp(self.C*x)

        return np.asarray([[V11, V12], [V21, V22]])

    def dV(self, x):
        dV11 = 0
        dV22 = 0
        if x > self.discont:
            dV12 = dV21 = self.B*self.C*math.exp(-self.C*x)
        else:
            dV12 = dV21 = self.C*self.B*math.exp(self.C*x)

        return np.asarray([[dV11, dV12], [dV21, dV22]])

# Plots adiabatic potential from diabatic representation

def plot_adiabatic_potential(model, x0, x1, num_iter, coupling_scaling_factor):
    x_linspace = np.linspace(x0, x1, num_iter)

    # adiabatic potential vectors
    adiabatic_1 = np.zeros(len(x_linspace))
    adiabatic_2 = np.zeros(len(x_linspace))

    # d12 vector
    d12 = np.zeros(len(x_linspace))
    d21 = np.zeros(len(x_linspace))
    for i in range(len(x_linspace)):
        x = x_linspace[i]

        # Get adiabatic representation
        potential, ev = model.get_adiabatic(x)
        adiabatic_1[i] = potential[0]
        adiabatic_2[i] = potential[1]

        # Calcualte d12
        grad_phi = model.get_d_wave_functions(x)
        d12[i] = ev[:, 0]@grad_phi[:, 1]
        d21[i] = ev[:, 1]@grad_phi[:, 0]

    plt.plot(x_linspace, adiabatic_1)
    plt.plot(x_linspace, adiabatic_2)
    plt.plot(x_linspace, d12*coupling_scaling_factor)
   # plt.plot(x_linspace, d21*coupling_scaling_factor)
    plt.show()


from enum import IntEnum

class Limits(IntEnum):
    # Максимальная длинна названия игридиента
    MAX_LEN_INGREDIENT_NAME = 100

    # Максимальная длинна еденицы измерения
    MAX_LEN_MEASUREMENT_UNIT = 20

    # Максимальная длинна имяни рецептв
    MAX_LEN_TAG_NAME = 200

    # Длинна цветового HEX-кода
    MAX_LEN_COLOR_HEX_CODE = 7

    # Максимальная длина слага
    MAX_LEN_SLUG = 200

    # Максимальная длинна имяни рецепта
    MAX_LEN_RECIPE_NAME = 200

    # Минимальное время приготовления рецепта в минутах
    MIN_COOKING_TIME = 1

    # Максимальная длина email (User)
    MAX_LEN_EMAIL_FIELD = 256


"""
https://leetcode.com/problems/add-one-row-to-tree/

https://leetcode.com/submissions/detail/107013179/
"""

from common.tree_node import TreeNode

class Solution(object):
    def addOneRow(self, root, v, d):
        """
        :type root: TreeNode
        :type v: int
        :type d: int
        :rtype: TreeNode
        """
        if d == 1:
            treeNode = TreeNode(v)
            treeNode.left = root
            return treeNode
        elif d == 2:
            left = root.left
            right = root.right
            root.left = TreeNode(v)
            root.right = TreeNode(v)
            root.left.left = left
            root.right.right = right
            return root
        else:
            if root.left:
                self.addOneRow(root.left, v, d - 1)
            if root.right:
                self.addOneRow(root.right, v, d - 1)
            return root

import unittest
from common.assert_tree_node_equal import assertTreeNodeEqual

class Test(unittest.TestCase):
    def test(self):
        solution = Solution()
        root = TreeNode(4)
        root.left = TreeNode(2)
        root.left.left = TreeNode(3)
        root.left.right = TreeNode(1)
        root.right = TreeNode(6)
        root.right.left = TreeNode(5)
        newRoot = TreeNode(4)
        newRoot.left = TreeNode(1)
        newRoot.left.left = TreeNode(2)
        newRoot.left.left.left = TreeNode(3)
        newRoot.left.left.right = TreeNode(1)
        newRoot.right = TreeNode(1)
        newRoot.right.right = TreeNode(6)
        newRoot.right.right.left = TreeNode(5)
        self.assertTrue(assertTreeNodeEqual(
            solution.addOneRow(root, 1, 2), newRoot))

if __name__ == '__main__':
    unittest.main()


import os
from datetime import datetime
from decimal import Decimal
from functools import partial
from typing import Any
from typing import Callable
from typing import Tuple

import arrow
from django.conf import settings
from django.db import models
from django.db.models import Sum
from django.utils.translation import ugettext_lazy as _
from django_fsm import RETURN_VALUE
from django_fsm import FSMField
from django_fsm import transition
from simple_history.models import HistoricalRecords

from cashflow.bill.choices import BillTypeChoices
from cashflow.bill.choices import BillYearChoices
from cashflow.bill.fees.management_fees import calculate_management_fees
from cashflow.bill.fees.membership_fees import calculate_membership_fees
from cashflow.bill.fees.rhapsody_fees import calculate_rhapsody_fees
from cashflow.bill.fees.upfront_fees import calculate_upfront_fees
from cashflow.bill.models_abstracts import BillAbstract
from cashflow.bill.models_abstracts import BillInvoiceManagment
from cashflow.bill.models_abstracts import BillStateManagement
from cashflow.bill.utils import generate_bill_template
from cashflow.bill.utils import get_document_context
from cashflow.bill.utils import get_output_path
from core_auth.choices import PreferredLanguageChoices
from core_management.models import CoreModel
from core_management.models import PlatformRelation
from ort_files.documentbuilder.utils.utils_file import convert_to_pdf
from ort_files.general.utils.filefield_utils import get_file
from ort_files.general.utils.s3_utils import get_s3_presigned_url
from ort_files.general.utils.s3_utils import save_to_s3
from utilities.exceptions import ForbiddenUpdateError

from .models_utils import BillUtils

def bill_upload_to(instance, filename):
    return "v2/bills/%s/%s" % (instance.id, filename)

class Bill(
    BillAbstract, BillStateManagement, BillInvoiceManagment, BillUtils, CoreModel
):
    platform_relation = models.OneToOneField(
        PlatformRelation,
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        verbose_name="Platform relation",
    )
    updated_at = models.DateTimeField(
        auto_now=True, null=True, blank=True, verbose_name="Updated at"
    )
    updated_by = models.ForeignKey(
        "core_auth.CustomUser",
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        related_name="bills_updated",
        verbose_name="Updated by",
    )
    type = models.CharField(
        max_length=140,
        null=True,
        blank=True,
        choices=BillTypeChoices.choices(),
        default="upfront_fees",
        verbose_name="Bill type",
    )

    year = models.IntegerField(
        null=True,
        blank=True,
        choices=BillYearChoices,
        default=datetime.now().year,
        verbose_name=_("Issuing year"),
    )
    invoice_number_deprecated = models.CharField(
        max_length=255, null=True, blank=True, verbose_name="Invoice number Deprecated"
    )

    invoice_number = models.CharField(
        max_length=255, null=True, blank=True, verbose_name="Invoice number"
    )
    investor_name = models.CharField(max_length=255, blank=True, null=True)
    cc_emails = models.TextField(blank=True, null=True)
    sendinblue_template_id = models.IntegerField(blank=False, null=True)
    last_sent = models.DateTimeField(null=True, blank=True, verbose_name="Last sent")
    # File fields
    file = models.FileField(
        null=True, blank=True, upload_to=bill_upload_to, verbose_name=_("Issuing file")
    )

    # # Relations
    investment = models.ForeignKey(
        "investment.Investment",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="bills",
        verbose_name="Investment",
    )
    investor = models.ForeignKey(
        "investor.Investor",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name="bills",
        verbose_name="Investor",
    )

    amount_due = models.DecimalField(
        max_digits=19,
        null=True,
        blank=True,
        decimal_places=2,
        default=Decimal("0.00"),
        verbose_name=_("Amount due"),
    )
    fees_amount_due = models.DecimalField(
        max_digits=19,
        null=True,
        blank=True,
        decimal_places=2,
        default=Decimal("0.00"),
        verbose_name=_("Fees Amount Due"),
    )

    log = HistoricalRecords(related_name="history")

    class Meta:
        verbose_name = "Bill"
        verbose_name_plural = "Bills"

    def save(self, *args, **kwargs):
        if not self.can_update(kwargs.get("update_fields", [])):
            raise ForbiddenUpdateError(f"Cannot Update {self.status} bill ")
        if self.type != BillTypeChoices.rhapsody_fees.name:
            self.update_fees_amount()
        self.update_investor_trail()
        super(Bill, self).save(*args, **kwargs)

    def get_cash_call(self) -> "CashCall":
        return self.cashcalls.last()

    def get_investor(self):
        return self.investment.investor if self.investment else self.investor

    def get_payin(self):
        return getattr(self, "payin", None)

    def get_fundraising(self):
        return self.investment.fundraising if self.investment else None

    def get_owner_email(self):
        investor = (
            self.investor
            if self.investor
            else self.investment.investor
            if self.investment
            else None
        )
        if investor:
            owner = investor.get_owner_user()
            return owner.email if owner else None
        return None

    def format_invoice(self, payin: dict):
        formatter_function_by_type: dict = {
            "management_fees": partial(get_document_context, self),
            "upfront_fees": self._format_upfront_fees_invoice_params,
            "credit_notes": self._format_upfront_fees_invoice_params,
            "rhapsody_fees": self._format_rhapsody_fees_invoice_params,
            "membership_fees": self._format_membership_fees_invoice_params,
        }
        formatter_function: Callable = formatter_function_by_type[self.type]
        return formatter_function(payin)

    def get_fee_function_by_type(self):
        bill_type = self.type
        fees_functions = {
            BillTypeChoices.upfront_fees.name: calculate_upfront_fees,
            BillTypeChoices.management_fees.name: calculate_management_fees,
            BillTypeChoices.rhapsody_fees.name: calculate_rhapsody_fees,
            BillTypeChoices.membership_fees.name: calculate_membership_fees,
        }
        return fees_functions.get(bill_type)

    def calculate_fees(self) -> Tuple[Decimal, Decimal]:
        fee_function = self.get_fee_function_by_type()
        return fee_function(self.investment, invested_amount=self.amount_due)

    def create_bill_file(self, payin: dict):
        if self.type == "management_fees":
            if self.file:
                self.file.delete()
            self.set_sendinblue_template_id()
            current_path, file = generate_bill_template(self, payin)
            output_path = get_output_path("docx", self)
            pdf_output_path = get_output_path("pdf", self)
            save_to_s3(current_path, output_path)
            convert_to_pdf(output_path, pdf_output_path)
            presigned_url = get_s3_presigned_url(pdf_output_path)
            file_content = get_file(presigned_url)
            self.file.save(name=pdf_output_path, content=file_content)
            os.remove(current_path)
        return self.file

    def set_sendinblue_template_id(self) -> None:
        preferred_language: PreferredLanguageChoices = self.get_owner_language()
        self.sendinblue_template_id = settings.SENDINBLUE_TEMPLATES_IDS[self.type][
            preferred_language
        ]
        self.save()


import json
from nuscenes import NuScenes
from pathlib import Path

sample_file = "data/nuScenes/v1.0-trainval/sample.json"
sample_data_file = "data/nuScenes/v1.0-trainval/sample_data.json"
sample_annotation_file = "data/nuScenes/v1.0-trainval/sample_annotation.json"
scene_file = "data/nuScenes/v1.0-trainval/scene.json"
root_path = "data/nuScenes"
version = "v1.0-trainval"

def _get_available_scenes(nusc):
  available_scenes = []
  print("total scene num:", len(nusc.scene))
  for scene in nusc.scene:
    scene_token = scene["token"]
    scene_rec = nusc.get("scene", scene_token)
    sample_rec = nusc.get("sample", scene_rec["first_sample_token"])
    sd_rec = nusc.get("sample_data", sample_rec["data"]["LIDAR_TOP"])
    has_more_frames = True
    scene_not_exist = False
    while has_more_frames:
      lidar_path, boxes, _ = nusc.get_sample_data(sd_rec["token"])
      if not Path(lidar_path).exists():
        scene_not_exist = True
        break
      else:
        break
      if not sd_rec["next"] == "":
        sd_rec = nusc.get("sample_data", sd_rec["next"])
      else:
        has_more_frames = False
    if scene_not_exist:
      continue
    available_scenes.append(scene)
  print("exist scene num:", len(available_scenes))
  return available_scenes

nusc = NuScenes(version=version, dataroot=root_path, verbose=True)
available_scenes = _get_available_scenes(nusc)
# available_scene_names = [s["name"] for s in available_scenes]
available_scene_token = [s["token"] for s in available_scenes]

with open(scene_file, 'r') as f:
  scenes = json.load(f)
out_scenes = []
with open('scene.json', 'w') as f:
  for scene in scenes:
      scene_token = scene['token']
      if scene_token in available_scene_token:
        out_scenes.append(scene)
  json.dump(out_scenes, f)

with open(sample_file, 'r') as f:
  samples = json.load(f)

out_sample = []
with open('sample.json', 'w') as f:
  for sample in samples:
      sample_scene_token = sample['scene_token']
      if sample_scene_token in available_scene_token:
        out_sample.append(sample)
  json.dump(out_sample, f)

available_sample_token = [s["token"] for s in out_sample]

with open(sample_data_file, 'r') as f:
  sample_datas = json.load(f)
out_sample_data = []
with open('sample_data.json', 'w') as f2:
  for sample_data in sample_datas:
      sample_token = sample_data['sample_token']
      if sample_token in available_sample_token:
        out_sample_data.append(sample_data)
  json.dump(out_sample_data, f2)

with open(sample_annotation_file, 'r') as f:
  sample_annotations = json.load(f)
out_sample_annotation = []
with open('sample_annotation.json', 'w') as f3:
  for sample_annotation in sample_annotations:
      sample_token = sample_annotation['sample_token']
      if sample_token in available_sample_token:
        out_sample_annotation.append(sample_annotation)
  json.dump(out_sample_annotation, f3)


import numpy as np
import math
import os, sys
import open3d as o3d

pyexample_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(pyexample_path)

from open3d_example import *

def scalable_integrate_rgb_frames(path_dataset, intrinsic, config):
    poses = []
    [color_files, depth_files] = get_rgbd_file_lists(path_dataset)
    n_files = len(color_files)
    n_fragments = int(math.ceil(float(n_files) / \
            config['n_frames_per_fragment']))
    volume = o3d.pipelines.integration.ScalableTSDFVolume(
        voxel_length=config["tsdf_cubic_size"] / 512.0,
        sdf_trunc=0.04,
        color_type=o3d.pipelines.integration.TSDFVolumeColorType.RGB8)

    pose_graph_fragment = o3d.io.read_pose_graph(
        join(path_dataset, config["template_refined_posegraph_optimized"]))

    for fragment_id in range(len(pose_graph_fragment.nodes)):
        pose_graph_rgbd = o3d.io.read_pose_graph(
            join(path_dataset,
                 config["template_fragment_posegraph_optimized"] % fragment_id))

        for frame_id in range(len(pose_graph_rgbd.nodes)):
            frame_id_abs = fragment_id * \
                    config['n_frames_per_fragment'] + frame_id
            print(
                "Fragment %03d / %03d :: integrate rgbd frame %d (%d of %d)." %
                (fragment_id, n_fragments - 1, frame_id_abs, frame_id + 1,
                 len(pose_graph_rgbd.nodes)))
            rgbd = read_rgbd_image(color_files[frame_id_abs],
                                   depth_files[frame_id_abs], False, config)
            pose = np.dot(pose_graph_fragment.nodes[fragment_id].pose,
                          pose_graph_rgbd.nodes[frame_id].pose)
            volume.integrate(rgbd, intrinsic, np.linalg.inv(pose))
            poses.append(pose)

    mesh = volume.extract_triangle_mesh()
    mesh.compute_vertex_normals()
    if config["debug_mode"]:
        o3d.visualization.draw_geometries([mesh])

    mesh_name = join(path_dataset, config["template_global_mesh"])
    o3d.io.write_triangle_mesh(mesh_name, mesh, False, True)

    traj_name = join(path_dataset, config["template_global_traj"])
    write_poses_to_log(traj_name, poses)

def run(config):
    print("integrate the whole RGBD sequence using estimated camera pose.")
    if config["path_intrinsic"]:
        intrinsic = o3d.io.read_pinhole_camera_intrinsic(
            config["path_intrinsic"])
    else:
        intrinsic = o3d.camera.PinholeCameraIntrinsic(
            o3d.camera.PinholeCameraIntrinsicParameters.PrimeSenseDefault)
    scalable_integrate_rgb_frames(config["path_dataset"], intrinsic, config)


import pickle
import random

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

class PreprocessingDataSet:

    def __init__(self):
        random.seed(1725)
        self.word_features = {}
        self.stop_words = stopwords.words('english')

    # for removing stopping word
    def remove_stopping_word(self, raw_data):
        filtered_list = []
        for w in raw_data:
            if not w in self.stop_words:
                filtered_list = [w]
        return filtered_list

    # have to implement this function to get more accuracy in result
    def getUniqueItems(self, items):
        uniquelist = []
        for item in items:
            if item not in uniquelist:
                uniquelist.append(item)
        return uniquelist

    def word_to_vector(self, word_features):
        return {token: idx for idx, token in enumerate(set(word_features))}

    def find_features(self, document):
        words = word_tokenize(document)
        features = {}
        for w in words:
            features[w] = (w in self.word_features.keys())
        return features

    def get_training_testing_dataset(self, training_percentage=0.5):
        spathneg = "dataset/negativesenwosc.txt"
        spathpos = "dataset/positivesenwosc.txt"

        short_pos = open(spathpos, "r", encoding='utf-8').read()
        short_neg = open(spathneg, "r", encoding='utf-8').read()

        # move this up here
        all_words = []
        documents = []

        # word tag and its meaning  J is for adjective,
        # V is for verb
        allowed_word_types = ["J", "V"]
        print("\n70 30 verb adjective")

        for p in short_pos.split('\n\n\n'):
            documents.append((p, "pos"))
            words = word_tokenize(p)
            pos = nltk.pos_tag(words)
            for w in pos:
                if w[1][0] in allowed_word_types:
                    all_words.append(w[0].lower())

        for p in short_neg.split('\n\n\n'):
            documents.append((p, "neg"))
            words = word_tokenize(p)
            pos = nltk.pos_tag(words)
            for w in pos:
                if w[1][0] in allowed_word_types:
                    all_words.append(w[0].lower())

        # saving documents "no need can be removed"
        save_documents = open("pickle_data_model_save/documents.pickle", "wb")
        pickle.dump(documents, save_documents)
        save_documents.close()

        # counting freq of words
        all_words = nltk.FreqDist(all_words)

        # selecting 5000 word for training
        self.word_features = list(all_words.keys())[:5000]

        # converting word to vector
        self.word_features = self.word_to_vector(self.word_features)

        # saving word_features for testing purpose
        save_word_features = open("pickle_data_model_save/word_features.pickle", "wb")
        pickle.dump(self.word_features, save_word_features)
        save_word_features.close()

        # prepating feature set with tag for training
        featuresets = [(self.find_features(rev), category) for (rev, category) in documents]

        # random shuffling for data set trainig for better accuracy
        random.shuffle(featuresets)
        length = len(featuresets)

        # preparing training and testing dataset for 70-30 percentage
        training_set = featuresets[:int(length * training_percentage)]
        testing_set = featuresets[int(length * training_percentage):]

        return (training_set, testing_set)


from __future__ import print_function

import sys
import argparse
import gzip
import json
import os
import pickle
from CNN import *
import tensorflow as tf
import matplotlib.pyplot as plt

import numpy as np

def one_hot(labels):
    """this creates a one hot encoding from a flat vector:
    i.e. given y = [0,2,1]
     it creates y_one_hot = [[1,0,0], [0,0,1], [0,1,0]]
    """
    classes = np.unique(labels)
    n_classes = classes.size
    one_hot_labels = np.zeros(labels.shape + (n_classes,))
    for c in classes:
        one_hot_labels[labels == c, c] = 1
    return one_hot_labels

def mnist(datasets_dir='./data'):
    if not os.path.exists(datasets_dir):
        os.mkdir(datasets_dir)
    data_file = os.path.join(datasets_dir, 'mnist.pkl.gz')
    if not os.path.exists(data_file):
        print('... downloading MNIST from the web')
        try:
            import urllib
            urllib.urlretrieve('http://google.com')
        except AttributeError:
            import urllib.request as urllib
        url = 'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz'
        urllib.urlretrieve(url, data_file)

    print('... loading data')
    # Load the dataset
    f = gzip.open(data_file, 'rb')
    try:
        train_set, valid_set, test_set = pickle.load(f, encoding="latin1")
    except TypeError:
        train_set, valid_set, test_set = pickle.load(f)
    f.close()

    test_x, test_y = test_set
    test_x = test_x.astype('float32')
    test_x = test_x.astype('float32').reshape(test_x.shape[0], 28, 28, 1)
    test_y = test_y.astype('int32')
    valid_x, valid_y = valid_set
    valid_x = valid_x.astype('float32')
    valid_x = valid_x.astype('float32').reshape(valid_x.shape[0], 28, 28, 1)
    valid_y = valid_y.astype('int32')
    train_x, train_y = train_set
    train_x = train_x.astype('float32')
    train_x = train_x.astype('float32').reshape(train_x.shape[0], 28, 28, 1)
    train_y = train_y.astype('int32')
    print('... done loading data')
    return train_x, one_hot(train_y), valid_x, one_hot(valid_y), test_x, one_hot(test_y)

def train_and_validate(x_train, y_train, x_valid, y_valid, num_epochs, lr, num_filters, batch_size, filter_size, verbose=True):

    # define convolutional neural network:
    cnn = Network()
    l1 = InitialLayer([None,28,28,1])
    l2 = ConvolutionalLayer(l1, filter_size, num_filters, pooling=tf.nn.max_pool, poolingsize=2)
    l3 = ConvolutionalLayer(l2, filter_size, num_filters, pooling=tf.nn.max_pool, poolingsize=2)
    l4 = FullyConnectedLayer(l3.flattened(), 128, activation=None)
    l5 = OutputLayer(l4, 10)
    cnn.addLayers([l1, l2, l3, l4, l5])

    print("... network configuration: " + ", ".join("x".join(str(x) for x in l.shape()[1:]) for l in cnn.layers))

    model = Calculation(cnn, lr)

    if verbose:
        print("... training network")

    learning_curve = []
    n_batches = x_train.shape[0] // batch_size
    n_validation_batches = x_valid.shape[0] // batch_size

    # optimize parameters with stochastic gradient descent
    for j in range(num_epochs):
        if verbose:
            print("....... epoch number {0}".format(j), end='', flush=True)
        perm = np.random.permutation(x_train.shape[0])
        for i in range(n_batches):
            x_batch = x_train[perm[(batch_size*i):(batch_size*(i+1))],:,:,:]
            y_batch = y_train[perm[(batch_size*i):(batch_size*(i+1))],:]
            model.optimize(x_batch, y_batch)
        accuracy = 0
        for i in range(n_validation_batches):
            x_batch = x_valid[(batch_size*i):(batch_size*(i+1)),:,:,:]
            y_batch = y_valid[(batch_size*i):(batch_size*(i+1)),:]
            accuracy += float(model.accuracy(x_batch, y_batch)) * x_batch.shape[0]
        learning_curve += [1 - accuracy / x_valid.shape[0]]

        if verbose:
            print(" ... error: {0:.2f}".format(learning_curve[-1]), flush=True)

    return learning_curve, model

def test(x_test, y_test, model):
    return 1 - float(model.accuracy(x_test, y_test))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--output_path", default="./", type=str, nargs="?",
                        help="Path where the results will be stored")
    parser.add_argument("--input_path", default="./", type=str, nargs="?",
                        help="Path where the data is located. If the data is not available it will be downloaded first")
    parser.add_argument("--learning_rate", default=0.05, type=float, nargs="?", help="Learning rate for SGD")
    parser.add_argument("--num_filters", default=16, type=int, nargs="?",
                        help="The number of filters for each convolution layer")
    parser.add_argument("--batch_size", default=32, type=int, nargs="?", help="Batch size for SGD")
    parser.add_argument("--epochs", default=12, type=int, nargs="?",
                        help="Determines how many epochs the network will be trained")
    parser.add_argument("--run_id", default=0, type=int, nargs="?",
                        help="Helps to identify different runs of an experiments")
    parser.add_argument("--filter_size", default=3, type=int, nargs="?",
                        help="Filter width and height")
    args = parser.parse_args()

    # hyperparameters
    lr = args.learning_rate
    num_filters = args.num_filters
    batch_size = args.batch_size
    epochs = args.epochs
    filter_size = args.filter_size

    # train and test convolutional neural network
    x_train, y_train, x_valid, y_valid, x_test, y_test = mnist(args.input_path)

    learning_curve, model = train_and_validate(x_train, y_train, x_valid, y_valid, epochs, lr, num_filters, batch_size, filter_size)

    plt.plot(learning_curve, label='training')
    plt.xlabel('epoche')
    plt.ylabel('validation error')
    plt.title("")
    plt.legend()
    plt.show()

    test_error = test(x_test, y_test, model)

    model.stop()

    # save results in a dictionary and write them into a .json file
    results = dict()
    results["lr"] = lr
    results["num_filters"] = num_filters
    results["batch_size"] = batch_size
    results["filter_size"] = filter_size
    results["learning_curve"] = learning_curve
    results["test_error"] = test_error

    path = os.path.join(args.output_path, "results")
    os.makedirs(path, exist_ok=True)

    fname = os.path.join(path, "results_run_%d.json" % args.run_id)

    fh = open(fname, "w")
    json.dump(results, fh)
    fh.close()


import os
import json
import time
from concurrent.futures import Future
from gbdxtools.rda.error import NotFound, BadRequest
import warnings
try:
    from urllib import urlencode
except ImportError:
    from urllib.parse import urlencode

VIRTUAL_RDA_URL = os.environ.get("VIRTUAL_RDA_URL", "https://rda.geobigdata.io/v1")

def resolve_if_future(future):
    if isinstance(future, Future):
        return future.result()
    else:
        return future

def req_with_retries(conn, url, retries=5):
    for i in range(retries):
        try:
            res = resolve_if_future(conn.get(url))
            if res.status_code != 502:
                return res
            elif res.status_code == 501:
                raise Exception('501 Auth error')
        except:
            pass
        time.sleep(0.5 * (i + 1))
    raise Exception('RDA is overloaded')

def get_graph_stats(conn, graph_id, node_id):
    warnings.warn('Graph API is deprecated')
    url = "{}/metadata/{}/{}/display_stats.json".format(VIRTUAL_RDA_URL, graph_id, node_id)
    req = req_with_retries(conn, url)
    #req = resolve_if_future(conn.get(url))
    if req.status_code == 200:
        return req.json()
    else:
        raise NotFound("Could not fetch stats for graph/node: {} / {}".format(graph_id, node_id))

def get_template_stats(conn, template_id, **kwargs):
    qs = urlencode(kwargs)
    url = "{}/template/{}/display_stats.json?{}".format(VIRTUAL_RDA_URL, template_id, qs)
    req = req_with_retries(conn, url)
    #req = resolve_if_future(conn.get(url))
    if req.status_code == 200:
        return req.json()
    else:
        raise NotFound("Could not fetch stats for template/args: {} / {}".format(template_id, kwargs))

def get_rda_graph(conn, graph_id):
    warnings.warn('Graph API is deprecated')
    url = "{}/graph/{}".format(VIRTUAL_RDA_URL, graph_id)
    #req = resolve_if_future(conn.get(url))
    req = req_with_retries(conn, url)
    if req.status_code == 200:
        return req.json()
    else:
        raise NotFound("No RDA graph found matching id: {}".format(graph_id))

def get_rda_graph_template(conn, template_id):
    # search for template metadata for template ID
    try:
        template_id = _search_for_rda_template(conn, template_id)
    except:
        pass
    url = "{}/template/{}".format(VIRTUAL_RDA_URL, template_id)
    #req = resolve_if_future(conn.get(url))
    req = req_with_retries(conn, url)
    if req.status_code == 200:
        return req.json()
    else:
        raise NotFound("No RDA Template found matching name or id: {}".format(template_id))

def _search_for_rda_template(conn, template_name):
    """
    Searches for template by name, eventually RDA will have named templates and this method goes away.
    :param conn:
    :param template_name:
    :return:
    """
    url = "{}/template/metadata/search?free-text={}".format(VIRTUAL_RDA_URL, template_name)
    #req = resolve_if_future(conn.get(url))
    req = req_with_retries(conn, url)
    if req.status_code == 200:
        request_json = req.json()
        # make sure list is not empty
        if request_json is not None:
            # make sure list contains 1 entry
            for template in request_json:
                if template.get("name") == template_name:
                    # parse template Id
                    return template.get("templateId")

    # if anything fails, raise as we should always get a template Id
    raise Exception("Error fetching template Id")

def register_rda_graph(conn, rda_graph):
    warnings.warn('Graph API is deprecated')
    url = "{}/graph".format(VIRTUAL_RDA_URL)
    res = resolve_if_future(conn.post(url, json.dumps(rda_graph, sort_keys=True),
                                      headers={'Content-Type': 'application/json'}))
    if res.status_code == 200:
        return res.text
    else:
        raise BadRequest("Problem registering graph: {}".format(res.text))

def get_rda_metadata(conn, rda_id, node='toa_reflectance'):
    warnings.warn('Graph API is deprecated')
    url = VIRTUAL_RDA_URL + "/metadata/{}/{}/metadata.json".format(rda_id, node)
    md_response = req_with_retries(conn, url)
    if md_response.status_code != 200:
        md_json = md_response.json()
        if 'error' in md_json:
            raise BadRequest("RDA error: {}. RDA Graph: {}".format(md_json['error'], rda_id))
        raise BadRequest("Problem fetching image metadata: status {} {}, graph_id: {}".format(md_response.status_code, md_response.reason, rda_id))
    else:
        md_json = md_response.json()
        return {
            "image": md_json["imageMetadata"],
            "georef": md_json.get("imageGeoreferencing", None),
            "rpcs": md_json.get("rpcSensorModel", None)
        }

def get_rda_template_metadata(conn, _id, **kwargs):
    qs = urlencode(kwargs)
    url = VIRTUAL_RDA_URL + "/template/{}/metadata?{}".format(_id, qs)
    md_response = req_with_retries(conn, url)
    if md_response.status_code != 200:
        md_json = md_response.json()
        if 'error' in md_json:
            raise BadRequest("RDA error: {}. RDA Graph: {}".format(md_json['error'], _id))
        raise BadRequest("Problem fetching image metadata: status {} {}, graph_id: {}".format(md_response.status_code, md_response.reason, _id))
    else:
        md_json = md_response.json()
        return {
            "image": md_json["imageMetadata"],
            "georef": md_json.get("imageGeoreferencing", None),
            "rpcs": md_json.get("rpcSensorModel", None)
        }

def create_rda_template(conn, graph):
    r = conn.post("{}/template".format(VIRTUAL_RDA_URL), json=graph).result()
    r.raise_for_status()
    return r.json()['id']

def materialize_template(conn, payload):
    r = conn.post("{}/template/materialize".format(VIRTUAL_RDA_URL), json=payload).result()
    r.raise_for_status()
    return r.json()['jobId']

def materialize_status(conn, job_id):
    r = conn.get("{}/template/materialize/status/{}".format(VIRTUAL_RDA_URL, job_id)).result()
    r.raise_for_status()
    return r.json()['status']


from bs4 import BeautifulSoup
from main.tools import banner, colors, template
import os
import requests
from bs4 import BeautifulSoup

def main():
    while True:
        os.system("clear")
        banner.main()
        banner.attack("Deniel of Service Attack")
        list_attacks = [
            "GoldenEye",
            "SlowHTTPTest",
            "THC-SSL-DOS",
            "Slowloris",
            "go back",
        ]
        for i in range(len(list_attacks)):
            print(colors.options, f"{i}) {list_attacks[i]}".title(), colors.reset)
        try:
            option = input(f"\n {colors.select}Select An Option ->{colors.reset}  ")
        except KeyboardInterrupt:
            template.exit_program()
        if option == "0":
            print("\n[+] GoldenEye")
            github = github_getting_text(
                "https://www.kali.org/tools/goldeneye/", "p", 0
            )
            template.template(
                "goldeneye",
                "goldeneye -h",
                github.strip(),
                {
                    "Goldeneye DDos Tool in Kali Linux": "https://www.geeksforgeeks.org/goldeneye-ddos-tool-in-kali-linux/",
                    "Golden Eye DDoS Tool: Installation and Tool usage with examples": "https://allabouttesting.org/golden-eye-ddos-tool-installation-and-tool-usage-with-examples/",
                    "DoS website in Kali Linux using GoldenEye": "https://www.blackmoreops.com/2015/05/18/dos-website-in-kali-linux-using-goldeneye/",
                    "DoS website with GoldenEye - Layer 7 DoS tool": "https://www.darkmoreops.com/2014/11/22/dos-website-with-goldeneye/",
                },
            )  # method=kali
        elif option == "1":
            print("\n[+] Slowhttptest")
            github = github_getting_text(
                "https://www.kali.org/tools/slowhttptest/", "p", 1
            )
            template.template(
                "slowhttptest",
                "slowhttptest -h",
                github.strip(),
                {
                    'How to perform a DoS attack "Slow HTTP" with SlowHTTPTest': "https://ourcodeworld.com/articles/read/949/how-to-perform-a-dos-attack-slow-http-with-slowhttptest-test-your-server-slowloris-protection-in-kali-linux",
                    "slowhttptest Usage Example": "https://www.kali.org/tools/slowhttptest/",
                    "DoS website using slowhttptest in Kali Linux ": "https://www.blackmoreops.com/2015/06/07/attack-website-using-slowhttptest-in-kali-linux/",
                    "Kali Linux - Stressing Tools": "https://www.tutorialspoint.com/kali_linux/kali_linux_stressing_tools.htm",
                    "How to perform SlowHTTPtest DOS attack ": "https://support.tetcos.com/support/solutions/articles/14000130254-how-to-perform-slowhttptest-dos-attack-through-netsim-emulator-",
                },
            )  # method is kali
        elif option == "2":
            print("\n[+] THC-SSL-DOS")
            github = github_getting_text(
                "https://www.kali.org/tools/thc-ssl-dos/", "p", 1
            )
            github += github_getting_text(
                "https://www.kali.org/tools/thc-ssl-dos/", "p", 2
            )
            github += github_getting_text(
                "https://www.kali.org/tools/thc-ssl-dos/", "p", 3
            )
            template.template(
                "thc-ssl-dos",
                "thc-ssl-dos -h",
                github.strip(),
                {
                    "THC-SSL DoS": "https://www.radware.com/security/ddos-knowledge-center/ddospedia/thc-ssl-dos/",
                    "thc-ssl-dos Usage Example": "https://www.kali.org/tools/thc-ssl-dos/",
                    "THC-SSL-DOS – DoS Tool Against Secure Web-Servers and for Testing SSL-Renegotiation": "https://kalilinuxtutorials.com/thc-ssl-dos/",
                    "The THC SSL DoS Threat": "https://resources.infosecinstitute.com/topic/thc-ssl-dos-threat/",
                },
            )  # method is kali
        elif option == "3":
            print("\n[+] SlowLoris")
            github = "Slowloris is a Low bandwidth  HTTP Denial of Service attack that affects threaded servers"
            template.template(
                "Slowloris",
                "python slowloris.py -h",
                github.strip(),
                {
                    "Slowloris DDOS Attack Tool in Kali Linux": "https://www.geeksforgeeks.org/slowloris-ddos-attack-tool-in-kali-linux/",
                    "What is Slowloris?": "https://www.imperva.com/learn/ddos/slowloris/",
                    "Performing a genuine slowloris attack": "https://ourcodeworld.com/articles/read/962/performing-a-genuine-slowloris-attack-slowhttp-of-indefinite-length-in-kali-linux",
                },
                link="https://github.com/gkbrk/slowloris.git",
                method="github",
                github_install="git clone https://github.com/gkbrk/slowloris.git",
                github_check="slowloris",
            )

        else:
            return

def github_getting_text(link, selector, indexvalue):
    print("Please Wait....\r", end="")
    URL = link
    try:
        r = requests.get(URL)
        soup = BeautifulSoup(r.content, "html.parser")
        paras = soup.select(selector)
        # check index value from test file
        return paras[indexvalue].text
    except:
        return f"{colors.red}Not Loaded Because No Internet Connection{colors.reset}"

if __name__ == "__main__":
    main()


"""build script for setuptools"""
from __future__ import absolute_import
import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="clitool2",
    version="1.1",
    author="Daryl Scott",
    author_email="daryl_scott@live.com",
    description="Create command line interface for one or more functions",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/daryl-scott/clitool2",
    packages=["clitool2"],
    install_requires=["python-dateutil"],
    classifiers=[
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent"
    ],
    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4"
)