import torch
import pytorch_lightning as pl
from torch.utils.data import DataLoader
import torch.nn.functional as F
from torch import nn
#import torchvision as tv
from NucleicNet import Burn, Fuel
import torchmetrics
from NucleicNet.Burn import Catalogue_Activation
#from torchmetrics.functional import accuracy
import NucleicNet.Burn.T1
import NucleicNet.Burn.LRS
import NucleicNet.Burn.DA
# ==================================
# Naive adaptor
# ==================================
class B1hw_NaiveRepeatB3hwFcLogits(pl.LightningModule):
    def __init__(self,
                    model   = None, # tv.models.resnet18(nested_module = True),
                    loss    = nn.CrossEntropyLoss(),
                    n_channel = 3,
                    n_class = 10):
        super().__init__()
        if help:

            """
            # NOTE This is a naive 'working' handle to take 1-D tensor forged to (b,1,h,w)
            #      and further forged to (b,3,h,w) by simply repeating the vector
            #      The critique here is that it does not handle batch-normalisation correctly,
            #      because the meaning of color channel mismatches. (it 'smudges' all the features)
            #      This maybe considered the toy model to be compared for debugging purpose.
            #      However, it also offers a template to construct and model with a FcLogit output
            # NOTE The "model" flag assumes the last layer being named 'fc' for fully connected
            """

        self.n_channel = n_channel
        self.n_class = n_class
        self.nested_module = model

        # ==========================================
        # Construct a fc "suffix" adapter
        # ==========================================

        for _, lastfc in self.nested_module.named_modules():
            pass
        if str(lastfc).startswith("Linear"):
            fcsize = int(str(lastfc).split(", out_features=")[-1].split(",")[0])
            fcsize_infeature = int(str(lastfc).split("in_features=")[-1].split(",")[0])
        else:
            raise KeyError("ABORTED. Provided nested_module model does not end with a fc layer.")

        #self.suffixadaptor = nn.Linear(fcsize, self.n_class, bias=True)
        self.nested_module.fc = nn.Linear(fcsize_infeature, self.n_class, bias=True)

        # NOTE The FC output imply logits. This should be a loss accepting logit
        self.loss = loss
        self.automatic_optimization = True
    def forward(self, x):
        #print(x.shape)
        x = Fuel.T1.B1hw_Bnhw(x, n_channel = self.n_channel) # NOTE prefixadaptor
        x = self.nested_module(x)
        #x = self.suffixadaptor(x)
        return x

    def training_step(self, batch, batch_idx):
        x, y = batch
        logits = self.forward(x)
        loss = self.loss(logits, y)
        ACC = torchmetrics.functional.accuracy(logits, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        PR = torchmetrics.functional.precision_recall(logits, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        self.log('train_precision', PR[0])
        self.log('train_recall', PR[1])
        self.log('train_acc', ACC)
        self.log('train_loss_s', loss, prog_bar=True)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y = batch
        logits = self.forward(x)
        CE = nn.CrossEntropyLoss()
        loss = CE(logits, y) #self.loss(logits, y) # NOTE Validate without the random
        self.log('val_loss_s', loss, prog_bar=True)
        self.log("val_loss", loss)
        ACC = torchmetrics.functional.accuracy(logits, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        PR = torchmetrics.functional.precision_recall(logits, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        self.log('val_acc', ACC)
        self.log('val_recall', PR[1])
        self.log('val_precision', PR[0])
        return loss

    def predict_step(self, batch, batch_idx):
        x, y = batch
        logits = self.forward(x)
        p = torch.softmax(logits, dim=1)
        return p

    #lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5,150,300,600,1200,], gamma=0.1)
    def configure_optimizers(self):
        optimizer = torch.optim.AdamW(self.parameters(), lr=1e-4)
        lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=100, factor=0.99,
                            verbose=True, threshold = 1e-5, min_lr= 1e-6, cooldown = 1000)
        return {
                "optimizer": optimizer,
                "lr_scheduler": {
                    "scheduler": lr_scheduler,
                    'reduce_on_plateau': True,
                    "interval": "step",
                    "frequency": 1,
                    "monitor": "train_loss_s",
                # If "monitor" references validation metrics, then "frequency" should be set to a
                # multiple of "trainer.check_val_every_n_epoch".
            },
        }

# =================================
# Logit Template
# =================================
class B1hw_FcLogits(pl.LightningModule):
    def __init__(self, #learning_rate,
                    model   = None,
                    User_Loss    = "CrossEntropyLoss", #nn.CrossEntropyLoss(),
                    hw_product = 480,
                    adaptive2dpool = (3,40),
                    #User_Activation = 'gelu',

                    n_class = 10,
                    BiasInSuffixFc = True,
                    AddMultiLabelSoftMarginLoss = False, # NOTE Worsen stuff?
                    User_lr = 1e-05,
                    User_AdamW_weight_decay = 1e-2,
                    User_min_lr = 1e-6,
                    User_CooldownInterval = 753,
                    User_LrScheduler = 'CosineAnnealingLR',
                    User_ShiftLrRatio = 0.5,
                    # NOTE basic overfit
                    User_Dropoutp = 0.5,
                    User_AddL1 = 0.9,
                    User_NoiseX = 0.1,
                    User_NoiseY = 0.1,
                    User_NoiseZ = 0.1,
                    device = torch.device('cuda'),
                    ManualInitiation = False,
                    User_Mixup = False,
                    User_LabelSmoothing = 0.0,
                    User_NeighborLabelSmoothAngstrom = 1.5,
                    User_InputDropoutp = 0.1,
                    User_FocalLossAlpha = 0.25,
                    User_FocalLossGamma = 2,
                    **kwargs,
        ):
        super().__init__()
        if help:

            """
            # NOTE This is a template to take (b,1,h,w)
            #      The user-defined model has to be a torch functional with batch normalisation defined correctly.
            #      the last layer will be flattened to be fed to a Fc for logit traiinng.
            """
        #self.learning_rate = learning_rate
        if User_lr is not None:
            self.learning_rate = User_lr
        self.n_class = n_class
        self.nested_module = model
        self.hw_product  = hw_product
        try:
            self.User_Activation = model.User_Activation
        except:
            self.User_Activation = 'relu'

        self.User_CooldownInterval = User_CooldownInterval
        self.User_min_lr = User_min_lr
        self.User_AdamW_weight_decay = User_AdamW_weight_decay
        self.User_LrScheduler = User_LrScheduler
        self.User_Dropoutp = User_Dropoutp
        self.AddMultiLabelSoftMarginLoss = AddMultiLabelSoftMarginLoss
        self.User_AddL1 = User_AddL1
        self.User_ShiftLrRatio = User_ShiftLrRatio
        self.User_NoiseX = User_NoiseX
        self.User_NoiseY = User_NoiseY
        self.User_Mixup = User_Mixup
        self.User_LabelSmoothing = User_LabelSmoothing
        self.User_NoiseZ = User_NoiseZ
        self.User_NeighborLabelSmoothAngstrom = User_NeighborLabelSmoothAngstrom
        self.User_InputDropoutp = User_InputDropoutp
        self.User_FocalLossAlpha = User_FocalLossAlpha
        self.User_FocalLossGamma = User_FocalLossGamma

        #self.AddL2 = AddL2
        self.device_ = device

        # Saving all entered hparam above
        self.save_hyperparameters(ignore = ["model_params", 'device', 'model', 'loss'])
        # ==========================================
        # Construct a fc "suffix" adapter
        # ==========================================
        self.prefix_layerD = nn.Sequential(
            nn.Dropout(p=self.User_InputDropoutp,inplace=False)
        )
        # Mapping to classification output
        self.suffix_layerA = nn.Sequential(
            #nn.AdaptiveAvgPool2d((adaptive2dpool[0],adaptive2dpool[1])), # NOTE referencing torchvision's implemetation a pooling layer can be addded
            nn.Flatten(start_dim=1, end_dim=- 1),
        )

        self.suffix_layerD = nn.Sequential(
                #nn.Dropout(p=self.User_Dropoutp, inplace=False),
                #NucleicNet.Burn.T1.NoisyLinear(self.hw_product, self.hw_product, bias=BiasInSuffixFc, sigma_init= self.User_NoiseZ),
                nn.Dropout(p=self.User_Dropoutp, inplace=False),
                nn.Linear(self.hw_product, self.hw_product, bias=BiasInSuffixFc),
                nn.Dropout(p=self.User_Dropoutp, inplace=False),
        )

        self.suffix_layerZ = nn.Sequential(
            nn.Linear(self.hw_product, self.n_class, bias=False)

        )

        if ManualInitiation:
            self._init_params()

        self.User_Loss = User_Loss # This is a string
        self.CatalogueLoss = {
            "CrossEntropyLoss" : nn.CrossEntropyLoss(label_smoothing=self.User_LabelSmoothing),
            "MixupSoftmaxLoss": NucleicNet.Burn.DA.MixupSoftmaxLoss(class_weights = None, reduction = 'mean', label_smooth_eps = self.User_LabelSmoothing),
            "SigmoidalFocalLoss": NucleicNet.Burn.T1.SigmoidalFocalLoss(alpha = self.User_FocalLossAlpha, gamma = self.User_FocalLossGamma,label_smoothing = self.User_LabelSmoothing),
            "MultiLabelSoftMarginLoss": nn.MultiLabelSoftMarginLoss(weight=None, size_average=None, reduce=None, reduction='mean')
        }

        self.loss = self.CatalogueLoss[self.User_Loss]
        self.automatic_optimization = True

    def forward(self, x):
        x = self.prefix_layerD(x)
        x = self.nested_module(x)
        # NOTE We cannot standardize the last layer as it will severely hurt performance. Adding noise here can be dangerous
        #if (self.training) & (self.User_NoiseZ > 0.0):
        #    x += self.User_NoiseZ *  torch.randn(x.shape, device = self.device_)# NOTE Noise added in training

        x = self.suffix_layerA(x)
        if self.User_Dropoutp > 0.0:
            x = self.suffix_layerD(x)
        x = self.suffix_layerZ(x)

        return x

    def training_step(self, batch, batch_idx):

        x, y, yyyy= batch

        # Noise
        x += self.User_NoiseX *  torch.randn(x.shape, device = self.device_)# NOTE Noise added in training
        logits = self.forward(x)

        p = F.softmax(logits,-1)

        #loss = self.loss(logits, yyyy)
        loss = self.loss(logits, yyyy)

        if self.User_AddL1 > 0.0:
            l1_reg = torch.tensor(0.0, device = self.device_)
            for _ , t in enumerate(self.parameters()):
                    l1_reg += torch.mean(torch.flatten(t).abs())
            loss += self.User_AddL1 * l1_reg

        # Visualisation scores
        if self.User_Mixup:
            y = torch.argmax(y, dim=1)

        #y_hot = torch.nn.functional.one_hot(y.type(torch.int64), num_classes=self.n_class)

        ACC = torchmetrics.functional.accuracy(p, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        PR = torchmetrics.functional.precision_recall(p, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        ACC2 = torchmetrics.functional.accuracy(p, y, top_k =2, average = 'weighted', num_classes = self.n_class)
        PR2 = torchmetrics.functional.precision_recall(p, y, top_k =2, average = 'weighted', num_classes = self.n_class)

        ACC_mi = torchmetrics.functional.accuracy(p, y, top_k =1, average = 'micro', num_classes = self.n_class)
        #PR_mi = torchmetrics.functional.precision_recall(p, y, top_k =1, average = 'micro', num_classes = self.n_class)
        ACC_ma = torchmetrics.functional.accuracy(p, y, top_k =1, average = 'macro', num_classes = self.n_class)
        #PR_ma = torchmetrics.functional.precision_recall(p, y, top_k =1, average = 'macro', num_classes = self.n_class)

        ACC2_mi = torchmetrics.functional.accuracy(p, y, top_k =2, average = 'micro', num_classes = self.n_class)
        #PR2_mi = torchmetrics.functional.precision_recall(p, y, top_k =2, average = 'micro', num_classes = self.n_class)
        ACC2_ma = torchmetrics.functional.accuracy(p, y, top_k =2, average = 'macro', num_classes = self.n_class)
        #PR2_ma = torchmetrics.functional.precision_recall(p, y, top_k =2, average = 'macro', num_classes = self.n_class)

        SF = self.CatalogueLoss["SigmoidalFocalLoss"]
        SM = self.CatalogueLoss["MultiLabelSoftMarginLoss"]
        self.log('train_softmargin',SM(p,yyyy) )
        self.log('train_sigmoidalfocal',SF(p,yyyy) )
        self.log('train_precision', PR[0])
        self.log('train_recall', PR[1])
        self.log('train_acc', ACC)
        self.log('train_precision2', PR2[0])
        self.log('train_recall2', PR2[1])
        self.log('train_acc2', ACC2)

        self.log('train_acc_mi', ACC_mi)
        #self.log('train_recall_mi', PR_mi[1])
        #self.log('train_precision_mi', PR_mi[0])
        self.log('train_acc2_mi', ACC2_mi)
        #self.log('train_recall2_mi', PR2_mi[1])
        #self.log('train_precision2_mi', PR2_mi[0])

        self.log('train_acc_ma', ACC_ma)
        #self.log('train_recall_ma', PR_ma[1])
        #self.log('train_precision_ma', PR_ma[0])
        self.log('train_acc2_ma', ACC2_ma)
        #self.log('train_recall2_ma', PR2_ma[1])
        #self.log('train_precision2_ma', PR2_ma[0])

        self.log('train_loss_s', loss, prog_bar=True)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y, yyyy = batch
        logits = self.forward(x)
        p = F.softmax(logits,-1)
        #CE = nn.CrossEntropyLoss()
        #loss = CE(logits, y) #self.loss(logits, y) # NOTE Validate without the random
        #loss_functional = nn.CrossEntropyLoss(label_smoothing=0.0)#self.User_LabelSmoothing)
        #loss = loss_functional(logits, y)
        loss = self.loss(logits, yyyy)

        #y_hot = torch.nn.functional.one_hot(y.type(torch.int64), num_classes=self.n_class)

        if self.User_Mixup:
            pass # NOTE Because mixup is not called in validation_start hook
        SF = self.CatalogueLoss["SigmoidalFocalLoss"]
        SM = self.CatalogueLoss["MultiLabelSoftMarginLoss"]
        self.log('val_softmargin',SM(p,yyyy) )
        self.log('val_sigmoidalfocal',SF(p,yyyy) )

        # NOTE I will just monitor L1
        l1_reg = torch.tensor(0.0, device = self.device_)
        for _ , t in enumerate(self.parameters()):
                l1_reg += torch.mean(torch.flatten(t).abs())

        self.log("L1", l1_reg)
        self.log('val_loss_s', loss, prog_bar=True)
        self.log("val_loss", loss)
        #print(p,y)
        ACC = torchmetrics.functional.accuracy(p, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        PR = torchmetrics.functional.precision_recall(p, y, top_k =1, average = 'weighted', num_classes = self.n_class)
        ACC2 = torchmetrics.functional.accuracy(p, y, top_k =2, average = 'weighted', num_classes = self.n_class)
        PR2 = torchmetrics.functional.precision_recall(p, y, top_k =2, average = 'weighted', num_classes = self.n_class)

        ACC_mi = torchmetrics.functional.accuracy(p, y, top_k =1, average = 'micro', num_classes = self.n_class)
        #PR_mi = torchmetrics.functional.precision_recall(p, y, top_k =1, average = 'micro', num_classes = self.n_class)
        ACC_ma = torchmetrics.functional.accuracy(p, y, top_k =1, average = 'macro', num_classes = self.n_class)
        #PR_ma = torchmetrics.functional.precision_recall(p, y, top_k =1, average = 'macro', num_classes = self.n_class)

        ACC2_mi = torchmetrics.functional.accuracy(p, y, top_k =2, average = 'micro', num_classes = self.n_class)
        #PR2_mi = torchmetrics.functional.precision_recall(p, y, top_k =2, average = 'micro', num_classes = self.n_class)
        ACC2_ma = torchmetrics.functional.accuracy(p, y, top_k =2, average = 'macro', num_classes = self.n_class)
        #PR2_ma = torchmetrics.functional.precision_recall(p, y, top_k =2, average = 'macro', num_classes = self.n_class)

        self.log('val_acc', ACC)
        self.log('val_recall', PR[1])
        self.log('val_precision', PR[0])
        self.log('val_acc2', ACC2)
        self.log('val_recall2', PR2[1])
        self.log('val_precision2', PR2[0])

        self.log('val_acc_mi', ACC_mi)
        #self.log('val_recall_mi', PR_mi[1])
        #self.log('val_precision_mi', PR_mi[0])
        self.log('val_acc2_mi', ACC2_mi)
        #self.log('val_recall2_mi', PR2_mi[1])
        #self.log('val_precision2_mi', PR2_mi[0])

        self.log('val_acc_ma', ACC_ma)
        #self.log('val_recall_ma', PR_ma[1])
        #self.log('val_precision_ma', PR_ma[0])
        self.log('val_acc2_ma', ACC2_ma)
        #self.log('val_recall2_ma', PR2_ma[1])
        #self.log('val_precision2_ma', PR2_ma[0])

        self.log("hp_metric", ACC)

        return loss

    def predict_step(self, batch, batch_idx):
        x, y = batch
        logits = self.forward(x)
        p = torch.softmax(logits, dim=1)
        return p

    def predict_p(self, batch):
        x = batch
        logits = self.forward(x)
        p = torch.softmax(logits, dim=1)
        return p

    def configure_optimizers(self):
        optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate, weight_decay = self.User_AdamW_weight_decay)
        #optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate, momentum=0.9,
        #                            weight_decay=0.01, nesterov=True)
        #optimizer = torch.optim.Adadelta(self.parameters(), lr=self.learning_rate, rho=0.9, eps=1e-06, weight_decay=0.001)

        lr_scheduler_ReduceLrEveryEpoch = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
                                patience=1, factor=0.1, mode='min',
                                verbose=True,
                                threshold_mode='rel',
                                threshold = 0.01, # NOTE In rel mode, dynamic_threshold = best * ( 1 + threshold ) in ‘max’ mode or best * ( 1 - threshold ) in min mode.
                                min_lr= self.User_min_lr, cooldown = self.User_CooldownInterval)

        if self.User_LrScheduler == 'CosineAnnealingLR':
            lr_scheduler =  torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max = self.User_CooldownInterval,
                                                                        eta_min=self.User_min_lr,
                                                                        last_epoch=-1, verbose=False)
        if self.User_LrScheduler == 'CosineAnnealingWarmRestarts':
            lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, self.User_CooldownInterval, T_mult=1,
                                                                        eta_min=self.User_min_lr,
                                                                        last_epoch=-1, verbose=False)
        if self.User_LrScheduler == 'DescendingCosineAnnealingLR_HalfEpoch':
            lr_scheduler = NucleicNet.Burn.LRS.DescendingCosineAnnealingLR_HalfEpoch(optimizer, T_max = self.User_CooldownInterval,
                                                                        eta_min=self.User_min_lr,
                                                                        last_epoch=-1, verbose=False,
                                                                        n_step_per_epoch = 500000,
                                                                        User_ShiftLrRatio = self.User_ShiftLrRatio,
                                                                        final_eta_min = 1e-10)
        if self.User_LrScheduler == 'SimpleMultistepCosineLRS':
            lr_scheduler = NucleicNet.Burn.LRS.SimpleMultistepCosineLRS(optimizer, last_epoch=-1, verbose=False,
                                                                        jumpeverystep = 20000,
                                                                        T_max = self.User_CooldownInterval,
                                                                        jumpfactor = 0.5, eta_min=self.User_min_lr,
                                                                        AssignSteps = [10000, 160000]
                                                                        )
        if self.User_LrScheduler == 'SimpleMultistepCosineLRS_SXPR':
            lr_scheduler = NucleicNet.Burn.LRS.SimpleMultistepCosineLRS(optimizer, last_epoch=-1, verbose=False,
                                                                        jumpeverystep = 20000,
                                                                        T_max = self.User_CooldownInterval,
                                                                        jumpfactor = 0.5, eta_min=self.User_min_lr,
                                                                        AssignSteps = [10000, 160000]
                                                                        )
        if self.User_LrScheduler == 'MultistepCosineAnnealingLRS':
            lr_scheduler = NucleicNet.Burn.LRS.MultistepCosineAnnealingLRS(optimizer, last_epoch=-1, verbose=False,
                                                                        jumpeverystep = 20000,
                                                                        T_max = self.User_CooldownInterval,
                                                                        jumpfactor = 0.5, eta_min=self.User_min_lr,
                                                                        AssignSteps = [10000] #[40000, 80000, 160000]
                                                                        )

        return (
                    {
                        "optimizer": optimizer,
                        "lr_scheduler": {
                            "scheduler": lr_scheduler,
                            'reduce_on_plateau': True,
                            "interval": "step",
                            "frequency": 1,
                            "monitor": "train_loss_s",
                        # If "monitor" references validation metrics, then "frequency" should be set to a
                        # multiple of "trainer.check_val_every_n_epoch".
                        },
                    },

        )

    def _init_params(self):

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity=Catalogue_Activation[self.User_Activation])
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

# We should try to move the inverse-vec here. possibly also some eigvec here after B1hw we can use einsum
# TODO TODO BVec_FcLogits(pl.LightningModule)

# =================================
# Resnet
# =================================
from torch import jit
# NOTE These are customised to take radial shell inputs. with B1hw (h is the shell, w is the feature)
class B1hw_BlockPreActResnet(jit.ScriptModule):
    def __init__(self,
                    n_FeatPerShell = 80,
                    n_Shell = 6,
                    n_ShellMix = 2,
                    act_fn = Catalogue_Activation['gelu'],
                    blockposition = 0,
                    User_NoiseZ = None,
                    device = torch.device('cuda'),
                    ):
        super().__init__()
        self.User_NoiseZ = User_NoiseZ
        self.blockposition = blockposition
        self.device_ = device
        if help:
            """
            # NOTE For some 1-D tensor f = (b,1,h*w) vectorized from some F = (b,1,h,w)
            #      f = Vec(F) NOTE This is done in T1.BVec_B1hw
            #      The input here is some F = (b,1,h,w)
            #      where f was structured by shell index h; index w is the feature index within a shell.
            #      The output is a Resnet block with color channel batch normalised correctly
            #      https://arxiv.org/abs/1603.05027
            #      TODO Downsample maybe implemented by recursion and stride.
            """
        #print('bpreact, n_feat',n_FeatPerShell)
        self.net = nn.Sequential(
            # NOTE Batch normalisation done right woth Preactivation
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),
        )
        self.net2a = nn.Sequential(
            #nn.BatchNorm2d(num_features = n_FeatPerShell),
            NucleicNet.Burn.T1.GhostBatchNorm2D(num_features= n_FeatPerShell, virtual_batch_size=128, eps=1e-05, momentum=0.1, affine=True),
        )
        self.net2b= nn.Sequential(
            act_fn(),
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),
        )
        self.net3 = nn.Sequential(
            # NOTE Convolve along shell extracting useful combination of features in adjacent shells
            NucleicNet.Burn.T1.B1hw_ShellConv(n_FeatPerShell = n_FeatPerShell, n_Shell = n_Shell, n_ShellMix = n_ShellMix),

            # NOTE  Batch normalisation done right woth Preactivation
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),
            #nn.BatchNorm2d(num_features = n_FeatPerShell),
            NucleicNet.Burn.T1.GhostBatchNorm2D(num_features= n_FeatPerShell, virtual_batch_size=128, eps=1e-05, momentum=0.1, affine=True),
            act_fn(),
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),

            #nn.AdaptiveMaxPool2d((int(n_Shell/2), int(n_FeatPerShell/2))),
            #torch.nn.ConvTranspose2d(1, 1, 2, stride=2, padding=0,
            #                        output_padding=0, groups=1, bias=False, dilation=1,
            #                        padding_mode='zeros', device=None, dtype=None),

            # NOTE Convolve along shell extracting useful combination of features in adjacent shells
            NucleicNet.Burn.T1.B1hw_ShellConv(n_FeatPerShell = n_FeatPerShell,n_Shell = n_Shell, n_ShellMix = n_ShellMix),
        )

    @jit.script_method
    def forward(self, x):

        z = self.net(x)
        z = self.net2a(z)
        if (self.blockposition <= 2) & (self.training) & (self.User_NoiseZ > 0.0):
            z += self.User_NoiseZ *  torch.randn(z.shape, device = self.device_)
        z = self.net2b(z)
        z = self.net3(z)
        out = z + x
        return out

#class B1hw_BlockBottleneckResnet(nn.Module):
class B1hw_BlockBottleneckResnet(jit.ScriptModule):
    def __init__(self,
                    n_FeatPerShell = 80,
                    n_channelbottleneck = 40,
                    n_Shell = 6,
                    n_ShellMix = 2,
                    act_fn = Catalogue_Activation['gelu'],
                    device = torch.device('cuda'),
                    blockposition = 0,
                    User_NoiseZ = None,
                    ):
        super().__init__()

        if help:
            """
            # NOTE For some 1-D tensor f = (b,1,h*w) vectorized from some F = (b,1,h,w)
            #      f = Vec(F) NOTE This is done in T1.BVec_B1hw
            #      The input here is some F = (b,1,h,w)
            #      where f was structured by shell index h; index w is the feature index within a shell.
            #      The output is a Resnet block with color channel batch normalised correctly
            #      https://arxiv.org/abs/1603.05027
            #      TODO Downsample maybe implemented by recursion and stride.
            # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
            # while original implementation places the stride at the first 1x1 convolution(self.conv1)
            # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
            # This variant is also known as ResNet V1.5 and improves accuracy according to
            # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.
            """
        self.n_Shell = n_Shell
        self.selectshell = torch.arange(self.n_Shell, device= device)
        self.act_fn = act_fn()
        self.net1 = nn.Sequential(
            # NOTE Batch normalisation done right woth Preactivation
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),
            nn.Conv2d(n_FeatPerShell, n_channelbottleneck, kernel_size=1, stride=1, bias=False),
            #nn.BatchNorm2d(num_features = n_channelbottleneck),
            NucleicNet.Burn.T1.GhostBatchNorm2D(num_features= n_channelbottleneck, virtual_batch_size=128, eps=1e-05, momentum=0.1, affine=True),
        )
        self.net2a = nn.Sequential(
            act_fn(),
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),

            #nn.AdaptiveMaxPool2d((int(n_Shell/2), int(n_channelbottleneck/2))),
            #torch.nn.ConvTranspose2d(1, 1, 2, stride=2, padding=0,
            #                        output_padding=0, groups=1, bias=False, dilation=1,
            #                        padding_mode='zeros', device=None, dtype=None),

        )
        self.widedrop = nn.Dropout(p=0.1, inplace=False)
        self.net2b = nn.Sequential(

            # NOTE Convolve along shell extracting useful combination of features in adjacent shells
            NucleicNet.Burn.T1.B1hw_ShellConv(n_FeatPerShell = n_channelbottleneck, n_Shell = n_Shell, n_ShellMix = n_ShellMix),

            # NOTE  Batch normalisation done right woth Preactivation
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),
            #nn.BatchNorm2d(num_features = n_channelbottleneck),
            NucleicNet.Burn.T1.GhostBatchNorm2D(num_features= n_channelbottleneck, virtual_batch_size=128, eps=1e-05, momentum=0.1, affine=True),
            act_fn(),
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),

            # NOTE Convolve along shell extracting useful combination of features in adjacent shells
            #NucleicNet.Burn.T1.B1hw_ShellConv(n_FeatPerShell = n_FeatPerShell,n_Shell = n_Shell, n_ShellMix = n_ShellMix),
            nn.Conv2d(in_channels = 1, out_channels = n_FeatPerShell,
                        kernel_size = (n_ShellMix,n_channelbottleneck),
                        padding=(n_ShellMix - 1 ,0),
                        stride=1, dilation=1, groups=1,
                        bias=False, padding_mode='zeros'),
                        #device=None, dtype=None),
            NucleicNet.Burn.T1.Bnhw_Permute(dimorder=(0,3,2,1)),

        )
        self.blockposition = blockposition
        self.device_ = device
        assert (User_NoiseZ is not None), 'ABORTED. B1hw_BlockBottleneckResnet User_NoiseZ is None'

        self.User_NoiseZ = User_NoiseZ

    @jit.script_method
    def forward(self, x):

        z = self.net1(x)

        if (self.blockposition <= 2) & (self.training) & (self.User_NoiseZ > 0.0):
            z += self.User_NoiseZ *  torch.randn(z.shape, device = self.device_)
        z = self.net2a(z)
        #if (self.blockposition <= 2):
        #    z=self.widedrop(z)
        # NOTE Worsen if drop here.
        z = self.net2b(z)
        #print(z.shape)
        z = z.index_select(2, self.selectshell)
        out = z + x
        out = self.act_fn(out)
        return out

Catalogue_BlockResnet = {"B1hw_BlockPreActResnet" : B1hw_BlockPreActResnet,
                        "B1hw_BlockBottleneckResnet" : B1hw_BlockBottleneckResnet}

# NOTE Resnet
#class B1hw_LayerResnet(nn.Module):
class B1hw_LayerResnet(jit.ScriptModule):
    def __init__(self,
                    n_FeatPerShell = 80,
                    n_Shell = 6,
                    n_ShellMix = 2,
                    User_Activation = 'gelu',
                    User_Block = "B1hw_BlockPreActResnet",
                    n_Blocks = 16,
                    #ShellIPCA = None,
                    ManualInitiation = False,
                    device = torch.device('cuda')):
        super(B1hw_LayerResnet, self).__init__()

        if help:
            """
            # NOTE For some 1-D tensor f = (b,1,h*w) vectorized from some F = (b,1,h,w)
            #      f = Vec(F) NOTE This is done in T1.BVec_B1hw
            #      The input here is some F = (b,1,h,w)
            #      where f was structured by shell index h; index w is the feature index within a shell.
            #      The output is a series of Resnet block
            #      TODO Downsample maybe implemented by recursion and stride.
            """

        self.n_Blocks = n_Blocks
        self.User_Activation = User_Activation
        self.User_Block = User_Block

        self.n_FeatPerShell = n_FeatPerShell
        self.n_Shell = n_Shell
        self.n_ShellMix = n_ShellMix

        self._create_network()
        if ManualInitiation:
            self._init_params()
        #self.save_hyperparameters()

    def _create_network(self):
      blocks = []
      for _ in range(self.n_Blocks):
            current_Block_Functional = Catalogue_BlockResnet[self.User_Block]
            current_Activation_Functional = Catalogue_Activation[self.User_Activation]
            blocks.append(
                current_Block_Functional(
                    n_FeatPerShell = self.n_FeatPerShell,
                    n_Shell = self.n_Shell,
                    n_ShellMix = self.n_ShellMix,
                    act_fn = current_Activation_Functional)
                # TODO With bottleneck layer
                    )

      self.blocks = nn.Sequential(*blocks)

    def _init_params(self):

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity=Catalogue_Activation[self.HparamDict["Activation"]])
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
    @jit.script_method
    def forward(self, x):

        x = self.blocks(x)
        return x

#class B1hw_LayerResnetBottleneck(nn.Module):
class B1hw_LayerResnetBottleneck(jit.ScriptModule):
    def __init__(self,
                    n_FeatPerShell = 80,
                    n_Shell = 6,
                    n_ShellMix = 2,
                    User_Activation = 'gelu',
                    User_n_channelbottleneck = 40,
                    User_Block = "B1hw_BlockPreActResnet",
                    n_Blocks = 16,
                    User_NoiseZ = 0.01,
                    #ShellIPCA = None,
                    ManualInitiation = False,
                    device = torch.device('cuda')):
        super(B1hw_LayerResnetBottleneck, self).__init__()

        if help:
            """
            # NOTE For some 1-D tensor f = (b,1,h*w) vectorized from some F = (b,1,h,w)
            #      f = Vec(F) NOTE This is done in T1.BVec_B1hw
            #      The input here is some F = (b,1,h,w)
            #      where f was structured by shell index h; index w is the feature index within a shell.
            #      The output is a series of Resnet block
            #      TODO Downsample maybe implemented by recursion and stride.
            """

        self.n_Blocks = n_Blocks
        self.User_Activation = User_Activation
        self.User_Block = User_Block
        self.User_NoiseZ = User_NoiseZ
        self.n_FeatPerShell = n_FeatPerShell
        self.n_Shell = n_Shell
        self.n_ShellMix = n_ShellMix
        self.n_channelbottleneck = User_n_channelbottleneck

        self._create_network()
        if ManualInitiation:
            self._init_params()
        #self.save_hyperparameters()

    def _create_network(self):
      blocks = []
      for _ in range(int(self.n_Blocks/2)):
            current_Block_Functional = Catalogue_BlockResnet[self.User_Block]
            current_Block_Bottleneck = Catalogue_BlockResnet["B1hw_BlockBottleneckResnet"]
            current_Activation_Functional = Catalogue_Activation[self.User_Activation]
            blocks.append(
                current_Block_Functional(
                    n_FeatPerShell = self.n_FeatPerShell,
                    n_Shell = self.n_Shell,
                    n_ShellMix = self.n_ShellMix,
                    act_fn = current_Activation_Functional,
                    blockposition = _ ,
                    User_NoiseZ = self.User_NoiseZ
                    )
                    )
            blocks.append(
                current_Block_Bottleneck(
                    n_FeatPerShell = self.n_FeatPerShell,
                    n_channelbottleneck = self.n_channelbottleneck,
                    n_Shell = self.n_Shell,
                    n_ShellMix = self.n_ShellMix,
                    act_fn = current_Activation_Functional,
                    blockposition = _ ,
                    User_NoiseZ = self.User_NoiseZ)

                    )

      self.blocks = nn.Sequential(*blocks)

    def _init_params(self):

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity=Catalogue_Activation[self.HparamDict["Activation"]])
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
    @jit.script_method
    def forward(self, x):

        x = self.blocks(x)
        #x = self.normlayer(x) # NOTE This is NOT a good idea. batch normalising the last layer will hurt performance
        return x
# =========================
# Legacy B1hw_FcLogits Config for optimizer
# =========================

def LEGACY_configure_optimizers(self):
        optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate, weight_decay = self.User_AdamW_weight_decay)
        #optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate, momentum=0.9)
        #optimizer = torch.optim.Adadelta(self.parameters(), lr=self.learning_rate, rho=0.9, eps=1e-06, weight_decay=0.001)
        if self.User_LrScheduler == 'ReduceLROnPlateau':
            lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=100, factor=0.951,
                                verbose=False,
                                # NOTE This threshold is the "within epsilon" for the loss to be counted as unchanged
                                # should be around 1e-4 to 5
                                # it alsoimplies the lowest minima acheivable,
                                threshold = 5e-5,
                                min_lr= self.User_min_lr, cooldown = self.User_CooldownInterval)
        if self.User_LrScheduler == 'CosineAnnealingLR':
            lr_scheduler =  torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max = self.User_CooldownInterval,
                                                                        eta_min=self.User_min_lr,
                                                                        last_epoch=-1, verbose=False)
        if self.User_LrScheduler == 'CosineAnnealingWarmRestarts':
            lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, self.User_CooldownInterval, T_mult=1,
                                                                        eta_min=self.User_min_lr,
                                                                        last_epoch=-1, verbose=False)
        return {
                "optimizer": optimizer,
                "lr_scheduler": {
                    "scheduler": lr_scheduler,
                    'reduce_on_plateau': True,
                    "interval": "step",
                    "frequency": 1,
                    "monitor": "train_loss_s",
                # If "monitor" references validation metrics, then "frequency" should be set to a
                # multiple of "trainer.check_val_every_n_epoch".
            },
        }

# =================================================================================
#    NucleicNet
#    Copyright (C) 2019-2022  Jordy Homing Lam, JHML. All rights reserved.
#
#    Acknowledgement.
#    JHML would like to thank Mingyi Xue and Joshua Chou for their patience and efforts
#    in the debugging process and Prof. Xin Gao and Prof. Xuhui Huang for their
#    continuous support.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Redistribution and use in source and binary forms, with or without
#    modification, are permitted provided that the following conditions are
#    met:
#
#    * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation and/or
#    other materials provided with the distribution.
#    * Cite our work at Lam, J.H., Li, Y., Zhu, L. et al. A deep learning framework to predict binding preference of RNA constituents on protein surface. Nat Commun 10, 4941 (2019). https://doi.org/10.1038/s41467-019-12920-0
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
# ==================================================================================


#!/usr/bin/python3
""" Script that takes in a letter and sends
    a POST request to url with the letter as
    a parameter. """
import requests
from sys import argv

if __name__ == "__main__":
    if len(argv) == 2:
        q = argv[1]
    else:
        q = ''
    url = 'http://0.0.0.0:5000/search_user'
    req = requests.post(url, data={'q': q})
    try:
        data = req.json()
        if not data:
            print('No result')
        else:
            print('[{}] {}'.format(data.get('id'), data.get('name')))
    except Exception:
        print('Not a valid JSON')


import pytest

from rss_parser.news_parser.news_parser import (NotRssFeedUrlError,
                                                rss_feed_type_checker,
                                                validate_url_is_rss_feed, parse_rss_feed_regularly)

@pytest.mark.parametrize('expected_exception, url', [(NotRssFeedUrlError, 'http://www.google.com')])
def test_validate_url_is_rss_feed(expected_exception, url):
    with pytest.raises(expected_exception):
        validate_url_is_rss_feed(url)

@pytest.mark.parametrize('url, expected_result', [('https://lifehacker.com/rss', True),
                                                  ('https://news.yahoo.com/rss', False),
                                                  ('https://rss.dw.com/xml/rss-ru-ger', False)])
def test_rss_feed_type_checker(url, expected_result):
    assert rss_feed_type_checker(url) == expected_result


import numpy as np
import itertools
import kernel_learning as kler
import matplotlib.pyplot as plt
import jax_math as jmath
import database_preparation as datprep
datapath = "/home/stuke/Databases/QM9_XYZ_below10/"

def cleanup_results(result_file, multiple_runs = False, Choose_Folder = False, rep_no = 1):
    ''' gets data from resultfile and returns plottable Curve objects
    Variables
    ---------
    resultsfile : string, path to file containing pickled Result objects
    multiple_runs : if True, calculate mean of runs with same lamda and sigma
    Choose_Folder: boolean, if True, file is directly stored to result_file.
                    if not, result file is stored in ./Pickled/Kernel_Results folder

    Returns
    -------
    this_curve : LearningResults object
    '''

    if not Choose_Folder:
        #print("your results were stored to ./Pickled/Kernel_Results/")
        result_file = "./Pickled/Kernel_Results/" + result_file + "_" + str(rep_no)+ "reps"

    plottable_curves = []
    if rep_no > 1:
        multiple_runs = True

    if multiple_runs:
        lamdas = []
        sigmas = []

    results_list = datprep.read_compounds(result_file)
    #print("len results_list:", len(results_list))

    for result in results_list:
        #print("type result:", type(result))
        lamda = result.lamda
        sigma = result.sigma
        xlist = result.set_sizes
        ylist = result.maes
        if not multiple_runs:
            name = curve_name(sigma, lamda)
            curve = CurveObj(name)
            curve.xnparray = xlist
            curve.ynparray = ylist

            plottable_curves.append(curve)

        else:
            lamdas.append(lamda)
            sigmas.append(sigma)

    #probably plottable_curves could already be returned here for False
    if multiple_runs:
        for l in list(set(lamdas)): #get all unique occurances for lamda
            for s in list(set(sigmas)): #get all unique occurances for sigma
                same_x = []
                same_y = []

                #find all results with these s and l
                for result in results_list:
                    if result.lamda == l and result.sigma == s:
                        same_x.append(result.set_sizes)
                        same_y.append(result.maes)
                #print("all arrays of same y:\n", same_y)
                #calculate average now
                av_ylist, yerror = jmath.calculate_mean(same_y)
                print("the calculated mean and it's error are:\n mean:", av_ylist, "\n error:", yerror)
                #add Curve object
                name = curve_name(s,l)
                curve = CurveObj(name)
                curve.xnparray = same_x[0]
                curve.ynparray = av_ylist
                curve.yerror = yerror

                plottable_curves.append(curve)

    return(plottable_curves)

def curve_name( sigma, lamda):
    name = ', sigma = %s, lambda = %s'% (str(sigma), str(lamda))
    return(name)

def plot_learning(set_sizes ,\
        maes,\
        labels = [],\
        xtitle = 'Training Set Size',\
        ytitle = 'MAE [kcal/mol]',\
        title = "sigma = 20\nlambda = 1e-3",\
        filename = "QML_learning"):

    '''
    plots x and y values of learning curves
    '''

    #standard settings for plotting:
    fontsize = 30

    plt.rc('font',       size=fontsize) # controls default text sizes
    plt.rc('axes',  titlesize=fontsize) # fontsize of the axes title
    plt.rc('axes',  labelsize=fontsize) # fontsize of the x and y labels
    plt.rc('xtick', labelsize=fontsize*0.8) # fontsize of the tick labels
    plt.rc('ytick', labelsize=fontsize*0.8) # fontsize of the tick labels
    plt.rc('legend', fontsize=fontsize*0.8) # legend fontsize
    plt.rc('figure',titlesize=fontsize*1.2) # fontsize of the figure title
    plt.rcParams['axes.titlepad'] = 20

    f, ax = plt.subplots(nrows = 1, ncols = 1, figsize = (12, 8))

    for m in range(len(maes)):
        mae_list = maes[m]
        name = labels[m]
        ax.loglog(set_sizes, mae_list, label = name, linewidth = 2)

    ax.spines['right'].set_color('none')
    ax.spines['top'].set_color('none')
    ax.spines['bottom'].set_position(('axes', -0.05))
    ax.spines['bottom'].set_color('black')
    ax.spines['left'].set_color('black')
    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    ax.spines['left'].set_position(('axes', -0.05))

    ax.xaxis.set_tick_params(width=2.5, length=20)
    ax.yaxis.set_tick_params(width=2.5, length=20)

    #set x and y axis label
    ax.set_xlabel(xtitle)
    ax.set_ylabel(ytitle)

    #add legend
    ax.legend()

    #save figure
    f.savefig(filename, bbox_inches = 'tight')

    return(print("figure was saved to", filename))

def plot_scatter(y_test, y_predicted, label = "OM",\
        title = "OM Representation Gaussian Kernel",\
        figuretitle = "Scatterplot_OM",\
        xtitle = "Atomic Energies [kcal/mol]",\
        ytitle = "Predicted Atomic Energies [kcal/mol]"):
    #standard settings for plotting:
    fontsize = 30

    plt.rc('font',       size=fontsize) # controls default text sizes
    plt.rc('axes',  titlesize=fontsize) # fontsize of the axes title
    plt.rc('axes',  labelsize=fontsize) # fontsize of the x and y labels
    plt.rc('xtick', labelsize=fontsize*0.8) # fontsize of the tick labels
    plt.rc('ytick', labelsize=fontsize*0.8) # fontsize of the tick labels
    plt.rc('legend', fontsize=fontsize*0.8) # legend fontsize
    plt.rc('figure',titlesize=fontsize*1.2) # fontsize of the figure title
    plt.rcParams['axes.titlepad'] = 20

    #prep figure
    f, ax = plt.subplots(nrows = 1, ncols = 1, figsize = (12, 8))

    ax.set_xlabel(xtitle)
    ax.set_ylabel(ytitle)

    ax.spines['right'].set_color('none')
    ax.spines['top'].set_color('none')
    ax.spines['bottom'].set_position(('axes', -0.05))
    ax.spines['bottom'].set_color('black')
    ax.spines['left'].set_color('black')
    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    ax.spines['left'].set_position(('axes', -0.05))

    ax.xaxis.set_tick_params(width=2.5, length=20)
    ax.yaxis.set_tick_params(width=2.5, length=20)

    st = f.suptitle(title)

    #plot results
    ax.scatter(y_test, y_predicted, label = label)

    #make x and y ticks the same
    multiplier = 10 ** -2
    min_tick = int(min(y_test)*multiplier) / multiplier

    plt.xticks(np.arange(min_tick, max(y_test)+100, 200))
    plt.yticks(np.arange(min_tick, max(y_test)+100, 200))
    #save figure

    f.savefig(figuretitle, bbox_inches = 'tight')
    return(print("figure was saved to", figuretitle))

def plot_curves(curve_list, file_title = "TrialLearning",\
        plottitle = 'Learning Curves of CM Eigenvector Representation on QM9 Dataset\n 1000 molecules, 2 Runs Averaged',\
        xtitle = 'Training Set Size',\
        ytitle = 'MAE [kcal/mol]',\
        multiple_runs = False,\
        include_title = False):

    '''plots learning curves from hartree to kcal/mol
    curve_list: list of CurveObj class objects
    file_title: string, where to store plot to
    plottitle = title over plot
    xtitle : string, title of x axis
    ytitle : string, title of y axis
    mutliple_runs : boolean, if true, plot curves with variation shadowed
    include_title : boolean, if true, plot title
    '''

    #standard settings for plotting:
    fontsize = 30

    plt.rc('font',       size=fontsize) # controls default text sizes
    plt.rc('axes',  titlesize=fontsize) # fontsize of the axes title
    plt.rc('axes',  labelsize=fontsize) # fontsize of the x and y labels
    plt.rc('xtick', labelsize=fontsize*0.8) # fontsize of the tick labels
    plt.rc('ytick', labelsize=fontsize*0.8) # fontsize of the tick labels
    plt.rc('legend', fontsize=fontsize*0.8) # legend fontsize
    plt.rc('figure',titlesize=fontsize*1.2) # fontsize of the figure title
    plt.rcParams['axes.titlepad'] = 20

    f, ax = plt.subplots(nrows = 1, ncols = 1, figsize = (12, 8))

    #plot title if included
    if include_title:
        st = f.suptitle(plottitle)

    ax.set_xlabel(xtitle)
    ax.set_ylabel(ytitle)

    ax.set_xscale('log')
    ax.set_yscale('log')

    ax.spines['right'].set_color('none')
    ax.spines['top'].set_color('none')
    ax.spines['bottom'].set_position(('axes', -0.05))
    ax.spines['bottom'].set_color('black')
    ax.spines['left'].set_color('black')
    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    ax.spines['left'].set_position(('axes', -0.05))

    ax.xaxis.set_tick_params(width=2.5, length=20)
    ax.yaxis.set_tick_params(width=2.5, length=20)

    #assign marker for better differentiation
    marker = itertools.cycle((',', '+', '.', 'o', '*', "1", "2", "3", "4", "8", "H", "D"))

    #all the plotting has to be done below
    for curve in curve_list:

        yarray = curve.ynparray

        ax.plot(curve.xnparray, curve.ynparray, linewidth = 2, marker = next(marker), label = curve.name)
        print('x:' ,curve.xnparray,'\ny:', yarray)

    #all the plotting has to be done above

    f.legend = ax.legend(loc = 'lower left')

    #handles, labels = ax.get_legend_handles_labels()

    #f.legend(handles, labels, bbox_to_anchor=(1,1), loc="upper left")

    figuretitle = "./Images/" + file_title + ".png"
    f.savefig(figuretitle, bbox_inches = 'tight')
    return(print("figure was saved to", figuretitle))


# @author Yu Tongxin
# @data 2020-08-28
import re
import xlwt
import os

def read_log():
    value = []
    value.append(["date", "type", "info"])
    with open("Apache_log/Apache_2k.log", "r") as log:
        for line in log:
            value.append(extract_data(line))
    book_name_xls = 'Apache_log.xls'
    sheet_name_xls = 'apache_log'
    if os.path.exists(book_name_xls):
        os.remove(book_name_xls)

    write_excel_xls(book_name_xls, sheet_name_xls, value)

def extract_data(line):
    list = []
    matchObj = re.match(r'(\[.*\]) (\[.*\]) (.*)', line, re.M | re.I)
    if matchObj:
        date = matchObj.group(1).strip().replace("[", "").replace("]", "")
        type = matchObj.group(2).strip().replace("[", "").replace("]", "")
        info = matchObj.group(3)
        list.append(date)
        list.append(type)
        list.append(info)
        return list
    else:
        print("No match!!")

def write_excel_xls(path, sheet_name, value):
    index = len(value)  # 获取需要写入数据的行数
    workbook = xlwt.Workbook()  # 新建一个工作簿
    sheet = workbook.add_sheet(sheet_name)  # 在工作簿中新建一个表格
    for i in range(0, index):
        for j in range(0, len(value[i])):
            sheet.write(i, j, value[i][j])  # 像表格中写入数据（对应的行和列）
    workbook.save(path)  # 保存工作簿
    print("xls格式表格写入数据成功！")

read_log()


 # Copy of spreadsheet here: https://docs.google.com/spreadsheets/d/1j-hdTuNznRYAwJ-f8-DVUjfmrvWIN1cJzEqffgLNPxo/edit#gid=0
# Shared this email with google sheets: petition@petition-180717.iam.gserviceaccount.com
# Installed: pip install gspread oauth2client

# gspread is a Python client library for the Google Sheets API.
# Here's the docs.
import gspread
from oauth2client.service_account import ServiceAccountCredentials

# This uses credentials for API.
scope = ['https://spreadsheets.google.com/feeds']
creds = ServiceAccountCredentials.from_json_keyfile_name('client_secret.json', scope)
client = gspread.authorize(creds)

# This opens the spreadsheet.
sheet = client.open("Petition").sheet1

# # Pulls all data.
# list_of_hashes = sheet.get_all_records()

# # Pulls row. (Left to right)
# list_of_hashes = sheet.row_values(1)

# # Pulls column. (Top to bottom)
# list_of_hashes = sheet.col_values(1)

# # Pulls cell.
# list_of_hashes = sheet.cell(2, 1).value

# # Writes to spreadsheet.
list_of_hashes = sheet.update_cell(1, 1, "TEST TEST TEST TEST")

# # Inserts row. (Left to right)
# row = ["I'm", "inserting"]
# index = 1
# list_of_hashes = sheet.insert_row(row, index)

# # Deletes rows.
# list_of_hashes = sheet.delete_row(1)

# # Counts all rows.
# list_of_hashes = sheet.row_count

print(list_of_hashes)


import os
import time
import logging as log

from virttest import virsh
from virttest import data_dir
from virttest import utils_disk
from virttest import utils_misc
from virttest.utils_test import libvirt
from virttest.libvirt_xml import vm_xml

# Using as lower capital is not the best way to do, but this is just a
# workaround to avoid changing the entire file.
logging = log.getLogger('avocado.' + __name__)

def run(test, params, env):
    """
    Test command: virsh qemu-agent-command.
    """
    vm_name = params.get("main_vm")
    vm = env.get_vm(vm_name)
    status_cmd = params.get("status_cmd", "")
    freeze_cmd = params.get("freeze_cmd", "")
    thaw_cmd = params.get("thaw_cmd", "")
    xml_backup = vm_xml.VMXML.new_from_inactive_dumpxml(vm_name)
    try:
        def get_dirty(session, frozen=False):
            """
            Get dirty data of guest
            """
            try:
                data_cmd = "cat /proc/meminfo | grep Dirty"
                if not frozen:
                    result = utils_misc.wait_for(lambda:
                                                 int(session.
                                                     cmd_output(data_cmd).
                                                     strip().split()[1]) != 0,
                                                 60)
                    if result:
                        return int(session.cmd_output(data_cmd).strip().
                                   split()[1])
                    else:
                        return 0
                    dirty_info = session.cmd_output(data_cmd).strip()
                    return int(dirty_info.split()[1])
                else:
                    result = utils_misc.wait_for(lambda:
                                                 int(session.
                                                     cmd_output(data_cmd).
                                                     strip().split()[1]) == 0,
                                                 60)
                    if result:
                        return 0
                    else:
                        return int(session.cmd_output(data_cmd).strip().
                                   split()[1])
            except (IndexError, ValueError) as details:
                test.fail("Get dirty info failed: %s" % details)

        device_source_path = os.path.join(data_dir.get_tmp_dir(), "disk.img")
        device_source = libvirt.create_local_disk("file", path=device_source_path,
                                                  disk_format="qcow2")
        vm.prepare_guest_agent()

        # Do operation before freeze guest filesystem
        session = vm.wait_for_login()
        tmp_file = "/mnt/test.file"
        try:
            # Create extra image and attach to guest, then mount
            old_parts = utils_disk.get_parts_list(session)
            ret = virsh.attach_disk(vm_name, device_source, "vdd")
            if ret.exit_status:
                test.fail("Attaching device failed before testing agent:%s" % ret.stdout.strip())
            time.sleep(1)
            new_parts = utils_disk.get_parts_list(session)
            added_part = list(set(new_parts).difference(set(old_parts)))
            session.cmd("mkfs.ext3 -F /dev/{0} && mount /dev/{0} /mnt".format(added_part[0]))

            # Generate dirty memory
            session.cmd("rm -f %s" % tmp_file)
            session.cmd_output("cp /dev/zero %s 2>/dev/null &" % tmp_file)
            # Get original dirty data
            org_dirty_info = get_dirty(session)
            fz_cmd_result = virsh.qemu_agent_command(vm_name, freeze_cmd,
                                                     ignore_status=True,
                                                     debug=True)
            libvirt.check_exit_status(fz_cmd_result)

            # Get frozen dirty data
            fz_dirty_info = get_dirty(session, True)
            st_cmd_result = virsh.qemu_agent_command(vm_name, status_cmd,
                                                     ignore_status=True,
                                                     debug=True)
            libvirt.check_exit_status(st_cmd_result)
            if not st_cmd_result.stdout.strip().count("frozen"):
                test.fail("Guest filesystem status is not frozen: %s"
                          % st_cmd_result.stdout.strip())

            tw_cmd_result = virsh.qemu_agent_command(vm_name, thaw_cmd,
                                                     ignore_status=True,
                                                     debug=True)
            libvirt.check_exit_status(tw_cmd_result)

            # Get thawed dirty data
            tw_dirty_info = get_dirty(session)
            st_cmd_result = virsh.qemu_agent_command(vm_name, status_cmd,
                                                     ignore_status=True,
                                                     debug=True)
            libvirt.check_exit_status(st_cmd_result)
            if not st_cmd_result.stdout.strip().count("thawed"):
                test.fail("Guest filesystem status is not thawed: %s"
                          % st_cmd_result.stdout.strip())
            logging.info("Original dirty data: %s" % org_dirty_info)
            logging.info("Frozen dirty data: %s" % fz_dirty_info)
            logging.info("Thawed dirty data: %s" % tw_dirty_info)
            if not tw_dirty_info or not org_dirty_info:
                test.fail("The thawed dirty data should not be 0!")
            if fz_dirty_info:
                test.fail("The frozen dirty data should be 0!")
        finally:
            # Thaw the file system that remove action can be done
            virsh.qemu_agent_command(vm_name, thaw_cmd, ignore_status=True)
            session.cmd("rm -f %s" % tmp_file)
            session.close()
    finally:
        xml_backup.sync()
        os.remove(device_source_path)


import os
import torch.optim as optim
import torch.nn as nn

from lpd.trainer import Trainer
from lpd.predictor import Predictor
from lpd.callbacks import SchedulerStep, StatsPrint, ModelCheckPoint, LossOptimizerHandler, CallbackMonitor
from lpd.extensions.custom_schedulers import DoNothingToLR
from lpd.enums import Phase, State, MonitorType, StatsType, MonitorMode
from lpd.metrics import BinaryAccuracyWithLogits, MetricBase
import lpd.utils.torch_utils as tu
import lpd.utils.general_utils as gu
import examples.utils as eu

gu.seed_all(42)  # BECAUSE ITS THE ANSWER TO LIFE AND THE UNIVERSE

save_to_dir = os.path.dirname(__file__) + '/trainer_checkpoint/'
trainer_file_name = 'trainer'

def get_parameters():
    # N is batch size; D_in is input dimension;
    # H is hidden dimension; D_out is output dimension.
    N, D_in, H, D_out = 64, 100, 100, 1
    num_epochs = 3
    data_loader = eu.examples_data_generator(N, D_in, D_out, binary_out=True)
    data_loader_steps = 100
    return N, D_in, H, D_out, num_epochs, data_loader, data_loader_steps

def get_trainer_base(D_in, H, D_out):
    device = tu.get_gpu_device_if_available()

    model = eu.get_basic_model(D_in, H, D_out).to(device)

    loss_func = nn.BCEWithLogitsLoss().to(device)

    optimizer = optim.Adam(model.parameters(), lr=1e-4)

    scheduler = DoNothingToLR() #CAN ALSO USE scheduler=None, BUT DoNothingToLR IS MORE EXPLICIT

    metrics = BinaryAccuracyWithLogits(name='Accuracy')

    return device, model, loss_func, optimizer, scheduler, metrics

def get_trainer(N, D_in, H, D_out, num_epochs, data_loader, data_loader_steps):
    device, model, loss_func, optimizer, scheduler, metrics = get_trainer_base(D_in, H, D_out)

    callbacks = [
                    LossOptimizerHandler(),
                    #ADDING ModelCheckPoint WITH save_full_trainer=True TO SAVE FULL TRAINER
                    ModelCheckPoint(checkpoint_dir=save_to_dir,
                                    checkpoint_file_name=trainer_file_name,
                                    callback_monitor=CallbackMonitor(monitor_type=MonitorType.LOSS,
                                                                     stats_type=StatsType.VAL,
                                                                     monitor_mode=MonitorMode.MIN),
                                    save_best_only=True,
                                    save_full_trainer=True),
                    SchedulerStep(),
                    # SINCE ACCURACY NEEDS TO GO UP AND INACCURACY NEEDS TO GO DOWN, LETS DEFINE CallbackMonitors for StatsPrint PER EACH METRIC
                    StatsPrint(train_metrics_monitors=CallbackMonitor(monitor_type=MonitorType.METRIC,
                                                                       stats_type=StatsType.TRAIN,
                                                                       monitor_mode=MonitorMode.MAX,
                                                                       metric_name='Accuracy'))
                ]

    trainer = Trainer(model=model,
                      device=device,
                      loss_func=loss_func,
                      optimizer=optimizer,
                      scheduler=scheduler,
                      metrics=metrics,
                      train_data_loader=data_loader,
                      val_data_loader=data_loader,
                      train_steps=data_loader_steps,
                      val_steps=data_loader_steps,
                      callbacks=callbacks,
                      name='Train-Save-Load-Predict-Example')
    return trainer

def run():
    N, D_in, H, D_out, num_epochs, data_loader, data_loader_steps = get_parameters()

    current_trainer = get_trainer(N, D_in, H, D_out, num_epochs, data_loader, data_loader_steps)

    # TRAINING WILL SAVE current_trainer IN ModelCheckPoint
    current_trainer.train(num_epochs)

    # YOU CAN ALSO SAVE THIS TRAINER MANUALLY, LIKE THE CODE BELOW
    current_trainer.save_trainer(save_to_dir, trainer_file_name + '_manual_save')

    # NOW LETS CREATE A PREDICTOR FROM THE SAVED FILE
    device = tu.get_gpu_device_if_available()
    model = eu.get_basic_model(D_in, H, D_out).to(device)
    predictor = Predictor(model=model, device=device)

    data_generator_for_predictions = eu.examples_prediction_data_generator(data_loader, data_loader_steps)

    #PREDICT ON A SINGLE SAMPLE
    sample = next(data_generator_for_predictions)[0]
    sample_prediction = predictor.predict_sample(sample)

    # PREDICT ON A SINGLE BATCH
    batch = next(data_generator_for_predictions)
    batch_prediction = predictor.predict_batch(batch)

    # PREDICTION ON A DATA LOADER
    data_loader_predictions = predictor.predict_data_loader(data_generator_for_predictions, data_loader_steps)


import json
import numpy as np
import webview
import os

class SOMScatterviewService:
    def __init__(self, ds):
        self.cache = {
            "DATASET_PATH": None,
            "DATASET": None,
            "AXES": None,
            "WEIGHTS": None,
            "WEIGHTS_NODES": None,
            "WEIGHTS_EDGES": None
        }
        self.datastore = ds

    # Update the current weights being shown according to epoch
    def update_scatter_som_weights_by_training_epoch(self, epoch):
        if not self.cache['WEIGHTS']:
            return None

        weights = self.cache["WEIGHTS"]["weightspb"][str(epoch)]
        width = self.cache["WEIGHTS"]["w"]
        height = self.cache["WEIGHTS"]["h"]
        nodes = []
        edges = []

        for point in weights:
            nodes.append({"x": float(point[0]), "y": float(point[1]), "z": float(point[2])})

        for i in range(len(weights)):
            center = nodes[i]
            if i % width == width - 1:
                if i != len(weights) - 1:
                    down = nodes[i + width]
                    edges.append([center, down])
            elif int(i / width) == height - 1:
                right = nodes[i + 1]
                edges.append([center, right])
            else:
                down = nodes[i + width]
                right = nodes[i + 1]
                edges.append([center, down])
                edges.append([center, right])

        self.cache['WEIGHTS_NODES'] = nodes
        self.cache['WEIGHTS_EDGES'] = edges

    def update_scatter_dataset(self, data):
        scatter = []
        counter = 0

        # Iterate through each line of coordinates
        for i in range(len(data)):
            # Append coordinate instance to dataset list
            scatter.append({"x": float(data[i][0]), "y": float(data[i][1]), "z": float(data[i][2]), "id": 'point_' + str(counter)})

        self.cache["DATASET"] = scatter

    def get_scatter_som_weights(self):
        return [self.cache["WEIGHTS_NODES"], self.cache["WEIGHTS_EDGES"]]

    # Return dataset points and axes
    def get_scatter_dataset(self):
        return self.cache["DATASET"]

    # Return SOM dimensions
    def upload_scatter_weights_from_json_file(self):
        # Get weights json file
        filename = webview.windows[0].create_file_dialog(webview.OPEN_DIALOG)

        # Validate json file
        if filename == None:
            return None

        if len(filename) < 1:
            return None
        filename = filename[0]
        if not os.path.exists(filename):
            return None

        # Open and process json file
        fields = json.loads(open(filename).read())
        self.cache['WEIGHTS'] = fields
        self.update_scatter_som_weights_by_training_epoch(0)
        return len(fields['weights'])

    # Return dataset dimensions
    def upload_scatter_dataset(self):
        # Get dataset file from application file upload dialog
        filename = webview.windows[0].create_file_dialog(webview.OPEN_DIALOG)

        # Check if selected file is valid
        if filename == None:
            return None
        if len(filename) < 1:
            return None
        filename = filename[0]
        if not os.path.exists(filename):
            return None

        # Open dataset file and store/update record
        self.cache["DATASET_PATH"] = os.path.abspath(filename)
        lines = open(filename).readlines()
        lines = [line.strip().split(",") for line in lines]
        self.update_scatter_dataset(lines)
        return os.path.basename(filename)

    # Clean up function
    def resetDatastore(self):
        self.cache = {
            "DATASET_PATH": None,
            "DATASET": None,
            "AXES": None,
            "WEIGHTS": None,
            "WEIGHTS_NODES": None,
            "WEIGHTS_EDGES": None
        }


import logging
import sys
from datetime import datetime

def create_logger(filename = None):
    if filename is None:
        now = datetime.now()
        filename = "log_" + now.strftime("%m_%d_%Y__%H_%M_%S")
    try:
        logger = logging.getLogger("main_log")
    except OSError:
        pass
    logger.setLevel(logging.DEBUG)
    # create a file handler
    handler = logging.FileHandler(filename)
    handler.setLevel(logging.DEBUG)
    # create a logging format
    # formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    formatter = MyFormatter()
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.DEBUG)
    # formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    formatter = MyFormatter()
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    return logger

# Custom formatter
class MyFormatter(logging.Formatter):
    err_fmt = "%(asctime)s - %(levelname)s - %(message)s"
    info_fmt = "%(asctime)s - %(levelname)s - %(message)s"
    dbg_fmt = "%(message)s"

    def __init__(self, fmt="%(levelno)s: %(msg)s"):
        logging.Formatter.__init__(self, fmt)

    def format(self, record):

        # Save the original format configured by the user
        # when the logger formatter was instantiated
        format_orig = self._fmt

        # Replace the original format with one customized by logging level
        if record.levelno == logging.DEBUG:
            self._fmt = MyFormatter.dbg_fmt

        elif record.levelno == logging.INFO:
            self._fmt = MyFormatter.info_fmt

        elif record.levelno == logging.ERROR:
            self._fmt = MyFormatter.err_fmt

        # Call the original formatter class to do the grunt work
        result = logging.Formatter.format(self, record)

        # Restore the original format configured by the user
        self._fmt = format_orig

        return result


import pytest
import pandas as pd
import numpy as np
from shapely import wkt

from tenzing.core.model_implementations import *

_test_suite = [
    pd.Series([1, 2, 3], name='int_series'),
    pd.Series([1, 2, 3], name='categorical_int_series', dtype='category'),
    pd.Series([1, 2, np.nan], name='int_nan_series'),

    pd.Series([1.0, 2.1, 3.0], name='float_series'),
    pd.Series([1.0, 2.5, np.nan], name='float_nan_series'),
    pd.Series([1.0, 2.0, 3.1], dtype='category', name='categorical_float_series'),

    pd.Series(['hello', 'world'], name='string_series'),
    pd.Series(['hello', 'world'], dtype='category', name='categorical_string_series'),
    pd.Series(['2017-10-01', '12/05/2017'], name='timestamp_string_series'),

    pd.Series([True, False], name='bool_series'),

    pd.Series([np.complex(0, 0), np.complex(1, 2), np.complex(3, -1), np.nan], name='complex_series'),
    pd.Series([np.complex(0, 0), np.complex(1, 2), np.complex(3, -1), np.nan], name='categorical_complex_series',
              dtype='category'),

    pd.Series([pd.datetime(2017, 3, 5), pd.datetime(2019, 12, 4)], name='timestamp_series'),

    pd.Series(['POINT (-92 42)', 'POINT (-92 42.1)', 'POINT (-92 42.2)'], name='geometry_string_series'),
    pd.Series([wkt.loads('POINT (-92 42)'), wkt.loads('POINT (-92 42.1)'),
               wkt.loads('POINT (-92 42.2)')], name='geometry_series'),
]

def make_pytest_parameterization(true_series):
    def mark_pytest_param(item):
        mark = pytest.mark.basic() if (item.name in true_series) else pytest.mark.xfail()
        return pytest.param(item, marks=mark, id=item.name)
    test_marks = [mark_pytest_param(item) for item in _test_suite]
    return pytest.mark.parametrize('series', test_marks)

@pytest.fixture(params=_test_suite, ids=lambda series: series.name)
def test_series(request):
    yield request.param

@make_pytest_parameterization(['int_series', 'int_nan_series'])
def test_int_contains(series):
    type = tenzing_integer
    assert series in type

@make_pytest_parameterization(['float_series', 'float_nan_series'])
def test_float_contains(series):
    type = tenzing_float
    assert series in type

@make_pytest_parameterization(['categorical_int_series', 'categorical_float_series', 'categorical_string_series',
                               'categorical_complex_series'])
def test_categorical_contains(series):
    type = tenzing_categorical
    assert series in type

@make_pytest_parameterization(['bool_series'])
def test_bool_contains(series):
    type = tenzing_bool
    assert series in type

@make_pytest_parameterization(['complex_series'])
def test_complex_contains(series):
    type = tenzing_complex
    assert series in type

@make_pytest_parameterization(['timestamp_series'])
def test_timestamp_contains(series):
    type = tenzing_timestamp
    assert series in type

@make_pytest_parameterization(['timestamp_string_series', 'string_series', 'geometry_string_series'])
def test_string_contains(series):
    type = tenzing_string
    assert series in type

@make_pytest_parameterization(['geometry_series'])
def test_geometry_contains(series):
    type = tenzing_geometry
    assert series in type


"""
从68个特征点提取出6个特征点
"""

import os
import tqdm

def thread_(txt_lst):
    for txt in tqdm.tqdm(txt_lst):
        with open(f'../val_lables/{txt}', 'r') as f:
            datas = f.readlines()
            with open(f'../6_val_txt/{txt}','w') as w:
                for data in datas:

                    data = data.split()
                    bbox = list(map(float, data[1:5]))
                    bbox = list(map(int, bbox))
                    tmp_lst = data[:6]
                    x = data[6::3]
                    y = data[7::3]

                    left_eye = [x[36], y[36]]
                    right_eye = [x[45], y[45]]
                    chin = [x[8], y[8]]
                    nose = [x[30], y[30]]

                    left_mouse = [x[48], y[48]]
                    right_mouse = [x[54], y[54]]
                    tmp_lst += left_eye
                    tmp_lst += right_eye
                    tmp_lst += nose
                    tmp_lst += left_mouse
                    tmp_lst += right_mouse
                    tmp_lst += chin
                    tmp_lst[5] = '6_ld'
                    w.write(" ".join(tmp_lst) + '\n')

if __name__ == '__main__':
    txt_lst = os.listdir('../val_lables')
    # n_thread = 6
    # every_len = len(txt_lst) // n_thread
    # Process = []
    # for i in range(n_thread + 1):
    thread_(txt_lst)


import streamlit as st
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

st.title('Books recommender')
html_temp = """

Books recommender

"""
st.markdown(html_temp,unsafe_allow_html= True)
books1 = pd.read_csv("https://raw.githubusercontent.com/asaikiran1999/books-recommender/main/data/books1.csv")
books2 = pd.read_csv("https://raw.githubusercontent.com/asaikiran1999/books-recommender/main/data/books2.csv")
books3 = pd.read_csv("https://raw.githubusercontent.com/asaikiran1999/books-recommender/main/data/books3.csv")
books4 = pd.read_csv("https://raw.githubusercontent.com/asaikiran1999/books-recommender/main/data/books4.csv")
books1= books1.append(books2)
books1= books1.append(books3)
books1= books1.append(books4)
books = books1.copy()
page = st.selectbox("Choose your page", ["Top 50 book recommendation", "selected book recommender"])
def paginator(label, items, items_per_page=10, on_sidebar=True):
	"""Lets the user paginate a set of items.
	Parameters
	----------
	label : str
	    The label to display over the pagination widget.
	items : Iterator[Any]
	    The items to display in the paginator.
	items_per_page: int
	    The number of items to display per page.
	on_sidebar: bool
	    Whether to display the paginator widget on the sidebar.

	Returns
	-------
	Iterator[Tuple[int, Any]]
	    An iterator over *only the items on that page*, including
	    the item's index.
	Example
	-------
	This shows how to display a few pages of fruit.
	>>> fruit_list = [
	...     'Kiwifruit', 'Honeydew', 'Cherry', 'Honeyberry', 'Pear',
	...     'Apple', 'Nectarine', 'Soursop', 'Pineapple', 'Satsuma',
	...     'Fig', 'Huckleberry', 'Coconut', 'Plantain', 'Jujube',
	...     'Guava', 'Clementine', 'Grape', 'Tayberry', 'Salak',
	...     'Raspberry', 'Loquat', 'Nance', 'Peach', 'Akee'
	... ]
	...
	... for i, fruit in paginator("Select a fruit page", fruit_list):
	...     st.write('%s. **%s**' % (i, fruit))
	"""

	# Figure out where to display the paginator
	if on_sidebar:
	    location = st.sidebar.empty()
	else:
	    location = st.empty()

	# Display a pagination selectbox in the specified location.
	items = list(items)
	n_pages = len(items)
	n_pages = (len(items) - 1) // items_per_page + 1
	page_format_func = lambda i: "Page %s" % i
	page_number = location.selectbox(label, range(n_pages), format_func=page_format_func)

	# Iterate over the items in the page to let the user display them.
	min_index = page_number * items_per_page
	max_index = min_index + items_per_page
	import itertools
	return itertools.islice(enumerate(items), min_index, max_index)

if page == "Top 50 book recommendation":
	items2 = []
	items3 = []
	x =  pd.read_csv("https://raw.githubusercontent.com/asaikiran1999/books-recommender/main/data/top50book.csv")
	for i in range(50):
		items2.append(x['Image-URL-M'][i])
		items3.append(x['Book-Title'][i])
	i=0

	image_iterator = paginator("Select next page", items2)
	indices_on_page, images_on_page = map(list, zip(*image_iterator))
	st.image(images_on_page, width=200)

elif page == "selected book recommender":
	users= pd.read_csv("https://raw.githubusercontent.com/asaikiran1999/books-recommender/main/data/Users.csv")
	ratings = pd.read_csv("https://raw.githubusercontent.com/asaikiran1999/books-recommender/main/data/Ratings.csv")
	ratings_and_books = ratings.merge(books,on='ISBN')
	x=ratings_and_books.groupby('User-ID').count()['Book-Rating']>200
	best_users=x[x].index
	filtered_ratings =ratings_and_books[ratings_and_books['User-ID'].isin(best_users)]
	y=filtered_ratings.groupby('Book-Title').count()['Book-Rating']>=50
	famous_books=y[y].index
	filtered_famous = filtered_ratings[filtered_ratings['Book-Title'].isin(famous_books)]
	pivot = filtered_famous.pivot_table(index ='Book-Title',columns='User-ID',values='Book-Rating')
	pivot.fillna(0,inplace=True)
	similarity_scores = cosine_similarity(pivot)

	def recommend(book_name):
	  index = np.where(pivot.index == book_name)[0][0]
	  similar_items =sorted(list(enumerate(similarity_scores[index])),key=lambda x:x[1],reverse=True)[1:6]
	  items=[]
	  for i in similar_items:
	    #print(pivot.index[i[0]])
	    temp_df = books[books['Book-Title']==pivot.index[i[0]]]
	    items.append(list(temp_df.drop_duplicates('Book-Title')['Image-URL-M']))
	  return items
	input = st.text_input("Book_Name","")
	st.text('most search books are :')
	st.text("Harry Potter and the Sorcerer's Stone (Book 1)")
	st.text('The Da Vinci Code')
	st.text('The Secret Life of Bees')
	st.text('Girl with a Pearl Earring')
	st.text('About a Boy')

	if st.button('recommend'):
		r=[]
		y=list(recommend(input))
		for i in range(5):
			r.append(y[i][0])
		image_iterators = paginator("Select next page", r)
		indices_on_pages, images_on_pages = map(list, zip(*image_iterators))
		st.image(images_on_pages, width=200)


import numpy as np
import matplotlib.pyplot as plt
from matplotlib import image as mpimg
from matplotlib import pyplot as plt
from unit_tests import TEST_IMG_DIR
from copy import deepcopy
import os
from carnd_vehicle_detection import DEFAULT_EXTRACT_PARAMS
from carnd_vehicle_detection.preprocess import single_img_features, extract_prediction_features, extract_global_hog_features, convert_color
from carnd_vehicle_detection.visualize import one_by_two_plot
from carnd_vehicle_detection.traverse_image import get_global_hog_features, get_image_hog_features, \
    get_hog_extraction_coordinates
import pickle
from carnd_vehicle_detection import ROOT_DIR
import os
import cv2

def _discard_alpha_channel(img):
    return img[:, :, :3]

FULL_IMAGE = _discard_alpha_channel(mpimg.imread(os.path.join(TEST_IMG_DIR, 'test_frame_1.png')))
SUBIMAGE_SKY_64_64 = _discard_alpha_channel(mpimg.imread(os.path.join(TEST_IMG_DIR, 'test_subframe_1_1.png')))
SUBIMAGE_ROAD_64_64 = _discard_alpha_channel(mpimg.imread(os.path.join(TEST_IMG_DIR, 'test_subframe_1_3.png')))
SUBIMAGE_ROAD_2_64_64 = _discard_alpha_channel(mpimg.imread(os.path.join(TEST_IMG_DIR, 'test_subframe_1_4.png')))
SUBIMAGE_ROAD_SCALED = _discard_alpha_channel(mpimg.imread(os.path.join(TEST_IMG_DIR, 'test_subframe_1_5.png')))
SUBIMAGE_ROAD_SCALED_2 = _discard_alpha_channel(mpimg.imread(os.path.join(TEST_IMG_DIR, 'test_subframe_1_6.png')))
SCALER_PATH = os.path.join(ROOT_DIR, 'classifier_scaler.p')
with open(SCALER_PATH, 'rb') as scalerfile:
    SCALER = pickle.load(scalerfile)

def visualize_feature_vectors():
    """Visualize the feature vectors chosen from a couple of subimages from a frame. The selections have been picked
    manually."""

    # Extract and scale the params for non-scaled windows
    features_sky = SCALER.transform(np.ravel(single_img_features(SUBIMAGE_SKY_64_64, **DEFAULT_EXTRACT_PARAMS)))
    features_road = SCALER.transform(np.ravel(single_img_features(SUBIMAGE_ROAD_64_64, **DEFAULT_EXTRACT_PARAMS)))
    features_road_2 = SCALER.transform(np.ravel(single_img_features(SUBIMAGE_ROAD_2_64_64, **DEFAULT_EXTRACT_PARAMS)))

    # Extract and scale params for 1.5 scaled windows
    extract_params = deepcopy(DEFAULT_EXTRACT_PARAMS)
    extract_params['pix_per_cell'] = 12
    features_road_scaled = SCALER.transform(np.ravel(single_img_features(SUBIMAGE_ROAD_SCALED, **extract_params)))
    features_road_scaled_2 = SCALER.transform(np.ravel(single_img_features(SUBIMAGE_ROAD_SCALED_2, **extract_params)))

    plt.imshow(FULL_IMAGE)
    plt.show()
    one_by_two_plot(SUBIMAGE_SKY_64_64, SUBIMAGE_ROAD_64_64, None, None, "sky", "Road")

    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 10))
    ax1.set_title("Non-vehicle, scaled features")
    ax1.plot(list(range(len(features_sky))), features_sky)
    ax2.set_title("Vehicle, scaled features")
    ax2.plot(list(range(len(features_road))), features_road)
    plt.show()

    window_params = {'window': 64, 'ystart': 0, 'ystop': None}

    img_tosearch = FULL_IMAGE[window_params['ystart']:window_params['ystop'], :, :]
    ctrans_tosearch = convert_color(img_tosearch, conv=extract_params['color_space'])

    wins_per_scale = {1: [((128, 400), (192, 464)), ((248, 400), (312, 464))], 1.5: [((120, 396), (216, 492)), ((144, 420), (240, 516))]}
    ref_feat_per_scale = {1: [features_road, features_road_2], 1.5: [features_road_scaled, features_road_scaled_2]}
    for scale in (1, 1.5):
        windows = wins_per_scale[scale]

        for window in windows:
            xpos, ypos = window[0]
            window_params['ypos'] = ypos
            window_params['xpos'] = xpos
            xleft, ytop = xpos * extract_params['pix_per_cell'], ypos * extract_params['pix_per_cell']
            window_params['xleft'] = xleft
            window_params['ytop'] = ytop
            window_params['scale'] = scale
            nblocks_per_window = (window_params['window'] // extract_params['pix_per_cell']) - extract_params['cell_per_block'] + 1
            window_params['nblocks_per_window'] = nblocks_per_window

            ch0 = ctrans_tosearch[:, :, 0]
            ch1 = ctrans_tosearch[:, :, 1]
            ch2 = ctrans_tosearch[:, :, 2]

            # Let us put the channels into a dict for easier dynamic lookup
            channels_dict = {0: ch0, 1: ch1, 2: ch2}
            global_hog_features = extract_global_hog_features(channels_dict, extract_params['orient'],
                                                              extract_params['pix_per_cell'],
                                                              extract_params['cell_per_block'],
                                                              extract_params['hog_channel'])
        params = {'global_hog_features': global_hog_features,
                  'window_params': window_params,
                  'extract_params': extract_params}

        scaled_features = extract_prediction_features(ctrans_tosearch, SCALER, **params)

        f2, (ax21, ax22) = plt.subplots(1, 2, figsize=(20, 10))
        ax21.set_title("Single-pass HOG extraction results")
        ax21.plot(list(range(len(scaled_features))), scaled_features)
        ax22.set_title("Per-image HOG extraction results")
        ax22.plot(list(range(len(ref_feat_per_scale[scale][0]))), ref_feat_per_scale[scale][0])
        f2.suptitle("First window for scale {}".format(scale))
        plt.show()

        # plt.plot(list(range(81)), features_vehicle[120:201] - subfeatures_vehicle[120:201])
        plt.plot(list(range(len(scaled_features))), ref_feat_per_scale[scale][0] - scaled_features)
        plt.show()

    # f3, (ax31, ax32) = plt.subplots(1, 2, figsize=(20, 10))
    # ax31.set_title("Final image, Single-pass HOG extraction results")
    # ax31.plot(list(range(len(features_road_2))), features_road_2)
    # ax32.set_title("Final image, Per-image 64x64 features")
    # ax32.plot(list(range(len(subfeatures_from_64_64))), subfeatures_from_64_64)
    # plt.show()

    # plt.plot(list(range(len(features_road_2))), subfeatures_from_64_64 - subfeatures_vehicle_2)
    # plt.plot(list(range(81)), ref_feat_per_scale[scale][1][120:201] - subfeatures_vehicle_2[120:201])
    # plt.show()

if __name__ == "__main__":
    visualize_feature_vectors()


# -*- coding: utf-8 -*-

import math

import torch

import utils

class NumberSys(torch.nn.Module):

    def __init__(self, matrix_size, int_size, logic_size):
        super(NumberSys, self).__init__()
        self.matrix_size = matrix_size
        self.int_size = int_size
        self.logic_size = logic_size

        # int
        self.zero = torch.nn.Parameter(
            torch.Tensor(self.matrix_size, self.matrix_size))
        self.M = torch.nn.Parameter(
            torch.Tensor(self.matrix_size, self.matrix_size))

        # independent
        # self.int = torch.nn.Parameter(torch.Tensor(self.int_size, self.matrix_size, self.matrix_size))

        # logic
        # maybe require logic representation to be orthonormal (inner product of matrices is trace)
        # check vector logics: https://en.wikipedia.org/wiki/Vector_logic
        self.logic = torch.nn.Parameter(
            torch.Tensor(self.logic_size, self.matrix_size, self.matrix_size))

        self.reset_parameters()

    def reset_parameters(self):
        # init int
        torch.nn.init.normal_(self.zero)
        torch.nn.init.kaiming_uniform_(self.M, a=math.sqrt(5))

        # independent
        # torch.nn.init.normal_(self.int)

        # init logic
        torch.nn.init.normal_(self.logic)

    def forward(self):
        int_list = [self.zero]
        for _ in range(self.int_size - 1):
            int_list.append(torch.matmul(self.M, int_list[-1]))
        # Tensor of (case, matrix_size, matrix_size)
        return torch.stack(int_list, dim=0), self.logic

        # independent
        # return self.int, self.logic

class Attribute(object):

    def __init__(self, name, space_size, mode):
        self.attr = name
        self.space_size = space_size
        self.mode = mode

    def construct(self, int_space, logic_space):
        if self.mode == "int":
            space = int_space[:self.space_size, :, :]
        elif self.mode == "logic":
            space = logic_space[:self.space_size, :, :]
        return space

    def forward(self, indices, space):
        # indices: Tensor of any shape
        # space: Tensor of shape (case, matrix_size, matrix_size)
        size = list(indices.shape)
        size.append(space.shape[-1])
        size.append(space.shape[-1])
        return torch.index_select(space, 0, indices.view(-1)).view(size)

    def decode(self, predict_matrix, space):
        # predict_matrix: Tensor of shape (batch, matrix_size, matrix_size)
        # space: Tensor of shape (case, matrix_size, matrix_size)
        predict_matrix = predict_matrix.unsqueeze(1).expand(
            -1, space.shape[0], -1, -1)
        dist = utils.squared_norm(predict_matrix - space.unsqueeze(0),
                                  dim=(2, 3))
        return torch.nn.functional.softmax(-dist, dim=-1)


#! /usr/bin/python3
"""Add audio to notes lacking it. This is an example of how to manipulate notes' fields using anki_tool, which you can use as a base for your own application.
Usage: anki_tool -q search . | anki_tool -q dump_fields | ./example_field_modifier.py | anki_tool -f replace_fields"""

import sys
import json
import re
import anki_tool

modified_notes = dict()
# reading json produced by dump_fields from stdin
for line in sys.stdin:
    notes = json.loads(line.rstrip())
    # getting back the original dict for each note
    for _id in notes:
        keys, values = notes[_id]
        notes[_id] = anki_tool.lists_to_ordered_dict(keys, values)

    for _id in notes:
        note = notes[_id]
        # if the note does not already contain audio
        if ('Front' in note and
            not re.search(r'\[sound:[^\]+]', note['Front'])):

            # getting first word in the field
            word = re.search(r'^\w+', note['Front']).group()
            # adding audio for that word
            note['Front'] += '<div>[sound:'+word+'.mp3]</div>'

            modified_notes[_id] = note

# prepare notes for output, replacing dict with lists so we don't lose the field
# order
for _id in modified_notes:
    modified_notes[_id] = anki_tool.ordered_dict_to_lists(modified_notes[_id])

# also print our modification in a human-readable way
for _id in modified_notes:
    print(_id, modified_notes[_id], file=sys.stderr)

# print result, which can be fed to replace_fields
print(json.dumps(modified_notes))


# -*- coding:utf-8 -*-
import sys
import os
"""直接运行此脚本，避免import失败的方法"""
__project_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))  # 项目目录
if __project_dir not in sys.path:
    sys.path.append(__project_dir)
from mongo_db import ObjectId
from mongo_db import get_obj_id
from api.data.item_module import User
from manage.company_module import Company
from manage.company_module import Employee
from manage.company_module import Dept
from manage.company_module import Post
import openpyxl

"""从数据库表格读取用户信息,再将对应的用户和公司建立关联"""

excel_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "excel")

def read_excel():
    """
    从excel读取用户信息,返回list
    return: [
    {"user_name": "张三","phone_num": "13848482925", "employee_number": 3434},
    {"user_name": "李四","phone_num": "15644525789", "employee_number: 3435},
    ...
     ]
    """
    file_path = os.path.join(excel_path, "华新中转场驾驶员使用保驾犬明细_2018_5_9.xlsx")
    excel = openpyxl.load_workbook(file_path)
    sheets = excel.sheetnames
    users = list()
    for sheet in sheets:
        sheet = excel.get_sheet_by_name(sheet)
        lines = list()
        for tr in sheet:
            first = tr[0].value
            if first is not None and (isinstance(first, int) or first.isdigit()):
                line = dict()
                line['employee_number'] = str(tr[1].value)
                line['real_name'] = tr[2].value
                line['phone_num'] = str(tr[3].value)
                line['line'] = tr[4].value
                lines.append(line)
        users.extend(lines)
    return users

def rebuild() -> None:
    """
    重建关系员工和公司
    :return:
    """
    us = read_excel()
    p_map = {x['phone_num']: {
        "real_name": x['real_name'],
        "employee_number": x['employee_number'],
        "line": x['line'],
    } for x in us}
    phones = list(p_map.keys())
    f = {"phone_num": {"$in": phones}}
    users = User.find_plus(filter_dict=f, to_dict=False)
    ids = list()
    for e in users:
        phone = e.get_attr('phone_num')
        temp = p_map[phone]
        e.set_attr('real_name', temp['real_name'])
        e.set_attr('employee_number', temp['employee_number'])
        e.set_attr('line', temp['line'])
        r = e.save_plus()
        if r is None:
            print("保存失败")
        else:
            ids.append(r)
    company_id = get_obj_id("59a671a0de713e3db34de8bf")  # 顺丰公司
    dept_id = get_obj_id("5abcac4b4660d3599207fe18")  # 顺丰华新分部
    post_id = get_obj_id("59a67348de713e3f43dcf0d7")  # 司机岗位
    Company.rebuild_relation(company_id=company_id, dept_id=dept_id, post_id=post_id, employee_list=ids)

if __name__ == "__main__":
    """修复顺丰员工和企业的关系"""
    rebuild()
    pass