# ===============
# best_ckpt=
# ===============
import os
import gc
import copy
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#import segmentation_models_pytorch as smp
from apex import amp
from contextlib import contextmanager
from albumentations import *
import torch
from torch.utils.data import DataLoader
from torch.optim.lr_scheduler import CosineAnnealingLR

import sys
sys.path.append("../severstal-src/")
from util import seed_torch, search_threshold
from losses import FocalLovaszLoss
from datasets import SeverDataset, MaskProbSampler
from logger import setup_logger, LOGGER
from trainer import train_one_epoch, validate
from scheduler import GradualWarmupScheduler
sys.path.append("../")
import segmentation_models_pytorch as smp
from sync_batchnorm import convert_model

# ===============
# Constants
# ===============
IMG_DIR = "../input/train_images/"
LOGGER_PATH = "log.txt"
FOLD_PATH = "../input/severstal_folds01.csv"
ID_COLUMNS = "ImageId"
N_CLASSES = 4

# ===============
# Settings
# ===============
SEED = np.random.randint(100000)
device = "cuda"
IMG_SIZE = (1600, 256)
CLR_CYCLE = 3
BATCH_SIZE = 16
EPOCHS = 101
FOLD_ID = 0
EXP_ID = "exp56_unet_seresnext"
CLASSIFICATION = True
EMA = True
EMA_START = 6
base_ckpt = 3
base_model = None
base_model_ema = None
base_model = "models/{}_fold{}_latest.pth".format(EXP_ID, FOLD_ID)
base_model_ema = "models/{}_fold{}_latest_ema.pth".format(EXP_ID, FOLD_ID)

setup_logger(out_file=LOGGER_PATH)
seed_torch(SEED)
LOGGER.info("seed={}".format(SEED))

@contextmanager
def timer(name):
    t0 = time.time()
    yield
    LOGGER.info('[{}] done in {} s'.format(name, round(time.time() - t0, 2)))

import os
import cv2
import torch
import random
import pydicom
import numpy as np
from torch.utils.data import Dataset
from torch.utils.data.sampler import Sampler

from util import rle2mask
from logger import LOGGER

class SeverDataset(Dataset):

    def __init__(self,
                 df,
                 img_dir,
                 img_size,
                 n_classes,
                 crop_rate=1.0,
                 id_colname="ImageId",
                 mask_colname=["EncodedPixels_{}".format(i) for i in range(1, 5)],
                 transforms=None,
                 means=[0.485, 0.456, 0.406],
                 stds=[0.229, 0.224, 0.225],
                 class_y=None,
                 cut_h=False,
                 crop_320=False
                 ):
        self.df = df
        self.img_dir = img_dir
        self.img_size = img_size
        self.transforms = transforms
        self.means = np.array(means)
        self.stds = np.array(stds)
        self.id_colname = id_colname
        self.mask_colname = mask_colname
        self.n_classes = n_classes
        self.crop_rate = crop_rate
        self.class_y = class_y
        self.cut_h = cut_h
        self.crop_320 = crop_320

    def __len__(self):
        return self.df.shape[0]

    def __getitem__(self, idx):
        cur_idx_row = self.df.iloc[idx]
        img_id = cur_idx_row[self.id_colname]
        img_id = "17e12e6cb.jpg"
        img_path = os.path.join(self.img_dir, img_id)

        img = cv2.imread(img_path)
        w, h, _ = img.shape
        mask = np.zeros((w, h, self.n_classes))
        for i, encoded in enumerate(cur_idx_row[self.mask_colname]):
            if encoded in "-1":
                continue
            else:
                mask[:, :, i] = rle2mask(encoded, (w, h))

        if self.crop_rate < 1:
            img, mask = random_wcropping(img, mask, is_random=True, ratio=self.crop_rate)
        if self.crop_320:
            img, mask = random_320cropping(img, mask)
        img = cv2.resize(img, self.img_size)
        mask = cv2.resize(mask, self.img_size)
        mask[mask != 0] = 1

        if self.transforms is not None:
            augmented = self.transforms(image=img, mask=mask)
            img = augmented['image']
            mask = augmented['mask']
        if self.cut_h:
            img, mask = cutout_h(img, mask, self.img_size, self.n_classes)

        img = img / 255
        img -= self.means
        img /= self.stds
        img = img.transpose((2, 0, 1))
        mask = mask.transpose((2, 0, 1))
        mask = torch.from_numpy(mask)

        if self.class_y is not None:
            class_y_ = self.class_y[idx]
            target = {"mask": mask, "class_y": torch.tensor(class_y_)}
        else:
            target = mask

        return torch.from_numpy(img), target, img_id

def validate(model, valid_loader, criterion, device, classification=False):
    model.eval()
    test_loss = 0.0
    true_ans_list = []
    preds_cat = []
    with torch.no_grad():

        for step, (features, targets, ids) in enumerate(valid_loader):
            features, targets = features.to(device), targets.to(device)
            print(ids)

            print(features)
            print()
            if classification:
                logits, _ = model(features)
                loss = criterion(logits, targets)
                print(_)
                print()
            else:
                logits = model(features)
                loss = criterion(logits, targets)

            test_loss += loss.item()
            print(loss.item())

            if not loss.item() > 0:
                print(logits)
                print()
                print(targets)
                kk
            #true_ans_list.append(targets.float().cpu().numpy().astype("int8"))
            #preds_cat.append(torch.sigmoid(logits).float().cpu().numpy().astype("float16"))

            del features, targets, logits
            gc.collect()

        #all_true_ans = np.concatenate(true_ans_list, axis=0)
        #all_preds = np.concatenate(preds_cat, axis=0)

    return test_loss / (step + 1)#, all_preds, all_true_ans

def main(seed):
    with timer('load data'):
        df = pd.read_csv(FOLD_PATH)
        y1 = (df.EncodedPixels_1 != "-1").astype("float32").values.reshape(-1, 1)
        y2 = (df.EncodedPixels_2 != "-1").astype("float32").values.reshape(-1, 1)
        y3 = (df.EncodedPixels_3 != "-1").astype("float32").values.reshape(-1, 1)
        y4 = (df.EncodedPixels_4 != "-1").astype("float32").values.reshape(-1, 1)
        y = np.concatenate([y1, y2, y3, y4], axis=1)

    with timer('preprocessing'):
        train_df, val_df = df[df.fold_id != FOLD_ID], df[df.fold_id == FOLD_ID]
        y_train, y_val = y[df.fold_id != FOLD_ID], y[df.fold_id == FOLD_ID]

        train_augmentation = Compose([
            Flip(p=0.5),
            OneOf([
                GridDistortion(p=0.5),
                OpticalDistortion(p=0.5, distort_limit=2, shift_limit=0.5)
            ], p=0.5),
            OneOf([
                RandomGamma(gamma_limit=(100,140), p=0.5),
                RandomBrightnessContrast(p=0.5),
                RandomBrightness(p=0.5),
                RandomContrast(p=0.5)
            ], p=0.5),
            OneOf([
                GaussNoise(p=0.5),
                Cutout(num_holes=10, max_h_size=10, max_w_size=20, p=0.5)
            ], p=0.5),
            ShiftScaleRotate(rotate_limit=20, p=0.5),
        ])
        val_augmentation = None

        train_dataset = SeverDataset(train_df, IMG_DIR, IMG_SIZE, N_CLASSES, id_colname=ID_COLUMNS,
                                    transforms=train_augmentation, crop_rate=1.0, class_y=y_train)
        val_dataset = SeverDataset(val_df, IMG_DIR, IMG_SIZE, N_CLASSES, id_colname=ID_COLUMNS,
                                  transforms=val_augmentation)
        train_sampler = MaskProbSampler(train_df, demand_non_empty_proba=0.6)
        train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=8)
        val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8)

        del train_df, val_df, df, train_dataset, val_dataset
        gc.collect()

    with timer('create model'):
        model = smp.Unet('se_resnext50_32x4d', encoder_weights="imagenet", classes=N_CLASSES, encoder_se_module=True,
                         decoder_semodule=True, h_columns=False, skip=True, act="swish", freeze_bn=True,
                         classification=CLASSIFICATION, attention_type="cbam", center=True)
        model = convert_model(model)
        if base_model is not None:
            model.load_state_dict(torch.load(base_model))
        model.to(device)

        criterion = torch.nn.BCEWithLogitsLoss()
        optimizer = torch.optim.Adam([
            {'params': model.decoder.parameters(), 'lr': 3e-3},
            {'params': model.encoder.parameters(), 'lr': 3e-4},
        ])
        if base_model is None:
            scheduler_cosine = CosineAnnealingLR(optimizer, T_max=CLR_CYCLE, eta_min=3e-5)
            scheduler = GradualWarmupScheduler(optimizer, multiplier=1.1, total_epoch=CLR_CYCLE*2, after_scheduler=scheduler_cosine)
        else:
            scheduler = CosineAnnealingLR(optimizer, T_max=CLR_CYCLE, eta_min=3e-5)

        model, optimizer = amp.initialize(model, optimizer, opt_level="O1", verbosity=0)

        if EMA:
            ema_model = copy.deepcopy(model)
            if base_model_ema is not None:
                ema_model.load_state_dict(torch.load(base_model_ema))
            ema_model.to(device)
        else:
            ema_model = None
        model = torch.nn.DataParallel(model)
        ema_model = torch.nn.DataParallel(ema_model)

    with timer('train'):
        valid_loss = validate(model, val_loader, criterion, device, classification=CLASSIFICATION)

if __name__ == '__main__':
    main(SEED)


import pytest

from keywords.ClusterKeywords import ClusterKeywords
from keywords.SyncGateway import (sync_gateway_config_path_for_mode,
                                  validate_sync_gateway_mode)
from keywords.tklogging import Logging
from keywords.constants import CLUSTER_CONFIGS_DIR
from keywords.utils import log_info, check_xattr_support, version_is_binary, clear_resources_pngs

from keywords.exceptions import ProvisioningError, FeatureSupportedError
from keywords.utils import host_for_url
from libraries.testkit import cluster
from utilities.cluster_config_utils import persist_cluster_config_environment_prop
from utilities.cluster_config_utils import get_load_balancer_ip
from libraries.testkit import prometheus

# This will be called once for the at the beggining of the execution of each .py file
# in the 'topology_specific_tests/multiple_syncgateways' directory.
# It will be torn down (code after the yeild) when all of the tests have executed in that file
@pytest.fixture(scope="module")
def params_from_base_suite_setup(request):
    log_info("Setting up 'params_from_base_suite_setup' ...")

    # pytest command line parameters
    server_version = request.config.getoption("--server-version")
    sync_gateway_version = request.config.getoption("--sync-gateway-version")
    disable_tls_server = request.config.getoption("--disable-tls-server")
    mode = request.config.getoption("--mode")
    skip_provisioning = request.config.getoption("--skip-provisioning")
    race_enabled = request.config.getoption("--race")
    cbs_ssl = request.config.getoption("--server-ssl")
    xattrs_enabled = request.config.getoption("--xattrs")
    sg_ce = request.config.getoption("--sg-ce")
    cbs_ce = request.config.getoption("--cbs-ce")
    sg_lb = request.config.getoption("--sg-lb")
    use_sequoia = request.config.getoption("--sequoia")
    no_conflicts_enabled = request.config.getoption("--no-conflicts")
    sg_ssl = request.config.getoption("--sg-ssl")
    use_views = request.config.getoption("--use-views")
    number_replicas = request.config.getoption("--number-replicas")
    sg_installer_type = request.config.getoption("--sg-installer-type")
    sa_installer_type = request.config.getoption("--sa-installer-type")
    sg_platform = request.config.getoption("--sg-platform")
    delta_sync_enabled = request.config.getoption("--delta-sync")
    cbs_platform = request.config.getoption("--cbs-platform")
    magma_storage_enabled = request.config.getoption("--magma-storage")
    prometheus_enabled = request.config.getoption("--prometheus-enable")
    hide_product_version = request.config.getoption("--hide-product-version")
    skip_couchbase_provision = request.config.getoption("--skip-couchbase-provision")
    enable_cbs_developer_preview = request.config.getoption("--enable-cbs-developer-preview")
    disable_persistent_config = request.config.getoption("--disable-persistent-config")
    cluster_config = request.config.getoption("--cluster-config")
    enable_server_tls_skip_verify = request.config.getoption("--enable-server-tls-skip-verify")
    disable_tls_server = request.config.getoption("--disable-tls-server")
    disable_admin_auth = request.config.getoption("--disable-admin-auth")
    trace_logs = request.config.getoption("--trace-logs")

    if xattrs_enabled and version_is_binary(sync_gateway_version):
        check_xattr_support(server_version, sync_gateway_version)

    if delta_sync_enabled and sync_gateway_version < "2.5":
        raise FeatureSupportedError('Delta sync feature not available for sync-gateway version below 2.5, so skipping the test')

    log_info("server_version: {}".format(server_version))
    log_info("sync_gateway_version: {}".format(sync_gateway_version))
    log_info("mode: {}".format(mode))
    log_info("skip_provisioning: {}".format(skip_provisioning))
    log_info("race_enabled: {}".format(race_enabled))
    log_info("cbs_ssl: {}".format(cbs_ssl))
    log_info("xattrs_enabled: {}".format(xattrs_enabled))
    log_info("sg_ce: {}".format(sg_ce))
    log_info("sg_lb: {}".format(sg_lb))
    log_info("no conflicts enabled {}".format(no_conflicts_enabled))
    log_info("sg_ssl: {}".format(sg_ssl))
    log_info("use_views: {}".format(use_views))
    log_info("number_replicas: {}".format(number_replicas))
    log_info("sg_installer_type: {}".format(sg_installer_type))
    log_info("sa_installer_type: {}".format(sa_installer_type))
    log_info("sg_platform: {}".format(sg_platform))
    log_info("delta_sync_enabled: {}".format(delta_sync_enabled))
    log_info("prometheus_enabled: {}".format(prometheus_enabled))
    log_info("hide_product_version: {}".format(hide_product_version))
    log_info("enable_cbs_developer_preview: {}".format(enable_cbs_developer_preview))
    log_info("disable_persistent_config: {}".format(disable_persistent_config))
    log_info("trace_logs: {}".format(trace_logs))

    # sg-ce is invalid for di mode
    if mode == "di" and sg_ce:
        raise FeatureSupportedError("SGAccel is only available as an enterprise edition")

    if no_conflicts_enabled and sync_gateway_version < "2.0":
        raise FeatureSupportedError('No conflicts feature not available for sync-gateway version below 2.0, so skipping the test')

    if ('test_scopes_and_collections' in request.node.name) and (use_views or cbs_ssl or magma_storage_enabled or (sync_gateway_version < "3.1.0")):
        pytest.skip("It is not necessary/possible to run the ISGR scopes and collections tests with user views, server ssl, magma or versions prior to 3.1.0")

    # Make sure mode for sync_gateway is supported ('cc' or 'di')
    validate_sync_gateway_mode(mode)

    # use base_cc cluster config if mode is "cc" or base_di cluster config if more is "di"
    cluster_config = "{}/{}{}".format(CLUSTER_CONFIGS_DIR, cluster_config, mode)
    # cluster_config = "{}/three_sync_gateways_{}".format(CLUSTER_CONFIGS_DIR, mode)
    sg_config = sync_gateway_config_path_for_mode("sync_gateway_default_functional_tests", mode)

    if use_views:
        log_info("Running SG tests using views")
        # Enable sg views in cluster configs
        persist_cluster_config_environment_prop(cluster_config, 'sg_use_views', True)
    else:
        log_info("Running tests with cbs <-> sg ssl disabled")
        # Disable sg views in cluster configs
        persist_cluster_config_environment_prop(cluster_config, 'sg_use_views', False)

    # Write the number of replicas to cluster config
    persist_cluster_config_environment_prop(cluster_config, 'number_replicas', number_replicas)

    if sg_ssl:
        log_info("Enabling SSL on sync gateway")
        persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_ssl', True)
    else:
        persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_ssl', False)

    # Add load balancer prop and check if load balancer IP is available
    if sg_lb:
        persist_cluster_config_environment_prop(cluster_config, 'sg_lb_enabled', True)
        log_info("Running tests with load balancer enabled: {}".format(get_load_balancer_ip(cluster_config)))
    else:
        log_info("Running tests with load balancer disabled")
        persist_cluster_config_environment_prop(cluster_config, 'sg_lb_enabled', False)

    if cbs_ssl:
        log_info("Running tests with cbs <-> sg ssl enabled")
        # Enable ssl in cluster configs
        persist_cluster_config_environment_prop(cluster_config, 'cbs_ssl_enabled', True)
    else:
        log_info("Running tests with cbs <-> sg ssl disabled")
        # Disable ssl in cluster configs
        persist_cluster_config_environment_prop(cluster_config, 'cbs_ssl_enabled', False)

    if xattrs_enabled:
        log_info("Running test with xattrs for sync meta storage")
        persist_cluster_config_environment_prop(cluster_config, 'xattrs_enabled', True)
    else:
        log_info("Using document storage for sync meta data")
        persist_cluster_config_environment_prop(cluster_config, 'xattrs_enabled', False)

    try:
        server_version
    except NameError:
        log_info("Server version is not provided")
        persist_cluster_config_environment_prop(cluster_config, 'server_version', "")
    else:
        log_info("Running test with server version {}".format(server_version))
        persist_cluster_config_environment_prop(cluster_config, 'server_version', server_version)

    try:
        sync_gateway_version
    except NameError:
        log_info("Sync gateway version is not provided")
        persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', "")
    else:
        log_info("Running test with sync_gateway version {}".format(sync_gateway_version))

        persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', sync_gateway_version)

    try:
        sg_platform
    except NameError:
        log_info("sg platform  is not provided, so by default it runs on Centos")
        persist_cluster_config_environment_prop(cluster_config, 'sg_platform', "centos", False)
    else:
        log_info("Running test with sg platform {}".format(sg_platform))
        persist_cluster_config_environment_prop(cluster_config, 'sg_platform', sg_platform, False)

    if no_conflicts_enabled:
        log_info("Running with no conflicts")
        persist_cluster_config_environment_prop(cluster_config, 'no_conflicts_enabled', True)
    else:
        log_info("Running with allow conflicts")
        persist_cluster_config_environment_prop(cluster_config, 'no_conflicts_enabled', False)

    if delta_sync_enabled:
        log_info("Running with delta sync")
        persist_cluster_config_environment_prop(cluster_config, 'delta_sync_enabled', True)
    else:
        log_info("Running without delta sync")
        persist_cluster_config_environment_prop(cluster_config, 'delta_sync_enabled', False)

    try:
        cbs_platform
    except NameError:
        log_info("cbs platform  is not provided, so by default it runs on Centos7")
        persist_cluster_config_environment_prop(cluster_config, 'cbs_platform', "centos7", False)
    else:
        log_info("Running test with cbs platform {}".format(cbs_platform))
        persist_cluster_config_environment_prop(cluster_config, 'cbs_platform', cbs_platform, False)

    if magma_storage_enabled:
        log_info("Running with magma storage")
        persist_cluster_config_environment_prop(cluster_config, 'magma_storage_enabled', True, False)
    else:
        log_info("Running without magma storage")
        persist_cluster_config_environment_prop(cluster_config, 'magma_storage_enabled', False, False)

    if hide_product_version:
        log_info("Suppress the SGW product Version")
        persist_cluster_config_environment_prop(cluster_config, 'hide_product_version', True)
    else:
        log_info("Running without suppress SGW product Version")
        persist_cluster_config_environment_prop(cluster_config, 'hide_product_version', False)

    if enable_cbs_developer_preview:
        log_info("Enable CBS developer preview")
        persist_cluster_config_environment_prop(cluster_config, 'cbs_developer_preview', True)
    else:
        log_info("Running without CBS developer preview")
        persist_cluster_config_environment_prop(cluster_config, 'cbs_developer_preview', False)

    if disable_persistent_config:
        log_info(" disable persistent config")
        persist_cluster_config_environment_prop(cluster_config, 'disable_persistent_config', True)
    else:
        log_info("Running without Centralized Persistent Config")
        persist_cluster_config_environment_prop(cluster_config, 'disable_persistent_config', False)

    if enable_server_tls_skip_verify:
        log_info("Enable server tls skip verify flag")
        persist_cluster_config_environment_prop(cluster_config, 'server_tls_skip_verify', True)
    else:
        log_info("Running without server_tls_skip_verify Config")
        persist_cluster_config_environment_prop(cluster_config, 'server_tls_skip_verify', False)

    if disable_tls_server:
        log_info("Disable tls server flag")
        persist_cluster_config_environment_prop(cluster_config, 'disable_tls_server', True)
    else:
        log_info("Enable tls server flag")
        persist_cluster_config_environment_prop(cluster_config, 'disable_tls_server', False)

    if disable_admin_auth:
        log_info("Disabled Admin Auth")
        persist_cluster_config_environment_prop(cluster_config, 'disable_admin_auth', True)
    else:
        log_info("Enabled Admin Auth")
        persist_cluster_config_environment_prop(cluster_config, 'disable_admin_auth', False)

    if trace_logs:
        log_info("Enabled trace logs for Sync Gateway")
        persist_cluster_config_environment_prop(cluster_config, 'trace_logs', True)
    else:
        persist_cluster_config_environment_prop(cluster_config, 'trace_logs', False)

    if sync_gateway_version < "2.0.0" and no_conflicts_enabled:
        pytest.skip("Test cannot run with no-conflicts with sg version < 2.0.0")

    # Skip provisioning if user specifies '--skip-provisoning' or '--sequoia'
    should_provision = True
    if skip_provisioning or use_sequoia:
        should_provision = False

    cluster_utils = ClusterKeywords(cluster_config)
    if should_provision:
        try:
            cluster_utils.provision_cluster(
                cluster_config=cluster_config,
                server_version=server_version,
                sync_gateway_version=sync_gateway_version,
                sync_gateway_config=sg_config,
                race_enabled=race_enabled,
                cbs_platform=cbs_platform,
                sg_ce=sg_ce,
                cbs_ce=cbs_ce,
                sg_platform=sg_platform,
                sg_installer_type=sg_installer_type,
                sa_installer_type=sa_installer_type,
                skip_couchbase_provision=skip_couchbase_provision
            )
        except ProvisioningError:
            logging_helper = Logging()
            logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=request.node.name)
            raise

    # Hit this intalled running services to verify the correct versions are installed
    cluster_utils.verify_cluster_versions(
        cluster_config,
        expected_server_version=server_version,
        expected_sync_gateway_version=sync_gateway_version
    )

    cluster_topology = cluster_utils.get_cluster_topology(cluster_config)
    need_sgw_admin_auth = (not disable_admin_auth) and sync_gateway_version >= "3.0"

    if prometheus_enabled:
        if not prometheus.is_prometheus_installed():
            prometheus.install_prometheus()
        cluster_topology = cluster_utils.get_cluster_topology(cluster_config)
        sg_url = cluster_topology["sync_gateways"][0]["public"]
        sg_ip = host_for_url(sg_url)
        prometheus.start_prometheus(sg_ip, sg_ssl, need_sgw_admin_auth)

    yield {"cluster_config": cluster_config,
           "mode": mode,
           "xattrs_enabled": xattrs_enabled,
           "sg_platform": sg_platform,
           "sync_gateway_version": sync_gateway_version,
           "disable_tls_server": disable_tls_server,
           "sg_ce": sg_ce,
           "prometheus_enabled": prometheus_enabled,
           "sg_ssl": sg_ssl,
           "cluster_topology": cluster_topology,
           "need_sgw_admin_auth": need_sgw_admin_auth,
           "disable_persistent_config": disable_persistent_config
           }

    log_info("Tearing down 'params_from_base_suite_setup' ...")

    # Stop all sync_gateway and sg_accels as test finished
    c = cluster.Cluster(cluster_config)
    c.stop_sg_and_accel()

    # Delete png files under resources/data
    clear_resources_pngs()
    if prometheus_enabled:
        prometheus.stop_prometheus(sg_ip, sg_ssl, need_sgw_admin_auth)

# This is called before each test and will yield the dictionary to each test that references the method
# as a parameter to the test method
@pytest.fixture(scope="function")
def params_from_base_test_setup(request, params_from_base_suite_setup):
    # Code before the yeild will execute before each test starts

    # pytest command line parameters
    collect_logs = request.config.getoption("--collect-logs")

    cluster_config = params_from_base_suite_setup["cluster_config"]
    mode = params_from_base_suite_setup["mode"]
    xattrs_enabled = params_from_base_suite_setup["xattrs_enabled"]
    sg_platform = params_from_base_suite_setup["sg_platform"]
    sync_gateway_version = params_from_base_suite_setup["sync_gateway_version"]
    disable_tls_server = params_from_base_suite_setup["disable_tls_server"]
    sg_ce = params_from_base_suite_setup["sg_ce"]
    prometheus_enabled = params_from_base_suite_setup["prometheus_enabled"]
    sg_ssl = params_from_base_suite_setup["sg_ssl"]
    cluster_topology = params_from_base_suite_setup["cluster_topology"]
    need_sgw_admin_auth = params_from_base_suite_setup["need_sgw_admin_auth"]
    disable_persistent_config = params_from_base_suite_setup["disable_persistent_config"]

    test_name = request.node.name
    log_info("Setting up test '{}'".format(test_name))

    # This dictionary is passed to each test
    yield {"cluster_config": cluster_config,
           "mode": mode,
           "xattrs_enabled": xattrs_enabled,
           "sg_platform": sg_platform,
           "sync_gateway_version": sync_gateway_version,
           "disable_tls_server": disable_tls_server,
           "sg_ce": sg_ce,
           "prometheus_enabled": prometheus_enabled,
           "sg_ssl": sg_ssl,
           "cluster_topology": cluster_topology,
           "need_sgw_admin_auth": need_sgw_admin_auth,
           "disable_persistent_config": disable_persistent_config
           }

    # Code after the yeild will execute when each test finishes
    log_info("Tearing down test '{}'".format(test_name))

    # Verify all sync_gateways and sg_accels are reachable
    c = cluster.Cluster(cluster_config)
    errors = c.verify_alive(mode)

    try:
        failures = request.node.rep_call.failed
    except Exception:
        failures = True

    # if the test failed pull logs
    if collect_logs or failures or len(errors) != 0:
        logging_helper = Logging()
        logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name)


"""
gruを用いたvae

vaeで時系列に見立てたMNISTを生成
"""
import math
import os
import numpy as np
from keras.layers import Input, InputLayer, Dense, RepeatVector, Lambda, TimeDistributed
from keras.layers import GRU
# from keras.layers import CuDNNGRU as GRU  # GPU用
from keras.models import Model, Sequential
from keras.callbacks import TensorBoard, EarlyStopping
from keras.optimizers import adam
from keras import backend as K
# データの準備

def prepare_data():
    """
    戻り値
      規格化された
      x_train, x_test
    """
    from keras.datasets import mnist
    (x_train, _), (x_test, _) = mnist.load_data()
    x_train, x_test = x_train/255, x_test/255
    # >>> x_train.shape, x_test.shape
    # ((60000, 28, 28), (10000, 28, 28))
    # これを(None, NUM_TIMESTEPS, NUM_INPUT_DIM)と対応させる
    global NUM_TIMESTEPS, NUM_INPUT_DIM
    _, NUM_TIMESTEPS, NUM_INPUT_DIM = x_train.shape
    return x_train, x_test

def seq_vae():
    """
    入力(input)
    ↓
    GRU(encoder)
    ↓
    内部状態
    ↓   ↓
    mean, log_var
    ↓
    zをサンプリング(ここまでencoder)
    ↓（このzを復元された内部状態だとして）
    GRU(decoder)
    ↓
    全結合層(出力)

    戻り値
     model
    """
    LATENT_DIM = 20

    def sampling(args):
        """
        z_mean, z_log_var=argsからzをサンプリングする関数
        戻り値
            z (tf.tensor):サンプリングされた潜在変数
        """
        z_mean, z_log_var = args
        batch = K.shape(z_mean)[0]
        dim = K.int_shape(z_mean)[1]
        # by default, random_normal has mean=0 and std=1.0
        epsilon = K.random_normal(shape=(batch, dim))
        # K.exp(0.5 * z_log_var)が分散に標準偏差になっている
        # いきなり標準偏差を求めてしまっても構わないが、負を許容してしまうのでこのようなトリックを用いている
        return z_mean + K.exp(0.5 * z_log_var) * epsilon

    # encoderの定義
    inputs = Input(shape=(NUM_TIMESTEPS, NUM_INPUT_DIM))
    # (None, NUM_TIMESTEPS,NUM_INPUT_DIM)
    x = GRU(LATENT_DIM)(inputs)
    # (None, LATENT_DIM)
    z_mean = Dense(LATENT_DIM, name='z_mean')(x)  # z_meanを出力
    # (None,LATENT_DIM)
    z_log_var = Dense(LATENT_DIM, name='z_log_var')(x)  # z_sigmaを出力
    # (None, LATENT_DIM)

    # use reparameterization trick to push the sampling out as input
    # note that "output_shape" isn't necessary with the TensorFlow backend
    z = Lambda(sampling, output_shape=(LATENT_DIM,), name='z')(
        [z_mean, z_log_var])  # 2つの変数を受け取ってランダムにサンプリング
    # (None, NATENT_DIM)
    encoder = Model(inputs, [z_mean, z_log_var, z], name="encoder")
    print("encoderの構成")
    encoder.summary()
    # encoder部分は入力を受けて平均、分散、そこからランダムサンプリングしたものの3つを返す

    # decoderの定義
    latent_inputs = RepeatVector(
        NUM_TIMESTEPS)(z)
    # (None, NUM_TIMESTEPS, LATENT_DIM)
    x = GRU(LATENT_DIM, return_sequences=True)(latent_inputs)
    # (None, NUM_TIMESTEPS, LATENT_DIM)
    outputs = TimeDistributed(
        Dense(NUM_INPUT_DIM, activation='sigmoid'))(x)
    # (None, NUM_TIMESTEPS, NUM_INPUT_DIM)

    # instantiate decoder model
    # decoder = Model(z, outputs, name='decoder')
    # print("decoderの構成")
    # decoder.summary()

    # デコーダーとエンコーダーの結合
    # encoderの出力の3つめ、つまりzを入力として、decoderを実行する
    vae = Model(inputs, outputs, name='seq_vae')

    # 損失関数をこのモデルに加える
    def loss(inputs, outputs):
        """
        損失関数の定義
        """
        from keras.losses import binary_crossentropy
        z_mean, z_log_var, _ = encoder(inputs)
        reconstruction_loss = binary_crossentropy(
            K.flatten(inputs), K.flatten(outputs))
        reconstruction_loss *= NUM_INPUT_DIM*NUM_TIMESTEPS
        kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
        kl_loss = K.sum(kl_loss, axis=-1)
        kl_loss *= -0.5
        return K.mean(reconstruction_loss + kl_loss)

    vae.add_loss(loss(inputs, outputs))
    print("seq_vaeの構成")
    vae.summary()
    return vae

if __name__ == "__main__":

    print("="*20+"preparating the data..."+"="*20)

    x_train, x_test = prepare_data()
    print("="*20+"summary of this model"+"="*20)
    seq_vae = seq_vae()

    seq_vae.compile(optimizer="adam")
    # 学習
    seq_vae.fit(x_train,
                epochs=100,
                batch_size=64,
                shuffle=True,
                validation_split=0.1,
                callbacks=[TensorBoard(log_dir="./MNIST_gru_vae/"), EarlyStopping(patience=3)])
    seq_vae.save('./MNIST_gru_vae/MNIST_gru_vae.h5')

    # 推論
    x_pred = seq_vae.predict(x_test)

    import matplotlib.pyplot as plt
    # データをプロット
    for true, pred in zip(x_test[::1000], x_pred[::1000]):
        plt.matshow(true)

        plt.matshow(pred)
        plt.show()

    encoder = K.function([seq_vae.input], [seq_vae.get_layer("z").output])
    decoder = K.function([seq_vae.get_layer("z").output],
                         [seq_vae.layers[-1].output])
    # 値の確認
    coded = encoder([x_test])[0]
    # zは毎回サンプリングされているので、そこそこ変動する
    print(np.max(coded[2]), np.min(coded[2]), np.mean(coded[2]))
    # ここから生成
    # seq_vae.layers[2].summary()
    gen = decoder([np.random.normal(0, 0.7, size=(20, 20))])[0]
    for g in gen:
        plt.matshow(g)
        plt.show()
"""
"""


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
import time
import json

chrome_driver_path = r"C:\Users\rv\Downloads\chromedriver_win32_88\chromedriver.exe"

def read_config_file():
    f = open(r'C:\Users\rv\Downloads\configData.txt')
    data_dic = json.load(f)
    return data_dic

data = read_config_file()
print(data['ext_id'])
print(data['urls'])

crx_extension_path = data['file_path'] + r'\jifpbeccnghkjeaalbbjmodiffmgedin.crx'

def download_crx():
    options = Options()
    options.headless = True
    options.add_extension(crx_extension_path)
    driver = webdriver.Chrome(executable_path=chrome_driver_path, options=options)
    # # to maximize the browser window
    driver.maximize_window()
    url = "chrome-extension://jifpbeccnghkjeaalbbjmodiffmgedin/crxviewer.html?crx=https%3A%2F%2Fclients2.google.com" \
          "%2Fservice%2Fupdate2%2Fcrx%3Fresponse%3Dredirect%26os%3Dwin%26arch%3Dx86-64%26os_arch%3Dx86-64%26nacl_arch" \
          "%3Dx86-64%26prod%3Dchromecrx%26prodchannel%3Dunknown%26prodversion%3D80.0.3987.116%26acceptformat%3Dcrx2" \
          "%2Ccrx3%26x%3Did%253D12345678%2526uc&zipname=12345678.zip "

    sourceUrl = url.replace('12345678', data['ext_id'])
    print(sourceUrl)
    driver.get(sourceUrl)
    time.sleep(10)

    if driver.find_element_by_id("download-link").is_displayed():
        driver.find_element_by_id("download-link").click()
        print(driver.find_element_by_id("download-link").get_attribute("href"))
        time.sleep(4)

    driver.close()

# upload extension and open the diff cat urls

def install_extn_open_cat_urls():
    options = Options()
    options.headless = True
    extension_path = data['file_path'] + '\\' + data['ext_id'] + ".zip"
    options.add_extension(extension_path)
    driver = webdriver.Chrome(executable_path=chrome_driver_path, options=options)
    # # to maximize the browser window
    driver.maximize_window()

    driver.get("https://www.google.com/")
    print(len(data['urls']))
    # Looping the script to open N new tabs

    for i in range(len(data['urls'])):
        driver.execute_script("window.open('','_blank');")
        driver.switch_to.window(driver.window_handles[-1])
        driver.get(data['urls'][i])
        time.sleep(5)
    driver.close()

# driver code
# download_crx()
install_extn_open_cat_urls()


from azure.common.credentials import ServicePrincipalCredentials
from azure.graphrbac import GraphRbacManagementClient
from lib.connector import UserConnector

AUTH_RESOURCE = "https://graph.windows.net"

class Connector(UserConnector):
    MappingName = 'Azureusers'
    Settings = {
        'tenant_id':        {'order': 1, 'example': '', 'default': ""},
        'client_id':        {'order': 2, 'example': "", 'default': ''},
        'secret':           {'order': 3, 'example': "", 'default': ''},
        'default_role':     {'order': 4, 'example': 25, 'type': int},
        'default_position': {'order': 5, 'example': 'Employee'},
    }

    FieldMappings = {
        'PERMISSIONS_ID': {'setting': "default_role"},
    }

    def _load_records(self, options):

        tenant_id = self.settings['tenant_id']
        client_id = self.settings['client_id']
        secret = self.settings['secret']

        credentials = ServicePrincipalCredentials(
            client_id=client_id,
            secret=secret,
            tenant=tenant_id,
            resource=AUTH_RESOURCE
        )

        graphrbac_client = GraphRbacManagementClient(
            credentials,
            tenant_id
        )

        self.logger.info('Getting user list Client ID [%s] Tenant ID [%s]',
            client_id, tenant_id)

        for user in graphrbac_client.users.list():
            yield user.__dict__


from __future__ import annotations

__all__ = ["BatchDict", "check_same_batch_size", "check_same_keys", "get_seq_lens"]

from collections.abc import (
    Hashable,
    ItemsView,
    Iterable,
    Iterator,
    KeysView,
    Sequence,
    ValuesView,
)
from typing import Any, TypeVar

import numpy as np
import torch
from coola import objects_are_allclose, objects_are_equal
from coola.utils.format import str_indent, str_mapping
from torch import Tensor

from redcat.base import BaseBatch

TBaseBatch = TypeVar("TBaseBatch", bound=BaseBatch)
# Workaround because Self is not available for python 3.9 and 3.10
# https://peps.python.org/pep-0673/
TBatchDict = TypeVar("TBatchDict", bound="BatchDict")

class BatchDict(BaseBatch[dict[Hashable, TBaseBatch]]):
    r"""Implements a batch object to represent a dictionary of batches.

    Args:
    ----
        data (dict): Specifies the dictionary of batches.

    Example usage:

    .. code-block:: pycon

        >>> import torch
        >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
        >>> batch = BatchDict(
        ...     {
        ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
        ...         "key2": BatchList(["a", "b"]),
        ...     }
        ... )
        >>> batch
        BatchDict(
          (key1): tensor([[0, 1, 2, 3, 4],
                    [5, 6, 7, 8, 9]], batch_dim=0, seq_dim=1)
          (key2): BatchList(data=['a', 'b'])
        )
    """

    def __init__(self, data: dict[Hashable, TBaseBatch]) -> None:
        if not isinstance(data, dict):
            raise TypeError(f"Incorrect type. Expect a dict but received {type(data)}")
        check_same_batch_size(data)
        self._data = data

    def __repr__(self) -> str:
        data_str = str_indent(str_mapping({key: repr(value) for key, value in self._data.items()}))
        return f"{self.__class__.__qualname__}(\n  {data_str}\n)"

    @property
    def batch_size(self) -> int:
        return next(iter(self._data.values())).batch_size

    @property
    def data(self) -> dict[Hashable, TBaseBatch]:
        return self._data

    #################################
    #     Dictionary operations     #
    #################################

    def __contains__(self, key: Hashable) -> bool:
        return key in self._data

    def __getitem__(self, key: Hashable) -> TBaseBatch:
        return self._data[key]

    def __iter__(self) -> Iterator[Hashable]:
        return iter(self._data)

    def __len__(self) -> int:
        return len(self._data)

    def __setitem__(self, key: Hashable, value: TBaseBatch) -> None:
        if value.batch_size != self.batch_size:
            raise RuntimeError(
                f"Incorrect batch size. Expected {self.batch_size} but received {value.batch_size}"
            )
        self._data[key] = value

    def get(self, key: Hashable, default: TBaseBatch | None = None) -> TBaseBatch | None:
        return self._data.get(key, default)

    def items(self) -> ItemsView:
        return self._data.items()

    def keys(self) -> KeysView:
        return self._data.keys()

    def values(self) -> ValuesView:
        return self._data.values()

    #################################
    #     Conversion operations     #
    #################################

    def to_data(self) -> dict:
        return {key: value.to_data() for key, value in self._data.items()}

    ###############################
    #     Creation operations     #
    ###############################

    def clone(self) -> TBatchDict:
        return self.__class__({key: batch.clone() for key, batch in self._data.items()})

    #################################
    #     Comparison operations     #
    #################################

    def allclose(
        self, other: Any, rtol: float = 1e-5, atol: float = 1e-8, equal_nan: bool = False
    ) -> bool:
        if not isinstance(other, self.__class__):
            return False
        return objects_are_allclose(
            self.data, other.data, rtol=rtol, atol=atol, equal_nan=equal_nan
        )

    def equal(self, other: Any) -> bool:
        if not isinstance(other, self.__class__):
            return False
        return objects_are_equal(self.data, other.data)

    ###########################################################
    #     Mathematical | advanced arithmetical operations     #
    ###########################################################

    def permute_along_batch(self, permutation: Sequence[int] | Tensor) -> TBatchDict:
        return self.__class__(
            {k: v.permute_along_batch(permutation) for k, v in self._data.items()}
        )

    def permute_along_batch_(self, permutation: Sequence[int] | Tensor) -> None:
        for value in self._data.values():
            value.permute_along_batch_(permutation)

    def permute_along_seq(self, permutation: Sequence[int] | Tensor) -> TBatchDict:
        r"""Permutes the data along the sequence dimension.

        The same permutation is applied on all the sequences. This
        method should be called only if all the sequences have the
        same length.

        This method only permutes the values that implement
        ``permute_along_seq``.

        Args:
        ----
            permutation (sequence or ``torch.Tensor`` of type long
                and shape ``(dimension,)``): Specifies the permutation
                to use on the data. The dimension of the permutation
                input should be compatible with the shape of the data.

        Returns:
        -------
            ``BatchDict``: A new batch with permuted data.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.permute_along_seq([2, 1, 3, 0, 4])
            BatchDict(
              (key1): tensor([[2, 1, 3, 0, 4],
                             [7, 6, 8, 5, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        out = {}
        for key, val in self._data.items():
            if hasattr(val, "permute_along_seq"):
                val = val.permute_along_seq(permutation)
            out[key] = val
        return self.__class__(out)

    def permute_along_seq_(self, permutation: Sequence[int] | Tensor) -> None:
        r"""Permutes the data along the sequence dimension.

        The same permutation is applied on all the sequences. This
        method should be called only if all the sequences have the
        same length.

        This method only permutes the values that implement
        ``permute_along_seq``.

        Args:
        ----
            permutation (sequence or ``torch.Tensor`` of type long
                and shape ``(dimension,)``): Specifies the permutation
                to use on the data. The dimension of the permutation
                input should be compatible with the shape of the data.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.permute_along_seq_([2, 1, 3, 0, 4])
            >>> batch
            BatchDict(
              (key1): tensor([[2, 1, 3, 0, 4],
                             [7, 6, 8, 5, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        for val in self._data.values():
            if hasattr(val, "permute_along_seq_"):
                val.permute_along_seq_(permutation)

    def shuffle_along_seq(self, generator: torch.Generator | None = None) -> TBatchDict:
        r"""Shuffles the data along the sequence dimension.

        This method should be called only if all the sequences have
        the same length.

        Args:
        ----
            generator (``torch.Generator`` or ``None``, optional):
                Specifies an optional random generator.
                Default: ``None``

        Returns:
        -------
            ``BatchDict``:  A new batch with shuffled data.

        Raises:
            RuntimeError if the batch has multiple sequence lengths.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.shuffle_along_seq()
            BatchDict(
              (key1): tensor([[...]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        seq_lens = get_seq_lens(self._data)
        if not seq_lens:
            return self
        if len(seq_lens) > 1:
            raise RuntimeError(
                f"Invalid operation because the batch has multiple sequence lengths: {seq_lens}"
            )
        return self.permute_along_seq(torch.randperm(seq_lens.pop(), generator=generator))

    def shuffle_along_seq_(self, generator: torch.Generator | None = None) -> None:
        r"""Shuffles the data along the sequence dimension.

        This method should be called only if all the sequences have
        the same length.

        Args:
        ----
            generator (``torch.Generator`` or ``None``, optional):
                Specifies an optional random generator.
                Default: ``None``

        Raises:
        ------
            RuntimeError if the batch has multiple sequence lengths.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.shuffle_along_seq()
            >>> batch
            BatchDict(
              (key1): tensor([[...]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        seq_lens = get_seq_lens(self._data)
        if not seq_lens:
            return
        if len(seq_lens) > 1:
            raise RuntimeError(
                f"Invalid operation because the batch has multiple sequence lengths: {seq_lens}"
            )
        self.permute_along_seq_(torch.randperm(seq_lens.pop(), generator=generator))

    ################################################
    #     Mathematical | point-wise operations     #
    ################################################

    ###########################################
    #     Mathematical | trigo operations     #
    ###########################################

    ##########################################################
    #    Indexing, slicing, joining, mutating operations     #
    ##########################################################

    def append(self, other: TBaseBatch) -> None:
        check_same_keys(self.data, other.data)
        for key, value in self._data.items():
            value.append(other[key])

    def cat_along_seq(self, batches: TBaseBatch | Sequence[TBaseBatch]) -> TBatchDict:
        r"""Concatenates the data of the batch(es) to the current batch
        along the sequence dimension and creates a new batch.

        Note that only the sequences are concatenated.

        Args:
        ----
            batches (``BatchDict`` or  ``Sequence``): Specifies the
                batch(es) to concatenate along the sequence dimension.

        Returns:
        -------
            ``BatchDict``: A batch with the concatenated data
                along the sequence dimension.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> b = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> b.cat_along_seq(
            ...     BatchDict({"key1": BatchedTensorSeq(torch.tensor([[10, 11, 12], [20, 21, 22]]))})
            ... )
            BatchDict(
              (key1): tensor([[ 0,  1,  2,  3,  4, 10, 11, 12],
                        [ 5,  6,  7,  8,  9, 20, 21, 22]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        if isinstance(batches, BatchDict):
            batches = [batches]
        out = {}
        for key, val in self._data.items():
            if hasattr(val, "cat_along_seq"):
                val = val.cat_along_seq([batch[key] for batch in batches])
            out[key] = val
        return self.__class__(out)

    def cat_along_seq_(self, batches: TBaseBatch | Sequence[TBaseBatch]) -> None:
        r"""Concatenates the data of the batch(es) to the current batch
        along the sequence dimension and creates a new batch.

        Note that only the sequences are concatenated.

        Args:
        ----
            batches (``BatchDict`` or  ``Sequence``): Specifies the
                batch(es) to concatenate along the sequence dimension.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> b = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> b.cat_along_seq_(
            ...     BatchDict({"key1": BatchedTensorSeq(torch.tensor([[10, 11, 12], [20, 21, 22]]))})
            ... )
            >>> b
            BatchDict(
              (key1): tensor([[ 0,  1,  2,  3,  4, 10, 11, 12],
                        [ 5,  6,  7,  8,  9, 20, 21, 22]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        if isinstance(batches, BatchDict):
            batches = [batches]
        for key, val in self._data.items():
            if hasattr(val, "cat_along_seq_"):
                val.cat_along_seq_([batch[key] for batch in batches])

    def chunk_along_batch(self, chunks: int) -> tuple[TBatchDict, ...]:
        keys = self._data.keys()
        batches = []
        for values in zip(*[batch.chunk_along_batch(chunks) for batch in self._data.values()]):
            batches.append(self.__class__({key: value for key, value in zip(keys, values)}))
        return tuple(batches)

    def extend(self, other: Iterable[BatchDict | Sequence[TBatchDict]]) -> None:
        for batch in other:
            self.append(batch)

    def index_select_along_batch(self, index: Tensor | Sequence[int]) -> TBatchDict:
        return self.__class__(
            {key: value.index_select_along_batch(index) for key, value in self._data.items()}
        )

    def index_select_along_seq(self, index: Tensor | Sequence[int]) -> TBatchDict:
        r"""Slices the batch along the sequence dimension at the given
        indices.

        Args:
        ----
            index (``torch.Tensor`` of type long and shape
                ``(seq_len,)`` or ``(batch_size, seq_len,)`` or
                ``Sequence``): Specifies the indices to select.

        Returns:
        -------
            ``BatchDict``: A new batch sliced along the sequence
                dimension at the given indices.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.index_select_along_seq([2, 4])
            BatchDict(
              (key1): tensor([[2, 4], [7, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
            >>> batch.index_select_along_seq(torch.tensor([2, 4]))
            BatchDict(
              (key1): tensor([[2, 4], [7, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
            >>> batch.index_select_along_seq(torch.tensor([[2, 4], [4, 3]]))
            BatchDict(
              (key1): tensor([[2, 4], [9, 8]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        out = {}
        for key, val in self._data.items():
            if hasattr(val, "index_select_along_seq"):
                val = val.index_select_along_seq(index)
            out[key] = val
        return self.__class__(out)

    def repeat_along_seq(self, repeats: int) -> TBatchDict:
        r"""Repeats the batch along the sequence dimension.

        Args:
        ----
            repeats (int): Specifies the number of times to repeat
                the batch along the sequence dimension.

        Returns:
        -------
            ``BatchDict``: A repeated version of the input batch.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.repeat_along_seq(2)
            BatchDict(
              (key1): tensor([[0, 1, 2, 3, 4, 0, 1, 2, 3, 4],
                        [5, 6, 7, 8, 9, 5, 6, 7, 8, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        out = {}
        for key, val in self._data.items():
            if hasattr(val, "repeat_along_seq"):
                val = val.repeat_along_seq(repeats)
            out[key] = val
        return self.__class__(out)

    def select_along_batch(self, index: int) -> dict:
        return {key: value.select_along_batch(index) for key, value in self._data.items()}

    def slice_along_batch(
        self, start: int = 0, stop: int | None = None, step: int = 1
    ) -> TBatchDict:
        return self.__class__(
            {key: value.slice_along_batch(start, stop, step) for key, value in self._data.items()}
        )

    def slice_along_seq(self, start: int = 0, stop: int | None = None, step: int = 1) -> TBatchDict:
        r"""Slices the batch in the sequence dimension.

        Args:
        ----
            start (int, optional): Specifies the index where the
                slicing of object starts. Default: ``0``
            stop (int, optional): Specifies the index where the
                slicing of object stops. ``None`` means last.
                Default: ``None``
            step (int, optional): Specifies the increment between
                each index for slicing. Default: ``1``

        Returns:
        -------
            ``BatchedTensorSeq``: A slice of the current batch.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.slice_along_seq(start=2)
            BatchDict(
              (key1): tensor([[2, 3, 4],
                        [7, 8, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
            >>> batch.slice_along_seq(stop=3)
            BatchDict(
              (key1): tensor([[0, 1, 2],
                        [5, 6, 7]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
            >>> batch.slice_along_seq(step=2)
            BatchDict(
              (key1): tensor([[0, 2, 4],
                        [5, 7, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        out = {}
        for key, val in self._data.items():
            if hasattr(val, "slice_along_seq"):
                val = val.slice_along_seq(start, stop, step)
            out[key] = val
        return self.__class__(out)

    def split_along_batch(
        self, split_size_or_sections: int | Sequence[int]
    ) -> tuple[TBatchDict, ...]:
        keys = self._data.keys()
        batches = []
        for values in zip(
            *[batch.split_along_batch(split_size_or_sections) for batch in self._data.values()]
        ):
            batches.append(self.__class__({key: value for key, value in zip(keys, values)}))
        return tuple(batches)

    def take_along_seq(self, indices: TBaseBatch | np.ndarray | Tensor | Sequence) -> TBatchDict:
        r"""Takes values along the sequence dimension.

        Args:
        ----
            indices (``BaseBatch`` or ``numpy.ndarray`` or
                ``torch.Tensor`` or `` Specifies the indices to take
                along the batch dimension.

        Returns:
        -------
            ``BatchDict``: The batch with the selected data.

        Example usage:

        .. code-block:: pycon

            >>> import torch
            >>> from redcat import BatchDict, BatchList, BatchedTensorSeq
            >>> batch = BatchDict(
            ...     {
            ...         "key1": BatchedTensorSeq(torch.arange(10).view(2, 5)),
            ...         "key2": BatchList(["a", "b"]),
            ...     }
            ... )
            >>> batch.take_along_seq(torch.tensor([[3, 0, 1], [2, 3, 4]]))
            BatchDict(
              (key1): tensor([[3, 0, 1],
                        [7, 8, 9]], batch_dim=0, seq_dim=1)
              (key2): BatchList(data=['a', 'b'])
            )
        """
        out = {}
        for key, val in self._data.items():
            if hasattr(val, "take_along_seq"):
                val = val.take_along_seq(indices)
            out[key] = val
        return self.__class__(out)

    ########################
    #     mini-batches     #
    ########################

    #################
    #     Other     #
    #################

    def summary(self) -> str:
        data_str = str_mapping({key: value.summary() for key, value in self._data.items()})
        return f"{self.__class__.__qualname__}(\n  {str_indent(data_str)}\n)"

def check_same_batch_size(data: dict[Hashable, BaseBatch]) -> None:
    r"""Checks if the all the batches in a group have the same batch
    size.

    Args:
    ----
        group (``BaseBatch`` or dict or sequence): Specifies the group
            of batches to check.

    Raises:
    ------
        RuntimeError if there are several batch sizes.

    Example usage:

    .. code-block:: pycon

        >>> from redcat import BatchedTensor, BatchedTensorSeq
        >>> from redcat.batchdict import check_same_batch_size
        >>> check_same_batch_size(
        ...     {
        ...         "key1": BatchedTensorSeq(torch.ones(2, 3)),
        ...         "key2": BatchedTensor(torch.ones(2, 6)),
        ...     }
        ... )
    """
    if not data:
        raise RuntimeError("The dictionary cannot be empty")
    batch_sizes = {val.batch_size for val in data.values()}
    if len(batch_sizes) != 1:
        raise RuntimeError(
            "Incorrect batch size. A single batch size is expected but received several values: "
            f"{batch_sizes}"
        )

def check_same_keys(data1: dict, data2: dict) -> None:
    r"""Checks if the dictionaries have the same keys.

    Args:
    ----
        data1 (dict): Specifies the first dictionary.
        data2 (dict): Specifies the second dictionary.

    Raises:
    ------
        RuntimeError if the keys are different.

    Example usage:

    .. code-block:: pycon

        >>> from redcat import BatchedTensor, BatchedTensorSeq
        >>> from redcat.batchdict import check_same_keys
        >>> check_same_keys(
        ...     {
        ...         "key1": BatchedTensorSeq(torch.ones(2, 3)),
        ...         "key2": BatchedTensor(torch.ones(2, 6)),
        ...     },
        ...     {
        ...         "key1": BatchedTensorSeq(torch.zeros(2, 4)),
        ...         "key2": BatchedTensor(torch.zeros(2, 4)),
        ...     },
        ... )
    """
    keys1 = set(data1.keys())
    keys2 = set(data2.keys())
    if keys1 != keys2:
        raise RuntimeError(f"Keys do not match: ({keys1} vs {keys2})")

def get_seq_lens(data: dict[Hashable, BaseBatch]) -> set[int]:
    r"""Gets the sequence lengths from the inputs.

    Args:
    ----
        data (dict): Specifies the data with the sequences.

    Returns:
    -------
        set: The sequence lengths.

    Example usage:

    .. code-block:: pycon

        >>> from redcat import BatchedTensor, BatchedTensorSeq
        >>> from redcat.batchdict import get_seq_lens
        >>> get_seq_lens(
        ...     {
        ...         "key1": BatchedTensorSeq(torch.ones(2, 3)),
        ...         "key2": BatchedTensor(torch.ones(2, 6)),
        ...     }
        ... )
        {3}
    """
    return {val.seq_len for val in data.values() if hasattr(val, "seq_len")}


from django.db import models
from main.models import User

class UserProfile(models.Model):
    user = models.OneToOneField(User, on_delete = models.CASCADE)
    bio = models.TextField(default="")
    post_amount = models.IntegerField(default=0)
    upvote_amount = models.IntegerField(default=0)

class DoctorProfile(models.Model):
    profile = models.OneToOneField(UserProfile, on_delete = models.CASCADE)
    comment_amount = models.IntegerField(default=0)
    rating_average = models.FloatField(default=0)

class Rating(models.Model):
    doctor = models.ForeignKey(DoctorProfile, on_delete = models.CASCADE)
    author = models.ForeignKey(User, on_delete = models.CASCADE)
    rating = models.IntegerField(default=0)
    date = models.DateTimeField(auto_now_add=True)
    comment = models.TextField(max_length=150)


import pygame
import math
import random

class filmgrain(object):
    """U-G-L-Y you aint got no alibi, you ugly, yeah yeah, you ugly!"""

    def __init__(self, screen_width, screen_height, num_scratches, min_length, max_length):
        self.screen_width = screen_width
        self.screen_height = screen_height
        self.min_length = min_length
        self.max_length = max_length
        self.num_scratches = num_scratches
        self.color = (0,0,0,127)
        self.framecount = 0
        self.start_point = None

    def update(self, dt):
        if self.framecount < 3:
            self.framecount += 1
        else:
            self.framecount = 0
            self.start_point = None

            if random.randint(0,10) == 1:
                rot = random.random()*6.28
                len = random.randint(self.min_length, self.max_length)

                x1 = random.randint(0, self.screen_width)
                y1 = random.randint(0, self.screen_width)
                x2 = math.cos(rot) * len
                y2 = math.sin(rot) * len

                self.start_point = (x1, y1)
                self.end_point = (x1-x2, y1-y2)

    def draw(self, screen):
        if self.start_point != None:
            pygame.draw.line(screen, self.color, self.start_point, self.end_point, 2)


import csv
import time
import operator
from collections import defaultdict, namedtuple
from operator import itemgetter
import matplotlib.pyplot as plt
import numpy as np
from collections import defaultdict
from datetime import datetime
import time

class PlotSpi:
	def __init__(self, logfile):
		self.logfilepath = "logs/" + logfile
	def readCSV(self):
		csv_file = open(self.logfilepath, mode='r')
		data = csv.DictReader(csv_file)
		return data
	def parselog(self, logtype):
		logdata = self.readCSV()
		num_log = len(logtype)
		log_dict = {}
		val_array = []
		for x in range(num_log):
			val_array.append([])
			log_dict[logtype[x]]=x
		flag=1
		for row in logdata:
			if flag:
				initial_time = time.mktime(datetime.strptime(row["timestamp"], "%Y-%m-%d %H:%M:%S").timetuple())
				flag=0
			val_array[log_dict[row["logid"]]].append((time.mktime(datetime.strptime(row["timestamp"], "%Y-%m-%d %H:%M:%S").timetuple()) - initial_time, row["timetaken"]))
		for x in range(num_log):
			plt.figure(x)
			x_val = [p[0] for p in val_array[x]]
			y_val = [float(p[1]) for p in val_array[x]]
			plt.plot(x_val, y_val)
			plt.title(logtype[x])
			plt.xlabel("simulation time")
			plt.ylabel(logtype[x]+"in milliseconds")
			plt.savefig(logtype[x]+"performance")

			plt.figure(x+num_log)
			plt.boxplot(x_val,y_val)
			plt.xlabel("Number of samples")
			plt.ylabel(logtype[x]+"in milliseconds")
			plt.savefig(logtype[x]+"performance_box", bbox_inches='tight')

def main():
	logfile = "./log.csv"
	inst = PlotSpi(logfile)
	inst_data = inst.parselog(["REGISTER", "SIGN_GS", "SIGN_HY", "VERIFYGM"])

main()


# DECISION TREE REGRESSION

# Importing liberaries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Importing dataset
dataset = pd.read_csv("Position_Salaries.csv")

# Seperating dependent and independent variables
x = dataset.iloc[:,1:2].values
y = dataset.iloc[:,2].values

# We dont need to split dataset in DECISION TREE REGRESSION
# Splitting the dataset into Training set and Test set
'''
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size = 0.2,random_state = 0)
'''

# We dont need apply feature scaling in DECISION TREE REGRESSION
# Feature Scaling -> Normalizing the range of data/vairiable values
'''
from sklearn.preprocessing import StandardScaler
sc_x = StandardScaler()
x_train = sc_x.fit_transform(x_train)
x_test = sc_x.transform(x_test)
# We can also rescale y if we need
sc_y = StandardScaler()
y_train = sc_y.fit_transform(y_train)
y_test = sc_y.transform(y_test)
'''

# Fitting DECISION TREE REGRESSION model to the dataset
from sklearn.tree import DecisionTreeRegressor
# We pass "random_state" value so that our result and the instructor result in video is same.
regressor = DecisionTreeRegressor(random_state = 0)
regressor.fit(x,y)

# Predicting a new result with DECISION TREE REGRESSION
y_pred = regressor.predict([[6.5]])

# This graph wouldn't show the correct graph because it can show non-linear and non contineous graph hence we used high resolution graph to plot
# Visualising the DECISION TREE REGRESSION results
'''
plt.scatter(x, y, color = 'red')
plt.plot(x, regressor.predict(x), color = 'blue')
plt.title('Truth or Bluff (DECISION TREE REGRESSION)')
plt.xlabel('Position level')
plt.ylabel('Salary')
plt.show()
'''

# Visualising the DECISION TREE REGRESSION results (for higher resolution and smoother curve)
# If we want the graph to be more accurate by making inputs complicated.
# It gives a much smoother curve.
x_grid = np.arange(min(x), max(x), 0.01)
x_grid = x_grid.reshape((len(x_grid), 1))
plt.scatter(x, y, color = 'red')
plt.plot(x_grid, regressor.predict(x_grid), color = 'blue')
plt.title('Truth or Bluff (DECISION TREE REGRESSION)')
plt.xlabel('Position level')
plt.ylabel('Salary')
plt.show()


import re
from pyspark.sql.functions import  udf
from pyspark.sql.types import StringType
from pyspark.sql.functions import monotonically_increasing_id
from pyspark.conf import SparkConf
from pyspark.sql import SparkSession
import pysolr
import json

#regex fonctions :
def cin(text):
    result = re.search(r'(?<=identité N°.)[A-Z][A-Z0-9][0-9]{5}', text)
    if result!=None:
        return result.group()
    else:
        print('None')

def no_compte(text):
    result =  re.search(r'[0-9]{4}[A-Z][0-9]{9}', text)
    if result!=None:
        return result.group()
    else:
        print('None')

def date_naissance(text) :
    result = re.search(r'\d{2}/\d{2}/\d{4}', text)
    if result!=None:
        return result.group()
    else:
        print('None')

def tel(text) :
    result = re.search(r'06[0-9]{8}', text)
    if result!=None:
        return result.group()
    else:
        print('None')

def name(text):
   tel = re.search('(?<=assurer :)(.*[A-Z\s]?)(?=Pièce)', text)
   if(tel==None):print('None')
   else :return re.match('(|[A-Z\s]+)(.|\s+)([A-Z\s]+)', tel.group()).group()

def File_path(text):
    return text.replace('file:/','').replace('output','input').replace('txt','pdf')

#------------------------------------------------------------------------------------------------------------#
if __name__ == "__main__":
    spark = SparkSession.builder.master("local") \
                        .appName('Spark_regex') \
                        .config(conf=SparkConf()).getOrCreate()

    rdd = spark.sparkContext.wholeTextFiles("C:/Users/asus/OneDrive/Documents/text_images/resources/output")
    list = spark.createDataFrame(rdd).collect()
    df = spark.createDataFrame(list,["file_path", "content"])

    #udfs functions :----------------------------------------------------------------------------------------#
    cin_udf = udf(cin, StringType())
    date_naissance_udf = udf(date_naissance,StringType())
    tel_udf = udf(tel ,StringType())
    name_udf = udf(name,StringType())
    no_compte_udf = udf(no_compte,StringType())
    file_path_udf = udf(File_path,StringType())
#------------------------------------------------------------------------------------------------------------#
    df1 = df.withColumn("No_Compte",no_compte_udf(df.content))
    df2 = df1.withColumn("Full_name",name_udf(df.content))
    df3 = df2.withColumn("CIN", cin_udf(df.content))
    df4 = df3.withColumn("Date_naissance",date_naissance_udf(df.content))
    df5 = df4.withColumn("Telephone",tel_udf(df.content))
    df6 = df5.withColumn("file_path",file_path_udf(df.file_path))
    list2 = df6.drop('content').collect()
    df7 = spark.createDataFrame(list2)
    df8 = df7.withColumn("id", monotonically_increasing_id())
    df8.show(truncate=False)
#--------------------------------------------------------------------------------------------------------------#
"""
#Solr integration :
results = df8.toJSON().map(lambda j: json.loads(j)).collect()
solr = pysolr.Solr('http://127.0.0.1:8983/solr/AWB2',  search_handler='/autocomplete')
#solr.delete(q='*:*')
solr.add(results)
solr.commit()
print("Send to solr successfully :) ")"""


# reads in csv and outputs the complete-data
# Requires the RWA spreadsheet be condensed into the columns which the application will use
# export the sheet as csv, and run through this script
# make sure the encoding on the CSV is utf-8 on export!

# TODO:: put into an array and iterate through them
# Columns:
# id_modified,format-catalogue,transcript,location_shelf,public_notes,1935_catalogue,title,author,date_published,no_of_vols,place_published,publisher,internet_link,Gale_Document_Number,ESTC,EEBO_BibNo,edition,1908_catalogue,folder_name,images,locationOfBook,permissions,missingvolnumber,spine

import os
import csv
from pathlib import Path

catalogue_file = "catalogue.csv"
output_filename = "complete-data.js"
output_path = Path(__file__).parent.absolute() / 'production-files' / output_filename

catalogue_arr = []

with open(catalogue_file, "r", encoding="utf-8") as file:
    # format csv data to dictionary
    file_reader = csv.DictReader(file)
    for row in file_reader:
        catalogue_arr.append(row)

def getBoolFromProperty(prop):
    affirmative = ['y', 'yes']
    return "true" if prop.lower() in affirmative else False

def createCompleteDataJSfromArr(arr, id_as_key = False):
    with open(output_path, mode="w", encoding="utf-8") as file:

        js_variable = "var COMPLETE_DATA ="

        # makes ID key of json
        if id_as_key:
            js_variable += "{"
        else:
            js_variable += "["

        file.write(js_variable)

        # compile book str to write to file
        for bk in arr:
            if(id_as_key):
                # use bookId as key, values are book attributes
                book = "\"" + str(bk["id"]) + "\":"
                book += "{"
            else:
                book = "[{"
                if bk["id"]: book += "\"id\":\"{}\",".format(bk["id"])

            if bk["title"]: book += "\"title\":\"{}\",".format(bk["title"])
            if bk["author"]: book += "\"author\":\"{}\",".format(bk["author"])
            if bk["pub_date"]: book += "\"pub_date\":\"{}\",".format(bk["pub_date"])
            if bk["publisher"]: book += "\"publisher\":\"{}\",".format(bk["publisher"])
            if bk["pub_loc"]: book += "\"pub_loc\":\"{}\",".format(bk["pub_loc"])
            if bk["edition"]: book += "\"edition\":\"{}\",".format(bk["edition"])
            if bk["shelf"]: book += "\"shelf\":\"{}\",".format(bk["shelf"])
            if bk["transcript"]: book += "\"transcript\":\"{}\",".format(bk["transcript"])
            if bk["format"]: book += "\"format\":\"{}\",".format(bk["format"])
            if bk["volumes"]: book += "\"volumes\":\"{}\",".format(bk["volumes"])
            if bk["missing"]: book += "\"missing\":\"{}\",".format(bk["missing"])
            if bk["current_loc"]: book += "\"current_loc\":\"{}\",".format(bk["current_loc"])

            #if bk["in_1908"]: book += "\"in_1908\":\"{}\",".format(bk["in_1908"])
            if bk["in_1908"]:
                temp = getBoolFromProperty(bk["in_1908"])
                if temp:
                    book += "\"in_1908\":{},".format(temp)

            #if bk["in_1935"]: book += "\"in_1935\":\"{}\",".format(bk["in_1935"])
            if bk["in_1935"]:
                temp = getBoolFromProperty(bk["in_1935"])
                if temp:
                    book += "\"in_1935\":{},".format(temp)

            if bk["notes"]: book += "\"notes\":\"{}\",".format(bk["notes"])
            if bk["ESTC"]: book += "\"ESTC\":\"{}\",".format(bk["ESTC"])
            if bk["EEBO"]: book += "\"EEBO\":\"{}\",".format(bk["EEBO"])
            if bk["Gale"]: book += "\"Gale\":\"{}\",".format(bk["Gale"])
            if bk["link"]: book += "\"link\":\"{}\",".format(bk["link"])
            if bk["images"]: book += "\"images\":\"{}\",".format(bk["images"])
            if bk["permissions"]: book += "\"permissions\":\"{}\",".format(bk["permissions"])

            if bk["folder"]: book += "\"folder\":\"{}\",".format(bk["folder"])
            if bk["spine"]:
                temp = getBoolFromProperty(bk["spine"])
                if temp:
                    book += "\"spine\":\"{}\",".format(temp)

            # remove extra comma
            book = book[:-1]

            book += "},"

            file.write(book)

        if(id_as_key):
            file.write("};")
        else:
            file.write("];")

# write arr to file
createCompleteDataJSfromArr(catalogue_arr, True)
print("Finished: production-files/complete-data.js")


from bs4 import BeautifulSoup
from time import sleep
import time
import requests
from random import randint
from html.parser import HTMLParser
import json

USER_AGENT = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'}
searching_url='https://html.duckduckgo.com/html/?q='
class SearchEngine:
	@staticmethod
	def search(query, sleep=True):
		if sleep: # Prevents loading too many pages too soon
			sleeptime = randint(10, 100)
			print("sleep for "+ str(sleeptime) +" seconds.")
			time.sleep(sleeptime)
		temp_url = '+'.join(query.split()) #for adding + between words for the query
		url = searching_url + temp_url
		#print(url)
		soup = BeautifulSoup(requests.get(url, headers=USER_AGENT).text,"html.parser")
		#print(soup)
		new_results = SearchEngine.scrape_search_result(soup)
		return new_results
	@staticmethod
	def scrape_search_result(soup):
		raw_results = soup.find_all("a", attrs = {"class" : "result__a"})
		results = []
		#implement a check to get only 10 results and also check that URLs must not be duplicated
		for result in raw_results:
			link = result.get('href')
			results.append(link)
		return results

def writeJson(dictionary):
  # Serializing json
	json_object = json.dumps(dictionary, indent=4)
	# Writing to sample.json
	with open("hw1.json", "w") as outfile:
		outfile.write(json_object)
def readJson():
  with open('hw1.json', 'r') as openfile:
    json_object = json.load(openfile)
    return json_object

resJSON={}
queries = []
with open('100QueriesSet4.txt') as f:
	while(True):
		line = f.readline()
		if(line):
			line = line[:-2]
			queries.append(line)
		else:
			break
f.close()
queries0 = queries[0:1]
queries1 = queries[0:25]
queries2 = queries[25:50]
queries3 = queries[50:75]
queries4 = queries[75:100]
for queryStr in queries4:
  resJSON=readJson()
  result = SearchEngine.search(queryStr,sleep=True)
  result= list(dict.fromkeys(result))
  print(queryStr)
  print(len(result))
  resJSON[queryStr]=result[:10]
  #print(resJSON)
  writeJson(resJSON)


# -*- coding: utf-8 -*-
__author__ = 'vahid'
import tg.i18n
from tg import response
from datetime import timedelta

def set_lang(lang):
    tg.i18n.set_lang(lang)
    response.set_cookie('lang',
                        lang,
                        max_age=timedelta(days=500))

def get_lang():
    languages = tg.i18n.get_lang(all=False)
    if languages:
        return languages[0][:2]
    else:
        return None


import logging

from auth.bkiam import constant
from auth.models import ResourceScope

logger = logging.getLogger(__name__)

class Convert:
    @classmethod
    def parse_iam_path_expression(cls, value):
        """
        解析权限中心拓扑授权表达式，处理成本地表示的路径
        :param value: "/project,12511/"
        :return: project&12511
        """
        # 解析路径，拼接成 resource&id的形式
        data = value.replace("/", "").split(",")
        return cls.to_local_path(data[0], data[1])

    @classmethod
    def to_local_path(cls, resource_type, object_id):
        """
        拼接本地路径
        :param resource_type: 资源类型
        :param separator: 分隔符
        :param object_id: 资源 id
        :return: example: project&1
        """
        return "{}{}{}".format(resource_type, constant.SEPARATOR, object_id)

    @classmethod
    def get_user_action_key(cls, user_id, action_id):
        """
        拼接权限中心用户的key
        :param user_id: 用户id
        :param action_id: 操作id
        :return: user_id$action_id
        """
        return "{}{}{}".format(user_id, constant.KEY_SEPARATOR, action_id)

    @classmethod
    def get_object_type(cls, object_id):
        """
        判断资源授权类型
        """
        if "#" in object_id:
            return constant.COMPLEX_PATH
        elif constant.SEPARATOR in object_id:
            return constant.SIMPLE_PATH
        else:
            return constant.INSTANCE

    @classmethod
    def build_resource_scopes(cls, resources):
        """
        构造list [auth.models.ResourceScope,auth.models.ResourceScope]
        """
        if not isinstance(resources, list):
            # 如果返回的是单个资源，转换成列表
            resources = [resources]
        resource_scopes = []
        for resource in resources:
            resource_scopes.append(
                ResourceScope(id=resource, type=cls.get_object_type(resource))
            )
        return resource_scopes

    @classmethod
    def to_local_action(self, iam_action_id):
        """
        将权限中心格式的action 转换成本地格式的 action
        :param iam_action_id: 权限中心权限操作id： project-delete
        :return: 本地 project.delete
        """
        return iam_action_id.replace("-", ".")