import nltk
import numpy as np
import pandas as pd
import json
import glob

import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
from gensim.models import TfidfModel

import spacy
import en_core_web_sm
from nltk.corpus import stopwords

import pyLDAvis
import pyLDAvis.gensim_models

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=UserWarning)

data = pd.read_csv("data:/Launch NLP.csv")["Description"]
# print(data)
data_1 = []
for value in data:
    data_1 += [value]
data = data_1
# print(data)

def lemmatization(texts, allowed_postags=["NOUN", "ADJ", "VERB", "ADV"]):
    nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"])
    texts_out = []
    for text in texts:
        doc = nlp(text)
        new_text = []
        for token in doc:
            if token.pos_ in allowed_postags:
                new_text.append(token.lemma_)
        final = " ".join(new_text)
        texts_out.append(final)
    return texts_out

lemmatized_texts = lemmatization(data)
print(lemmatized_texts[0])

def gen_words(texts):
    final = []
    for text in texts:
        new = gensim.utils.simple_preprocess(text, deacc=True)
        final.append(new)
    return final

data_words = gen_words(lemmatized_texts)
print(data_words[0])

id2word = corpora.Dictionary(data_words)
corpus = []
for text in data_words:
    new = id2word.doc2bow(text)
    corpus.append(new)

print(corpus[0])
lda_model = gensim.models.ldamodel.LdaModel(
    corpus=corpus,
    id2word=id2word,
    num_topics=2,
    random_state=100,
    update_every=1,
    alpha="auto"
)

# pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim_models.prepare(lda_model, corpus, id2word, mds="mmds", R=30)
pyLDAvis.save_html(vis, 'lda_naga1.html')


#%%

import datetime
import pandas as pd
from collections import Counter

#%%

df=pd.read_json('new_parsed_tweets_closed.json')
df2=pd.read_csv('parsed_new.csv', encoding='windows-1252')

#%%

all_hashtags=[]
for hashtags in df['hashtags_list']:
    for item in list(hashtags):
        all_hashtags.append(item)
        #print(item)

all_hashtags=sorted(all_hashtags)

#%%
stats=(Counter(all_hashtags ))
common_hashtags=stats.most_common(30)
comm=[]
for i in common_hashtags:
    comm.append(i[0])
#%%
print(comm)
a = datetime.date.today()
numdays = 13
date_list = []
for x in range (0, numdays):
    date_list.append(a - datetime.timedelta(days = x))

print (date_list)
print(date_list[0])
df['created_at']=pd.to_datetime(df['created_at'])
df['created_at']=df['created_at'].apply(lambda x: x.replace(tzinfo=None))
df['created_at']=df['created_at'].dt.date
new_data=[]
for date in date_list:
    df_filtered=df[((df["created_at"]==date))]
    print(df_filtered)
    for hashtag in comm:
        new_row=[0]*6
        #print(new_row)
        for index, row in df_filtered.iterrows():
            #print(row
            #print(row["created_at"].day )
            if hashtag in row["hashtags_list"]:
                new_row[0]=date
                new_row[1]=hashtag
                new_row[2]+=1
                new_row[3]+=row["retweet_count"]
                new_row[4]+=row["favorite_count"]
                new_row[5]+=row["user_followers_count"]
                # print("luj")
                # print(new_row)
                #df.drop(index)
        new_data.append(new_row)
        #print(new_data)

new_df=pd.DataFrame.from_records(new_data)
print(new_df)


from Crypto.Cipher import DES3
from Crypto import Random

def main():
  key = b"The secret key a"
  iv = Random.new().read(DES3.block_size)

  # Encryption
  encryption_suite = DES3.new(key, DES3.MODE_CBC, iv)
  cipher_text = encryption_suite.encrypt(b"Hello world!    ")
  print(cipher_text)

  # Decryption
  decryption_suite = DES3.new(key, DES3.MODE_CBC, iv)
  plain_text = decryption_suite.decrypt(cipher_text)
  print(plain_text)

if __name__ == '__main__':
  main()


import itertools
import os
import pathlib

DOSE_EXTENSIONS = ["dcm"]
DOSE_PREFIX = ["dose"]
VARIANCE_EXTENSIONS = ["dcm"]
VARIANCE_PREFIX = ["variance"]
DENSITY_EXTENSIONS = ["npy"]
DENSITY_PREFIX = ["density"]

def make_dataset(dir, max_dataset_size=float("inf")):
    doses = []
    assert os.path.isdir(dir), '%s is not a valid directory' % dir

    for root, _, fnames in sorted(os.walk(dir)):
        for fname in fnames:
            if is_dose_file(fname):
                path = os.path.join(root, fname)
                doses.append(path)
    return doses[:min(max_dataset_size, len(doses))]

def make_variance_dataset(dir, max_dataset_size=float("inf")):
    variances = []
    assert os.path.isdir(dir), '%s is not a valid directory' % dir

    for root, _, fnames in sorted(os.walk(dir)):
        for fname in fnames:
            if is_variance_file(fname):
                path = os.path.join(root, fname)
                variances.append(path)
    return variances[:min(max_dataset_size, len(variances))]

def make_density_dataset(dir, max_dataset_size=float("inf")):
    densities = []
    assert os.path.isdir(dir), '%s is not a valid directory' % dir

    for root, _, fnames in sorted(os.walk(dir)):
        for fname in fnames:
            if is_density_file(fname):
                path = os.path.join(root, fname)
                densities.append(path)
    return densities[:min(max_dataset_size, len(densities))]

def is_dose_file(filename):
    filename = pathlib.Path(filename)
    filename = filename.name
    return any(filename.endswith(extension) and filename.startswith(prefix)
               for extension, prefix in itertools.product(DOSE_EXTENSIONS, DOSE_PREFIX))

def is_variance_file(filename):
    filename = pathlib.Path(filename)
    filename = filename.name
    return any(filename.endswith(extension) and filename.startswith(prefix)
               for extension, prefix in itertools.product(VARIANCE_EXTENSIONS, VARIANCE_PREFIX))

def is_density_file(filename):
    filename = pathlib.Path(filename)
    filename = filename.name
    return any(filename.endswith(extension) and filename.startswith(prefix)
               for extension, prefix in itertools.product(DENSITY_EXTENSIONS, DENSITY_PREFIX))


"""
目标：完成登录业务层实现
"""
# 导包 unittest ApiLogin
import unittest
from InterfaceTest.api.api_login import ApiLogin
from parameterized import parameterized
from InterfaceTest.tools.read_json import ReadJson

# 读取数据函数
def get_data():
    datas = ReadJson("login_more.json").read_json()
    arrs = []
    # 使用遍历获取所有的value
    for data in datas.values():
        arrs.append((data.get("url"),
                     data.get("mobile"),
                     data.get("code"),
                     data.get("expect_result"),
                     data.get("status_code")))
    return arrs

# 新建测试类
class TestLogin(unittest.TestCase):
    # 新建测试方法
    @parameterized.expand(get_data())
    def test_login(self, url, mobile, code, expect_result, status_code):
        # 暂时存放数据 url mobile code
        # url = "http://ttapi.research.itcast.cn/app/v1_0/authorizations"
        # mobile = "16602710209"
        # code = "882477"
        # 调用登录方法
        s = ApiLogin().api_post_login(url, mobile, code)
        # 调试使用
        print("查看响应结果：", s.json())
        # 断言 响应信息 及 状态码
        # self.assertEquals("ok", s.json()['message'])
        self.assertEquals(expect_result, s.json()['message'])
        # 断言响应状态码
        # self.aseertEquals(201, s.status_code)
        self.aseertEquals(status_code, s.status_code)

if __name__ == '__main__':
    unittest.main()


import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import time
import random
from contextlib import contextmanager
from adabelief_pytorch import AdaBelief
import functools
from omegaconf import OmegaConf
from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold
from collections import defaultdict, Counter
import sys
from tqdm import tqdm
from typing import Tuple
from torch.utils.data import Dataset
from torch.utils.data.dataloader import DataLoader
# from pytorch_lightning.metrics.functional.classification import auroc
import cv2
from pytorch_lightning import LightningDataModule
from sklearn import model_selection
import albumentations as A
from sklearn.metrics import roc_auc_score, accuracy_score, average_precision_score, precision_score
from pytorch_lightning.core.lightning import LightningModule

from albumentations import (
    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,
    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,
    IAAAdditiveGaussianNoise, Transpose, CLAHE, MultiplicativeNoise, IAASharpen
)
from torch.autograd import Variable
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold

from albumentations.pytorch import ToTensorV2
from albumentations import ImageOnlyTransform
import matplotlib.pyplot as plt
from pathlib import Path
# import timm
from google.cloud import storage
from torchvision import models
import torch.nn as nn
from torch.nn import functional as F
import torch
from torch.nn.parameter import Parameter
# from torch.optim import Adam, SGD
from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau
import pytorch_lightning as pl
import torch.optim as optim
from src.optimizer import get_optimizer
from src.loss import get_criterion
from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.metrics import AveragePrecision
from pytorch_lightning.metrics.functional import average_precision
import wandb
import warnings

warnings.simplefilter('ignore')
sys.path.append("../pytorch-image-models")
import timm
from timm.utils.agc import adaptive_clip_grad

train = pd.read_csv('../data/Training_Set/RFMiD_Training_Labels.csv')
extra = pd.read_csv("../extra/use_df.csv").iloc[:0, :]
extra["fold"] = -1
test = train.iloc[:640, :]
# test = pd.read_csv('../data/sample_submission.csv')
rand = random.randint(0, 100000)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def seed_torch(seed=42):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

def multi_disease_avg_score(y_true: np.ndarray, y_pred: np.ndarray):
    map_score = average_precision_score(y_true=y_true, y_score=y_pred, average=None)
    map_score = np.nan_to_num(map_score, nan=0.0).mean()

    scores = []
    for i in range(len(y_true[0])):
        if y_true[:, i].mean() > 0.0:
            auc = roc_auc_score(y_true=y_true[:, i], y_score=y_pred[:, i])
            scores.append(auc)
        else:
            scores.append(0.0)
    auc_score = np.mean(scores)
    return 0.5 * map_score + 0.5 * auc_score

# ====================================================
# Transforms
# ====================================================
def get_transforms(img_size, data):
    if data == 'train':
        return Compose([
            Resize(img_size, img_size),
            # RandomResizedCrop(img_size, img_size, scale=(0.85, 1.0)),
            # CLAHE(clip_limit=(1, 4), p=1),
            # MultiplicativeNoise(p=1),
            # IAASharpen(p=1),
            HorizontalFlip(p=0.5),
            Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225],
            ),
            IAAAdditiveGaussianNoise(),
            ToTensorV2(),
        ])

    elif data == 'valid':
        return Compose([
            Resize(img_size, img_size),
            Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225],
            ),
            ToTensorV2(),
        ])

# ====================================================
# Dataset
# ====================================================
class TrainDataset(Dataset):
    def __init__(self, cfg, df, transform=None, inference=False):
        self.cfg = cfg
        self.df = df
        self.file_names = df['ID'].values
        # target_cols = # df.drop(columns=["ID", "fold"]).columns if target_cols == "all" else target_cols
        self.labels = df[cfg.base.target_cols].values
        self.transform = transform
        self.inference = inference

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.file_names[idx]
        if "right" in str(file_name) or "left" in str(file_name):
            file_path = f'{self.cfg.base.extra_path}/{file_name}.png'
        else:
            file_path = f'{self.cfg.base.train_path}/{file_name}.png'
        image = cv2.imread(file_path)
        # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        try:
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        except:
            print(file_path)

        if self.transform:
            augmented = self.transform(image=image)
            image = augmented['image']

        label = torch.tensor(self.labels[idx]).float()

        if self.inference:
            return image
        else:
            return image, label

class TestDataset(Dataset):
    def __init__(self, cfg, df, transform=None):
        self.cfg = cfg
        self.df = df
        self.file_names = df['ID'].values
        self.transform = transform

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.file_names[idx]
        file_path = f'{self.cfg.base.test_path}/{file_name}.png'
        image = cv2.imread(file_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        if self.transform:
            augmented = self.transform(image=image)
            image = augmented['image']
        return image

class CHIZUDataModule(LightningDataModule):
    def __init__(
            self,
            cfg,
            train_df,
            val_df,
            aug_p: float = 0.5,
            val_pct: float = 0.2,
            img_sz: int = 224,
            batch_size: int = 64,
            num_workers: int = 4,
            fold_id: int = 0,
    ):
        super().__init__()
        self.cfg = cfg
        self.aug_p = aug_p
        self.val_pct = val_pct
        self.img_sz = img_sz
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.fold_id = fold_id

        self.train_df = train_df
        self.val_df = val_df

    def train_dataloader(self):
        train_dataset = TrainDataset(self.cfg, self.train_df,
                                     transform=get_transforms(self.img_sz, data="train"))

        return DataLoader(
            train_dataset,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
            shuffle=True,
            pin_memory=True,
            drop_last=True
        )

    def val_dataloader(self):
        valid_dataset = TrainDataset(self.cfg, self.val_df,
                                     transform=get_transforms(self.img_sz, data="valid"))
        return DataLoader(
            valid_dataset,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
            shuffle=False,
            pin_memory=True,
            drop_last=False
        )

def gem(x, p=3, eps=1e-6):
    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)

class GeM(nn.Module):
    def __init__(self, p=3, eps=1e-6):
        super(GeM, self).__init__()
        self.p = Parameter(torch.ones(1) * p)
        self.eps = eps

    def forward(self, x):
        return gem(x, p=self.p, eps=self.eps)

    def __repr__(self):
        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(
            self.eps) + ')'

__SPLITS__ = {
    "MultilabelStratifiedKFold": MultilabelStratifiedKFold
}

def get_scheduler(cfg, optimizer):
    scheduler_name = cfg.scheduler.name

    if scheduler_name is None:
        return
    else:
        return optim.lr_scheduler.__getattribute__(scheduler_name)(optimizer, **cfg.scheduler.param)

def get_split(cfg):
    if hasattr(model_selection, cfg.split.name):
        return model_selection.__getattribute__(cfg.split.name)(**cfg.split.param)
    elif __SPLITS__.get(cfg.split.name) is not None:
        return __SPLITS__[cfg.split.name](**cfg.split.param)
    else:
        raise NotImplementedError

class sub_loss(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, outputs, labels):
        if labels.sum() == 0:
            return 1
        else:
            outputs = torch.sigmoid(outputs)
            score = np.nan_to_num(
                average_precision_score(labels.cpu().detach().numpy(), outputs.cpu().detach().numpy(), average=None),
                nan=0.0).mean()
            loss = torch.tensor(1 - score, requires_grad=True)
            return loss

class CHIZUModel(LightningModule):
    def __init__(self, cfg):
        super().__init__()

        self.cfg = cfg
        self.wd = 1e-6
        self.model_name = cfg.model.model_name
        self.model = timm.create_model(cfg.model.model_name, pretrained=cfg.model.pretrained)

        if "nfnet" in self.model_name:
            # n_features = self.model.num_classes
            n_features = self.model.num_features
            self.model.head.fc = nn.Linear(n_features, cfg.base.target_size)
        elif "efficient" not in self.model_name:
            n_features = self.model.fc.in_features
            self.model.fc = nn.Linear(n_features, cfg.base.target_size)
        else:
            self.model.classifier = nn.Linear(self.model.num_features, cfg.base.target_size)

        if self.cfg.model.gem:
            self.model.avg_pool = GeM()

        self.optimizer = get_optimizer(cfg, self.model)
        self.scheduler = get_scheduler(cfg, self.optimizer)
        self.criterion = get_criterion(cfg)
        self.sigmoid = nn.Sigmoid()
        # self.sub_loss = nn.BCEWithLogitsLoss()
        # self.ap = AveragePrecision(num_classes=cfg.base.target_size)
        # self.ap_list = [AveragePrecision(num_classes=1) for _ in range(cfg.base.target_size)]

    def forward(self, x):
        x = self.model(x)
        # x = self.sigmoid(x)
        return x

    def training_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        # sub_loss = self.sub_loss(y_hat, y)

        # for j in range(self.cfg.base.target_size):
        #     if j == 0:
        #         sub_loss = 1 - self.ap_list[j](torch.sigmoid(y_hat[:, j]), y[:, j])
        #     else:
        #         loss_ = 1 - self.ap_list[j](torch.sigmoid(y_hat[:, j]), y[:, j])
        #         loss_[loss_ != loss_] = 0
        #         sub_loss += loss_
        #
        # loss = loss * 0.5 + sub_loss * 0.5
        # self.log("train_loss", loss, prog_bar=True)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        # sub_loss = self.sub_loss(y_hat, y)
        # loss = loss * 0.5 + sub_loss * 0.5

        # self.log("valid_loss", loss, prog_bar=True)
        return loss, y_hat.cpu().numpy(), y.cpu().numpy()

    def validation_epoch_end(self, input_):
        auc_l = 0
        acc_l = 0
        acc_f = 0
        # map_l = 0
        for j in range(self.cfg.base.target_size):
            loss_list, y_hat_list, y_list = np.array([]), np.array([]), np.array([])
            for i, (loss, y_hat, y) in enumerate(input_):
                # y_hat_list = np.append(y_hat_list, y_hat.argmax(1))
                y_hat_list = np.append(y_hat_list, y_hat[:, j])
                y_list = np.append(y_list, y[:, j])

            y_hat_list = sigmoid(y_hat_list)
            try:
                auc = roc_auc_score(y_list, y_hat_list)
            except ValueError:
                auc = 0
            acc = accuracy_score(y_list, np.round(y_hat_list))
            # ap = precision_score(y_list, np.round(y_hat_list), average="macro")
            auc_l += auc / self.cfg.base.target_size
            acc_l += acc / self.cfg.base.target_size
            # map_l += ap / self.cfg.base.target_size

            num = "{0:02d}".format(j + 1)
            if j == 0:
                self.log(f"{self.cfg.base.target_cols[j]}-auc", auc, prog_bar=True)
            else:
                self.log(f"{self.cfg.base.target_cols[j]}-auc", auc, prog_bar=False)

            if j == 0:
                auc_f = auc

        for i, (loss, y_hat, y) in enumerate(input_):
            loss_list = np.append(loss_list, float(loss.cpu()))

        self.log("valid_loss", loss_list.mean(), prog_bar=True)
        self.log("valid auc", auc_l, prog_bar=True)

        all_pred, all_true = np.empty((0, self.cfg.base.target_size), dtype=int), np.empty(
            (0, self.cfg.base.target_size), dtype=int)
        for i, (loss, y_hat, y) in enumerate(input_):
            # y_hat_list = np.append(y_hat, y_hat.argmax(1))
            all_pred = np.append(all_pred, y_hat, axis=0)
            all_true = np.append(all_true, y, axis=0)

        map = average_precision_score(all_true, np.round(all_pred), average=None)
        map = np.nan_to_num(map, nan=0.0)
        for j in range(self.cfg.base.target_size):
            self.log(f"{self.cfg.base.target_cols[j]}-map", map[j], prog_bar=False)

    """
    loss_list, all_pred, all_true = np.empty((0, self.cfg.base.target_size), dtype=int),np.empty((0, self.cfg.base.target_size), dtype=int),np.empty((0, self.cfg.base.target_size), dtype=int)
    for i, (loss, y_hat, y) in enumerate(input_):
        # y_hat_list = np.append(y_hat, y_hat.argmax(1))
        all_pred = np.append(all_pred, y_hat, axis=0)
        all_true = np.append(all_true, y, axis=0)

    """

    def configure_optimizers(self):
        optimizer = self.optimizer
        scheduler = self.scheduler

        return [optimizer], [scheduler]

    def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_idx, cloosure, second_order_closure=None,
                       on_tpu=False,
                       using_native_amp=False, using_lbfgs=False):
        adaptive_clip_grad(self.model.parameters(), clip_factor=0.01, eps=1e-3, norm_type=2.0)
        optimizer.step(closure=cloosure)
        optimizer.zero_grad()

class SAMModel(CHIZUModel):
    def __init__(self, cfg):
        super().__init__(cfg)

    def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx,
                       optimizer_closure, on_tpu, using_native_amp, using_lbfgs):
        optimizer.first_step(closure=optimizer_closure, zero_grad=True)
        optimizer.second_step(closure=optimizer_closure, zero_grad=True)

def test_inf(cfg, dataset, model, model_path):
    model = model.load_from_checkpoint(model_path, cfg=cfg, model_name=cfg.model.model_name).to(device)
    model.freeze()
    model.eval()
    test_loader = DataLoader(
        dataset,
        batch_size=cfg.model.batch_size,
        num_workers=cfg.base.num_workers,
        shuffle=False,
        pin_memory=True,
        drop_last=False
    )

    for i, img in tqdm(enumerate(test_loader)):
        y_hat = model(img.to(device))
        if i == 0:
            pred = y_hat.cpu().numpy()
        else:
            pred = np.append(pred, y_hat.cpu().numpy(), axis=0)

    pred = sigmoid(pred)

    return pred

def train_loop(cfg, folds, fold):
    if cfg.wandb.use:
        wandb.init(
            name=cfg.wandb.name + f"-fold-{fold}-{rand}",
            project=cfg.wandb.project,
            tags=cfg.wandb.tags + [str(rand)],
            reinit=True
        )
        wandb_logger = WandbLogger(
            name=cfg.wandb.name + f"-fold-{fold}-{rand}",
            project=cfg.wandb.project,
            tags=cfg.wandb.tags + [str(rand)]
        )
        wandb_logger.log_hyperparams(dict(cfg))
        wandb_logger.log_hyperparams(dict({"rand": rand, "fold": fold, }))

    trn_idx = folds[folds['fold'] != fold].index
    val_idx = folds[folds['fold'] == fold].index

    train_folds = folds.loc[trn_idx].reset_index(drop=True)
    valid_folds = folds.loc[val_idx].reset_index(drop=True)

    train_folds = pd.concat(
        [
            train_folds,
            extra
        ],
        # axis=1
    )

    data_module = CHIZUDataModule(
        cfg,
        train_folds,
        valid_folds,
        aug_p=0.5,
        img_sz=cfg.model.size,
        batch_size=cfg.model.batch_size,
        num_workers=cfg.base.num_workers,
        # fold_id=fold,
    )
    if cfg.optimizer.name == "SAM":
        model = SAMModel(
            cfg
        )
    else:
        model = CHIZUModel(
            cfg
        )

    checkpoint_callback = ModelCheckpoint(
        dirpath=f'../exp9/{rand}',
        filename=f"fold-{fold}",
        # save_top_k=3,
        mode='min',
    )

    trainer = pl.Trainer(
        gpus=-1,
        max_epochs=cfg.model.epochs,
        gradient_clip_val=0.1,
        precision=16,
        logger=wandb_logger if "wandb_logger" in locals() else None,
        callbacks=[checkpoint_callback]
    )

    trainer.fit(model=model, datamodule=data_module)
    model_path = checkpoint_callback.best_model_path

    test_set = TestDataset(
        cfg,
        test,
        get_transforms(
            cfg.model.size,
            "valid"
        )
    )
    pred = test_inf(cfg, test_set, model, model_path)
    oof = None

    if cfg.base.oof:
        val_set = TrainDataset(
            cfg,
            valid_folds,
            get_transforms(
                cfg.model.size,
                "valid"
            ),
            inference=True
        )
        oof = test_inf(cfg, val_set, model, model_path)
    return pred, oof

def main(cfg):
    seed_torch(seed=cfg.base.seed)

    folds = train.copy()
    Fold = get_split(cfg)
    for n, (train_index, val_index) in enumerate(Fold.split(folds, folds[cfg.base.target_cols])):
        folds.loc[val_index, 'fold'] = int(n)
    folds['fold'] = folds['fold'].astype(int)

    oof_df = train.copy()
    test_pred = test.copy()
    test_pred.iloc[:, 1:] = 0

    for fold in range(cfg.base.n_fold):
        if fold in cfg.base.trn_fold:
            fold_pred, fold_oof = train_loop(cfg, folds, fold)
            if cfg.base.oof:
                oof_df.iloc[folds["fold"] == fold, 1:] = fold_oof

            test_pred[list(cfg.base.target_cols)] += fold_pred / len(cfg.base.trn_fold)

    test_pred[["ID"] + list(cfg.base.target_cols)].to_csv(
        f"../exp9/{rand}/{rand}_{cfg.base.n_fold}_{len(cfg.base.trn_fold)}.csv", index=False)
    # oof_df.to_csv(f"../exp2/{rand}/{rand}_oof.csv", index=False)

if __name__ == "__main__":
    main(OmegaConf.load("../yaml/9.yaml"))


import sqlite3

class MemeDatabase:
    """
    Meme database interface

    :param str site:
        The name of the StackExchange site as an identifier (used as the table
        name)
    """
    def __init__(self, site):
        self.site = site
        self.conn = sqlite3.connect('memes.db')
        cursor = self.conn.cursor()
        cursor.execute(f"create table if not exists {site} (question_id int unique)")
        cursor.execute(f"create table if not exists memes (meme_id int unique, meme_name text, blacklisted bool, include_random bool)")
        cursor.close()

    def __repr__(self):
        return f"<MemeDatabase object for site {self.site}>"

    def insert_memes(self, memes):
        """
        Insert memes
        """
        cursor = self.conn.cursor()
        try:
            cursor.executemany("insert or ignore into memes values (?, ?, 0, 1)", memes)
            self.conn.commit()
        except sqlite3.IntegrityError:
            pass
        cursor.close()

    def select_random_meme(self):
        """
        Select a random meme ID
        """
        cursor = self.conn.cursor()
        cursor.execute(f"select meme_id from memes where include_random and not blacklisted order by random() limit 1")
        result = cursor.fetchone()
        cursor.close()
        return result[0]

    def select_meme(self, id):
        """
        Look up a specific meme by ID
        """
        cursor = self.conn.cursor()
        cursor.execute(f"select meme_name from memes where meme_id = ?", (id, ))
        result = cursor.fetchone()
        cursor.close()
        return result[0]

    def search_for_meme(self, search):
        """
        Look up a specific meme by searching
        """
        cursor = self.conn.cursor()
        cursor.execute(f"select * from memes where lower(meme_name) like ?", (f'%{search}%', ))
        results = cursor.fetchall()
        cursor.close()
        return results

    def blacklist_meme(self, id):
        """
        Blacklist a meme ID
        """
        cursor = self.conn.cursor()
        cursor.execute(f"update memes set blacklisted = 1 where meme_id = ?", (id, ))
        self.conn.commit()
        cursor.close()

    def insert_question(self, id):
        """
        Insert a question ID
        """
        cursor = self.conn.cursor()
        cursor.execute(f"insert into {self.site} values (?)", (id, ))
        self.conn.commit()
        cursor.close()

    def question_is_known(self, id):
        """
        Return True if the provided question ID is already in the database,
        otherwise return False
        """
        cursor = self.conn.cursor()
        cursor.execute(f"select 1 from {self.site} where question_id = ?", (id, ))
        result = bool(cursor.fetchone())
        cursor.close()
        return result


# coding: utf-8

# ### Problem:
#
# *The dataset has information collected from a company which faces issue with employees leaving the company. The database has information collected from different employees who have left or still staying in the company. The company wants to use your expertise in identifying which is the major contributor for employees leaving the company.*
#
# 1. Satisfaction Level - ranges between 0 & 1 - gives the satisfaction level of the employee
# 2. last_evaluation - ranges between 0 & 1 - defines the normalised employee's rating in the last appraisal
# 3. number_project - numeric - No of projects the employee has worked on so far
# 4. average_monthly_hours - Average amount of hours employee spends in the office per month
# 5. time_spend_company - Time employee has spent in the company (in months)
# 6. Work_accident - categorical - If the employee has encountered any accident in the work environment
# 7. Department - Categorical - Department in which the employee is working/ has worked
# 8. Salary - Categorical - Divided into low,medium & high

# ### Read the dataset in hr-analytics.csv file

# In[105]:

# Import all Libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plot

# In[106]:

hr_data = pd.read_csv("hr-analytics.csv")
hr_data.head()

# ### Check the variables in the dataset

# In[107]:

hr_data.info()

# ### Observe the shape and the type of dataset

# In[108]:

hr_data.shape

# ### Check sample records of dataset

# In[109]:

hr_data.sample(10)

# ### Use correlation for numerical variables & scatter matrix to observe the dependency between variables (Drop the dependent variable if the abs(correlation) with dependent variable is <0.01)

# In[110]:

pd.plotting.scatter_matrix(hr_data)

# In[111]:

corr = hr_data.corr()
corr

# In[112]:

corr[abs(corr['left']) < 0.01]

# In[113]:

hr_data_new = hr_data.drop(corr[abs(corr['left']) < 0.01].index, axis=1)
hr_data_new.info()

# In[114]:

plt.figure(figsize=(12,6))
sns.heatmap(corr,cmap='plasma', vmax=1,vmin=-1)

# ### Label Encode the categorical variables `Department` and `salary` (Use Label Encoder from Scikit learn)

# In[115]:

#Use Label Encoder to encode class values
from sklearn.preprocessing import LabelEncoder
labelencoder = LabelEncoder()

# In[116]:

hr_data_new['Department'] = labelencoder.fit_transform(hr_data_new['Department'])
hr_data_new['salary'] = labelencoder.fit_transform(hr_data_new['salary'])
hr_data_new.head()

# ### Divide into feature set and target set (70:30)

# In[117]:

X = hr_data_new.drop('left',axis=1)
y = hr_data_new['left']

# ### Standardize the dataset using Standard scaler function

# In[118]:

from sklearn.preprocessing import StandardScaler
std_scale = StandardScaler()

# In[119]:

X_scale = pd.DataFrame(std_scale.fit_transform(X), columns=X.columns)
X_scale.head()

# ### Covariance Matrix
#
# **Hint:** check np.cov() in numpy. By default, it assumes each row represents a variable, with observations in the columns. So, transponse of the dataframe should be given as input.
#
# Covariance = np.cov(X_std.T)
#
# Covariance matrix is required for calculating Eigen vectors & values manually
#

# In[120]:

covar = np.cov(X_scale.T)
covar

# ### Eigen Values and Vectors
#
# **Hint:** check np.linalg.eig() in numpy. It takes covariance matrix as input and gives the eigen values, eigen vectors as output.
#
# Example: `eig_vals, eig_vecs = np.linalg.eig(covariance_matrix)`
#

# In[121]:

eig_vals, eig_vecs = np.linalg.eig(covar)
print(eig_vals, eig_vecs)

# ## Sort the Eigen values
#
# ###  We need to choose the top eigen values (as it explains the most variance) and the corresponding vectors associated with that.
# ### It makes sense to treat eigen vectors & values as pairs and then sort them. Sort the eigen pairs array using the first column (Eigen value) - Hint: Use .sort function

# `eig_pairs = [(np.abs(eig_vals[i]), eig_vecs[:,i]) for i in range(len(eig_vals))]`

# In[122]:

eig_pairs = [(np.abs(eig_vals[i]), eig_vecs[:,i]) for i in range(len(eig_vals))]

# In[123]:

eig_pairs

# In[124]:

eig_pairs.sort(reverse=True)

# In[125]:

eig_pairs

# ### Explained Variance
#
# **Hint:**
#
# `total = Sum of Eigen Values
#  variance_explained = [(each / total)*100 for each eigenvalue]`

# In[126]:

total = sum(eig_vals)
total

# In[127]:

variance_explained = (eig_vals / total)*100

# In[128]:

variance_explained

# ### Plot the Explained Variance
#
# Use the bar plot to plot var_exp using matplotlib
#
# `check plt.bar()`

# In[129]:

plot.bar(np.arange(len(variance_explained)), variance_explained,0.5)

# ## For Visualization purpose

#
# #### Projection Matrix considering only 2 PCs.

# In[130]:

w = np.hstack((eig_pairs[0][1].reshape(8,1), eig_pairs[1][1].reshape(8,1)))
print(w)

# #### Projection Onto the New Feature Space

# In[131]:

Y = X_scale.dot(w)
Y.shape

# ### Visual exploration of data distribution on first three major PCA dimensions
#

# In[132]:

get_ipython().magic(u'matplotlib inline')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.decomposition import PCA

fig = plt.figure(1, figsize=(8, 6))
ax = Axes3D(fig, elev=-150, azim=110)

X_test_reduced = PCA(n_components=3).fit_transform(X_scale)
print(X_test_reduced[:,1])
ax.scatter(X_test_reduced[:, 0], X_test_reduced[:, 1], X_test_reduced[:, 2], c=pd.DataFrame(X_scale).iloc[:,0].values,
           cmap=plt.cm.Set1, edgecolor='k', s=40)

ax.set_title("First three PCA directions")
ax.set_xlabel("1st eigenvector")
ax.w_xaxis.set_ticklabels([])
ax.set_ylabel("2nd eigenvector")
ax.w_yaxis.set_ticklabels([])
ax.set_zlabel("3rd eigenvector")
ax.w_zaxis.set_ticklabels([])

plt.show()

# ### Factor Analysis

# In[133]:

from sklearn.decomposition import FactorAnalysis
famodel = FactorAnalysis(n_components=8, random_state=10).fit(X_scale)

# In[134]:

print(famodel)

# In[135]:

print(famodel.components_)

# In[136]:

# Use Absolute values, so its simple to interpret
df_factors = pd.DataFrame(abs(famodel.components_),columns=X.columns.values,
                          index=['Comp0','Comp1','Comp2','Comp3','Comp4','Comp5','Comp6','Comp7']).transpose()
df_factors

# In[137]:

plt.figure(figsize=(20,10))
sns.heatmap(df_factors,cmap='plasma', annot=True, vmin=0,vmax=1)

# ### Perform factor analysis using the function FactorAnalysis ( Use No of components = 9)

# In[138]:

# Done Above

# ### Print the factors

# In[139]:

# Done Above

# ### Plot the factors using heatmap

# In[140]:

# Done above

# ### Use the heat map to come up with no of factors and its reasoning

# In[141]:

df_factors.sum()

# ## Based on the above data Comp0, Comp1 & Comp2 factors (first 3 factors) loads majority of the features.
#
# Comp0 ==> Loads time_spend_company, average_monthly_hours, number_project (Working Hours or Engagement Time)
# Comp1 ==> Loads work_accident, satisfaction_level (Safe Environment)
# Comp2 ==> Loads promotion_last_5years
#
# #### Kindly note time_spend_company should be ideally grouped with promotion_last_5years indicating Employee growth. Hence this data requires rotation. Post rotation, the components or factors will be:
#
# Comp0 ==> "Employee Engagement" which would group average_monthly_hours, number_project
# Comp1 ==> "Safe Environment" which would group work_accident, satisfaction_level
# Comp3 ==> "Employee Growth" which would group promotion_last_5year, time_spend_compant

# ### Communality Score

# In[142]:

df_factors.transpose().sum()

# # Thank You


#!/usr/bin/python

import sys
import telnetlib
import re

host = "localhost"
port = "8888"
teamname = "teamtest"

""" try to make an example"""

def example_telnet(host=host,port=port,teamname=teamname):
    """start the conversation"""
    tn = telnetlib.Telnet(host,port)
    tn.read_until("Language/team name",20)
    tn.write(teamname + "\n")
    pattern = r"N(\d+) E(\d+) S(\d+) W(\d+) P(\??[0-9]?)"
    pt = re.compile(pattern)
    tn.expect(pt, 20)
    tn.write("exit\n")
    tn.write(r"^]")
    return tn

def compose_order(N=None,S=None,W=None,E=None):
    """compose an order string to send"""
    if N is None:
        N=0
    if S is None:
        S=0
    if W is None:
        W=0
    if E is None:
        E = 0
    template = """N{0} E{1} S{2} W{3} P{4}\n"""
    return template.format(N,E,S,W,"?")

def parse_answer(answer=None):
    """parses the request to make it int"""
    assert answer is not None
    r = pt.match(answer).groups()
    return {"N":int(r[0]),
            "E":int(r[1]),
            "S":int(r[2]),
            "W":int(r[3]),
            "P":r[4]}

if __name__ == "__main__":
    example_telnet()


from scipy import linalg
import numpy as np

def soltrinf(A,b):

    if np.linalg.det(A) == 0 :
        print('La matriz es singular')
        return None

    n = A.shape[0]
    x = np.zeros(n)

    for i in range(n):
        x[i] = b[i] - np.dot( A[i,:i ], x[:i] )
        x[i] = x[i] / A[i,i]

    return x

def soltrsup(A, b):
    n = A.shape[0]
    x = b

    for idx in range(n-1, -1, -1):
        for jdx in range(n-1, idx, -1):
            x[idx] = x[idx] - A[idx, jdx] * x[jdx]
        x[idx] = x[idx] / A[idx, idx]

    return x

def sollu(A, b):
    P, L, U = linalg.lu(A)
    y = P.T @ b
    z = soltrinf(L, y)
    x = soltrsup(U, z)

    return x


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/mx/Dropbox (MIT)/Science/Code/allesfitter/allesfitter/general_output.py
# Compiled at: 2019-01-24 13:45:24
"""
Created on Fri Oct  5 01:10:51 2018

@author:
Maximilian N. Günther
MIT Kavli Institute for Astrophysics and Space Research,
Massachusetts Institute of Technology,
77 Massachusetts Avenue,
Cambridge, MA 02109,
USA
Email: maxgue@mit.edu
Web: www.mnguenther.com
"""
from __future__ import print_function, division, absolute_import
import seaborn as sns
sns.set(context='paper', style='ticks', palette='deep', font='sans-serif', font_scale=1.5, color_codes=True)
sns.set_style({'xtick.direction': 'in', 'ytick.direction': 'in'})
sns.set_context(rc={'lines.markeredgewidth': 1})
import numpy as np, matplotlib.pyplot as plt, os, sys, warnings
from astropy.time import Time
warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
warnings.filterwarnings('ignore', category=np.RankWarning)
from exoworlds.lightcurves import lightcurve_tools as lct
from . import config
from .utils import latex_printer
from .computer import update_params, calculate_model, rv_fct, flux_fct, calculate_baseline, calculate_yerr_w

def logprint(*text):
    print(*text)
    original = sys.stdout
    with open(os.path.join(config.BASEMENT.outdir, 'logfile_' + config.BASEMENT.now + '.log'), 'a') as (f):
        sys.stdout = f
        print(*text)
    sys.stdout = original

def draw_initial_guess_samples(Nsamples=1):
    if Nsamples == 1:
        samples = np.array([config.BASEMENT.theta_0])
    else:
        samples = config.BASEMENT.theta_0 + config.BASEMENT.init_err * np.random.randn(Nsamples, len(config.BASEMENT.theta_0))
    return samples

def plot_panel(datadir):
    config.init(datadir)
    fig, axes = plt.subplots(2, 1, figsize=(20, 10))
    for inst in config.BASEMENT.settings['inst_phot']:
        axes[0].plot(config.BASEMENT.data[inst]['time'], config.BASEMENT.data[inst]['flux'], marker='.', ls='none', label=inst)

    for inst in config.BASEMENT.settings['inst_rv']:
        axes[1].plot(config.BASEMENT.data[inst]['time'], config.BASEMENT.data[inst]['rv'], marker='.', ls='none', label=inst)

    axes[0].legend()
    axes[1].legend()
    return (
     fig, axes)

def afplot(samples, companion):
    """
    Inputs:
    -------
    samples : array
        samples from the initial guess, or from the MCMC / Nested Sampling posteriors
    """
    N_inst = len(config.BASEMENT.settings['inst_all'])
    if 'do_not_phase_fold' in config.BASEMENT.settings and config.BASEMENT.settings['do_not_phase_fold']:
        fig, axes = plt.subplots(N_inst, 1, figsize=(6, 4 * N_inst))
        styles = ['full']
    else:
        if config.BASEMENT.settings['phase_variations']:
            fig, axes = plt.subplots(N_inst, 5, figsize=(30, 4 * N_inst))
            styles = ['full', 'phase', 'phase_variation', 'phasezoom', 'phasezoom_occ']
        elif config.BASEMENT.settings['secondary_eclipse']:
            fig, axes = plt.subplots(N_inst, 4, figsize=(24, 4 * N_inst))
            styles = ['full', 'phase', 'phasezoom', 'phasezoom_occ']
        else:
            fig, axes = plt.subplots(N_inst, 3, figsize=(18, 4 * N_inst))
            styles = ['full', 'phase', 'phasezoom']
        axes = np.atleast_2d(axes)
        for i, inst in enumerate(config.BASEMENT.settings['inst_all']):
            for j, style in enumerate(styles):
                if ('zoom' in style) & (inst in config.BASEMENT.settings['inst_rv']):
                    axes[(i, j)].axis('off')
                elif (inst in config.BASEMENT.settings['inst_phot']) & (companion not in config.BASEMENT.settings['companions_phot']):
                    axes[(i, j)].axis('off')
                elif (inst in config.BASEMENT.settings['inst_rv']) & (companion not in config.BASEMENT.settings['companions_rv']):
                    axes[(i, j)].axis('off')
                else:
                    plot_1(axes[(i, j)], samples, inst, companion, style)

    plt.tight_layout()
    return (fig, axes)

def plot_1(ax, samples, inst, companion, style, timelabel='Time'):
    """
    Inputs:
    -------
    ax : matplotlib axis

    samples : array
        Prior or posterior samples to plot the fit from

    inst: str
        Name of the instrument (e.g. 'TESS')

    companion : None or str
        None or 'b'/'c'/etc.

    style:
        'full' / 'phase' / 'phasezoom'

    timelabel:
        'Time' / 'Time_since'

    Notes:
    ------
    yerr / epoch / period:
        come from the initial_guess value or the MCMC median (not from individual samples)

    """
    params_median, params_ll, params_ul = get_params_from_samples(samples)
    if inst in config.BASEMENT.settings['inst_phot']:
        key = 'flux'
        ylabel = 'Relative Flux'
    elif inst in config.BASEMENT.settings['inst_rv']:
        key = 'rv'
        ylabel = 'RV (km/s)'
    else:
        raise ValueError('inst should be listed in inst_phot or inst_rv...')
    if samples.shape[0] == 1:
        alpha = 1.0
    else:
        alpha = 0.1
    if style == 'full':
        x = config.BASEMENT.data[inst]['time']
        if timelabel == 'Time_since':
            x = np.copy(x)
            objttime = Time(x, format='jd', scale='utc')
            xsave = np.copy(x)
            x -= x[0]
        y = config.BASEMENT.data[inst][key]
        yerr_w = calculate_yerr_w(params_median, inst, key)
        ax.errorbar(x, y, yerr=yerr_w, fmt='b.', capsize=0, rasterized=True)
        if config.BASEMENT.settings['color_plot']:
            ax.scatter(x, y, c=x, marker='o', rasterized=True, cmap='inferno', zorder=11)
        if timelabel == 'Time_since':
            ax.set(xlabel='Time since %s [days]' % objttime[0].isot[:10], ylabel=ylabel, title=inst)
        else:
            if timelabel == 'Time':
                ax.set(xlabel='Time (BJD)', ylabel=ylabel, title=inst)
            if x[(-1)] - x[0] < 1:
                dt = 0.0013888888888888887
            else:
                dt = 0.020833333333333332
            xx = np.arange(x[0], x[(-1)] + dt, dt)
            for i in range(samples.shape[0]):
                s = samples[i, :]
                p = update_params(s)
                model = calculate_model(p, inst, key, xx=xx)
                baseline = calculate_baseline(p, inst, key, xx=xx)
                ax.plot(xx, model + baseline, 'r-', alpha=alpha, rasterized=True, zorder=12)

        if timelabel == 'Time_since':
            x = np.copy(xsave)
    else:
        x = config.BASEMENT.data[inst]['time']
        baseline_median = calculate_baseline(params_median, inst, key)
        y = config.BASEMENT.data[inst][key] - baseline_median
        yerr_w = calculate_yerr_w(params_median, inst, key)
        if 'phasezoom' in style:
            zoomfactor = params_median[(companion + '_period')] * 24.0
        else:
            zoomfactor = 1.0
        if inst in config.BASEMENT.settings['inst_rv']:
            for other_companion in config.BASEMENT.settings['companions_rv']:
                if companion != other_companion:
                    model = rv_fct(params_median, inst, other_companion)[0]
                    y -= model

            phase_time, phase_y, phase_y_err, _, phi = lct.phase_fold(x, y, params_median[(companion + '_period')], params_median[(companion + '_epoch')], dt=0.002, ferr_type='meansig', ferr_style='sem', sigmaclip=False)
            if len(x) > 500:
                ax.plot(phi * zoomfactor, y, 'b.', color='lightgrey', rasterized=True)
                ax.errorbar(phase_time * zoomfactor, phase_y, yerr=phase_y_err, fmt='b.', capsize=0, rasterized=True, zorder=11)
            else:
                ax.errorbar(phi * zoomfactor, y, yerr=yerr_w, fmt='b.', capsize=0, rasterized=True, zorder=11)
            ax.set(xlabel='Phase', ylabel=ylabel + '- Baseline', title=inst + ', companion ' + companion + ' only')
            xx = np.linspace(-0.25, 0.75, 1000)
            for i in range(samples.shape[0]):
                s = samples[i, :]
                p = update_params(s, phased=True)
                model = rv_fct(p, inst, companion, xx=xx)[0]
                ax.plot(xx * zoomfactor, model, 'r-', alpha=alpha, rasterized=True, zorder=12)

        elif inst in config.BASEMENT.settings['inst_phot']:
            if 'phase_variation' in style:
                dt = 0.01
            else:
                dt = 0.002
            phase_time, phase_y, phase_y_err, _, phi = lct.phase_fold(x, y, params_median[(companion + '_period')], params_median[(companion + '_epoch')], dt=dt, ferr_type='meansig', ferr_style='sem', sigmaclip=False)
            if len(x) > 500:
                if 'phase_variation' not in style:
                    ax.plot(phi * zoomfactor, y, 'b.', color='lightgrey', rasterized=True)
                    ax.errorbar(phase_time * zoomfactor, phase_y, yerr=phase_y_err, fmt='b.', capsize=0, rasterized=True, zorder=11)
                else:
                    ax.plot(phase_time * zoomfactor, phase_y, 'b.', rasterized=True, zorder=11)
            else:
                ax.errorbar(phi * zoomfactor, y, yerr=yerr_w, fmt='b.', capsize=0, rasterized=True, zorder=11)
                if config.BASEMENT.settings['color_plot']:
                    ax.scatter(phi * zoomfactor, y, c=x, marker='o', rasterized=True, cmap='inferno', zorder=11)
                ax.set(xlabel='Phase', ylabel=ylabel + '- Baseline', title=inst + ', companion ' + companion)
                if style == 'phasezoom':
                    xx = np.linspace(-4.0 / zoomfactor, 4.0 / zoomfactor, 1000)
                elif style == 'phasezoom_occ':
                    xx = np.linspace((-4.0 + zoomfactor / 2.0) / zoomfactor, (4.0 + zoomfactor / 2.0) / zoomfactor, 1000)
                else:
                    xx = np.linspace(-0.25, 0.75, 1000)
                for i in range(samples.shape[0]):
                    s = samples[i, :]
                    p = update_params(s, phased=True)
                    model = flux_fct(p, inst, companion, xx=xx)
                    ax.plot(xx * zoomfactor, model, 'r-', alpha=alpha, rasterized=True, zorder=12)

        if 'phasezoom' in style:
            ax.set(xlim=[-4, 4], xlabel='$\\mathrm{ T - T_0 \\ (h) }$')
        if 'phasezoom_occ' in style:
            ax.set(xlim=[-4 + zoomfactor / 2.0, 4 + zoomfactor / 2.0], ylim=[0.999, 1.0005], xlabel='$\\mathrm{ T - T_0 \\ (h) }$')
        if 'phase_variation' in style:
            ax.set(ylim=[0.9999, 1.0001])

def get_params_from_samples(samples):
    """
    read MCMC results and update params
    """
    theta_median = np.percentile(samples, 50, axis=0)
    theta_ul = np.percentile(samples, 16, axis=0)
    theta_ll = np.percentile(samples, 84, axis=0)
    params_median = update_params(theta_median)
    params_ll = update_params(theta_ll)
    params_ul = update_params(theta_ul)
    return (
     params_median, params_ll, params_ul)

def save_table(samples, mode):
    """
    Inputs:
    -------
    samples : array
        posterior samples
    mode : string
        'mcmc' or 'ns'
    """
    params, params_ll, params_ul = get_params_from_samples(samples)
    with open(os.path.join(config.BASEMENT.outdir, mode + '_table.csv'), 'w') as (f):
        f.write('############ Fitted parameters ############\n')
        for i, key in enumerate(config.BASEMENT.allkeys):
            if key not in config.BASEMENT.fitkeys:
                f.write(key + ',' + str(params[key]) + ',' + '(fixed),\n')
            else:
                f.write(key + ',' + str(params[key]) + ',' + str(params_ll[key]) + ',' + str(params_ul[key]) + '\n')

def save_latex_table(samples, mode):
    """
    Inputs:
    -------
    samples : array
        posterior samples
    mode : string
        'mcmc' or 'ns'
    """
    params_median, params_ll, params_ul = get_params_from_samples(samples)
    label = 'none'
    with open(os.path.join(config.BASEMENT.outdir, mode + '_latex_table.txt'), 'w') as (f):
        with open(os.path.join(config.BASEMENT.outdir, mode + '_latex_cmd.txt'), 'w') as (f_cmd):
            f.write('parameter & value & unit & fit/fixed \\\\ \n')
            f.write('\\hline \n')
            f.write('\\multicolumn{4}{c}{\\textit{Fitted parameters}} \\\\ \n')
            f.write('\\hline \n')
            for i, key in enumerate(config.BASEMENT.allkeys):
                if key not in config.BASEMENT.fitkeys:
                    value = str(params_median[key])
                    f.write(config.BASEMENT.labels[i] + ' & $' + value + '$ & ' + config.BASEMENT.units[i] + '& (fixed) \\\\ \n')
                    f_cmd.write('\\newcommand{\\' + key.replace('_', '') + '}{' + label + '$=' + value + '$} \n')
                else:
                    value = latex_printer.round_tex(params_median[key], params_ll[key], params_ul[key])
                    f.write(config.BASEMENT.labels[i] + ' & $' + value + '$ & ' + config.BASEMENT.units[i] + '& \\\\ \n')
                    f_cmd.write('\\newcommand{\\' + key.replace('_', '') + '}{' + label + '$=' + value + '$} \n')

def show_initial_guess(do_logprint=True, initial_guess_plot=True, return_figs=False):
    """
    Inputs:
    -------
    datadir : str
        the working directory for allesfitter
        must contain all the data files
        output directories and files will also be created inside datadir

    Outputs:
    --------
    This will output information into the console,
    and create a file called datadir/results/initial_guess.pdf
    """
    if do_logprint:
        logprint('\nSettings:')
        logprint('--------------------------')
        for key in config.BASEMENT.settings:
            if config.BASEMENT.settings[key] != '':
                logprint(('{0: <30}').format(key), ('{0: <15}').format(str(config.BASEMENT.settings[key])))
            else:
                logprint(('\n{0: <30}').format(key))

        logprint('\nParameters:')
        logprint('--------------------------')
        for i, key in enumerate(config.BASEMENT.params):
            if key in config.BASEMENT.fitkeys:
                ind = np.where(config.BASEMENT.fitkeys == key)[0][0]
                logprint(('{0: <30}').format(key), ('{0: <15}').format(str(config.BASEMENT.params[key])), ('{0: <5}').format('free'), ('{0: <30}').format(str(config.BASEMENT.bounds[ind])))
            elif config.BASEMENT.params[key] != '':
                logprint(('{0: <30}').format(key), ('{0: <15}').format(str(config.BASEMENT.params[key])), ('{0: <5}').format('set'))
            else:
                logprint(('\n{0: <30}').format(key))

        logprint('\nndim:', config.BASEMENT.ndim)
    if initial_guess_plot == True:
        samples = draw_initial_guess_samples()
        if return_figs == False:
            for companion in config.BASEMENT.settings['companions_all']:
                fig, axes = afplot(samples, companion)
                fig.savefig(os.path.join(config.BASEMENT.outdir, 'initial_guess_' + companion + '.pdf'), bbox_inches='tight')
                plt.close(fig)

        else:
            fig_list = []
            for companion in config.BASEMENT.settings['companions_all']:
                fig, axes = afplot(samples, companion)
                fig_list.append(fig)

            return fig_list

def get_labels(datadir, as_type='dic'):
    config.init(datadir)
    if as_type == '2d_array':
        return config.BASEMENT.labels
    if as_type == 'dic':
        labels_dic = {}
        for key in config.BASEMENT.fitkeys:
            ind = np.where(config.BASEMENT.allkeys == key)[0]
            labels_dic[key] = config.BASEMENT.labels[ind][0]

        return labels_dic


#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
#reload(sys)
import importlib
importlib.reload(sys)
try:
    from PyQt5.QtGui import *
    from PyQt5.QtCore import *
    from PyQt5.QtWidgets import *
except ImportError:
    # needed for py3+qt4
    # ref: http://pyqt.sourceforge.net/Docs/PyQt4/incompatible_apis.html
    # ref:
    # http://stackoverflow.com/questions/21217399/pyqt4-qtcore-qvariant-object-instead-of-a-string
    if sys.version_info.major >= 3:
        import sip
        sip.setapi('QVariant', 2)
    from PyQt4.QtGui import *
    from PyQt4.QtCore import *

HEAD_COLOR = QColor(255, 0, 0)
COVER_COLOR = QColor(0, 255, 0)
RANGE_COLOR = QColor(255, 255, 184)
DISABLED_COLOR = QColor(255, 0, 0)
PEN_COLOR = QColor(255, 255, 255)
HIGHLIGHT_COLOR = QColor(0, 0, 255)

class RangeSlider(QWidget):
    OVER_CAPACITY = 100
    splitSize = 10
    releaseCallback = None

    def __init__(self, *args, **kwargs):
        super(RangeSlider, self).__init__(*args, **kwargs)
        self.initUI()

    def initUI(self):
        self.setMinimumSize(1, 30)
        self.value = self.OVER_CAPACITY
        self.bound = 6.0
        self.lPos = 0
        self.rPos = 0
        self.pos = 0
        self.description = ''
        self.inBounding = False
        self.inControl = 'Other'
        self.num = []
        self.setCapacity(self.OVER_CAPACITY)
        self.setPosition(0, 0)
        self.setEnabled(False)

    def setValue(self, value):
        self.rPos = self.fixedRange('Right', self.rangeToPos(value))

    def setCurrentPosition(self, value):
        self.pos = self.rangeToPos(value)
        self.repaint()

    def setCapacity(self, value):
        self.OVER_CAPACITY = value
        self.setSplitNum(self.OVER_CAPACITY)
        self.lPos = 0
        self.rPos = self.rangeToPos(self.OVER_CAPACITY)

    def setSplitNum(self, capacity):
        self.num = []
        for i in range(1, self.splitSize, 1):
            self.num.append(int(capacity * float(i) / float(self.splitSize)))

    def setEventDescription(self, description):
        self.description = description
        self.repaint()

    def paintEvent(self, e):
        qp = QPainter()
        qp.begin(self)
        self.drawWidget(qp)
        qp.end()

    def setReleaseCallback(self, e):
        self.releaseCallback = e

    def mousePressEvent(self, ev):
        value = ev.pos().x()
        clickBound = self.inBoundRange(value)
        if clickBound in ['Right', 'Left']:
            self.inBounding = True
            self.inControl = clickBound
        else:
            self.inBounding = False
            self.inControl = None

    def mouseMoveEvent(self, ev):
        if self.inBounding and self.inControl is 'Right':
            self.rPos = float(ev.pos().x())
            self.setValue(self.posToRange(self.rPos))
            self.repaint()
        elif self.inBounding and self.inControl is 'Left':
            self.lPos = float(ev.pos().x())
            self.repaint()

    def mouseReleaseEvent(self, ev):
        if self.releaseCallback is not None:
            if self.inBounding and self.inControl is 'Right':
                self.releaseCallback(('End', self.posToRange(self.rPos)))
            elif self.inBounding and self.inControl is 'Left':
                self.releaseCallback(('Start', self.posToRange(self.lPos)))

    def setPosition(self, lPos, rPos):
        lPos = lPos if lPos < rPos else rPos
        lPos = self.rangeToPos(lPos)
        rPos = self.rangeToPos(rPos)
        self.lPos = lPos
        self.rPos = rPos
        self.repaint()

    def posToRange(self, value):
        width = self.size().width()
        return int(float(value) * (self.OVER_CAPACITY / float(width)))

    def rangeToPos(self, value):
        width = self.size().width()
        return int(float(value) * (float(width) / self.OVER_CAPACITY))

    def drawWidget(self, qp):

        def drawText(position, text):
            pen = QPen(QColor(20, 20, 20), 1, Qt.SolidLine)
            qp.setPen(pen)
            qp.setBrush(Qt.NoBrush)
            metrics = qp.fontMetrics()
            fw = metrics.width(str(text))
            qp.drawText(position-fw/2, h/2, str(text))

        def drawTitle(position, text, size):
            o_font = qp.font()
            font = QFont('Serif', size, QFont.Light)
            pen = QPen(QColor(20, 20, 20), 1, Qt.SolidLine)
            qp.setFont(font)
            qp.setPen(pen)
            qp.setBrush(Qt.NoBrush)
            metrics = qp.fontMetrics()
            fw = metrics.width(str(text))
            qp.drawText(position-fw/2, h-5, str(text))
            qp.setFont(o_font)

        font = QFont('Serif', 7, QFont.Light)
        qp.setFont(font)

        size = self.size()
        w = size.width()
        h = size.height()

        step = int(round(w / 10))
        bound = self.bound
        hBound = self.bound / 2
        rPos = self.rPos
        lPos = self.lPos
        if self.inControl is 'Right':
            rPos = self.fixedRange('Right', self.rPos)
        elif self.inControl is 'Left':
            lPos = self.fixedRange('Left', self.lPos)
        self.lPos = lPos
        self.rPos = rPos
        lValue = int(lPos)
        rValue = int(rPos)
        Range = rValue - lValue

        if self.isEnabled():
            if Range >= bound:
                qp.setPen(PEN_COLOR)
                qp.setBrush(HEAD_COLOR)
                qp.drawRect(lValue - hBound, 0, Range + bound, h)
                if Range - bound > 0:
                    qp.setPen(PEN_COLOR)
                    qp.setBrush(RANGE_COLOR)
                    qp.drawRect(lValue + hBound, 0, Range - bound, h)
            else:
                qp.setPen(PEN_COLOR)
                qp.setBrush(HEAD_COLOR)
                qp.drawRect(lValue - hBound, 0, Range + bound, h)
                qp.setPen(PEN_COLOR)
                qp.setBrush(COVER_COLOR)
                qp.drawRect(float(lValue) + float(Range) / 2 - (float(bound - Range) / 2), 0, float(bound - Range), h)
        else:
            qp.setPen(PEN_COLOR)
            qp.setBrush(DISABLED_COLOR)
            qp.drawRect(lValue - hBound, 0, Range + bound, h)

        qp.setPen(PEN_COLOR)
        qp.setBrush(HIGHLIGHT_COLOR)
        qp.drawRect(self.pos - hBound, 0, bound, h)

        if len(self.description) > 0:
            drawTitle(lValue + float(Range) / 2, self.description, 10)

        pen = QPen(QColor(20, 20, 20), 1,
            Qt.SolidLine)

        qp.setPen(pen)
        qp.setBrush(Qt.NoBrush)
        qp.drawRect(0, 0, w-1, h-1)

        if len(self.num) > 0:
            j = 0
            for i in range(step, self.splitSize * step, step):
                qp.drawLine(i, 0, i, 5)
                drawText(i, str(self.num[j]))
                j = j + 1

    def fixedRange(self, posType, value):
        lPos = self.lPos
        rPos = self.rPos
        fixedValue = value
        rBound = self.rangeToPos(int(self.OVER_CAPACITY))
        if posType is 'Right':
            if value <= lPos:
                fixedValue = int(lPos)
            elif value > rBound:
                fixedValue = int(rBound)
        elif posType is 'Left':
            if value >= rPos:
                fixedValue = int(rPos)
            elif value < 0:
                fixedValue = int(0)
        return fixedValue

    def inBoundRange(self, value):
        w = self.size().width()
        rPos = self.rPos
        lPos = self.lPos
        hBound = self.bound / 2
        rIn = rPos - hBound <= value and value <= rPos + hBound
        lIn = lPos - hBound <= value and value <= lPos + hBound
        rhIn = float(w) - hBound <= value and value <= float(w) + hBound
        lhIn = 0 - hBound <= value and value <= 0 + hBound
        if (rIn and not lIn) or (rIn and lIn and lhIn) or (rIn and lIn and not lhIn and not rhIn):
            return "Right"
        elif (lIn and not rIn) or (lIn and rIn and rhIn):
            return "Left"
        return "Other"


import pandas as pd
import numpy as np
import tensorflow as tf
import keras
from keras.models import Sequential
from keras.layers.core import Dense, Activation, Flatten, Dropout
from keras.layers.convolutional import Convolution2D
from keras.layers.pooling import MaxPooling2D
import os
import matplotlib.pyplot as plt
from scipy import misc

### Experiments
# 1) Using only center images work better
# 2) Udacity data doesn't improve ?
# 3) A wider network doesn't seem to work. (24 in deeper layers)
# 4) Using a bigger batch size (around 100 and too big). Might have improved accuracy.
# 5) Dropout? Not really working.
# 6) Adding another fully connected layer (a total of three fc)? No
# 7) Using longer training (10 epochs)? No. Use five????
# 8) USE VALID PADDING !!!!!!
# 9) Increasing steering correction from 0.10 to 0.27... YEAHHH it worked
# 10) Crop top and bottom of image? Overshoots with steering correcion 0.27. NOT WORKING
# 11) Dropout? Not working

### Load data ##############################################################
# Bool to select remote or host computing.
useRemote = False
if useRemote == True:
    dirDataOut = "/media/slow-storage/DataFolder/Peter/SelfDriving/CollectedSets"
else:
    dirDataOut = "/home/repete/DataFolder/SelfDriving/CollectedSets"

# The number of epochs used in training.
nEpochs = 8

# Parameters for selecting subsets of the data.
# The image and csv-files have been pre-generated have been generated in a another script.
# The scripts resamples images by 2 and I have also smoothened steering angles (I did not used a joystick)
selectedNames = sorted(['peter','mikkel','udacity']) # Data to be included.
smoothSteering = sorted(['peter','mikkel','udacity'])
# An image resizing of 2 have been used.
imageResizing = 2
# Steerings angles have been smoothened with a average kernel of nSize.
nSize = 5

# Loads a specific data-folder based on above specification.
inputName = "Resizing" + str(imageResizing) + "_AddedData" + ''.join(selectedNames) + '_SmoothSteeringAmount' + str(nSize) + "To" +''.join(smoothSteering)
# Dir of images.
dirInputImages = os.path.join(dirDataOut,inputName,"IMG")
# Dir of cvs-files with data.
data = pd.read_csv(os.path.join(dirDataOut,inputName,'driving_log.csv'))
# Select batch size.
batchSize = 100

### Split data into train, validation and test set.
# Random indices are generated to select random data for train, validation and test set.
randIndex = np.random.permutation(data.shape[0])
# Select the amount of data used for validation and test set.
pVal = 0.1
pTest = 0.1
dataTrain = data.reindex([randIndex[0                           :data.shape[0]*(1-pVal-pTest)]])
dataVal = data.reindex(  [randIndex[data.shape[0]*(1-pVal-pTest):data.shape[0]*(1-pTest)]])
dataTest = data.reindex( [randIndex[data.shape[0]*(1-pTest)     :]])

print("Total number of samples: ",data.shape[0]) # Number of training samples:  12562
print("Number of training samples: ",dataTrain.shape[0]) # Number of training samples:  12562
print("Number of validation samples: ",dataVal.shape[0]) # Number of validation samples:  1570
print("Number of test samples: ",dataTest.shape[0])      # Number of test samples:  1571

# A python generator is used for selecting batches used in training, validation and test.
# Bools specify wether to use normalization, mirroring and left/right cameras.
def generatorGetBatch(data,dirInputImages,batchSize,normalizePerImage=True, plotting=False, doMirror = True, useLeftRight = True):
    # SteeringCorrection.
    # a) Added to the steeringAngle, when the left camera is used.
    # b) Subtracted from the steeringAngle, when the left camera is used.
    steeringCorrection = 0.27

    # Bool to specify if crop top and bottom of input images.
    # This haven't improve performanced and therefore not included.
    doCrop = False

    imgNames = ['center','left','right']
    # The while-True loop ensured that the generator will never run out of samples.
    while True:
        # Data is shuffled for every epoch.
        dataShuffled = data.reindex(np.random.permutation(data.index))

        # For every next()-call a batch is collected and returned using the yield-cmd.
        for iBatch in range(0,data.shape[0],batchSize):
            # Lists for appending images (X) and steeringsangles (y) to a batch.
            X = []
            y = []
            augType = []

            # Create a batch
            dfBatch = dataShuffled.iloc[iBatch:iBatch+batchSize]

            # Run through all samples in the batch.
            for _, out in dfBatch.iterrows():
                # Steering angle
                steering = out['steering']

                # Use left and right camera.
                if useLeftRight:
                    # center, left or right is selected at random.
                    augmentationType = imgNames[np.random.randint(3)]
                    # Steering correction is perform for only left and right camera.
                    if(augmentationType=="left"):
                        steering = steering+steeringCorrection
                    if(augmentationType=="right"):
                        steering = steering-steeringCorrection
                else:
                    # Only center camera is used. imgName[0] = 'center'
                    augmentationType = imgNames[0] # Use always center.

                # The image directory
                dirImg = out[augmentationType]
                dirImg = os.path.join(dirInputImages,dirImg.split("/")[-1])

                # Reads img.
                img = misc.imread(dirImg)

                # If enabled: Visualize image.
                if(plotting):
                    plt.figure()
                    plt.imshow(img)

                # If enabled: Performs normalization per-image
                if(normalizePerImage):
                    mu = np.mean(np.reshape(img,(-1,3)),axis=0)
                    sig = np.sqrt(np.var(np.reshape(img,(-1,3)),axis=0))
                    img = (img-mu)/sig

                # If enabled: Img and steering angle is mirrored half the time.
                if doMirror and np.random.randint(2)==0:
                    #mCount = mCount+1
                    #print(mCount)
                    steering = -steering
                    img = np.fliplr(img)

                # If enabled: Crops top (sky) and bottom (car) of the image.
                if doCrop :
                    pTopCrop  = 0.2
                    topCrop = img.shape[0]*pTopCrop
                    pBottomCrop  = 0.1
                    bottomCrop = img.shape[0]-img.shape[0]*pBottomCrop
                    print("Crop")
                    img = img[topCrop:bottomCrop,:,:]
                # Image and steering is appended to lists
                X.append(img)
                y.append(steering)
                augType.append(augmentationType)
            # Images and steering is converted into a numpy-array and returned.
            yield np.array(X), np.array(y)

# Test and visualize for a single batch
test = generatorGetBatch(data,dirInputImages,10,True,False,True,True)
#test = overFitBatch(data,10,True,True )
for iBatch in range(0,1):
    Xtest,ytest = next(test)

dimX = Xtest.shape[1:4]
img = Xtest[0,:,:,:]

### Create network #########################################################
if(False):

    model = Sequential()
    # Valid padding have been used for all convolutions.
    padType = 'valid'

    nKernels = 12
    kernelSize = 5
    # Apply a 5x5 convolution with 12 output filters on input image
    model.add(Convolution2D(nKernels, kernelSize, kernelSize, border_mode=padType, input_shape=(dimX)))
    model.add(Activation('relu'))

    # Apply max-pooling for subsamling the image.
    model.add(MaxPooling2D((2, 2)))

    nKernels = 24
    kernelSize = 5
    # Apply a 5x5 convolution with 24 output filters on input image
    model.add(Convolution2D(nKernels, kernelSize, kernelSize, border_mode=padType))
    model.add(Activation('relu'))

    # Apply max-pooling for subsamling the image.
    model.add(MaxPooling2D((2, 2)))

    nKernels = 24
    kernelSize = 3
    # Apply a 3x3 convolution with 24 output filters on input image
    model.add(Convolution2D(nKernels, kernelSize, kernelSize, border_mode=padType))
    model.add(Activation('relu'))

    # Apply max-pooling for subsamling the image.
    model.add(MaxPooling2D((2, 2)))

    nKernels = 24
    kernelSize = 3
    # Apply a 3x3 convolution with 24 output filters on input image
    model.add(Convolution2D(nKernels, kernelSize, kernelSize, border_mode=padType))
    model.add(Activation('relu'))

    # Apply max-pooling for subsamling the image.
    model.add(MaxPooling2D((2, 2)))

    # Feature map returned for the fully convolutional part of the network is flattened into a 1D-vector.
    model.add(Flatten())

    # A fully connected layer with 200 neurons/kernels with dropout.
    model.add(Dense(200))
    model.add(Activation('relu'))
    model.add(Dropout(0.10))

    # A fully connected layer with 200 neurons/kernels with dropout.
    model.add(Dense(100))
    model.add(Activation('relu'))
    model.add(Dropout(0.10))

    # The output must be a single value (a single neuron) to control the vehicle.
    model.add(Dense(1))

    # The mean-squared-error is used as a loss-function with an adam-optimizer.
    model.compile(loss='mean_squared_error', optimizer='adam')
    model.summary()

    # fit_generator is used for training the model.
    model.fit_generator(generatorGetBatch(dataTrain,dirInputImages,batchSize,True,False,True,True),samples_per_epoch=dataTrain.shape[0], nb_epoch=nEpochs,validation_data=generatorGetBatch(dataTest,dirInputImages,batchSize,True,False,True,True),nb_val_samples=dataTest.shape[0])

    ### Model is stored-
    # serialize model to JSON
    model_json = model.to_json()
    with open("model.json", "w") as json_file:
        json_file.write(model_json)
    # serialize weights to HDF5
    model.save_weights("model.h5")
    print("Saved model to disk")


import io
import multiprocessing as mp
import sys
import tkinter as tk
import tkinter.ttk as ttk

from config import Config
from os import path
from tkinter import messagebox
from frames.frame_patch import PatchExecutableFrame
from frames.frame_download import DownloadTranslationsFrame
from frames.frame_translate_external_files import TranslateExternalFiles

class MainWindow(tk.Tk):
    def save_settings_repeatedly(self, config, delay=500):
        if hasattr(self, 'notebook'):
            nb = self.notebook
            if nb.tabs():
                config['last_tab_opened'] = nb.tabs().index(nb.select())

        self.after(delay, self.save_settings_repeatedly, config, delay)
        config.save_settings()

    def init_notebook(self, config: Config):
        notebook = ttk.Notebook()
        notebook.pack(fill='both', expand=1)

        notebook.add(DownloadTranslationsFrame(notebook, config), text='Download translations')
        notebook.add(PatchExecutableFrame(notebook, config, debug=self.app.debug), text='Patch executable file')
        notebook.add(TranslateExternalFiles(notebook, config), text='Translate external text files')

        tab = config['last_tab_opened']
        if 0 <= tab < len(notebook.tabs()):
            notebook.select(tab)

        return notebook

    def check_for_errors(self, stderr, delay=100):
        if stderr.getvalue():
            messagebox.showerror('Unhandled Exception', stderr.getvalue())
            stderr.truncate(0)
            stderr.seek(0)
        self.after(delay, self.check_for_errors, stderr, delay)

    def init_error_handler(self):
        executable = path.split(sys.executable)[1]
        if executable.startswith('pythonw') or not executable.startswith('python'):  # if no console attached
            stderr = io.StringIO()
            sys.stderr = stderr
            self.check_for_errors(stderr)

    def __init__(self, app):
        super().__init__()
        self.app = app
        self.init_error_handler()
        self.notebook = self.init_notebook(self.app.config)

class App:
    def init_config(self):
        config_name = '.df-translate.json'
        user_directory = path.expanduser('~')

        default_config = dict(last_tab_opened=0)
        config = Config(default_config)

        if not self.ignore_config_file:
            config_path = path.join(user_directory, config_name)
            config.load_settings(config_path)

        return config

    def autosave_config(self):
        if not self.ignore_config_file:
            self.window.bind('<Destroy>', lambda _: self.config.save_settings())  # Save settings on quit
            self.window.save_settings_repeatedly(self.config, delay=500)  # Save settings every 500 ms

    def __init__(self, ignore_config_file=False, debug=False):
        self.ignore_config_file = ignore_config_file
        self.debug = debug
        self.config = self.init_config()
        self.window = MainWindow(self)
        self.autosave_config()
        self.window.mainloop()

if __name__ == '__main__':
    mp.freeze_support()
    App(ignore_config_file='--noconfig' in sys.argv, debug='--debug' in sys.argv)


import scrapy

from ..utils.parsers import Parsers
from ..utils.write_excel import ExcelRW

class BookSpider(scrapy.Spider):
    name = 'BuyBook'

    params = {'keywords': '解忧杂货店'}
    allow_domains = [
        'https://www.amazon.cn/s',
        'http://search.dangdang.com/',
        'https://search.jd.com/Search'
    ]
    amazon_url = allow_domains[0] + '?k=' + params['keywords'] + '&i=stripbooks'

    dd_url = allow_domains[1] + '?key=' + params['keywords'] + '&act=input'

    jd_url = allow_domains[2] + '?enc=utf-8&keyword=' + params['keywords']

    urls = [amazon_url, dd_url, jd_url]

    def start_requests(self):
        for url in self.urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        print('============================================', response)
        parsers = Parsers()
        excel = ExcelRW()
        page = response.url.split("/")[2]
        excel_path = './static_files/excels/'
        header_data = [['书名', 50], ['作者', 40], ['价格', 10], ['出版社', 40], ['时间', 10],
                       ['星级', 10], ['评价数', 10], ['详情链接', 50]]
        if page == 'www.amazon.cn':
            # 保存当页数据
            data = parsers.get_amaze_data(response)
            excel.save_excel('Amazon Book', data, excel_path + self.params['keywords'] + '_book.xlsx', header_data)

            # 爬取下一页
            max_page = response.xpath('//ul[@class="a-pagination"]/li/text()').extract()[-1]
            try:
                max_page = int(max_page) if max_page and int(max_page) else 0
                for i in range(2, max_page + 1):
                    next_page = response.urljoin(self.urls[0] + '&page=' + str(i))
                    yield scrapy.Request(next_page, callback=self.parse)
            except Exception as e:
                print('FBI WARNING!!!!!!:', e)

        elif page == 'search.dangdang.com':
            data = parsers.get_dd_data(response)
            excel.save_excel('DangDang Book', data, excel_path + self.params['keywords'] + '_book.xlsx', header_data)

            max_page = int(response.xpath('//div[@class="paging"]/ul/li/a/text()').extract()[-2])
            for i in range(2, max_page + 1):
                next_page = response.urljoin(self.urls[1] + '&page_index=' + str(i))
                yield scrapy.Request(next_page, callback=self.parse)

        elif page == 'search.jd.com':
            data = parsers.get_jd_data(response)
            excel.save_excel('JD Book', data, excel_path + self.params['keywords'] + '_book.xlsx', header_data)

            # 京东页码比较特别，比如：page=1返回第一组数据显示在第一页，page=2时返回第二组数据但是与第一组数据一起显示在第一页，以此类推
            max_page = int(response.xpath('//div[@id="J_topPage"]/span[@class="fp-text"]/i/text()').extract_first()) * 2
            for i in range(4, max_page, 2):
                next_page = response.urljoin(self.urls[2] + '&page=' + str(i))
                yield scrapy.Request(next_page, callback=self.parse)

        # 写入html文件
        # filename = '../../static_files/web/' + page + ".html"
        # with open(filename, 'ab') as f:
        #     if page == 'www.amazon.cn':
        #         f.write(response.xpath('//ul[@id="s-results-list-atf"]').extract_first(default='not-found').encode())
        #     elif page == 'search.dangdang.com':
        #         f.write(response.xpath('//div[@id="search_nature_rg"]/ul').extract_first(default='not-found').encode())
        #     elif page == 'search.jd.com':
        #         f.write(response.xpath('//div[@id="J_goodsList"]').extract_first(default='not-found').encode())

        # data = [{'book_name': '', 'author': '', 'money': float(0),
        #          'publisher': '', 'time': '', 'like': int(''), 'buy_count': int('')}]


import sys
from pgu import gui
import pygame
import threading
import websocket
from client.ws_events import *

RESX = 600
RESY = 400
FPS = 40

CONNECT_SUCCESS = False
#

# class App(websocket.WebSocketApp):
#     def send(self, data, opcode=0x1):
#         super().send(data, opcode=opcode)

def on_message(ws, message):
    print(type(message))
    print(message)
    if message != 'Sorry':
        win._quit = True
        main_thread.start()
        login = input_login.value
        # message = {"login": login}
        message = "login"
        # message = "Exit"
        # ws.send(json.dumps(message))
        ws.send(message)
    else:
        input_error.value = message
        ws.close()
        # ws = websocket.WebSocketApp("ws://127.0.1.1:8888/websocket", on_message=on_message)
    # chat.value += message + '\n'

# Должна отправлять сообщения
def on_btn_send(value):
    mail = input_field.value
    chat.value += mail + '\n'
    ws.send(mail)
    input_field.value = ''

def on_btn_connect(btn_event):
    ws_thread.start()
    login = input_login.value

# First Window
screen = pygame.display.set_mode((RESX, RESY))

win = gui.Desktop()
win.connect(gui.QUIT, win.quit, None)

#
sign = gui.Table()
label_login = gui.Label("nick")
label_error = gui.Label("error")
input_login = gui.Input(size=20)
button_connect = gui.Button("Connect")
input_error = gui.Input(size=20)

# Стол
sign.tr()
sign.td(label_login)
sign.td(input_login)
sign.td(button_connect)
sign.tr()
sign.td(label_error)
sign.td(input_error)
button_connect.connect(gui.CLICK, on_btn_connect, "Connect")

win.init(widget=sign)

# INIT

# Second Window
app = gui.Desktop()
app.connect(gui.QUIT, app.quit, None)

# SOCKETS

button_send = gui.Button('Send message')
grid = c = gui.Table()
input_field = gui.Input(size=20)
chat = gui.TextArea()
button_quit = gui.Button("Quit")

# Размер чата
chat.style.height = 200
chat.style.width = 300

# Стол
grid.tr()
grid.td(input_field)
grid.tr()
grid.td(chat)
grid.tr()
grid.td(button_send)

button_send.connect(gui.CLICK, on_btn_send, "Send")
app.init(widget=grid)
clock = pygame.time.Clock()

def main():
    done = True
    while done:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                print("!!")
                done = False
                ws.close()
                break
            else:
                app.event(event)

        dt = clock.tick(FPS)
        screen.fill((0, 0, 0))
        app.paint()
        pygame.display.flip()

# Так надо
ws = websocket.WebSocketApp("ws://127.0.1.1:8888/websocket", on_data=on_data, on_error=on_errors)
# init threads
ws_thread = threading.Thread(target=ws.run_forever)
main_thread = threading.Thread(target=main)

win.run()


from rest_framework import serializers
from . import models

class MeasurementSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.Measurement
        fields = ('name',)

class IngredientSerializer(serializers.ModelSerializer):
    class Meta:
        model = models.Ingredient
        fields = ('name',)

class RecipeIngredientSerializer(serializers.ModelSerializer):
    ingredient = IngredientSerializer(read_only=True)
    unit = MeasurementSerializer(read_only=True)

    class Meta:
        model = models.RecipeIngredient
        fields = ('ingredient', 'amount', 'unit')

class RecipeSummarySerializer(serializers.HyperlinkedModelSerializer):
    """
    Generates a summary variant of the recipe model
    """
    url = serializers.HyperlinkedIdentityField(
        view_name='recipe-detail',
        lookup_field='slug'
    )

    class Meta:
        model = models.Recipe
        fields = (
            'name', 'description', 'cooking_instructions', 'preparation_time', 'cooking_time', 'category', 'course',
            'cuisine', 'created', 'modified', 'url')
        read_only_fields = ['created', 'modified']
        extra_kwargs = {
            'course': {
                'lookup_field': 'slug'
            },
            'cuisine': {
                'lookup_field': 'slug'
            },
            'category': {
                'lookup_field': 'slug'
            }
        }
        lookup_field = 'slug'

class RecipeSerializer(serializers.HyperlinkedModelSerializer):
    """
    A serializer for a full recipe
    """
    url = serializers.HyperlinkedIdentityField(
        view_name='recipe-detail',
        lookup_field='slug'
    )

    ingredients = RecipeIngredientSerializer(many=True, read_only=True)

    class Meta:
        model = models.Recipe
        fields = (
            'name', 'description', 'cooking_instructions', 'preparation_time', 'cooking_time', 'category', 'course',
            'cuisine', 'ingredients', 'created', 'modified', 'url')
        read_only_fields = ['created', 'modified']
        extra_kwargs = {
            'course': {
                'lookup_field': 'slug'
            },
            'cuisine': {
                'lookup_field': 'slug'
            },
            'category': {
                'lookup_field': 'slug'
            }
        }
        lookup_field = 'slug'

class CategorySerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = models.Category
        fields = ('id', 'name',)
        lookup_field = 'slug'

class CuisineSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = models.Cuisine
        fields = ('id', 'name', 'slug')
        lookup_field = 'slug'

class CourseSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = models.Course
        fields = ('id', 'name', 'slug')
        lookup_field = 'slug'


import sqlite3
import os

class TweetCache(object):
	def __init__(self, db = "twitter.db"):
		if not os.path.exists(db):
			build_db = True
		else:
			build_db = False
		self.conn = sqlite3.connect(db)
		self.c = self.conn.cursor()
		if build_db:
			self._build_database()
		self.db = db

	def _build_database():
		self.c.execute("""
		CREATE TABLE users (
			int user_id PRIMARY KEY,
			int friend_count,
			int follower_count,
			text description,
			text url
		)
		""")
		self.c.execute("""
		CREATE TABLE friends (
			int user_id PRIMARY KEY,
			int target_id INDEX,
		)
		""")

	def _check_for_user(user_id):
		self.c.execute("""
		SELECT COUNT(*) from
		users where user_id = ?;
		""", (user_id))
		is_user = self.c.fetchone()
		return is_user

	def is_cached(user_id):
		return self._check_for_user(user_id)

	def friend_info_is_cached(friend_id):
		self.c.execute("""
		SELECT COUNT(*) from
		users where user_id = ?;
		""", (user_id))
		is_user = self.c.fetchone()
		self.c.exexute("""
		SELECT COUNT(*) from
		friends where friend_id = ?;
		""")
		is_friend = self.c.fetchone()
		return us_user and is_friend

	def _add_user(user_obj):
		user_id = user_obj['id']
		friend_count = user_obj['friends_count']
		follower_count = user_obj['followers_count']
		description = user_obj['description']
		url = user_obj['url']
		self.c.execute("""
		INSERT INTO users values(
			?, ?, ?, ?, ?
		)
		""",(user_id, friend_count, follower_count,
			description, url))
		self.conn.commit()

	def cache_user_info(user_obj):
		user_id = user_obj['id']
		user_present = self._check_for_user(user_id)
		if not user_present:
			self._add_user(user_obj)

	def cache_friend_info(user_id, friends):
		pass

if __name__ == "__main__":
	# test creation:
	pass