import torch
from base.models.model_container import ModelContainer
from base.train.metrics import accuracy
from base.data.data_container import DataContainer
from base.train.common_trainer import CommonTrainer
import torch.nn as nn
import torch.optim as optim
import matplotlib.pyplot as plt
import torch.nn.functional as F
import random
from real_cnn_model.train.loss import SoftCrossEntropy, NegLogRatioLoss, KLDivLogits, InfoRadiusLoss
from data.imagenet import mutual_remove

class CNNTrainer(CommonTrainer):
    def __init__(self, cfg, model_container: ModelContainer, data_container: DataContainer, **kwargs):
        super(CNNTrainer, self).__init__(cfg, model_container, data_container)
        print(f'Initializing trainer {self.__class__.__name__}...')
        self.init_env()
        self.init_optimizer()
        self.init_scheduler()
        self.prep_train()
        self.loss_func = nn.CrossEntropyLoss()
        self.debug = getattr(self.cfg, 'debug', False)
        self.debug_input = getattr(self.cfg, 'debug_input', False)
        self.debug_labels = getattr(self.cfg, 'debug_labels', False)

    def init_optimizer(self, **kwargs):
        """
        Initialize self.optimizer using self.cfg.
        """

        # Choose optimizer
        model = self.model_container.models['model']
        opt_type = self.cfg.optimizer
        freeze = getattr(self.cfg, 'freeze', False) or getattr(self.cfg, 'train_classifier', False)

        if opt_type == 'SGD':
            print('Using SGD as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.SGD(model.parameters(), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
        elif opt_type == 'Adam':
            print('Using Adam as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.Adam(model.parameters(), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)

    def train(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']

        self.model_container.set_train(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])

        self.optimizer.zero_grad()

        pred = self.model_container.infer('model', input_data, False)

        loss = self.loss_func(pred, label)

        acc_1, acc_5 = accuracy(pred.cpu(), label.cpu(), topk=(1, min(5, pred.shape[-1])))
        loss.backward()
        self.optimizer.step()

        if self.debug:
            if self.debug_input:
                self.inspect_input(input_data)
            if self.debug_labels:
                self.inspect_labels(pred, label, acc_1)

        return loss.item(), acc_1, acc_5

    def test(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']

        self.model_container.set_eval(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])

        pred = self.model_container.infer('model', input_data, True)
        tot_num = len(pred)

        acc_1, acc_5 = accuracy(pred.cpu(), label.cpu(), topk=(1, min(5, pred.shape[-1])))

        num_correct = int(acc_1 * tot_num)

        if self.debug:
            if self.debug_input:
                self.inspect_input(input_data)
            if self.debug_labels:
                self.inspect_labels(pred, label, acc_1)

        return acc_1, acc_5, num_correct, tot_num

    def save_model(self, total_epoch, total_iter, total_val_iter, **kwargs):
        """
        Save model on self.exp_save_dir.

        Args:
            total_epoch: Current total epoch
            total_iter: Current total number of iterations
            total_val_iter: Current total number of validation iterations
        """

        save_mode = self.cfg.save_by
        multi_gpu = torch.cuda.device_count() >= 1 and self.cfg.parallel

        model = self.model_container.models['model']
        if multi_gpu:
            if save_mode == 'iter':
                if total_iter % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'iter': total_iter,
                        'state_dict': model.module.state_dict(),
                    }, save_dir)

            elif save_mode == 'epoch':
                if total_epoch % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'epoch': total_epoch,
                        'state_dict': model.module.state_dict(),
                    }, save_dir)
        else:
            if save_mode == 'iter':
                if total_iter % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'iter': total_iter,
                        'state_dict': model.state_dict(),
                    }, save_dir)

            elif save_mode == 'epoch':
                if total_epoch % self.cfg.save_every == 0:
                    save_dir = self.exp_save_dir / 'model_log' / f'checkpoint_epoch_{total_epoch}_iter_{total_iter}_val_{total_val_iter}.tar'
                    torch.save({
                        'epoch': total_epoch,
                        'state_dict': model.state_dict(),
                    }, save_dir)

    def prep_train(self):
        """
        Funcion called before training begins. Auxiliary function for initializng different training
        configurations such as model parallelism and weight freezing.
        """
        # Prepare for data parallelism
        self.model_container.load_saved()
        self.model_container.parallelize(['model'], self.devices)
        self.model_container.print_model_size(['model'])

    def inspect_labels(self, pred, label, acc_1):
        # Debugging utility for inspecting ground truth / predicted labels
        thres = getattr(self.cfg, 'acc_threshold', 0.1)
        if acc_1 < thres:
            unq, cnt = torch.unique(pred.argmax(-1), return_counts=True)
            print("GT label: " + self.data_container.dataset[self.cfg.mode].labels[label[0]] + f" {label[0]}")
            print("Most predicted: " + self.data_container.dataset[self.cfg.mode].labels[unq[cnt.argmax()]] + f" {unq[cnt.argmax()]}")

    def inspect_input(self, input_data):
        inspect_channel = getattr(self.cfg, 'inspect_channel', 0)
        inspect_index = getattr(self.cfg, 'inspect_index', 0)

        if inspect_index == 'random':
            inspect_index = random.randint(0, len(input_data) - 1)

        # Debugging utility for visualizing input data
        if type(inspect_channel) is int:
            tmp = input_data[inspect_index].permute(1, 2, 0)[:, :, inspect_channel]
            plt.imshow(255 * tmp.cpu().numpy())
            plt.show()

        if inspect_channel == 'all':
            fig = plt.figure(figsize=(50, 50))

            tmp = input_data[inspect_index].permute(1, 2, 0)
            num_channel = tmp.shape[-1]

            for i in range(num_channel):
                fig.add_subplot(num_channel // 2, 2, i + 1)
                plt.imshow(255 * tmp[..., i].cpu().numpy())

            plt.show()

        if inspect_channel == 'color':
            vis_data = input_data[inspect_index].permute(1, 2, 0)
            pos_ev = (vis_data[..., 0] != 0)
            neg_ev = (vis_data[..., 1] != 0)
            vis_img = torch.ones([vis_data.shape[0], vis_data.shape[1], 3], device=vis_data.device)
            vis_img[pos_ev] = torch.tensor([[1., 0, 0]], device=vis_data.device)
            vis_img[neg_ev] = torch.tensor([[0, 0, 1.]], device=vis_data.device)

            plt.imshow(vis_img.cpu())
            plt.show()

        if inspect_channel == 'color_chn':
            vis_data = input_data[inspect_index].permute(1, 2, 0)
            pos_ev = (vis_data[..., 0] != 0)
            neg_ev = (vis_data[..., 1] != 0)
            pos_img = torch.ones([vis_data.shape[0], vis_data.shape[1], 3], device=vis_data.device)
            pos_img[pos_ev] = torch.tensor([[1., 0, 0]], device=vis_data.device)
            neg_img = torch.ones([vis_data.shape[0], vis_data.shape[1], 3], device=vis_data.device)
            neg_img[neg_ev] = torch.tensor([[0, 0, 1.]], device=vis_data.device)

            fig = plt.figure(figsize=(50, 50))
            fig.add_subplot(1, 2, 1)
            plt.imshow(pos_img.cpu().numpy())
            fig.add_subplot(1, 2, 2)
            plt.imshow(neg_img.cpu().numpy())
            plt.show()

class EvTTACNNTrainer(CNNTrainer):
    def __init__(self, cfg, model_container: ModelContainer, data_container: DataContainer, **kwargs):
        super(EvTTACNNTrainer, self).__init__(cfg, model_container, data_container)
        print(f'Initializing trainer {self.__class__.__name__}...')
        self.loss_type = getattr(cfg, 'loss_type', 'entropy')
        if self.loss_type == 'entropy':
            self.loss_func = SoftCrossEntropy(1.)
        self.entropy_func = SoftCrossEntropy(1.)
        self.cos_loss_func = nn.CosineSimilarity(dim=-1)
        self.kl_loss_func = KLDivLogits()
        self.kl_distrib_loss_func = KLDivLogits(tgt_distrib=True)
        self.kl_log_loss_func = KLDivLogits(input_distrib=True)
        self.info_rad_loss_func = InfoRadiusLoss()
        if getattr(self.cfg, 'print_consistent', False):
            self.avg_consistent = 0.0
            self.counter = 0
        if getattr(self.cfg, 'num_train_samples', None) is not None:
            self.num_tot_train = len(self.data_container.dataset['train'])
            self.sample_counter = 0
        self.threshold_confidence = getattr(self.cfg, 'threshold_confidence', False)
        self.sigma_thres = getattr(cfg, 'sigma_thres', 0.25)

    def z_test_remove(self, input_data, augment_input_data = None):
        pos_count = (input_data[:, 0] != 0).flatten(1, 2).sum(-1)
        neg_count = (input_data[:, 1] != 0).flatten(1, 2).sum(-1)
        trans_val = self.data_container.dataset['train'].geary_trans(pos_count / neg_count)
        trans_mean = trans_val.mean()
        trans_std = trans_val.std()

        sigma = self.sigma_thres
        right_z_val = (trans_mean - (-sigma)) / (trans_std / (len(trans_val) ** (1 / 2)))
        right_p_val = self.data_container.dataset['train'].p_value(right_z_val)

        left_z_val = (trans_mean - (sigma)) / (trans_std / (len(trans_val) ** (1 / 2)))
        left_p_val = self.data_container.dataset['train'].p_value(left_z_val)

        if right_p_val < 0.1:
            input_data = mutual_remove(input_data, 0, 1)
        elif left_p_val > 0.9:
            input_data = mutual_remove(input_data, 1, 0)

        if augment_input_data is not None:
            return input_data, augment_input_data
        else:
            return input_data

    def train(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']
        augment_input_data = data_dict['augment_input_data']

        self.model_container.set_train(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])
            augment_input_data = augment_input_data.to(self.devices[0])

        if getattr(self.cfg, 'z_test_channel', False):
            input_data, augment_input_data = self.z_test_remove(input_data, augment_input_data)

        self.optimizer.zero_grad()

        total_pred = self.model_container.infer('model', torch.cat([input_data, augment_input_data], dim=0), False)

        label_pred = total_pred[:input_data.shape[0]]  # (B, L)
        label_augment_pred = total_pred[input_data.shape[0]:]  # (K * B, L)

        label_augment_pred = label_augment_pred.reshape(input_data.shape[0], getattr(self.cfg, 'num_sentry_augment', 3), -1)  # (B, K, L)
        label_idx = label_pred.argmax(-1).unsqueeze(-1)  # (B, 1)
        label_augment_idx = label_augment_pred.argmax(-1)  # (B, K)
        consistent = ((label_augment_idx == label_idx).sum(-1) / float(getattr(self.cfg, 'num_sentry_augment', 3)) > 0.5)  # (B, )

        total_pred = torch.cat([label_pred.unsqueeze(1), label_augment_pred], dim=1) # (B, K + 1, L)
        anchor_pred = label_pred

        # Optimize entropy of anchor_pred
        loss = self.loss_func(anchor_pred, anchor_pred, return_vector=True)
        loss[~consistent] *= getattr(self.cfg, 'sentry_inconsistent_fill', -1)
        loss = loss[loss != 0.].mean()

        # Set indices to compare with anchor
        other_idx = torch.stack([torch.arange(getattr(self.cfg, 'num_sentry_augment', 3)) + 1] * input_data.shape[0], dim=0)

        for idx in range(getattr(self.cfg, 'num_sentry_augment', 3)):
            tgt_pred_1 = total_pred[torch.arange(input_data.shape[0]), other_idx[:, idx], ...]
            tgt_pred_2 = anchor_pred

            if getattr(self.cfg, 'detach_tgt', False):  # Optionally detach gradients
                tgt_pred_1 = tgt_pred_1.detach()
                tgt_pred_2 = tgt_pred_2.detach()

            mutual_loss_type = getattr(self.cfg, 'mutual_loss_type', 'kl')
            loss += 0.5 * self.kl_loss_func(anchor_pred, tgt_pred_1) \
            + 0.5 * self.kl_loss_func(total_pred[torch.arange(input_data.shape[0]), other_idx[:, idx], ...], tgt_pred_2)

            # Target used for entropy loss
            entropy_tgt_pred = total_pred[torch.arange(input_data.shape[0]), other_idx[:, idx], ...]

            entropy_loss = self.loss_func(entropy_tgt_pred, entropy_tgt_pred, return_vector=True)
            entropy_loss[~consistent] *= getattr(self.cfg, 'sentry_inconsistent_fill', -1)
            loss += entropy_loss[entropy_loss != 0.].mean()

        tot_num = len(label_pred)

        loss.backward()
        self.optimizer.step()

        acc_1, acc_5 = accuracy(label_pred.cpu(), label.cpu(), topk=(1, min(5, label_pred.shape[-1])))
        num_correct = int(acc_1 * tot_num)

        return loss.item(), acc_1, acc_5, num_correct, tot_num

    def test(self, data_dict, **kwargs):
        input_data = data_dict['input_data']
        label = data_dict['label']

        self.model_container.set_eval(['model'])

        if self.use_cuda:
            input_data, label = input_data.to(self.devices[0]), label.to(self.devices[0])

        if getattr(self.cfg, 'z_test_channel', False):
            input_data = self.z_test_remove(input_data)

        pred = self.model_container.infer('model', input_data, True)
        tot_num = len(pred)

        acc_1, acc_5 = accuracy(pred.cpu(), label.cpu(), topk=(1, min(5, pred.shape[-1])))

        num_correct = int(acc_1 * tot_num)

        if self.debug:
            if self.debug_input:
                self.inspect_input(input_data)
            if self.debug_labels:
                self.inspect_labels(pred, label, acc_1)

        return acc_1, acc_5, num_correct, tot_num

    def run_epoch(self):
        # Train
        print(f"This is {self.tracker.epoch}-th epoch.")

        self.tracker.init_epoch(mode='train')

        skip_train = getattr(self.cfg, 'skip_train', False)

        if not skip_train:
            self.train_epoch()

            print(f"Epoch {self.tracker.epoch} training loss = {self.tracker.total_train_loss.get_avg():.4f}")
            print(f"Total training accuracy = {(self.tracker.get_train_acc()):.4f}")

        # Validate
        print(f"Epoch {self.tracker.epoch} validation!")

        self.tracker.init_epoch(mode='val')
        self.validate_epoch()
        print(f"Total validation accuracy = {(self.tracker.get_val_acc()):.4f}")

        # Update scheduler
        if self.scheduler_type == 'reduce_plateau':
            self.scheduler.step(self.tracker.get_val_acc())
        elif self.scheduler_type is None:
            pass
        else:
            self.scheduler.step()

        # Save model by epoch (depends on cfg)
        if self.cfg.save_by == 'epoch':
            self.save_model(self.tracker.epoch, self.tracker.total_iter, self.tracker.total_val_iter)

    def train_batch(self, data_dict):
        self.tracker.init_batch(mode='train')
        self.tracker.start_infer_timing()

        loss, acc_1, acc_5, train_correct, train_num = self.train(data_dict)
        self.tracker.end_infer_timing()

        write_dict = {'training loss': loss, 'top 1 training accuracy': acc_1, 'top 5 training accuracy': acc_5, 'load time': self.tracker.load_time}
        print_dict = {'Iter': self.tracker.batch_idx, 'training loss': loss, 'top 1 training acc': acc_1, 'top 5 training acc': acc_5,
        'infer time': self.tracker.infer_time, 'load time': self.tracker.load_time}
        self.print_state(print_dict)
        self.write(self.tracker.total_iter, write_dict)

        self.tracker.total_train_loss.update(loss)
        self.tracker.total_train_correct.accumulate(train_correct)
        self.tracker.total_train_num.accumulate(train_num)

        if self.cfg.save_by == 'iter':
            self.save_model(self.tracker.epoch, self.tracker.total_iter, self.tracker.total_val_iter)

    def init_optimizer(self, **kwargs):
        """
        Initialize self.optimizer using self.cfg.
        """

        # Choose optimizer
        model = self.model_container.models['model']
        opt_type = self.cfg.optimizer
        freeze = getattr(self.cfg, 'freeze', False) or getattr(self.cfg, 'train_classifier', False)

        if opt_type == 'SGD':
            print('Using SGD as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.SGD(model.parameters(), lr=self.cfg.learning_rate, momentum=self.cfg.momentum, weight_decay=self.cfg.weight_decay)
        elif opt_type == 'Adam':
            print('Using Adam as optimizer')
            if freeze:
                print('Freezing weights!')
                self.optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)
            else:
                self.optimizer = optim.Adam(model.parameters(), lr=self.cfg.learning_rate, weight_decay=self.cfg.weight_decay)


# py -m pip install dnspython

import dns.resolver, dns.reversename

strDomains = """
ya.ru
yawdadad1.ru
2020.kbsh.rzd.ru
"""

strIps = """
8.8.8.8
83.234.113.47
"""

domains = strDomains.split()
ips = strIps.split()

for ip in ips:
    addr = dns.reversename.from_address(ip)
    try:
        A = dns.resolver.resolve(addr, 'PTR')
    except dns.resolver.NXDOMAIN:
        print(ip," XXX")
    except Exception:
        print(ip," неизвестная ошибка")
    else:
        for i in A.response.answer:
            for j in i.items:
                print(ip," ",j)

for domain in domains:
    try:
        A = dns.resolver.resolve(domain, 'A')
    except dns.resolver.NXDOMAIN:
        print(domain," 0.0.0.0")
    except Exception:
        print(domain," неизвестная ошибка")
    else:
        for i in A.response.answer:
            for j in i.items:
#                print(domain," ",j.address)
                print(domain," ",j)


from urllib.parse import urlparse

import requests
from django.core.files.base import ContentFile
from django.core.management.base import BaseCommand
from places.models import Place, Image

def get_filename(url):
    image_path = urlparse(url).path
    filename = image_path.split('/')[-1]
    return filename

class Command(BaseCommand):
    help = 'Loads places description and photos to database'

    def add_arguments(self, parser):
        parser.add_argument('url', type=str)

    def handle(self, *args, **options):
        response = requests.get(options['url'])
        response.raise_for_status()
        payload = response.json()

        place, created = Place.objects.update_or_create(
            title=payload['title'],
            description_short=payload['description_short'],
            lng=payload['coordinates']['lng'],
            lat=payload['coordinates']['lat'],
            defaults={'description_long': payload['description_long']}
        )

        image_urls = payload['imgs']
        if not created:
            place.imgs.all().delete()

        for number, url in enumerate(image_urls):
            image = Image.objects.create(place=place, number=number)
            image_response = requests.get(url)
            image_response.raise_for_status()
            content = ContentFile(image_response.content)
            filename = get_filename(url)
            image.photo.save(filename, content, save=True)


from abc import ABC, abstractmethod
from dataclasses import dataclass, field
import datetime as dt
from decimal import Decimal
from itertools import groupby
from typing import List, Optional

from dateutil.relativedelta import relativedelta

from domain.common.investments import StockInvestment
from domain.common.investment_position import InvestmentPosition, StockPosition
from domain.common.investment_position_summary import StockPositionSummary
from domain.common.investment_summary import InvestmentSummary, StockSummary
from domain.common.portfolio_item import PortfolioItem

DATE_FORMAT = "%Y%m%d"

@dataclass
class InvestmentConsolidated(PortfolioItem, ABC):
    initial_date: dt.date = dt.datetime.max.date()
    history: List[InvestmentPosition] = field(default_factory=list)

    def __post_init__(self):
        if not isinstance(self.initial_date, dt.date):
            self.initial_date = dt.datetime.strptime(
                str(self.initial_date), DATE_FORMAT
            ).date()

    def to_json(self) -> dict:
        ret = {
            **super().to_json(),
            "initial_date": self.initial_date.strftime(DATE_FORMAT),
            "history": sorted(
                [h.to_dict() for h in self.history], key=lambda h: h["date"]
            ),
        }
        return ret

    @abstractmethod
    def add_investment(self, investment):
        """Adds the investment"""

    def export_investment_summary(self) -> Optional[InvestmentSummary]:
        """Creates an InvestmentSummary objects based on history"""

class StockPositionWrapper:
    def __init__(self, data: StockPosition):
        self.data = data
        self.next: Optional[StockPositionWrapper] = None
        self.prev: Optional[StockPositionWrapper] = None

    @property
    def date(self) -> dt.date:
        return self.data.date

    @property
    def amount(self):
        if self.prev:
            return self.prev.amount + self.data.bought_amount - self.data.sold_amount
        return self.data.bought_amount - self.data.sold_amount

    @property
    def bought_amount(self):
        if self.prev and self.prev.amount > 0:
            return self.prev.bought_amount + self.data.bought_amount
        return self.data.bought_amount

    @property
    def bought_value(self):
        if self.prev and self.prev.amount > 0:
            return self.prev.bought_value + self.data.bought_value
        return self.data.bought_value

    @property
    def average_price(self):
        bought_amount = self.bought_amount
        if bought_amount > 0:
            return (self.bought_value / bought_amount).quantize(Decimal("0.01"))
        return 0

    @property
    def gross_sold_value(self):
        if self.prev:
            return self.prev.position_sold_value + self.data.sold_value
        return self.data.sold_value

    @property
    def invested_sold_value(self):
        if self.prev:
            return (
                    self.prev.current_invested_sold_value
                    + self.data.sold_amount * self.average_price
            )
        return self.data.sold_amount * self.average_price

    @property
    def current_invested_sold_value(self):
        if self.prev and self.prev.amount > 0:
            return (
                    self.prev.current_invested_sold_value
                    + self.data.sold_amount * self.average_price
            )
        return self.data.sold_amount * self.average_price

    @property
    def current_invested_value(self):
        if self.prev:
            return (self.bought_value - self.current_invested_sold_value).quantize(
                Decimal("0.01")
            )
        return (self.data.bought_value - self.current_invested_sold_value).quantize(
            Decimal("0.01")
        )

    @property
    def node_invested_value(self):
        return self.data.bought_value - self.data.sold_amount * self.average_price

    @property
    def node_invested_sold_value(self):
        return self.data.sold_amount * self.average_price

    @property
    def realized_profit(self):
        return (self.gross_sold_value - self.invested_sold_value).quantize(
            Decimal("0.01")
        )

    # @property
    # def gross_value(self):
    #     return self.amount * self.data.close_price
    #
    # @property
    # def prev_adjusted_gross_value(self):
    #     if self.prev and self.prev.amount > 0:
    #         if self.data.sold_amount > 0:
    #             prev_month_amount = self.prev.amount - round(
    #                 self.node_invested_sold_value / self.prev.average_price
    #             )
    #         else:
    #             prev_month_amount = self.prev.amount
    #         return prev_month_amount * self.prev.data.close_price
    #     return 0

    @property
    def month_variation_percent(self):
        if self.prev and self.prev.amount > 0:
            prev_gross_value = self.prev_adjusted_gross_value + self.data.bought_value
        else:
            prev_gross_value = self.current_invested_value
        if prev_gross_value > 0:
            return ((self.gross_value * 100 / prev_gross_value) - 100).quantize(
                Decimal("0.01")
            )
        return 0

class StockPositionWrapperLinkedList:
    def __init__(self):
        self.head: Optional[StockPositionWrapper] = None
        self.tail: Optional[StockPositionWrapper] = None
        self._current: Optional[StockPositionWrapper] = None
        self._first_interaction = False

    def __iter__(self):
        self._first_interaction = True
        self._current = self.head
        return self

    def __next__(self) -> StockPositionWrapper:
        if not self._current:
            raise StopIteration
        if self._first_interaction:
            self._first_interaction = False
            return self._current
        self._current = self._current.next
        if not self._current:
            raise StopIteration
        return self._current

    def push(self, new_val: StockPosition):
        new_node = StockPositionWrapper(new_val)
        new_node.next = self.head
        if self.head is not None:
            self.head.prev = new_node
        self.head = new_node

    def insert(self, prev_node, new_val):
        if prev_node is None:
            return
        new_node = StockPositionWrapper(new_val)
        new_node.next = prev_node.next
        prev_node.next = new_node
        new_node.prev = prev_node
        if new_node.next:
            new_node.next.prev = new_node
        else:
            self.tail = new_node

    def append(self, new_val):
        new_node = StockPositionWrapper(new_val)
        new_node.next = None

        if self.head is None:
            new_node.prev = None
            self.head = new_node
            self.tail = new_node
            return

        last = self.head
        while last.next:
            last = last.next
        last.next = new_node
        new_node.prev = last
        self.tail = new_node

    def find(self, func):
        curr = self.head
        while curr:
            if func(curr):
                return curr
            curr = curr.next
        return None

    @staticmethod
    def list_print(node):
        while node:
            print(node.data),
            node = node.next

@dataclass
class StockConsolidated(InvestmentConsolidated):
    ticker: str = field(default="")
    alias_ticker: str = field(default="")

    def __post_init__(self):
        super().__post_init__()
        self.history = [
            StockPosition(**h) if isinstance(h, dict) else h for h in self.history
        ]

    @property
    def sk(self) -> str:
        sk = f"TICKER#{self.ticker}"
        return sk

    def to_json(self) -> dict:
        _dict = super().to_json()
        _dict["alias_ticker"] = self.alias_ticker
        _dict["ticker"] = self.ticker

        return _dict

    def current_ticker_name(self):
        return self.alias_ticker or self.ticker

    def __add__(self, other):
        initial_date = min(self.initial_date, other.initial_date)
        return StockConsolidated(
            subject=self.subject,
            ticker=self.ticker,
            alias_ticker=self.alias_ticker,
            initial_date=initial_date,
            history=self.history + other.history,
        )

    def find_position_in_history(self, date: dt.date) -> Optional[StockPosition]:
        return next(
            (position for position in self.history if position.date == date),
            None,
        )

    def create_empty_position(self, date: dt.date) -> StockPosition:
        p = StockPosition(date)
        self.history.append(p)
        return p

    def add_investment(self, investment: StockInvestment):
        self.initial_date = min(self.initial_date, investment.date)
        if investment.alias_ticker:
            self.alias_ticker = investment.alias_ticker

        position = self.find_position_in_history(investment.date)
        if not position:
            position = self.create_empty_position(investment.date)

        position.add_investment(investment)
        if position.is_empty():
            self.history.remove(position)

    def add_investments(self, investments: List[StockInvestment]):
        for investment in investments:
            self.add_investment(investment)

    def export_investment_summary(self) -> Optional[StockSummary]:
        wrapper = self.monthly_stock_position_wrapper_linked_list()
        if not wrapper.tail:
            return

        current = wrapper.tail
        previous = current.prev

        previous_position = None
        latest_position = StockPositionSummary(
            date=current.data.date,
            amount=current.amount,
            invested_value=current.current_invested_value,
            bought_value=current.data.bought_value,
            average_price=current.average_price,
        )
        if previous:
            previous_position = StockPositionSummary(
                date=previous.data.date,
                amount=previous.amount,
                bought_value=previous.data.bought_value,
                invested_value=previous.current_invested_value,
            )

        return StockSummary(
            self.ticker,
            alias_ticker=self.alias_ticker,
            latest_position=latest_position,
            previous_position=previous_position,
        )

    def grouped_by_month_stock_position(self) -> List[StockPosition]:
        grouped_positions = []
        for date, positions in groupby(
                sorted(self.history, key=lambda h: h.date),
                key=lambda p: p.date.replace(day=1),
        ):
            grouped = StockPosition(date)
            for position in positions:
                grouped = grouped + position
            grouped_positions.append(grouped)
        return grouped_positions

    def monthly_stock_position_wrapper_linked_list(
            self,
            to_date: dt.date = None,
    ) -> StockPositionWrapperLinkedList:
        doubly = StockPositionWrapperLinkedList()

        prev_date = self.initial_date.replace(day=1)
        for h in self.grouped_by_month_stock_position():
            if h.date > prev_date + relativedelta(months=1):
                while prev_date < h.date:  # filling the gaps
                    doubly.append(StockPosition(prev_date))
                    prev_date += relativedelta(months=1)
            doubly.append(h)
            prev_date = h.date.replace(day=1) + relativedelta(months=1)

        if to_date and doubly.tail:
            while doubly.tail.date < to_date:
                doubly.append(StockPosition(prev_date))
                prev_date += relativedelta(months=1)

        return doubly


import threading

from flask import Flask, render_template
import ssh

from gevent import pywsgi

app = Flask(__name__)

host_info = {}
cfg_dict = {}

@app.before_first_request
def start_ssh():
    global cfg_dict, host_info
    ssh_pool, cfg_dict = ssh.init()
    for (ssh_id, (host, ssh_cfg)) in enumerate(ssh_pool.items()):
        t = threading.Thread(target=ssh.query_host, args=(ssh_id, host, ssh_cfg, host_info, cfg_dict['timeout']))
        t.start()

@app.route('/')
def hello_world():
    return "hello world"

@app.route('/gpu', methods=['GET', 'POST'])
def gpu():
    global host_info, cfg_dict
    host_i = dict(sorted(host_info.items(), key=lambda item: item[0]))
    if cfg_dict['warmup'] > 0:
        refresh = 5
        cfg_dict['warmup'] -= 1
    else:
        refresh = cfg_dict['refresh']
    return render_template('gpu.html', title='Who Moved My GPU?', host_info=host_i, refresh_time=refresh)

@app.route('/fix', methods=['GET', 'POST'])
def fix():
    return render_template('fix.html')

if __name__ == '__main__':
    server = pywsgi.WSGIServer(('0.0.0.0', 1301), app)
    server.serve_forever()


'''
06 - Unix timestamps

Datetimes are sometimes stored as Unix timestamps: the  number of  seconds  since
January 1, 1970. This is especially common with computer infrastructure, like the
log files that websites keep when they get visitors.

Instructions

- Complete the for loop to loop over timestamps.
- Complete the code to turn each timestamp ts into a datetime.
'''
# Import datetime
from datetime import datetime

# Starting timestamps
timestamps = [1514665153, 1514664543]

# Datetime objects
dts = []

# Loop
for ts in timestamps:
  dts.append(datetime.fromtimestamp(ts))

# Print results
print(dts)

'''
<script.py> output:
    [datetime.datetime(2017, 12, 30, 20, 19, 13), datetime.datetime(2017, 12, 30, 20, 9, 3)]

The largest number that some older computers can hold in one variable is 2147483648, which as
a Unix timestamp is in January 2038. On that day, many computers which haven't been upgraded
will fail.
'''

18_Working with Dates and Times in Python/02_Combining Dates and Times/06_Unix timestamps.py


import requests
import json
import datetime as dt

today = dt.datetime.now()
yesterday = today - dt.timedelta(days=1)
tomorrow = today + dt.timedelta(days=1)

# B100005528
# J100005670

def getLunch(day, month):
    school = 'J100005670'
    url = f'https://schoolmenukr.ml/api/high/{school}?date={day}&allergy=hidden&month={month}'
    response = requests.get(url)
    school_menu = json.loads(response.text)
    lunch = school_menu['menu'][0]['lunch']
    for menu in lunch:
        print(menu)

print('\n오늘급식:\n')
getLunch(today.day, today.month)
print('\n내일급식:\n')
getLunch(tomorrow.day, tomorrow.month)
print('\n어제급식:\n')
getLunch(yesterday.day, yesterday.month)


from kafka import KafkaProducer
from datetime import datetime
import time
from json import dumps
import requests

TOPIC_NAME_CONS = "topic-weather"
BOOTSTRAP_SERVERS_CONS = 'localhost:9092'

def get_data(url):

    response_data = requests.get(url=url)
    data = response_data.json()
    weather = {}
    event_datetime = datetime.now()
    weather["event_datetime"] 		= event_datetime.strftime("%Y-%m-%d %H:%M")
    weather["humidity"] 			= data["main"]["humidity"]
    weather["pressure"] 			= data["main"]["pressure"]
    weather["temperature"] 			= data["main"]["temp"]
    weather["temperature_max"] 		= data["main"]["temp_max"]
    weather["temperature_min"] 		= data["main"]["temp_min"]
    weather["temperature_feels"] 	= data["main"]["feels_like"]
    weather["city_name"] 			= data["name"]
    weather["wind_deg"] 			= data["wind"]["deg"]
    weather["wind_speed"] 			= data["wind"]["speed"]

    return weather

if __name__ == "__main__":
    print("Open Weather API Data | Kafka Producer Application Started ... ")

    kafka_producer_obj = KafkaProducer(bootstrap_servers=BOOTSTRAP_SERVERS_CONS,
                             value_serializer=lambda x: dumps(x).encode('utf-8'))

    city_names = ["Hamburg","Berlin","Frankfurt am Main","Stuttgart","Munich","Düsseldorf","Köln","Dresden","Leipzig"]

    i = 0
    while True:
        try:

            for city in city_names:
                api_key = "API KEY"
                url = "https://api.openweathermap.org/data/2.5/weather?q={0}&appid={1}".format(city,api_key)
                weather_data = get_data(url)
                i = i + 1
                print("<<<<<<<<<<  Live Straeming Data >>>>>>>>>>")
                print("Printing message id: " + str(i))
                print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
                print("Sending message to Kafka topic: " + TOPIC_NAME_CONS)
                print("Message to be sent: ", weather_data)
                kafka_producer_obj.send(TOPIC_NAME_CONS, weather_data)

        except Exception as ex:
            print("Event Message Construction Failed. ")
            print(ex)

        time.sleep(10)


import time, re
from itertools import groupby
from helpers import *

t = time.time()

input = "1113122113"

# Part 1
def splitSequence(input):
    return [m.group(0) for m in re.finditer(r"(\d)\1*", input)]

def lookAndSay(seq):
    seq = splitSequence(seq)
    say = "".join(map(lambda c: str(len(c)) + c[0], seq))
    return say

for i in range(45):
    input = lookAndSay(input)

dropstar(19, len(input), t)

# Part 2
def lookAndSay2(number):
    return "".join(str(len(list(g))) + k for k, g in groupby(number))

# 1113122113
# 132113
# 3113
# 1113
# 13
# 3

input = "3"
for i in range(55):
    input = lookAndSay2(input)

dropstar(20, len(input), t)


import numpy as np
import matplotlib.pyplot as plt

plt.rc("text", usetex = True)
plt.rc("text.latex", preamble = r"\usepackage[utf8]{inputenc} \usepackage{siunitx} \usepackage{physics} \sisetup{round-mode = figures} \sisetup{round-precision = 3}")
plt.rc("font", **{"family": "serif", "serif": ["Computer Modern"]})

data = np.genfromtxt("actividad_1.txt")
data[:,1] = -data[:,1]

fit = np.polyfit(data[:,0],data[:,1],1)
sigma_B = np.sqrt(np.sum((data[:,1]-fit[1]-fit[0]*data[:,0])**2)/(len(data)-2))
Delta = len(data)*np.sum(data[:,0]**2)-(np.sum(data[:,0]))**2
sigma_pendiente = sigma_B*np.sqrt(len(data)/Delta)
sigma_intercepto = sigma_B*np.sqrt(np.sum((data[:,0])**2)/Delta)
print("se obtuvo B = %f I + %f con incertidumbres %f en los campos magneticos, %f en la pendiente y %f en el intercepto." % (fit[0], fit[1], sigma_B, sigma_pendiente, sigma_intercepto))
I = np.array([np.amin(data[:,0]),np.amax(data[:,0])])
np.save("calibracion", fit)

fig, ax = plt.subplots()
ax.errorbar(data[:,0], data[:,1], xerr = 0.01, yerr = 10, c = "k", fmt = "o")
ax.plot(I, fit[0]*I + fit[1], c = "k")
ax.set_xlabel(r"$I_B(\si{\A})$")
ax.set_ylabel(r"$B(\si{\milli\tesla})$")
ax.set_title(r"Curva de calibraci\'on")
fig.savefig("calibracion.png")
plt.close(fig)


# from importlib.resources import path
from django.urls import path
from . import views

app_name = 'posts'

urlpatterns = [
    # главная страница
    path('', views.index, name='index'),
    # <slug:название группы>
    path('group/<slug:slug>/', views.group_posts, name='group_list'),
    # Профайл пользователя
    path('profile/<slug:username>/', views.profile, name='profile'),
    # Просмотр записи
    path('posts/<int:post_id>/', views.post_detail, name='post_detail'),
    path('create/', views.post_create, name='post_create'),
    path('posts/<int:post_id>/edit/', views.post_edit, name='post_edit'),
    path(
        'posts/<int:post_id>/comment/',
        views.add_comment,
        name='add_comment'
    ),
    path('follow/', views.follow_index, name='follow_index'),
    # Подписаться на автора
    path(
        'profile/<str:username>/follow/',
        views.profile_follow,
        name='profile_follow'
    ),
    # Отписаться от автоора
    path(
        'profile/<str:username>/unfollow/',
        views.profile_unfollow,
        name='profile_unfollow'
    ),
]


from pathlib import Path
import logging
import os
import sys

logger = logging.getLogger("defaultLogger")

def get_newest(path: str, file_extension: str):
    """
    Fetchest name of the newest file in a folder

    args:
    ---

    returns:
    ---
    newest_file -- str
        Name of the newest file in a folder

    """
    logger.info(f"Getting first timestamp from {path}")
    files = list(Path(path).rglob(f"*{file_extension}*"))
    if not files:
        logger.info(f"No files found in {path}")
        logger.warning("EXITING")
        # BUG: NEED A BETTER WAY OF EXITING THE FUNCTION BECAUSE THIS
        # STOPS THE LOOPING THROUGH FILES
        sys.exit(0)

    # linux only
    # newest_file = str(max([f for f in files], key=lambda item: item.stat().st_ctime))
    # cross platform
    newest_file = str(max(files, key=os.path.getmtime))
    return newest_file


#! /usr/bin/python
import sys
import warnings
from selenium import webdriver
from appium import webdriver as app_driver
import json
from utils.my_logger import log
from config.get_config_data import get_sys_yml_data
from config.PATH import os, DRIVERS

def __get_driver_path(driver_type):
    """
    获取 driver 文件路径
    :param driver_type: 传入 driver 类型, 系统自动 配合 系统配置文件中的文件名, 匹配项目中是否存储有 driver 文件
    :return 有的话，返回 绝对路径
            无的话，返回 None
    """

    driver_type = driver_type.lower()
    if driver_type not in ['chrome', 'firefox', 'ie', 'safari']:
        raise Exception(f'不支持的 driver 类型: {driver_type}')

    current_sys_type = sys.platform
    sys_yml_data = get_sys_yml_data()

    if current_sys_type == 'win32':
        current_sys_type = 'win'
    else:
        current_sys_type = current_sys_type

    driver_name = sys_yml_data['driver_path'][current_sys_type][driver_type]
    if driver_name:
        if os.path.exists(driver_name):  # 如果是 绝对路径的
            driver_path = driver_name
        else:  # 如果是 名字 + driver存放于项目中
            driver_path = os.path.join(DRIVERS, current_sys_type, driver_name)

        if os.path.exists(driver_path):
            return driver_path
        else:
            return None
    else:
        return None

def get_h5_driver(driver_type='chrome'):
    driver_type = driver_type.lower()
    driver_path = __get_driver_path(driver_type)
    if driver_path is None:
        driver_path = 'default_path'

    mobile_emulation = {"deviceName": "iPhone 5/SE"}
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
    chrome_options.add_experimental_option('w3c', False)
    warnings.simplefilter("ignore", ResourceWarning)

    log.info(f'driver路径: {driver_path}')
    log.info('打开H5浏览器。')
    if driver_path:
        driver = webdriver.Chrome(options=chrome_options, executable_path=driver_path)
    else:
        driver = webdriver.Chrome(options=chrome_options)
    driver = __check_webdriver_to_undefined(driver)
    return driver

def get_web_driver(driver_type='chrome'):
    driver_type = driver_type.lower()
    driver_path = __get_driver_path(driver_type)
    if driver_path is None:
        driver_path = 'default_path'

    log.info(f'driver路径: {driver_path}')
    log.info('打开web浏览器。')
    if driver_path:
        driver = webdriver.Chrome(executable_path=driver_path)
    else:
        driver = webdriver.Chrome()
    driver = __check_webdriver_to_undefined(driver)
    driver.maximize_window()
    return driver

def get_app_driver(appium_port, desired_caps):
    """
    打开APP
    :param appium_port: appium端口号
    :param desired_caps: 移动设备的关键信息
    """
    log.info('=' * 40 + '  设备初始化信息  ' + '=' * 40)
    log.info('appium_port：{}'.format(appium_port))
    log.info('desired_capabilities：{}'.format(json.dumps(desired_caps, indent=2, ensure_ascii=False)))
    log.info('=' * 40 + '  设备初始化信息  ' + '=' * 40)
    log.info('打开APP。')
    driver = app_driver.Remote(command_executor='http://127.0.0.1:{}/wd/hub'.format(appium_port),
                               desired_capabilities=desired_caps)
    return driver

def __check_webdriver_to_undefined(driver):
    """
    去掉 selenium 反爬检测
    :param driver: driver
    :return: driver
    """
    driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
        'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
    })
    return driver


# -*- coding: utf-8 -*-
from __future__ import print_function, division

#------------------------------------------------------------------------------
# SENTIMENT MODEL - BUILD MODEL
#------------------------------------------------------------------------------

# Functions:
# [1] ...

# Version: 2.0
# Last edited: 21 Dec 2016
# Edited by: Minh PHAN

#------------------------------------------------------------------------------
# Global variables and settings
#------------------------------------------------------------------------------

# Seting working directory
import os
os.chdir('D:\\SentimentTM\\23_sentiment_analysis\\model')

#------------------------------------------------------------------------------
# Initiating
#------------------------------------------------------------------------------

import pandas as pd
import numpy as np

import time

from sklearn import preprocessing

from sklearn.grid_search import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import BaggingClassifier
from sklearn.dummy import DummyClassifier

from scipy import sparse
import xgboost as xgb
import pickle

#------------------------------------------------------------------------------
# Self-defined functions
#------------------------------------------------------------------------------

# Function to calculate the probability of each target class
# Can replace this function with predict_proba()
def classification_ProbClass(clf, data_test_new, labelEncoder, n_top):

    # Estimate prob of belonging to each class
    target_predProb = clf.predict_proba(data_test_new) # Matrix containing prob for each row of data_test
    idx_sort = np.argsort(-target_predProb,axis=1) # Indexes of array if sorted largest prob first
    i_top = idx_sort[:,idx_sort.shape[1]-n_top:idx_sort.shape[1]] # Index n_top predicted clases, order decreasing prob

    # Top n probabilities, decreasing order
    target_predProb_top = np.vstack([target_predProb[i,i_top[i,:]] for i in range(target_predProb.shape[0])]) # prob of top clases, order decreasing prob

    # Top n classes, convert class code to real target value
    realTop_target = labelEncoder.inverse_transform(i_top)

    # Create string output that combines pred class and prob, for easy visualization
    assign_class_prob = ([zip(realTop_target[i,:].astype(np.str), target_predProb_top[i,:].astype(np.str))
                          for i in range(target_predProb.shape[0])])

    pred_class_prob = [] # np.zeros((len(assign_class_prob),), dtype=object)
    for i in  range(len(assign_class_prob)):
        row = assign_class_prob[i]
        str_row = ''
        for j in range(len(row)):
            str_row = str_row + '='.join(row[j]) + ', '

        str_row = str_row[0:-2] # Remove last space and coma

        #pred_class_prob[i] = str_row
        pred_class_prob.append(str_row)

    return target_predProb_top, realTop_target, pred_class_prob

# Function to select best features
def feature_selection(data_train, target_train, data_test, scoring_type):

    # Fewature selection using linear svm
    tuned_parameters = [{'C': [0.05,0.1, 0.5, 1 ,5,10,50,100,500], 'penalty':["l1","l2"], 'dual': [False],'class_weight': ['auto'] }]
    clf_featureSelect = GridSearchCV(LinearSVC(), tuned_parameters, cv=10, scoring=scoring_type)
    clf_featureSelect.fit(data_train,target_train)
    data_train_new = clf_featureSelect.transform(data_train)
    data_test_new = clf_featureSelect.transform(data_test)

    return data_train_new, data_test_new, clf_featureSelect

# Function to run the classifier and save the models
def classifyTwitter(dataIn_training, dataIn_test):

    # Drop NA data
    dataIn_training = dataIn_training.dropna()
    dataIn_test = dataIn_test.dropna()
    dataIn_training = dataIn_training.reset_index(drop=True)
    dataIn_test = dataIn_test.reset_index(drop=True)

    # Enocde target variable
    # To decode: labelEncoder.inverse_transform([2, 2, 1])='neutral,..
    target_training_org = dataIn_training.polarity.values
    target_test_org = dataIn_test.polarity.values
    labelEncoder = preprocessing.LabelEncoder()
    labelEncoder.fit(target_training_org)
    target_train = labelEncoder.transform(target_training_org)
    target_test = labelEncoder.transform(target_test_org)

    # Drop target variable
    dataIn_training.drop('polarity', axis=1, inplace=True)
    dataIn_test.drop('polarity', axis=1, inplace=True)

    # Sparse data
    data_train_sparse = sparse.csr_matrix(dataIn_training.as_matrix())
    data_test_sparse = sparse.csr_matrix(dataIn_test.as_matrix())

    # Scale data
    # Need to preprocess data: norlaize--> probelm: if sparse, not possible
    # to standarize (if d-mean/std--> 0 besomes sometihng--> dense)
    # either no standarization, or standarization [0,1] o just by std
    use_scaler = False
    if use_scaler == True:
        scaler = preprocessing.StandardScaler(with_mean=False).fit(data_train_sparse)
        data_train = scaler.transform(data_train_sparse)
        data_test = scaler.transform(data_test_sparse)
    else:
        data_train = data_train_sparse
        data_test = data_test_sparse
        scaler = None

    # Fewature selection
    # Need to save the original data_trin/test for the decoding of one-hot
    use_featureSelect = False
    scoring_type = 'f1_weighted'
    if use_featureSelect == True:
        start = time.time()
        data_train_new, data_test_new, clf_featureSelect = feature_selection(data_train,target_train,data_test,scoring_type)
        end = time.time()
        print('Feature Select time =', str(end-start))
        print('new shape= ', data_test_new.shape)
    else:
       data_train_new, data_test_new=data_train, data_test
       clf_featureSelect = None

    # Convert training data back to arrary
    # Sparse matrix has error when running cross validation
    data_train_new = data_train_new.toarray()
    data_test_new = data_test_new.toarray()

    # Train xgboost model with GridSearchCV
#    start = time.time()
#    xgb_model = xgb.XGBClassifier({'nthread':4, 'objective':'multi:softprob',
#                                   'num_class':3, 'random_state':0})
#    tuned_parameters = {'max_depth':[2,4,6,8],
#                        'n_estimators':[10,50,100,200,400,600]}
#    clf_xgb = GridSearchCV(xgb_model, tuned_parameters, cv=10, scoring=scoring_type,
#                           verbose=10)
#
#    clf_xgb.fit(data_train_new, target_train)
#    target_pred_xgb = clf_xgb.predict(data_test_new)
#    end = time.time()
#
#    print('xgboost classifier:')
#    print(clf_xgb.best_score_)
#    print(clf_xgb.best_params_)
#    print('clf_xgb time = ', str(end-start))
#    print(classification_report(target_test, target_pred_xgb))
#    print(confusion_matrix(target_test, target_pred_xgb))
#    acc_xgb = clf_xgb.score(data_test_new, target_test)
#    print(acc_xgb)

    # Re-train xgboost model with best params to evaluate
    xgb_best_params = xgb.XGBClassifier(nthread=4, objective='multi:softprob',
                                        seed=0, n_estimators=600, max_depth=2)
    xgb_best_params.fit(data_train_new, target_train)
    target_pred_xgb = xgb_best_params.predict(data_test_new)

    print('xgboost classifier:')
    print(classification_report(target_test, target_pred_xgb))
    print(confusion_matrix(target_test, target_pred_xgb))
    acc_xgb = xgb_best_params.score(data_test_new, target_test)
    print(acc_xgb)

    # Save xgboost model with best params
    xgb_best_params = xgb.XGBClassifier(nthread=4, objective='multi:softprob',
                                        seed=0, n_estimators=600, max_depth=2)
    train_all = np.concatenate((data_train_new, data_test_new), axis=0)
    target_all = np.concatenate((target_train, target_test), axis=0)
    xgb_best_params.fit(train_all, target_all)

    file_out = '.\\output\\sentiment_classifier_model.pkl'
    save_data = [labelEncoder, xgb_best_params]
    pickle.dump(save_data, open(file_out, 'wb'))

    # Dummy calssifier (baseline to compare)
    clf_dummy = DummyClassifier(strategy='stratified', random_state=0)
    clf_dummy.fit(data_train_new, target_train)
    target_pred_dummy = clf_dummy.predict(data_test_new)

    print('Dummy classifier:')
    print(classification_report(target_test, target_pred_dummy))
    print(confusion_matrix(target_test, target_pred_dummy))
    acc_dummy = clf_dummy.score(data_test_new, target_test)
    print(acc_dummy)

    # Create output results
    dataIn_test['polarity'] = labelEncoder.inverse_transform(target_test)
    dataOut = dataIn_test

    col_output = 'EstimatedPolarity_XGB'
    dataOut[col_output] = labelEncoder.inverse_transform(target_pred_xgb)

    n_top = 3 # chose top 3 classes (in this case only 3 in total but other cases can  bemore)
    target_predProb_top, realTop_target, pred_class_prob = classification_ProbClass(xgb_best_params, data_test_new, labelEncoder, n_top)

    col_output = 'EstimatedPolarity_XGB_prob'
    dataOut[col_output] = pred_class_prob

    return dataOut

#------------------------------------------------------------------------------
# MAIN
#------------------------------------------------------------------------------

if __name__ == '__main__':

    # Read data in and train sentiment classifier model
    fileNameIn_train = '.\\output\\tweeti-b.distFULL_featureEncoded.tsv'
    fileNameIn_test = '.\\output\\tweeti-b.dev.distFULL_featureEncoded.tsv'

    dataIn_training = pd.read_csv(fileNameIn_train, sep='\t', encoding='utf-8')
    dataIn_test = pd.read_csv(fileNameIn_test, sep='\t', encoding='utf-8')

    dataOut_test = classifyTwitter(dataIn_training, dataIn_test)

    # Merge and export results
    fileNameIn_test_withMsg = '.\\input\\data_challenge2013\\tweeti-b.dev.distFULL.tsv'
    fileNameOut_test_withMsg = '.\\output\\tweeti-b.dev.distFULL_classificationOut_withMSg.csv'

    dataIn_test_withMsg = pd.read_csv(fileNameIn_test_withMsg, sep='\t')
    colNames = list(dataIn_test_withMsg.columns)
    dataIn_test_withMsg['EstimatedPolarity_XGB_prob'] = dataOut_test['EstimatedPolarity_XGB_prob']
    dataIn_test_withMsg = dataIn_test_withMsg[colNames[0:3] + ['EstimatedPolarity_XGB_prob'] + [colNames[3]]]

    dataIn_test_withMsg.to_csv(fileNameOut_test_withMsg, index=False)

#------------------------------------------------------------------------------


import logging
import logging.handlers

LOG_FILENAME = 'test.log'

logger = logging.getLogger('Fluentd-Test')
logger.setLevel(logging.DEBUG)

log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=200000, backupCount=5)
handler.setLevel(logging.DEBUG)
handler.setFormatter(log_formatter)

logger.addHandler(handler)

def main():
    logger.info('Info Test')
    logger.error('Error Test')
    logger.debug('Debug Test')

if __name__ == '__main__':
    main()


#!/usr/bin/python3

"""
Defines the model of State
"""
from sqlalchemy import MetaData, Column, Integer, String
from sqlalchemy.orm import declarative_base

mymetadata = MetaData()
Base = declarative_base(metadata=mymetadata)

class State(Base):
    """
    Class with id and name attributes of each state
    """
    __tablename__ = 'states'
    id = Column(Integer, unique=True, nullable=False, primary_key=True)
    name = Column(String(128), nullable=False)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Bokeh Visualization Template

This template is a general outline for turning your data into a
visualization using Bokeh.
"""

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

import os
import pandas as pd

# Bokeh Libraries
from bokeh.plotting import figure, show
from bokeh.io import output_file
from bokeh.models import ColumnDataSource, CategoricalColorMapper
from bokeh.models import NumeralTickFormatter
from bokeh.layouts import gridplot

# Get File Directory
WORK_DIR = os.path.dirname((os.path.realpath(__file__)))

# Read the csv files
player_stats = pd.read_csv(WORK_DIR + '/data/2017-18_playerBoxScore.csv',
                           parse_dates=['gmDate'])
team_stats = pd.read_csv(WORK_DIR + '/data/2017-18_teamBoxScore.csv',
                         parse_dates=['gmDate'])
standings = pd.read_csv(WORK_DIR + '/data/2017-18_standings.csv',
                        parse_dates=['stDate'])

# Output inline in the notebook
output_file('phi-gm-linked-selections.html',
            title='76ers Percentages vs. Win-Loss')

# Isolate relevant data
phi_gm_stats_2 = (
    team_stats[(team_stats['teamAbbr'] == 'PHI')
               & (team_stats['seasTyp'] == 'Regular')].
    loc[:, ['gmDate', 'team2P%', 'team3P%', 'teamPTS', 'opptPTS']].sort_values(
        'gmDate'))

# Add game number
phi_gm_stats_2['game_num'] = range(1, len(phi_gm_stats_2) + 1)

# Derive a win_loss column
win_loss = []
for _, row in phi_gm_stats_2.iterrows():

    # If the 76ers score more points, it's a win
    if row['teamPTS'] > row['opptPTS']:
        win_loss.append('W')
    else:
        win_loss.append('L')

# Add the win_loss data to the DataFrame
phi_gm_stats_2['winLoss'] = win_loss

# Store the data in a ColumnDataSource
gm_stats_cds = ColumnDataSource(phi_gm_stats_2)

# Create a CategoricalColorMapper that
# assigns specific colors to wins and losses
win_loss_mapper = CategoricalColorMapper(factors=['W', 'L'],
                                         palette=['Green', 'Red'])

# Specify the tools
toolList = ['lasso_select', 'tap', 'reset', 'save']

# Create a figure relating the percentages
pctFig = figure(title='2PT FG % vs 3PT FG %, 2017-18 Regular Season',
                plot_height=400,
                plot_width=400,
                tools=toolList,
                x_axis_label='2PT FG%',
                y_axis_label='3PT FG%')

# Draw with circle markers
pctFig.circle(x='team2P%',
              y='team3P%',
              source=gm_stats_cds,
              size=12,
              color='black')

# Format the y-axis tick labels as percenages
pctFig.xaxis[0].formatter = NumeralTickFormatter(format='00.0%')
pctFig.yaxis[0].formatter = NumeralTickFormatter(format='00.0%')

# Create a figure relating the totals
totFig = figure(title='Team Points vs Opponent Points, 2017-18 Regular Season',
                plot_height=400,
                plot_width=400,
                tools=toolList,
                x_axis_label='Team Points',
                y_axis_label='Opponent Points')

# Draw with square markers
totFig.square(x='teamPTS',
              y='opptPTS',
              source=gm_stats_cds,
              size=10,
              color=dict(field='winLoss', transform=win_loss_mapper))

# Create layout
grid = gridplot([[pctFig, totFig]])

# Visualize
show(grid)


from models import AppConfig as ac
from controllers import Webdriver as wd
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.common.keys import Keys
from models import UserConfig as uc
from controllers import Variaveis as v

class Tema:
  @staticmethod
  def preencher(data):
    try:
      temaSplitted = data.split('//')
      campoTema = wd.Webdriver.wait["regular"].until(EC.element_to_be_clickable(
        (By.XPATH, ac.AppConfig.xpaths["publicacao"]["temaSelect"])),
        message="Não foi possível localizar ou clicar no campo 'Tema'")
      campoTema.click()
      time.sleep(0.3)

      campoBuscarTema = wd.Webdriver.wait["regular"].until(EC.element_to_be_clickable(
        (By.XPATH, ac.AppConfig.xpaths["publicacao"]["buscarTemaInput"])),
        message="Não foi possível localizar ou clicar no campo 'Buscar tema'")
      wd.Webdriver.nav.execute_script("arguments[0].setAttribute('value',arguments[1])",campoBuscarTema, "")
      time.sleep(0.3)
      campoBuscarTema.send_keys(temaSplitted[0])
      time.sleep(1.5)

      if (len(temaSplitted) == 2):
        i = 1
        while (i < int(temaSplitted[1])):
          campoBuscarTema.send_keys(Keys.ARROW_DOWN)
          time.sleep(0.3)
          i += 1

      campoBuscarTema.send_keys(Keys.ENTER)
      wd.Webdriver.waitLoadingModal()

      campoTemaPreenchido = wd.Webdriver.wait["regular"].until(EC.element_to_be_clickable(
        (By.XPATH, ac.AppConfig.xpaths["publicacao"]["temaSelect"])),
        message="Não foi possível localizar ou clicar no campo 'Tema' preenchido")

      return {"log": f"Tema selecionado: {campoTemaPreenchido.text}", "type": "n"}

    except Exception as e:
      return wd.Webdriver.handleExceptions(e, "tema")

  @staticmethod
  def buscar(filetext):
    try:
      termos = v.Variaveis.atribuirValorVariaveis(uc.UserConfig.obterAutoTemasAssuntos())
      for termo, obj in termos.items():
        if termo.lower() in filetext.lower():
          return {"log": f"Tema identificado: {obj['tema']}", "type": "n", "return": obj["tema"]}
      raise Exception("não foi localizado um termo correspondente a um tema no conteúdo do documento.")
    except Exception as e:
      return {"log": f"Falha ao selecionar tema: {e}", "type": "e", "e": e, "return": False}


"""Explicitly create vshifts

Revision ID: d0841384d5d7
Revises: e3ef522bd3d9
Create Date: 2021-07-05 22:05:52.743905

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd0841384d5d7'
down_revision = 'e3ef522bd3d9'
branch_labels = None
depends_on = None

def upgrade():
    op.create_table (
        "volgisticsshifts",
        sa.Column("_id", sa.Integer, primary_key=True),
        sa.Column("volg_id", sa.Integer, nullable=False),
        sa.Column("assignment", sa.String(), nullable=True),
        sa.Column("site", sa.String(), nullable=True),
        sa.Column("from_date",  sa.Date, nullable=False),
        sa.Column("hours",  sa.DECIMAL, nullable=False)
    )

    op.execute("""CREATE  INDEX vs_volg_id_idx
                    ON public.volgisticsshifts USING btree (volg_id);"""
                    )

    op.create_unique_constraint( "uq_shift", "volgisticsshifts",  ["volg_id", "assignment", "from_date", "hours"] )

def downgrade():
    op.drop_table("volgisticsshifts")


# -*- coding: utf-8 -*-

"""
@date: 2020/11/3 上午10:22
@file: act_helper.py
@author: zj
@description:
"""

import torch.nn as nn

def get_act(type):
    if type == 'ReLU':
        return nn.ReLU
    else:
        raise ValueError(f'{type} does not exists')


import training
import scoring
import deployment
import diagnostics
import reporting
import json
import os
import sys
from pathlib import Path
import subprocess
import pandas as pd

with open('config.json','r') as f:
    config = json.load(f)

##################Check and read new data
#first, read ingestedfiles.txt
file_list_path = os.path.join(config["prod_deployment_path"], config['datalist_fname'])
ingested_files = []
with open(file_list_path, 'r') as f:
    for line in f:
        ingested_files.append(line)
#second, determine whether the source data folder has files that aren't listed in ingestedfiles.txt
data_files = list(Path(config["input_folder_path"]).glob('*.csv'))

##################Deciding whether to proceed, part 1
#if you found new data, you should proceed. otherwise, do end the process here

new_files = False
for file in data_files:
    for complete in ingested_files:
        if complete != file.name:
            new_files = True
            break
if not new_files:
    print("No new data found, exiting!")
    sys.exit()

print("Found new data!")
subprocess.run(args=['python', 'ingestion.py'])

##################Checking for model drift
#check whether the score from the deployed model is different from the score from the model that uses the newest ingested data
old_file_path = os.path.join(config["prod_deployment_path"], config['log_fname'])
with open(old_file_path, 'r') as f:
    score = float(f.readline())

new_data = pd.read_csv(data_files[0])
new_score = scoring.score_model(new_data, production=True)
new_score = 0.1

##################Deciding whether to proceed, part 2
#if you found model drift, you should proceed. otherwise, do end the process here
print('New Score: ' + str(new_score))
print('Old Score: ' + str(score))
if new_score < score:
    print('Model drift has occured, retraining...')
else:
    print('Model has not drifted, exiting...')
    sys.exit()

##################Re-deployment
#if you found evidence for model drift, re-run the deployment.py script
subprocess.run(args=['python', 'training.py'])
subprocess.run(args=['python', 'deployment.py'])

print('Model retrained and deployed!')

##################Diagnostics and reporting
#run diagnostics.py and reporting.py for the re-deployed model
subprocess.run(args=['python', 'apicalls.py'])
subprocess.run(args=['python', 'reporting.py'])


import openai
from time import time, sleep
from halo import Halo
import textwrap
import yaml
import os
from langchain.document_loaders import UnstructuredURLLoader
from langchain.document_loaders import SeleniumURLLoader
from urllib.parse import urlparse

urls = [
    "https://openai.com/policies/usage-policies"
]

loader = SeleniumURLLoader(urls=urls)
data = loader.load()

prompts = [
'Can you give me a very clear explanation of the core assertions, implications, and mechanics elucidated in this paper?',

"Can you explain the value of this in basic terms? Like you're talking to a CEO. So what? What's the bottom line here?",

'Can you give me an analogy or metaphor that will help explain this to a broad audience.',
]

###     file operations

def extract_domain(url):
    parsed_url = urlparse(url)
    domain = parsed_url.netloc
    return domain

# Example usage
website_link = "https://openai.com/policies/usage-policies"
domain_name = extract_domain(website_link)
print(domain_name)

def save_file(filepath, content):
    with open(filepath, 'w', encoding='utf-8') as outfile:
        outfile.write(content)

def open_file(filepath):
    with open(filepath, 'r', encoding='utf-8', errors='ignore') as infile:
        return infile.read()

###     API functions

def chatbot(conversation, model="gpt-3.5-turbo", temperature=0):
    max_retry = 7
    retry = 0
    while True:
        try:
            spinner = Halo(text='Thinking...', spinner='dots')
            spinner.start()

            response = openai.ChatCompletion.create(model=model, messages=conversation, temperature=temperature)
            text = response['choices'][0]['message']['content']

            spinner.stop()

            return text, response['usage']['total_tokens']
        except Exception as oops:
            print(f'\n\nError communicating with OpenAI: "{oops}"')
            if 'maximum context length' in str(oops):
                a = conversation.pop(0)
                print('\n\n DEBUG: Trimming oldest message')
                continue
            retry += 1
            if retry >= max_retry:
                print(f"\n\nExiting due to excessive errors in API: {oops}")
                exit(1)
            print(f'\n\nRetrying in {2 ** (retry - 1) * 5} seconds...')
            sleep(2 ** (retry - 1) * 5)

def chat_print(text):
    formatted_lines = [textwrap.fill(line, width=120, initial_indent='    ', subsequent_indent='    ') for line in text.split('\n')]
    formatted_text = '\n'.join(formatted_lines)
    print('\n\n\nCHATBOT:\n\n%s' % formatted_text)

if __name__ == '__main__':
    # instantiate chatbot, variables
    openai.api_key = open_file('key_openai.txt').strip()

    # Get list of all PDF files in the input folder
    pdf_files = [f for f in os.listdir('input/') if f.endswith('.pdf')]

    for pdf_file in pdf_files:
        # Check if the report already exists in the output folder
        filename = 'output_web/' + pdf_file.replace('.pdf', '.txt')
        if os.path.exists(filename):
            continue

        # Open the PDF file
        with open('input/' + pdf_file, 'rb') as file:
            pdf_reader = PyPDF2.PdfReader(file)
            paper = ''
            for page_num in list(range(0,len(pdf_reader.pages))):
                page = pdf_reader.pages[page_num]
                paper += page.extract_text()

        if len(paper) > 22000:
            paper = paper[0:22000]
        ALL_MESSAGES = [{'role':'system', 'content': paper}]
        report = ''
        for p in prompts:
            ALL_MESSAGES.append({'role':'user', 'content': p})
            response, tokens = chatbot(ALL_MESSAGES)
            chat_print(response)
            ALL_MESSAGES.append({'role':'assistant', 'content': response})
            report += '\n\n\n\nQ: %s\n\nA: %s' % (p, response)

        # Save the report in the output folder with the same name as the PDF file but with .txt extension
        save_file(filename, report.strip())

# Create a new BeautifulSoup object and define the basic structure of the HTML document
soup = BeautifulSoup("<title>Consolidated Policies</title>", "html.parser")

# For each text file
for c in data:
    content = c.page_content

    # Create a new section in the HTML document for this file
    section = soup.new_tag('section')

    # Add a header with the name of the file (without the .txt extension)
    header = soup.new_tag('h1')
    header.string = c.metadata
    section.append(header)

    # Split the content into Q&A pairs and add them to the section
    section.append(content)

    # Add the section to the body of the HTML document
    soup.body.append(section)

# Write the HTML document to a file
with open('web_report.html', 'w') as file:
    file.write(str(soup.prettify()))


from PIL import Image
from pathlib import Path
from torchvision import transforms
from torchvision.datasets import MNIST
from tqdm import tqdm
import cv2
import numpy as np
import torch

class MNISTF1(MNIST):
    SIZE = 64
    SMALL_START = 18
    SMALL_END = 46

    def __init__(self,
                 root,
                 train=True,
                 transform=None,
                 target_transform=None,
                 download=False):
        super(MNISTF1, self).__init__(root, train, transform, target_transform,
                                      download)
        self.root = root
        self.transform = transform
        self.train = train  # training set or test set

        if download:
            self.download()

        if not self._check_exists():
            raise RuntimeError('Dataset not found.' +
                               ' You can use download=True to download it')

        if self.train:
            data_file = self.training_file
        else:
            data_file = self.test_file
        self.data, self.targets = torch.load(
            Path(self.root, self.processed_folder, data_file))

        self.num_positives = 0
        self._build()

    def _build(self):
        data_, targets_ = [], []

        print("Building dataset...")
        for [img, target] in tqdm(zip(self.data, self.targets)):
            img = img.numpy()

            # We create "small" and
            small = np.zeros([self.SIZE, self.SIZE])
            small[self.SMALL_START:self.SMALL_END, self.SMALL_START:self.
                  SMALL_END] += img
            data_.append(small)
            targets_.append((target, 0))

            # "big" versions of each image in the MNIST dataset
            big = cv2.resize(img,
                             dsize=(self.SIZE, self.SIZE),
                             interpolation=cv2.INTER_CUBIC)

            data_.append(big)
            targets_.append((target, 1))

            # We count the number of positive examples for use in our algorithm
            self.num_positives += 1

        self.data = data_
        self.targets = targets_

    def __getitem__(self, index):
        img, target = self.data[index], self.targets[index]

        img = Image.fromarray(img)
        if self.transform is not None:
            img = self.transform(img)
        else:
            img = transforms.ToTensor()(img)

        # (class, big_or_small, index)
        target = torch.tensor([int(target[0]), int(target[1]), index])

        return img, target


import os
import webapp2
import jinja2
import urllib2
from xml.dom import minidom

from google.appengine.ext import db

template_dir = os.path.join(os.path.dirname(__file__), "templates")
jinja_env = jinja2.Environment(loader = jinja2.FileSystemLoader(template_dir),
                                autoescape = True)

IP_URL = "http://ip-api.com/xml/"

def get_coords(ip):
    ip = "4.2.2.2"
    url = IP_URL + ip
    content = None
    try:
        content = urllib2.urlopen(url).read()
    except URLError:
        return

    if content:
        d = minidom.parseString(content)
        lat_object = d.getElementsByTagName("lat")
        lon_object = d.getElementsByTagName("lon")
        if lat_object and lon_object and lat_object[0].childNodes[0].nodeValue and lon_object[0].childNodes[0].nodeValue:
            return db.GeoPt(lat_object[0].childNodes[0].nodeValue, lon_object[0].childNodes[0].nodeValue)

class Handler(webapp2.RequestHandler):
    def write(self, *a, **kw):
        self.response.out.write(*a, **kw)

    def render_str(self, template, **params):
        t = jinja_env.get_template(template)
        return t.render(params)

    def render(self, template, **kw):
        self.write(self.render_str(template, **kw))

class Art(db.Model):
    title = db.StringProperty(required = True)
    art = db.TextProperty(required = True)
    created = db.DateTimeProperty(auto_now_add = True)
    coords = db.GeoPtProperty()

class MainPage(Handler):
    def render_front(self, title="", art="", error=""):
        arts = db.GqlQuery( "SELECT * FROM Art "
                            "ORDER BY created DESC "
                            "LIMIT 10")
        self.render("front.html", title=title, art=art, error=error, arts = arts)

    def get(self):
        self.write(repr(get_coords(self.request.remote_addr)))
        self.render_front()

    def post(self):
        title = self.request.get("title")
        art = self.request.get("art")

        if title and art:
            a = Art(title = title, art = art)
            coords = get_coords(self.request.remote_addr)
            if coords:
                a.coords = coords
            a.put()

            self.redirect("/")
        else:
            error = "we need both a title and some artwork"
            self.render_front(title, art, error)

app = webapp2.WSGIApplication([ ('/', MainPage)], debug=True)


from scipy.fftpack import fft
import matplotlib.pyplot as plt
import numpy as np
from obspy.signal.util import next_pow_2

#Wiggle plots
def wigb(data,t_axis,offsets,amx):
	''' This function plots seismic traces as wiggle plot
	data: Is a matrix of seismic data where the columns are the seismic traces
	t_axis: Is your time axis
	offsets: vector of each trace offset
	amx: is a screen gain to the data'''
	amx2=amx*np.max(data)
	i=0
	for offset in offsets:

		x=np.divide(data[:,i],amx2)+offset
		i=i+1

		plt.plot(x,t_axis,'k')
		plt.fill_betweenx(t_axis,offset,x,where=(x>offset),color='k')

	plt.ylabel('Time (s)')
	plt.xlabel('Offset (m)')
	plt.xlim((offsets[0]-1.,offsets[-1]+1.))
	plt.ylim((0.0,t_axis[-1]))
	plt.gca().invert_yaxis()
	#plt.show()

def pick_wigb(data,t_axis,offsets,filename,amx):
	''' This function enables picking and saving on a seismic wiggle plot
	data: Is a matrix of seismic data where the columns are the seismic traces
	t_axis: Is your time axis
	offsets: vector of each trace offset
	filename: Is the output filename
	amx: is a screen gain to the data'''
	amx2=amx*np.max(data)
	i=0
	for offset in offsets:

		x=np.divide(data[:,i],amx2)+offset
		i=i+1

		plt.plot(x,t_axis,'k')
		plt.fill_betweenx(t_axis,offset,x,where=(x>offset),color='k')

	plt.ylabel('Time (s)')
	plt.xlabel('Offset (m)')
	plt.gca().invert_yaxis()
	plt.xlim((offsets[0]-1.,offsets[-1]+1.))
	picks = plt.ginput( n=0, timeout=0, show_clicks=True, mouse_add=1, mouse_pop=3, mouse_stop=2)
	np.savetxt(filename, picks)
	plt.show()

def interpret_refract(name):
	'''Select the maximum time each event is observed'''

	M=np.loadtxt(name)

	x=M[:,0]
	t_axis=M[:,1]
	plt.plot(x,t_axis,'k.-')
	plt.ylabel('Time (s)')
	plt.xlabel('Offset (m)')
	picks = plt.ginput( n=0, timeout=0, show_clicks=True, mouse_add=1, mouse_pop=3, mouse_stop=2)
	plt.show()
	ninterfaces=len(picks) #number of layers
	M2=np.array([ninterfaces,0])

	M4=np.vstack([M2,picks,M])
	filename="r_"+name
	np.savetxt(filename, M4)

def ampli_spec(D,header):
	'''Plots the amplitude spectrum for each seismic trace of data matrix D
	in a same graph'''

	dt=header["dt"]
	nx=header['tracl']
	ns=header['ns']
	k=next_pow_2(ns)
	nf=4*(2^k)

	t=np.linspace(0.0,ns*dt,ns)
	tf=np.linspace(0.0,1.0/(2*dt),nf/2)
	i=0
	while i<nx:
		df=fft(D[:,i],nf)
		i=i+1
		plt.plot(tf,2.0/dt*np.abs(df[0:int(nf/2)]))

	plt.ylabel('Amplitude')
	plt.xlabel('Frequency (Hz)')
	plt.grid()
	plt.show()


import re, json, os

def extract_question(in_file, out_file):
    cc1_all_text = open(in_file, 'r', encoding='utf8').read()

    rlt = []
    for item in re.finditer(r'
', cc1_all_text):
        rlt.append(item.start())

    print(f'find {len(rlt)}')

    def has_images(question_body):
        img_s = question_body.find('<img ')
        if img_s != -1:
            return True
        else:
            return False

    question_list = []
    need_images = []
    for pos in rlt:
        span1 = cc1_all_text.find('', pos) + 6
        span1_end = cc1_all_text.find('', span1)
        question_order = cc1_all_text[span1 : span1_end]

        question_body_s = cc1_all_text.find('', span1_end) + 5
        question_body_e = cc1_all_text.find('
', question_body_s)
        question_body = cc1_all_text[question_body_s : question_body_e].replace('\n', '').replace('\r', '').replace('
', '\n')
        question_body = re.sub(r'^\s+', '', question_body)

        last_end = question_body_e
        def find_selection():
            nonlocal last_end
            sel_s = cc1_all_text.find('
', last_end)
            sel_s = cc1_all_text.find('<input', sel_s)
            sel_s = cc1_all_text.find('>', sel_s) + 1
            sel_e = cc1_all_text.find('
', sel_s)
            sel = cc1_all_text[sel_s:sel_e].replace('\n', ' ').replace('\r', '').replace('
', '\n')
            sel = re.sub(r'^\s+', '', sel)
            last_end = sel_e + 4
            return sel

        a = find_selection()
        b = find_selection()
        c = find_selection()
        d = find_selection()

        answer_s = cc1_all_text.find('class="answer_right">', last_end) + 21
        answer_e = cc1_all_text.find('', answer_s)
        answer = cc1_all_text[answer_s: answer_e]

        my_answer_s = cc1_all_text.find('class="answer_wrong">', answer_e) + 21
        my_answer_e = cc1_all_text.find('', my_answer_s)
        my_answer = cc1_all_text[my_answer_s: my_answer_e]

        jiexi_s = cc1_all_text.find('
', my_answer_e) + 25
        jiexi_e = cc1_all_text.find('
', jiexi_s)
        jiexi = cc1_all_text[jiexi_s:jiexi_e]
        jiexi = re.sub('^\s+', '', jiexi, flags=re.M)

        question_list.append([question_order, question_body, a, b, c, d, answer, my_answer, jiexi])
        if has_images(question_body):
            need_images.append(question_order)

        print(f'{question_order} {question_body[:50]}...')

    print('write to file: ', out_file,', count: ' , len(question_list))
    print(f'need images: ' + ','.join(need_images))
    open(out_file, 'w', encoding='utf8').write(json.dumps(question_list, ensure_ascii=False, indent=2))

html_files = os.listdir('html')
json_files = list(map(lambda x:x[:-5] + '.json', html_files))

for i in range(len(html_files)):
    in_file =  'html/' + html_files[i]
    out_file = 'json/' + json_files[i]
    print('process', in_file)
    if os.path.isfile(out_file):
        print('\t', 'already exists, skip', in_file)
    else:
        extract_question(in_file, out_file)


import json

file = open('weather.txt', 'r')
rows = file.readlines()

print(len(rows))
Table = []
for row in rows:
	Table.append([int(row[2:5]), int(row[6:14]), int(row[15:20]), int(row[21:26]), int(row[27:])])

print(Table[:10])
Names = ['Station', 'Date', 'Temp', 'RainTime', 'RainAmmount']

data = []
for i in range(0, len(Table)):
	month = {}
	for j in range(0, len(Names)):
		month[Names[j]] = Table[i][j]
	data.append(month)

with open('weather.json', 'w') as outfile:
    json.dump(data, outfile)


import requests
import re
import os
url='http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=detail&fr=&sf=1&fmq=1447473655189_R&pv=&ic=0&nc=1&z=&se=&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=%E9%95%BF%E8%80%85%E8%9B%A4'
html = requests.get(url).text
urls = re.findall('"objURL":"(.*?)"', html)
if not os.path.isdir('C:\\Users\\wangz\\Desktop\\dytt'):
    os.mkdir('C:\\Users\\wangz\\Desktop\\dytt')

index = 1
for url in urls:
    print("Doading:",url)
    res=requests.get(url)
    filename = os.path.join('C:\\Users\\wangz\\Desktop\\dytt', str(index) + ".jpg")
    with open(filename, 'wb') as f:
        f.write(res.content)
        index += 1


from django.conf.urls import url
from .views import PersonaDetailView, PersonaListView, ResponsableListView
from .views import InstitucionListView
from .views import ParticipanteListView

urlpatterns = [
    url(r'^responsables/$', ResponsableListView.as_view(),
        name='responsable-list'),
    url(r'^personas/$', PersonaListView.as_view(), name='personas-list'),
    url(r'^personas/(?P<id>\d+)/$', PersonaDetailView.as_view(),
        name='personas-detail'),
    url(r'^instituciones/$', InstitucionListView.as_view(),
        name='instituciones-list'),
    url(r'^participantes/$', ParticipanteListView.as_view(),
        name='participante-list'),
]


# -*- coding:utf-8 -*-
'''
Created on 2017年11月14日

@author: 任胜勇

这是一个关于清晰经纬度的数据
现在有两种方法
现有的数据是每个商铺对应用户的id,经纬度
1.
    用用户的经纬度找一个二维平面的最高区间，保留边界用户，删除用户
2.
    以商铺的经纬度为中心画圆，寻找一个最合适的半径作为范围划分
3.
    计算信息与商店的距离，去掉一定数目距离的大值（认为异常）
我采用了第三种方法
'''

import os
import numpy as np
import pandas as pd
from tqdm import tqdm
from math import radians, cos, sin, asin, sqrt

def haversine(lon1, lat1, lon2, lat2):  # 经度1，纬度1，经度2，纬度2 （十进制度数）
    """
    Calculate the great circle distance between two points
    on the earth (specified in decimal degrees)
    """
    # 将十进制度数转化为弧度
    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])

    # haversine公式
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
    c = 2 * asin(sqrt(a))
    r = 6371  # 地球平均半径，单位为公里
    return c * r * 1000

# 含有shop对应用户数据的pd文件路径
pd_path = r'G:\customer&fixed-position\data\shop\person2shop/'
# 保存了shop路径的文件
npy_path = r'G:\customer&fixed-position\data\shop\shop_lc/'
shop_list = os.listdir(pd_path)

shop_lc = os.listdir(npy_path)
shop_dis = dict();
dis_sum = 0;
dis_real_sum = 0;
shop_dis_real = dict()
for sp in tqdm(shop_list):

    pr_dis = dict()
    pr_dis['dis'] = [];
    pr_dis['time_stamp'] = [];
    pr_dis['user_id'] = []
    pr_dis['longitude'] = [];
    pr_dis['latitude'] = []
    shop_lc = np.load(npy_path + sp.split('.')[0] + '.npy')
    # 获取店铺的经纬度
    s_lg = shop_lc[0];
    s_lt = shop_lc[1]
    shop_data = pd.read_csv(pd_path + sp)
    # 初始化，新增distance
    shop_data['distance_sq'] = 999
    # 计算data距离

    for _, sp_data in shop_data.iterrows():
        lg = sp_data.longitude;
        lt = sp_data.latitude
        dis = haversine(lg, lt, shop_lc[0], shop_lc[1])
        pr_dis['dis'].append(dis)
        pr_dis['time_stamp'].append(sp_data.time_stamp)
        pr_dis['user_id'].append(sp_data.user_id)
        # 经纬度有小数点记录不精确，使用字符串记录
        pr_dis['longitude'].append(str(sp_data.longitude))
        pr_dis['latitude'].append(str(sp_data.latitude))

    df = pd.DataFrame({'user_id': pr_dis['user_id'],
                       'dis': pr_dis['dis'],
                       'time_stamp': pr_dis['time_stamp'],
                       'longitude': pr_dis['longitude'],
                       'latitude': pr_dis['latitude']})
    df.to_csv(r'G:\customer&fixed-position\data\shop\shop_dis/' + sp.split('.')[0] + '.csv')


import argparse
import textwrap
import sys
import os

def run_matlab_parser():
    usage = '''\
        run_matlab <command> [options]
        Commands:
            install            Installation of the MATLAB compiler runtime (MCR)
            run                Running the MATLAB function after installation
        Run run_matlab <command> -h for help on a specific command.
        '''
    parser = argparse.ArgumentParser(
        description='run_matlab: Command Line MATLAB Function Caller with Automatic MATLAB Compiler Runtime (MCR) Installation',
        usage=textwrap.dedent(usage)
    )

    from .version import __version__
    parser.add_argument('--version', action='version', version=f'run_matlab {__version__}')

    parser.add_argument('command', nargs='?', help='Subcommand to run')

    return parser

def install_parser():
    parser = MyParser(
        description='Installation of the MATLAB compiler runtime (MCR)',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        prog='run_matlab install'
    )

    parser.add_argument(
        '-d',
        dest='installation_dir',
        type=str,
        default=os.path.expanduser('~')+'/run_matlab',
        help='Directory wherein the MATLAB compiler runtime (MCR) will be installed.')

    parser.add_argument(
        '-v',
        dest='matlab_version',
        type=str,
        default='R2013b',
        help='MATLAB version.')

    parser.add_argument(
        '-r',
        dest='runtime_version',
        type=str,
        default='8.2',
        help='MATLAB Compiler Runtime version.')

    return parser

def run_parser():
    parser = MyParser(
        description='Running the MATLAB function after installation',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        prog='run_matlab run'
    )

    parser.add_argument(
        'function_dir',
        type=str,
        help='Directory wherein the standalone MATLAB function is located.')

    parser.add_argument(
        'function_name',
        type=str,
        help='Name of the standalone MATLAB function.')

    parser.add_argument(
        'function_args',
        nargs='*',
        type=str,
        help='Arguments for the standalone MATLAB function.')

    parser.add_argument(
        '-d',
        dest='installation_dir',
        type=str,
        default=os.path.expanduser('~')+'/run_matlab',
        help='Directory wherein the MATLAB compiler runtime (MCR) was installed.')

    parser.add_argument(
        '-v',
        dest='matlab_version',
        type=str,
        default='R2013b',
        help='MATLAB version.')

    parser.add_argument(
        '-r',
        dest='runtime_version',
        type=str,
        default='8.2',
        help='MATLAB Compiler Runtime version.')

    return parser

class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)


import yfinance as yf
import streamlit as st
import pandas as pd
import numpy as np
from datetime import date
from datetime import timedelta
import matplotlib

st.write("""
# Simple Stock Price App
Shown are the stock closing price and volume of Google!
""")

today1 = date.today()
today0= today1 - timedelta(days = 2) #yesterday
now= today0.strftime("%Y-%m-%d")
# https://towardsdatascience.com/how-to-get-stock-data-using-python-c0de1df17e75
#define the ticker symbol
tickerSymbols = ["MSFT", "GOOGL", "AAPL"]
#get data on this ticker
#tickerData = yf.tickerSymbols
data = yf.download(tickerSymbols, start='2010-01-20', end=now)
#print(tickerData)
#get the historical prices for this ticker
# tickers =['FNF', 'ASML', 'GOOGL', 'CVS']
# data = yf.download(tickers, group_by="ticker", period='1y')
#tickerDf = yf.download(tickerData, period='1y') #, start='2020-5-31', end='2020-5-31')
# Open	High	Low	Close	Volume	Dividends	Stock Splits
#print(tickerDf)

st.write("""
## Closing Price
""")
st.line_chart(data.Close)
st.write("""
## Volume Price
""")
st.line_chart(data.Volume)

# chart_data = pd.DataFrame(
#      np.random.randn(20, 3),
#      columns=['a', 'b', 'c'])

# st.line_chart(chart_data)


from dataclasses import dataclass
from typing import List

from .hparams import HyperParams

@dataclass
class ROMEHyperParams(HyperParams):
    # Method
    layers: List[int]
    fact_token: str
    v_num_grad_steps: int
    v_lr: float
    v_weight_decay: float
    clamp_norm_factor: float
    kl_factor: float

    # Module templates
    rewrite_module_tmp: str
    mlp_module_tmp: str

    @classmethod
    def from_name(cls, name: str):
        data = dict(
            fact_token='subject_last',
            v_num_grad_steps=20,
            v_lr=1e-1,
            v_weight_decay=1e-2,
            clamp_norm_factor=4,
            kl_factor=0.0625,
        )
        if name == 'llama-7b':
            data.update(
                dict(
                    layers=[5],
                    rewrite_module_tmp='model.layers.{}.mlp.down_proj',
                    mlp_module_tmp='model.layers.{}.mlp',
                ))
        elif name == 'llama-13b':
            data.update(
                dict(
                    layers=[10],
                    rewrite_module_tmp='model.layers.{}.mlp.down_proj',
                    mlp_module_tmp='model.layers.{}.mlp',
                ))
        elif name == 'chatglm-6b':
            data.update(
                dict(
                    layers=[5],
                    rewrite_module_tmp=
                    'transformer.encoder.layers.{}.mlp.dense_4h_to_h',
                    mlp_module_tmp='transformer.encoder.layers.{}.mlp',
                ))
        else:
            raise NotImplementedError(f'{name} not supported.')

        return cls(**data)


#!/usr/bin/env python2
#-*- coding:utf-8 -*-

import os
import argparse
import multiprocessing

def filter_data(input_file, output_file):
    print('Process %s' % input_file)
    with open(input_file, 'r') as inf:
        with open(output_file, 'w') as of:
            for line in inf:
                try:
                    line.decode('utf-8')
                    of.write(line)
                except:
                    continue

def filter_data_list(input_list, output_folder, process_count=2):
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    pool = multiprocessing.Pool(process_count)
    with open(input_list) as inf:
        for line in inf:
            line = line.strip()
            if line.startswith('#'):
                continue
            output_file_name = os.path.basename(line)
            output_file_path = os.path.join(output_folder, output_file_name)
            pool.apply_async(filter_data, (line, output_file_path))
    pool.close()
    pool.join()

def filter_data_folder(input_folder, output_folder, process_count=2):
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    pool = multiprocessing.Pool(process_count)
    for root, _, files in os.walk(input_folder):
        for f in files:
            input_path = os.path.join(root, f)
            output_path = os.path.join(output_folder, f)
            pool.apply_async(filter_data, (input_path, output_path))
    pool.close()
    pool.join()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--input-file', '-i', nargs='+', default=[])
    parser.add_argument('--output-file', '-o', default=[])
    parser.add_argument('--input-list', '-il', nargs='+', default=[])
    parser.add_argument('--input-folder', '-if', nargs='+', default=[])
    parser.add_argument('--output-folder', '-of', default='')
    parser.add_argument('--process-count', default=0)
    args = parser.parse_args()

    if not args.input_file and not args.input_list and not args.input_folder:
        parser.error('No input data.')
    if not args.output_file and not args.output_folder:
        parser.error('No output path.')
    if (args.input_list or args.input_folder) and not args.output_folder:
        parser.error('"--input-list" or "--input-folder" is set, '
                     'but missing "--output-folder".')
    if len(args.input_file) > 1 and not args.output_folder:
        parser.error('input files are more than one, '
                     'but missing "--output-folder".')

    cpu_core = multiprocessing.cpu_count() if not args.process_count else args.process_count

    if args.input_list:
        for input_list in args.input_list:
            filter_data_list(input_list, args.output_folder, cpu_core)
    if args.input_folder:
        for folder in args.input_folder:
            filter_data_folder(folder, args.output_folder, cpu_core)
    if len(args.input_file) == 1:
        if not os.path.exists(args.output_folder):
            os.makedirs(args.output_folder)
        filter_data(args.input_file[0], args.output_file)
    elif len(args.input_file) > 1:
        if not os.path.exists(args.output_folder):
            os.makedirs(args.output_folder)
        for f in args.input_file:
            file_name = os.path.basename(f)
            output_file_path = os.path.join(args.output_folder, file_name)
            filter_data(f, output_file_path)


import sys
from io import StringIO
import unittest

class TestClass(unittest.TestCase):
    maxDiff = None
    def assertIO(self, input, output):
        stdout, stdin = sys.stdout, sys.stdin
        sys.stdout, sys.stdin = StringIO(), StringIO(input)
        resolve()
        sys.stdout.seek(0)
        out = sys.stdout.read()[:-1]
        sys.stdout, sys.stdin = stdout, stdin
        self.assertEqual(out, output)

    def test_Sample_Input_1(self):
        input = """3 7"""
        output = """2"""

    def test_Sample_Input_2(self):
        input = """4 10"""
        output = """12"""

    def test_Sample_Input_3(self):
        input = """1 1000000"""
        output = """392047955148"""

import sys
sys.setrecursionlimit(500*500)

from math import gcd
from functools import reduce
from itertools import product
from itertools import combinations
from itertools import accumulate # 累積和作るやつ
import numpy as np
from collections import deque
from collections import defaultdict
from heapq import heappop, heappush

alpha2num = lambda c: ord(c) - ord('a')
num2alpha = lambda c: chr(c+97)
popcnt = lambda x: bin(x).count("1")

dx = [-1, 0, 1, 0]
dy = [0, -1, 0, 1]

def resolve():
  # https://atcoder.jp/contests/abc206/editorial/2129
  L, R = map(int, input().split(" "))
  L = max(2, L)
  def f1(l, r):
    return ((r-l+1)*(r-l))//2

  from collections import defaultdict
  # f2 => g
  memo = [defaultdict(int) for _ in range(R+1)]
  def f2(l, r):
    if l == r: return 0
    # print(l, r)
    if memo[r][l] != 0: return memo[r][l]
    temp = 0
    for i in range(2, r):
      temp += f2((l-1)//i+1, r//i)
    # print(l, r, f1(l, r) - temp)
    memo[r][l] = f1(l, r) - temp
    return memo[r][l]

  f3 = sum([R//i-1 for i in range(L, R+1)])

  ans = 2*(f1(L, R) - f2(L, R) - f3)
  print(ans)

if __name__ == "__main__":
  unittest.main()


"""
Script for cleaning up old preview environments to save resources.
"""

import os
from datetime import datetime, timedelta
import re
import requests
import pytz

utc=pytz.UTC

def job():
    try:
        HUMANITEC_URL = os.getenv("HUMANITEC_URL", default="https://api.humanitec.io")
        HUMANITEC_TOKEN = os.getenv("HUMANITEC_TOKEN")
        HUMANITEC_ORG = os.getenv("HUMANITEC_ORG")
        HUMANITEC_APP = os.getenv("HUMANITEC_APP")
        HUMANITEC_ENV_REGEX = os.getenv("HUMANITEC_ENV_REGEX")
        HUMANITEC_ENV_MAX_AGE_DAYS = os.getenv("HUMANITEC_ENV_MAX_AGE_DAYS", default="7")
    except:
        print("Missing environment variables.")
        exit()

    preview_env_regex = re.compile(rf"{HUMANITEC_ENV_REGEX}")
    max_env_age = datetime.utcnow() - timedelta(days=float(HUMANITEC_ENV_MAX_AGE_DAYS))
    max_env_age = utc.localize(max_env_age)

    headers = {"Authorization": f"Bearer {HUMANITEC_TOKEN}", "Content-Type": "application/json"}
    resp_env_list = requests.get(f"{HUMANITEC_URL}/orgs/{HUMANITEC_ORG}/apps/{HUMANITEC_APP}/envs", headers=headers)
    env_list = resp_env_list.json()
    if resp_env_list.status_code == 200:
        for env in env_list:
            if re.fullmatch(preview_env_regex, env['id']):
                print(f"ENV: {env['id']}, last deployed: {env['last_deploy']['created_at']} matches regex.")
                env_last_deploy_date = datetime.strptime(env['last_deploy']['created_at'], "%Y-%m-%dT%H:%M:%S.%f%z")
                if env_last_deploy_date <= max_env_age:
                    print(f"{env['id']}'s last deployment is older than {HUMANITEC_ENV_MAX_AGE_DAYS} day(s).")
                    resp_env_delete = requests.delete(f"{HUMANITEC_URL}/orgs/{HUMANITEC_ORG}/apps/{HUMANITEC_APP}/envs/{env['id']}", headers=headers)
                    if resp_env_delete.status_code == 204:
                        print(f"ENV {env['id']} has been deleted.")
                    else:
                        print(f"Delete request failed with status {resp_env_delete.status_code}")
    else:
        print(f"Listing environments failed with status {resp_env_list.status_code}")

if __name__ == "__main__":
    job()