import torch
from torch import nn
from opt import get_opts
import os
import glob
import imageio
import numpy as np
import cv2
from einops import rearrange

# data
from torch.utils.data import DataLoader
from datasets import dataset_dict

from datasets.ray_utils import axisangle_to_R, get_rays

# models
from kornia.utils.grid import create_meshgrid3d
from models.networks import NGP
from models.rendering import render, MAX_SAMPLES

# optimizer, losses
from apex.optimizers import FusedAdam
from torch.optim.lr_scheduler import CosineAnnealingLR
from losses import NeRFLoss

# metrics
from torchmetrics import (
    PeakSignalNoiseRatio,
    StructuralSimilarityIndexMeasure
)
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity

from utils import slim_ckpt

import warnings; warnings.filterwarnings("ignore")

import time

def depth2img(depth):
    depth = (depth-depth.min())/(depth.max()-depth.min())
    depth_img = cv2.applyColorMap((depth*255).astype(np.uint8),
                                  cv2.COLORMAP_TURBO)

    return depth_img

### Args
hparams = get_opts()
if hparams.val_only and (not hparams.ckpt_path):
    raise ValueError('You need to provide a @ckpt_path for validation!')

device = "cuda"

### NeRF hyperparameters
S = 16
loss_func = NeRFLoss()
train_psnr = PeakSignalNoiseRatio(data_range=1)
val_psnr = PeakSignalNoiseRatio(data_range=1)
val_ssim = StructuralSimilarityIndexMeasure(data_range=1)

if hparams.eval_lpips:
    val_lpips = LearnedPerceptualImagePatchSimilarity('vgg')
    for p in val_lpips.net.parameters():
        p.requires_grad = False

model = NGP(scale=hparams.scale)
G = model.grid_size
model.register_buffer('density_grid',
    torch.zeros(model.cascades, G**3))
model.register_buffer('grid_coords',
    create_meshgrid3d(G, G, G, False, dtype=torch.int32).reshape(-1, 3))
model.to(device, dtype=torch.float)

### Dataset Inits
dataset = dataset_dict[hparams.dataset_name]
kwargs = {'root_dir': hparams.root_dir,
          'downsample': hparams.downsample}
train_dataset = dataset(split=hparams.split, **kwargs)
train_dataset.batch_size = hparams.batch_size

test_dataset = dataset(split='test', **kwargs)

train_dataloader = DataLoader(train_dataset,
                          num_workers=16,
                          persistent_workers=True,
                          batch_size=None,
                          pin_memory=True)

test_dataloader = DataLoader(test_dataset,
                          num_workers=8,
                          batch_size=None,
                          pin_memory=True)

directions = train_dataset.directions.to(device)
train_poses = train_dataset.poses.to(device)

print(directions.shape)
print(train_poses.shape)
print(test_dataset.poses.shape)
print()

### Optimizers --> note here there is no optimization for extrinsics yet
net_params = model.parameters()

net_opt = FusedAdam(net_params, hparams.lr, eps=1e-15)

net_sch = CosineAnnealingLR(net_opt,
                            hparams.num_epochs,
                            hparams.lr/30)

model.mark_invisible_cells(train_dataset.K.to(device),
                                        train_poses,
                                        train_dataset.img_wh)

model.train()
global_step = 0

for epoch in range(hparams.num_epochs):
    for batch in train_dataloader:

        ### Move to cuda
        batch["rgb"] = batch["rgb"].to(device)
        batch["img_idxs"] = batch["img_idxs"].to(device)

        if global_step%S == 0:
            model.update_density_grid(0.01*MAX_SAMPLES/3**0.5,
                                           warmup=global_step<256,
                                           erode=hparams.dataset_name=='colmap')

        poses = train_poses[batch['img_idxs']]
        rays_o, rays_d = get_rays(directions[batch['pix_idxs']], poses)

        kwargs = {'test_time': False}
        if hparams.dataset_name in ['colmap', 'nerfpp']:
            kwargs['exp_step_factor'] = 1/256

        results = render(model, rays_o, rays_d, **kwargs)

        loss_d = loss_func(results, batch)

        loss = sum(lo.mean() for lo in loss_d.values())

        print(loss)
        print(loss.dtype)

        net_opt.zero_grad()
        print("zero grad")
        loss.backward()
        print("backward")
        net_opt.step()
        print("opt step")

        net_sch.step()
        print("sched step")

        global_step += 1

        print(loss)
        print(results['total_samples']/len(rays_o))

        ### Logging
        print("PSNR")
        with torch.no_grad():
            print(train_psnr(results['rgb'], batch['rgb']))
        # print(psnr)

        exit()


from typing import Dict, List
import re

def confidence_cm(
    region: List[float],
    subfam_counts: Dict[str, float],
    subfams: List[str],
    subfam_rows: List[int],
    repeats: int,
    node_confidence_bool: bool,
) -> List[float]:
    """
    Computes confidence values for competing annotations using alignment and tandem
    repeat scores. Loops through the array once to find sum of 2^every_hit_score in
    region, then loops back through to calculate confidence. Converts the alignment
    score to account for lambda before summing.

    If command line option for subfam_counts, this is included in confidence math.

    input:
    region: list of scores for competing annotations
    subfam_counts: dict that maps subfam name to it's count info
    subfams: list of subfam names
    subfam_rows: list of subfamily rows that correspond to the the subfams of the region scores
    repeats: number of tandem repeat scores found at the end of the region
    node_confidence_bool: if False (0) - confidence for filling DP matrices, if True - confidence for nodes

    output:
    confidence_list: list of confidence values for competing annotations, each input alignment
    and tandem repeat score will have one output confidence score

    >>> counts = {"s1": .33, "s2": .33, "s3": .33}
    >>> subs = ["s1", "s2", "s3"]
    >>> conf = confidence_cm([2, 1, 1], counts, subs, [0, 1, 2], 0, False)
    >>> f"{conf[0]:.2f}"
    '0.50'
    >>> f"{conf[1]:.2f}"
    '0.25'
    >>> f"{conf[2]:.2f}"
    '0.25'

    >>> conf = confidence_cm([0, 100, 100], 0, subs, [0, 1, 2], 0, False)
    >>> f"{conf[0]:.2f}"
    '0.01'

    >>> conf = confidence_cm([0, 100, 100], 0, subs, [0, 1, 2], 0, True)
    >>> f"{conf[0]:.2f}"
    '0.00'

    >>> counts = {"s1": .31, "s2": .31, "s3": .31, "Tandem Repeat": .06}
    >>> subs = ["s1", "s2", "s3", "Tandem Repeat"]
    >>> conf = confidence_cm([2, 1, 0.7], counts, subs, [0, 1, 3], 1, False)
    >>> f"{conf[0]:.2f}"
    '0.65'
    >>> f"{conf[1]:.2f}"
    '0.32'
    >>> f"{conf[2]:.2f}"
    '0.03'
    """

    confidence_list: List[float] = []
    score_total: int = 0

    # if command line option to include subfam_counts
    if subfam_counts:
        # alignment scores
        for index in range(len(region) - repeats):
            subfam: str = subfams[subfam_rows[index]]
            m = re.search(r"(.+?)#.+", subfams[subfam_rows[index]])
            if m:
                subfam = m.group(1)
            converted_score = (2 ** int(region[index])) * subfam_counts[subfam]
            confidence_list.append(converted_score)
            score_total += converted_score
        # TR scores
        for index in range(len(region) - repeats, len(region)):
            subfam = subfams[subfam_rows[index]]
            m = re.search(r"(.+?)#.+", subfams[subfam_rows[index]])
            if m:
                subfam = m.group(1)
            tr_score = (2 ** int(region[index])) * subfam_counts[subfam]
            confidence_list.append(tr_score)
            score_total += tr_score

    # don't include subfam counts (default)
    else:
        # alignment scores
        for index in range(len(region) - repeats):
            converted_score = 2 ** int(region[index])
            confidence_list.append(converted_score)
            score_total += converted_score
        # TR scores
        for index in range(len(region) - repeats, len(region)):
            tr_score = 2 ** int(region[index])
            confidence_list.append(tr_score)
            score_total += tr_score

    for index in range(len(region)):
        confidence_list[index] = confidence_list[index] / score_total

    # if skip state confidence is < 1 %, increase it to 1 % and normalize all
    # others do not do this when computing node confidence (skip state is not
    # used)
    if confidence_list[0] < 0.01 and not node_confidence_bool:
        summ = 0.0
        for i in range(1, len(confidence_list)):
            summ += confidence_list[i]

        confidence_list[0] = 0.01
        for i in range(1, len(confidence_list)):
            confidence_list[i] = confidence_list[i] * 0.99 / summ

    return confidence_list

def confidence_only(
    region: List[int],
    lambs: List[float],
) -> List[float]:
    """
    Computes confidence values for competing annotations using alignment scores. Loops
    through the array once to find sum of 2^every_hit_score in
    region, then loops back through to calculate confidence. Converts the alignment
    score to account for lambda before summing.

    input:
    lambdaa: lambda for score matrix used
    region: list of scores for competing annotations
    subfams: list of subfam names

    output:
    confidence_list: list of confidence values for competing annotations

    >>> reg = [100, 55, 1]
    >>> lambs = [.1227] * 3
    >>> conf = confidence_only(reg, lambs)
    >>> conf
    [0.9843787551069454, 0.015380918048546022, 0.0002403268445085316]
    """

    confidence_list: List[float] = []
    score_total: int = 0

    # alignment scores
    for index in range(len(region)):
        converted_score = 2 ** int(region[index] * lambs[index])
        confidence_list.append(converted_score)
        score_total += converted_score

    for index in range(len(region)):
        confidence_list[index] = confidence_list[index] / score_total

    return confidence_list


import setuptools
from setuptools import find_packages

with open("requirements.txt", mode="r") as file:
    requirements = file.read().split("\n")

setuptools.setup(
    name="CNN_Spotify_Music",
    version="0.0.1",
    description="Apache Beam pipeline that uses Convolutional Neural Networks to predict music genre from audio files",
    classifiers=[
        "Intended Audience :: W207 - Fall 2022 - Section 04",
        "Programming Language :: Python :: 3.8",
        "Topic :: Experience :: Final Project"
    ],
    packages=find_packages() + ['.'],
    install_requires = requirements,
    python_requires = '>=3.8'
)


"""
Yinglan Chen

======================================
           DOCUMENTATION
======================================
VARIABLES:
x_t_i = 1 if city i is visited at timestep t , represented as a tuple (i,t)
           range: node i in [0,n-1], timestep t in [0, n-1], inclusive

CONSTRAINTS:
- optional: if fix start and end point, x_0_start = 1, x_{n}_end = 1
- each city is visited once: forall i, sum_t x_t_i = 1
- each timestep visits only one city: forall t, sum_i x_t_i = 1
- connectivity: for each t, for each node n, x_n_t => OR x_nn_{t+1} where nn are neighbors

OBJECTIVE:
- sum of t from 0 to n-1: x_i_t * x_j_t+1 * d_ij

"""

import sys
import math
import random
import itertools
from parse import parse
from gurobipy import *

filename = sys.argv[1]

# Parse argument
n, dist, adj = parse(filename)

# create model
m = Model()

# n cities, n timestep
vars = m.addVars(n,n,vtype=GRB.BINARY, name="x")

# 0. if fix start and end
start = 1
end = 3
# m.addConstr(vars[start,0], GRB.EQUAL, 1)
# m.addConstr(vars[end,n], GRB.EQUAL, 1)

# 1. each city is visited once
for city in range(n):
    m.addConstr(vars.sum(city,'*') == 1)

# 2. each timestep visits only one city
for t in range(n):
    print(vars.sum('*',t))
    m.addConstr(vars.sum('*',t) == 1)

# 3. connectivity
for timestep in range(n-1):
    for node in range(n):
        # node t implies neighbor t+1
        neighbors = adj[node]
        constr = LinExpr()
        constr.add(1-vars[node, timestep])
        for neighbor in neighbors:
            constr.add(vars[neighbor, timestep+1])
        # bug, wrong indentation before, add constr after loop through neighbors
        # print(constr)
        c = m.addConstr(constr, GRB.GREATER_EQUAL, 1)
        # print(c)

# opt
goal = QuadExpr()
for t in range(n-1):
    for i in range(n):
        for j in range(n):
            w = dist[i][j]
            goal.add(vars[(i,t)] * vars[(j,t+1)] * w)

m.setObjective(goal, GRB.MINIMIZE)

m.optimize()

print('')

print('Optimal cost: %g' % m.objVal)
print("print out solution in ascending timestep")
vals = m.getAttr('x', vars)
for timestep in range(n):
    for i in range(n):
            if (vals[(i,timestep)] > 0):
                print("timestep ", timestep, ": ", i)
print('')


import decimal
from decimal import Decimal

from django.conf import settings
from shop.models import Product

class Cart(object):
    """Класс для работы с корзиной."""

    def __init__(self, request) -> None:
        """Инициализация объекта корзины."""
        self.session = request.session
        cart = self.session.get(settings.CART_SESSION_ID)
        if not cart:
            cart = self.session[settings.CART_SESSION_ID] = {}
        self.cart = cart

    def add(self, product, quantity=1, override_quantity=False) -> None:
        """Добавление товара в корзину или обновление его количества."""
        product_id = str(product.id)
        if product_id not in self.cart:
            self.cart[product_id] = {
                "quantity": 0,
                "price": str(product.price),
            }
        if override_quantity:
            self.cart[product_id]["quantity"] = quantity
        else:
            self.cart[product_id]["quantity"] += quantity
        self.save()

    def save(self) -> None:
        self.session.modified = True

    def remove(self, product) -> None:
        """Удаление товара из корзины."""
        product_id = str(product.id)
        if product_id in self.cart:
            del self.cart[product_id]
            self.save()

    def __iter__(self):
        """Перебор корзины."""
        product_ids = self.cart.keys()
        products = Product.objects.filter(id__in=product_ids)

        cart = self.cart.copy()
        for product in products:
            cart[str(product.id)]["product"] = product

        for item in cart.values():
            item["price"]: decimal = Decimal(item["price"])
            item["total_price"]: decimal = item["price"] * item["quantity"]
            yield item

    def __len__(self) -> int:
        """Возвращает общее количество товаров в корзине."""
        return len(self.cart)

    def get_total_price(self) -> Decimal:
        """Возвращает общую стоимость товаров в корзине."""
        return sum(
            Decimal(item["price"]) * item["quantity"]
            for item in self.cart.values()
        )

    def clear(self) -> None:
        """Очищение корзины."""
        del self.session[settings.CART_SESSION_ID]
        self.save()


import pathlib

import sqlalchemy as sa
from pyramid import config
import pytest

from clld.db.meta import DBSession, Base, CustomModelMixin
from clld.db.models import common
from clld.web.datatables.language import Languages
from pytest_clld._app import ExtendedTestApp
from clld_glottologfamily_plugin.models import Family, HasFamilyMixin
from clld_glottologfamily_plugin.datatables import MacroareaCol, FamilyLinkCol

@pytest.fixture()
def glottolog_repos():
    return pathlib.Path(__file__).parent / 'repos'

class LanguageWithFamily(CustomModelMixin, common.Language, HasFamilyMixin):
    pk = sa.Column(sa.Integer, sa.ForeignKey('language.pk'), primary_key=True)
    macroarea = sa.Column(sa.Unicode)

@pytest.fixture(scope='session')
def language_with_family():
    return LanguageWithFamily

class LanguagesWithFamily(Languages):
    def col_defs(self):
        return [
            MacroareaCol(self, 'ma', language_cls=LanguageWithFamily),
            FamilyLinkCol(self, 'f', language_cls=LanguageWithFamily),
        ]

@pytest.fixture
def engine():
    DBSession.bind = sa.create_engine('sqlite://')
    yield DBSession
    DBSession.remove()

@pytest.fixture
def db():
    Base.metadata.bind = DBSession.bind
    Base.metadata.create_all()

    DBSession.add(common.Dataset(id='1', name='test app', domain='example.org'))
    family = Family(id='f', name='family', description='desc', jsondata=dict(icon=1))
    DBSession.add(LanguageWithFamily(id='l1', family=family))
    DBSession.add(LanguageWithFamily(id='l2'))
    DBSession.add(LanguageWithFamily(id='abcd1236'))
    DBSession.add(LanguageWithFamily(id='abcd1234'))
    return DBSession

@pytest.fixture(scope='module')
def testapp():
    def main():
        cfg = config.Configurator(settings={
            'sqlalchemy.url': 'sqlite://',
            'mako.directories': [
                'clld:web/templates',
                'clld_glottologfamily_plugin:templates'
            ]})
        cfg.include('clld.web.app')
        cfg.include('clld_glottologfamily_plugin')
        cfg.register_datatable('languages', LanguagesWithFamily)
        return cfg.make_wsgi_app()

    DBSession.remove()
    wsgi_app = main()
    Base.metadata.bind = DBSession.bind
    Base.metadata.create_all()

    DBSession.add(common.Dataset(id='1', name='test app', domain='example.org'))
    family = Family(id='f', name='family', description='desc', jsondata=dict(icon=1))
    DBSession.add(LanguageWithFamily(id='l1', family=family))
    DBSession.add(LanguageWithFamily(id='l2'))
    DBSession.add(LanguageWithFamily(id='abcd1236'))
    yield ExtendedTestApp(wsgi_app)


import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models
from torchvision.models.resnet import BasicBlock, Bottleneck
from torch.autograd import Variable
import numpy.random as npr
import random
import math

class ResNet(nn.Module):
    def __init__(self, block=BasicBlock, layers=[2, 2, 2, 2], num_classes=10):
        # change params: block and layers to swift from res18 to other models
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
        self.avgpool = nn.AvgPool2d(7, stride=1)
        self.fc_ = nn.Linear(512 * block.expansion, num_classes)

        self.pecent = 1/3

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

        self.register_buffer('pre_features', torch.zeros(128, 512))
        self.register_buffer('pre_weight1', torch.ones(128, 1))

    def _make_layer(self, block, planes, blocks, stride=1):

        downsample = None

        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def is_patch_based(self):
        return False

    def forward(self, x, gt=0, flag=False, epoch=0):

        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        if flag:
            interval = 10
            if epoch % interval == 0:
                self.pecent = 3.0 / 10 + (epoch / interval) * 2.0 / 10

            self.eval()
            x_new = x.clone().detach()
            x_new = Variable(x_new.data, requires_grad=True)
            x_new_view = self.avgpool(x_new)
            x_new_view = x_new_view.view(x_new_view.size(0), -1)
            output = self.fc_(x_new_view)
            class_num = output.shape[1]
            index = gt
            num_rois = x_new.shape[0]
            num_channel = x_new.shape[1]
            H = x_new.shape[2]
            HW = x_new.shape[2] * x_new.shape[3]
            one_hot = torch.zeros((1), dtype=torch.float32).cuda()
            one_hot = Variable(one_hot, requires_grad=False)
            sp_i = torch.ones([2, num_rois]).long()
            sp_i[0, :] = torch.arange(num_rois)
            sp_i[1, :] = index
            sp_v = torch.ones([num_rois])
            one_hot_sparse = torch.sparse.FloatTensor(sp_i, sp_v, torch.Size([num_rois, class_num])).to_dense().cuda()
            one_hot_sparse = Variable(one_hot_sparse, requires_grad=False)
            one_hot = torch.sum(output * one_hot_sparse)
            self.zero_grad()
            one_hot.backward()
            grads_val = x_new.grad.clone().detach()
            grad_channel_mean = torch.mean(grads_val.view(num_rois, num_channel, -1), dim=2)
            channel_mean = grad_channel_mean
            grad_channel_mean = grad_channel_mean.view(num_rois, num_channel, 1, 1)
            spatial_mean = torch.sum(x_new * grad_channel_mean, 1)
            spatial_mean = spatial_mean.view(num_rois, HW)
            self.zero_grad()

            choose_one = random.randint(0, 9)
            if choose_one <= 4:
                # ---------------------------- spatial -----------------------
                spatial_drop_num = math.ceil(HW * 1 / 3.0)
                th18_mask_value = torch.sort(spatial_mean, dim=1, descending=True)[0][:, spatial_drop_num]
                th18_mask_value = th18_mask_value.view(num_rois, 1).expand(num_rois, 49)
                mask_all_cuda = torch.where(spatial_mean > th18_mask_value, torch.zeros(spatial_mean.shape).cuda(),
                                            torch.ones(spatial_mean.shape).cuda())
                mask_all = mask_all_cuda.reshape(num_rois, H, H).view(num_rois, 1, H, H)
            else:
                # -------------------------- channel ----------------------------
                vector_thresh_percent = math.ceil(num_channel * 1 / 3.2)
                vector_thresh_value = torch.sort(channel_mean, dim=1, descending=True)[0][:, vector_thresh_percent]
                vector_thresh_value = vector_thresh_value.view(num_rois, 1).expand(num_rois, num_channel)
                vector = torch.where(channel_mean > vector_thresh_value,
                                     torch.zeros(channel_mean.shape).cuda(),
                                     torch.ones(channel_mean.shape).cuda())
                mask_all = vector.view(num_rois, num_channel, 1, 1)

            # ----------------------------------- batch ----------------------------------------
            cls_prob_before = F.softmax(output, dim=1)
            x_new_view_after = x_new * mask_all
            x_new_view_after = self.avgpool(x_new_view_after)
            x_new_view_after = x_new_view_after.view(x_new_view_after.size(0), -1)
            x_new_view_after = self.fc_(x_new_view_after)
            cls_prob_after = F.softmax(x_new_view_after, dim=1)

            sp_i = torch.ones([2, num_rois]).long()
            sp_i[0, :] = torch.arange(num_rois)
            sp_i[1, :] = index
            sp_v = torch.ones([num_rois])
            one_hot_sparse = torch.sparse.FloatTensor(sp_i, sp_v, torch.Size([num_rois, class_num])).to_dense().cuda()
            before_vector = torch.sum(one_hot_sparse * cls_prob_before, dim=1)
            after_vector = torch.sum(one_hot_sparse * cls_prob_after, dim=1)
            change_vector = before_vector - after_vector - 0.0001
            change_vector = torch.where(change_vector > 0, change_vector, torch.zeros(change_vector.shape).cuda())
            th_fg_value = torch.sort(change_vector, dim=0, descending=True)[0][int(round(float(num_rois) * self.pecent))]
            drop_index_fg = change_vector.gt(th_fg_value).long()
            ignore_index_fg = 1 - drop_index_fg
            not_01_ignore_index_fg = ignore_index_fg.nonzero()[:, 0]
            mask_all[not_01_ignore_index_fg.long(), :] = 1

            self.train()
            mask_all = Variable(mask_all, requires_grad=True)
            x = x * mask_all

        x = self.avgpool(x)
        features = x.view(x.size(0), -1)
        logits = self.fc_(features)

        return logits,features

def main():
    #resnet18 = models.resnet18(pretrained=True)    # downloading pretrained resnet to local env

    resnet18 = models.resnet18(pretrained=False)
    resnet18.load_state_dict(torch.load('./models/resnet/resnet18-5c106cde.pth'))

    model_a = ResNet(num_classes=10)      # animal
    model_v = ResNet(num_classes=8)       # vehicle
    pretrained_dict = resnet18.state_dict()
    model_dict_a = model_a.state_dict()
    model_dict_v = model_v.state_dict()

    pretrained_dict_a = {k:v for k,v in pretrained_dict.items() if k in model_dict_a}
    pretrained_dict_v = {k:v for k,v in pretrained_dict.items() if k in model_dict_v}
    model_dict_a.update(pretrained_dict_a)
    model_dict_v.update(pretrained_dict_v)
    model_a.load_state_dict(model_dict_a)
    model_v.load_state_dict(model_dict_v)

    torch.save(model_a.state_dict(),'./models/resnet/nico_animal.pkl')
    torch.save(model_v.state_dict(),'./models/resnet/nico_vehicle.pkl')

if __name__ == '__main__':
    main()


import hashlib
import os
import re
import time
import zipfile

from bs4 import BeautifulSoup
from flask import Blueprint, json, request
from flask import redirect
from flask import send_from_directory
from flask_login import login_required
from qiniu import Auth
from sqlalchemy import func
from lightniwa import app

from database import Article, User, db_session, engine, Book, Volume, Anime, Count, Chapter
from lightniwa.helper import api_helper

mod = Blueprint('api', __name__, url_prefix='/api/v1')

def update_info():
    resp = {}
    resp['versionCode'] = 22
    resp['url'] = 'https://ltype.me/static/LightNiwa_v0.8.5_build20190328.apk'
    resp['message'] = '\n*API已经大改需要更新(以前写的太烂我已经看不懂了)\n1.修复新版无法下载和闪退的BUG\n2.优化部分UI'
    return resp

@mod.route('/checkUpdate')
@mod.route('/checkupdate')
def check_update():
    resp = update_info()
    resp['versionCode'] = 1
    return api_helper.dumps(resp)

@mod.route('/version')
def version():
    return api_helper.wrap_resp(update_info())

@mod.route('/search')
def search():
    check_bot(request, 30)
    version_code = request.args.get('version_code')
    version_code = int((version_code, 0)[not version_code])
    keyword = request.args.get('keyWord')
    if not keyword:
        keyword = request.args.get('keyword')
    if keyword.find('ln') == 0:
        keyword = keyword.replace('ln', '')
    result = db_session.query(Book, func.sum(Volume.download).label('download')).join(Volume, Volume.book_id == Book.id) \
        .filter(func.concat(Volume.name, Book.name, Book.author, Book.illustrator).like('%' + keyword + '%')) \
        .group_by(Book.id).order_by(Volume.download.desc())
    resp = []
    for item in result:
        book = item.Book.to_json()
        book['download'] = int(item.download)
        resp.append(book)
    if version_code >= app.config['VERSION_CODE']:
        return api_helper.wrap_resp(resp)
    else:
        return api_helper.dumps(resp)

@mod.route('/book/<int:book_id>')
def book(book_id):
    check_bot(request, 30)
    b = Book.query.filter_by(id=book_id).first()
    volumes = Volume.query.filter_by(book_id=book_id).all()
    resp = b.to_json()
    resp['volumes'] = [v.to_json() for v in volumes]
    return api_helper.wrap_resp(resp)

@mod.route('/volume/<int:volume_id>', methods=['GET', 'POST'])
def volume_get(volume_id):
    check_bot(request, 30)
    v = Volume.query.filter_by(id=volume_id).first()
    resp = v.to_json()
    return json.dumps(resp, ensure_ascii=False), 200, {'ContentType': 'application/json'}

@login_required
@mod.route('/volume/<int:volume_id>', methods=['PATCH'])
def volume_patch(volume_id):
    v = Volume.query.filter_by(id=volume_id).first()
    print('name:%s' % request.form['name'])
    print('desc:%s' % request.form['desc'])
    print('file:%s' % request.files['file'].filename)
    # v.name = name
    # v.description = desc
    resp = v.to_json()
    return json.dumps(resp, ensure_ascii=False), 200, {'ContentType': 'application/json'}

@login_required
@mod.route('/volume/new', methods=['POST'])
def volume_post():
    v = Volume()
    data = request.json
    name = data['name']
    desc = data['desc']
    # v.name = name
    # v.description = desc
    print('name:%s,desc:%s' % (name, desc))
    resp = v.to_json()
    return json.dumps(resp, ensure_ascii=False), 200, {'ContentType': 'application/json'}

@mod.route('/chapter/<int:volume_id>', methods=['GET'])
def chapter_get(volume_id):
    chapters = Chapter.query.filter_by(vol_id=volume_id).all()
    resp = [c.to_json() for c in chapters]
    return json.dumps(resp, ensure_ascii=False), 200, {'ContentType': 'application/json'}

@login_required
@mod.route('/chapter/<int:volume_id>', methods=['POST'])
def chapter_post(volume_id):
    name = request.form.get('name')
    editor = request.form.get('editor')
    soup = BeautifulSoup(editor)
    for p in soup.find_all('p'):
        print(p.get_text())
    # print(editor)
    v = Volume.query.filter_by(id=volume_id).first()
    c = Chapter(book_id=v.book_id, vol_id=v.id, name=name, content=editor)
    return json.dumps({}, ensure_ascii=False), 200, {'ContentType': 'application/json'}

@login_required
@mod.route('/chapter/<int:chapter_id>', methods=['PATCH'])
def chapter_patch(chapter_id):
    data = request.json
    print('data:%s' % data)
    name = data['name']
    desc = data['desc']
    print('name:%s, desc:%s' % (name, desc))
    return json.dumps({}, ensure_ascii=False), 200, {'ContentType': 'application/json'}

latest_cache = []
latest_update_time = 0

@mod.route('/latest')
@mod.route('/latestupdate')
def latest():
    version_code = request.args.get('version_code')
    version_code = int((version_code, 0)[not version_code])
    global latest_cache
    global latest_update_time
    if not latest_cache or latest_update_time + 3600 < time.time():
        result = db_session.query(Book, Volume).join(Volume, Volume.book_id == Book.id) \
            .order_by(Volume.id.desc()).limit(30)
        resp = []
        if version_code >= app.config['VERSION_CODE']:
            for line in result:
                item = line.Volume.to_json()
                item['book'] = line.Book.to_json()
                resp.append(item)
            latest_cache = resp
            latest_update_time = time.time()
        else:
            for line in result:
                item = {}
                item['vol_id'] = line.Volume.id
                item['vol_index'] = line.Volume.index
                item['vol_name'] = line.Volume.name
                item['vol_cover'] = line.Volume.cover
                item['vol_description'] = line.Volume.description
                item['book_id'] = line.Book.id
                item['book_name'] = line.Book.name
                item['book_author'] = line.Book.author
                item['book_illustrator'] = line.Book.illustrator
                resp.append(item)
            return api_helper.dumps(resp)
    return api_helper.wrap_resp(latest_cache)

@mod.route('/popular')
def popular():
    version_code = request.args.get('version_code')
    version_code = int((version_code, 0)[not version_code])
    resp = []
    result = db_session.query(Book, func.sum(Volume.download).label('total')) \
        .join(Volume, Volume.book_id == Book.id) \
        .group_by(Book.id).order_by(Volume.download.desc()).limit(20)
    if version_code >= app.config['VERSION_CODE']:
        for item in result:
            book = item.Book.to_json()
            book['total'] = int(item.total)
            resp.append(book)
        return api_helper.wrap_resp(resp)
    else:
        for line in result:
            item = {}
            item['book_id'] = line.Book.id
            item['book_name'] = line.Book.name
            item['book_author'] = line.Book.author
            item['book_illustrator'] = line.Book.illustrator
            item['book_publisher'] = line.Book.publisher
            item['book_cover'] = line.Book.cover
            item['total'] = int(line.total)
            resp.append(item)
        return api_helper.dumps(resp)

@mod.route('/anime/<int:month>')
@mod.route('/animes/<int:month>')
def anime(month):
    result = db_session.query(Book, Anime).join(Anime, Anime.book_id == Book.id).filter(Anime.month == month)
    resp = []
    for line in result:
        resp.append(line.Book.to_json())
    return api_helper.wrap_resp(resp)

@mod.route('/cover/image/<file_dir>/<file_name>.jpg')
def cover(file_dir, file_name):
    check_bot(request, 30)
    key = 'cover/image/%s/%s.jpg' % (file_dir, file_name)
    # if not os.path.isfile(app.config['PRIVATE_PATH'] + '/' + key):
    # lkcore.download_img('/' + key)
    # return 404
    q = Auth(app.config['QN_ACCESS_KEY'], app.config['QN_SECRET_KEY'])
    base_url = 'http://%s/%s' % (app.config['QN_BUCKET_DOMAIN'], key)
    private_url = q.private_download_url(base_url, expires=3600)
    # global cover_cache_time
    # if cover_cache_time + 3000 < time.time():
    #     q = Auth(qiniu.access_key, qiniu.secret_key)
    #     key1 = 'cover/image/christmas/cover_wa2_1.jpg'
    #     key2 = 'cover/image/christmas/cover_wa2_2.jpg'
    #     key3 = 'cover/image/christmas/cover_wa2_3.jpg'
    #     christmas_cover[0] = q.private_download_url('http://%s/%s' % (qiniu.bucket_domain, key1), expires=3600)
    #     christmas_cover[1] = q.private_download_url('http://%s/%s' % (qiniu.bucket_domain, key2), expires=3600)
    #     christmas_cover[2] = q.private_download_url('http://%s/%s' % (qiniu.bucket_domain, key3), expires=3600)
    #     cover_cache_time = time.time()
    # private_url = christmas_cover[random.randrange(0, 3)]
    return redirect(private_url)

# volume_id.zip
# |--info
# |--|--book.json
# |--|--volume.json
# |--|--chapters.json
# |--content
# |--|--chapter_id.json
# |--img
# |--|--|md5('/cover/image/20120803/20120803215640_24363.jpg').jpg
@mod.route('/download/volume/<int:volume_id>')
def download_volume(volume_id):
    check_bot(request, 30)
    Volume.query.filter_by(id=volume_id).update({"download": (Volume.download + 1)})

    db_session.commit()

    result = db_session.query(Book, Volume).join(Volume, Volume.book_id == Book.id).filter(
        Volume.id == volume_id).first()
    if not result:
        return 404
    path = '/zip/%s/%s.zip' % (result.Book.id, volume_id)
    if os.path.exists(app.config['PRIVATE_PATH'] + path):
        q = Auth(app.config['QN_ACCESS_KEY'], app.config['QN_SECRET_KEY'])
        base_url = 'http://%s/%s' % (app.config['QN_BUCKET_DOMAIN'], path)
        private_url = q.private_download_url(base_url, expires=3600)
        return send_from_directory(app.config['PRIVATE_PATH'] + '/zip/%s' % result.Book.id, '%s.zip' % volume_id)
    else:
        if not os.path.isdir(os.path.dirname(app.config['PRIVATE_PATH'] + path)):
            os.makedirs(os.path.dirname(app.config['PRIVATE_PATH'] + path))

        vol_cover = '/%s/img/%s.jpg' \
                    % (volume_id, hashlib.md5(result.Volume.cover.encode('utf-8')).hexdigest())
        if result.Book.cover:
            book_cover = '/%s/img/%s.jpg' \
                         % (volume_id, hashlib.md5(result.Book.cover.encode('utf-8')).hexdigest())
        else:
            book_cover = vol_cover
        book_json = {}
        book_json['book_id'] = result.Book.id
        book_json['author'] = result.Book.author
        book_json['illustrator'] = result.Book.illustrator
        book_json['publisher'] = result.Book.publisher
        book_json['name'] = result.Book.name
        book_json['cover'] = book_cover
        book_json['description'] = result.Volume.description

        vol_json = {}
        vol_json['index'] = result.Volume.index
        vol_json['book_id'] = result.Book.id
        vol_json['volume_id'] = result.Volume.id
        vol_json['name'] = result.Volume.id
        vol_json['cover'] = vol_cover
        vol_json['description'] = result.Volume.description

        try:
            # create a empty zip file
            zf = zipfile.ZipFile(app.config['PRIVATE_PATH'] + path, 'w', zipfile.ZIP_DEFLATED)
            zfi = zipfile.ZipInfo('info/')
            zf.writestr(zfi, '')
            zfi = zipfile.ZipInfo('img/')
            zf.writestr(zfi, '')
            zfi = zipfile.ZipInfo('content/')
            zf.writestr(zfi, '')

            # lkcore.download_img(result.Volume.cover)
            zf.write(app.config['PRIVATE_PATH'] + result.Volume.cover,
                     'img/%s.jpg' % hashlib.md5(result.Volume.cover.encode('utf-8')).hexdigest())

            chapters = Chapter.query.filter_by(vol_id=volume_id).all()
            chapters_json = []
            for row in chapters:
                chapter_json = {}
                chapter_json['index'] = row.index
                chapter_json['book_id'] = row.book_id
                chapter_json['volume_id'] = row.vol_id
                chapter_json['chapter_id'] = row.id
                chapter_json['name'] = row.name
                chapters_json.append(chapter_json)

                # lkcore.download_txt(row['chapter_id'], row['file_path'])
                contents = []
                f = open(app.config['PRIVATE_PATH'] + row.file_path, 'r', encoding='utf-8')
                for line in f.readlines():
                    m = re.search(r"^/[^\s]*?\.(jpg|jpeg|png)", line)
                    is_img = '0'
                    if m:
                        is_img = '1'
                        img_path = m.group(0)
                        # lkcore.download_img(img_path)
                        if os.path.exists(app.config['PRIVATE_PATH'] + img_path):
                            encode_path = hashlib.md5(img_path.encode('utf-8')).hexdigest() + \
                                          os.path.splitext(img_path)[1]
                            zf.write(app.config['PRIVATE_PATH'] + img_path, 'img/%s' % encode_path)
                            line = '/%s/img/%s' % (volume_id, encode_path)
                        else:
                            line = '缺少插画，请联系管理'

                    content = {}
                    content['index'] = len(contents)
                    content['is_img'] = is_img
                    content['content'] = line
                    contents.append(content)
                zf.writestr('content/%s.json' % row.id, json.dumps(contents, ensure_ascii=False))

            zf.writestr('info/book.json', json.dumps(book_json, ensure_ascii=False))
            zf.writestr('info/volume.json', json.dumps(vol_json, ensure_ascii=False))
            zf.writestr('info/chapters.json', json.dumps(chapters_json, ensure_ascii=False))
            zf.close()
        except FileNotFoundError as e:
            print(e)
            zf.close()
            os.remove(zf.filename)
            raise e
        except OSError as e:
            print(e)
            zf.close()
            os.remove(zf.filename)
    return send_from_directory(os.path.dirname(zf.filename), '%s.zip' % volume_id)

def zipdir(path, ziph):
    # ziph is zipfile handle
    for root, dirs, files in os.walk(path):
        for file in files:
            ziph.write(os.path.join(root, file))

def async_download_volume(volume_id):
    v = Volume.query.filter_by(id=volume_id).first()
    # if not v:
    #     if 0 < v.id < 10000:
    # threading.Thread(target=lkcore.download_volume(volume_id, 1)).start()

    # threading.Thread(target=lkcore.download_volume(volume_id, 1)).start()

def check_bot(req, limit):
    ip = req.remote_addr
    now = int(str(time.time()).split('.')[0])
    count = Count.query.filter_by(ip=ip).first()
    if count:
        if count.last_time < now - 60 * 1:
            count.frequency = 1
        else:
            if count.frequency >= limit:
                raise FileNotFoundError
            count.frequency += 1
        count.total += 1
        count.last_time = now
    else:
        count = Count(ip=ip, last_time=now, frequency=0, total=0)
        db_session.add(count)


from flask import Blueprint, make_response, request

from slamd.common.error_handling import SlamdUnprocessableEntityException, ValueNotSupportedException
from slamd.common.session_backup.session_service import SessionService

session_blueprint = Blueprint('session', __name__, url_prefix='/session')

"""
    The form that serves these endpoints is found in the navbar. As such it is not created using WTForms.
    It relies on CSRF token elements in the body of the page for authentication.
    These are usually provided by other WTForms. However, some pages may not have form elements on them, in particular
    the landing page.
    These pages require the manual addition of CSRF token fields.
"""

@session_blueprint.route('/', methods=['GET'])
def save_session():
    json_string = SessionService.convert_session_to_json_string()
    response = make_response(json_string.encode())
    filename = SessionService.create_default_filename()
    response.headers['Content-Disposition'] = f'attachment; filename={filename}'
    response.mimetype = 'text/json'
    return response

@session_blueprint.route('/', methods=['POST'])
def load_session():
    if 'file' not in request.files:
        raise SlamdUnprocessableEntityException(message='Request did not contain a file')

    file_extension = request.files['file'].filename.lower().split('.')[-1]
    if file_extension != 'json':
        raise ValueNotSupportedException(message=f'Invalid file type: {file_extension}')

    # Flask checks that the 'Content-Length' header is present.
    # Otherwise, it rejects the request and returns a status code 400 - Bad Request.
    length_of_file = int(request.headers['CONTENT_LENGTH'])
    try:
        file_as_string = request.files['file'].read(length_of_file).decode()
    except UnicodeDecodeError:
        raise SlamdUnprocessableEntityException(message='Error while attempting to decode JSON file')

    SessionService.load_session_from_json_string(file_as_string)

    # In the frontend, Javascript will reload the page automatically if it receives an OK response
    # Actual content of response does not matter
    return ''

@session_blueprint.route('/', methods=['DELETE'])
def clear_session():
    SessionService.clear_session()

    # In the frontend, Javascript will reload the page automatically if it receives an OK response
    # Actual content of response does not matter
    return ''


import torch
from torchvision import datasets
from torchvision import transforms
import torch.utils.data as Data
from sklearn import metrics

from GoogLeNet import GoogLeNet
from resnet_18 import ResNet_18
from resnet_50 import ResNet_50
def test(model):

    test_transformer = transforms.Compose(
        [
            transforms.RandomHorizontalFlip(0.2),
            transforms.RandomRotation(68),
            transforms.RandomGrayscale(0.2),
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.5, 0.5, 0.5],
                                 std=[0.5, 0.5, 0.5])
        ])

    test_data = datasets.ImageFolder("./dataset2-master/images/TEST", transform=test_transformer)
    test_loader = Data.DataLoader(dataset=test_data, batch_size=32, shuffle=False, num_workers=2)

    device=torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model=model.to(device)

    test_corrects=0.0
    test_num=0
    with torch.no_grad():
        for data_x,data_y in test_loader:
            data_x=data_x.to(device)
            data_y=data_y.to(device)

            model.eval()
            output=model(data_x)

            pre_lab=torch.argmax(output,dim=1)
            test_corrects+=torch.sum(pre_lab==data_y)
            test_num+=data_x.size(0)

    test_acc=test_corrects.double().item()/test_num
    print("the accurate rate is {:.3f}".format(test_acc))
    f1=metrics.f1_score(data_y,pre_lab,average="samples")
    print("the f1 score is {:.3f}".format(f1))

if __name__=="__main__":
    model=ResNet_50()
    model.load_state_dict(torch.load("ResNet_50_best_model.pt"))

    test(model)


# -*- coding: utf-8 -*-
"""
Created on Thu Jul  6 13:41:45 2017

@author: ZHILANGTAOSHA
"""

import pandas as pd
import geohash
from math import radians, cos, sin, asin, sqrt
# import itertools
import numpy as np
import math
import xgboost as xgb

def distance_haver(lon1, lat1, lon2, lat2):
    dx = np.absolute(lon1 - lon2)  # 经度差
    dy = np.absolute(lat1 - lat2)  # 维度差
    b = (lat1 + lat2) / 2.0
    Lx = (dx / 57.2958) * 6371004.0 * np.cos(b / 57.2958)
    Ly = 6371004.0 * (dy / 57.2958)
    L = (Lx ** 2 + Ly ** 2) ** 0.5
    return L

def train_tztq(train):
    zh = train[['geohashed_start_loc', 'geohashed_end_loc']]
    zh.loc[:, 'pl'] = 1
    zh = zh.groupby(['geohashed_start_loc', 'geohashed_end_loc'])['pl'].count().reset_index()
    zh['max_dd'] = zh[['geohashed_start_loc', 'geohashed_end_loc']].max(axis=1)
    zh['min_dd'] = zh[['geohashed_start_loc', 'geohashed_end_loc']].min(axis=1)

    # 统计相同两个点的关联个数
    zh = zh.groupby(['max_dd', 'min_dd'])['pl'].agg({'zz': np.sum, 'gs': np.size}).reset_index()
    print(zh.shape)

    # 换算为经纬度
    zh.loc[:, 'start_jw'] = zh.max_dd.map(geohash.decode)
    zh.loc[:, 'start_j'] = zh.start_jw.map(lambda x: x[0])
    zh.loc[:, 'start_w'] = zh.start_jw.map(lambda x: x[1])
    zh.loc[:, 'end_jw'] = zh.min_dd.map(geohash.decode)
    zh.loc[:, 'end_j'] = zh.end_jw.map(lambda x: x[0])
    zh.loc[:, 'end_w'] = zh.end_jw.map(lambda x: x[1])
    del zh['start_jw'], zh['end_jw']

    # 计算经纬度的具体距离
    zh.loc[:, 'juli'] = zh.apply(lambda x: distance_haver(x['start_j'], x['start_w'], x['end_j'], x['end_w']), axis=1)

    # 建立最后的地点画像

    # 计算大的维度的地点对应个数
    max_dd_tz = zh.groupby('max_dd').agg({'gs': np.size})
    # 计算训练集中地点出现的总次数
    max_dd_tz['zz_gs'] = zh.groupby('max_dd')['zz'].agg({'zz_gs': np.sum})

    # 计算
    # 地点所至最大距离
    max_dd_tz['juli_max'] = zh.groupby('max_dd')['juli'].max()
    # 地点所至最小距离
    max_dd_tz['juli_min'] = zh.groupby('max_dd')['juli'].min()
    # 地点所至中值
    max_dd_tz['juli_median'] = zh.groupby('max_dd')['juli'].median()

    # 计算大的维度的地点对应个数
    min_dd_tz = zh.groupby('min_dd').agg({'gs': np.size})
    # 计算训练集中地点出现的总次数
    min_dd_tz['zz_gs'] = zh.groupby('min_dd')['zz'].agg({'zz_gs': np.sum})

    # 计算
    # 地点所至最大距离
    min_dd_tz['juli_max'] = zh.groupby('min_dd')['juli'].max()
    # 地点所至最小距离
    min_dd_tz['juli_min'] = zh.groupby('min_dd')['juli'].min()
    # 地点所至中值
    min_dd_tz['juli_median'] = zh.groupby('min_dd')['juli'].median()

    # 拼接所有地点

    dd_tz = pd.concat([max_dd_tz, min_dd_tz])
    return dd_tz

def datateime_slice(train):
    hour_sx = {0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 2, 7: 4, 8: 4, 9: 4, 10: 2, 11: 2, 12: 5, 13: 5, 14: 2, 15: 3,
               16: 3, 17: 6, 18: 6, 19: 6, 20: 3, 21: 3, 22: 1, 23: 1}
    week_sx = {0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 2, 6: 2}
    train.loc[:, 'starttime'] = pd.to_datetime(train.starttime)
    train['weekday_time'] = train.starttime.dt.weekday
    train['hour_time'] = train.starttime.dt.hour
    train['hour_sx'] = train.hour_time.map(hour_sx)
    train['week_sx'] = train.weekday_time.map(week_sx)
    return (train)

def cal_distance_start_end(zh):
    js = zh[~zh.tj_dd.isnull()].drop_duplicates()
    # 换算为经纬度
    js.loc[:, 'start_jw'] = js.iloc[:, 0].map(geohash.decode)
    js.loc[:, 'start_j'] = js.start_jw.map(lambda x: x[0])
    js.loc[:, 'start_w'] = js.start_jw.map(lambda x: x[1])
    js.loc[:, 'end_jw'] = js.iloc[:, 1].map(geohash.decode)
    js.loc[:, 'end_j'] = js.end_jw.map(lambda x: x[0])
    js.loc[:, 'end_w'] = js.end_jw.map(lambda x: x[1])
    del js['start_jw'], js['end_jw']
    # 计算经纬度的具体距离
    js.loc[:, 'juli'] = js.apply(lambda x: distance_haver(x['start_j'], x['start_w'], x['end_j'], x['end_w']), axis=1)
    return js

train = pd.read_csv('/root/Documents/play/mobike/source/train.csv')
# test = pd.read_csv('../input/test.csv')
# example = pd.read_csv('../input/sample_submission.csv',header=None)

print(train.shape)

train1 = ['2017-05-14',
          '2017-05-15',
          '2017-05-16',
          '2017-05-12',
          '2017-05-13',
          '2017-05-10',
          '2017-05-11']

train2 = ['2017-05-18',
          '2017-05-19',
          '2017-05-23',
          '2017-05-24',
          '2017-05-20',
          '2017-05-22',
          '2017-05-21']

train2 = train[train.starttime.map(lambda x: x[:10]).isin(train2)]
train = train[train.starttime.map(lambda x: x[:10]).isin(train1)]

train2 = datateime_slice(train2)
train = datateime_slice(train)

pj1 = train[['geohashed_start_loc', 'geohashed_end_loc', 'hour_sx', 'week_sx']]
pj1 = pj1.rename(columns={'geohashed_end_loc': 'tj_dd'})
pj1 = pj1.drop_duplicates()
z1 = pd.merge(train2, pj1, on=['geohashed_start_loc', 'hour_sx', 'week_sx'], how='left')
del z1['starttime']

z1_not_null = z1[z1.tj_dd.isnull()]
z1 = z1[~z1.tj_dd.isnull()]
zh = z1.loc[:, ('geohashed_start_loc', 'tj_dd')]
js = cal_distance_start_end(zh)
z1 = pd.merge(z1, js[['geohashed_start_loc', 'tj_dd', 'juli']], on=['geohashed_start_loc', 'tj_dd'], how='left')

train_dd_tz = train_tztq(train)
train_dd_tz = train_dd_tz.reset_index()
train_dd_tz = train_dd_tz.rename(columns={'index': 'geohashed_start_loc'})

wpj = train_dd_tz[train_dd_tz.zz_gs > 100]

z1_all_more = z1[z1.geohashed_start_loc.isin(wpj.geohashed_start_loc)]

z1 = z1[~z1.geohashed_start_loc.isin(wpj.geohashed_start_loc)]

z1 = pd.merge(z1, train_dd_tz, on='geohashed_start_loc')
train_dd_tz = train_dd_tz.rename(columns={'geohashed_start_loc': 'tj_dd'})
z1 = pd.merge(z1, train_dd_tz, on='tj_dd')

z1['label'] = z1.geohashed_end_loc == z1.tj_dd
z1.label = z1.label.map(int)

featurelist = [i for i in z1.columns if
               i not in ['orderid', 'geohashed_start_loc', 'geohashed_end_loc', 'tj_dd', 'label']]

params = {
    'objective ': 'binary:logistic',
    'eval_metric': 'map',
    'max_depth': 11,
    'min_child_weight': 0.9,
    'max_delta_step': 10,
    'eta': 0.3
}

del z1['tj_dd']

from sklearn.cross_validation import train_test_split

x, y = train_test_split(z1, test_size=0.3)

x = xgb.DMatrix(x[featurelist], label=x.label)
y = xgb.DMatrix(y[featurelist], label=y.label)

test = pd.read_csv('/root/Documents/play/mobike/source/test.csv')
test = datateime_slice(test)

pj1 = train2[['geohashed_start_loc', 'geohashed_end_loc', 'hour_sx', 'week_sx']]
pj1 = pj1.rename(columns={'geohashed_end_loc': 'tj_dd'})
pj1 = pj1.drop_duplicates()
z2 = pd.merge(test, pj1, on=['geohashed_start_loc', 'hour_sx', 'week_sx'], how='left')
del z2['starttime']

zh = z2.loc[:, ('geohashed_start_loc', 'tj_dd')]
js = cal_distance_start_end(zh)
z2 = pd.merge(z2, js[['geohashed_start_loc', 'tj_dd', 'juli']], on=['geohashed_start_loc', 'tj_dd'], how='left')

train_dd_tz = train_tztq(train2)
train_dd_tz = train_dd_tz.reset_index()
train_dd_tz = train_dd_tz.rename(columns={'index': 'geohashed_start_loc'})

z2 = pd.merge(z2, train_dd_tz, on='geohashed_start_loc')
train_dd_tz = train_dd_tz.rename(columns={'geohashed_start_loc': 'tj_dd'})
z2 = pd.merge(z2, train_dd_tz, on='tj_dd')
del z2['geohashed_start_loc']
# z2 = xgb.DMatrix(z2[featurelist])

watchlist = [(x, 'train'), (y, 'eval')]
evals_result = {}
num_round = 1000
bst = xgb.train(params, x, num_round, watchlist, evals_result=evals_result, early_stopping_rounds=20, maximize=True)


from typing import Optional

from construct import Container

from external_asset_ism_ismc_generation_tool.common.bit_reader import BitReader
from external_asset_ism_ismc_generation_tool.common.logger.i_logger import ILogger
from external_asset_ism_ismc_generation_tool.common.logger.logger import Logger
from external_asset_ism_ismc_generation_tool.mp4_data_parser.model.atom.avcc_atom import Avcc
from external_asset_ism_ismc_generation_tool.mp4_data_parser.model.atom.hvc1_atom import Hvc1
from external_asset_ism_ismc_generation_tool.mp4_data_parser.model.atom.hvcc_atom import Hvcc

class AtomsDataParser:
    __logger: ILogger = Logger("AtomsDataParser")

    @classmethod
    def redefine_logger(cls, logger: ILogger):
        cls.__logger = logger

    @staticmethod
    def parse_avcc(sample_info: Container) -> Optional[Avcc]:
        if sample_info:
            sample_info_data = sample_info[0]['data']
            configuration_version = sample_info_data[0]
            profile = sample_info_data[1]
            profile_compatibility = sample_info_data[2]
            level = sample_info_data[3]
            nalu_length_size = 1 + (sample_info_data[4] & 3)
            num_seq_params = sample_info_data[5] & 31
            sequence_param_length = int.from_bytes(sample_info_data[6:8], "big")
            sequence_parameters = sample_info_data[8:8 + sequence_param_length]
            current = 8 + sequence_param_length
            num_pic_params = sample_info_data[current]
            current += 1
            picture_param_length = int.from_bytes(sample_info_data[current:current + 2], "big")
            current = current + 2
            picture_parameter = sample_info_data[current:current + picture_param_length]
            return Avcc(configuration_version, profile, profile_compatibility, level, nalu_length_size, sequence_parameters, picture_parameter)

    @staticmethod
    def parse_hvc1_data(hvc1_data: bytes) -> Optional[Hvc1]:
        if hvc1_data:
            reader = BitReader(hvc1_data)
            predefined1 = reader.get_bits(16)
            reserved1 = reader.get_bits(16)
            predefined2 = reader.get_bits(12 * 8)
            width = reader.get_bits(16)
            height = reader.get_bits(16)
            horiz_resolution = reader.get_bits(32)
            vert_resolution = reader.get_bits(32)
            reserved2 = reader.get_bits(32)
            frame_count = reader.get_bits(16)
            compressor_name_full_size = 32 * 8
            compressor_name_length_size = 8
            compressor_name_length = reader.get_bits(compressor_name_length_size)
            compressor_name = reader.get_bits(compressor_name_length)
            reserved3 = reader.get_bits(compressor_name_full_size - compressor_name_length - compressor_name_length_size)
            depth = reader.get_bits(16)
            predefined3 = reader.get_bits(16)
            data = reader.read_bytes(len(hvc1_data) - reader.tell())

            return Hvc1(width=width,
                        height=height,
                        horiz_resolution=horiz_resolution,
                        vert_resolution=vert_resolution,
                        compressor_name=compressor_name,
                        data=data)

    @staticmethod
    def parce_hvcc_data(hvc1_data: bytes) -> Optional[Hvcc]:
        if not hvc1_data:
            return None
        _HVCC_ATOM_TYPE = 'hvcC'
        atom_header_size = 4
        atom_size = hvc1_data[0:4]
        atom_type = hvc1_data[4:8].decode("utf-8")
        start_byte = 8
        while atom_type != _HVCC_ATOM_TYPE:
            if len(hvc1_data) < 8:
                return None
            skip = atom_size - atom_header_size
            start_byte = 8 + skip
            atom_size = hvc1_data[start_byte: start_byte + 32]
            start_byte += 32
            atom_type = hvc1_data[start_byte: start_byte + 32].decode("utf-8")
            start_byte += 32

        data = hvc1_data[start_byte: len(hvc1_data)]
        configuration_version = data[0]
        general_profile_space = (data[1] >> 6) & 0x03
        general_tier_flag = (data[1] >> 5) & 0x01
        general_profile = (data[1]) & 0x1F
        general_profile_compatibility_flags = data[2:6]
        general_constraint_indicator_flags = int.from_bytes(data[6:12], byteorder='big')
        general_level = data[12]
        reserved1 = (data[13] >> 4) & 0x0F
        min_spatial_segmentation = int.from_bytes(data[13:14], byteorder='big') & 0x0FFF
        reserved2 = (data[15] >> 2) & 0x3F
        parallelism_type = data[15] & 0x03
        reserved3 = (data[16] >> 2) & 0x3F
        chroma_format = data[16] & 0x03
        reserved4 = (data[17] >> 3) & 0x1F
        luma_bit_depth = 8 + (data[17] & 0x07)
        reserved5 = (data[18] >> 3) & 0x1F
        chroma_bit_depth = 8 + (data[18] & 0x07)
        average_frame_rate = int.from_bytes(data[19:21], byteorder='big')
        constant_frame_rate = (data[21] >> 6) & 0x03
        num_temporal_layers = (data[21] >> 3) & 0x07
        temporal_id_nested = (data[21] >> 2) & 0x01
        nalu_length_size = 1 + (data[21] & 0x03)
        num_seq = data[22]

        current = 23
        nalus: list = []
        for i in range(0, num_seq):
            array_completeness = data[current] >> 7 & 0x01
            reserved = (data[current] >> 6) & 0x01
            nalu_type = data[current] & 0x3F
            current += 1

            nalu_count = int.from_bytes(data[current: current + 2], 'big')
            current += 2

            for j in range(0, nalu_count):
                nalu_length = int.from_bytes(data[current: current + 2], 'big')
                current += 2
                nalus.append(data[current:current + nalu_length])
                current += nalu_length

        return Hvcc(configuration_version,
                    general_profile_space,
                    general_profile,
                    general_tier_flag,
                    general_profile_compatibility_flags,
                    general_constraint_indicator_flags,
                    general_level,
                    min_spatial_segmentation,
                    parallelism_type,
                    chroma_format,
                    chroma_bit_depth,
                    luma_bit_depth,
                    average_frame_rate,
                    constant_frame_rate,
                    num_temporal_layers,
                    temporal_id_nested,
                    nalu_length_size,
                    nalus)


import traceback
from typing import Any

from quart import ResponseReturnValue, request as view_request
from quart.views import View

from webapps.language.decorators.singleton import Singleton

from webapps.model.auth.access.access_errors import SerializableObjectNotAvialable
from webapps.model.identifier import ModelIdentifier
from webapps.model.auth.access.tic_tok_depot import SerializableObjectDepot
from webapps.modules.coroutinpromise.promisepool import PromisePool

from webapps.modules.lumber.lumber import Lumber

from webapps.modules.plugin.endpoints import PluginEndpoint
from webapps.modules.requests.httpcallbuilder import HttpCallBuilder
from webapps.modules.requests.httpheaders import HttpHeaders

from webapps.plugins.tplus.actuators.api_broker_call import TplusOpenapiBrokerHttpCall
from webapps.plugins.tplus.actuators.app_ticket_call import TplusTicketHttpCall
from webapps.plugins.tplus.actuators.app_token_call import TplusAppTokenHttpCall
from webapps.plugins.tplus.endpoints.auth.ticket import AppTicketEndpoints

from webapps.plugins.tplus.model.auth.app_ticket import AppTicket
from webapps.plugins.tplus.model.auth.app_token import AppToken
from webapps.plugins.tplus.model.dao.api_broker_request import TplusRestfulapiBrokerRequest
from webapps.plugins.tplus.model.dao.tplus_errors import AppTicketExpired, AppTokenExpired, AppTokenInvalid
from webapps.plugins.tplus.model.properties.tplus_endpoint_profile import TplusEndpointProfile
from webapps.plugins.tplus.model.properties.tplus_openapi_properties import TplusOpenApiProperties

@Singleton
class TplusOpenapiBroker(PluginEndpoint):

    _timber = Lumber.timber("endpoints")
    _err_timber = Lumber.timber("error")

    _ARGKEY_GRANT_TYPE_    = "grantType"
    _ARGVLU_REFRESH_TOKEN_ = "refresh_token"

    _ARGKEY_REFRESH_TOKEN_ = "refreshToken"

    class BrokerView(View):

        methods = ["GET", "POST"]

        def __init__(self, endpoint: PluginEndpoint) -> None:
            super().__init__()
            self._endpoint = endpoint

        async def dispatch_request(self, **kwargs: Any) -> ResponseReturnValue:

            if not TplusRestfulapiBrokerRequest._REQUEST_HEADER_CONTENT_TYPE_ in view_request.headers:
                return "A valid request must contain specified: 'Content-Type' as 'application/json'."

            content_type = view_request.headers.get(TplusRestfulapiBrokerRequest._REQUEST_HEADER_CONTENT_TYPE_)
            if (content_type != HttpHeaders._HDV_MIME_JSON_):
                return 'Content-Type not supported!'

            request = TplusRestfulapiBrokerRequest(await view_request.get_json())
            try:
                return await self._endpoint.request(request.api_path, request.api_method, request.body)
            except Exception as error:
                TplusOpenapiBroker._err_timber.error(f"{error}")
                TplusOpenapiBroker._err_timber.error(f"{error.args}")

                return "Request data is invalid/malformatted", 404

    def __init__(self, name: str, profile: TplusEndpointProfile, props: TplusOpenApiProperties) -> None:
        super().__init__(name, profile)

        assert profile and props

        self._profile = profile
        self._props = props

        self._conf_vault = dict(profile.valueset | props.valueset)

        self._seri_obj_depot = SerializableObjectDepot()
        self._http_call_builder = HttpCallBuilder(props.base_url)

        self._tok_identifier = ModelIdentifier(props.plugin_qualifier, profile.endpoint_qualifier, self.url)
        self._tic_identifier = AppTicketEndpoints.query_subscription()

        header_dic = {
            TplusOpenApiProperties._APP_KEY_: props.app_key,
            TplusOpenApiProperties._APP_SECRET_: props.app_secret
        }
        self._http_call_builder.headers = HttpHeaders(defaults= header_dic)

    def get_app_ticket(self) -> AppTicket:
        app_tic = AppTicket(self._seri_obj_depot.get(self._tic_identifier))

        if not app_tic.is_valid:
            raise AppTicketExpired(f"{app_tic.ticket} has expired. Lifetime: {app_tic.lifetime}.")

        return app_tic

    def get_token(self) -> AppToken:
        app_tok = AppToken(self._seri_obj_depot.get(self._tok_identifier))

        if not app_tok.is_valid:
            if not app_tok.is_expired:
                raise AppTokenInvalid(f"{app_tok.access_token_lifetime}")
            else:
                raise AppTicketExpired(f"{app_tok.refresh_token_lifetime}")

        return app_tok

    def put_token(self, app_tok) -> None:
        self._seri_obj_depot.put(app_tok, self._tok_identifier)

    async def try_fetch_ticket(self) -> AppTicket:
        ticket = None
        try:
            ticket = self.get_app_ticket()
        except (SerializableObjectNotAvialable, AppTicketExpired) as error:
            TplusOpenapiBroker._timber.info(f"App Ticket Error - not avialable or expired: '{error}', RENEWING.")
            ticket = await self.renew_app_ticket()
        finally:
            return ticket

    async def try_fetch_token(self) -> AppToken:
        token = None
        try:
            token = self.get_token()
        except (SerializableObjectNotAvialable, AppTokenExpired, AppTokenInvalid) as error:
            TplusOpenapiBroker._timber.info(f"App Token not avialable or expired: '{error}', RENEWING needed.")

            ticket = await self.try_fetch_ticket()
            token = await self.exchange_for_token(ticket)
            self.put_token(token)
        finally:
            return token

    async def renew_app_ticket(self):
        TplusOpenapiBroker._timber.debug(f"Calling renew_app_ticket()")

        http_call = self._http_call_builder.build(TplusTicketHttpCall, self._conf_vault)

        try:
            await http_call.refresh_app_ticket()
            app_tic = await PromisePool.wish(self._tic_identifier, None)
        except Exception as error:
            TplusOpenapiBroker._err_timber.error(f"Skipped waiting for app-ticket promise because: {error} - {error.args}")
            raise(error)

        return app_tic

    async def exchange_for_token(self, app_tic: AppTicket= None):
        TplusOpenapiBroker._timber.debug("Calling exchange_for_token()")

        http_call = self._http_call_builder.build(TplusAppTokenHttpCall, self._conf_vault)

        exchange_params = {
            TplusEndpointProfile._APP_TICKET_: app_tic.ticket,
            TplusOpenApiProperties._CERTIFICATE_: self._props.certificate
        }

        token_txt = await http_call.exchange_app_token(json = exchange_params)
        return AppToken(token_txt)

    async def refresh_token(self, app_tok: AppToken):
        TplusOpenapiBroker._timber.debug("Calling refresh_token()")

        refresh_params = {
            TplusOpenapiBroker._ARGKEY_GRANT_TYPE_: TplusOpenapiBroker._ARGVLU_REFRESH_TOKEN_,
            TplusOpenapiBroker._ARGKEY_REFRESH_TOKEN_: app_tok.refresh_token
        }

        http_call = self._http_call_builder.build(TplusAppTokenHttpCall, self._conf_vault)

        token_json = await http_call.refresh_app_token(url_params= refresh_params)
        return AppToken(token_json)

    async def request(self, api_path: str, api_method: str, request_body: dict):

        app_token = await self.try_fetch_token()
        headers = HttpHeaders(defaults={TplusOpenapiBrokerHttpCall._HDR_APP_TOKEN_: app_token.access_token})

        http_call = self._http_call_builder.build(TplusOpenapiBrokerHttpCall, self._conf_vault, api_path= api_path, api_method= api_method)
        http_call.update_http_headers(headers)

        return await http_call.broker_api_call(api_path, api_method, json= request_body)

    @property
    def view(self):
        return TplusOpenapiBroker.BrokerView.as_view(self._profile.name, self)


#!/usr/bin/python
# -*- coding: utf-8 -*-
import argparse
# from functools import lru_cache
# from itertools import permutations
from copy import deepcopy
import time
# import networkx as nx
# import collections

def arguments():
    # Handle command line arguments
    parser = argparse.ArgumentParser(description='Adventofcode.')
    parser.add_argument('-f', '--file', required=True)

    args = parser.parse_args()

    return args

class VMVersionTwo():
    def __init__(self, id):
        self.reset()
        self.instructions = None
        self.id = id

    def reset(self):
        self.memory = dict()
        self.incomming_message_queue = list()
        self.outgoing_message_queue = None
        self.state = 'terminated'  # running, waiting, terminated, queue
        self.last_index = 0
        self.num_of_send_transactions = 0

    def is_digit(self, n):
        try:
            int(n)
            return True
        except ValueError:
            return False

    def run_vm(self):
        self.memory['p'] = self.id
        self.state = 'running'
        idx = deepcopy(self.last_index)
        while True:
            cmd = self.instructions[idx]
            inst = cmd[0]
            first_registrer = cmd[1]
            jump = 1

            if first_registrer not in self.memory and first_registrer.isalpha():
                self.memory[first_registrer] = 0
            if len(cmd) > 2:
                second_registrer = cmd[2]
                if second_registrer not in self.memory and second_registrer.isalpha():
                    self.memory[second_registrer] = 0

            if inst == 'snd':
                if first_registrer.isalpha():
                    value = self.memory[first_registrer]
                else:
                    value = int(first_registrer)
                self.outgoing_message_queue.incomming_message_queue.append(value)
                self.outgoing_message_queue.state = 'queue'
                self.num_of_send_transactions += 1
            elif inst == 'rcv':
                if len(self.incomming_message_queue) == 0:
                    self.state = 'waiting'
                    self.last_index = idx
                    break
                else:
                    self.memory[first_registrer] = self.incomming_message_queue[0]
                    del self.incomming_message_queue[0]
            if inst == 'set':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] = value
            elif inst == 'add':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] += value
            elif inst == 'mul':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] *= value
            elif inst == 'mod':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] = self.memory[first_registrer] % value
            elif inst == 'jgz':
                jumping = False
                if self.is_digit(first_registrer):
                    if int(first_registrer) > 0:
                        jumping = True
                elif self.memory[first_registrer] > 0:
                    jumping = True
                if jumping:
                    if self.is_digit(second_registrer):
                        jump = int(second_registrer)
                    else:
                        jump = self.memory[second_registrer]
            idx += jump

class VM():
    def __init__(self):
        self.reset()
        self.instructions = None

    def reset(self):
        self.memory = dict()
        self.last_sound = None
        self.recoved_frequency = None

    def is_digit(self, n):
        try:
            int(n)
            return True
        except ValueError:
            return False

    def run_vm(self):
        idx = 0
        while True:
            cmd = self.instructions[idx]
            inst = cmd[0]
            first_registrer = cmd[1]
            jump = 1

            if first_registrer not in self.memory and first_registrer.isalpha():
                self.memory[first_registrer] = 0
            if len(cmd) > 2:
                second_registrer = cmd[2]
                if second_registrer not in self.memory and second_registrer.isalpha():
                    self.memory[second_registrer] = 0

            if inst == 'set':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] = value
            elif inst == 'snd':
                self.last_sound = int(self.memory[first_registrer])
            elif inst == 'add':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] += value
            elif inst == 'mul':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] *= value
            elif inst == 'mod':
                if self.is_digit(second_registrer):
                    value = int(second_registrer)
                else:
                    value = self.memory[second_registrer]
                self.memory[first_registrer] = self.memory[first_registrer] % value
            elif inst == 'jgz':
                jumping = False
                if self.is_digit(first_registrer):
                    if int(first_registrer) > 0:
                        jumping = True
                elif self.memory[first_registrer] > 0:
                    jumping = True
                if jumping:
                    if self.is_digit(second_registrer):
                        jump = int(second_registrer)
                    else:
                        jump = self.memory[second_registrer]
            elif inst == 'rcv':
                if self.memory[first_registrer] != 0:
                    self.recoved_frequency = self.last_sound
                    break
            idx += jump

def main():
    startTime = time.time()
    args = arguments()
    with open(args.file) as file:
        input_file = file.read().splitlines()
        input_file = [x.split() for x in input_file]
    vm = VM()
    vm.instructions = input_file
    vm.run_vm()
    print(f'Part1: {vm.recoved_frequency}')
    vm_one = VMVersionTwo(0)
    vm_two = VMVersionTwo(1)
    vm_one.instructions = input_file
    vm_two.instructions = input_file
    vm_one.outgoing_message_queue = vm_two
    vm_two.outgoing_message_queue = vm_one

    vms = [vm_one, vm_two]
    running_vm_idx = 0
    while True:
        if all(x.state == 'waiting' for x in vms):
            break
        if vms[running_vm_idx].state == 'waiting':
            if running_vm_idx == 0:
                running_vm_idx = 1
            else:
                running_vm_idx = 0
        else:
            vms[running_vm_idx].run_vm()

    print(f'Part2: {vm_two.num_of_send_transactions}')
    print(f'Execution time in seconds: {(time.time() - startTime)}')

if __name__ == '__main__':
    main()


from bs4 import BeautifulSoup
from urllib.parse import urlencode, urlunparse
import json
import os
import requests
import sys

def price(listing):
    price_node = listing.find('a', class_='text-price')
    try:
        price = list(price_node.children)[2]
    except:
        price = price_node.get_text()

    price = (
        price
            .strip()
            .replace(',', '')
            .replace(u'\xa3', '')
    )

    try:
        return int(price)
    except:
        return None

def beds(listing):
    try:
        return int(
            listing
                .find('span', class_='num-beds')
                .get_text()
                .strip()
        )
    except:
        return None

def parse_listing(listing):
    return {
        'listing_id': int(listing.attrs['data-listing-id']),
        'price': price(listing),
        'latitude': listing
            .find('meta', itemprop='latitude')
            .attrs['content'],
        'longitude': listing
            .find('meta', itemprop='longitude')
            .attrs['content'],
        'beds': beds(listing)
    }

def parse_listings(listings):
    return map(parse_listing, listings)

params = {
    'q': 'london',
    'results_sort': 'newest_listings',
    'search_source': 'home',
    'price_frequency': 'per_month',
    'radius': 0,
    'identifier': 'london',
    'page_size': 100,
    'pn': 1
}

url = urlunparse([
    'http',
    'www.zoopla.co.uk',
    'to-rent/property/london/',
    '',
    urlencode(params),
    ''
])

soup = BeautifulSoup(requests.get(url).content, 'html5lib')

print(len(soup.find_all('li', attrs={'data-listing-id': True})))

#listings = parse_listings()

# list(map(json.dumps, listings))