"""Make a new thread by calling clone

That is to say, make a new thread the normal way.

"""
from __future__ import annotations
from dataclasses import dataclass
from rsyscall._raw import ffi # type: ignore
from rsyscall.concurrency import OneAtATime
from rsyscall.epoller import AsyncFileDescriptor
from rsyscall.handle import Stack, WrittenPointer, Pointer, FutexNode, FileDescriptor, Task, FutexNode
from rsyscall.loader import Trampoline, NativeLoader
from rsyscall.memory.allocator import Arena
from rsyscall.memory.ram import RAM
from rsyscall.monitor import AsyncChildProcess, ChildProcessMonitor
from rsyscall.struct import Int32
from rsyscall.tasks.base_sysif import BaseSyscallInterface
from rsyscall.tasks.connection import SyscallConnection
from rsyscall.near.sysif import SyscallHangup
import contextlib
import logging
import rsyscall.far as far
import trio
import typing as t

from rsyscall.sched import CLONE
from rsyscall.signal import SIG
from rsyscall.sys.mman import PROT, MAP
from rsyscall.sys.wait import W

__all__ = [
    'ChildExit',
    'MMRelease',
    'ChildSyscallInterface',
    'launch_futex_monitor',
    'clone_child_task',
    'ForkThread',
]

class ChildExit(SyscallHangup):
    "The task we were sending syscalls to has exited"
    pass

class MMRelease(SyscallHangup):
    """The task we were sending syscalls to has either exited or exec'd; either way it can no longer respond

    More concretely, the task has left its old address space - it has called the
    mm_release kernel function (hence the name of this class).

    """
    pass

class ChildSyscallInterface(BaseSyscallInterface):
    """A connection to an rsyscall server that is one of our child processes

    We take as arguments here not only a SyscallConnection, but also an
    AsyncChildProcess monitoring the child process to which we will send
    syscalls.

    This is useful for situations where we can't rely on getting an EOF if the
    other side of a connection dies. That will happen, for example, whenever the
    child process is sharing a file descriptor table with us. In those
    situations, we need some other means to detect that a syscall will never be
    responded to, and signal it to the caller by throwing SyscallHangup.

    In this class, we detect a hangup while waiting for a syscall response by
    simultaneously monitoring the child process. If the child process exits, we
    stop waiting for the syscall response and throw SyscallHangup back to the
    caller.

    This is not just a matter of failure cases, it's also important for normal
    functionality. Detecting a hangup is our only way to discern whether a call
    to exit() was successful, and rsyscall.near.exit treats receiving an
    RsycallHangup as successful.

    We also take a futex_process: AsyncChildProcess. This is also used for
    normal functionality: futex_process should exit when the process has
    successfully called exec. This is again our only way of detecting a
    successful call to exec, and rsyscall.near.execve treats receiving an
    RsycallHangup as successful. (Concretely, futex_process will also exit when
    the process exits, not just when it execs, but that's harmless)

    A better way of detecting exec success would be great...

    """
    def __init__(self,
                 rsyscall_connection: SyscallConnection,
                 server_process: AsyncChildProcess,
                 futex_process: t.Optional[AsyncChildProcess],
    ) -> None:
        super().__init__(rsyscall_connection)
        self.server_process = server_process
        self.futex_process = futex_process
        self.logger = logging.getLogger(f"rsyscall.ChildSyscallInterface.{int(self.server_process.process.near)}")
        self.running_read = OneAtATime()

    @contextlib.asynccontextmanager
    async def _throw_on_child_exit(self) -> t.AsyncGenerator[None, None]:
        """Monitor the child process and throw if it exits or execs

        Naturally, if the child does exit or exec while we're in the context
        manager body, we'll cancel the context manager body so that we don't
        spend forever waiting on a dead child.

        This is useful for detecting a situation where we've sent a request and
        will never receive a response, particularly for syscalls as documented
        in the module docstring.

        The application to syscalls is the primary purpose of this method; but
        this method is also useful for some other thread implementations, so we
        expose it with this relatively generic interface.

        """
        child_exited = False
        futex_exited = False
        got_result = False
        async with trio.open_nursery() as nursery:
            async def server_exit() -> None:
                await self.server_process.waitpid(W.EXITED)
                nonlocal child_exited
                child_exited = True
                nursery.cancel_scope.cancel()
            async def futex_exit() -> None:
                if self.futex_process is not None:
                    await self.futex_process.waitpid(W.EXITED)
                    nonlocal futex_exited
                    futex_exited = True
                    nursery.cancel_scope.cancel()
            nursery.start_soon(server_exit)
            nursery.start_soon(futex_exit)
            yield
            got_result = True
            nursery.cancel_scope.cancel()
        if got_result:
            return
        elif child_exited:
            # this takes precedence over MMRelease, since it gives us more information
            raise ChildExit()
        elif futex_exited:
            raise MMRelease()

    async def _read_syscall_responses_direct(self) -> None:
        async with self._throw_on_child_exit():
            await self.rsyscall_connection.read_pending_responses()
        self.logger.debug("returning after reading some syscall responses")

    async def _read_pending_responses(self) -> None:
        async with self.running_read.needs_run() as needs_run:
            if needs_run:
                self.logger.debug("running read_syscall_responses_direct")
                await self._read_syscall_responses_direct()
                self.logger.debug("done with read_syscall_responses_direct")

async def launch_futex_monitor(ram: RAM,
                               loader: NativeLoader, monitor: ChildProcessMonitor,
                               futex_pointer: WrittenPointer[FutexNode]) -> AsyncChildProcess:
    """Launch a process to wait on a futex; then we monitor the process to monitor the futex

    This process calls futex(futex_pointer, FUTEX_WAIT, futex_pointer.value) and
    then exits, so this process will exit if and when the futex has FUTEX_WAKE
    called on it.

    Sadly, this is the best we can do with integrating futexes into our event
    loop. There used to be a way to get a file descriptor to represent a futex,
    but it was removed because it was racy.

    Something better would be really great - especially because it would allow
    incorporating pthreads locks and other shared memory concurrency mechanisms
    based on futexes, into a normal event loop.

    """
    async def op(sem: RAM) -> t.Tuple[Pointer[Stack], WrittenPointer[Stack]]:
        stack_value = loader.make_trampoline_stack(Trampoline(
            loader.futex_helper_func, [
                int(futex_pointer.near + ffi.offsetof('struct futex_node', 'futex')),
                futex_pointer.value.futex]))
        stack_buf = await sem.malloc(Stack, 4096)
        stack = await stack_buf.write_to_end(stack_value, alignment=16)
        return stack
    stack = await ram.perform_batch(op)
    futex_process = await monitor.clone(CLONE.VM|CLONE.FILES, stack)
    # wait for futex helper to SIGSTOP itself,
    # which indicates the trampoline is done and we can deallocate the stack.
    state = await futex_process.waitpid(W.EXITED|W.STOPPED)
    if state.state(W.EXITED):
        raise Exception("thread internal futex-waiting task died unexpectedly", state)
    # resume the futex_process so it can start waiting on the futex
    await futex_process.kill(SIG.CONT)
    # TODO uh we need to actually call something to free the stack
    return futex_process

async def clone_child_task(
        parent: ForkThread,
        flags: CLONE,
        trampoline_func: t.Callable[[FileDescriptor], Trampoline],
) -> t.Tuple[AsyncChildProcess, Task]:
    """Clone a new child process and setup the sysif and task to manage it

    We rely on trampoline_func to take a socket and give us a native function call with
    arguments that will speak the rsyscall protocol over that socket.

    We also create a futex process, which we use to monitor the ctid futex.
    This process allows us to detect when the child successfully finishes an
    exec; see the docstring of ChildSyscallInterface.  Because we set
    CLONE.CHILD_CLEARTID, the ctid futex will receive a FUTEX_WAKE when the
    child process exits or execs, and the futex process will accordingly exit.

    """
    # Open a channel which we'll use for the rsyscall connection
    [(access_sock, remote_sock)] = await parent.connection.open_async_channels(1)
    # Create a trampoline that will start the new process running an rsyscall server
    trampoline = trampoline_func(remote_sock)
    # Force these flags to be used
    flags |= CLONE.VM|CLONE.FILES|CLONE.IO|CLONE.SYSVSEM
    # TODO it is unclear why we sometimes need to make a new mapping here, instead of
    # allocating with our normal allocator; all our memory is already MAP.SHARED, I think.
    # We should resolve this so we can use the normal allocator.
    arena = Arena(await parent.task.mmap(4096*2, PROT.READ|PROT.WRITE, MAP.SHARED))
    async def op(sem: RAM) -> t.Tuple[t.Tuple[Pointer[Stack], WrittenPointer[Stack]],
                                                       WrittenPointer[FutexNode]]:
        stack_value = parent.loader.make_trampoline_stack(trampoline)
        stack_buf = await sem.malloc(Stack, 4096)
        stack = await stack_buf.write_to_end(stack_value, alignment=16)
        futex_pointer = await sem.ptr(FutexNode(None, Int32(0)))
        return stack, futex_pointer
    # Create the stack we'll need, and the zero-initialized futex
    stack, futex_pointer = await parent.ram.perform_batch(op, arena)
    # it's important to start the processes in this order, so that the thread
    # process is the first process started; this is relevant in several
    # situations, including unshare(NEWPID) and manipulation of ns_last_pid
    child_process = await parent.monitor.clone(flags|CLONE.CHILD_CLEARTID, stack, ctid=futex_pointer)
    futex_process = await launch_futex_monitor(
        parent.ram, parent.loader, parent.monitor, futex_pointer)
    # Create the new syscall interface, which needs to use not just the connection,
    # but also the child process and the futex process.
    syscall = ChildSyscallInterface(SyscallConnection(access_sock, access_sock),
                                    child_process, futex_process)
    # Set up the new task with appropriately inherited namespaces, tables, etc.
    # TODO correctly track all the namespaces we're in
    if flags & CLONE.NEWPID:
        pidns = far.PidNamespace(child_process.process.near.id)
    else:
        pidns = parent.task.pidns
    task = Task(syscall, child_process.process,
                parent.task.fd_table, parent.task.address_space, pidns)
    task.sigmask = parent.task.sigmask
    # Move ownership of the remote sock into the task and store it so it isn't closed
    remote_sock_handle = remote_sock.move(task)
    syscall.store_remote_side_handles(remote_sock_handle, remote_sock_handle)
    return child_process, task

from rsyscall.epoller import Epoller
from rsyscall.network.connection import Connection, ConnectionThread
class ForkThread(ConnectionThread):
    def __init__(self,
                 task: Task,
                 ram: RAM,
                 epoller: Epoller,
                 connection: Connection,
                 loader: NativeLoader,
                 monitor: ChildProcessMonitor,
    ) -> None:
        super().__init__(task, ram, epoller, connection)
        self.loader = loader
        self.monitor = monitor

    def _init_from(self, thr: ForkThread) -> None: # type: ignore
        super()._init_from(thr)
        self.loader = thr.loader
        self.monitor = thr.monitor

    async def _fork_task(self, flags: CLONE) -> t.Tuple[AsyncChildProcess, Task]:
        return await clone_child_task(
            self, flags, lambda sock: Trampoline(self.loader.server_func, [sock, sock]))


import os
import sys
import csv
import json
import subprocess
from glob import glob
from os import listdir
from os.path import isfile, join

services = ["features-service", "languagetool", "ncs", "news", "ocvn", "proxyprint", "restcountries", "scout-api",
            "scs", "erc20-rest-service", "genome-nexus", "person-controller", "problem-controller", "rest-study",
            "spring-batch-rest", "spring-boot-sample-app", "user-management", "cwa-verification", "market",
            "project-tracking-system"]
tools = ["evomaster-whitebox_data", "restler_data", "resttestgen_data", "restest_data", "bboxrt_data",
         "schemathesis_data", "tcases_data", "dredd_data", "evomaster-blackbox_data", "apifuzzer_data"]
class_name = ["app.coronawarn", "com.giassi.microservice", "com.test.sampleapp", "com.github.chrisgleissner",
              "org.restscs", "se.devscout.scoutapi", "com.in28minutes.rest.webservices.restfulwebservices",
              "eu.fayder.restcountries", "io.github.proxyprint.kitchen", "com.pfa.pack", "com.sw.project",
              "com.mongodb.starter", "org.devgateway", "org.tsdes.spring.examples.news", "org.restncs", "market",
              "org.languagetool", "org.cbioportal.genome_nexus", "org.javiermf.features", "io.blk.erc20"]

name = sys.argv[1]
curdir = os.getcwd()
res = ""
# for i in range(10):
for j in range(20):
    t_line = [0, 0, 0, 0, 0, 0]
    c_line = [0, 0, 0, 0, 0, 0]
    t_branch = [0, 0, 0, 0, 0, 0]
    c_branch = [0, 0, 0, 0, 0, 0]
    t_method = [0, 0, 0, 0, 0, 0]
    c_method = [0, 0, 0, 0, 0, 0]
    error = 0
    unique_err = 0
    crucial = 0
    mypath = os.path.join(curdir, name + "_data/" + services[j])
    if os.path.isdir(mypath):
        onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
        for dir_file in onlyfiles:
            if '_1.csv' in dir_file:
                with open(os.path.join(mypath, dir_file)) as f:
                    lines = f.readlines()
                    start = False
                    for line in lines:
                        if not start:
                            start = True
                            continue
                        element = line.split(',')
                        if "EMDriver" in element[2] or "EmbeddedControl" in element[2]:
                            continue
                        c_branch[0] = c_branch[0] + int(element[6])
                        t_branch[0] = t_branch[0] + int(element[5]) + c_branch[0]
                        c_line[0] = c_line[0] + int(element[8])
                        t_line[0] = t_line[0] + int(element[7]) + c_line[0]
                        c_method[0] = c_method[0] + int(element[12])
                        t_method[0] = t_method[0] + c_method[0] + int(element[11])
            elif '_2.csv' in dir_file:
                with open(os.path.join(mypath, dir_file)) as f:
                    lines = f.readlines()
                    start = False
                    for line in lines:
                        if not start:
                            start = True
                            continue
                        element = line.split(',')
                        if "EMDriver" in element[2] or "EmbeddedControl" in element[2]:
                            continue
                        c_branch[1] = c_branch[1] + int(element[6])
                        t_branch[1] = t_branch[1] + int(element[5]) + c_branch[1]
                        c_line[1] = c_line[1] + int(element[8])
                        t_line[1] = t_line[1] + int(element[7]) + c_line[1]
                        c_method[1] = c_method[1] + int(element[12])
                        t_method[1] = t_method[1] + int(element[11]) + c_method[1]
            elif '_3.csv' in dir_file:
                with open(os.path.join(mypath, dir_file)) as f:
                    lines = f.readlines()
                    start = False
                    for line in lines:
                        if not start:
                            start = True
                            continue
                        element = line.split(',')
                        if "EMDriver" in element[2] or "EmbeddedControl" in element[2]:
                            continue
                        c_branch[2] = c_branch[2] + int(element[6])
                        t_branch[2] = t_branch[2] + int(element[5]) + c_branch[2]
                        c_line[2] = c_line[2] + int(element[8])
                        t_line[2] = t_line[2] + int(element[7]) + c_line[2]
                        c_method[2] = c_method[2] + int(element[12])
                        t_method[2] = t_method[2] + int(element[11]) + c_method[2]
            elif '_4.csv' in dir_file:
                with open(os.path.join(mypath, dir_file)) as f:
                    lines = f.readlines()
                    start = False
                    for line in lines:
                        if not start:
                            start = True
                            continue
                        element = line.split(',')
                        if "EMDriver" in element[2] or "EmbeddedControl" in element[2]:
                            continue
                        c_branch[3] = c_branch[3] + int(element[6])
                        t_branch[3] = t_branch[3] + int(element[5]) + c_branch[3]
                        c_line[3] = c_line[3] + int(element[8])
                        t_line[3] = t_line[3] + int(element[7]) + c_line[3]
                        c_method[3] = c_method[3] + int(element[12])
                        t_method[3] = t_method[3] + int(element[11]) + c_method[3]
            elif '_5.csv' in dir_file:
                with open(os.path.join(mypath, dir_file)) as f:
                    lines = f.readlines()
                    start = False
                    for line in lines:
                        if not start:
                            start = True
                            continue
                        element = line.split(',')
                        if "EMDriver" in element[2] or "EmbeddedControl" in element[2]:
                            continue
                        c_branch[4] = c_branch[4] + int(element[6])
                        t_branch[4] = t_branch[4] + int(element[5]) + c_branch[4]
                        c_line[4] = c_line[4] + int(element[8])
                        t_line[4] = t_line[4] + int(element[7]) + c_line[4]
                        c_method[4] = c_method[4] + int(element[12])
                        t_method[4] = t_method[4] + int(element[11]) + c_method[4]
            elif '_6.csv' in dir_file:
                with open(os.path.join(mypath, dir_file)) as f:
                    lines = f.readlines()
                    start = False
                    for line in lines:
                        if not start:
                            start = True
                            continue
                        element = line.split(',')
                        if "EMDriver" in element[2] or "EmbeddedControl" in element[2]:
                            continue
                        c_branch[5] = c_branch[5] + int(element[6])
                        t_branch[5] = t_branch[5] + int(element[5]) + c_branch[5]
                        c_line[5] = c_line[5] + int(element[8])
                        t_line[5] = t_line[5] + int(element[7]) + c_line[5]
                        c_method[5] = c_method[5] + int(element[12])
                        t_method[5] = t_method[5] + int(element[11]) + c_method[5]
            elif 'error' in dir_file:
                with open(os.path.join(mypath, dir_file), 'r') as f:
                    data = json.load(f)
                unique = []
                for d in data:
                    uniq = d[:d.find('\n')]
                    if uniq not in unique:
                        unique.append(uniq)

                error = error + len(data)
                unique_err = unique_err + len(unique)

                for uni in unique:
                    flag = True
                    for n in class_name:
                        if n in uni:
                            flag = False
                    if flag:
                        crucial = crucial + 1
    res = res + "0,0,0,0,0,0\n"

    for k in range(6):
        if t_line[k] != 0:
            line = c_line[k] / t_line[k]
        else:
            line = 0
        if t_branch[k] != 0:
            branch = c_branch[k] / t_branch[k]
        else:
            branch = 0
        if t_method[k] != 0:
            method = c_method[k] / t_method[k]
        else:
            method = 0
        res = res + str(line) + ',' + str(branch) + ',' + str(method) + ',' + str(error / 7) + ',' + str(
            unique_err / 7) + ',' + str(crucial / 7) + '\n'

with open('res.csv', 'w') as f:
    f.write(res)

subprocess.call('mv res.csv ' + name + "_data", shell=True)


import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.utils.data
import torch.utils.data.distributed
from torch.utils.data.dataloader import default_collate
from tensorboardX import SummaryWriter
import sys
import os
import argparse
from lib.config.hrnet_config import update_config
from lib.config.hrnet_config import config
from lib.models.Hrnet import HigherResolutionNet
from lib.datasets.trip_dataloader import trip_retrieval
from pathlib import Path
# loss
from lib.utils.loss import triplet_loss_cl
from lib.models.Vit_header import VitEncoder

class retrieval_net(nn.Module):
    def __init__(self,cfg, is_train = True, is_transform=True):
        super(retrieval_net,self).__init__()
        self.backbone = HigherResolutionNet(cfg, is_train=is_train)
        self.is_transform = is_transform
        if self.is_transform:
            self.self_attention = VitEncoder(cfg.MODEL_EXTRA.STAGE4.NUM_CHANNELS[0])
    def forward(self, images):
        features = self.backbone(images)
        batch_size = features.shape[0]
        if self.is_transform:
            output_feature = self.self_attention(features)
        else:
            extracted_feature = nn.functional.adaptive_avg_pool2d(features,(1,1))
            output_feature = extracted_feature.reshape(batch_size,-1)
        # need to normalize the feature
        output_feature = output_feature / torch.norm(output_feature,dim=-1,keepdim=True)
        return output_feature

def parse_args():
    parser = argparse.ArgumentParser(description='Train keypoints network')
    parser.add_argument(
        '--cfg', help='experiment configure file name', required=True, type=str)
    parser.add_argument(
        '--transform', help='whether using transform', default=False ,type=bool)
    args, rest = parser.parse_known_args()
    update_config(args.cfg) # 把config的文件更新过去
    return args

def get_optimizer(model):
    lr = config.TRAIN.LR
    optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.module.parameters()), lr=lr) # 整体模型权重均全部重新训练
    return model, optimizer

def load_backbone(model,pretrained_file):
    pretrained_state_dict = torch.load(pretrained_file)
    model_state_dict_backbone = model.module.backbone.state_dict()
    prefix_b = 'backbone.'
    new_pretrained_state_dict_bacbone = {}
    for k, v in pretrained_state_dict.items():
        if k.replace(prefix_b, "") in model_state_dict_backbone and v.shape == model_state_dict_backbone[k.replace(prefix_b, "")].shape:     #.replace(prefix, "") .replace(prefix, "")
            new_pretrained_state_dict_bacbone[k.replace(prefix_b, "")] = v
    print("load statedict from {}".format(pretrained_file))
    model.module.backbone.load_state_dict(new_pretrained_state_dict_bacbone)
    return model

def load_checkpoint(model, optimizer, output_dir, filename='checkpoint.pth.tar'):
    file = os.path.join(output_dir, filename)
    if os.path.isfile(file):
        checkpoint = torch.load(file)
        start_epoch = checkpoint['epoch']
        metrics = checkpoint['loss']
        model.module.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        print('=> load checkpoint {} (epoch {})'
              .format(file, start_epoch))

        return start_epoch, model, optimizer, metrics

    else:
        print('=> no checkpoint found at {}'.format(file))
        return 0, model, optimizer, np.inf

def save_checkpoint(states, is_best, output_dir, filename='checkpoint.pth.tar'):
    torch.save(states, os.path.join(output_dir, filename))
    if is_best and 'state_dict' in states:
        torch.save(states['state_dict'],
                   os.path.join(output_dir, 'model_best.pth.tar'))

def main():
    args = parse_args() # 读取 cfg 参数，config表示之后需要看一下
    result_log_dir = Path(config.OUTPUT_DIR)
    result_log_dir.mkdir(parents=True, exist_ok=True)

    gpus = [int(i) for i in config.GPUS.split(',')]
    image_folder = '/Extra/panzhiyu/img_retrieval/shopee-product-matching/train_images'
    train_dataset = trip_retrieval(image_folder,is_train = True)
    test_dataset = trip_retrieval(image_folder,is_train = False)
    train_loader = torch.utils.data.DataLoader(
        train_dataset,
        batch_size=config.TRAIN.BATCH_SIZE * len(gpus),
        shuffle=config.TRAIN.SHUFFLE,
        num_workers=config.WORKERS,
        pin_memory=True)

    test_loader = torch.utils.data.DataLoader(
        test_dataset,
        batch_size=config.TEST.BATCH_SIZE * len(gpus),
        shuffle=True,
        num_workers=config.WORKERS,
        pin_memory=True)

    cudnn.benchmark = config.CUDNN.BENCHMARK
    torch.backends.cudnn.deterministic = config.CUDNN.DETERMINISTIC
    torch.backends.cudnn.enabled = config.CUDNN.ENABLED
    print('=> Constructing models ..')
    model = retrieval_net(config, is_train= True, is_transform=args.transform)
    with torch.no_grad():
        model = torch.nn.DataParallel(model, device_ids=gpus).cuda()
    model, optimizer = get_optimizer(model)
    start_epoch = config.TRAIN.BEGIN_EPOCH
    end_epoch = config.TRAIN.END_EPOCH
    least_test_loss = np.inf # enough large

    if config.NETWORK.PRETRAINED_BACKBONE: # no pretrained test
        print(f'Using backbone {config.NETWORK.PRETRAINED_BACKBONE}')
        model = load_backbone(model, config.NETWORK.PRETRAINED_BACKBONE) # load POSE ESTIMATION BACKBONE
    if config.TRAIN.RESUME:
        start_epoch, model, optimizer, metrics_load = load_checkpoint(model, optimizer, config.OUTPUT_DIR) # TODO: Load the A1 metrics
        least_test_loss = metrics_load

    tb_log_dir = Path(os.path.join(config.OUTPUT_DIR,'tensorboard_log'))
    tb_log_dir.mkdir(parents=True, exist_ok=True)

    writer_dict = {
        'writer': SummaryWriter(log_dir=str(tb_log_dir)),
        'train_global_steps': 0,
        'valid_global_steps': 0,
    }

    print('=> Training...')
    device=torch.device('cuda')
    for epoch in range(start_epoch, end_epoch):
        print('Epoch: {}'.format(epoch))
        train_sim_loss = AverageMeter()
        test_sim_loss = AverageMeter()
        trip_class_loss = triplet_loss_cl()
        # The train part
        model.train()
        for i, batch in enumerate(train_loader):
            # import pdb; pdb.set_trace()
            q_images, g_images = batch
            if q_images.shape[0] == 1:
                continue # cannot do the triplet loss
            q_images = q_images.to(device)
            g_images = g_images.to(device)

            q_features = model(q_images)
            g_features = model(g_images)

            # calculate the loss as triplet
            trip_loss = trip_class_loss(q_features,g_features)
            train_sim_loss.update(trip_loss.item())
            optimizer.zero_grad()
            trip_loss.backward()
            optimizer.step()

            if i % config.PRINT_FREQ == 0:
                gpu_memory_usage = torch.cuda.memory_allocated(0)
                msg = f'Epoch:[{epoch}][{i}/{len(train_loader)}]\t'\
                        f'Loss_trip: {train_sim_loss.val:.3f}({train_sim_loss.avg:.3f})\t'\
                        f'Memory {gpu_memory_usage:.1f}'
                print(msg)
                writer = writer_dict['writer']
                global_steps = writer_dict['train_global_steps']
                writer.add_scalar('train_loss_trip', train_sim_loss.avg, global_steps)
                writer_dict['train_global_steps'] = global_steps + 1

        # store the first model
        if epoch ==  0:
            model_name =os.path.join(config.OUTPUT_DIR,
                                          f'epoch{epoch}_state.pth.tar')
            print('saving current model state to {}'.format(
                model_name))
            torch.save(model.module.state_dict(), model_name)
        # The eval part
        model.eval()
        for i, batch in enumerate(test_loader):
            q_images, g_images = batch
            if q_images.shape[0] == 1:
                continue # cannot do the triplet loss
            q_images = q_images.to(device)
            g_images = g_images.to(device)

            q_features = model(q_images)
            g_features = model(g_images)

            # calculate the loss as triplet
            trip_loss = trip_class_loss(q_features,g_features)
            test_sim_loss.update(trip_loss.item())

            if i % config.PRINT_FREQ == 0:
                gpu_memory_usage = torch.cuda.memory_allocated(0)
                msg = f'Test:[{epoch}][{i}/{len(test_loader)}]\t'\
                        f'Loss_trip: {test_sim_loss.val:.3f}({test_sim_loss.avg:.3f})\t'\
                        f'Memory {gpu_memory_usage:.1f}'
                print(msg)
                writer = writer_dict['writer']
                global_steps = writer_dict['valid_global_steps']
                writer.add_scalar('test_loss_trip', test_sim_loss.avg, global_steps)
                writer_dict['test_global_steps'] = global_steps + 1

        test_loss = test_sim_loss.avg
        # compare the loss
        if test_loss < least_test_loss:
            least_test_loss = test_loss
            best_model = True
        else:
            best_model = False

        save_checkpoint({
            'epoch': epoch + 1,
            'state_dict': model.module.state_dict(),
            'loss': test_loss,
            'optimizer': optimizer.state_dict(),
        }, best_model, config.OUTPUT_DIR)

    final_model_state_file = os.path.join(config.OUTPUT_DIR,
                                          'final_state.pth.tar')
    torch.save(model.module.state_dict(), final_model_state_file)

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

if __name__ == '__main__':
    main()


import datetime
import pytz

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.utils import timezone

from autoslug import AutoSlugField

class Popup(models.Model):
    title = models.CharField(_('title'), max_length=255)
    published = models.BooleanField(_('published'), default=True)
    slug = AutoSlugField(
        _('slug'), populate_from='title',
        blank=True, editable=True, unique_with='language'
    )
    language = models.CharField(
        _('language'), max_length=255,
        choices=settings.LANGUAGE_CHOICES, default='all'
    )

    content = models.TextField(_('content'), null=True, blank=True)

    auto_open = models.BooleanField(_('auto open'), default=False)
    auto_open_delay = models.PositiveIntegerField(
        _('delay'), help_text=_('Delay in seconds before popup is shown.'),
        null=True, blank=True
    )
    auto_open_remind_me_later_btn = models.BooleanField(
        _('"Remind me later" button'), default=True
    )
    auto_open_dont_show_again_btn = models.BooleanField(
        _('"Don\'t show this again" button'), default=True
    )
    auto_open_from_hour = models.TimeField(
        _('from hour'), default=datetime.time(hour=0)
    )
    auto_open_to_hour = models.TimeField(
        _('to hour'), default=datetime.time(hour=23, minute=59, second=59)
    )

    class Meta:
        app_label = 'popups'
        ordering = ('title',)
        verbose_name = _('popup')
        verbose_name_plural = _('popups')

    def __str__(self):
        return self.title

    def is_auto_open(self):
        if not self.auto_open:
            return False
        now = timezone.now().astimezone(pytz.timezone(settings.TIME_ZONE)) \
                      .time()
        if self.auto_open_from_hour < self.auto_open_to_hour:
            return (
                self.auto_open_from_hour <= now and
                self.auto_open_to_hour >= now
            )
        else:
            return not (
                self.auto_open_from_hour >= now and
                self.auto_open_to_hour <= now
            )

MATCH_CHOICES = (
    ('exact', _('Exact match')),
    ('startswith', _('Starts with'))
)

class PopupPath(models.Model):
    popup = models.ForeignKey(Popup, related_name='paths')
    path = models.CharField(_('path'), max_length=255)
    match = models.CharField(
        _('match'), max_length=255,
        choices=MATCH_CHOICES, default='exact'
    )

    class Meta:
        app_label = 'popups'
        verbose_name = _('path')
        verbose_name_plural = _('paths')


"""Test of the object-store container module."""
import hashlib
import io
import os
import random
import shutil
import tempfile
import uuid
import zlib

import psutil
import pytest

from disk_objectstore import Container
import disk_objectstore.utils as utils
import disk_objectstore.exceptions as exc

class UnopenableBytesIO(io.BytesIO):

    def __enter__(self):
        raise AttributeError('__enter__ method disabled for UnopenableBytesIO')

def _assert_empty_repo(container):
    """Check that the container has no objects and no pack files.

    :param container: a Container.
    """
    counts = container.count_objects()
    assert counts['packed'] == 0, (
        'The container should be empty at the beginning '
        '(but there are {} packed objects)'.format(counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should be empty at the beginning '
        '(but there are {} loose objects)'.format(counts['loose'])
    )
    assert counts['pack_files'] == 0, (
        'The container should be empty at the beginning '
        '(but there are {} pack files)'.format(counts['pack_files'])
    )

def _add_objects_loose_loop(container, data):
    """Add loose objects to a container, within a for loop.

    :param container: a Container
    :param data: a dictionary where the key is an arbitrary identifier, and the value is a byte string.
    :return: a dictionary where the keys are the object UUIDs as returned by the container, and the values
      are the keys in the original ``data`` dictionary.
    """
    retval = {}
    for key, content in data.items():
        obj_uuid = container.add_object(content)
        retval[obj_uuid] = key
    return retval

def _add_objects_directly_to_pack(container, data, compress):
    """Add objects directly to pack files.

    :param container: a Container
    :param data: a dictionary where the key is an arbitrary identifier, and the value is a byte string.
    :return: a dictionary where the keys are the object UUIDs as returned by the container, and the values
      are the keys in the original ``data`` dictionary.
    :param compress: whether to use compression or not.
    """
    # Store sorted lists of keys and values
    keys = list(data.keys())
    values = [data[key] for key in keys]

    obj_uuids = container.add_objects_to_pack(values, compress=compress)

    return dict(zip(obj_uuids, keys))

def _get_data_and_md5_loop(container, obj_uuids):
    """Get the MD5 of the data stored under the given container, one at a time in a loop.

    :param container: a Container
    :param obj_uuids: a list of object UUIDs
    :return: a dictionary where the keys are the object UUIDs and the values are the MD5 hexdigests.
    """
    retval = {}
    for obj_uuid in obj_uuids:
        retrieved_content = container.get_object_content(obj_uuid)
        retval[obj_uuid] = hashlib.md5(retrieved_content).hexdigest()
    return retval

def _get_data_and_md5_bulk(container, obj_uuids):
    """Get the MD5 of the data stored under the given container as a single bulk operation.

    :param container: a Container
    :param obj_uuids: a list of object UUIDs
    :return: a dictionary where the keys are the object UUIDs and the values are the MD5 hexdigests.
    """
    retval = {}
    retrieved_contents = container.get_object_contents(obj_uuids)
    for obj_uuid in retrieved_contents:
        retval[obj_uuid] = hashlib.md5(retrieved_contents[obj_uuid]).hexdigest()
    return retval

@pytest.mark.parametrize('retrieve_bulk', [True, False])
def test_add_get_loose(temp_container, generate_random_data, retrieve_bulk):
    """Add a number of objects (one by one, loose) to the container.

    Then retrieve them and check the content is correct."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    # Store
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    counts = temp_container.count_objects()
    assert counts['packed'] == 0, (
        'The container should have no packed objects '
        '(but there are {} instead)'.format(counts['packed'])
    )
    assert counts['loose'] == len(obj_md5s), (
        'The container should have {} loose objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    if retrieve_bulk:
        retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)
    else:
        retrieved_md5s = _get_data_and_md5_loop(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('use_compression,retrieve_bulk', [(True, True), (True, False), (False, True), (False, False)])
def test_add_get_with_packing(temp_container, generate_random_data, use_compression, retrieve_bulk):
    """Add a number of objects (one by one, loose) to the container.

    Then retrieve them and check the content is correct."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    # Store
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    # Pack all loose objects
    temp_container.pack_all_loose(compress=use_compression)

    counts = temp_container.count_objects()
    assert counts['packed'] == len(obj_md5s), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    if retrieve_bulk:
        retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)
    else:
        retrieved_md5s = _get_data_and_md5_loop(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('use_compression', [True, False])
def test_directly_to_pack_content(temp_container, generate_random_data, use_compression):
    """Add a number of objects directly to packs.

    Then retrieve them and check the content is correct (always bulk retrieve for simplicity)."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    obj_md5s = _add_objects_directly_to_pack(temp_container, data, compress=use_compression)

    counts = temp_container.count_objects()
    assert counts['packed'] == len(data), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(data), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('use_compression,open_streams', [(True, True), (True, False), (False, True), (False, False)])
def test_directly_to_pack_streamed(temp_container, generate_random_data, use_compression, open_streams):
    """Add a number of objects directly to packs, using streams.

    Then retrieve them and check the content is correct (always bulk retrieve for simplicity)."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    # Store sorted lists of keys and values
    keys = list(data.keys())
    values = [data[key] for key in keys]

    if open_streams:
        streams = []
        streams_copy = []
        with tempfile.TemporaryDirectory() as temp_dir:
            # Read a given fixed order, otherwise later when reconstructing obj_md5s we would have a problem
            for key, content in zip(keys, values):
                file_path = os.path.join(temp_dir, key)
                with open(file_path, 'bw') as fhandle:
                    fhandle.write(content)
                streams.append(utils.LazyOpener(file_path, mode='rb'))
                streams_copy.append(utils.LazyOpener(file_path, mode='rb'))
            obj_uuids = temp_container.add_streamed_objects_to_pack(
                streams, compress=use_compression, open_streams=True
            )
            # I check that instead it fails if I forget to open the streams (and that it does not create side effects)
            # The error that I get here would be: "LazyOpener' object has no attribute 'read'"
            with pytest.raises(AttributeError):
                temp_container.add_streamed_objects_to_pack(streams_copy, compress=use_compression, open_streams=False)

    else:
        streams = [UnopenableBytesIO(value) for value in values]
        streams_copy = [UnopenableBytesIO(value) for value in values]
        obj_uuids = temp_container.add_streamed_objects_to_pack(streams, compress=use_compression, open_streams=False)
        # I check that instead it fails if I try to open but I am not allowed to
        # (and that it does not create side effects). Note that I disabled explicitly the __enter__ method
        # in the class UnopenableBytesIO to make sure that this raises
        with pytest.raises(AttributeError):
            temp_container.add_streamed_objects_to_pack(streams_copy, compress=use_compression, open_streams=True)

    obj_md5s = dict(zip(obj_uuids, keys))

    counts = temp_container.count_objects()
    assert counts['packed'] == len(data), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(data), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('loose_prefix_len,pack_prefix_len', [(0, 2), (2, 2), (3, 2), (0, 3), (2, 3), (3, 3)])
def test_prefix_lengths(temp_dir, generate_random_data, pack_prefix_len, loose_prefix_len):
    """Check if the prefix lengths are honored."""
    container = Container(temp_dir)
    container.init_container(clear=True, pack_prefix_len=pack_prefix_len, loose_prefix_len=loose_prefix_len)
    # Check that the `get_folder` method returns the expected folder name
    assert container.get_folder() == os.path.realpath(temp_dir)

    assert container.loose_prefix_len == loose_prefix_len
    assert container.pack_prefix_len == pack_prefix_len

    _assert_empty_repo(container)
    data = generate_random_data()

    # Store
    obj_md5s = _add_objects_loose_loop(container, data)

    loose_firstlevel = os.listdir(container._get_loose_folder())  # pylint: disable=protected-access
    assert len(loose_firstlevel) > 0
    if loose_prefix_len == 0:
        assert all(len(inode) == 32 for inode in loose_firstlevel)
    else:
        assert all(len(inode) == loose_prefix_len for inode in loose_firstlevel)

    counts = container.count_objects()
    assert counts['packed'] == 0, (
        'The container should have 0 packed objects '
        '(but there are {} instead)'.format(counts['packed'])
    )
    assert counts['loose'] == len(obj_md5s), (
        'The container should have {} loose objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['loose'])
    )

    retrieved_md5s = _get_data_and_md5_bulk(container, obj_md5s.keys())
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

    # Pack all loose objects
    container.pack_all_loose()

    pack_firstlevel = os.listdir(container._get_pack_folder())  # pylint: disable=protected-access
    assert len(pack_firstlevel) > 0
    assert all(len(inode) == pack_prefix_len for inode in pack_firstlevel)

    counts = container.count_objects()
    assert counts['packed'] == len(obj_md5s), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    retrieved_md5s = _get_data_and_md5_bulk(container, obj_md5s.keys())
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

    # Test also the validation functions
    valid_pack_ids = ['0' * pack_prefix_len, 'a' * pack_prefix_len]
    invalid_pack_ids = ['0' * (pack_prefix_len + 1), 'a' * (pack_prefix_len + 1), 'g' * pack_prefix_len]
    valid_loose_prefixes = ['0' * loose_prefix_len, 'a' * loose_prefix_len]
    invalid_loose_prefixes = [
        '0' * (loose_prefix_len + 1), 'a' * (loose_prefix_len + 1), 'g' * loose_prefix_len if loose_prefix_len else 'g'
    ]

    for pack_id in valid_pack_ids:
        assert container._is_valid_pack_id(  # pylint: disable=protected-access
            pack_id
        ), "'{}' should be valid".format(pack_id)
    for pack_id in invalid_pack_ids:
        assert not container._is_valid_pack_id(  # pylint: disable=protected-access
            pack_id
        ), "'{}' should be invalid".format(pack_id)
    for loose_prefix in valid_loose_prefixes:
        assert container._is_valid_loose_prefix(  # pylint: disable=protected-access
            loose_prefix
        ), "'{}' should be valid".format(loose_prefix)
    for loose_prefix in invalid_loose_prefixes:
        assert not container._is_valid_loose_prefix(  # pylint: disable=protected-access
            loose_prefix
        ), "'{}' should be invalid".format(loose_prefix)

    # I close the container, as this is needed on Windows
    container.close()

@pytest.mark.parametrize('loose_prefix_len,pack_prefix_len', [(-1, 2), (2, -1), (2, 0)])
def test_invalid_prefix_lengths(temp_dir, pack_prefix_len, loose_prefix_len):
    """Check if the prefix lengths are honored."""
    container = Container(temp_dir)
    with pytest.raises(ValueError):
        container.init_container(clear=True, pack_prefix_len=pack_prefix_len, loose_prefix_len=loose_prefix_len)

def test_initialisation(temp_dir):
    """Test that the initialisation function works as expected."""
    container = Container(temp_dir)
    assert not container.is_initialised

    with pytest.raises(exc.NotInitialised):
        _ = container.loose_prefix_len
    with pytest.raises(exc.NotInitialised):
        _ = container.pack_prefix_len

    # Check that the session cannot be obtained before initialising
    with pytest.raises(FileNotFoundError):
        container._get_session(create=False, raise_if_missing=True)  # pylint: disable=protected-access
    assert container._get_session(create=False, raise_if_missing=False) is None  # pylint: disable=protected-access

    container.init_container()
    assert container.is_initialised

    #This call should go through
    container.init_container(clear=True)
    assert container.is_initialised

    with pytest.raises(FileExistsError) as excinfo:
        container.init_container()
    assert 'already exists' in str(excinfo.value)

    # I artificially remove one of the folders: it should notice and say it's not initialised
    os.rmdir(os.path.join(container.get_folder(), 'sandbox'))
    assert not container.is_initialised

    # Close open files (the DB)
    container.close()

    # Remove the folder
    shutil.rmtree(temp_dir)
    assert not container.is_initialised
    # Make back the folder: I leave temp_dir in a consistent state, and moreover I
    # check that the empty folder is again considered non-initialised
    os.mkdir(temp_dir)
    assert not container.is_initialised

    # Create an empty file
    with open(os.path.join(container.get_folder(), 'somefile'), 'w'):
        pass
    # Final re-initialisation
    with pytest.raises(FileExistsError) as excinfo:
        container.init_container()
    assert 'already some file or folder' in str(excinfo.value)

# Only three options: if pack_objects is False, the values of compress_packs is ignored
@pytest.mark.parametrize('pack_objects,compress_packs', [(True, True), (True, False), (False, False)])
def test_unknown_uuids(temp_container, generate_random_data, pack_objects, compress_packs):
    """Put some data in the container, then check that unknown UUIDs raise the correct error."""
    # Generate and store some data, just not to have an empty container
    data = generate_random_data()
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    if pack_objects:
        temp_container.pack_all_loose(compress=compress_packs)

    # Pick any valid UUID
    obj_uuids = list(obj_md5s.keys())

    # 5 unknown UUIDs + one invalid string
    unknown_uuids = [uuid.uuid4().hex for _ in range(5)] + ['invalid-uuid-string']

    # Loop read
    for unknown_uuid in unknown_uuids:
        with pytest.raises(exc.NotExistent):
            temp_container.get_object_content(unknown_uuid)

        # Currently, the exception is actually raised when accessing the stream,
        # not at stream creation
        stream = temp_container.get_object_stream(unknown_uuid)
        with pytest.raises(exc.NotExistent):
            with stream:
                pass

    # 6 invalid UUIDs + all valid ones
    uuids_list = unknown_uuids + obj_uuids
    # I shuffle so they are really in random order
    random.shuffle(uuids_list)

    ##### Bulk reads from here on
    # skip_if_missing=True, get contents
    contents = temp_container.get_object_contents(uuids_list, skip_if_missing=True)
    # The retrieved values should be only the valid ones
    assert set(contents.keys()) == set(obj_uuids)
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

    # skip_if_missing=True, get streams
    missing = []
    check_md5s = {}
    with temp_container.get_object_streams_and_size(uuids_list, skip_if_missing=True) as triplets:
        for obj_uuid, stream, size in triplets:
            if stream is None:
                assert size == 0
                missing.append(obj_uuid)
            else:
                check_md5s[obj_uuid] = stream.read()
                assert len(check_md5s[obj_uuid]) == size
    # The retrieved values should be only the valid ones
    assert missing == []
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

    # skip_if_missing=False, get objects
    contents = temp_container.get_object_contents(uuids_list, skip_if_missing=False)
    # There should be only one return value
    for unknown_uuid in unknown_uuids:
        # Check that it's there, that it's noe, and pop it
        assert contents.pop(unknown_uuid) is None
    # After popping, I should be left with the same case as above
    assert set(contents.keys()) == set(obj_uuids)
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

    # skip_if_missing=False, get streams
    missing = []
    check_md5s = {}
    with temp_container.get_object_streams_and_size(uuids_list, skip_if_missing=False) as triplets:
        for obj_uuid, stream, size in triplets:
            if stream is None:
                assert size == 0
                missing.append(obj_uuid)
            else:
                check_md5s[obj_uuid] = stream.read()
                assert len(check_md5s[obj_uuid]) == size
    # The retrieved values should be only the valid ones
    assert set(missing) == set(unknown_uuids)
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

@pytest.mark.parametrize('compress_packs', [True, False])
def test_sizes(temp_container, generate_random_data, compress_packs):
    """Check that the information on size is reliable."""
    size_info = temp_container.get_total_size()
    assert size_info['total_size_packed'] == 0
    assert size_info['total_size_packed_on_disk'] == 0
    assert size_info['total_size_packfiles_on_disk'] == 0
    assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
    assert size_info['total_size_loose'] == 0

    data = generate_random_data()
    total_object_size = sum(len(value) for value in data.values())
    obj_md5s = _add_objects_loose_loop(temp_container, data)
    # Try to count size after retrieving, just to be sure
    assert sum(
        len(content) for content in temp_container.get_object_contents(obj_md5s.keys()).values()
    ) == total_object_size

    # Check the size for loose objects
    size_info = temp_container.get_total_size()
    assert size_info['total_size_packed'] == 0
    assert size_info['total_size_packed_on_disk'] == 0
    assert size_info['total_size_packfiles_on_disk'] == 0
    assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
    assert size_info['total_size_loose'] == total_object_size

    # Pack without compression
    temp_container.pack_all_loose(compress=compress_packs)
    # Try to count size after retrieving, just to be sure
    assert sum(
        len(content) for content in temp_container.get_object_contents(obj_md5s.keys()).values()
    ) == total_object_size

    if compress_packs:
        # Compress data manually to get compressed size
        # In the current version, compression level is hardcoded.
        # If this becomes a parameter, we need to change this test
        # Note that until when we want to support py3.5, we cannot specify the
        # level as a keyword argument, as this was added only in python 3.6
        compressed_data = {
            key: zlib.compress(val, temp_container._COMPRESSLEVEL)  # pylint: disable=protected-access
            for key, val in data.items()
        }
        total_compressed_size = sum(len(value) for value in compressed_data.values())

        size_info = temp_container.get_total_size()
        assert size_info['total_size_packed'] == total_object_size
        assert size_info['total_size_packed_on_disk'] == total_compressed_size
        assert size_info['total_size_packfiles_on_disk'] == total_compressed_size
        assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
        assert size_info['total_size_loose'] == 0
    else:
        size_info = temp_container.get_total_size()
        assert size_info['total_size_packed'] == total_object_size
        assert size_info['total_size_packed_on_disk'] == total_object_size
        assert size_info['total_size_packfiles_on_disk'] == total_object_size
        assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
        assert size_info['total_size_loose'] == 0

def test_get_object_streams_closes(temp_container, generate_random_data):
    """Test that get_object_streams_and_size closes intermediate streams."""
    data = generate_random_data()
    # Store
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    # I get all objects first - this will actually internally go through the same function
    # `get_object_streams_and_size`, but I need to do it as this might open additional files,
    # namely the SQLite DB (possibly more than one file due to the fact it's open in WAL mode).
    # The following checks are still meaningful, I check that if I do it again I don't open more files.
    temp_container.get_object_contents(obj_md5s.keys())

    current_process = psutil.Process()
    start_open_files = len(current_process.open_files())

    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True):
        # I don't use the triplets
        pass
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    print(current_process.open_files())
    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True) as triplets:
        # I loop over the triplets, but I don't do anything
        for _ in triplets:
            pass
    # Check that at the end nothing is left open
    print(current_process.open_files())
    assert len(current_process.open_files()) == start_open_files

    # I actually read the content
    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True) as triplets:
        # I loop over the triplets, but I don't do anything
        for _, stream, _ in triplets:
            stream.read()
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    ##### Same test after packing
    temp_container.pack_all_loose()
    # I get all objects first, again - this is because it might have closed the DB files while packing
    temp_container.get_object_contents(obj_md5s.keys())
    # I now update the count
    start_open_files = len(current_process.open_files())

    with temp_container.get_object_streams_and_size(obj_md5s.keys()):
        # I don't use the triplets
        pass
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    with temp_container.get_object_streams_and_size(obj_md5s.keys()) as triplets:
        # I loop over the triplets, but I don't do anything
        for _ in triplets:
            pass
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    # I actually read the content
    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True) as triplets:
        # I loop over the triplets, but I don't do anything
        for _, stream, _ in triplets:
            stream.read()
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files


import tornado.ioloop
import tornado.web
from tornado import gen

#import pg
import ptdb

class DBHandler:
    inst = None
    def __init__(self, db):
        self.db = db
        DBHandler.inst = self

    @gen.coroutine
    def insertRow(self, str1):
        cur = yield self.db.cursor()
        sqlstr = ('INSERT INTO "tTest"(str1) VALUES (%s) RETURNING id1;')
        sqlarr = (str1, )
        yield cur.execute(sqlstr, sqlarr)

    @gen.coroutine
    def listRow(self):
        cur = yield self.db.cursor()
        sqlstr = ('SELECT id1, str1 FROM "tTest";')
        yield cur.execute(sqlstr)
        rows = cur.fetchall()

        ret_rows = []

        for row in rows:
            ret_rows.append({
                "id" : row[0],
                "str" : row[1]
            })

        return ret_rows

class MainHandler(tornado.web.RequestHandler):
    @gen.coroutine
    def get(self):
        val = self.get_argument("str", default="")
        yield DBHandler.inst.insertRow(val)
        lst = yield DBHandler.inst.listRow()
        for ll in lst:
            print(ll)

        self.write("")

if __name__ == "__main__":
    app = tornado.web.Application([
        (r"/", MainHandler),
    ])
    app.listen(8763)
    db = ptdb.Connection("cultural107", "localhost", "5432", "cultural107", "cultural107")
    DBHandler(db)
    tornado.ioloop.IOLoop.current().start()


# -*- coding: utf-8 -*-

# Import required libraries
import os
from random import randint

import plotly.plotly as py

import flask
import dash
from dash.dependencies import Input, Output, State, Event
import dash_core_components as dcc
import dash_html_components as html
import plotly.graph_objs as go
import os
from datetime import datetime as dt
import base64
import pandas

# Setup the app
# Make sure not to change this file name or the variable names below,
# the template is configured to execute 'server' on 'app.py'
server = flask.Flask(__name__)
server.secret_key = os.environ.get('secret_key', str(randint(0, 1000000)))
app = dash.Dash(__name__, server=server)

colors = {
    'background': '#999999',
    'text': '#7FDBFF'
}

all_options = {
    'Invoice': ['Concrete', 'Landscaping', 'Site Work'],
    'Estimate': ['Concrete', 'Landscaping', 'Site Work']
}

def create_logo():
	return dcc.Graph(
        id='life-exp-vs-gdp',
        figure={
            'data': [
                go.Scatter(
                    x=[],
                    y=[],
                    text=[],
                    mode='markers',
                    opacity=0.7,
                    marker={
                        'size': 15,
                        'line': {'width': 0.5, 'color': 'white'}
                    },
                )
            ],
            'layout': go.Layout(
            	images=[dict(
        			source="https://raw.githubusercontent.com/CullenBoldt/msystems/master/logo.png",
        			xref="paper", yref="paper",
        			x=.4, y=-.7,
        			sizex=2, sizey=2,
        			xanchor="center", yanchor="bottom")],
        		autosize=False, height=100, width=165,
        		xaxis={},
            	yaxis={},
            	margin={'l': 40, 'b': 40, 't': 10, 'r': 10},
            	legend={'x': 0, 'y': 1},
           		hovermode='closest'

            )

    })

app.layout = html.Div(style={'backgroundColor': colors['background']}, children=[

	html.Div(style={'backgroundColor': colors['background']},children=[      # page 1

    	html.H2(children="MSystems Concrete Estimate", style={'width': '100%', 'padding': '10px 0px 0px 0px'}),
    	html.Hr(),

        html.Div(style={'backgroundColor': colors['background']},children=[ # row 1

    		# Field for cost per yard of concrete
    		html.Div([
    			html.H4(children='''
        				Concrete:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Volume concrete:
    					'''),

    			dcc.Input(
    				id='concrete_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/yard concrete:
    					'''),
    			dcc.Input(
    				id='concrete_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),   #end subpage 1

		html.Div(style={'backgroundColor': colors['background']},children=[ # row 2

    		# Field for cost per yard of Backfill
    		html.Div([
    			html.H4(children='''
        				Backfill:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Volume backfill:
    					'''),

    			dcc.Input(
    				id='backfill_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot backfill:
    					'''),
    			dcc.Input(
    				id='backfill_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 2

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 3

    		# Field for cost per yard of Base
    		html.Div([
    			html.H5(children='''
        				Base:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Quantity base:
    					'''),

    			dcc.Input(
    				id='base_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/ton base:
    					'''),
    			dcc.Input(
    				id='base_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 3

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 4

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Trucking:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Quantity base:
    					'''),

    			dcc.Input(
    				id='trucking_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/ton base trucking:
    					'''),
    			dcc.Input(
    				id='trucking_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 4

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 6

    		# Field for cost trucking base
    		html.Div([
    			html.H4(children='''
        				Finish:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='finish_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='finish_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 6

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 6.5

    		# Field for cost trucking base
    		html.Div([
    			html.H4(children='''
        				Form Set:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='form_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='form_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 6.5

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 7

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Steel Tying:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='tying_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='tying_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 7

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 7.5

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Plaster:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='plaster_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='plaster_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 7.5

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 8

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Cable and Steel:
    					''',className = "two columns" ),
    			html.Div([
    				html.Div(children='''
        				Job price:
    					'''),

    			dcc.Input(
    				id='cable_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.H5(children='''
        				Bagging/Cover:
    					''',className = "three columns" ),

    			html.Div([
    				html.Div(children='''
        				Job Price:
    					'''),
    			dcc.Input(
    				id='poly_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "three columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 5

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 8

    		# Field for cost trucking base
    		html.Div([
    			html.H4(children='''
        				Pump Truck:
    					''',className = "two columns" ),
    			html.Div([
    				html.Div(children='''
        				Job Price:
    					'''),

    			dcc.Input(
    				id='truck_price',
    				placeholder='0.00',
    				type='text',
    				value='1'),
    			],className = "four columns" ),

    			html.H4(children='''
        				Profit:
    					''',className = "two columns" ),
    			html.Div([
    				html.Div(children='''
        				Percentage: e.g.   15 = 15%
    					'''),

    			dcc.Input(
    				id='profit',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "three columns" ),

    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 8

    ], className = "page" ), # end page 1

    html.Div(style={'backgroundColor': colors['background']},children=[  # page 2
    	html.A([ 'Print PDF' ],
			className="button no-print",
    		style=dict(position="absolute", top=-40, right=0)),

    	html.Div([
    		html.Div([create_logo()],className = "three column" ),
    		#html.H4(children='''Concrete Estimate:''',className = "five column" ),
    		#html.H3(children="MSystems Concrete Estimate", style={'width': '50%', 'padding': '100px 0px 0px 0px'}),
    	],  className = "row" ),

		html.Div([
			#html.H4(children='''Concrete Estimate:'''),
			html.Div(id='total_out')

		],  className = "row" ), # end final row

    	html.Div([
        	html.Table(id='total_out', className = "w3-table-all w3-tiny"),
            ]),

        html.Div([
        	dcc.Graph(id='pie',config={'displayModeBar': False})], style={'width': '100%', 'padding': '0px 0px 80px 1px','backgroundColor': colors['background']},
            ),

    	html.Div(children='''*This estimate is a best professional assessment including the cost of
    	hiring any subcontractors, the price of materials, and any other labor involved.'''),

    ], className = "page" ), # end page 2

])

#best professional assessment – including the cost of hiring any subcontractors, the price of materials, and any other labor involved.

#Turn key Form setting:  Backfill,  Steel tying, Finishing, Base Material, Concrete

#Cable packages and steel, Bagging poly and cover poly,

@app.callback(
    Output(component_id='total_out', component_property='children'),
    [Input(component_id='concrete_price', component_property='value'),
    Input(component_id='concrete_volume', component_property='value'),
    Input(component_id='backfill_price', component_property='value'),
    Input(component_id='backfill_volume', component_property='value'),
    Input(component_id='base_price', component_property='value'),
    Input(component_id='base_volume', component_property='value'),
    Input(component_id='trucking_price', component_property='value'),
    Input(component_id='trucking_volume', component_property='value'),
    Input(component_id='cable_price', component_property='value'),
    Input(component_id='poly_price', component_property='value'),
    Input(component_id='finish_price', component_property='value'),
    Input(component_id='finish_volume', component_property='value'),
    Input(component_id='form_price', component_property='value'),
    Input(component_id='form_volume', component_property='value'),
    Input(component_id='tying_price', component_property='value'),
    Input(component_id='tying_volume', component_property='value'),
    Input(component_id='plaster_price', component_property='value'),
    Input(component_id='plaster_volume', component_property='value'),
    Input(component_id='truck_price', component_property='value'),
    Input(component_id='profit', component_property='value')]
)
def generate_table(conp,conv,backp,backv,basep,basev,truckbp,truckbv,cable,poly,finishp,finishv,formp,formv,tyingp,tyingv,plasterp,plasterv,truckp,profit):

	dict = {'Concrete' : float(conp)*float(conv), 'Backfill' : float(backp)*float(backv),
			'Base Material' : float(basep)*float(basev), 'Base Trucking': float(truckbv)*float(truckbp),
			'Cable' : float(cable), 'Poly' : float(poly), 'Finish': float(finishp)*float(finishv),
			'Form': float(formp)*float(formv), 'Tying':float(tyingp)*float(tyingv),
			'Plaster':float(plasterp)*float(plasterv), 'Pump Truck' :float(truckp)}

	profit_margin = float(profit) * .01

	for i in dict.keys():
		dict[i] = dict[i]/(1-profit_margin)

	total = format(round(sum(dict.values()),2),',')

	for i in dict.keys():
		dict[i] = str(format(round(dict[i],2), ','))

	table = []
	for row in dict.items():
		html_row = []
		for i in range(len(row)):
			html_row.append( html.Td([ row[i] ]) )
		table.append( html.Tr( html_row, style={'height': '5'}))

	table.append(html.Tr([html.Td([ "Total Estimate"]), html.Td([total])]))
	return table

@app.callback(
    Output(component_id='pie', component_property='figure'),
    [Input(component_id='concrete_price', component_property='value'),
    Input(component_id='concrete_volume', component_property='value'),
    Input(component_id='backfill_price', component_property='value'),
    Input(component_id='backfill_volume', component_property='value'),
    Input(component_id='base_price', component_property='value'),
    Input(component_id='base_volume', component_property='value'),
    Input(component_id='trucking_price', component_property='value'),
    Input(component_id='trucking_volume', component_property='value'),
    Input(component_id='cable_price', component_property='value'),
    Input(component_id='poly_price', component_property='value'),
    Input(component_id='finish_price', component_property='value'),
    Input(component_id='finish_volume', component_property='value'),
    Input(component_id='form_price', component_property='value'),
    Input(component_id='form_volume', component_property='value'),
    Input(component_id='tying_price', component_property='value'),
    Input(component_id='tying_volume', component_property='value'),
    Input(component_id='plaster_price', component_property='value'),
    Input(component_id='plaster_volume', component_property='value'),
    Input(component_id='truck_price', component_property='value'),
    Input(component_id='profit', component_property='value')]
)
def update_pie(conp,conv,backp,backv,basep,basev,truckbp,truckbv,cable,poly,finishp,finishv,formp,formv,tyingp,tyingv,plasterp,plasterv,truckp,profit):

	price_dict = {'Concrete' : float(conp)*float(conv), 'Backfill' : float(backp)*float(backv),
			'Base Material' : float(basep)*float(basev), 'Base Trucking': float(truckbv)*float(truckbp),
			'Cable' : float(cable), 'Poly' : float(poly), 'Finish': float(finishp)*float(finishv),
			'Form': float(formp)*float(formv), 'Tying':float(tyingp)*float(tyingv),
			'Plaster':float(plasterp)*float(plasterv), 'Pump Truck' :float(truckp)}

	profit_margin = float(profit) * .01

	for i in price_dict.keys():
		price_dict[i] = round(price_dict[i]/(1-profit_margin),2)

	layout_pie = {}
	data = [
		dict(
            type='pie',
            labels=[i for i in price_dict.keys()],
            values=[i for i in price_dict.values()],
            name='Production Breakdown',
            hoverinfo="label+value+percent",
            textinfo="label+percent+name",
            rotation = 270,
            hole=0 ,
            marker=dict(
                colors=['#fac1b7', '#a9bb95', '#92d8d8']
            ),
            domain={"x": [0, .8], 'y':[0, 1]},
        	)
	]
	#layout_pie['images'] = [dict(
        #source="https://raw.githubusercontent.com/CullenBoldt/msystems/master/logo.png",
        #xref="paper", yref="paper",
        #x=0.1, y=1,
        #sizex=0.4, sizey=0.4,
        #xanchor="center", yanchor="bottom"
      #)]
	layout_pie['font'] = dict(color='#000000')
	layout_pie['legend'] = dict(font=dict(color='#000000', size='15'), orientation='v', bgcolor='rgba(0,0,0,0)')
	layout_pie['paper_bgcolor']= '#999999'

	figure = dict(data=data, layout= layout_pie)
	return figure

if 'DYNO' in os.environ:
    app.scripts.append_script({
        'external_url': 'https://cdn.rawgit.com/chriddyp/ca0d8f02a1659981a0ea7f013a378bbd/raw/e79f3f789517deec58f41251f7dbb6bee72c44ab/plotly_ga.js'
    })

external_css = [ "https://cdnjs.cloudflare.com/ajax/libs/normalize/7.0.0/normalize.min.css",
        "https://cdnjs.cloudflare.com/ajax/libs/skeleton/2.0.4/skeleton.min.css",
        "//fonts.googleapis.com/css?family=Raleway:400,300,600",
        "https://cdn.rawgit.com/plotly/dash-app-stylesheets/5047eb29e4afe01b45b27b1d2f7deda2a942311a/goldman-sachs-report.css",
        "https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css",
        "https://www.w3schools.com/w3css/4/w3.css"]

for css in external_css:
    app.css.append_css({ "external_url": css })

external_js = [ "https://code.jquery.com/jquery-3.2.1.min.js",
        "https://cdn.rawgit.com/plotly/dash-app-stylesheets/a3401de132a6d0b652ba11548736b1d1e80aa10d/dash-goldman-sachs-report-js.js" ]

for js in external_js:
    app.scripts.append_script({ "external_url": js })

# Run the Dash app
if __name__ == '__main__':
    app.server.run(debug=True, threaded=True)


from Bio.SeqUtils.ProtParam import ProteinAnalysis
from Bio import SeqIO
from Bio.SeqUtils import ProtParamData
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import string

sequences = []

matplotlib.rc('font',family='Arial')

fig, axs = plt.subplots(4, sharex=True, sharey=True)

titles = ['Hxt1p','Hxt2p','Hxt3p','Hxt4p']

axs[0].set_xticks(np.arange(0, 570, 50))

fn = open('hydrophobicOutput.txt', 'w')
for rec in SeqIO.parse("../sequenceAnalysis/sequences/yeast_transporters_protein.fasta", 'fasta'):
	sequences.append(str(rec.seq))

for i in range(len(sequences)):
	X = ProteinAnalysis(sequences[i])
	kd = X.protein_scale(ProtParamData.kd, 19, edge=0.1)
	num_1 = [g + 10 for g in range(len(kd))]
	axs[i].plot(num_1, kd)
	axs[i].plot(num_1, [1.6 for g in range(len(kd))], color='red', linewidth=0.4)
	axs[i].grid(True)
	axs[i].margins(tight=True, x=-0, y=None)

	for k in kd:
		if k >= 1.6:
			fn.write(titles[i] + '\t'  + sequences[i][kd.index(k)]
			 + '\t'+ str(kd.index(k)+10) + '\t'+  str(k) + '\n')

b = 0
for ax in axs:
    ax.text(15, 2, string.ascii_uppercase[b],
                            size=10, weight='bold', fontname="Arial")
    b += 1
fig.add_subplot(111, frameon=False)
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.ylabel("Hydropathy Index", labelpad=10, fontweight='bold')
plt.xlabel("Residue Position", labelpad=10, fontweight='bold')
plt.subplots_adjust(hspace=0.1)
plt.title("Hydrophobicity of Hxt1-4p", fontweight='bold',fontsize=14)

fn.close()

plt.tight_layout()

plt.savefig('Hydrophobicity.png', dpi=800)


#-------------------------------------------------------------------------------
# Name:        util.py
# Purpose:     1. find last error record in the database  fawn-monitor.appspot.com/fawn/queryLastRecord
#              2. query record in specific date range fawn-monitor.appspot.com/fawn/queryRecord
#              3. find last error record in the database  fawn-monitor.appspot.com/fdacs/queryLastRecord
#              4. query record in specific date range fawn-monitor.appspot.com/fdacs/queryRecord
# Author:      Dawei Jia
#
# Created:     12/19/2013
# Copyright:   (c) DaweiJia 2013
# Licence:     <your licence>
#-------------------------------------------------------------------------------
import logging
import datetime
import database
from monitor import FawnMonitor
from google.appengine.api import users
#from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext import db
import webapp2
class QueryLastFawnRecord(webapp2.RequestHandler):
    '''query last fawn record'''

    def get(self):

        query = db.GqlQuery("SELECT * FROM Record \
                            ORDER BY record_time DESC")
        showRecordHelper.showLastRecord(self,query.get())

class QueryFawnRecord(webapp2.RequestHandler):
    '''query fawn record in date range'''
    def get(self):
        html = showRecordHelper.queryHtml(self)
        self.response.out.write(html)
        if self.request.get("from") != "":
            startTime = self.request.get("from").split('/')
            endTime = self.request.get("to").split('/')
            query = db.GqlQuery("""SELECT * FROM Record
                                WHERE record_time >= DATE(%s,%s,%s) and record_time <= DATE(%s,%s,%s)
                                ORDER BY record_time DESC
                                """% (startTime[2],startTime[0],startTime[1],endTime[2],endTime[0],str(eval(endTime[1])+1)))
            results = query.fetch(100)
            showRecordHelper.showRecord(self,results,startTime, endTime)
        self.response.out.write("""
""")

class QueryLastFdacsRecord(webapp2.RequestHandler):
    '''query last fdacs record '''
    def get(self):

        query = db.GqlQuery("SELECT * FROM FdacsRecord \
                            ORDER BY record_time DESC")

        showRecordHelper.showLastRecord(self,query.get())

class QueryFdacsRecord(webapp2.RequestHandler):
    '''query fdacs record in date range'''
    def get(self):
        html = showRecordHelper.queryHtml(self)
        self.response.out.write(html)
        if self.request.get("from") != "":
            startTime = self.request.get("from").split('/')
            endTime = self.request.get("to").split('/')
            query = db.GqlQuery("""SELECT * FROM FdacsRecord
                                WHERE record_time >= DATE(%s,%s,%s) and record_time <= DATE(%s,%s,%s)
                                ORDER BY record_time DESC
                                """% (startTime[2],startTime[0],startTime[1],endTime[2],endTime[0],str(eval(endTime[1])+1)))
            results = query.fetch(10000)
            showRecordHelper.showRecord(self,results,startTime,endTime)

        self.response.out.write("""
""")

class showRecordHelper():
    '''show Record Helper'''
    dicts = {"QueryFdacsRecord":"fdacs","QueryFawnRecord":"fawn"}

    @classmethod
    def queryHtml(self,resp):
        '''build query html'''
        return """

            <meta charset="utf-8">
            <link rel="stylesheet" href="http://code.jquery.com/ui/1.10.3/themes/smoothness/jquery-ui.css">

            <link rel="stylesheet" href="/resources/demos/style.css">

              <form action="/%s/queryRecord" method="get">
                From
                <input type="text" id="from" name="from">
                to
                <input type="text" id="to" name="to">

<input type="submit" value="Query">

              </form>
           """ % self.dicts[resp.__class__.__name__]
    @classmethod
    def showLastRecord(self,resp,result):
        '''show last record '''
        resp.response.out.write("----The last error message----
")
        if result is None:
            resp.response.out.write("NO RECORD IN THE DATABASE !")
            return
        if result.error_code != '200':
            resp.response.out.write("NO RESPONSE FROM SERVER ! @ " + str(result.record_time))
            return
        else:
            stnIdList = result.error_details.split(",")
            errorTimeList = result.error_time.split(",")
            for id in stnIdList:
                resp.response.out.write("""NO UPDATE STATION %s @ %s
""" %(id, str(errorTimeList[stnIdList.index(id)])))
            return

    @classmethod
    def showRecord(self,resp,results,start,end):
        '''show record in a certain date range'''
        resp.response.out.write("----RESULT FROM %s TO %s----
" % ("/".join(start), "/".join(end)))
        if len(results) == 0:
            resp.response.out.write("NO RECORD IN THE DATABASE !" )
            return
        resp.response.out.write("""

                                        error_code
                                        error_details
                                        error_time
                                        record_time

""")

        for result in results:
            error_detail_list = str(result.error_details).split(',')
            error_time_list = str(result.error_time).split(',')
            for each_detail in error_detail_list:
                index = error_detail_list.index(each_detail)
                each_time = error_time_list[index]
                resp.response.out.write("""

                                            %s
                                            %s
                                            %s
                                            %s

"""%(str(result.error_code),each_detail,each_time,str(result.record_time)))

        return

class Option(webapp2.RequestHandler):

    def get(self):
        path = self.request.path
        self.response.out.write("""

Choose your action

            1.check monitor
            2.query record
            3.query last record

        """ % (path,path,path))
        return

application = webapp2.WSGIApplication(
                                    [('/fawn/queryLastRecord',QueryLastFawnRecord),
                                     ('/fawn/queryRecord',QueryFawnRecord),
                                     ('/fdacs/queryLastRecord',QueryLastFdacsRecord),
                                     ('/fdacs/queryRecord',QueryFdacsRecord),
                                     ('/fawn', Option),
                                     ('/fdacs', Option)],
                                    debug = True)


import triangle as tr

import numpy as np
import matplotlib.pyplot as plt
from numpy.polynomial.legendre import leggauss as roots

from mpl_toolkits import mplot3d
from matplotlib import tri
import matplotlib as mpl
from matplotlib.lines import Line2D
from ..Utils import isBetween
import pandas as pd

class Geometria:
    def __init__(this, vertices):
        this.vertices = vertices
        this.cbe = []
        this.cbn = []
class Delaunay1V(Geometria):
    def __init__(this, vertices, params,cbe = [],valorCBE=10, plot=False):
        this.bordesCBE = cbe
        this.valorCBE= valorCBE
        this.params = params
        super().__init__(vertices)
        this.seg = []
        for i in range(len(this.vertices)-1):
            this.seg.append([i,i+1])
        this.seg.append([i+1,0])
        this.original = dict(vertices=np.array(this.vertices),segments=np.array(this.seg))
        this.triangular = tr.triangulate(this.original,this.params)
        if plot:
            tr.compare(plt, this.original, this.triangular)
        count = 0
        for i in this.triangular['segments']:
            if count > 1:
                if np.sum(np.isin(np.array(this.cbe)[:,0], i[0]))<1:
                    this.cbe.append([i[0],0])
                if np.sum(np.isin(np.array(this.cbe)[:,0], i[1]))<1:
                    this.cbe.append([i[1],0])
            else:
                this.cbe.append([i[0],0])
            if np.sum(np.isin(np.array(this.cbe)[:,0], i[1]))<1:
                    this.cbe.append([i[1],0])
            count+=1
        this.diccionarios = this.triangular['triangles'].tolist()
        this.tipos = np.zeros([len(this.diccionarios)]).astype(str)
        this.tipos[:] = 'T1V'
        this.gdls = this.triangular['vertices'].tolist()
    def darNodosCB(this,segmento):
        a = []
        ps=this.original['vertices'][this.original['segments'][segmento]].tolist()
        for i,p in enumerate(this.triangular['vertices']):
            if isBetween(ps[0], ps[1], p):
                a.append(i)
        return np.array(a)
    def generarCB(this,bordes,valor=0):
        cb = []
        this.bordesCBE = bordes
        this.valorCBE= valor
        for segmento in bordes:
            nodos = this.darNodosCB(segmento)
            cbe = np.zeros([len(nodos),2])
            cbe[:,0] = nodos
            cbe[:,1] = valor
            cb+= cbe.tolist()
        return cb
    def generarDatos(this):
        m = []
        for t in this.triangular['triangles'].tolist():
            coordsModelo = this.original['vertices'].flatten().tolist()
            # Coordendas de cada triangulo
            coords = np.array(this.gdls)[np.ix_(t)]
            cx = [np.average(coords[:,0])]
            cy = [np.average(coords[:,1])]
            coords = coords.flatten().tolist()
            #Saber si es cb
            cb = [np.any(np.isin(t,np.array(this.cbe)[:,0]))*1]
            fila = coordsModelo+coords+cx+cy+cb
            m.append(fila)
        return m
    def areaRefiner(this,model,norm):
        X1 = 'X1'
        X2 = 'Y1'
        X3 = 'X2'
        X4 = 'Y2'
        X5 = 'X2'
        X6 = 'Y3'
        X7 = 'X4'
        X8 = 'Y4'
        X9 = 'X5'
        X10 = 'Y5'
        X11 = 'X6'
        X12 = 'Y6'
        X13 = 'CX'
        X14 = 'CY'
        X15 = 'XE1'
        X16 = 'YE1'
        X17 = 'XE2'
        X18 = 'YE2'
        X19 = 'XE3'
        X20 = 'YE3'
        X21 = 'CB'
        dt = pd.DataFrame.from_records(this.generarDatos())
        dt.columns = [X1,X2,X3,X4,X5,X6,X7,X8,X9,X10,X11,X12,X13,X14,X15,X16,X17,X18,X19,X20,X21]
        xs = norm(dt)
        ys = model(xs.values)
        this.triangular['triangle_max_area'] = ys
        tnueva = tr.triangulate(this.triangular,'ra')
        tr.compare(plt,this.triangular,tnueva)
        this.triangular = tnueva
        this.diccionarios = this.triangular['triangles'].tolist()
        this.tipos = np.zeros([len(this.diccionarios)]).astype(str)
        this.tipos[:] = 'T1V'
        this.gdls = this.triangular['vertices'].tolist()
        this.cbe = this.generarCB(this.bordesCBE,this.valorCBE)

def Imesh(tf,tw,a,b,params,plot=True):
    corners = [[0,0],[a,0],[a,tf],[a/2+tw/2,tf],[a/2+tw/2,tf+b],[a,tf+b],[a,2*tf+b],[0,2*tf+b],[0,tf+b],[a/2-tw/2,tf+b],[a/2-tw/2,tf],[0,tf]]
    seg = []
    for i in range(len(corners)-1):
        seg.append([i,i+1])
    seg.append([i+1,0])
    A = dict(vertices=np.array(corners),segments=np.array(seg))
    B = tr.triangulate(A,params)
    if plot:
        tr.compare(plt, A, B)
    return B,corners

def _strdelaunay(constrained=True,delaunay=True,a=None,q=None):
    p = ''
    if constrained:
        p = 'p'
    if a == None:
        a = ''
    else:
        a = 'a'+format(a)
    D = ''
    if delaunay:
        D = 'D'
    if q == None:
        q=''
    else:
        if type(q) == int:
            if q > 35:
                raise "No sepuedecrearunatriangulacion conangulos menores a 35 grados"
        q = 'q'+format(q)
    return p+a+D+q+'i'
def generarGeometria(triang):
    cbe = []
    count = 0
    for i in triang['segments']:
        if count > 1:
            if np.sum(np.isin(np.array(cbe)[:,0], i[0]))<1:
                cbe.append([i[0],0])
            if np.sum(np.isin(np.array(cbe)[:,0], i[1]))<1:
                cbe.append([i[1],0])
        else:
            cbe.append([i[0],0])
        if np.sum(np.isin(np.array(cbe)[:,0], i[1]))<1:
                cbe.append([i[1],0])
        count+=1
    g = Geometria([-1])

    g.triangular = triang
    g.diccionarios = triang['triangles'].tolist()
    g.gdls = triang['vertices'].tolist()
    g.tipos = np.zeros([len(g.diccionarios)]).astype(str)
    g.tipos[:] = 'T1V'

    return g


import RPi.GPIO as GPIO
import notifier
import time
import logging

gpio_pin = 7

logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)

def startListeningForRing():
    try:
        GPIO.setmode(GPIO.BOARD)
        GPIO.setup(gpio_pin, GPIO.IN)

        GPIO.add_event_detect(gpio_pin, GPIO.FALLING, callback = ringHandler, bouncetime = 800)
        print("Listening for ring signal on GPIO PIN #" + str(gpio_pin))

        while True:
            time.sleep(0.2)
        print("Stopped listing for Ring")

    except KeyboardInterrupt:
        GPIO.cleanup()

def ringHandler(pin):
    localtime = time.asctime( time.localtime(time.time()) )
    print("Doorbell rang at: " + localtime)

    notifier.sendNotification()


import hmac
import hashlib
import json
import api
from flask import Flask
from flask import request

app = Flask(__name__)
app.config.from_json('config.json')

@app.route("/", methods=['GET', 'POST'])
def index():
    body = request.data
    signature = request.headers['x-pyrus-sig']
    secret = str.encode(app.config['SECRET_KEY'])

    if _is_signature_correct(body, secret, signature):
        return _prepare_response(body.decode('utf-8'))

def _is_signature_correct(message, secret, signature):
    digest = hmac.new(secret, msg=message, digestmod=hashlib.sha1).hexdigest()
    return hmac.compare_digest(digest, signature.lower())

def _prepare_response(body):
    task = json.loads(body)["task"]
    mails = api.getMails(task["fields"][0]["value"])
    api.sendMails(mails, "Вебинар " + task["fields"][0]["value"], task["fields"][1]["value"])
    return {"text": "Письма разосланы",
            "approval_choice": "approved"}

if __name__ == "__main__":
    app.run()