from __future__ import print_function
from math import ceil
import numpy as np
import sys
import pdb

import torch
import torch.optim as optim
import torch.nn as nn

import generator
import discriminator
import helpers

CUDA = True
VOCAB_SIZE = 15000
MAX_SEQ_LEN = 40
START_LETTER = 0
BATCH_SIZE = 8
MLE_TRAIN_EPOCHS = 10
ADV_TRAIN_EPOCHS = 10
POS_NEG_SAMPLES = 100000

GEN_EMBEDDING_DIM = 300
GEN_HIDDEN_DIM = 256
DIS_EMBEDDING_DIM = 64
DIS_HIDDEN_DIM = 64

oracle_samples_path = './oracle_samples.trc'
oracle_state_dict_path = './oracle_EMBDIM32_HIDDENDIM32_VOCAB5000_MAXSEQLEN20.trc'
pretrained_gen_path = './gen_MLEtrain_EMBDIM32_HIDDENDIM32_VOCAB5000_MAXSEQLEN20.trc'
pretrained_dis_path = './dis_pretrain_EMBDIM_64_HIDDENDIM64_VOCAB5000_MAXSEQLEN20.trc'

from tqdm import tqdm
def getLength(line):
    vec=[int(token) for token in line.split()]
    return vec.index(0)
def calculatePPL(gen,testpath):
    testset=loadData(testpath)
    testset_tensor=torch.tensor(testset)

    length=[]
    with open(testpath,'r') as fin:
        for line in fin:
            length.append(getLength(line))
    length=np.array(length)

    nll_all=[]
    TEST_SIZE=testset_tensor.shape[0]
    for i in tqdm(range(0, TEST_SIZE)):
        inp, target = helpers.prepare_generator_batch(testset_tensor[i:i + 1], start_letter=START_LETTER,
                                                              gpu=CUDA)
        nll = gen.batchNLLLoss(inp, target)
        nll_all.append(float(nll.data.cpu()))
    nll_all=np.array(nll_all)

    return np.mean(2**(nll_all/length))

def train_generator_MLE(gen, gen_opt, real_data_samples, epochs):
    """
    Max Likelihood Pretraining for the generator
    """
    for epoch in range(epochs):
        print('epoch %d : ' % (epoch + 1), end='')
        sys.stdout.flush()
        total_loss = 0

        for i in range(0, POS_NEG_SAMPLES, BATCH_SIZE):
            inp, target = helpers.prepare_generator_batch(real_data_samples[i:i + BATCH_SIZE], start_letter=START_LETTER,
                                                          gpu=CUDA)
            gen_opt.zero_grad()
            loss = gen.batchNLLLoss(inp, target)
            loss.backward()
            gen_opt.step()

            total_loss += loss.data.item()

            if (i / BATCH_SIZE) % ceil(
                            ceil(POS_NEG_SAMPLES / float(BATCH_SIZE)) / 10.) == 0:  # roughly every 10% of an epoch
                print('.', end='')
                sys.stdout.flush()
        # Generate LSTM samples
        path='output/MSE-{}.samples'.format(epoch)
        generateSamples(gen, path)

        # each loss in a batch is loss per sample
        total_loss = total_loss / ceil(POS_NEG_SAMPLES / float(BATCH_SIZE)) / MAX_SEQ_LEN

        print(' average_train_NLL = %.4f' % (total_loss))

def train_generator_PG(gen, gen_opt, validation_data_samples, dis, num_batches,_id=0):
    """
    The generator is trained using policy gradients, using the reward from the discriminator.
    Training is done for num_batches batches.
    """

    for batch in range(num_batches):
        s = gen.sample(BATCH_SIZE*2)        # 64 works best
        inp, target = helpers.prepare_generator_batch(s, start_letter=START_LETTER, gpu=CUDA)
        rewards = dis.batchClassify(target)

        gen_opt.zero_grad()
        pg_loss = gen.batchPGLoss(inp, target, rewards)
        pg_loss.backward()
        gen_opt.step()

    # Generate LSTM samples
    path='output/ADV-{}.samples'.format(_id)
    generateSamples(gen, path)

    #validation_loss=0
    #VAL_SIZE=validation_data_samples.shape[0]
    #for i in range(0, VAL_SIZE, BATCH_SIZE):
    #    inp, target = helpers.prepare_generator_batch(validation_data_samples[i:i + BATCH_SIZE], start_letter=START_LETTER,
    #                                                  gpu=CUDA)
    #    gen_opt.zero_grad()
    #    loss = gen.batchNLLLoss(inp, target)
    #    validation_loss+= loss
    #helpers.batchwise_oracle_nll(gen, oracle, POS_NEG_SAMPLES, BATCH_SIZE, MAX_SEQ_LEN,
                   #                                start_letter=START_LETTER, gpu=CUDA)

    # print(' validation_loss = %.4f' % validation_loss)

def train_discriminator(discriminator, dis_opt, real_data_samples, generator, trainset, d_steps, epochs):
    """
    Training the discriminator on real_data_samples (positive) and generated samples from generator (negative).
    Samples are drawn d_steps times, and the discriminator is trained for epochs epochs.
    """

    # generating a small validation set before training (using oracle and generator)
    len_sampled=100
    perm=np.random.permutation(trainset.shape[0])
    pos_val=torch.tensor(trainset[perm[:len_sampled]])
    neg_val = generator.sample(len_sampled)
    if CUDA:
        pos_val=pos_val.cuda()
    val_inp, val_target = helpers.prepare_discriminator_data(pos_val, neg_val, gpu=CUDA)

    for d_step in range(d_steps):
        s = helpers.batchwise_sample(generator, POS_NEG_SAMPLES, BATCH_SIZE)
        dis_inp, dis_target = helpers.prepare_discriminator_data(real_data_samples, s, gpu=CUDA)
        for epoch in range(epochs):
            print('d-step %d epoch %d : ' % (d_step + 1, epoch + 1), end='')
            sys.stdout.flush()
            total_loss = 0
            total_acc = 0

            for i in range(0, 2 * POS_NEG_SAMPLES, BATCH_SIZE):
                inp, target = dis_inp[i:i + BATCH_SIZE], dis_target[i:i + BATCH_SIZE]
                dis_opt.zero_grad()
                out = discriminator.batchClassify(inp)
                loss_fn = nn.BCELoss()
                loss = loss_fn(out, target)
                loss.backward()
                dis_opt.step()

                total_loss += loss.data.item()
                total_acc += torch.sum((out>0.5)==(target>0.5)).data.item()

                if (i / BATCH_SIZE) % ceil(ceil(2 * POS_NEG_SAMPLES / float(
                        BATCH_SIZE)) / 10.) == 0:  # roughly every 10% of an epoch
                    print('.', end='')
                    sys.stdout.flush()

            total_loss /= ceil(2 * POS_NEG_SAMPLES / float(BATCH_SIZE))
            total_acc /= float(2 * POS_NEG_SAMPLES)

            val_pred = discriminator.batchClassify(val_inp)
            print(' average_loss = %.4f, train_acc = %.4f, val_acc = %.4f' % (
                total_loss, total_acc, torch.sum((val_pred>0.5)==(val_target>0.5)).data.item()/200.))

from torch import Tensor

def generateSamples(gen, path):
    with open(path,'w') as fout:
        for i in range(10):
            samples=gen.sample(1000)
            samples=np.array(Tensor.cpu(samples.data))

            for i in range(samples.shape[0]):
                row=[]
                for j in range(samples.shape[1]):
                    row.append(str(samples[i][j]))
                fout.write(' '.join(row)+'\n')

def loadData(filepath):
    ret=[]
    with open(filepath,'r') as fin:
        for line in fin:
            ret.append([int(token) for token in line.split()])
    return np.array(ret)

if __name__=="__main__":

    trainset=loadData('./dataset/train.vec')
    validationset=loadData('./dataset/valid.vec')
    testset=loadData('./dataset/test.vec')

    trainset_tensor=torch.tensor(trainset)
    validationset_tensor=torch.tensor(validationset)
    oracle=None

    gen = generator.Generator(GEN_EMBEDDING_DIM, GEN_HIDDEN_DIM, VOCAB_SIZE, MAX_SEQ_LEN, gpu=CUDA)
    dis = discriminator.Discriminator(DIS_EMBEDDING_DIM, DIS_HIDDEN_DIM, VOCAB_SIZE, MAX_SEQ_LEN, gpu=CUDA)

    if CUDA:
        #oracle = oracle.cuda()
        gen = gen.cuda()
        dis = dis.cuda()
        oracle_samples = trainset_tensor.cuda()
        trainset_tensor=trainset_tensor.cuda()
        validationset_tensor=validationset_tensor.cuda()
        ### debug
    #MLE_TRAIN_EPOCHS=1
    #########

    # GENERATOR MLE TRAINING
    print('Starting Generator MLE Training...')
    gen_optimizer = optim.Adam(gen.parameters(), lr=1e-3)
    train_generator_MLE(gen, gen_optimizer, trainset_tensor, MLE_TRAIN_EPOCHS)

    torch.save(gen.state_dict(), pretrained_gen_path)
    gen.load_state_dict(torch.load(pretrained_gen_path))

    # PRETRAIN DISCRIMINATOR
    print('\nStarting Discriminator Training...')
    d_steps=10

    ######debug######
    # d_steps=1   ###
    #################
    dis_optimizer = optim.Adagrad(dis.parameters())
    train_discriminator(dis, dis_optimizer, trainset_tensor, gen, trainset, d_steps, 3)

    torch.save(dis.state_dict(), pretrained_dis_path)
    dis.load_state_dict(torch.load(pretrained_dis_path))

    # ADVERSARIAL TRAINING
    print('\nStarting Adversarial Training...')
    d_steps=5

    ### debug ##
    # d_step=1####
    ############

    for epoch in range(ADV_TRAIN_EPOCHS):
        print('\n--------\nEPOCH %d\n--------' % (epoch+1))
        # TRAIN GENERATOR
        print('\nAdversarial Training Generator : ', end='')
        sys.stdout.flush()
        train_generator_PG(gen, gen_optimizer, validationset_tensor, dis, 1, _id=epoch)

        # TRAIN DISCRIMINATOR
        print('\nAdversarial Training Discriminator : ')
        train_discriminator(dis, dis_optimizer, trainset_tensor, gen, trainset, d_steps, 3)

        # Generate seqGAN samples
        path='output/seqGAN-epoch{}.samples'.format(epoch)
        generateSamples(gen, path)


from setuptools import setup, find_packages

REQUIRES = ['logbook', 'elasticsearch>=2.0.0', 'python-dateutil', 'pybling', 'python-docx', 'openpyxl', 'xlrd',
            'pyzmq', 'ujson', 'pyprinter']

setup(
    name='blingindexers',
    version='1.0',
    install_requires=REQUIRES,
    packages=find_packages()
)


# -*- coding: utf-8 -*-
"""
Created on Mon Jul  8 17:08:57 2019

@author: Lvov_AS
"""
'''
Реализовать следующую логику: получать при помощи requests данные сервиса
https://jsonplaceholder.typicode.com/
(сущность можно выбрать любую, например:
https://jsonplaceholder.typicode.com/comments),
выводить в консоль все пары заголовки, сохранять полученный json в файл на диск

'''
import requests

__author__ = 'ASLvov'

'''def get_from_site(sitename):
    r = requests.get(sitename)
    print(r.status_code)
    print(r.headers)
    print(r.content)'''

def get_from_site(sitename):
    r = requests.get(sitename)
    print(r.status_code, r.headers)
    print(r.content)

    # s = 'http://petstore.swagger.io/v2/pet/89?tags=string&filter=sad'

if __name__ == '__main__':
    site = 'https://jsonplaceholder.typicode.com/'
    get_from_site(site)
    #find_pet_by_tag('string')

tceh/requests, urllib, read, write, json/read, write, json/requests.py


from .controls.control_base     import ControlBase
from .controls.control_file     import ControlFile
from .controls.control_slider   import ControlSlider
from .controls.control_text     import ControlText
from .controls.control_checkbox import ControlCheckBox
from .controls.control_label    import ControlLabel
from .controls.control_button   import ControlButton
try:
    from .controls.control_player import ControlPlayer
except:
    print("ControlPlayer is not available")
from .web.applications import ApplicationsLoader
from .web.middleware   import PyFormsMiddleware
import uuid, os, shutil, base64, inspect
import base64, dill, simplejson, filelock
from confapp import conf
from django.template.loader import render_to_string
from .utils import make_lambda_func

class no_columns(object):
    def __init__(self, *args):  self.items = args
    def __getitem__(self,index): return self.items[index]
    def __setitem__(self,index,value): self.items[index] = value
    def __len__(self):  return len(self.items)
    def __iter__(self):
        self._index = -1; return self
    def __next__(self):
        self._index += 1
        if self._index>=len(self.items): raise StopIteration
        return self.items[self._index]

class segment(object):
    def __init__(self, *args, **kwargs):
        self.css = kwargs.get('css', '')
        self.items = args
    def __getitem__(self,index): return self.items[index]
    def __setitem__(self,index,value): self.items[index] = value
    def __len__(self):  return len(self.items)
    def __iter__(self):
        self._index = -1; return self
    def __next__(self):
        self._index += 1
        if self._index>=len(self.items): raise StopIteration
        return self.items[self._index]

class BaseWidget(object):
    """
    The class implements a application window
    """
    TITLE           = None
    LAYOUT_POSITION = None
    CSS             = ''

    FORM_NO_ROW_ALIGNMENT = 0

    REFRESH_TIMEOUT = None #time in milliseconds to refresh the application

    def __init__(self, *args, **kwargs):
        self._formset       = None
        self._splitters     = []
        self._title         = kwargs.get('title', args[0] if len(args)>0 else self.TITLE)
        self._formLoaded    = False
        self._controls      = []
        self._html          = ''
        self._js            = ''
        self._close_widget  = False

        self.init_form_result = None

        self._uid =  self.UID if hasattr(self, 'UID') else str(uuid.uuid4())

        self._messages        = []
        self._js_code2execute = [];

        self.parent = kwargs.get('parent_win', None)
        self.is_new_app = True

        PyFormsMiddleware.add(self)

    ############################################################################
    ############ FUNCTIONS #####################################################
    ############################################################################

    def init_form(self, parent=None):
        """
        Generate the module Form
        """

        self._html = ''
        self._js = ''
        self._controls = [c.init_form() for c in self.controls.values()]
        if self._formset != None:
            self._html += self.generate_panel(self._formset, add_field_class=False)

        parent_code = 'undefined'
        if parent: parent_code = "'{0}'".format(parent.uid)

        extra_data = {'refresh_timeout': self.REFRESH_TIMEOUT, 'messages':self._messages}

        modulename = inspect.getmodule(self).__name__ + '.' + self.__class__.__name__

        self._js = '[{0}]'.format(",".join(self._controls))
        self._html += """

        """.format(modulename, self._js, self.uid, parent_code, simplejson.dumps(extra_data))
        self._formLoaded = True

        self._messages = []
        self.mark_to_update_client()

        res =  {
            'code': self._html,
            'css': self.CSS,
            'title': self._title,
            'app_id':self.uid,
            'refresh_timeout':
            self.REFRESH_TIMEOUT
        }

        return res

    def generate_segment(self, row):
        """
        Generate the html to organize the formset in segments
        """
        html  = "
".format(row.css)
        html += self.generate_panel(list(row), add_field_class=False)
        html += "
"
        return html

    def generate_segments(self, formsetdict):
        """
        Generate the html to organize the formset in segments
        """
        html = ''
        for key, item in sorted(formsetdict.items()):
            if item==True: continue

            html += "
{0}
".format(key[key.find(':')+1:])
            html += "
"
            html += self.generate_panel(item, add_field_class=False)
            html += "
"
        return html

    def generate_tabs(self, formsetdict):
        """
        Generate QTabWidget for the module form
        @param formset: Tab form configuration
        @type formset: dict
        """
        tabs_head = ""
        tabs_body = ""
        tab_id = uuid.uuid4()

        for index, (key, item) in enumerate( sorted(formsetdict.items()) ):
            active = 'active' if index==0 else ''
            tabs_body += "
{2}
".format(tab_id, index, self.generate_panel(item), active, tab_id, index)
            tabs_head += "
{0}
".format(key[key.find(':')+1:], active, tab_id, index)

        return """
{1}
{2}""".format(tab_id, tabs_head, tabs_body)

    def generate_control(self, row):
        control = self.controls.get(row, None)
        if control==None:
            if   row==' ':                   return "

"
            elif row.startswith('h1:'):      return "
{0}
".format(row[3:])
            elif row.startswith('h1-right:'):return "
{0}
".format(row[9:])
            elif row.startswith('h2:'):      return "
{0}
".format(row[3:])
            elif row.startswith('h2-right:'):return "
{0}
".format(row[9:])
            elif row.startswith('h3:'):      return "
{0}
".format(row[3:])
            elif row.startswith('h3-right:'):return "
{0}
".format(row[9:])
            elif row.startswith('h4:'):      return "
{0}
".format(row[3:])
            elif row.startswith('h4-right:'):return "
{0}
".format(row[9:])
            elif row.startswith('h5:'):      return "
{0}
".format(row[3:])
            elif row.startswith('h5-right:'):return "
{0}
".format(row[9:])
            elif row.startswith('info:'):    return "
{0}
".format(row[5:])
            elif row.startswith('warning:'): return "
{0}
".format(row[8:])
            elif row.startswith('alert:'):   return "
{0}
".format(row[6:])
            elif row.startswith('msg:'):     return "
{0}
".format(row[4:])
            elif row == '-':                 return "

"
            else:                            return row
        else:
            return str(control)

    def generate_panel(self, formset, add_field_class=True):
        """
        Generate a panel for the module form with all the controls formset format example:

        .. code-block:: python

            [
                ('_video', '_arenas', '_run'),
                {
                    "Player": [
                        '_threshold',
                        "_player",
                        "=",
                        "_results",
                        "_query"
                    ],
                    "Background image":[
                        (' ', '_selectBackground', '_paintBackground'),
                        '_image'
                    ]
                },
                "_progress"
            ]

        **tuple**: displays the controls horizontally.

        **list**: displays the controls vertically.

        **dict**: displays the controls in Tabs.

        **'||'**: splits the controls with a horizontal line.

        **'='**: splits the controls with a vertical line.

        """
        if formset is None:
            return ''
        elif '=' in formset:
            index = list( formset ).index('=')
            return "

{top}

{bottom}

".format(
                splitter_id = uuid.uuid4(),
                top         = self.generate_panel(formset[0:index]),
                bottom      = self.generate_panel(formset[index+1:])
            )
        elif '||' in formset:
            index = list( formset ).index('||')
            return "

{left}

{right}

".format(
                splitter_id = uuid.uuid4(),
                left        = self.generate_panel(formset[0:index]),
                right       = self.generate_panel(formset[index+1:])
            )

        if isinstance(formset, (tuple,no_columns) ):
            layout  = "
".format(self.__get_fields_class(formset))
            for row in formset:
                layout += self.generate_panel( row, add_field_class )
            return layout+"
"

        elif isinstance(formset, segment):
            return self.generate_segment(formset)

        elif isinstance(formset, list):
            layout  = ""
            for row in formset:
                if row == ' ': layout += "

"
                layout += self.generate_panel( row, add_field_class )
            return layout

        elif isinstance(formset, dict ):
            return self.generate_tabs(formset)

        else:
            return self.generate_control(formset)

    def save_form(self, data={}, path=None):
        """
        Called to save the form

        TODO
        """
        pass

    def load_form(self, data, path=None):
        """
        Called to load a form

        TODO
        """
        pass

    def save_window(self):
        pass

    def load_window(self):
        pass

    def load_form_filename(self, filename):
        """
        Load the forms from a file

        TODO
        """
        pass

    def close(self):
        """
        Close the application
        """
        self.mark_to_update_client()
        self._close_widget = True

    def message(self, msg, title=None, msg_type=None):
        """
        Write a simple message
        """
        msg = { 'type': msg_type if msg_type else '', 'messages':msg if isinstance(msg, list) else [msg], 'title':title }
        self._messages.append(msg)
        self.mark_to_update_client()
    def success(self,   msg, title=None):
        """
        Write a success message
        """
        self.message(msg, title, msg_type='success')
    def info(self,      msg, title=None):
        """
        Write a info message
        """
        self.message(msg, title, msg_type='info')
    def warning(self,   msg, title=None):
        """
        Write a warning message
        """
        self.message(msg, title, msg_type='warning');
    def alert(self,     msg, title=None):
        """
        Write a alert message
        """
        self.message(msg, title, msg_type='error')

    def message_popup(self, msg, title='', buttons=None, handler=None, msg_type='success'):
        """
        Show a popup message window
        """
        self._active_popup_msg = PopupWindow(title, msg, buttons, handler, msg_type='success', parent_win=self)
        return self._active_popup_msg
    def success_popup(self, msg, title='', buttons=None, handler=None):
        """
        Show a popup success message window
        """
        return self.message_popup(msg, title, buttons, handler, msg_type='success')
    def info_popup(self, msg, title='', buttons=None, handler=None):
        """
        Show a popup info message window
        """
        return self.message_popup(msg, title, buttons, handler, msg_type='info')
    def warning_popup(self, msg, title='', buttons=None, handler=None):
        """
        Show a popup warning message window
        """
        return self.message_popup(msg, title, buttons, handler, msg_type='warning')
    def alert_popup(self, msg, title='', buttons=None, handler=None):
        """
        Show a popup alert message window
        """
        return self.message_popup(msg, title, buttons, handler, msg_type='alert')

    ##########################################################################
    ############ WEB functions ###############################################
    ##########################################################################

    def __get_fields_class(self, row):
        """
        Get the css class to be used on the controls organization
        """
        if len(row)>=1 and row[0]==self.FORM_NO_ROW_ALIGNMENT: return 'no-alignment'
        if isinstance(row, no_columns): return 'no-alignment'

        if   len(row)==2: return 'two'
        elif len(row)==3: return 'three'
        elif len(row)==4: return 'four'
        elif len(row)==5: return 'five'
        elif len(row)==6: return 'six'
        elif len(row)==7: return 'seven'
        elif len(row)==8: return 'eight'
        elif len(row)==9: return 'nine'
        elif len(row)==10: return 'ten'
        elif len(row)==11: return 'eleven'
        elif len(row)==12: return 'twelve'
        elif len(row)==13: return 'thirteen'
        elif len(row)==14: return 'fourteen'
        elif len(row)==15: return 'fiveteen'
        elif len(row)==16: return 'sixteen'
        elif len(row)==17: return 'seventeen'
        elif len(row)==18: return 'eighteen'
        elif len(row)==19: return 'nineteen'
        elif len(row)==20: return 'twenty'
        elif len(row)==21: return 'twentyone'
        elif len(row)==22: return 'twentytwo'
        else: return ''

    def commit(self):
        """
        Save all the application updates to a file, so it can be used in the next session.
        """
        for key, item in self.controls.items(): item.commit()

        user = PyFormsMiddleware.user()
        # save the modifications
        userpath = os.path.join(
            conf.PYFORMS_WEB_APPS_CACHE_DIR,
            '{0}-{1}'.format(user.pk, user.username)
        )
        if not os.path.exists(userpath): os.makedirs(userpath)

        app_path = os.path.join(userpath, "{0}.app".format(self.uid) )

        lock = filelock.FileLock("lockfile.txt")
        with lock.acquire(timeout=4):
            with open(app_path, 'wb') as f:
                dill.dump(self, f)

    def execute_js(self, code):
        """
        This function executs a javascript on the client side
        """
        self._js_code2execute.append(code)

    def mark_to_update_client(self):
        """
        Used to flag pyforms that the application was updated and the updates should be sent to the client side
        """
        request = PyFormsMiddleware.get_request()
        if  request is not None and \
            hasattr(request,'updated_apps'):
            request.updated_apps.add_top(self)

    def load_serialized_form(self, params):
        """
        Load the json parameters sent by the client side
        """
        widgets = []

        if hasattr(self, 'parent') and isinstance(self.parent, (str,str)):
            self.parent = PyFormsMiddleware.get_instance(self.parent)

        for key, value in params.items():
            control = self.controls.get(key, None)
            if control!=None:
                if control.__class__.__name__=='ControlEmptyWidget':
                    widgets.append( (control, params[key]) )
                else:
                    control.deserialize(params[key])

        for control, data in widgets: control.deserialize(data)

        if 'event' in params.keys():
            control = params['event']['control']
            if control in self.controls.keys():
                item = self.controls[control]
                func = getattr(item, params['event']['event'])
                func()
            elif control=='self':
                func = getattr(self, params['event']['event'])
                func()

    def serialize_form(self):
        """
        Serialize the Form to a control
        """
        res = {
            'uid':              self.uid,
            'layout_position':  self.LAYOUT_POSITION if hasattr(self, 'LAYOUT_POSITION') else 5,
            'title':            self.title,
            'close_widget':     self._close_widget,
            'js-code':          list(self._js_code2execute)
        }

        self._js_code2execute = []

        if len(self._messages)>0:
            res.update({'messages': self._messages})
            self._messages = []

        for key, item in self.controls.items():

            if item.was_updated:
                res[item._name] = item.serialize()
                try:
                    if isinstance(item, ControlPlayer ) and item._value!=None and item._value!='':
                        item._value.release() #release any open video
                except:
                    pass

        return res

    @classmethod
    def has_permissions(cls, user):
        """
        This class method, verifies if a user has permissions to execute the application
        """
        if hasattr(cls, 'AUTHORIZED_GROUPS'):
            if user.is_superuser and 'superuser' in cls.AUTHORIZED_GROUPS:
                return True
            if user.groups.filter(name__in=cls.AUTHORIZED_GROUPS).exists():
                return True
        else:
            return True

        return False

    def has_session_permissions(self, user):
        """
        It verifies if a user has permissions to execute the application during the runtime.
        """
        return True

    ##########################################################################
    ############ EVENTS ######################################################
    ##########################################################################

    def before_close_event(self):
        """
        Function called before the Form is closed.

        TODO
        """
        pass

    ##########################################################################
    ############ WEB events ##################################################
    ##########################################################################

    def refresh_event(self):
        """
        Event called every X time defined by refresh_timeout variable.
        """
        pass

    ############################################################################
    ############ Properties ####################################################
    ############################################################################

    @property
    def controls(self):
        """
        Return all the form controls from the the module
        """
        result = {}
        for name, var in vars(self).items():
            if isinstance(var, ControlBase):
                var.parent  = self
                var._name   = name
                result[name]= var

        return result

    @property
    def form(self):
        """
        Return the basewidget html. The html is based on the 'basewidget-template.html' template
        """
        return render_to_string(
            os.path.join('pyforms', 'basewidget-template.html'),
            {'application_html': self._html, 'application_id': self.uid}
        )

    @property
    def title(self):
        """
        Return and set the title of the application
        """
        return self._title

    @title.setter
    def title(self, value): self._title = value

    @property
    def mainmenu(self):
        """
        Return and set the mainmenu

        TODO
        """
        return None

    @mainmenu.setter
    def mainmenu(self, value):
        pass

    @property
    def formset(self):
        """
        Return and set the controls organization in the form
        """
        return self._formset

    @formset.setter
    def formset(self, value): self._formset = value

    @property
    def uid(self):
        """
        Return and set the application unique identifier
        """
        return self._uid

    @uid.setter
    def uid(self, value): self._uid = value

    @property
    def visible(self):
        """
        Return a boolean indicating if the form is visible or not
        """
        return True

    ############################################################################
    ############ WEB Properties ################################################
    ############################################################################

    @property
    def js(self):
        """
        Return the form javascript
        """
        return self._js

class PopupWindow(BaseWidget):
    LAYOUT_POSITION = 4

    def __init__(self, title, msg, buttons, handler, msg_type, parent_win=None):
        BaseWidget.__init__(self, title, parent_win=parent_win)

        self._label = ControlLabel(default=msg)
        #self._label.css = msg_type

        if buttons:
            buttons_formset = []
            for i, b in enumerate(buttons):
                name = 'button_{0}'.format(i)
                setattr(self, name, ControlButton(b))
                getattr(self, name ).value = make_lambda_func(handler, popup=self, button=b)
                buttons_formset.append(name)

        self.formset = ['_label'] + [no_columns(buttons_formset)]


# coding=utf-8
"""
wecubek8s.server.scheduler
~~~~~~~~~~~~~~~~~~~~~~~~~

本模块提供定时清理能力

"""

from __future__ import absolute_import

import os
import shutil
import time
import datetime
import logging
from pytz import timezone
from talos.core import config
from talos.core import logging as mylogger

from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.executors.pool import ThreadPoolExecutor

CONF = config.CONF
LOG = logging.getLogger(__name__)

jobstores = {'default': MemoryJobStore()}
executors = {'default': ThreadPoolExecutor(5)}
job_defaults = {'coalesce': False, 'max_instances': 1}

# def cleanup_cached_dir():
#     try:
#         max_delta = 24 * 60 * 60
#         base_dir = CONF.pakcage_cache_dir
#         if os.path.exists(base_dir):
#             for name in list(os.listdir(base_dir)):
#                 fullpath = os.path.join(base_dir, name)
#                 path_stat = os.stat(fullpath)
#                 if time.time() - path_stat.st_atime > max_delta:
#                     LOG.info('remove dir/file: %s, last access: %s', fullpath, path_stat.st_atime)
#                     if os.path.isdir(fullpath):
#                         shutil.rmtree(fullpath, ignore_errors=True)
#                     elif os.path.isfile(fullpath):
#                         os.remove(fullpath)
#     except Exception as e:
#         LOG.exception(e)

def rotate_log():
    try:
        logs = [CONF.log.gunicorn_access, CONF.log.gunicorn_error, CONF.log.path]
        extend_logs = getattr(CONF.log, 'loggers', [])
        for l in extend_logs:
            if l.get('path'):
                logs.append(l['path'])
        max_file_keep = 30
        for log_file in logs:
            results = []
            base_dir = os.path.dirname(log_file)
            if os.path.exists(base_dir):
                for name in list(os.listdir(base_dir)):
                    fullpath = os.path.join(base_dir, name)
                    if os.path.isfile(fullpath):
                        if fullpath.startswith(log_file + '.'):
                            timestamp = 0
                            try:
                                timestamp = int(fullpath.rsplit('.', 1)[1])
                                # ignore which not endswith datetime
                                results.append((timestamp, fullpath))
                            except Exception as e:
                                pass

            results.sort(key=lambda item: item[0])
            while len(results) >= max_file_keep:
                timestamp, fullpath = results.pop(0)
                try:
                    LOG.info('remove file: %s', fullpath)
                    os.remove(fullpath)
                except Exception as e:
                    LOG.info('remove file: %s error: %s', fullpath, str(e))
        for log_file in logs:
            new_log_file = log_file + '.' + datetime.datetime.now().strftime('%Y%m%d%H%M%S')
            if os.path.exists(log_file):
                try:
                    LOG.info('rename file: %s to %s', log_file, new_log_file)
                    os.rename(log_file, new_log_file)
                except Exception as e:
                    LOG.info('rename file: %s to %s error: %s', log_file, new_log_file, str(e))
    except Exception as e:
        LOG.exception(e)

def main():
    config.setup(os.environ.get('WECUBEK8S_CONF', '/etc/wecubek8s/wecubek8s.conf'),
                 dir_path=os.environ.get('WECUBEK8S_CONF_DIR', '/etc/wecubek8s/wecubek8s.conf.d'))
    mylogger.setup()
    tz_info = timezone(CONF.timezone)
    try:
        if CONF.platform_timezone:
            prefix = 'ENV@'
            value = CONF.platform_timezone
            if value.startswith(prefix):
                env_name = value[len(prefix):]
                value = os.getenv(env_name, default='')
            tz_info = timezone(value)
    except Exception as e:
        LOG.exception(e)
    scheduler = BlockingScheduler(jobstores=jobstores,
                                  executors=executors,
                                  job_defaults=job_defaults,
                                  timezone=tz_info)
    # scheduler.add_job(cleanup_cached_dir, 'cron', hour='*')
    scheduler.add_job(rotate_log, 'cron', hour=3, minute=5)
    try:
        scheduler.start()
    except (KeyboardInterrupt, SystemExit):
        pass

if __name__ == '__main__':
    main()


from bottle import route, run, template
import lc

@route("/")
def inicio():
    """ Atiende la petición GET / """
    ruta = "."
    carpeta = lc.Carpeta(ruta)
    carpeta.obtener_elementos()

    html = """
    <!DOCTYPE html>

Lista de archivos

            {tabla}

    """
    tabla = []  # Lista de filas
    for e in carpeta.elementos:
        # Determina si un elemento e es un objeto Archivo o
        # Carpeta
        if isinstance(e, lc.Archivo):
            # Si es archivo, lo agregamos tal cual
            fila = f"
{e.tamanio}	{e.fecha}	{e.nombre}
"
            tabla.append(fila)
        else:
            # Es una carpeta y procesamos sus elementos
            e.obtener_elementos()
            for f in e.elementos:
                fila = f"
{f.tamanio}	{f.fecha}	{f.nombre}
"
                tabla.append(fila)

    # Unimos todas las fila separadas por un salto de línea
    tabla = "\n".join(tabla)
    # Integrar la tabla en el html replanzando la etiqueta.
    html = html.replace("{tabla}", tabla)

    # return template("
Hola {{ nom }}!
", nom=nombre)
    return html

# GET /hola/nombre
@route("/hola/<nombre>")
def inicio(nombre):
    """ Atiende la petición GET /hola/nombre """
    return template("
Hola {{ nom }}!
", nom=nombre)

def lista_elementos(carpeta):
    """ Genera una lista con los elementos de carpeta """
    elementos = []
    for e in carpeta.elementos:
        elementos.append(e.dict)
        if isinstance(e, lc.Carpeta):
            e.obtener_elementos()
            elementos += lista_elementos(e)

    return elementos

# Funciones para atender las peticones de la API
@route("/api/json")
def apijson():
    """ Atendiendo la petición GET /api/json """
    ruta = "."
    carpeta = lc.Carpeta(ruta)
    carpeta.obtener_elementos()

    arch_dict = {
        "Archivos": []
    }
    arch_dict["Archivos"] = lista_elementos(carpeta)

    return arch_dict

# Equivalente a la ejecución del main()
run(host="localhost", port=8000)


#!/usr/bin/env python
# This code will check the folder and some subfolder assigned by the user to see if the mp-*** listed in a text file are available or not
# If available, the selected property at selected concentration and temperature will be reported
# If not, N.A would be typed in front of the list
# By: Alireza Faghaninia

#!/usr/bin/env python

import argparse
import os

def find_properties(filename, n, T): #find the transport properties at a given n and T from aMoBT output
	count = 0
	mobility = 0
	conductivity = 0
	thermopower = 0
	at_correct_n = False
	if os.path.exists(filename):
		with open(filename) as aMoBT_output:
			for row in aMoBT_output:
				line = row.split()
				if at_correct_n:
					if float(line[0]) == T:
						mobility = float(line[1])
						conductivity = float(line[2])
						thermopower = float(line[3])
						at_correct_n = False
				if len(line) > 3:
					if 'Carrier' in line[0] and float(line[3]) == n:
						at_correct_n = True
	return mobility, conductivity, thermopower

def find_effective_mass(filename):
	m_e = 0.0000
	m_h = 0.0000
	if os.path.exists(filename):
		with open(filename) as aMoBT_output:
			for row in aMoBT_output:
				line = row.split()
				if len(line) >= 2:
					if line[0] == '***Proceed':
						if 'm*_e/m' in line[7]:
							m_e = float(line[9])
						if 'm*_h/m' in line[7]:
							m_h = float(line[9])
	return m_e, m_h

if __name__ == "__main__":
	parser = argparse.ArgumentParser()
	parser.add_argument("-f", "--filename", help="The filename that contains the list, default=checklist",	required=False, default="checklist")
	parser.add_argument("-n", "--n", help="Carrier concentration", required=False, default=1e21)
	parser.add_argument("-T", "--T", help="Temperature(K)", required=False, default=300)
	parser.add_argument("-d", "--free_e", help="To use free-electron density of states or not ('''true''' or '''false''')", required=False, default="true")
	args = parser.parse_args()
	folders = ['./', 'new_TEs/', 'new_TCOs/']
	n_type_folder = '1_n-type_aMoBT_free-e=' + args.free_e
	p_type_folder = '2_p-type_aMoBT_free-e=' + args.free_e
	clist = []
	with open(args.filename,'r') as raw_list:
		for line in raw_list:
			if len(line)>1:
				supposed_id = line.split()[0]
				if ('mp-' in supposed_id) or ('mvc-' in supposed_id):
					clist.append(line.split()[0])
	stat = open('status.txt', 'w')
	rem = open('remaining.txt', 'w')
	stat.write('%30s%15s%12s %10s%10s%10s%10s\n' % ('location of mp-id (if any)', 'mu-cm2/V.s', 'sigma-S/cm', 'S-uV/K', 'p_mu', 'p_sigma', 'p_S' ))
	for c in clist:
		proceed = False
		for subf in folders:
			if os.path.exists(subf + c):
				proceed = True
				c = subf + c
		if proceed:
			stat.write('%30s\n' % c)
		else:
			stat.write('%30s%15s\n' % (c, 'N.A.'))
			mpstart = c.find('mp-')
			rem.write(c[mpstart:] + '\n')

	stat.close()
	rem.close()
	mobility_n, conductivity_n, thermopower_n = find_properties(n_type_folder + '/aMoBT_output.txt', args.n, args.T)
	mobility_p, conductivity_p, thermopower_p = find_properties(p_type_folder + '/aMoBT_output.txt', args.n, args.T)


import cffi
import faulthandler
import os
import re
import sys

SELF_DIR = os.path.dirname(os.path.abspath(__file__))
TOPLEVEL = os.path.abspath(SELF_DIR + "/../") + "/"

# The reference implementation is at the top level.
sys.path.append(TOPLEVEL)

HEADERS = [
    "umash.h",
    "t/umash_test_only.h",
]

FFI = cffi.FFI()

def read_stripped_header(path):
    """Returns the contents of a header file without preprocessor directives."""
    ret = ""
    in_directive = False
    with open(path) as f:
        for line in f:
            if in_directive or re.match(r"^\s*#", line):
                in_directive = line.endswith("\\\n")
            else:
                in_directive = False
                # HACK: ignore the C++ guard
                if line != 'extern "C" {\n' and line != "}\n":
                    ret += line
    return ret

for header in HEADERS:
    FFI.cdef(read_stripped_header(TOPLEVEL + header))

C = FFI.dlopen(os.getenv("UMASH_TEST_LIB", TOPLEVEL + "umash_test_only.so"))

# Pass in a copy of stderr in case anyone plays redirection tricks.
faulthandler.enable(os.dup(2))


import subprocess

def hello_pubsub(event, context):
    """Background Cloud Function to be triggered by Pub/Sub.
    Args:
         event (dict):  The dictionary with data specific to this type of
         event. The `data` field contains the PubsubMessage message. The
         `attributes` field will contain custom attributes if there are any.
         context (google.cloud.functions.Context): The Cloud Functions event
         metadata. The `event_id` field contains the Pub/Sub message ID. The
         `timestamp` field contains the publish time.
    """
    import base64

    print("""This Function was triggered by messageId {} published at {}
    """.format(context.event_id, context.timestamp))

    if 'data' in event:
        name = base64.b64decode(event['data']).decode('utf-8')
    else:
        name = 'World'
    print('Hello {}!'.format(name))
    which_gsutil()

def which_gsutil():

    p = subprocess.run(
      [
        "which",
        "gsutil"
      ],
      capture_output=True
    )
    print(p.stdout)

if __name__ == "__main__":
    which_gsutil()


from benchmark.test_types.framework_test_type import FrameworkTestType

import json

class DBTestType(FrameworkTestType):
  def __init__(self):
    args = ['db_url']
    FrameworkTestType.__init__(self, name='db',
      accept_header=self.accept_json,
      requires_db=True, args=args)

  def get_url(self):
    return self.db_url

  def verify(self, base_url):
    '''Ensures body is valid JSON with a key 'id' and a key
    'randomNumber', both of which must map to integers
    '''

    url = base_url + self.db_url
    body = self._curl(url)

    # Empty response
    if body is None:
      return [('fail','No response', url)]
    elif len(body) == 0:
      return [('fail','Empty Response', url)]

    # Valid JSON?
    try:
      response = json.loads(body)
    except ValueError as ve:
      return [('fail',"Invalid JSON - %s" % ve, url)]

    problems = []

    # We are allowing the single-object array
    # e.g. [{'id':5, 'randomNumber':10}] for now,
    # but will likely make this fail at some point
    if type(response) == list:
      response = response[0]
      problems.append( ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url) )

      # Make sure there was a JSON object inside the array
      if type(response) != dict:
        problems.append( ('fail', 'Response is not a JSON object or an array of JSON objects', url) )
        return problems

    problems += self._verifyObject(response, url)

    if len(problems) == 0:
      return [('pass','',url)]
    else:
      return problems

  def _verifyObject(self, db_object, url, max_infraction='fail'):
    '''Ensure the passed item is a JSON object with
    keys 'id' and 'randomNumber' mapping to ints.
    Separate method allows the QueryTestType to
    reuse these checks'''

    problems = []

    if type(db_object) != dict:
      got = str(db_object)[:20]
      if len(str(db_object)) > 20:
        got = str(db_object)[:17] + '...'
      return [(max_infraction, "Expected a JSON object, got '%s' instead" % got, url)]

    # Make keys case insensitive
    db_object = {k.lower(): v for k,v in db_object.iteritems()}

    if "id" not in db_object:
      problems.append( (max_infraction, "Response has no 'id' key", url) )
    if "randomnumber" not in db_object:
      problems.append( (max_infraction, "Response has no 'randomNumber' key", url) )

    # Ensure we can continue on to use these keys
    if "id" not in db_object or "randomnumber" not in db_object:
      return problems

    try:
      float(db_object["id"])
    except ValueError as ve:
      problems.append( (max_infraction, "Response key 'id' does not map to a number - %s" % ve, url) )

    try:
      float(db_object["randomnumber"])
    except ValueError as ve:
      problems.append( (max_infraction, "Response key 'randomNumber' does not map to a number - %s" % ve, url) )

    if type(db_object["id"]) != int:
      problems.append( ('warn', '''Response key 'id' contains extra quotations or decimal points.
        This may negatively affect performance during benchmarking''', url) )

    # Tests based on the value of the numbers
    try:
      response_id = float(db_object["id"])
      response_rn = float(db_object["randomnumber"])

      if response_id > 10000 or response_id < 1:
        problems.append( ('warn', "Response key 'id' should be between 1 and 10,000", url))
    except ValueError:
      pass

    return problems


# @author: xws    time: 2019/8/29 17:41

import pika

connection=pika.BlockingConnection(pika.ConnectionParameters("localhost"))
channel = connection.channel()

# exchange = 'm1', 秘书的名称
# exchange_type = 'topic'，秘书的工作方式：将消息发送给所有的队列,模糊匹配
channel.exchange_declare(exchange = 'm3',exchange_type = 'topic')

# 随机生成并声明一个队列
result = channel.queue_declare(queue = '',exclusive = True)
# 获取队列的名称
queue_name = result.method.queue

# 让exchange和queue进行绑定，使得消息发送到此队列中
# routing_key ： 指明绑定的匹配密钥
# routing_key中  # * ： 模糊匹配
# * ：匹配一个单词  例如：xws.s
# # ：匹配多个单词  例如：xws.s.d.d.d
channel.queue_bind(exchange = 'm3',queue = queue_name,routing_key="xws.#")

# 回调函数
def callback(ch, method, properties, body):
    # 获取 routing_key的值
    print(method.routing_key)
    print("消费者接受到了任务: %r" % body)

# 消费消息
channel.basic_consume(queue=queue_name,auto_ack = True,on_message_callback=callback)

# 开始消费
channel.start_consuming()

消费者_订阅者_2.py


# ----------------------------------------------------------------------------
# Copyright (c) 2016-2018, QIIME 2 development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------

import biom
import pandas as pd
from sklearn.metrics import pairwise_distances
from scipy.spatial.distance import squareform
from scipy.cluster.hierarchy import linkage
from skbio import TreeNode
from q2_feature_table import merge

from ._collate_fingerprint import collate_fingerprint
from ._match import match_label
from ._semantics import CSIDirFmt

def build_tree(relabeled_fingerprints: pd.DataFrame) -> TreeNode:
    '''
    This function makes a tree of relatedness between mass-spectrometry
    features using molecular substructure fingerprints.
    '''
    distmat = pairwise_distances(X=relabeled_fingerprints,
                                 Y=None, metric='jaccard')
    distsq = squareform(distmat, checks=False)
    linkage_matrix = linkage(distsq, method='average')
    tree = TreeNode.from_linkage_matrix(linkage_matrix,
                                        relabeled_fingerprints.index.tolist())
    return tree

def merge_feature_data(fdata: pd.DataFrame):
    '''
    This function merges feature data from multiple feature tables. The
    resulting table is indexed by MD5 hash mapped to unique feature
    identifiers in the original feature tables.
    '''
    for idx, data in enumerate(fdata):
        data['table_number'] = str(idx+1)
    merged_fdata = pd.concat(fdata)
    dupl_bool = merged_fdata.index.duplicated(keep='first')
    duplicates = merged_fdata[dupl_bool].index.unique()
    if len(duplicates) == 0:
        return merged_fdata
    else:
        for idx in duplicates:
            merged_fdata.loc[idx, '#featureID'] = (',').join(
                list(merged_fdata.loc[idx, '#featureID']))
            merged_fdata.loc[idx, 'table_number'] = (',').join(
                list(merged_fdata.loc[idx, 'table_number']))
        merged_fdata = merged_fdata[~dupl_bool]
        return merged_fdata

def make_hierarchy(csi_results: CSIDirFmt,
                   feature_tables: biom.Table,
                   qc_properties: bool = True) -> (TreeNode, biom.Table,
                                                   pd.DataFrame):
    '''
    This function generates a hierarchy of mass-spec features based on
    predicted chemical fingerprints. It filters the feature table to
    retain only the features with fingerprints and relables each feature with
    a hash (MD5) of its binary fingerprint vector.

    Parameters
    ----------
    csi_results : CSIDirFmt
        one or more CSI:FingerID output folder
    feature_table : biom.Table
        one or more feature tables with mass-spec feature intensity per sample
    qc_properties : bool, default True
        flag to filter molecular properties to keep only PUBCHEM fingerprints

    Raises
    ------
    ValueError
        If ``feature_table`` in empty
        If collated fingerprint table is empty
    UserWarning
        If features in collated fingerprint table are not a subset of
        features in ``feature_table``

    Returns
    -------
    skbio.TreeNode
        a tree of relatedness of molecules
    biom.Table
        merged feature table that is filtered to contain only the
        features present in the tree
    pd.DataFrame
        merged feature data
    '''

    fps, fts, fdata = [], [], []
    if len(feature_tables) != len(csi_results):
        raise ValueError("The feature tables and CSI results should have a "
                         "one-to-one correspondance.")
    for feature_table, csi_result in zip(feature_tables, csi_results):
        if feature_table.is_empty():
            raise ValueError("Cannot have empty feature table")
        fingerprints = collate_fingerprint(csi_result, qc_properties)
        relabeled_fp, matched_ft, feature_data = match_label(fingerprints,
                                                             feature_table)
        fps.append(relabeled_fp)
        fts.append(matched_ft)
        fdata.append(feature_data)
    merged_fdata = merge_feature_data(fdata)
    merged_fps = pd.concat(fps)
    merged_fps = merged_fps[~merged_fps.index.duplicated(keep='first')]
    merged_fts = merge(fts, overlap_method='error_on_overlapping_sample')
    tree = build_tree(merged_fps)

    return tree, merged_fts, merged_fdata


#!/usr/bin/python
# Solved by Bogdan Trif @       Completed on Fri, 6 Jan 2017, 21:42
#The  Euler Project  https://projecteuler.net
'''
Special subset sums: testing        -       Problem 105

Let S(A) represent the sum of elements in set A of size n.
We shall call it a special sum set if for any two non-empty disjoint subsets, B and C, the following properties are true:

1.      S(B) ≠ S(C); that is, sums of subsets cannot be equal.
2.      If B contains more elements than C then S(B) > S(C).

For example, {81, 88, 75, 42, 87, 84, 86, 65} is not a special sum set because 65 + 87 + 88 = 75 + 81 + 84,
whereas {157, 150, 164, 119, 79, 159, 161, 139, 158} satisfies both rules for all possible subset pair combinations and S(A) = 1286.

Using sets.txt (right click and "Save Link/Target As..."), a 4K text file with one-hundred sets containing
seven to twelve elements (the two examples given above are the first two sets in the file),
identify all the special sum sets, A1, A2, ..., Ak, and find the value of S(A1) + S(A2) + ... + S(Ak).

NOTE: This problem is related to Problem 103 and Problem 106.

'''
import time
from itertools import combinations

filename = "pb105_sets.txt"
def load_file(filename):
    with open(filename) as f:
        matrix = [list(map(int, line.split(","))) for line in f.readlines()]
    f.close()
    return matrix

print(load_file(filename ))

def isSpecialSumSet(s):
    uSet = set(s)
    for i in range(2,len(s)):
        maxSet = max(uSet)
        for a in combinations(s, i):
            ss = sum(a)
            if ss <= maxSet or ss in uSet: return False
            else: uSet.add(ss)
    return True

F = load_file(filename )

print('\n--------------------------TESTS------------------------------')

print('\n================  My FIRST SOLUTION,  VERY FAST, 40 ms ===============\n')
t1  = time.time()

def solution_pb105() :
    cnt , Set_sum = 0, 0
    for i in range(len(F)):
        S = F[i]
        if isSpecialSumSet(S) == True :
            cnt+=1
            Set_sum += sum(S)
            print(str(cnt)+'.   '  ,S)

    return print('\nAnswer : \t', Set_sum )

solution_pb105()                #   Answer : 	 73702

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')                  #       Completed in : 49.002647 ms

# print('\n===============OTHER SOLUTIONS FROM THE EULER FORUM ==============')
# print('\n--------------------------SOLUTION 1,   --------------------------')
# t1  = time.time()
#
#
#
# t2  = time.time()
# print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')
#
# print('\n--------------------------SOLUTION 2,   --------------------------')
# t1  = time.time()
#
#
#
# t2  = time.time()
# print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')
#
# print('\n--------------------------SOLUTION 3,   --------------------------')
# t1  = time.time()
#
#
#
# t2  = time.time()
# print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')
#
# print('\n--------------------------SOLUTION 4,   --------------------------')
# t1  = time.time()
#
#
#
# t2  = time.time()
# print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')
#
# print('\n--------------------------SOLUTION 5,   --------------------------')
# t1  = time.time()
#
#
#
# t2  = time.time()
# print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')
#
#
# print('\n--------------------------SOLUTION 6,   --------------------------')
# t1  = time.time()
#
#
#
# t2  = time.time()
# print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')
#

Project EULER/pb105 Special subset sums - testing.py


# -*- coding: utf-8 -*-

import unittest
import yaml
# import logging
from time import sleep
from jetplex import Procedure
# from jetplex import Execution
from jetplex import Fixture

class SimpleTestCase(unittest.TestCase):
    def testTheTester(self):
        self.assertEquals("good", "good", "good should be good")

hello_world_config = """
hello test fixture:
- {fixturevar1: hello fixture}
hello world procedure:
- set: {procvar1: hello procedure}
- get procedure variable: {lim: EQ, target: hello procedure}
- my condition
- my conditional step: {condition: $my condition}
- get fixture variable: {lim: EQ, target: hello fixture}
"""

class HelloProcedure(Procedure):
    def get_procedure_variable(self):
        return self.procvar1

    def get_fixture_variable(self):
        return self.fixturevar1

    def my_condition(self):
        return False  # because I want to skip my conditional step

    def my_conditional_step(self):
        pass

sleepy_config = """
sleepy procedure:
- be sleepy
- do nothing
"""

class SleepyProcedure(Procedure):
    def be_sleepy(self):
        sleep(1)

    def do_nothing(self):
        pass

class JetPlexTest(unittest.TestCase):
    def testHelloWorld(self):
        configuration = yaml.load(hello_world_config)
        hello_test_fixture = Fixture(
            name='hello test fixture',
            yaml_data=configuration
            )
        hello_test_procedure = HelloProcedure(
            name='hello world procedure',
            yaml_data=configuration,
            )
        hello_test_procedure.run(
            fixture=hello_test_fixture,
            serial_number='1',
            )

    def testSleepy(self):
        configuration = yaml.load(sleepy_config)
        sleepy_procedure = SleepyProcedure(
            name='sleepy procedure',
            yaml_data=configuration,
            )
        sleepy_procedure.run()

if __name__ == '__main__':
    # logging.basicConfig(level=logging.DEBUG)
    unittest.main()


import re
import time
from copy import copy
from json import dumps

import allure
from logger import logger
from resources.rus.translate.action_panel import action_panel as buttons_name
from resources.rus.translate.modal_editor_field import fields as translate_field
from resources.rus.translate.selector import columns as translate_columns
from settings.exceptions import SeleniumWebFormError, MismatchError
from steps.object import compare_web_elements_with_table
from steps.selenium.selenium_utils import close_datepicker, wait_modal_dialog_closing, gen_xpath
from steps.selenium.xpath.buttons import button
from steps.selenium.xpath.fields import fields
from steps.selenium.xpath.general import menu
from steps.selenium.xpath.modules import modules
from steps.selenium.xpath.table import table
from steps.waitings import waiting_selenium_activity

class FrontendGeneralWebElement:
    @staticmethod
    def __pprint(x):
        return dumps(x, ensure_ascii=False, indent=4, sort_keys=True)

    def __init__(self, browser):
        self.browser = browser

    def __get_index_in_list(self, _list, key):
        try:
            return _list.index(key) + 1
        except ValueError:
            raise ValueError('%r is not in \n\t%s' % (key, self.__pprint(_list)))

    # @when('fill fields on "{module}" to')
    @allure.step('Вводим значения полей в модуле {1!r}')
    def fill_fields_in_module(self, module_, table_fill, date_picker=False, pause=0) -> dict:
        xpath_default = None
        multistring_list = (
            'Рецептура',
        )

        xpath = gen_xpath(self.browser, module_) + fields['input_%s' % module_]

        title_elements = None

        # если ввод происходит в таблицу
        if 'td' in xpath:
            title_elements = [v.text for v in self.browser.get_by_xpath(
                modules[module_] + table['table_head'],
                is_elements_list=True,
                custom_time_out=1)]
            field_table = True
        else:
            time.sleep(.2)
            field_table = False

        for key, value in (table_fill.items() if isinstance(table_fill, dict) else table_fill):
            if field_table:
                # переводим из таблицы переводом по таблице и получаем индекс поля
                key = self.__get_index_in_list(title_elements, translate_columns.get(key, key))
            else:
                # просто переводим элемент
                key = translate_field.get(key, key)

            # получаем значения, и преобразуем в строку, селениум не умеем вводить числа
            if key in multistring_list:
                xpath_default, xpath = xpath, modules[module_] + fields['multistring_input_%s' % module_]
            elif xpath_default:
                xpath, xpath_default = xpath_default, None

            xpath_input = xpath % key
            value = str(value)
            multi_input = True if '\\t' in value else False

            element = self.browser.get_by_xpath(xpath_input, is_elements_list=multi_input, custom_time_out=10)

            # Разбиваем текст, если для ввода используется несколько окон
            texts = value.split('\\t') if multi_input else value

            if date_picker:
                # Закрываем не хороший датепикер
                close_datepicker(self.browser)

            # Для ввода в насколько объектов относящихся к одному родителю
            if multi_input:
                for i, text in enumerate(texts):
                    element[i].text_input(text)
            else:
                element.text_input(texts)

            time.sleep(pause)

        # у фронта задержка 400 милисекунд, мы тоже подождем.
        time.sleep(.45)
        # Потом подожем, что все действия завершены
        waiting_selenium_activity(self.browser)
        logger.attach_info('Таблица ввода', table_fill)

        return table_fill

    # @when('choose "{name}" item in dropdown menu')
    @allure.step('Выбрали элемент из выпадающего списка {1!r}')
    def choose_item_from_dropdown(self, name):
        # выбор элемента для создания
        xpath = menu['dropdown_el'] % translate_field.get(name, name)

        elements = self.browser.get_by_xpath(xpath, custom_time_out=10, is_elements_list=True)
        # кендо шаблон при каждом обращение к списку создает новый список, селениум не гарантирует порядка,
        # поэтому бежим по спику и кликаем только по активному элементу
        for i in range(3):
            check = False
            for el in elements:
                if el.is_displayed():
                    el.click()
                    check = True
                    break
            else:
                logger.error(
                    'Ни один элемент не определился как доступный для действия (попытка %r)' % i)
                time.sleep(.1)

            if check:
                break
        else:
            raise SeleniumWebFormError(
                'Не смог выбрать элемент %r из списка %s' % (
                    name,
                    dumps({elements[0].identify: [el.is_displayed() for el in elements]}, indent=2, ensure_ascii=False)
                )
            )

        waiting_selenium_activity(self.browser)

    # @when('click to "{name}" field with dropdown on "{module}"')
    @allure.step('Раскрыли выпадающий список {1!r} в модуле {2!r}')
    def expand_field_dropdown(self, name, module_, field_table=False):
        # расскрыть дропдаун
        if 'editor_selector' in module_:
            xpath = modules[module_] + menu[module_]['dropdown']
            field_table = True
        else:
            xpath = menu[module_]['dropdown']

        if field_table:
            title_elements = [v.text for v in self.browser.get_by_xpath(
                modules[module_] + table['table_head'],
                is_elements_list=True,
                custom_time_out=1)]
            # переводим из таблицы переводом по таблице и получаем индекс поля
            name = self.__get_index_in_list(title_elements, translate_columns.get(name, name))

        else:
            # просто переводим элемент
            name = translate_field.get(name, name)

        # Проверяем и закрываем е*** datepicker
        close_datepicker(self.browser)

        el = self.browser.get_by_xpath(xpath % name, custom_time_out=10)
        # Бывает что объъект ниже области видимости, спускаемся к нему
        self.browser.scroll(el)
        el.click()

        waiting_selenium_activity(self.browser)

    @allure.step('Нажимаем на кнопку добавления элемента (плюсик) {1!r} в модуле {2!r}')
    # @when('click to add button to "{name}" field with dropdown on "{module}"')
    def add_item_from_field(self, name, module_, field_table=False):
        # создание тэга по + в категории тэг и еще где-то
        xpath = '{}{}'.format(
            menu[module_]['add'],
            self.browser.get_current_modal_dialog_id if 'modal' in module_ else ''
        )

        if 'editor_selector' in module_:
            field_table = True

        if field_table:
            title_elements = [v.text for v in self.browser.get_by_xpath(
                modules[module_] + table['table_head'],
                is_elements_list=True,
                custom_time_out=1)]
            # переводим из таблицы переводом по таблице и получаем индекс поля
            name = self.__get_index_in_list(title_elements, translate_columns.get(name, name))
        else:
            # просто переводим элемент
            name = translate_field.get(name, name)

        self.browser.get_by_xpath(xpath % name, custom_time_out=10).click()

        waiting_selenium_activity(self.browser)

    @allure.step('Проверяем наличие элементов в выпадающем списке')
    # @then('"{module}" has default "{table}" items')
    def dropdown_has_items(self, table_items, only_left_part=False, type_el='dropdown', skip_objects=True):
        # проверка элементов в контейнерах
        xpath = menu[type_el]
        # Берем только левую часть элемента
        if only_left_part:
            xpath += '//span[contains(@class, "refEditor__categoryItem__title")]'

        error_list = compare_web_elements_with_table(self.browser.get_by_xpath(xpath, is_elements_list=True),
                                                     copy(table_items), skip_objects=skip_objects)
        waiting_selenium_activity(self.browser)
        if error_list:
            raise MismatchError('Нашли не соотвествия при проверки объектов', error_list)

    @allure.step('Нажимаем на кнопку {1!r} в {2}')
    def button_click(self, element_name, locate, module_=None, type_web_element='button_name'):
        """

        :param element_name:
        :param locate:
        :param module_:
        :param type_web_element:
        :return:
        """
        _rus_world = re.compile("[а-яА-Я]+")

        modal_id = xpath = ''
        # Добавляем путь до муодуля
        if module_:
            # Если дело происходит в модалке, надо получить индификатор модалки, что бы не промазать
            xpath = modal_id = gen_xpath(self.browser, module_)

        # Пробуем получить name из подготовлено списка, если не смогли получить,
        #  и есть русские буквы, что искать надо по тексту
        element_name = buttons_name.get(element_name, element_name)

        if _rus_world.search(element_name) and type_web_element == 'button_name':
            type_web_element = 'button_text'

        xpath += (button[locate] if locate else '') + button[type_web_element] % element_name

        web_element = self.browser.get_by_xpath(xpath)
        # Есть поблемы с тултипом, поэтому для начала наводим на кнопку.
        self.browser.move_to_element(web_element, sleep=0)
        web_element.click()
        waiting_selenium_activity(self.browser)

        # Ждем закрытия модальных окон, если кнопка была в модалке.
        if module_ == 'modal' and locate != 'action_panel':
            # Есть ситуации когда, скрипты фронта не успевают отработать, и клик не проходит.
            # Ждем 1 секунду и кликаем повторно
            if wait_modal_dialog_closing(self.browser, xpath=modal_id, time_out=1, _return=True):
                logger.warning('Не успели отработать скрипты со стороны фронта')
                web_element.click()
                waiting_selenium_activity(self.browser)
                wait_modal_dialog_closing(self.browser, xpath=modal_id)

        self.browser.refresh_list_opened_modal_id()

    # @then("*" "*" has "*" default buttons)
    @allure.step('Проверяем наличие кнопок {1}')
    def check_button_in_module(self, locate, table_element, module_=None, element='buttons'):
        xpath = '{}{}{}'.format(
            gen_xpath(self.browser, module_),
            button[locate] if locate else '',
            button[element if module_ != 'modal' else 'modal_buttons']
        )

        if element == 'dropdown':
            # Только дропдаун конвертим, все остальные таблицы, должны взяты 100% со словарей
            # и уже норм индификатор иметь
            table_element = {buttons_name.get(key, key): value for key, value in table_element.items()}

        error_list = compare_web_elements_with_table(
            self.browser.get_by_xpath(xpath, is_elements_list=True),
            table_element
        )
        if error_list:
            raise SeleniumWebFormError('В ходе проверки кнопок выявлены несоответствия', error_list)

    @allure.step('Проверяем список нотификаций полученный через селениум')
    def check_notifications_list(self, messages_list, double_message_error=False):
        """

        :param messages_list: передаются только те нотификации, которые пользователь ожидает увидеть в браузере
        :param double_message_error:
        :return:
        """

        xpath = modules['notification'] + '/p'

        mismatch_list = set()

        new_format = re.compile('^|\?|\n|\r|\t|$')

        notification_list = [new_format.sub('', v.text).strip() for v in self.browser.get_by_xpath(
            xpath,
            is_elements_list=True,
            custom_time_out=5,
            need_fail=list()
        )]

        # В notification_list передаются только те нотификации, которые пользователь видит в браузере
        result = set(notification_list) - set(messages_list)  # лишнии нотификации

        # Данная переменная необходима для того, чтобы проверить наличие "левых" нотификаций
        result_excessive_elements = set(messages_list) - set(notification_list)  # не проверенные

        if result_excessive_elements:
            mismatch_list.add('Не были найдены следующие нотификации: %r' % result_excessive_elements)

        if result:
            mismatch_list.add('Найдены лишние нотификации, не участвующие в проверке: %r' % result)

        if double_message_error:
            for el in notification_list:
                if notification_list.count(el) > 1:
                    mismatch_list.add('Существует несколько дубликатов нотификации: %r' % el)
                    return

        if mismatch_list:
            raise MismatchError('Нашли не соотвествия при проверки объектов', list(mismatch_list))

        # После всех проверок, удаляем все нотификации отображающиеся на текущий момент в браузере
        self.browser.api_qa_remove_notification()

    @allure.step('Проверяем если открытое модальное окно, только с текстом без полей')
    def item_successfully_created(self):
        # проверка если открытое модальное окно, только с текстом без полей

        xpath_body = gen_xpath(self.browser, 'modal')

        element_body = self.browser.get_by_xpath(xpath_body, need_fail=None) if xpath_body else None

        if element_body is not None:
            xpath_body += "//div"

            if not self.browser.get_by_xpath(xpath_body, need_fail=None, custom_time_out=0.1):
                raise SeleniumWebFormError('Открыто модальное окно c полями!!!')


from flask import Flask, render_template, redirect, url_for
from flask_wtf import Form
from flask_wtf.csrf import CsrfProtect
from wtforms import RadioField
import os
SECRET_KEY = 'development'

app = Flask(__name__)
CsrfProtect(app)
app.config.from_object(__name__)

class SimpleForm(Form):
    example = RadioField('Label', choices=[('value','Test_Matias'),('value_two','DES0001-4957')])

@app.route('/',methods=['post','get'])
def hello_world():
  form = SimpleForm()
  option = form.example.data
  if option=='value':
    if os.path.exists('static/Test_Matias'):
        print('I exist')
    else:
        print('no')

    #return redirect('/static/Test_Matias/test_modified.html')
    return redirect('/first')
  elif option == 'value_two':
    return redirect ('https://www.yahoo.com')
  return render_template('example.html',form=form)

@app.route('/first')
def first():
    return render_template('home.html', number="Test tile", folder='Test_Matias')

@app.route('/second')
def second():
    return 'second page'

if __name__ == '__main__':
  app.run(debug=True)


def iterable(obj):
    try:
        iter(obj)
        return True
    except TypeError:
        return False

for element in [1, ['a', 'b', 'c'], {'a':1,'b':2}, "string", 1.0035, (1,2,3)]:
    print(element, "Iterable? ", iterable(element))

class xrange:
    def __init__(self, n):
        self.i = 0
        self.n = n

    def __iter__(self):
        return self
    def next(self):
        if self.i < self.n:
            i = self.i
            self.i += 1
            return i
        else:
            raise StopIteration()

x = xrange(5)

import itertools as it
for x,y in it.izip(['a','b','c'],[1,2,3]):
    print(x,y)


import pymongo
from pymongo import MongoClient
from requests import get
from bs4 import BeautifulSoup

urls = []
url = 'https://houseofindia-limbach.de/speisekarte.html'

# Lists to store the scraped data in
names = []
prices = []
ingredients = []
types = []
classes = ["tr", "td"]

#for ittr in range(0, 1):

response = get(url)

html_soup = BeautifulSoup(response.text, 'html.parser')
title = html_soup.find_all('title')
dish_container = html_soup.find_all('tr')
dishType = html_soup.find_all('td', {"colspan": "4"})
dishName = html_soup.find_all('td', {"colspan": "2"})
dishDisc = html_soup.find_all('td', {"colspan": "3"})
dishPrice = html_soup.find_all('td', {"class": "right"})
#print(dishDisc)

# Extract data from individual dish container
for i in dishName:
    name = i.text
    names.append(name)
    #print(name)
for i in dishDisc:
    description = i.text
    ingredients.append(description)
    #print(description)
for i in dishPrice:
    price = i.text
    prices.append(price)
    #print(price)

for container in dishType:
    #h2 = container.find_all('h2')
    types = container.find_all('span')

    #print(types)
    j=0
    for  i in types:
        if(j == 0):
            type = i.text
            #print(type)
            types.append(type)
            j=j+1

import json

try:
    conn = MongoClient('mongodb://localhost:27017')
    print("Connected successfully!!!")
except:
    print("Could not connect to MongoDB")

db = conn.foodworld

collection = db.dishes

i=0
while i < len(price) :
    food = {'dishName': name[i], 'dishPrice':price[i], 'dishIngredients':description[i], 'type':type}
    i=i+1
    print(json.dumps(food))
    r1 = collection.insert_one(food)
    #


import os
import h5py
import numpy as np

def delete_file(filename):
	#deletes a file if it exsits
	try:
		os.remove(filename)
	except OSError:
		pass
#--------------------------Data Loader--------------------------------#

def load_data(data_dir):
	train_data=h5py.File('%s/train_data.h5'%data_dir)
	val_data=h5py.File('%s/val_data.h5'%data_dir)
	img_mean= np.array(train_data['img_mean'])

	train_labels=(train_data['hist_labels'], train_data['raw_labels'])
	val_labels=(val_data['hist_labels'], val_data['raw_labels'])

	return train_data['images'], train_labels, val_data['images'], val_labels, img_mean

#--------------------------Batch Loader-------------------------------#

#kernel=1.0*np.array([5, 20, 80, 100, 80, 20, 5])
kernel=1.0*np.array([5, 20, 50, 80, 100, 100, 100, 80, 50, 20, 5])
kernel/=np.sum(kernel)
def vectorize_labels(labels, opt):

	# threshold
	#labels[labels<23]=23
	#labels[labels>105]=105
	#labels-=23

	k_size=len(kernel)
	pad= k_size/2

	label_vec=np.zeros((len(labels),opt.num_classes+k_size-1))
	for idx, lab in enumerate(labels):
		label_vec[idx][lab:lab+k_size]=kernel
	label_vec=label_vec[:,pad:-pad]

	#renormalize rows
	label_vec/=label_vec.sum(axis=1)[:, np.newaxis]

	return label_vec

def get_random_batch(data, labels, opt):

	hist_labels, raw_labels= labels
	rand_idx=np.random.choice(data.shape[0], opt.batch_size, replace=False)
	rand_idx=sorted(rand_idx) #h5py is weird
	batch_data= np.subtract(data[rand_idx], opt.img_mean)
	batch_labels=1.0*vectorize_labels(raw_labels[0][rand_idx], opt)
	#batch_labels=raw_labels[0][rand_idx]+1.0 #Torch <3

	if 'staged' in opt.model:
		batch_labels=[batch_labels, raw_labels[1][rand_idx]+1.0, raw_labels[2][rand_idx]+1.0]

	return batch_data, batch_labels

def get_batches_in_sequence(data, labels, opt):

	hist_labels, raw_labels= labels

	for i in xrange(data.shape[0]/opt.batch_size):
		st, end= opt.batch_size*i, opt.batch_size*i + opt.batch_size
		batch_data= np.subtract(data[st:end], opt.img_mean)
		batch_labels= 1.0*vectorize_labels(raw_labels[0][st:end], opt)
		#batch_labels=raw_labels[0][st:end]+1.0 # Torch <3
		batch_truth= raw_labels[0][st:end]

		if 'staged' in opt.model:
			batch_labels=[batch_labels, raw_labels[1][st:end]+1.0, raw_labels[2][st:end]+1.0]

		yield batch_data, batch_labels, batch_truth


'''Based on https://github.com/google-research/dads'''

from gym.spaces import Tuple, Discrete, Dict
import logging
import numpy as np
import tree
from collections import defaultdict

import ray
from ray.rllib.agents.qmix.mixers import VDNMixer, QMixer
from ray.rllib.agents.qmix.model import RNNModel, _get_size
from ray.rllib.agents.ppo.ppo_torch_policy import ppo_surrogate_loss, vf_preds_fetches, ValueNetworkMixin, kl_and_loss_stats
from ray.rllib.env.multi_agent_env import ENV_STATE
from ray.rllib.env.wrappers.group_agents_wrapper import GROUP_REWARDS
from ray.rllib.evaluation.metrics import LEARNER_STATS_KEY
from ray.rllib.evaluation.postprocessing import compute_gae_for_sample_batch, \
    Postprocessing
from ray.rllib.models.torch.torch_action_dist import TorchCategorical
from ray.rllib.policy.policy import Policy
from ray.rllib.policy.torch_policy import TorchPolicy
from ray.rllib.policy.rnn_sequencing import chop_into_sequences
from ray.rllib.policy.sample_batch import SampleBatch
from ray.rllib.models.catalog import ModelCatalog
from ray.rllib.models.modelv2 import _unpack_obs, restore_original_dimensions
from ray.rllib.utils.framework import try_import_torch
from ray.rllib.utils.annotations import override
from ray.rllib.utils.torch_ops import apply_grad_clipping, \
    convert_to_torch_tensor, explained_variance, sequence_mask, \
    convert_to_non_torch_type
from ray.rllib.agents.custom_ppo.skill_dynamics_model import SkillDynamics
from ray.rllib.agents.custom_ppo.model import PPODADSAgent
from ray.rllib.execution.rollout_ops import StandardizeFields
from ray.rllib.utils.sgd import minibatches, standardize

# Torch must be installed.
torch, nn = try_import_torch(error=True)

logger = logging.getLogger(__name__)

class DADSPPOPolicy(TorchPolicy):
    def __init__(self, obs_space, action_space, config):

        # update policy attr for loss calculation
        self.framework = config['framework'] = 'torch'
        self.kl_coeff = config['kl_coeff']
        self.kl_target = config['kl_target']
        self.entropy_coeff = config['entropy_coeff']
        self.cur_lr = config['lr']
        # setup ._value() for gae computation
        self.setup_value(config)
        self.dist_class, logit_dim = ModelCatalog.get_action_dist(
                    action_space, config["model"], framework='torch')

        self.model = PPODADSAgent(obs_space, action_space, logit_dim,
                                  config['model'], name='ppo_dads_agent')

        super().__init__(obs_space, action_space, config,
                         model=self.model,
                         loss=None, # we calculate the loss inside learn_on_batch
                         action_distribution_class=self.dist_class
                         )
        # Merge Model's view requirements into Policy's.
        self.view_requirements.update(self.model.view_requirements)
        # Perform test runs through postprocessing- and loss functions.
        self._initialize_loss_from_dummy_batch(
            auto_remove_unneeded_view_reqs=True,
            stats_fn=None,
        )
        self.ppo_opt = torch.optim.Adam(self.model.parameters(), lr=self.config["lr"])
        self.minibatch_size = config['sgd_minibatch_size']
        self.ppo_epochs = config['ppo_epochs']
        self.dynamics_epochs = config['dynamics_epochs']
        self.use_dynamics = config['use_dynamics']
        # create dynamics model
        if self.use_dynamics:
            self.dynamics = SkillDynamics(self.config)
            self.dynamics_opt = torch.optim.Adam(self.dynamics.parameters(), lr=self.config["dynamics_lr"])

    @override(TorchPolicy)
    def postprocess_trajectory(self,
                                   sample_batch,
                                   other_agent_batches=None,
                                   episode=None):
        return sample_batch

    @override(TorchPolicy)
    def compute_actions_from_input_dict(self,
                                        input_dict,
                                        explore=None,
                                        timestep=None,
                                        episodes=None,
                                        **kwargs):

        explore = explore if explore is not None else self.config["explore"]
        with torch.no_grad():
            input_dict = self._lazy_tensor_dict(input_dict)
            # state_batches for RNN
            state_batches = [
                input_dict[k] for k in input_dict.keys() if "state_in" in k[:8]
            ]
            seq_lens = np.array([1] * len(input_dict["obs"])) if state_batches else None
            dist_inputs, state_out = self.model(input_dict, state_batches, seq_lens)
            action_dist = self.dist_class(dist_inputs, self.model)
            # Get the exploration action from the forward results.
            actions, logp = \
                self.exploration.get_exploration_action(
                    action_distribution=action_dist,
                    timestep=timestep,
                    explore=explore)
            # add extra info to the trajectory
            extra_info = {}
            # get values from the critic after doing inference for the actions
            extra_info[SampleBatch.VF_PREDS] = self.model.value_function()
            # Action-dist inputs.
            if dist_inputs is not None:
                extra_info[SampleBatch.ACTION_DIST_INPUTS] = dist_inputs

            # Action-logp and action-prob.
            if logp is not None:
                extra_info[SampleBatch.ACTION_PROB] = \
                    torch.exp(logp.float())
                extra_info[SampleBatch.ACTION_LOGP] = logp

            # Update our global timestep by the batch size.
            self.global_timestep += len(input_dict[SampleBatch.CUR_OBS])

            return convert_to_non_torch_type((actions, state_out, extra_info))

    @override(TorchPolicy)
    def learn_on_batch(self, train_batch):
        # print(type(train_batch))
        # Turn the values into tensors
        # train_batch_tensor = self._lazy_tensor_dict(train_batch)
        # train_batch_tensor = train_batch_tensor
        # restore_original_dimensions()
        # print(train_batch_tensor.keys())
        # update the skill dynamics

        # Set Model to train mode.
        if self.model:
            self.model.train()
        if self.dynamics:
            self.dynamics.train()

        stats = defaultdict(int)
        if self.use_dynamics:
            c = 0
            for ep in range(self.dynamics_epochs):
                for mb in minibatches(train_batch, self.minibatch_size): # minibatches(train_batch.copy(), self.minibatch_size)
                    c += 1
                    mb["is_training"] = True
                    minibatch = self._lazy_tensor_dict(mb)

                    obs = _unpack_obs(minibatch['obs'],
                                        self.model.options['orig_obs_space'],torch)
                    next_obs = _unpack_obs(minibatch['new_obs'],
                                        self.model.options['orig_obs_space'],torch)
                    dynamics_obs = obs['dynamics_obs']
                    next_dynamics_obs = next_obs['dynamics_obs'] - obs['dynamics_obs']
                    z = obs['z']

                    log_prob = self.dynamics.get_log_prob(dynamics_obs,z,next_dynamics_obs,training=True)
                    dynamics_loss = -torch.mean(log_prob)
                    orth_loss = self.dynamics.orthogonal_regularization()
                    l2_loss = self.dynamics.l2_regularization()
                    if self.config['dynamics_orth_reg']:
                        dynamics_loss += orth_loss
                    if self.config['dynamics_l2_reg'] and not self.config['dynamics_spectral_norm']:
                        dynamics_loss += l2_loss
                    self.dynamics_opt.zero_grad()
                    dynamics_loss.backward()
                    if self.config['grad_clip']:
                        grad_norm = nn.utils.clip_grad_norm_(
                            self.dynamics.parameters(), self.config['grad_clip'])
                    self.dynamics_opt.step()
                    stats['dynamics_loss'] += dynamics_loss.item()
                    stats['orth_loss'] += orth_loss.item()
                    stats['l2_loss'] += l2_loss.item()
            stats['dynamics_loss'] /= c
            stats['orth_loss'] /= c
            stats['l2_loss'] /= c

            self.dynamics.eval()
            # compute intrinsic reward
            with torch.no_grad():
                batch = self._lazy_tensor_dict(train_batch)
                obs = _unpack_obs(batch['obs'],
                                        self.model.options['orig_obs_space'],torch)
                next_obs = _unpack_obs(batch['new_obs'],
                                    self.model.options['orig_obs_space'],torch)
                z = obs['z']
                dynamics_obs = obs['dynamics_obs']
                next_dynamics_obs = next_obs['dynamics_obs'] - obs['dynamics_obs']

                dads_reward, info = self.dynamics.compute_reward(dynamics_obs,z,next_dynamics_obs)
                dads_reward = self.config['dads_reward_scale'] * dads_reward.numpy()
                # # replace the reward column in train_batch
                # print(train_batch['rewards'].shape)
                train_batch['rewards'] = dads_reward
                stats['avg_dads_reward'] = dads_reward.mean()
                stats['num_skills_higher_prob'] = info['num_higher_prob']

        # calculate GAE for dads reward here?
        trajs = train_batch.split_by_episode()
        processed_trajs = []
        for traj in trajs:
            processed_trajs.append(compute_gae_for_sample_batch(self,traj))
        batch = SampleBatch.concat_samples(processed_trajs)

        # train_batch = compute_gae_for_sample_batch(self, self._lazy_numpy_dict(train_batch))
        # train_batch = self._lazy_tensor_dict(train_batch)
        # update agent using RL algo
        # split to minibatches
        c = 0
        for ep in range(self.ppo_epochs):
            # batch.shuffle()
            for mb in minibatches(batch, self.minibatch_size):
                c += 1
                mb["is_training"] = True
                # minibatch = mb.copy()
                mb['advantages'] = standardize(mb['advantages'])
                minibatch = self._lazy_tensor_dict(mb)
                # compute the loss
                loss_out = ppo_surrogate_loss(self,self.model,self.dist_class,minibatch)
                # compute gradient
                self.ppo_opt.zero_grad()
                # the learning_rate is already used in ppo_surrogate_loss
                loss_out.backward()
                # grad norm
                if self.config['grad_clip']:
                    grad_norm = nn.utils.clip_grad_norm_(
                        self.model.parameters(), self.config['grad_clip'])
                self.ppo_opt.step()
                # log stats
                stats['ppo_loss'] += loss_out.item()
        stats['ppo_loss'] /= c
        # add more info about the loss
        stats.update(kl_and_loss_stats(self, train_batch))

                #  {
                #     "loss": loss_out.item(),
                #     'test': 1
                #     # "grad_norm": grad_norm
                #     # if isinstance(grad_norm, float) else grad_norm.item(),
                # }
        return {LEARNER_STATS_KEY: stats}

    def setup_value(self, config):
        # When doing GAE, we need the value function estimate on the
        # observation.
        if config["use_gae"]:
            # Input dict is provided to us automatically via the Model's
            # requirements. It's a single-timestep (last one in trajectory)
            # input_dict.
            if config["_use_trajectory_view_api"]:
                def value(**input_dict):
                    model_out, _ = self.model.from_batch(
                        convert_to_torch_tensor(input_dict, self.device),
                        is_training=False)
                    # [0] = remove the batch dim.
                    return self.model.value_function()[0]
        else:
            def value(*args, **kwargs):
                return 0.0

        self._value = value


#!/usr/bin/env python
import os
import zipfile
from argparse import ArgumentParser

class ZipFile(zipfile.ZipFile):
    def __init__(self, *args, charset=None, **kwargs):
        super().__init__(*args, **kwargs)
        self.from_charset = 'cp437'
        self.charset = 'cp437'
        self._refine_filelist(charset)

    def _refine_filelist_with_charset(self, charset):
        for item in self.filelist:
            raw_name = item.filename.encode(self.from_charset)
            new_name = raw_name.decode(charset)
            item.filename = new_name
            self.NameToInfo[new_name] = item
        self.charset = charset

    def _refine_filelist(self, charset=None):
        # When the charset is specified,
        # the specified charset is used first.
        if charset:
            for c in charset:
                try:
                    self._refine_filelist_with_charset(c)
                    return
                except UnicodeDecodeError:
                    pass

        try:
            import chardet
            all_filename = b''.join(
                [f.filename.encode(self.from_charset) for f in self.filelist])
            charset = chardet.detect(all_filename).get(
                'encoding') or self.from_charset

            self._refine_filelist_with_charset(charset)
        except ImportError:
            pass

def listZip(zipf, charset):
    with ZipFile(zipf, 'r', charset=charset) as zipObj:
        print("Archive:  %s" % zipf)
        print("  Length      Date    Time    Name")
        print("---------  ---------- -----   ----")
        infoList = zipObj.infolist()
        totalNum = 0
        totalSize = 0
        for fInfo in infoList:
            totalNum += 1
            totalSize += fInfo.file_size
            print("%9d" % fInfo.file_size, end=' ')
            print("%04d-%02d-%02d" %
                  (fInfo.date_time[0], fInfo.date_time[1], fInfo.date_time[2]), end=' ')
            print("%02d:%02d" %
                  (fInfo.date_time[3], fInfo.date_time[4]), end='  ')
            print("{}({})".format(fInfo.filename, zipObj.charset))
        print("---------                     -------")
        print("%9d" % totalSize, end='')
        print("                   ", end='')
        print("%d files" % totalNum)

def parse_args():
    parser = ArgumentParser()
    parser.add_argument('-l', '--list', action='store_true',
                        help='list files in zip file', dest='islist', default=False)
    parser.add_argument('-d', dest='exdir', default='./',
                        help='extract files into exdir')
    parser.add_argument('-O', dest='charset', nargs='*',
                        help='specify a character encoding')
    parser.add_argument('file', nargs='+', help='target file or directory')
    return parser.parse_args()

def main():
    args = parse_args()
    charset = args.charset
    for f in args.file:
        if args.islist:
            listZip(f, charset=charset)
        else:
            with ZipFile(f, 'r', charset=charset) as zipObj:
                zipObj.extractall(path=args.exdir)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('herders', '0025_auto_20150724_1701'),
    ]

    operations = [
        migrations.AlterField(
            model_name='gameevent',
            name='day_of_week',
            field=models.IntegerField(blank=True, null=True, choices=[(6, b'Sunday'), (0, b'Monday'), (1, b'Tuesday'), (2, b'Wednesday'), (3, b'Thursday'), (4, b'Friday'), (5, b'Saturday')]),
        ),
    ]