import pytest

from moj_analytics.auth0_client import Client, Group, Permission, Role

class TestAuth0Client(object):
    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials",
        "given_access_to_the_management_api",
    )
    def test_it_can_create_a_client(self, auth0):

        client = auth0.management.get_or_create(Client(name="test-client"))

        assert client["client_id"] == "new-client-id"
        assert client["client_secret"] == "new-client-secret"

    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials",
        "given_access_to_the_management_api",
    )
    def test_it_gets_all_clients(self, auth0):
        clients = auth0.management.get_all(Client)
        client_names = [client["name"] for client in clients]

        assert len(clients) == 100
        assert "client1" in client_names
        assert "client50" in client_names
        # Return all clients, including the ones on subsequent pages
        assert "client100" in client_names

    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials",
        "given_access_to_the_management_api",
    )
    def test_it_can_update_a_client(self, auth0):
        client = auth0.management.get_or_create(Client(name="test-client"))
        client.update(web_origins=["foo"])

        assert client["web_origins"] == ["foo"]

    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials",
        "given_access_to_the_authorization_api",
        "given_a_permission_exists",
    )
    def test_it_can_get_a_permission(self, auth0, responses, state):

        permission = auth0.authorization.get(
            Permission(name="perm1", applicationId="app1")
        )

        if len(state["permissions"]) == 0:
            assert permission is None

        else:
            assert permission

    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials", "given_access_to_the_authorization_api"
    )
    def test_it_can_create_a_permission(self, auth0):

        permission = auth0.authorization.get_or_create(
            Permission(name="view:app", applicationId="new-client-id")
        )

        assert permission["description"] == "view:app"

    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials",
        "given_access_to_the_authorization_api",
    )
    def test_it_can_create_a_role(self, auth0):

        role = auth0.authorization.create(
            Role(name="role1", applicationId="new-client-id")
        )

        assert role["description"] == "role1"

    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials",
        "given_access_to_the_authorization_api",
        "given_a_role_exists",
        "given_a_permission_exists",
    )
    def test_it_can_add_a_permission_to_a_role(self, auth0):

        role = auth0.authorization.get(Role(name="role1", applicationId="app1"))
        permission = auth0.authorization.get(
            Permission(name="perm1", applicationId="app1")
        )

        role.add_permission(permission)

        assert permission["_id"] in role["permissions"]

    @pytest.mark.usefixtures(
        "given_valid_api_client_credentials",
        "given_access_to_the_authorization_api",
        "given_a_group_exists",
        "given_a_role_exists",
    )
    def test_it_can_add_a_role_to_a_group(self, auth0, responses):

        group = auth0.authorization.get(Group(name="group1"))
        role = auth0.authorization.get(Role(name="role1", applicationId="app1"))

        group.add_role(role)

        calls = [
            call
            for call in responses.calls
            if call.request.method == "PATCH"
            and call.request.url.endswith("groups/{_id}/roles".format(**group))
        ]
        assert len(calls) == 1


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.main, name='stock_Main'),
    url(r'^stock/all_available/$', views.all_stock, name='all_available'),
    url(r'^stock_form/$', views.stockform, name='stock_form'),
    url(r'^stock/delete/(?P<lineid>\d+)/$', views.stock_delete, name='stock_delete'),
    url(r'^stock/edit/(?P<lineid>\d+)/$', views.stock_edit, name='stock_edit'),
    url(r'^stock/all_stock/$', views.all_stock, name='all_stock'),

]


"""
Distribution functions used in GLM
"""

# Author: Christian Lorentzen <lorentzen.ch@googlemail.com>
# License: BSD 3 clause
#
# TODO(1.3): remove file
#       This is only used for backward compatibility in _GeneralizedLinearRegressor
#       for the deprecated family attribute.

from abc import ABCMeta, abstractmethod
from collections import namedtuple
import numbers

import numpy as np
from scipy.special import xlogy

DistributionBoundary = namedtuple("DistributionBoundary", ("value", "inclusive"))

class ExponentialDispersionModel(metaclass=ABCMeta):
    r"""Base class for reproductive Exponential Dispersion Models (EDM).

    The pdf of :math:`Y\sim \mathrm{EDM}(y_\textrm{pred}, \phi)` is given by

    .. math:: p(y| \theta, \phi) = c(y, \phi)
        \exp\left(\frac{\theta y-A(\theta)}{\phi}\right)
        = \tilde{c}(y, \phi)
            \exp\left(-\frac{d(y, y_\textrm{pred})}{2\phi}\right)

    with mean :math:`\mathrm{E}[Y] = A'(\theta) = y_\textrm{pred}`,
    variance :math:`\mathrm{Var}[Y] = \phi \cdot v(y_\textrm{pred})`,
    unit variance :math:`v(y_\textrm{pred})` and
    unit deviance :math:`d(y,y_\textrm{pred})`.

    Methods
    -------
    deviance
    deviance_derivative
    in_y_range
    unit_deviance
    unit_deviance_derivative
    unit_variance

    References
    ----------
    https://en.wikipedia.org/wiki/Exponential_dispersion_model.
    """

    def in_y_range(self, y):
        """Returns ``True`` if y is in the valid range of Y~EDM.

        Parameters
        ----------
        y : array of shape (n_samples,)
            Target values.
        """
        # Note that currently supported distributions have +inf upper bound

        if not isinstance(self._lower_bound, DistributionBoundary):
            raise TypeError(
                "_lower_bound attribute must be of type DistributionBoundary"
            )

        if self._lower_bound.inclusive:
            return np.greater_equal(y, self._lower_bound.value)
        else:
            return np.greater(y, self._lower_bound.value)

    @abstractmethod
    def unit_variance(self, y_pred):
        r"""Compute the unit variance function.

        The unit variance :math:`v(y_\textrm{pred})` determines the variance as
        a function of the mean :math:`y_\textrm{pred}` by
        :math:`\mathrm{Var}[Y_i] = \phi/s_i*v(y_\textrm{pred}_i)`.
        It can also be derived from the unit deviance
        :math:`d(y,y_\textrm{pred})` as

        .. math:: v(y_\textrm{pred}) = \frac{2}{
            \frac{\partial^2 d(y,y_\textrm{pred})}{
            \partialy_\textrm{pred}^2}}\big|_{y=y_\textrm{pred}}

        See also :func:`variance`.

        Parameters
        ----------
        y_pred : array of shape (n_samples,)
            Predicted mean.
        """

    @abstractmethod
    def unit_deviance(self, y, y_pred, check_input=False):
        r"""Compute the unit deviance.

        The unit_deviance :math:`d(y,y_\textrm{pred})` can be defined by the
        log-likelihood as
        :math:`d(y,y_\textrm{pred}) = -2\phi\cdot
        \left(loglike(y,y_\textrm{pred},\phi) - loglike(y,y,\phi)\right).`

        Parameters
        ----------
        y : array of shape (n_samples,)
            Target values.

        y_pred : array of shape (n_samples,)
            Predicted mean.

        check_input : bool, default=False
            If True raise an exception on invalid y or y_pred values, otherwise
            they will be propagated as NaN.
        Returns
        -------
        deviance: array of shape (n_samples,)
            Computed deviance
        """

    def unit_deviance_derivative(self, y, y_pred):
        r"""Compute the derivative of the unit deviance w.r.t. y_pred.

        The derivative of the unit deviance is given by
        :math:`\frac{\partial}{\partialy_\textrm{pred}}d(y,y_\textrm{pred})
             = -2\frac{y-y_\textrm{pred}}{v(y_\textrm{pred})}`
        with unit variance :math:`v(y_\textrm{pred})`.

        Parameters
        ----------
        y : array of shape (n_samples,)
            Target values.

        y_pred : array of shape (n_samples,)
            Predicted mean.
        """
        return -2 * (y - y_pred) / self.unit_variance(y_pred)

    def deviance(self, y, y_pred, weights=1):
        r"""Compute the deviance.

        The deviance is a weighted sum of the per sample unit deviances,
        :math:`D = \sum_i s_i \cdot d(y_i, y_\textrm{pred}_i)`
        with weights :math:`s_i` and unit deviance
        :math:`d(y,y_\textrm{pred})`.
        In terms of the log-likelihood it is :math:`D = -2\phi\cdot
        \left(loglike(y,y_\textrm{pred},\frac{phi}{s})
        - loglike(y,y,\frac{phi}{s})\right)`.

        Parameters
        ----------
        y : array of shape (n_samples,)
            Target values.

        y_pred : array of shape (n_samples,)
            Predicted mean.

        weights : {int, array of shape (n_samples,)}, default=1
            Weights or exposure to which variance is inverse proportional.
        """
        return np.sum(weights * self.unit_deviance(y, y_pred))

    def deviance_derivative(self, y, y_pred, weights=1):
        r"""Compute the derivative of the deviance w.r.t. y_pred.

        It gives :math:`\frac{\partial}{\partial y_\textrm{pred}}
        D(y, \y_\textrm{pred}; weights)`.

        Parameters
        ----------
        y : array, shape (n_samples,)
            Target values.

        y_pred : array, shape (n_samples,)
            Predicted mean.

        weights : {int, array of shape (n_samples,)}, default=1
            Weights or exposure to which variance is inverse proportional.
        """
        return weights * self.unit_deviance_derivative(y, y_pred)

class TweedieDistribution(ExponentialDispersionModel):
    r"""A class for the Tweedie distribution.

    A Tweedie distribution with mean :math:`y_\textrm{pred}=\mathrm{E}[Y]`
    is uniquely defined by it's mean-variance relationship
    :math:`\mathrm{Var}[Y] \propto y_\textrm{pred}^power`.

    Special cases are:

    ===== ================
    Power Distribution
    ===== ================
    0     Normal
    1     Poisson
    (1,2) Compound Poisson
    2     Gamma
    3     Inverse Gaussian

    Parameters
    ----------
    power : float, default=0
            The variance power of the `unit_variance`
            :math:`v(y_\textrm{pred}) = y_\textrm{pred}^{power}`.
            For ``0<power<1``, no distribution exists.
    """

    def __init__(self, power=0):
        self.power = power

    @property
    def power(self):
        return self._power

    @power.setter
    def power(self, power):
        # We use a property with a setter, to update lower and
        # upper bound when the power parameter is updated e.g. in grid
        # search.
        if not isinstance(power, numbers.Real):
            raise TypeError("power must be a real number, input was {0}".format(power))

        if power <= 0:
            # Extreme Stable or Normal distribution
            self._lower_bound = DistributionBoundary(-np.Inf, inclusive=False)
        elif 0 < power < 1:
            raise ValueError(
                "Tweedie distribution is only defined for power<=0 and power>=1."
            )
        elif 1 <= power < 2:
            # Poisson or Compound Poisson distribution
            self._lower_bound = DistributionBoundary(0, inclusive=True)
        elif power >= 2:
            # Gamma, Positive Stable, Inverse Gaussian distributions
            self._lower_bound = DistributionBoundary(0, inclusive=False)
        else:  # pragma: no cover
            # this branch should be unreachable.
            raise ValueError

        self._power = power

    def unit_variance(self, y_pred):
        """Compute the unit variance of a Tweedie distribution
        v(y_\textrm{pred})=y_\textrm{pred}**power.

        Parameters
        ----------
        y_pred : array of shape (n_samples,)
            Predicted mean.
        """
        return np.power(y_pred, self.power)

    def unit_deviance(self, y, y_pred, check_input=False):
        r"""Compute the unit deviance.

        The unit_deviance :math:`d(y,y_\textrm{pred})` can be defined by the
        log-likelihood as
        :math:`d(y,y_\textrm{pred}) = -2\phi\cdot
        \left(loglike(y,y_\textrm{pred},\phi) - loglike(y,y,\phi)\right).`

        Parameters
        ----------
        y : array of shape (n_samples,)
            Target values.

        y_pred : array of shape (n_samples,)
            Predicted mean.

        check_input : bool, default=False
            If True raise an exception on invalid y or y_pred values, otherwise
            they will be propagated as NaN.
        Returns
        -------
        deviance: array of shape (n_samples,)
            Computed deviance
        """
        p = self.power

        if check_input:
            message = (
                "Mean Tweedie deviance error with power={} can only be used on ".format(
                    p
                )
            )
            if p < 0:
                # 'Extreme stable', y any real number, y_pred > 0
                if (y_pred <= 0).any():
                    raise ValueError(message + "strictly positive y_pred.")
            elif p == 0:
                # Normal, y and y_pred can be any real number
                pass
            elif 0 < p < 1:
                raise ValueError(
                    "Tweedie deviance is only defined for power<=0 and power>=1."
                )
            elif 1 <= p < 2:
                # Poisson and compound Poisson distribution, y >= 0, y_pred > 0
                if (y < 0).any() or (y_pred <= 0).any():
                    raise ValueError(
                        message + "non-negative y and strictly positive y_pred."
                    )
            elif p >= 2:
                # Gamma and Extreme stable distribution, y and y_pred > 0
                if (y <= 0).any() or (y_pred <= 0).any():
                    raise ValueError(message + "strictly positive y and y_pred.")
            else:  # pragma: nocover
                # Unreachable statement
                raise ValueError

        if p < 0:
            # 'Extreme stable', y any real number, y_pred > 0
            dev = 2 * (
                np.power(np.maximum(y, 0), 2 - p) / ((1 - p) * (2 - p))
                - y * np.power(y_pred, 1 - p) / (1 - p)
                + np.power(y_pred, 2 - p) / (2 - p)
            )

        elif p == 0:
            # Normal distribution, y and y_pred any real number
            dev = (y - y_pred) ** 2
        elif p < 1:
            raise ValueError(
                "Tweedie deviance is only defined for power<=0 and power>=1."
            )
        elif p == 1:
            # Poisson distribution
            dev = 2 * (xlogy(y, y / y_pred) - y + y_pred)
        elif p == 2:
            # Gamma distribution
            dev = 2 * (np.log(y_pred / y) + y / y_pred - 1)
        else:
            dev = 2 * (
                np.power(y, 2 - p) / ((1 - p) * (2 - p))
                - y * np.power(y_pred, 1 - p) / (1 - p)
                + np.power(y_pred, 2 - p) / (2 - p)
            )
        return dev

class NormalDistribution(TweedieDistribution):
    """Class for the Normal (aka Gaussian) distribution."""

    def __init__(self):
        super().__init__(power=0)

class PoissonDistribution(TweedieDistribution):
    """Class for the scaled Poisson distribution."""

    def __init__(self):
        super().__init__(power=1)

class GammaDistribution(TweedieDistribution):
    """Class for the Gamma distribution."""

    def __init__(self):
        super().__init__(power=2)

class InverseGaussianDistribution(TweedieDistribution):
    """Class for the scaled InverseGaussianDistribution distribution."""

    def __init__(self):
        super().__init__(power=3)

EDM_DISTRIBUTIONS = {
    "normal": NormalDistribution,
    "poisson": PoissonDistribution,
    "gamma": GammaDistribution,
    "inverse-gaussian": InverseGaussianDistribution,
}


import os
import numpy as np
import scipy.io as sio

import torch
from torch.utils.data import DataLoader, TensorDataset

__all__ = ['Cost2100DataLoader']

class Cost2100DataLoader(object):
    r""" PyTorch DataLoader for COST2100 dataset.
    """

    def __init__(self, root, batch_size, num_workers, scenario):
        assert os.path.isdir(root), root
        assert scenario in {"in", "out"}, scenario
        self.batch_size = batch_size
        self.num_workers = num_workers

        dir_test = os.path.join(root, f"DATA_Htest{scenario}.mat")
        dir_raw = os.path.join(root, f"DATA_HtestF{scenario}_all.mat")
        channel, nt, nc, nc_expand = 2, 32, 32, 125

        # Test data loading, including the sparse data and the raw data
        data_test = sio.loadmat(dir_test)['HT']
        data_test = torch.tensor(data_test, dtype=torch.float32).view(
            data_test.shape[0], channel, nt, nc)

        raw_test = sio.loadmat(dir_raw)['HF_all']
        real = torch.tensor(np.real(raw_test), dtype=torch.float32)
        imag = torch.tensor(np.imag(raw_test), dtype=torch.float32)
        raw_test = torch.cat((real.view(raw_test.shape[0], nt, nc_expand, 1),
                              imag.view(raw_test.shape[0], nt, nc_expand, 1)), dim=3)
        self.test_dataset = TensorDataset(data_test, raw_test)

    def __call__(self):
        test_loader = DataLoader(self.test_dataset,
                                 batch_size=self.batch_size,
                                 num_workers=self.num_workers,
                                 shuffle=False)

        return test_loader


import json

config = {
  "type": "cube",
  "lcd": {
      "height": 240,
      "width": 240,
      "invert": 1,
      "dir": 96
  },
  "board_info": {
      'BOOT_KEY': 16,
      'LED_R': 13,
      'LED_G': 12,
      'LED_B': 14,
      'LED_W': 32,
      'BACK': 11,
      'ENTER': 10,
      'NEXT': 16,
      'WIFI_TX': 6,
      'WIFI_RX': 7,
      'WIFI_EN': 8,
      'I2S0_MCLK': 19,
      'I2S0_SCLK': 35,
      'I2S0_WS': 33,
      'I2S0_IN_D0': 34,
      'I2S0_OUT_D2': 18,
      'I2C_SDA': 31,
      'I2C_SCL': 30,
      'SPI_SCLK': 21,
      'SPI_MOSI': 8,
      'SPI_MISO': 15,
      'SPI_CS': 20,
  }
}

cfg = json.dumps(config)
print(cfg)

try:
  with open('/flash/config.json', 'rb') as f:
    tmp = json.loads(f.read())
    print(tmp)
    if tmp["type"] != config["type"]:
      raise Exception('config.json no exist')
except Exception as e:
  with open('/flash/config.json', "w") as f:
    f.write(cfg)
  import machine
  machine.reset()


#!/usr/bin/env python
# -- coding: utf-8 --
#%%
import os
import matplotlib.pyplot as plt
from keras.models import load_model
from keras import backend as K
import pandas
import numpy as np
#%%
x_val=[]
y_val=[]
train_data = pandas.read_csv('train.csv') #training data

for i in range(0,4000):
    temp = np.array(list(map(int, train_data.loc[i, 'feature'].split())))
    temp = temp.reshape(1, 48,48, 1)
    x_val.append(temp)
    temp = np.zeros((7, ),dtype=np.int)
    temp[int(train_data.loc[i, 'label'])] = 1
    y_val.append(temp)
x_val = np.array(x_val)
y_val = np.array(y_val)

#%%

emotion_classifier = load_model('Final_Model_v3.h5')
layer_dict = dict([layer.name, layer] for layer in emotion_classifier.layers[0:])

input_img = emotion_classifier.input
name_ls = ["conv2d_24"]
collect_layers = [ K.function([input_img, K.learning_phase()], [layer_dict[name].output]) for name in name_ls ]

choose_id = 17
photo = x_val[choose_id]
for cnt, fn in enumerate(collect_layers):
    im = fn([photo, 0]) #get the output of that layer
    fig = plt.figure(figsize=(15, 13))
    nb_filter = im[0].shape[3]
    for i in range(nb_filter):
        ax = fig.add_subplot(nb_filter/15, 15, i+1)
        ax.imshow(im[0][0, :, :, i], cmap='BuGn')
        plt.xticks(np.array([]))
        plt.yticks(np.array([]))
        plt.tight_layout()
    fig.suptitle('Output of layer{} (Given image{})'.format(cnt, choose_id))


# coding: utf-8
import json
import optparse
import sys

import requests

from acmd import USER_ERROR, SERVER_ERROR, OK
from acmd import tool, error, log
from acmd.tools.tool_utils import get_argument, get_command, create_task_id

parser = optparse.OptionParser("acmd workflows [options] [list|start]")
parser.add_option("-r", "--raw",
                  action="store_const", const=True, dest="raw",
                  help="output raw response data")

MODELS_PATH = "/etc/workflow/models.json"
INSTANCES_PATH = '/etc/workflow/instances'

@tool('workflows', ['list', 'start'])
class WorkflowsTool(object):
    """
     See: https://docs.adobe.com/docs/en/cq/5-6-1/workflows/wf-extending/wf-rest-api.html
    """

    @staticmethod
    def execute(server, argv):
        options, args = parser.parse_args(argv)
        action = get_command(args)
        actionarg = get_argument(args)

        if action == 'list' or action == 'ls':
            return list_workflows(server, options)
        elif action == 'start':
            model = actionarg
            if len(args) >= 4:
                path = get_argument(args, i=3)
                return start_workflow(server, options, model, path)
            else:
                ret = OK
                for path in sys.stdin:
                    ret = ret | start_workflow(server, options, model, path.strip())

                return ret
        else:
            error('Unknown workflows action {a}\n'.format(a=action))
            return USER_ERROR

def _get_name(model):
    name = model.replace('/etc/workflow/models/', '').\
        replace('/jcr:content/model', '')
    return name

def list_workflows(server, options):
    url = server.url(MODELS_PATH)
    resp = requests.get(url, auth=server.auth)
    if resp.status_code != 200:
        error("Unexpected error code {code}: {content}".format(
            code=resp.status_code, content=resp.content))
        return SERVER_ERROR
    data = resp.json()
    if options.raw:
        sys.stdout.write(json.dumps(data, indent=4))
    else:
        for model_item in data:
            model_name = _get_name(model_item['uri'])
            sys.stdout.write("{}\n".format(model_name))
    return OK

def _asset_path(path):
    if path.startswith('/content/dam') and not path.endswith('/jcr:content/renditions/original'):
        return path + '/jcr:content/renditions/original'
    return path

def start_workflow(server, options, model, path):
    task_id = create_task_id(model)

    form_data = dict(
        model='/etc/workflow/models/{}/jcr:content/model'.format(model),
        payload=_asset_path(path),
        payloadType='JCR_PATH',
        workflowTitle=task_id,
        startComment=''
    )
    log(_asset_path(path))

    url = server.url(INSTANCES_PATH)
    resp = requests.post(url, auth=server.auth, data=form_data)
    if resp.status_code != 201:
        error("Unexpected error code {code}: {content}".format(
            code=resp.status_code, content=resp.content))
        return SERVER_ERROR

    output = resp.content if options.raw else task_id
    sys.stdout.write("{}\n".format(output))
    return OK


from selenium.webdriver.chrome.webdriver import WebDriver
import time
import math

def calc(x):
  return str(math.log(abs(12*math.sin(int(x)))))
try:
    link = 'http://suninjuly.github.io/redirect_accept.html'
    driver = WebDriver()
    driver.get(link)

    but1 = driver.find_element_by_xpath('//button[@type="submit"]').click()
    new_window = driver.window_handles[1]
    driver.switch_to_window(new_window)
    x = driver.find_element_by_xpath('//span[@id="input_value"]').text
    y = calc(x)
    pole = driver.find_element_by_xpath('//input[@id="answer"]').send_keys(y)
    but2 = driver.find_element_by_xpath('//button[@type="submit"]').click()

finally:
    # ожидание чтобы визуально оценить результаты прохождения скрипта
    time.sleep(10)
    # закрываем браузер после всех манипуляций
    driver.quit()


#coding=utf-8
# Create your views here.
from django.shortcuts import render_to_response
from django.http import HttpResponse,HttpResponseRedirect
from login_views import login_required_user as login_required
from models import *
from forms import *
from django.conf import settings

from createbatch import *
from xml.etree import ElementTree
from hashlib import sha1
from uuid import uuid4
#from bson.objectid import ObjectId
import os
import shutil #文件复制实用工具
import time
import random
import zipfile
import urllib2
import urllib
import json

# 全局项目设置
APP_DIR = 'generate'
CSS_STYLE = 'smoothness'
MEDIA_DIR = os.path.join(settings.MEDIA_ROOT, APP_DIR)

# 发码接口全局设置
REQUEST_SERVER = 'http://10.5.8.29:8081/'
CHECK_ZONE = 'check_zone/'
CHECK_BATCH = 'check_batch/'
GET_CONTROL = 'get_control/'
GET_KLASS = 'get_klass/'
GET_PROJECT_LIST = 'get_project_list/'
GET_PROJECT_INFO = 'get_project_info/'
GET_CODE_COUNT_INFO = 'get_code_count_info/'
GET_CODE_LIST = 'get_code_list'
POST_CONTROL = 'post_control/'

# 首页视图
@login_required
def index(request):
    user_name = request.session['user_name']
    user_type = '普通用户'
    user_obj = User.objects.get(user_name = user_name)
    if user_obj.user_level == 0:
        user_type = '系统用户'

    template_map = {
        'user_name' : user_name,
        'user_type' : user_type,
        'css_style' : CSS_STYLE,
        'app_dir' : APP_DIR
    }

    return render_to_response('%s/index.html' % APP_DIR, template_map)

def go_index(request):
    return HttpResponseRedirect('/%s/index/' % APP_DIR)

@login_required
def code_add(request):
    tpl_list = []
    project_list = []

    if request.method == 'POST':
        tpl_id = request.POST.get('tpl_id',0)
        project_id = request.POST.get('project_id',0)
        gutter = request.POST.get('gutter',1)
        aspect = request.POST.get('aspect',1)
        width = request.POST.get('width',240)
        frame = request.POST.get('frame',1)
        bgcolor = request.POST.get('bgcolor','')
        border = request.POST.get('border',1)
        batch_count = request.POST.get('batch_count',1)

        user_name = request.session['user_name']
        user_obj = User.objects.get(user_name = user_name)

        project_obj = Project.objects.get(id = project_id)

        options_dict = {
            'tpl_id' : int(tpl_id),
            'user_id' : user_obj.id,
            'user_name' : user_obj.user_name,
            'user_email' : user_obj.user_email,
            'project_id' : int(project_id),
            'project_name' : project_obj.project_name,
            'batch_count' : int(batch_count),
            'gutter' : int(gutter),
            'aspect' : int(aspect),
            'width' : int(width),
            'frame' : int(frame),
            'bgcolor' : bgcolor,
            'border' : int(border)
        }

        request_data = {
            'control_str' : project_obj.project_control
        }

        request_string = urllib.urlencode(request_data)
        request_url = '%s%s?%s' % (REQUEST_SERVER, GET_CODE_COUNT_INFO, request_string)

        try:
            req = urllib2.urlopen(request_url)
            count_info = json.loads(req.read())
            sum_count = count_info['sum_count']
            use_count = count_info['use_count']
        except Exception:
            sum_count = 0
            use_count = 0

        batch_sum = (sum_count - use_count) / 4096

        if batch_sum < int(batch_count):
            return HttpResponse('您请求生成项目的彩码预分配批次不足，请确认后再次提交生成请求！')
        else:
            from tasks import *
            generate_code_image.delay(options_dict)

            return HttpResponse('彩码生成请求成功，本次将生成%d个批次，共计%d个彩码图片，系统正在处理中，彩码生成完毕后会以邮件的形式通知用户，请注意查收邮件!' % (int(batch_count), int(batch_count) * 4096))
            #return HttpResponse('sdf%dsdf%d' % (int(batch_count), 4096))

    else:
        user_name = request.session['user_name']
        user_id = User.objects.get(user_name = user_name).id

        # 获取用户所有项目列表
        project_id_list = []
        company_id_list = []

        projectlinkuser_obj_list = ProjectLinkUser.objects.filter(user_id = user_id)
        for projectlinkuser_obj in projectlinkuser_obj_list:
            if projectlinkuser_obj.project_id == '':
                company_id_list.append(projectlinkuser_obj.company_id)
            else:
                project_id_list.append(projectlinkuser_obj.project_id)

        for company_id in company_id_list:
            companylinkproject_obj_list = CompanyLinkProject.objects.filter(company_id = company_id)
            for companylinkproject_obj in companylinkproject_obj_list:
                project_id_list.append(companylinkproject_obj.project_id)

        project_id_list = {}.fromkeys(project_id_list).keys()

        for project_id in project_id_list:
            try:
                project_obj = Project.objects.get(id = int(project_id))
            except Exception:
                pass

            project_list.append(project_obj)

        # ~获取用户所有项目列表

        # 获取可用模板列表
        userlinktemplate_obj_list = UserLinkTemplate.objects.filter(user_id = user_id)

        for userlinktemplate_obj in userlinktemplate_obj_list:
            try:
                template_obj = Template.objects.get(id = userlinktemplate_obj.tpl_id, tpl_status = True)
                tpl_dict = {
                    'tpl_id' : template_obj.id,
                    'tpl_name' : template_obj.tpl_name,
                    'tpl_isopen' : template_obj.tpl_isopen
                }
                tpl_list.append(tpl_dict)
            except Exception:
                pass

        template_obj_list = Template.objects.filter(tpl_isopen = True, tpl_status = True)

        for template_obj in template_obj_list:
            tpl_dict = {
                'tpl_id' : template_obj.id,
                'tpl_name' : template_obj.tpl_name,
                'tpl_isopen' : template_obj.tpl_isopen
            }
            tpl_list.append(tpl_dict)

        # ~获取可用模板列表

    template_map = {
        'tpl_list' : tpl_list,
        'project_list' : project_list,
        'css_style' : CSS_STYLE,
        'app_dir' : APP_DIR
    }

    return render_to_response('%s/index/code_add.html' % APP_DIR, template_map)

@login_required
def code_list(request):
    try:
        current_page = int(request.GET['page'])
    except Exception:
        current_page = 1

    user_name = request.session['user_name']
    user_id = User.objects.get(user_name = user_name).id

    # 每页显示行数
    count_page = 15
    # 列表总条数
    sum_list = len(GenerateRecord.objects.filter(user_id = user_id))
    # 总页数
    sum_page = sum_list / count_page

    if sum_list % count_page != 0:
        sum_page = sum_page + 1

    if current_page < 1:
        current_page = 1

    if current_page > sum_page and sum_page != 0:
        current_page = sum_page

    start_index = (current_page - 1) * count_page
    end_index = current_page * count_page

    record_obj_list = GenerateRecord.objects.filter(user_id = user_id)[start_index : end_index]
    record_list = []

    count = 0
    flag = True

    for record in record_obj_list:
        project_obj = Project.objects.get(id = int(record.project_id))

        if count % 2 == 0:
            flag = True
        else:
            flag = False

        record_dict = {
            'project_name' : project_obj.project_name,
            'batch_count' : record.batch_count,
            'code_sum' : record.batch_count * 4096,
            'uuid' : record.generate_dir,
            'generate_date' : record.generate_date,
            'flag' : flag
        }
        record_list.append(record_dict)

        count = count + 1

    template_map = {
        'record_list' : record_list,
        'css_style' : CSS_STYLE,
        'app_dir' : APP_DIR,
        'sum_page' : sum_page,
        'count_page' : count_page,
        'current_page' : current_page
    }

    return render_to_response('%s/index/code_list.html' % APP_DIR, template_map)

@login_required
def tpl_add(request):
    tpl_form = TemplateForm()
    tpl_name = request.POST.get('tpl_name', '')
    bRet = False

    if request.method == 'POST':
        tpl_form = TemplateForm(request.POST)

        if tpl_form.is_valid():

            bRet = tpl_form.upload(request)

            if bRet:
                tpl_form.new(request)

    template_map = {
        'tpl_form' : tpl_form,
        'bRet' : bRet,
        'tpl_name' : tpl_name,
        'css_style' : CSS_STYLE,
        'app_dir' : APP_DIR
    }

    return render_to_response('%s/index/tpl_add.html' % APP_DIR, template_map)

@login_required
def tpl_list(request):
    user_name = request.session['user_name']
    user_id = User.objects.get(user_name = user_name).id

    ult_obj_list = UserLinkTemplate.objects.filter(user_id = user_id)

    tpl_obj_list = []

    for ult_obj in ult_obj_list:
        try:
            tpl_obj = Template.objects.get(id = ult_obj.tpl_id)
            tpl_obj_list.append(tpl_obj)
        except Exception:
            pass

    try:
        current_page = int(request.GET['page'])
    except Exception:
        current_page = 1

    # 每页显示行数
    count_page = 15
    # 列表总条数
    sum_list = len(ult_obj_list)
    # 总页数
    sum_page = sum_list / count_page

    if sum_list % count_page != 0:
        sum_page = sum_page + 1

    if current_page < 1:
        current_page = 1

    if current_page > sum_page and sum_page != 0:
        current_page = sum_page

    start_index = (current_page - 1) * count_page
    end_index = current_page * count_page

    tpl_obj_list = tpl_obj_list[start_index : end_index]

    template_map = {
        'tpl_list' : tpl_obj_list,
        'count_page' : count_page,
        'current_page' : current_page,
        'sum_page' : sum_page,
        'css_style' : CSS_STYLE,
        'app_dir' : APP_DIR
    }

    return render_to_response('%s/index/tpl_list.html' % APP_DIR, template_map)

@login_required
def tpl_delete(request):
    tpl_name = request.GET.get('tpl_name','')

    tpl_obj = Template.objects.get(tpl_name = tpl_name)
    tpl_id = tpl_obj.id
    tpl_obj.delete()
    UserLinkTemplate.objects.get(tpl_id = tpl_id).delete()

    return HttpResponse('ok')

@login_required
def tpl_status(request):
    tpl_name = request.GET.get('tpl_name','')

    tpl_obj = Template.objects.get(tpl_name = tpl_name)
    if tpl_obj.tpl_status is True:
        tpl_obj.tpl_status = False
    else:
        tpl_obj.tpl_status = True

    tpl_obj.save()

    return HttpResponse('ok')

@login_required
def tpl_open(request):
    tpl_name = request.GET.get('tpl_name','')

    tpl_obj = Template.objects.get(tpl_name = tpl_name)
    if tpl_obj.tpl_isopen is True:
        tpl_obj.tpl_isopen = False
    else:
        tpl_obj.tpl_isopen = True

    tpl_obj.save()

    return HttpResponse('ok')

@login_required
def get_batch_sum(request):
    project_id = request.GET.get('project_id', 0)

    project_obj = Project.objects.get(id = int(project_id))

    request_data = {
        'control_str' : project_obj.project_control
    }

    request_string = urllib.urlencode(request_data)
    request_url = '%s%s?%s' % (REQUEST_SERVER, GET_CODE_COUNT_INFO, request_string)

    try:
        req = urllib2.urlopen(request_url)
        count_info = json.loads(req.read())
        sum_count = count_info['sum_count']
        use_count = count_info['use_count']
    except Exception:
        sum_count = 0
        use_count = 0

    batch_sum = (sum_count - use_count) / 4096

    return  HttpResponse(batch_sum)

@login_required
def get_tpl_name(request):
    tpl_id = request.GET.get('tpl_id', 0)

    tpl_obj = Template.objects.get(id = int(tpl_id))

    return HttpResponse(tpl_obj.tpl_name)

def get_preview(request):
    uuid = request.GET.get('uuid', '')

    preview_count = 20

    code_obj_list = []

    if uuid != '':
        code_obj_list = Code.objects.filter(code_image_addr__contains = uuid)[:preview_count]

    template_map = {
        'code_list' : code_obj_list,
        'css_style' : CSS_STYLE,
        'app_dir' : APP_DIR
    }

    return render_to_response('%s/get_preview.html' % APP_DIR, template_map)

@login_required
def get_downloads(request):
    uuid = request.GET.get('uuid', '')

    download_root  = os.path.join(MEDIA_DIR, 'downloads')

    file_list = []

    if uuid != '':
        download_dir = os.path.join(download_root, uuid)
        file_list = os.listdir(download_dir)

    template_map = {
        'file_list' : file_list,
        'uuid' : uuid,
        'css_style' : CSS_STYLE,
        'app_dir' : APP_DIR
    }

    return render_to_response('%s/get_downloads.html' % APP_DIR, template_map)


import pytest
import mock

import requests
import requests_mock

import acme.tools.retry as retry

class TestRetry:
    def test__no_retry(self):
        @retry.retry(Exception, 2)
        def f(v):
            v[0] -= 1
        #
        value = [2]
        f(value)
        # one call without retries
        assert value[0] == 1

    def test__exception_not_match(self):
        @retry.retry(AttributeError, 2)
        def f(v):
            v[0] -= 1
            raise RuntimeError
        #
        with pytest.raises(RuntimeError):
            value = [3]
            f(value)
        # one call without retries
        assert value[0] == 2

    def test__0_attempts(self):
        @retry.retry(Exception, 0)
        def f(v):
            v[0] -= 1
            raise Exception
        #
        with pytest.raises(Exception):
            value = [1]
            f(value)
        # one call - ignore zero
        assert value[0] == 0

    def test__1_attempt(self):
        @retry.retry(Exception, 1)
        def f(v):
            v[0] -= 1
            raise Exception
        #
        with pytest.raises(Exception):
            value = [1]
            f(value)
        # one call
        assert value[0] == 0

    def test__2_attempts(self):
        @retry.retry(Exception, 2)
        def f(v):
            v[0] -= 1
            raise Exception
        #
        with pytest.raises(Exception):
            value = [2]
            f(value)
        # two calls
        assert value[0] == 0

    def test__default_attempts(self):
        @retry.retry(Exception)
        def f(v):
            v[0] -= 1
            raise Exception
        #
        with pytest.raises(Exception):
            value = [3]
            f(value)
        # three calls
        assert value[0] == 0

class TestRetryWrapper:
    def test__args(self):
        def f(i):
            return i
        assert retry.wrapper(
            context={
                'function': f,
                'args': (1,),
            },
        ) == 1

    def test__kwargs(self):
        def f(i):
            return i
        assert retry.wrapper(
            context={
                'function': f,
                'kwargs': {'i': 1},
            },
        ) == 1

    def test__exception_not_match(self):
        def f(v):
            v[0] += 1
            raise AttributeError
        with pytest.raises(AttributeError):
            value = [0]
            retry.wrapper(
                context={
                    'function': f,
                    'args': (value,),
                },
                attempts=10,
                exceptions=RuntimeError,
            )
        assert value[0] == 1

    def test__1_attempts(self):
        def f(v):
            v[0] += 1
            raise AttributeError
        with pytest.raises(AttributeError):
            value = [0]
            retry.wrapper(
                context={
                    'function': f,
                    'args': (value,),
                },
                attempts=1,
                exceptions=AttributeError,
            )
        assert value[0] == 1

    def test__2_attempts(self):
        def f(v):
            v[0] += 1
            raise AttributeError
        with pytest.raises(AttributeError):
            value = [0]
            retry.wrapper(
                context={
                    'function': f,
                    'args': (value,),
                },
                attempts=2,
                exceptions=AttributeError,
            )
        assert value[0] == 2

class TestRequestsRetryWrapper:
    def test__no_raise(self):
        with requests_mock.mock() as m:
            m.get(
                'http://example.com/',
                text='OK',
                status_code=200,
            )
            #
            result = retry.requests_wrapper(
                context={
                    'function': 'get',
                    'args': (
                        'http://example.com/',
                    ),
                },
            )
            assert result.ok
            assert result.text == 'OK'

    def test__raises(self):
        __RETRY_SLEEP = retry.RETRY_SLEEP
        retry.RETRY_SLEEP = 0.0
        with pytest.raises(requests.exceptions.RequestException):
            result = retry.requests_wrapper(
                context={
                    'function': 'get',
                    'args': (
                        'invalid',
                    ),
                },
            )
        retry.RETRY_SLEEP = __RETRY_SLEEP

    def test__not_ok(self, monkeypatch):
        with requests_mock.mock() as m:
            m.get(
                'http://example.com/',
                text='ERROR',
                status_code=404,
            )
            #
            __RETRY_SLEEP = retry.RETRY_SLEEP
            retry.RETRY_SLEEP = 0.0
            result = retry.requests_wrapper(
                context={
                    'function': 'get',
                    'args': (
                        'http://example.com/',
                    ),
                },
            )
            retry.RETRY_SLEEP = __RETRY_SLEEP
            assert not result.ok
            assert result.text == 'ERROR'


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import os

# Get the path of the script
file_path = os.path.dirname(os.path.realpath(__file__))

font = {'family' : 'normal',
        'weight' : 'normal',
        'size'   : 15}

matplotlib.rc('font', **font)

data = pd.read_excel(file_path + '/../../results.xlsx','FigS2-S3',index_col=0)
biomass = pd.read_excel(file_path + '/../../results.xlsx','Table1 & Fig1',index_col=[0,1])
ind_num = pd.read_excel(file_path + '/../../results.xlsx','Table S1',index_col=[0,1])
data.loc['Plants','Number of individuals'] = np.nan
data.loc['Chordates','Number of individuals'] = np.nan

data.loc['Bacteria',['Biomass [Gt C]','Uncertainty']] = [biomass.loc['Bacteria','Biomass [Gt C]'].sum(),biomass.loc[('Bacteria','Terrestrial deep subsurface'),'Total uncertainty']]
data.loc['Archaea',['Biomass [Gt C]','Uncertainty']] = [biomass.loc['Archaea','Biomass [Gt C]'].sum(),biomass.loc[('Archaea','Terrestrial deep subsurface'),'Total uncertainty']]
data.loc['Plants',['Biomass [Gt C]','Uncertainty']] = [biomass.loc[('Plants','Plants'),'Biomass [Gt C]'],biomass.loc[('Plants','Plants'),'Uncertainty']]
data.loc['Plants (trees)',['Biomass [Gt C]','Uncertainty']] = [biomass.loc[('Plants','Plants'),'Biomass [Gt C]'],biomass.loc[('Plants','Plants'),'Uncertainty']]
data.loc['Fungi',['Biomass [Gt C]','Uncertainty']] = [biomass.loc['Fungi','Biomass [Gt C]'].sum(),biomass.loc[('Fungi','Terrestrial'),'Total uncertainty']]
data.loc['Arthropods',['Biomass [Gt C]']] = biomass.loc[[('Animals','Terrestrial arthropods'),('Animals','Marine arthropods')],'Biomass [Gt C]'].sum()
data.loc['Annelids','Biomass [Gt C]'] = biomass.loc[('Animals','Annelids'),'Biomass [Gt C]']
data.loc['Molluscs','Biomass [Gt C]'] = biomass.loc[('Animals','Molluscs'),'Biomass [Gt C]']
data.loc['Fish',['Biomass [Gt C]','Uncertainty']] = [biomass.loc[('Animals','Fish'),'Biomass [Gt C]'],biomass.loc[('Animals','Fish'),'Uncertainty']]
data.loc['Nematodes','Biomass [Gt C]'] = biomass.loc[('Animals','Nematodes'),'Biomass [Gt C]']
data.loc['Humans','Biomass [Gt C]'] = biomass.loc[('Animals','Humans'),'Biomass [Gt C]']
data.loc['Livestock','Biomass [Gt C]'] = biomass.loc[('Animals','Livestock'),'Biomass [Gt C]']
data.loc['Cnidarians','Biomass [Gt C]'] = biomass.loc[('Animals','Cnidarians'),'Biomass [Gt C]']
data.loc['Protists',['Biomass [Gt C]','Uncertainty']] = [biomass.loc['Fungi','Biomass [Gt C]'].sum(),biomass.loc[('Protists','Marine'),'Total uncertainty']]
data.loc['Wild birds','Biomass [Gt C]'] = biomass.loc[('Animals','Wild birds'),'Biomass [Gt C]']
data.loc['Chordates','Biomass [Gt C]'] = biomass.loc[('Animals',['Fish','Livestock','Humans','Wild mammals','Wild birds']),'Biomass [Gt C]'].sum()

data.loc['Bacteria','Number of individuals'] = ind_num.loc['Bacteria','Number of individuals'].sum()
data.loc['Archaea','Number of individuals'] = ind_num.loc['Archaea','Number of individuals'].sum()
data.loc['Plants (trees)','Number of individuals'] = ind_num.loc['Plants','Number of individuals'].sum()
data.loc['Fungi','Number of individuals'] = ind_num.loc['Fungi','Number of individuals'].sum()
data.loc['Arthropods','Number of individuals'] = ind_num.loc[[('Animals','Terrestrial arthropods'),('Animals','Marine arthropods')],'Number of individuals'].sum()
data.loc['Annelids','Number of individuals'] = ind_num.loc[('Animals','Annelids'),'Number of individuals']
data.loc['Molluscs','Number of individuals'] = ind_num.loc[('Animals','Molluscs'),'Number of individuals']
data.loc['Cnidarians','Number of individuals'] = ind_num.loc[('Animals','Cnidarians'),'Number of individuals']
data.loc['Fish','Number of individuals'] = ind_num.loc[('Animals','Fish'),'Number of individuals']
data.loc['Nematodes','Number of individuals'] = ind_num.loc[('Animals','Nematodes'),'Number of individuals']
data.loc['Humans','Number of individuals'] = ind_num.loc[('Animals','Humans'),'Number of individuals']
data.loc['Livestock','Number of individuals'] = ind_num.loc[('Animals','Livestock'),'Number of individuals']
data.loc['Wild birds','Number of individuals'] = ind_num.loc[('Animals','Wild birds'),'Number of individuals']
data.loc['Protists','Number of individuals'] = ind_num.loc['Protists','Number of individuals'].sum()

lower_error = data['Biomass [Gt C]'] - data['Biomass [Gt C]']/data['Uncertainty']
upper_error = data['Biomass [Gt C]']*data['Uncertainty'] - data['Biomass [Gt C]']

# Fig S3
figs2 = data[~pd.isnull(data['Number of species'])]
figs3 = data[~pd.isnull(data['Number of individuals'])]

plt.figure()
ax= plt.gca()

plt.errorbar(figs2['Number of species'],
             figs2['Biomass [Gt C]'],
             yerr=[lower_error.loc[figs2.index],upper_error.loc[figs2.index]],
             axes=ax,
             marker='.',
             capthick=1.5,
             elinewidth=1.5,
             ecolor='#bfbfbf',
             ms=20,
             fmt='',
             ls='None',
             color='k')

ax.set_xscale("log", nonposx='clip')
ax.set_yscale("log", nonposy='clip')
for txt in (data[data['Number of species'] != np.nan].index):
    ax.annotate(txt,(data.loc[txt,'Number of species']*1.1,data.loc[txt,'Biomass [Gt C]']*1.11))
plt.ylabel('Biomass [Gt C]', fontsize=15)
plt.xlabel('Number of species', fontsize=15)
plt.xlim([1e3,1e7])

plt.title('Figure S3')
plt.savefig(file_path + '/../output/figS3.pdf')
plt.savefig(file_path + '/../output/figS3.svg')

# Fig S2

plt.figure()
ax= plt.gca()

plt.errorbar(figs3['Number of individuals'],
             figs3['Biomass [Gt C]'],
             yerr=[lower_error.loc[figs3.index],upper_error.loc[figs3.index]],
             axes=ax,
             marker='.',
             capthick=1.5,
             elinewidth=1.5,
             ecolor='#bfbfbf',
             ms=20,
             fmt='',
             ls='None',
             color='k')

ax.set_xscale("log", nonposx='clip')
ax.set_yscale("log", nonposy='clip')
for txt in (data[data['Number of individuals'] != np.nan].index):
    ax.annotate(txt,(data.loc[txt,'Number of individuals']*1.1,data.loc[txt,'Biomass [Gt C]']*1.1))
plt.ylabel('Biomass [Gt C]', fontsize=15)
plt.xlabel('Number of individuals', fontsize=15)
plt.xticks([1e10,1e15,1e20,1e25,1e30])
plt.title('Figure S2')
plt.savefig(file_path + '/../output/figS2.pdf')
plt.savefig(file_path + '/../output/figS2.svg')

plt.show()


# Copyright 2017-2019 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import fnmatch
import csv
import hashlib
import logging
import re
import os
import shlex
import shutil
import struct
import sys
import threading
import time

import six
from six.moves import shlex_quote

import yaml

# Move any import that's expensive or seldom used into function
from guild import util

log = logging.getLogger("guild")

# Legacy support for functionality moved to _api
from guild import _api
NoCurrentRun = _api.NoCurrentRun
current_run = _api.current_run

function_pattern = re.compile(r"([a-zA-Z0-9_\-\.]*)\[(.*)\]\s*$")
function_arg_delimiter = ":"

RESTART_NEEDED_STATUS = ("pending",)

class ArgValueError(ValueError):

    def __init__(self, arg):
        super(ArgValueError, self).__init__(arg)
        self.arg = arg

class MissingRequiredFlags(ValueError):

    def __init__(self, missing):
        super(MissingRequiredFlags, self).__init__(missing)
        self.missing = missing

class InvalidFlagChoice(ValueError):

    def __init__(self, val, flag):
        super(InvalidFlagChoice, self).__init__(val, flag)
        self.val = val
        self.flag = flag

class InvalidFlagValue(ValueError):

    def __init__(self, val, flag, msg):
        super(InvalidFlagValue, self).__init__(val, flag, msg)
        self.val = val
        self.flag = flag
        self.msg = msg

class ProcessError(Exception):
    pass

class RunOutput(object):

    DEFAULT_WAIT_TIMEOUT = 10

    def __init__(self, run, proc=None, quiet=False, output_cb=None):
        assert run
        self._run = run
        self._quiet = quiet
        self._output_cb = output_cb
        self._output_lock = threading.Lock()
        self._open = False
        self._proc = None
        self._output = None
        self._index = None
        self._out_tee = None
        self._err_tee = None
        if proc:
            self.open(proc)

    @property
    def closed(self):
        return not self._open

    def open(self, proc):
        self._assert_closed()
        if proc.stdout is None:
            raise RuntimeError("proc stdout must be a PIPE")
        if proc.stderr is None:
            raise RuntimeError("proc stderr must be a PIPE")
        self._proc = proc
        self._output = self._open_output()
        self._index = self._open_index()
        self._out_tee = threading.Thread(target=self._out_tee_run)
        self._err_tee = threading.Thread(target=self._err_tee_run)
        self._out_tee.start()
        self._err_tee.start()
        self._open = True

    def _assert_closed(self):
        if self._open:
            raise RuntimeError("already open")
        assert self._proc is None
        assert self._output is None
        assert self._index is None
        assert self._out_tee is None
        assert self._err_tee is None

    def _open_output(self):
        path = self._run.guild_path("output")
        return open(path, "wb")

    def _open_index(self):
        path = self._run.guild_path("output.index")
        return open(path, "wb")

    def _out_tee_run(self):
        assert self._proc
        self._gen_tee_run(self._proc.stdout, sys.stdout, 0)

    def _err_tee_run(self):
        assert self._proc
        self._gen_tee_run(self._proc.stderr, sys.stderr, 1)

    def _gen_tee_run(self, input_stream, output_stream, stream_type):
        assert self._output
        assert self._index
        os_read = os.read
        os_write = os.write
        input_fileno = input_stream.fileno()
        if not self._quiet:
            stream_fileno = output_stream.fileno()
        else:
            stream_fileno = None
        output_fileno = self._output.fileno()
        index_fileno = self._index.fileno()
        time_ = time.time
        lock = self._output_lock
        line = []
        while True:
            b = os_read(input_fileno, 1)
            if not b:
                break
            with lock:
                if stream_fileno is not None:
                    os_write(stream_fileno, b)
                if b < b"\x09": # non-printable
                    continue
                line.append(b)
                if b == b"\n":
                    line_bytes = b"".join(line)
                    os_write(output_fileno, line_bytes)
                    line = []
                    entry = struct.pack(
                        "!QB", int(time_() * 1000), stream_type)
                    os_write(index_fileno, entry)
                    if self._output_cb:
                        try:
                            self._output_cb.write(line_bytes)
                        except Exception:
                            log.exception(
                                "error in output callback (will be removed)")
                            self._output_cb = None

    def wait(self, timeout=DEFAULT_WAIT_TIMEOUT):
        self._assert_open()
        self._out_tee.join(timeout)
        self._err_tee.join(timeout)

    def _assert_open(self):
        if not self._open:
            raise RuntimeError("not open")
        assert self._proc
        assert self._output
        assert self._index
        assert self._out_tee
        assert self._err_tee

    def close(self):
        lock = self._acquire_output_lock()
        try:
            self._close()
        finally:
            lock.release()

    def _acquire_output_lock(self, timeout=60):
        """Polling verison of acquire to support timeouts on Python 2."""
        timeout_at = time.time() + timeout
        while time.time() < timeout_at:
            if self._output_lock.acquire(False):
                return self._output_lock
            time.sleep(1)
        raise RuntimeError("timeout")

    def _close(self):
        self._assert_open()
        self._output.close()
        self._index.close()
        if self._output_cb:
            try:
                self._output_cb.close()
            except Exception:
                log.exception("closing output callback")
        assert not self._out_tee.is_alive()
        assert not self._err_tee.is_alive()
        self._proc = None
        self._output = None
        self._index = None
        self._out_tee = None
        self._err_tee = None
        self._open = False

    def wait_and_close(self, timeout=DEFAULT_WAIT_TIMEOUT):
        self.wait(timeout)
        self.close()

def resolve_file(filename):
    return util.find_apply([
        _abs_file,
        _cmd_file,
        _model_file,
        _cwd_file
    ], filename)

def _abs_file(filename):
    if os.path.isabs(filename):
        return filename
    return None

def _cmd_file(filename):
    assert "CMD_DIR" in os.environ
    filename = os.path.join(os.environ["CMD_DIR"], filename)
    if os.path.exists(filename):
        return filename
    return None

def parse_flags(args):
    return dict([parse_flag_arg(os.path.expanduser(arg)) for arg in args])

def parse_flag_arg(arg):
    parts = arg.split("=", 1)
    if len(parts) == 1:
        raise ArgValueError(arg)
    else:
        return parts[0], parse_arg_val(parts[1])

def parse_arg_val(s):
    if s == "":
        return s
    parsers = [
        (int, ValueError),
        (float, ValueError),
        (_yaml_parse, (ValueError, yaml.YAMLError)),
    ]
    for p, e_type in parsers:
        try:
            return p(s)
        except e_type:
            pass
    return s

def _yaml_parse(s):
    """Uses yaml module to parse s to a Python value.

    First tries to parse as an unnamed flag function with at least two
    args and, if successful, returns s unmodified. This prevents yaml
    from attempting to parse strings like '1:1' which it considers to
    be timestamps.
    """
    try:
        name, args = parse_function(s)
    except ValueError:
        pass
    else:
        if name is None and len(args) >= 2:
            return s
    return yaml.safe_load(s)

def format_flag_val(val):
    if val is True:
        return "yes"
    elif val is False:
        return "no"
    elif val is None:
        return "null"
    elif isinstance(val, list):
        return _format_flag_list(val)
    elif isinstance(val, (six.string_types, float)):
        return _yaml_format(val)
    else:
        return str(val)

def _format_flag_list(val_list):
    joined = ", ".join([format_flag_val(val) for val in val_list])
    return "[%s]" % joined

def _yaml_format(val):
    formatted = yaml.safe_dump(val).strip()
    if formatted.endswith("\n..."):
        formatted = formatted[:-4]
    return formatted

def format_flag_arg(name, val):
    return "%s=%s" % (name, format_flag_val(val))

class TFEvents(object):

    def __init__(self, logdir):
        self.logdir = logdir
        self._writer = None

    def add_scalars(self, scalars, global_step=None):
        self._ensure_writer()
        self._writer.add_summary(self._scalars_summary(scalars), global_step)

    @staticmethod
    def _scalars_summary(scalars):
        import tensorflow as tf
        value = [
            tf.summary.Summary.Value(tag=tag, simple_value=val)
            for tag, val in scalars
        ]
        return tf.summary.Summary(value=value)

    def _ensure_writer(self):
        import tensorflow as tf
        if not self._writer:
            self._writer = tf.summary.FileWriter(self.logdir, max_queue=0)

    def flush(self):
        if self._writer:
            self._writer.flush()

    def close(self):
        if self._writer:
            self._writer.close()
            self._writer = None

    def __enter__(self):
        return self

    def __exit__(self, *_args):
        self.close()

def tfevents(subdir=None, run=None):
    if not run:
        run = current_run()
    if subdir:
        logdir = os.path.join(run.path, subdir)
    else:
        logdir = run.path
    return TFEvents(logdir)

def exit(msg, exit_status=1):
    """Exit the Python runtime with a message.
    """
    sys.stderr.write(os.path.basename(sys.argv[0]))
    sys.stderr.write(": ")
    sys.stderr.write(msg)
    sys.stderr.write("\n")
    sys.exit(exit_status)

def parse_op_args(args):
    if len(args) < 2:
        exit("usage: %s COMMAND [ARG...]" % args[0])
    return args[1], args[2:]

def args_to_flags(args):
    flags, _args = args_to_flags2(args)
    return flags

def args_to_flags2(args):
    flags = {}
    extra = []
    name = None
    for arg in args:
        if arg[:2] == "--":
            name = arg[2:]
            flags[name] = True
        elif arg[:1] == "-":
            val = parse_arg_val(arg)
            if isinstance(val, (int, float)):
                flags[name] = val
            elif len(arg) == 2:
                name = arg[1]
                flags[name] = True
            elif len(arg) > 2:
                name = None
                flags[arg[1]] = arg[2:]
        elif name is not None:
            flags[name] = parse_arg_val(arg)
            name = None
        else:
            extra.append(arg)
    return flags, extra

def global_dest(global_name, flags):
    dest = cur = {}
    for name in global_name.split("."):
        cur = cur.setdefault(name, {})
    cur.update(flags)
    return dest

def find_file(path):
    return util.find_apply([_cwd_file, _model_file], path)

def _cwd_file(path):
    if os.path.exists(path):
        return path
    return None

def _model_file(path):
    model_path = os.getenv("MODEL_PATH")
    if model_path:
        for root in model_path.split(os.path.pathsep):
            full_path = os.path.join(root, path)
            if os.path.exists(full_path):
                return full_path
    return None

def coerce_flag_value(val, flagdef):
    """Coerces a flag value based on flagdef settings."""
    if val is None or not flagdef or not flagdef.type:
        return val
    if isinstance(val, list):
        return [coerce_flag_value(x, flagdef) for x in val]
    elif flagdef.type == "string":
        return _try_coerce_flag_val(val, str, flagdef)
    elif flagdef.type == "int":
        if isinstance(val, float):
            raise ValueError("invalid value for type 'int'")
        return _try_coerce_flag_val(val, int, flagdef)
    elif flagdef.type == "float":
        return _try_coerce_flag_val(val, float, flagdef)
    elif flagdef.type == "number":
        if isinstance(val, (float, int)):
            return val
        return _try_coerce_flag_val(val, (int, float), flagdef)
    elif flagdef.type in ("path", "existing-path"):
        return _resolve_rel_path(val)
    else:
        log.warning(
            "unknown flag type '%s' for %s - cannot coerce",
            flagdef.type, flagdef.name)
        return val

def _try_coerce_flag_val(val, funs, flagdef):
    if not isinstance(funs, tuple):
        funs = (funs,)
    for f in funs:
        try:
            return f(val)
        except ValueError as e:
            log.debug("value error applying %s to %r: %s", f, val, e)
    raise ValueError("invalid value for type '%s'" % flagdef.type)

def _resolve_rel_path(val):
    if val and not os.path.isabs(val):
        return os.path.abspath(val)
    return val

def validate_flag_vals(vals, opdef):
    _check_missing_flags(vals, opdef)
    _check_flag_vals(vals, opdef)

def _check_missing_flags(vals, opdef):
    missing = _missing_flags(vals, opdef)
    if missing:
        raise MissingRequiredFlags(missing)

def _missing_flags(vals, opdef):
    return [
        flag for flag in opdef.flags
        if flag.required and _flag_missing(vals.get(flag.name))
    ]

def _flag_missing(val):
    if val is None or val == "":
        return True
    return False

def _check_flag_vals(vals, opdef):
    for flag in opdef.flags:
        val = vals.get(flag.name)
        _check_flag_choice(val, flag)
        _check_flag_type(val, flag)
        _check_flag_range(val, flag)

def _check_flag_choice(val, flag):
    if (val and flag.choices and not flag.allow_other and
        val not in [choice.value for choice in flag.choices]):
        raise InvalidFlagChoice(val, flag)

def _check_flag_type(val, flag):
    if flag.type == "existing-path":
        if val and not os.path.exists(val):
            raise InvalidFlagValue(val, flag, "%s does not exist" % val)

def _check_flag_range(val, flag):
    if val is None:
        return
    if flag.min is not None and val < flag.min:
        raise InvalidFlagValue(
            val, flag, "out of range (less than min %s)" % flag.min)
    if flag.max is not None and val > flag.max:
        raise InvalidFlagValue(
            val, flag, "out of range (greater than max %s)" % flag.max)

def copy_source(run, opdef):
    _copy_source(
        opdef.guildfile.dir,
        opdef.modeldef.source,
        run.guild_path("source"))

def _copy_source(src_base, source_config, dest_base):
    to_copy = _source_to_copy(src_base, source_config)
    if not to_copy:
        log.debug("no source to copy")
        return
    for src, src_rel_path in to_copy:
        dest = os.path.join(dest_base, src_rel_path)
        log.debug("copying source %s to %s", src, dest)
        util.ensure_dir(os.path.dirname(dest))
        _try_copy_file(src, dest)

def _source_to_copy(src_dir, source_config):
    to_copy = []
    seen_dirs = set()
    for root, dirs, files in os.walk(src_dir, followlinks=True):
        seen_dirs.add(os.path.realpath(root))
        _del_excluded_dirs(dirs, root, seen_dirs)
        for name in files:
            path = os.path.join(root, name)
            rel_path = os.path.relpath(path, src_dir)
            if _to_copy(path, rel_path, source_config):
                to_copy.append((path, rel_path))
    return to_copy

def _try_copy_file(src, dest):
    try:
        shutil.copyfile(src, dest)
    except (IOError, OSError) as e:
        # This is not an error we want to stop an operation for. Log
        # and continue.
        if log.getEffectiveLevel() <= logging.DEBUG:
            log.exception("copy %s to %s", src, dest)
        else:
            log.warning("could not copy source file %s: %s", src, e)

def _del_excluded_dirs(dirs, root, seen_dirs):
    _del_env_dirs(dirs, root)
    _del_dot_dir(dirs)
    _del_seen_dirs(dirs, root, seen_dirs)

def _del_env_dirs(dirs, root):
    for name in dirs:
        if _is_env_dir(os.path.join(root, name)):
            dirs.remove(name)

def _del_dot_dir(dirs):
    for d in list(dirs):
        if d[:1] == ".":
            dirs.remove(d)

def _del_seen_dirs(dirs, root, seen):
    for dir_name in dirs:
        real_path = os.path.realpath(os.path.join(root, dir_name))
        if real_path in seen:
            dirs.remove(dir_name)

def _is_env_dir(path):
    return os.path.exists(os.path.join(path, "bin", "activate"))

def _to_copy(path, rel_path, source_config):
    last_match = None
    for spec in source_config.specs:
        if _source_match(rel_path, spec):
            last_match = spec
    if last_match:
        return _to_copy_for_spec(last_match)
    return _is_text_file(path)

def _source_match(rel_path, spec):
    return any((fnmatch.fnmatch(rel_path, p) for p in spec.patterns))

def _to_copy_for_spec(spec):
    return spec.type == "include"

def _is_text_file(path):
    from guild import binaryornot
    return not binaryornot.is_binary(path)

def split_main(main):
    if isinstance(main, list):
        return main
    # If main is None, this call will block (see
    # https://bugs.python.org/issue27775)
    return shlex.split(main or "")

# Alias
split_cmd = split_main

def wait_for_proc(p, stop_after_min, poll_interval=5, kill_delay=30):
    stop_at = time.time() + stop_after_min * 60
    while time.time() < stop_at:
        time.sleep(poll_interval)
        returncode = p.poll()
        if returncode is not None:
            return returncode
    log.info(
        "Stopping process early (pid %i) - %i minute(s) elapsed",
        p.pid, stop_after_min)
    return _terminate(p, poll_interval, kill_delay)

def _terminate(p, poll_interval, kill_delay):
    kill_at = time.time() + kill_delay
    p.terminate()
    while p.poll() is None and time.time() < kill_at:
        time.sleep(poll_interval)
    if p.poll() is None:
        log.warning("Process did not terminate (pid %i), killing", p.pid)
        p.kill()
        time.sleep(poll_interval)
    returncode = p.poll()
    if returncode not in (0, -15):
        raise ProcessError(
            "Process did not terminate gracefully (pid %i)"
            % p.pid)
    return returncode

def init_logging():
    import guild.log
    level = int(os.getenv("LOG_LEVEL", logging.WARN))
    format = os.getenv("LOG_FORMAT", "%(levelname)s: [%(name)s] %(message)s")
    guild.log.init_logging(level, {"_": format})
    globals()["log"] = logging.getLogger("guild")

def print_trials(trials):
    from guild import cli
    data, cols = _trials_table_data(trials)
    cli.table(data, cols)

def _trials_table_data(trials):
    names = set()
    data = []
    for i, flags in enumerate(trials):
        row = {"_trial": i + 1}
        data.append(row)
        if flags:
            row.update(
                {name: format_flag_val(flags[name])
                 for name in flags})
            names.update(flags)
    heading = {name: name for name in names}
    heading["_trial"] = "#"
    return [heading] + data, ["_trial"] + sorted(names)

def save_trials(trials, path):
    data, cols = _trials_table_data(trials)
    cols.remove("_trial") # Don't include trial number in CSV
    with open(path, "w") as f:
        out = csv.writer(f)
        for row in data:
            out.writerow([row.get(name, "") for name in cols])

def op_flag_encoder(op):
    import importlib
    spec = op.opdef.flag_encoder
    if not spec:
        return None
    parts = spec.split(":")
    if len(parts) != 2:
        log.warning(
            "invalid flag decoder %r - must be MODULE:FUNCTION",
            spec)
        return None
    mod_name, fun_name = parts
    try:
        mod = importlib.import_module(mod_name)
    except Exception as e:
        if log.getEffectiveLevel() <= logging.DEBUG:
            log.exception("importing %s", mod_name)
        else:
            log.warning(
                "cannot load flag decoder %r: %s",
                spec, e)
        return None
    fun = getattr(mod, fun_name, None)
    if fun is None:
        log.warning(
            "cannot load flag decoder %r: no such attribute in %s",
            spec, mod_name)
        return None
    return fun

def ensure_exit_status(run, exit_status):
    from guild import op as oplib
    run_exit_status = run.get("exit_status")
    if run_exit_status is None:
        run.write_attr("exit_status", exit_status)
    oplib.delete_pending(run)

def format_op_desc(run, nowarn=False, seen_protos=None):
    seen_protos = seen_protos or set()
    opref = run.opref
    base_desc = _base_op_desc(opref, nowarn)
    return _apply_batch_desc(base_desc, run, seen_protos)

def _base_op_desc(opref, nowarn):
    if opref.pkg_type == "guildfile":
        return _format_guildfile_op(opref)
    elif opref.pkg_type == "package":
        return _format_package_op(opref)
    elif opref.pkg_type == "script":
        return _format_script_op(opref)
    elif opref.pkg_type == "builtin":
        return _format_builtin_op(opref)
    elif opref.pkg_type == "pending":
        return _format_pending_op(opref)
    elif opref.pkg_type == "test":
        return _format_test_op(opref)
    else:
        if not nowarn:
            log.warning(
                "cannot format op desc, unexpected pkg type: %s (%s)",
                opref.pkg_type, opref.pkg_name)
        return "?"

def _format_guildfile_op(opref):
    parts = []
    gf_dir = _guildfile_dir(opref)
    if gf_dir:
        parts.extend([gf_dir, os.path.sep])
    if opref.model_name:
        parts.extend([opref.model_name, ":"])
    parts.append(opref.op_name)
    return "".join(parts)

def _guildfile_dir(opref):
    from guild import config
    gf_dir = os.path.dirname(opref.pkg_name)
    relpath = os.path.relpath(gf_dir, config.cwd())
    if relpath == ".":
        return ""
    return re.sub(r"\.\./(\.\./)+", ".../", _ensure_dot_path(relpath))

def _ensure_dot_path(path):
    if path[0:1] == ".":
        return path
    return os.path.join(".", path)

def _format_package_op(opref):
    return "%s/%s:%s" % (opref.pkg_name, opref.model_name, opref.op_name)

def _format_script_op(opref):
    return _format_guildfile_op(opref)

def _format_builtin_op(opref):
    return opref.op_name

def _format_pending_op(opref):
    return "<pending %s>" % opref.op_name

def _format_test_op(opref):
    return "%s:%s" % (opref.model_name, opref.op_name)

def _apply_batch_desc(base_desc, run, seen_protos):
    import guild.run
    try:
        proto_dir = run.guild_path("proto")
    except TypeError:
        # Occurs for run proxies that don't support guild_path - punt
        # with generic descriptor. (TODO: implement explicit behavior
        # in run interface + proxy)
        proto_dir = ""
    if not os.path.exists(proto_dir):
        return base_desc
    if proto_dir in seen_protos:
        # We have a cycle - drop this proto_dir
        return base_desc
    proto_run = guild.run.Run("", proto_dir)
    proto_op_desc = format_op_desc(proto_run, seen_protos)
    parts = [proto_op_desc]
    if not base_desc.startswith("+"):
        parts.append("+")
    parts.append(base_desc)
    return "".join(parts)

def run_params_for_restart(run, user_specified_params=None):
    """Returns params for use in run command for a restart of run.

    The set of applicable params in the run "run_params" attribute are
    considered. If user_specified_params contains a non-default value
    (i.e. the user has indicated she wants to use a specific value)
    that param will not be included in the result. If
    user_specified_params is None (default) then all applicable params
    for a restart that are defined in run are returned.
    """
    # Note about applicable run params:
    #
    # A limited number of params could possibly apply to args - those
    # are listed here. This list has to be maintained as new args are
    # added to the run command. Params must be included where the user
    # would reasonably assume applicability and never in cases where
    # the use of the parameter would be clearly surprising to the user
    # (e.g. reusing the 'yes' param, which would alter the expected
    # behavior of the command on a restart/rerun).
    #
    # Params that are saved as run attrs or otherwise available under
    # the run guild path (e.g. opspec, label, flags) should NOT be
    # returned in this value in the interest of elimiting redundancy
    # and potential mismtach bugs. Anyone needing those values MUST
    # read them via run attrs or applicable run interface
    # (e.g. opref in the case of opsec).
    #
    applicable_run_params = [
        "disable_plugins",
        "force_flags",
        "gpus",
        "max_trials",
        "maximize",
        "minimize",
        "no_gpus",
        "opt_flags",
        "optimizer",
        "random_seed",
    ]
    from guild.commands.run import run as run_cmd
    run_params = run.get("run_params", {})
    if not isinstance(run_params, dict):
        return
    baseline_params = run_cmd.make_context("", []).params
    result = {}
    for name in run_params:
        val = _coerce_run_param(name, run_params[name])
        if name not in applicable_run_params:
            continue
        if user_specified_params is None:
            result[name] = val
            continue
        try:
            user_specified_val = user_specified_params[name]
        except KeyError:
            result[name] = val
            continue
        if user_specified_val != baseline_params[name]:
            continue
        result[name] = val
    return result

def _coerce_run_param(name, val):
    """Ensures that named param is valid for the run command."""
    if name == "flags":
        return tuple(val)
    return val

def flags_hash(flags):
    flag_parts = [
        "%s:%s" % (name, format_flag_val(val))
        for name, val in sorted(flags.items())
    ]
    to_hash = "\n".join(flag_parts).encode()
    return hashlib.md5(to_hash).hexdigest()

def restart_needed(run, flags):
    return run.status in RESTART_NEEDED_STATUS or run.get("flags") != flags

def parse_function(s):
    if not isinstance(s, six.string_types):
        raise ValueError("requires string")
    m = function_pattern.match(s)
    if not m:
        raise ValueError("not a function")
    name = m.group(1) or None
    args_raw = m.group(2).strip()
    if args_raw:
        args_s = args_raw.split(function_arg_delimiter)
    else:
        args_s = []
    args = [parse_arg_val(arg.strip()) for arg in args_s]
    return name, tuple(args)

def flag_assigns(flags):
    def fmt(val):
        if isinstance(val, float):
            val = round(val, 4)
        return shlex_quote(format_flag_val(val))
    return [
        "%s=%s" % (name, fmt(flags[name]))
        for name in sorted(flags)
    ]

def opdef_model_paths(opdef):
    return _opdef_paths(opdef) + _model_parent_paths(opdef.modeldef)

def _opdef_paths(opdef):
    if not opdef.guildfile.dir:
        return []
    abs_gf_dir = os.path.abspath(opdef.guildfile.dir)
    if opdef.python_path is not None:
        return [os.path.join(abs_gf_dir, p) for p in opdef.python_path]
    return [abs_gf_dir]

def _model_parent_paths(modeldef):
    return [os.path.abspath(parent.dir) for parent in modeldef.parents]

def _patch_yaml_safe_loader():
    # Credit: https://stackoverflow.com/users/1307905/anthon
    # Ref:    https://stackoverflow.com/questions/30458977/
    #         yaml-loads-5e-6-as-string-and-not-a-number
    loader = yaml.SafeLoader
    loader.add_implicit_resolver(
        u'tag:yaml.org,2002:float',
        re.compile(u'''^(?:
        [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
        |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
        |\\.[0-9_]+(?:[eE][-+][0-9]+)?
        |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
        |[-+]?\\.(?:inf|Inf|INF)
        |\\.(?:nan|NaN|NAN))$''', re.X),
        list(u'-+0123456789.'))

_patch_yaml_safe_loader()


from sqlalchemy import create_engine
from sqlalchemy import Column, String
from sqlalchemy.orm import sessionmaker
from dotenv import load_dotenv
import os
import sys

from .models import Base, Book
from datetime import datetime

load_dotenv()

def _create_session():
    db_url = os.getenv("DATABASE_URL")

    if 'pytest' in sys.argv[0]:
        db_url += '_test'

    if not db_url:
        raise EnvironmentError('Need to set (TEST_)DATABASE_URL')

    engine = create_engine(db_url, echo=True)
    Base.metadata.create_all(engine)
    create_session = sessionmaker(bind=engine)
    return create_session()

session = _create_session()

def get_books(year=None):
    if year is not None:
        books = session.query(Book)
        beg_of_year = datetime(datetime.today().year, "01", "01")
        books = books.filter(Book.read >= beg_of_year).all()
    else:
        books = session.query(Book)

    books = books.order_by(Book.read.desc())
    return books.all()

def add_books(books):
    books = books if isinstance(books, list) else books.items()
    for book in books:
        session.add(Book(goodreads_book_id=book["id"],
                         title=book["title"],
                         author=book["author"],
                         rating=book["rating"],
                         read=book["read"],
                         isbn=book["isbn"]))
    session.commit()


import sys
from django.db import IntegrityError, transaction
from django.contrib.auth.models import User
from django.core.management.base import NoArgsCommand
from emailusernames.utils import _email_to_username

class Command(NoArgsCommand):
    def handle_noargs(self, **options):
        """Upgrade the existing User table to be usable by django-email-as-username.

        This is specifically designed for versions 1.3 and above of the package."""

        title = "Upgrading User table for django-email-as-username"
        sys.stdout.write("=====\n" + title + "\n=====\n")
        sys.stdout.flush()

        total = User.objects.count()
        failed = 0

        for user in User.objects.all():
            user.username = _email_to_username(user.email)
            try:
                sid = transaction.savepoint()
                user.save()
                transaction.savepoint_commit(sid)
            except IntegrityError:
                failed += 1
                sys.stdout.write(
                    "Could not convert user with username '%s' because the email "
                    "<%s> is already taken.\n" % (user.username, user.email)
                    )
                sys.stdout.flush()
                transaction.savepoint_rollback(sid)

        sys.stdout.write("Converted %d of %d users (%d failed).\n" % (total - failed, total, failed))


import praw

class Redditor(object):
    """
    Facade class for getting reddit posts
    """
    def __init__(self, client_id, client_secret, user_agent):
        """
        Initialize Redditor
        :param client_id: client id provided by Reddit
        :param client_secret: client secret provided by Reddit
        :param user_agent: user agent
        """
        self.reddit = praw.Reddit(client_id=client_id,
                                  client_secret=client_secret,
                                  user_agent=user_agent)

    def hot_submissions(self, limit, subreddit, *args):
        """
        Gets a random hot not stickied submission
        :param limit: Max amount of hot posts to add
        :param subreddit: First subreddit to look up the submissions
        :param args: Additional subreddits to look up the submissions
        :return: List of hot submissions from selected subreddits
        """
        submissions = []  # list of submissions

        # add non-stickied submissions from subreddit
        for submission in self.reddit.subreddit(subreddit).hot(limit=limit):
            if not submission.stickied:
                submissions.append(submission)

        # add non-stickied submissions from additional subreddits
        for sub in args:
            for submission in self.reddit.subreddit(sub).hot(limit=limit):
                if not submission.stickied:
                    submissions.append(submission)

        return submissions

    def random_hot(self, limit, subreddit, *args):
        """
        Retrieve random hot post from subreddit and additional subreddits specified in *args
        :param limit: The amount of hot posts to look up from subreddit
        :param subreddit: First subreddit to look up the submissions
        :param args: Additional subreddits to look up the submissions
        :return: Random hot submission
        """
        from random import choice
        return choice(self.hot_submissions(limit, subreddit, *args))


import numpy as np
import scipy.integrate as si
from numpy.ma import log, exp
from scipy.constants import K2C, C2K
from ..package_tools import Exporter
from ..constants import epsilon, kappa, P0, Rd, Lv, Cp_d

exporter = Exporter(globals())

sat_pressure_0c = 6.112  # mb

@exporter.export
def potential_temperature(pressure, temperature):
    r'''Calculate the potential temperature.

    Uses the Poisson equation to calculation the potential temperature
    given `pressure` and `temperature`.

    Parameters
    ----------
    pressure : array_like
        The total atmospheric pressure in mb
    temperature : array_like
        The temperature in Kelvin

    Returns
    -------
    array_like
        The potential temperature corresponding to the the temperature and
        pressure.

    See Also
    --------
    dry_lapse

    Notes
    -----
    Formula:

    .. math:: \Theta = T (P_0 / P)^\kappa

    Examples
    --------
    >>> metpy.calc.potential_temperature(800., 273.)
    290.9814150577374

    '''

    # Factor of 100 converts mb to Pa. Really need unit support here.
    return temperature * (P0 / (pressure * 100))**kappa

# Dividing P0 by 100 converts to mb
@exporter.export
def dry_lapse(pressure, temperature, starting_pressure=P0 / 100):
    r'''Calculate the temperature at a level assuming only dry processes
    operating from the starting point.

    This function lifts a parcel starting at `temperature` and
    `starting_pressure` to the level given by `pressure`, conserving potential
    temperature.

    Parameters
    ----------
    pressure : array_like
        The atmospheric pressure level of interest in mb
    temperature : array_like
        The starting temperature in Kelvin
    starting_pressure : array_like
        The pressure at the starting point. Defaults to P0 (1000 mb).

    Returns
    -------
    array_like
       The resulting parcel temperature, in Kelvin, at level `pressure`

    See Also
    --------
    moist_lapse : Calculate parcel temperature assuming liquid saturation
                  processes
    parcel_profile : Calculate complete parcel profile
    potential_temperature
    '''

    return temperature * (pressure / starting_pressure)**kappa

@exporter.export
def moist_lapse(pressure, temperature):
    r'''
    Calculate the temperature at a level assuming liquid saturation processes
    operating from the starting point.

    This function lifts a parcel starting at `temperature`
    this is calculating moist pseudo-adiabats. The starting pressure should be
    the first item in the `pressure` array.

    Parameters
    ----------
    pressure : array_like
        The atmospheric pressure level of interest in mb
    temperature : array_like
        The starting temperature in Kelvin

    Returns
    -------
    array_like
       The temperature corresponding to the the starting temperature and
       pressure levels, with the shape determined by numpy broadcasting rules.

    See Also
    --------
    dry_lapse : Calculate parcel temperature assuming dry adiabatic processes
    parcel_profile : Calculate complete parcel profile

    Notes
    -----
    This function is implemented by integrating the following differential
    equation:

    .. math:: \frac{dT}{dP} = \frac{1}{P} \frac{R_d T + L_v r_s}
                                {C_{pd} + \frac{L_v^2 r_s \epsilon}{R_d T^2}}

    This equation comes from [1]_.

    References
    ----------

    .. [1] Bakhshaii, A. and R. Stull, 2013: Saturated Pseudoadiabats--A
           Noniterative Approximation. J. Appl. Meteor. Clim., 52, 5-15.
    '''

    # Factor of 100 converts mb to Pa. Really need unit support here.
    def dt(t, p):
        rs = mixing_ratio(saturation_vapor_pressure(K2C(t)), p)
        return (1. / p) * ((Rd * t + Lv * rs) /
                           (Cp_d + (Lv * Lv * rs * epsilon / (Rd * t * t))))
    return si.odeint(dt, np.atleast_1d(temperature).squeeze(),
                     pressure.squeeze()).T.squeeze()

@exporter.export
def lcl(pressure, temperature, dewpt, max_iters=50, eps=1e-2):
    r'''Calculate the lifted condensation level (LCL) using from the starting
    point.

    The starting state for the parcel is defined by `temperature`, `dewpoint`,
    and `pressure`.

    Parameters
    ----------
    pressure : array_like
        The starting atmospheric pressure in mb
    temperature : array_like
        The starting temperature in Kelvin
    dewpt : array_like
        The dew point in Kelvin

    Returns
    -------
    array_like
        The LCL in mb.

    Other Parameters
    ----------------
    max_iters : int, optional
        The maximum number of iterations to use in calculation, defaults to 50.
    eps : float, optional
        The desired absolute error in the calculated value, defaults to 1e-2.

    See Also
    --------
    parcel_profile

    Notes
    -----
    This function is implemented using an iterative approach to solve for the
    LCL. The basic algorithm is:
    1. Find the dew point from the LCL pressure and starting mixing ratio
    2. Find the LCL pressure from the starting temperature and dewpoint
    3. Iterate until convergence

    The function is guaranteed to finish by virtue of the `maxIters` counter.
    '''

    w = mixing_ratio(saturation_vapor_pressure(K2C(dewpt)), pressure)
    p = pressure
    while max_iters:
        td = C2K(dewpoint(vapor_pressure(p, w)))
        new_p = pressure * (td / temperature) ** (1. / kappa)
        if np.abs(new_p - p).max() < eps:
            break
        p = new_p
        max_iters -= 1
    return new_p

@exporter.export
def parcel_profile(pressure, temperature, dewpt):
    r'''Calculate the profile a parcel takes through the atmosphere, lifting
    from the starting point.

    The parcel starts at `temperature`, and `dewpt`, lifed up
    dry adiabatically to the LCL, and then moist adiabatically from there.
    `pressure` specifies the pressure levels for the profile.

    Parameters
    ----------
    pressure : array_like
        The atmospheric pressure in mb. The first entry should be the starting
        point pressure.
    temperature : array_like
        The temperature in Kelvin
    dewpt : array_like
        The dew point in Kelvin

    Returns
    -------
    array_like
        The parcel temperatures at the specified pressure levels.

    See Also
    --------
    lcl, moist_lapse, dry_lapse
    '''

    # Find the LCL
    l = np.atleast_1d(lcl(pressure[0], temperature, dewpt))

    # Find the dry adiabatic profile, *including* the LCL
    press_lower = np.concatenate((pressure[pressure > l], l))
    t1 = dry_lapse(press_lower, temperature, pressure[0])

    # Find moist pseudo-adiabatic profile starting at the LCL
    t2 = moist_lapse(np.concatenate((l, pressure[pressure < l])), t1[-1])

    # Return LCL *without* the LCL point
    return np.concatenate((t1[:-1], t2[1:]))

@exporter.export
def vapor_pressure(pressure, mixing):
    r'''Calculate water vapor (partial) pressure

    Given total `pressure` and water vapor `mixing` ratio, calculates the
    partial pressure of water vapor.

    Parameters
    ----------
    pressure : array_like
        total atmospheric pressure
    mixing : array_like
        dimensionless mass mixing ratio

    Returns
    -------
    array_like
        The ambient water vapor (partial) pressure in the same units as
        `pressure`.

    See Also
    --------
    saturation_vapor_pressure, dewpoint
    '''

    return pressure * mixing / (epsilon + mixing)

@exporter.export
def saturation_vapor_pressure(temperature):
    r'''Calculate the saturation water vapor (partial) pressure

    Parameters
    ----------
    temperature : array_like
        The temperature in degrees Celsius.

    Returns
    -------
    array_like
        The saturation water vapor (partial) presure in mb.

    See Also
    --------
    vapor_pressure, dewpoint

    Notes
    -----
    Instead of temperature, dewpoint may be used in order to calculate
    the actual (ambient) water vapor (partial) pressure.

    The formula used is that from Bolton 1980 [2] for T in degrees Celsius:

    .. math:: 6.112 e^\frac{17.67T}{T + 243.5}

    References
    ----------
    .. [2] Bolton, D., 1980: The Computation of Equivalent Potential
           Temperature. Mon. Wea. Rev., 108, 1046-1053.
    '''

    return sat_pressure_0c * exp(17.67 * temperature / (temperature + 243.5))

@exporter.export
def dewpoint_rh(temperature, rh):
    r'''Calculate the ambient dewpoint given air temperature and relative
    humidity.

    Parameters
    ----------
    temperature : array_like
        Temperature in degrees Celsius
    rh : array_like
        Relative humidity expressed as a ratio in the range [0, 1]

    Returns
    -------
    array_like
        The dew point temperature in degrees Celsius, with the shape
        of the result being determined using numpy's broadcasting rules.

    See Also
    --------
    dewpoint, saturation_vapor_pressure
    '''

    return dewpoint(rh * saturation_vapor_pressure(temperature))

@exporter.export
def dewpoint(e):
    r'''Calculate the ambient dewpoint given the vapor pressure.

    Parameters
    ----------
    e : array_like
        Water vapor partial pressure in mb

    Returns
    -------
    array_like
        Dew point temperature in degrees Celsius.

    See Also
    --------
    dewpoint_rh, saturation_vapor_pressure, vapor_pressure

    Notes
    -----
    This function inverts the Bolton 1980 [3] formula for saturation vapor
    pressure to instead calculate the temperature. This yield the following
    formula for dewpoint in degrees Celsius:

    .. math:: T = \frac{243.5 log(e / 6.112)}{17.67 - log(e / 6.112)}

    References
    ----------
    .. [3] Bolton, D., 1980: The Computation of Equivalent Potential
           Temperature. Mon. Wea. Rev., 108, 1046-1053.
    '''

    val = log(e / sat_pressure_0c)
    return 243.5 * val / (17.67 - val)

@exporter.export
def mixing_ratio(part_press, tot_press):
    r'''Calculates the mixing ratio of gas given its partial pressure
    and the total pressure of the air.

    There are no required units for the input arrays, other than that
    they have the same units.

    Parameters
    ----------
    part_press : array_like
        Partial pressure of the constituent gas
    tot_press : array_like
        Total air pressure

    Returns
    -------
    array_like
        The (mass) mixing ratio, unitless (e.g. Kg/Kg or g/g)

    See Also
    --------
    vapor_pressure
    '''

    return epsilon * part_press / (tot_press - part_press)