import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.gaussian_process.kernels import ExpSineSquared
from sklearn.kernel_ridge import KernelRidge
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVR, LinearSVC, LinearSVR
from sklearn.model_selection import cross_val_score
from sklearn import linear_model
from sklearn.preprocessing import PolynomialFeatures
from sklearn.metrics import r2_score
from sklearn.utils import shuffle
import matplotlib.ticker as ticker

# For SVR we need Feature Scaling
scaler = StandardScaler()

dataset = pd.read_pickle('./dataframe.pkl')
#train = dataset.iloc[:400]
#test = dataset.iloc[400:]

print(dataset)

#dataset selection
train_datasets = ['RW1', 'RW7','RW4', 'RW5','RW10', 'RW11', 'RW13', 'RW14', 'RW20','RW21', 'RW22', 'RW23', 'RW25', 'RW26', 'RW27']
test_datasets = ['RW8', 'RW12','RW6','RW15', 'RW24', 'RW28']

train = dataset[dataset.ID.isin(train_datasets)]
test = dataset[dataset.ID.isin(test_datasets)]

#feature selected to train the model
#features_cols = ['Qdelta', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'CV', 'V10', 'CT', 'T2', 'T3', 'T4', 'T5', 'T6', 'T7', 'T8', 'T9', 'T10']
features_cols = ['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'T1', 'T2', 'T3', 'T4', 'T5', 'T6', 'T7', 'T8', 'T9', 'T10']
features_colsTest = ['ID','C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'T1', 'T2', 'T3', 'T4', 'T5', 'T6', 'T7', 'T8', 'T9', 'T10']
X = train.loc[:, features_cols]

#X = np.array(X)

#response vector for train
response_col = ['Capacità']
y = pd.Series(train['Capacità'])

#get the test models
X_test = test.loc[:, features_colsTest]
#X_test = np.array(X_test)

y_labels = test.loc[:, ['ID', 'Capacità']]

X_testRW8 = X_test.where(X_test['ID'] == 'RW8').dropna()
X_testRW6 = X_test.where(X_test['ID'] == 'RW6').dropna()
X_testRW12 = X_test.where(X_test['ID'] == 'RW12').dropna()
X_testRW15 = X_test.where(X_test['ID'] == 'RW15').dropna()
X_testRW24 = X_test.where(X_test['ID'] == 'RW24').dropna()
X_testRW28 = X_test.where(X_test['ID'] == 'RW28').dropna()
X_testRW8.pop('ID')
X_testRW6.pop('ID')
X_testRW15.pop('ID')
X_testRW12.pop('ID')
X_testRW24.pop('ID')
X_testRW28.pop('ID')

y_labelsRW8 = y_labels.where(y_labels['ID'] == 'RW8').dropna()
y_labelsRW6 = y_labels.where(y_labels['ID'] == 'RW6').dropna()
y_labelsRW12 = y_labels.where(y_labels['ID'] == 'RW12').dropna()
y_labelsRW15 = y_labels.where(y_labels['ID'] == 'RW15').dropna()
y_labelsRW24 = y_labels.where(y_labels['ID'] == 'RW24').dropna()
y_labelsRW28 = y_labels.where(y_labels['ID'] == 'RW28').dropna()

y_labelsRW8.pop('ID')
y_labelsRW6.pop('ID')
y_labelsRW12.pop('ID')
y_labelsRW15.pop('ID')
y_labelsRW24.pop('ID')
y_labelsRW28.pop('ID')

x_labels = test.loc[:, ['ID', 'Qstart','Qend']]

aRW8 = x_labels.where(x_labels['ID'] == 'RW8').dropna()
aRW6 = x_labels.where(x_labels['ID'] == 'RW6').dropna()
aRW15 = x_labels.where(x_labels['ID'] == 'RW15').dropna()
aRW24 = x_labels.where(x_labels['ID'] == 'RW24').dropna()
aRW28 = x_labels.where(x_labels['ID'] == 'RW28').dropna()
aRW12 = x_labels.where(x_labels['ID'] == 'RW12').dropna()
aRW8.pop('ID')
aRW6.pop('ID')
aRW15.pop('ID')
aRW24.pop('ID')
aRW28.pop('ID')
aRW12.pop('ID')

#train the model

param_grid={
            'C': [0.0001, 0.001, 0.1, 1, 100, 1000, 1250, 1500],
            'epsilon': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
            'gamma': ['auto','scale','auto_deprecated']
        }

svr_rbf = SVR(kernel='rbf')

clf = GridSearchCV(svr_rbf, param_grid, cv=3, scoring='neg_mean_squared_error', verbose=0, n_jobs=-1)

clf.fit(X, y)

print(clf.best_params_)

test_predictionsRW8 =  clf.predict(X_testRW8)
test_predictionsRW6 =  clf.predict(X_testRW6)
test_predictionsRW12 =  clf.predict(X_testRW12)
test_predictionsRW15 =  clf.predict(X_testRW15)
test_predictionsRW24 =  clf.predict(X_testRW24)
test_predictionsRW28 =  clf.predict(X_testRW28)

print("\n-----R2 SCORE----")
print("RW8 -->", r2_score(y_labelsRW8, test_predictionsRW8))
print("RW6 -->", r2_score(y_labelsRW6, test_predictionsRW6))
print("RW12 -->", r2_score(y_labelsRW12, test_predictionsRW12))
print("RW15 -->", r2_score(y_labelsRW15, test_predictionsRW15))
print("RW24 -->", r2_score(y_labelsRW24, test_predictionsRW24))
print("RW28 -->", r2_score(y_labelsRW28, test_predictionsRW28))
print("---------")

print("\n-----MSE SCORE----")
print("RW8 -->", mean_squared_error(y_labelsRW8, test_predictionsRW8))
print("RW6 -->", mean_squared_error(y_labelsRW6, test_predictionsRW6))
print("RW12 -->", mean_squared_error(y_labelsRW12, test_predictionsRW12))
print("RW15 -->", mean_squared_error(y_labelsRW15, test_predictionsRW15))
print("RW24 -->", mean_squared_error(y_labelsRW24, test_predictionsRW24))
print("RW28 -->", mean_squared_error(y_labelsRW28, test_predictionsRW28))
print("---------")

print(np.mean([mean_squared_error(y_labelsRW8, test_predictionsRW8), mean_squared_error(y_labelsRW6, test_predictionsRW6)
                  , mean_squared_error(y_labelsRW12, test_predictionsRW12), mean_squared_error(y_labelsRW15, test_predictionsRW15)
                  ,mean_squared_error(y_labelsRW24, test_predictionsRW24), mean_squared_error(y_labelsRW28, test_predictionsRW28)]))

#new_loc = np.round(np.arange(aRW8['Qstart'].iloc[0], aRW8['Qend'].iloc[-1], (aRW8['Qend'].iloc[-1] - aRW8['Qstart'].iloc[0])/len(locs) ),2)

colors = ['#2300A8', '#00A658']
plt.subplot(2, 3, 2)
frequency = 3
x_ax = range(y_labelsRW8.shape[0])
plt.scatter(x_ax, y_labelsRW8, s=5, color=colors[0], label = "original")
plt.plot(x_ax, test_predictionsRW8, lw=1.5, color =colors[1], label="predicted")
plt.xticks(x_ax[::6],np.round(np.arange(aRW8['Qstart'].iloc[0], aRW8['Qend'].iloc[-1], (aRW8['Qend'].iloc[-1] - aRW8['Qstart'].iloc[0])/len(x_ax)))[::6], rotation = 45)
plt.yticks(y_labelsRW8.values[::6], np.round(np.arange(y_labelsRW8['Capacità'].iloc[-1], y_labelsRW8['Capacità'].iloc[0], (y_labelsRW8['Capacità'].iloc[0]-y_labelsRW8['Capacità'].iloc[-1])/len(y_labelsRW8.values)), 2)[::-1])
plt.xlabel("Time (h)")
plt.ylabel("Capacity (Ah)")
plt.title('RW8')
ax = plt.gca()
ax.tick_params(axis = 'both', which = 'major', labelsize = 8.5)
plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)
plt.figlegend(loc = 'center left')

plt.subplot(2, 3, 1)
x_ax = range(y_labelsRW6.shape[0])
plt.scatter(x_ax, y_labelsRW6, s=5, color=colors[0], label = "original")
plt.plot(x_ax, test_predictionsRW6, lw=1.5, color =colors[1], label="predicted")
plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)
plt.xticks(x_ax[::3],np.round(np.arange(aRW6['Qstart'].iloc[0], aRW6['Qend'].iloc[-1], (aRW6['Qend'].iloc[-1] - aRW6['Qstart'].iloc[0])/len(x_ax)))[::3], rotation = 45)
plt.yticks(y_labelsRW6.values[::3], np.round(np.arange(y_labelsRW6['Capacità'].iloc[-1], y_labelsRW6['Capacità'].iloc[0], (y_labelsRW6['Capacità'].iloc[0]-y_labelsRW6['Capacità'].iloc[-1])/len(y_labelsRW6.values)), 2)[::-1])
plt.xlabel("Time (h)")
plt.ylabel("Capacity (Ah)")
plt.title('RW6')
ax = plt.gca()
ax.tick_params(axis = 'both', which = 'major', labelsize = 8.5)

plt.subplot(2, 3, 4)
x_ax = range(y_labelsRW15.shape[0])
plt.scatter(x_ax, y_labelsRW15, s=5, color=colors[0], label = "original")
plt.plot(x_ax, test_predictionsRW15, lw=1.5, color =colors[1], label="predicted")
locs, labels=plt.xticks()
plt.xticks(x_ax[::6],np.round(np.arange(aRW15['Qstart'].iloc[0], aRW15['Qend'].iloc[-1], (aRW15['Qend'].iloc[-1] - aRW15['Qstart'].iloc[0])/len(x_ax)))[::6], rotation = 45)
plt.yticks(y_labelsRW15.values[::6], np.round(np.arange(y_labelsRW15['Capacità'].iloc[-1], y_labelsRW15['Capacità'].iloc[0], (y_labelsRW15['Capacità'].iloc[0]-y_labelsRW15['Capacità'].iloc[-1])/len(y_labelsRW15.values)), 2)[::-1])
plt.xlabel("Time (h)")
plt.ylabel("Capacity (Ah)")
plt.title('RW15')
plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)
ax = plt.gca()
ax.tick_params(axis = 'both', which = 'major', labelsize = 8.5)

plt.subplot(2, 3, 5)
x_ax = range(y_labelsRW24.shape[0])
plt.scatter(x_ax, y_labelsRW24, s=5, color=colors[0], label = "original")
plt.plot(x_ax, test_predictionsRW24, lw=1.5, color =colors[1], label="predicted")
plt.xticks(x_ax[::frequency],np.round(np.arange(aRW24['Qstart'].iloc[0], aRW24['Qend'].iloc[-1], (aRW24['Qend'].iloc[-1] - aRW24['Qstart'].iloc[0])/len(x_ax)))[::frequency], rotation = 45)
plt.yticks(y_labelsRW24.values[::3], np.round(np.arange(y_labelsRW24['Capacità'].iloc[-1], y_labelsRW24['Capacità'].iloc[0], (y_labelsRW24['Capacità'].iloc[0]-y_labelsRW24['Capacità'].iloc[-1])/len(y_labelsRW24.values)), 2)[::-1])
plt.xlabel("Time (h)")
plt.ylabel("Capacity (Ah)")
plt.title('RW24')
plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)
ax = plt.gca()
ax.tick_params(axis = 'both', which = 'major', labelsize = 8.5)

plt.subplot(2, 3, 6)
x_ax = range(y_labelsRW28.shape[0])
plt.scatter(x_ax, y_labelsRW28, s=5, color=colors[0], label = "original")
plt.plot(x_ax, test_predictionsRW28, lw=1.5, color =colors[1], label="predicted")
plt.xticks(x_ax[::frequency],np.round(np.arange(aRW28['Qstart'].iloc[0], aRW28['Qend'].iloc[-1], (aRW28['Qend'].iloc[-1] - aRW28['Qstart'].iloc[0])/len(x_ax)))[::frequency], rotation = 45)
plt.yticks(y_labelsRW28.values[::3], np.round(np.arange(y_labelsRW28['Capacità'].iloc[-1], y_labelsRW28['Capacità'].iloc[0], (y_labelsRW28['Capacità'].iloc[0]-y_labelsRW28['Capacità'].iloc[-1])/len(y_labelsRW28.values)), 2)[::-1])
plt.xlabel("Time (h)")
plt.ylabel("Capacity (Ah)")
plt.title('RW28')
plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)
ax = plt.gca()
ax.tick_params(axis = 'both', which = 'major', labelsize = 8.5)

plt.subplot(2, 3, 3)
x_ax = range(y_labelsRW12.shape[0])
plt.scatter(x_ax, y_labelsRW12, s=5, color=colors[0], label = "original")
plt.plot(x_ax, test_predictionsRW12, lw=1.5, color =colors[1], label="predicted")
plt.xticks(x_ax[::8],np.round(np.arange(aRW12['Qstart'].iloc[0], aRW12['Qend'].iloc[-1], (aRW12['Qend'].iloc[-1] - aRW12['Qstart'].iloc[0])/len(x_ax)))[::8], rotation = 45)
plt.yticks(y_labelsRW12.values[::8], np.round(np.arange(y_labelsRW12['Capacità'].iloc[-1], y_labelsRW12['Capacità'].iloc[0], (y_labelsRW12['Capacità'].iloc[0]-y_labelsRW12['Capacità'].iloc[-1])/len(y_labelsRW12.values)), 2)[::-1])
plt.xlabel("Time (h)")
plt.ylabel("Capacity (Ah)")
plt.title('RW12')
plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)
ax = plt.gca()
ax.tick_params(axis = 'both', which = 'major', labelsize = 8.5)

plt.show()


import logging
from flask_cors import cross_origin
from flask import request
from flask_restplus import Resource
from api.wallet.business.holdings import get_holdings, get_holding, get_holdings_historical
from api.wallet.serializers import holding
from api.restplus import api
from api import auth
from cache import cache

log = logging.getLogger(__name__)

ns = api.namespace('wallet', description='Operations related to wallet Holdings')

@ns.route('/')
class HoldingsCollection(Resource):
    @api.marshal_list_with(holding)
    @auth.requires_auth
    def get(self):
        """
        Returns list of Holdings
        """
        # auth = request.headers.get("Authorization", None)

        # print(auth)
        cache_key = 'auth:' + request.headers.get("Authorization", None)
        rv = cache.get(cache_key)
        if rv is None:
            userInfo = auth.get_userinfo_with_token()
            rv = userInfo['email']
            cache.set(cache_key, rv, timeout=60 * 50)
        # return rv, 200

        response = get_holdings(rv)
        return response, 200

@ns.route('/<string:ticker>')
@api.response(404, 'Product not found.')
class HoldingItem(Resource):
    @auth.requires_auth
    @api.marshal_with(holding)
    def get(self, ticker):
        """
        Returns list of Holdings
        # """
        cache_key = 'auth:' + request.headers.get("Authorization", None)
        rv = cache.get(cache_key)
        if rv is None:
            userInfo = auth.get_userinfo_with_token()
            rv = userInfo['email']
            cache.set(cache_key, rv, timeout=60 * 50)

        response = get_holding(rv, ticker)
        return response, 200

# @api.marshal_list_with(holding)
@auth.requires_auth
@ns.route('/historical/')
class HistoricalHoldings(Resource):
    def get(self):
        cache_key = 'auth:' + request.headers.get("Authorization", None)
        rv = cache.get(cache_key)
        if rv is None:
            userInfo = auth.get_userinfo_with_token()
            rv = userInfo['email']
            cache.set(cache_key, rv, timeout=60 * 50)
        response = get_holdings_historical("test")
        return response, 200


# -*- coding: utf-8 -*
import re
import datetime

from RequestData import req_data

# rsshub的json接口
url_rsshub = "https://rsshub.dashwood.xyz/leetcode/submission/"

def get_rsshub_us(contury, user):
    # 当天时间
    now = datetime.date.today()
    otherStyleTime = str(int(now.strftime('%m'))) + '/' + str(int(now.strftime('%d'))) + '/' + str(
        int(now.strftime('%Y')))
    # 请求数据
    url_rs = url_rsshub + contury + "/" + user + ".json"  # rsshub的json接口
    response = req_data(url_rs)

    if (response.status_code != 404):
        if (user != "problemset"):
            # 初步处理数据
            user_data = response.json(strict=False)
            title = user_data["title"]
            description = user_data["description"]
            items = user_data["items"]
            url = user_data["home_page_url"]
            username = re.findall(r"\n(.+?)\n", title)[0].strip()  # NathanDai
            SolvedQuestion = re.findall(r"Solved Question: \n(.+?)\n", description)[0].strip()  # 30 / 1050
            AcceptedSubmission = re.findall(r"Accepted Submission: \n(.+?)\n", description)[0].strip()  # 31 / 48
            AcceptanceRate = re.findall(r"Acceptance Rate: \n(.+?)\n", description)[0].strip()  # 64.6 %
            avatar = re.findall(r"src="(.+?)">", description)[0].strip()
            lastSubmission = re.findall("(.*),", items[0]["date_published"])[0]
            last = re.findall(r"\d+\.?\d*", str(lastSubmission))
            todayS = 0  # 今天提交的次数
            todayA = 0  # 今天打卡数
            checked = 0  # 今天是否打卡
            if (lastSubmission == otherStyleTime):
                checked = 1
                for item in items:
                    if (re.findall("(.*),", item["date_published"])[0] == otherStyleTime):
                        todayS = todayS + 1;
                        if (re.findall(r"\n(.+?)\n", item["summary"])[0].strip() == "Accepted"):
                            todayA = todayA + 1
            solvedQ = re.findall(r"\d+\.?\d*", SolvedQuestion)[0]  # solvedQ
            acceptS = re.findall(r"\d+\.?\d*", AcceptedSubmission)[0]  # acceptS
            submission = re.findall(r"\d+\.?\d*", AcceptedSubmission)[1]  # submission
            acceptR = re.findall("(.*) %", AcceptanceRate)[0]  # acceptR
            lastSubmission = last[2] + '-' + last[0] + '-' + last[1]
            sum = []
            sum.append(url)
            sum.append(avatar)
            sum.append(username)
            sum.append(solvedQ)
            sum.append(acceptS)
            sum.append(submission)
            sum.append(acceptR)
            sum.append(lastSubmission)
            sum.append(todayA)
            sum.append(todayS)
            sum.append(checked)
            return sum
    else:
        print("找不到该用户")
        return 0


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest

from cryptography import utils
from cryptography.exceptions import UnsupportedAlgorithm, InternalError
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.backends.openssl.backend import backend, Backend
from cryptography.hazmat.primitives import interfaces
from cryptography.hazmat.primitives.ciphers import Cipher
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.ciphers.modes import CBC

@utils.register_interface(interfaces.Mode)
class DummyMode(object):
    name = "dummy-mode"

    def validate_for_algorithm(self, algorithm):
        pass

@utils.register_interface(interfaces.CipherAlgorithm)
class DummyCipher(object):
    name = "dummy-cipher"

class TestOpenSSL(object):
    def test_backend_exists(self):
        assert backend

    def test_is_default(self):
        assert backend == default_backend()

    def test_openssl_version_text(self):
        """
        This test checks the value of OPENSSL_VERSION_TEXT.

        Unfortunately, this define does not appear to have a
        formal content definition, so for now we'll test to see
        if it starts with OpenSSL as that appears to be true
        for every OpenSSL.
        """
        assert backend.openssl_version_text().startswith("OpenSSL")

    def test_supports_cipher(self):
        assert backend.cipher_supported(None, None) is False

    def test_register_duplicate_cipher_adapter(self):
        with pytest.raises(ValueError):
            backend.register_cipher_adapter(AES, CBC, None)

    @pytest.mark.parametrize("mode", [DummyMode(), None])
    def test_nonexistent_cipher(self, mode):
        b = Backend()
        b.register_cipher_adapter(
            DummyCipher,
            type(mode),
            lambda backend, cipher, mode: backend._ffi.NULL
        )
        cipher = Cipher(
            DummyCipher(), mode, backend=b,
        )
        with pytest.raises(UnsupportedAlgorithm):
            cipher.encryptor()

    def test_handle_unknown_error(self):
        with pytest.raises(InternalError):
            backend._handle_error_code(0, 0, 0)

        with pytest.raises(InternalError):
            backend._handle_error_code(backend._lib.ERR_LIB_EVP, 0, 0)

        with pytest.raises(InternalError):
            backend._handle_error_code(
                backend._lib.ERR_LIB_EVP,
                backend._lib.EVP_F_EVP_ENCRYPTFINAL_EX,
                0
            )

        with pytest.raises(InternalError):
            backend._handle_error_code(
                backend._lib.ERR_LIB_EVP,
                backend._lib.EVP_F_EVP_DECRYPTFINAL_EX,
                0
            )

    def test_ssl_ciphers_registered(self):
        meth = backend._lib.TLSv1_method()
        ctx = backend._lib.SSL_CTX_new(meth)
        assert ctx != backend._ffi.NULL
        backend._lib.SSL_CTX_free(ctx)

    def test_evp_ciphers_registered(self):
        cipher = backend._lib.EVP_get_cipherbyname(b"aes-256-cbc")
        assert cipher != backend._ffi.NULL

    def test_error_strings_loaded(self):
        # returns a value in a static buffer
        err = backend._lib.ERR_error_string(101183626, backend._ffi.NULL)
        assert backend._ffi.string(err) == (
            b"error:0607F08A:digital envelope routines:EVP_EncryptFinal_ex:"
            b"data not multiple of block length"
        )


import ast
import inspect
import cast
from pycparser import c_ast

def constant(name):
    """
    Check if *name* is a known constant and return an equivalent
    c_ast.Constant(). If it is unknown, return None.
    """
    known = ('pi', 'e', 'ln2', 'ln10')

    if name in known:
        return c_ast.Constant('float', name)

    return None

class PythonToC(ast.NodeVisitor):
    def __init__(self):
        self.result = None

    def generic_visit(self, node):
        ops = {ast.Add: '+', ast.Sub: '-', ast.Mult: '*', ast.Div: '/',
               ast.Gt: '>', ast.GtE: '>=', ast.Lt: '<', ast.LtE: '<=',
               ast.Eq: '==', ast.NotEq: '!=',
               ast.And: '&&', ast.Or: '||',
               ast.Invert: '~', ast.Not: '!', ast.UAdd: '+', ast.USub: '-'}

        if type(node) in ops:
            self.result = ops[type(node)]
        else:
            for _, c in node.children():
                self.visit(c)

    def visit_Return(self, node):
        self.result = c_ast.Return(python_to_c_ast(node.value))

    def visit_UnaryOp(self, node):
        self.result = c_ast.UnaryOp(python_to_c_ast(node.op),
                                    python_to_c_ast(node.operand))

    def visit_BinOp(self, node):
        if isinstance(node.op, ast.Pow):
            self.result = c_ast.FuncCall(c_ast.ID('pow'),
                                         c_ast.ExprList([python_to_c_ast(node.left),
                                                         python_to_c_ast(node.right)]))
        else:
            self.result = c_ast.BinaryOp(python_to_c_ast(node.op),
                                         python_to_c_ast(node.left),
                                         python_to_c_ast(node.right))

    def visit_BoolOp(self, node):
        conds = [python_to_c_ast(v) for v in node.values]
        self.result = cast.chain(python_to_c_ast(node.op), conds)

    def visit_If(self, node):
        else_branch = None if not node.orelse else python_to_c_ast(node.orelse)
        self.result = c_ast.If(python_to_c_ast(node.test),
                               python_to_c_ast(node.body),
                               else_branch)

    def visit_For(self, node):
        if isinstance(node.iter, ast.Name):
            # We are iterating over a named iterable which we will convert to
            # an array iteration
            body = python_to_c_ast(node.body)
            self.result = c_ast.For(None, None, None, body)

            # Mark this for-loop to be processed once input data is known
            setattr(self.result, '_extra', (node.target.id, node.iter.id))
        elif isinstance(node.iter, ast.Call) and node.iter.func.id == 'range':
            var = node.target.id
            args = [python_to_c_ast(arg) for arg in node.iter.args]
            frm = c_ast.Constant('int', '0') if len(args) == 1 else args[0]
            to = args[0] if len(args) == 1 else args[1]
            step = c_ast.Constant('int', '1') if len(args) <= 2 else args[2]

            init = cast.TypeDecl(var, 'int', frm)
            cond = c_ast.BinaryOp('<', c_ast.ID(var), to)
            update = c_ast.ExprList([c_ast.BinaryOp('+=', c_ast.ID(var), step)])

            self.result = c_ast.For(init, cond, update, python_to_c_ast(node.body))

    def visit_Compare(self, node):
        self.result = c_ast.BinaryOp(python_to_c_ast(node.ops[0]),
                                     python_to_c_ast(node.left),
                                     python_to_c_ast(node.comparators[0]))

    def visit_Name(self, node):
        self.result = constant(node.id)

        if not self.result:
            self.result = c_ast.ID(node.id)

    def visit_Num(self, node):
        self.result = c_ast.Constant('int', str(node.n))

    def visit_Index(self, node):
        self.result = python_to_c_ast(node.value)

    def visit_Subscript(self, node):
        self.result = c_ast.ArrayRef(python_to_c_ast(node.value),
                                     python_to_c_ast(node.slice))

    def visit_Assign(self, node):
        # We resolve tuple assignments by creating an assignment for each tuple
        # pair. All assignments are then converted to a compound statement,
        # that means all variables _must_ be declared beforehands.
        if isinstance(node.targets[0], ast.Tuple):
            assignments = []

            lvalues = (python_to_c_ast(e) for e in node.targets[0].elts)
            rvalues = (python_to_c_ast(e) for e in node.value.elts)

            for lvalue, rvalue in zip(lvalues, rvalues):
                assignments.append(c_ast.Assignment('=', lvalue, rvalue))

            self.result = c_ast.Compound(assignments)
        else:
            lvalue = python_to_c_ast(node.targets[0])
            rvalue =  python_to_c_ast(node.value)
            self.result = c_ast.Assignment('=', lvalue, rvalue)

    def visit_AugAssign(self, node):
        op = python_to_c_ast(node.op)
        lvalue = python_to_c_ast(node.target)
        rvalue =  python_to_c_ast(node.value)
        self.result = c_ast.Assignment(op + '=', lvalue, rvalue)

    def visit_Tuple(self, node):
        elements = [python_to_c_ast(e) for e in node.elts]
        self.result = c_ast.ExprList(elements)

    def visit_Attribute(self, node):
        # strip off attribute accesses
        self.result = constant(node.attr)

        if not self.result:
            self.result = c_ast.ID(node.attr)

    def visit_Call(self, node):
        args = [python_to_c_ast(arg) for arg in node.args]
        exprs = c_ast.ExprList(args)
        ctypes = ['int', 'float']

        if isinstance(node.func, ast.Name) and node.func.id in ctypes:
            self.result = cast.CastDecl(node.func.id, exprs)
        else:
            self.result = c_ast.FuncCall(python_to_c_ast(node.func), exprs)

    def visit_FunctionDef(self, node):
        params = [python_to_c_ast(arg) for arg in node.args.args]
        rtype = c_ast.IdentifierType(['void'])
        tdecl = c_ast.TypeDecl(node.name, ['__kernel'], rtype)
        ftype = c_ast.FuncDecl(c_ast.ParamList(params), tdecl)
        decl = c_ast.Decl(node.name, None, None, None, ftype, None, None)
        body = python_to_c_ast(node.body)
        self.result = c_ast.FuncDef(decl, None, body)

    def visit_Expr(self, node):
        # Ignore docstrings which are standalone expressions with a Str object.
        pass

def python_to_c_ast(py_node):
    if isinstance(py_node, list):
        body = (python_to_c_ast(stmt) for stmt in py_node)
        return c_ast.Compound([stmt for stmt in body if stmt is not None])
    else:
        v = PythonToC()
        v.visit(py_node)
        return v.result

def parse(func):
    """
    Turn *func*'s Python AST into a pycparser AST for subsequent
    optimization and OpenCL code generation.
    """
    source = inspect.getsource(func)
    tree = ast.parse(source)
    return python_to_c_ast(tree.body[0])


#! /usr/bin/env python
#

"""
Inspired by https://github.com/ufeindt/marshaltools
"""
import json
import requests
import pandas
import os
import numpy as np
from . import io

import matplotlib.pyplot as mpl
# list of effects:
#    list_program_sources.cgi | auth, data={'programidx' : str(programidx)}  # list of sources associated to the program
#    list_programs.cgi | auth # list of program you belong to
#    view_source.cgi | auth, data={'name' : name}
#    source_summary.cgi | auth, data={'sourceid' : str(source['id'])}
#    add_spec.cgi       | auth, data=payload,files=files
#                =>  payload = {'sourceid' : str(source['id']),'spectype':args.spectype,'programid':get_prog_id(args.prog_name),'instrumentid':get_inst_id(telname),
#                            'format':fformat,'obsdate':obsdate,'exptime':exptime,'observer':user,'reducedby':reducer,'class':"",'redshift':"",
#                            'phase':"",'comment':"",'commit':'yes','submit':'upload the file'}
#
#   growth_treasures_transient.cgi?cutprogramidx=%d
#
#marshal_root = 'http://skipper.caltech.edu:8080/cgi-bin/growth/'
#summary_url 	= marshal_root  + 'source_summary.cgi?sourceid=%s'
#listprog_url = marshal_root + 'list_programs.cgi' # list of program you belong to
#scanning_url = marshal_root + 'growth_treasures_transient.cgi'
#saving_url = marshal_root   + 'save_cand_growth.cgi?candid=%s&program=%s'
#savedsources_url = marshal_root + 'list_program_sources.cgi'
#rawsaved_url = marshal_root + 'list_sources_bare.cgi'
#annotate_url = marshal_root + 'edit_comment.cgi'
#ingest_url = marshal_root + 'ingest_avro_id.cgi'
#
#

MARSHAL_BASEURL = "http://skipper.caltech.edu:8080/cgi-bin/growth/"
from .io import LOCALSOURCE

def _account_id_declined_(username, password):
    """ This returns True if the login information has been rejected"""
    r = requests.post( MARSHAL_BASEURL+'list_programs.cgi',  auth=(username, password) )
    return "This server could not verify that you" in r.text

#############################
#                           #
# Stand Alone Functions     #
#                           #
#############################
def get_target_data(name):
    """ provide a name (or list of names) and get its/there marshal information
    IMPORTANT: This function is slow, but it takes the same amount of time if you provide 1 or any number of targets.
               So better provide a long list of target name at once.
    Returns
    -------
    pandas.DataFrame
    """
    m = MarshalAccess()
    m.load_target_sources()
    return m.get_target_data(name)

def download_spectra(name, dirout="default", auth=None, verbose=False, **kwargs):
    """Download all spectra for a source in the marshal as a tar.gz file

    Parameters:
    -----------
    name: [str]
        Name of a target on the marshal.

    dirout: [str] -optional-
        Directory where the data should be stored.
        Additional options:
        - `dirout=None`: The spectra are not saved be returned
        - `dirout='default'`: The spectra will be saved in native target location
                              (`$ZTFDATA`/marshal/spectra/`name`)
                              Spectra saved here can be recovered using `get_local_spectra`
                              * This is favored *

    auth: [str,str] -optional-
        Marshal [username, password]

    verbose: [bool] -optional-
        Prints to know what is going on.

    **kwargs goes to ztfquery.io.download_single_url()

    Returns
    -------
    None (or list of data if `dirout=None`)
    """
    # fileout is saved later to manage decompression
    import tarfile
    from io import BytesIO
    response = io.download_single_url(MARSHAL_BASEURL+'batch_spec.cgi',
                                   fileout=None,
                                   data={"name":name},
                                   auth=io._load_id_("marshal") if auth is None else auth,
                                   cookies="no_cookies", show_progress=False,
                                   **kwargs)

    tar = tarfile.open(fileobj=BytesIO( response.content ), mode='r')

    # No directory out? Then reformated data returned
    if dirout is None or dirout in ["None"]:
        if verbose: print("Data returned (dirout=None)")
        out = {member.name:tar.extractfile(member).read().decode("utf-8").splitlines() for member in tar.getmembers()}
        return out
    # Directory given, then dump data there:
    if dirout in ["default"]:
        dirout = target_spectra_directory(name)

    if verbose: print("Data will be stored here: %s"%dirout)
    if not os.path.exists(dirout):
        os.makedirs(dirout)

    tar.extractall(dirout)

# -------------- #
#  PLOT LC       #
# -------------- #

GENERIC = dict(alpha=1, mew=0.4, mec="0.7", ecolor="0.7", ls="None")
PROP    = { # ZTF
            "ZTF:r":dict(marker="o",ms=7,  mfc="C3"),
            "ZTF:g":dict(marker="o",ms=7,  mfc="C2"),
            "ZTF:i":dict(marker="o",ms=7, mfc="C1"),
            # Swift
            "UVOT:B":   dict(marker="s",  ms=5, mfc="C0"),
            "UVOT:u":   dict(marker="s",  ms=5, mfc=mpl.cm.Blues(0.7)),
            "UVOT:UVM2":dict(marker="s", ms=5, mfc=mpl.cm.Purples(0.6)),
            "UVOT:UVW2":dict(marker="s", ms=5, mfc=mpl.cm.Purples(0.8)),
            "UVOT:UVW1":dict(marker="s", ms=5, mfc=mpl.cm.Purples(0.4)),
            "UVOT:V":   dict(marker="s", ms=5, mfc=mpl.cm.Greens(0.9)),
            #
            "IOO:u":   dict(marker="d", ms=6,mfc=mpl.cm.Blues(0.6)),
            "IOO:g":   dict(marker="d", ms=6,mfc=mpl.cm.Greens(0.6)),
            "IOO:r":   dict(marker="d", ms=6,mfc=mpl.cm.Reds(0.7)),
            "IOO:i":   dict(marker="d",ms=6, mfc=mpl.cm.Oranges(0.6)),
            "IOO:z":   dict(marker="d", ms=6,mfc=mpl.cm.binary(0.8))
            }
for v in PROP.values():
    for k,v_ in GENERIC.items():
        v[k]=v_

def plot_lightcurve(lc_dataframe, ax=None, title=None, show_legend=True):
    """ """
    import matplotlib.pyplot as mpl
    from astropy.time import Time
    if ax is None:
        fig = mpl.figure(figsize=[7,4])
        ax  = fig.add_axes([0.1,0.12,0.67,0.8])
    else:
        fig = ax.figure

    lc_dataframe["inst_filter"] = [d.split("+")[-1].replace('"',"")
                            for d in lc_dataframe["instrument"]+":"+lc_dataframe["filter"]]

    # DataPoints
    for filter_ in np.unique(lc_dataframe["inst_filter"]):
        if filter_ not in  PROP:
            print("WARNING: Unknown instrument: %s | magnitude not shown"%filter_)
            continue

        jd, mag, magerr = lc_dataframe[lc_dataframe["inst_filter"].isin([filter_]) &
                                       ~lc_dataframe["magpsf"].isin([99.00])][
                            ["jdobs","magpsf","sigmamagpsf"]
                        ].values.T

        ax.errorbar([Time(jd_, format="jd").datetime for jd_ in jd],
                     mag, yerr= magerr,
                     label="%s"%filter_, **PROP[filter_.replace('"',"")])
    # Upper Limits
    ax.invert_yaxis()
    for filter_ in np.unique(lc_dataframe["inst_filter"]):
        if filter_ not in  PROP:
            print("WARNING: Unknown instrument: %s | upper limits not shown"%filter_)
            continue

        jdup, upmag = lc_dataframe[lc_dataframe["inst_filter"].isin([filter_]) &
                                 lc_dataframe["magpsf"].isin([99.00])][
                            ["jdobs","limmag"]
                        ].values.T
        ax.errorbar([Time(jd_, format="jd").datetime for jd_ in jdup],
                                upmag, yerr=0.15, lolims=True,alpha=0.3,
                                    color=PROP[filter_.replace('"',"")]["mfc"],
                            ls="None",
                                    label="_no_legend_")

    ax.set_ylabel("magnitude", fontsize="large")
    ax.set_xlabel("Time", fontsize="large")
    if title is not None:
        ax.set_title(title)
    if show_legend:
        ax.legend(loc=[1.02,0.], fontsize="medium" )
    return {"ax":ax, "fig":fig}

# -------------- #
#  Data I/O      #
# -------------- #
# - What at ?
def target_spectra_directory(name):
    """ where Marshal spectra are stored """
    return LOCALSOURCE+"marshal/spectra/%s/"%name

def target_lightcurves_directory(name):
    """ where Marshal lightcurves are stored """
    return LOCALSOURCE+"marshal/lightcurves/%s/"%name

# - Get the FullPathes
def get_local_spectra(name):
    """ returns list of fullpath of spectra on your computer for the given target name.
    Remark: These spectra have to be stored in the native `$ZTFDATA`/marshal/spectra/`name`
    """
    dir_ = target_spectra_directory(name)
    return {d:open(dir_+"%s"%d).read().splitlines() for d in os.listdir( dir_ )}

def get_local_lightcurves(name):
    """ returns list of fullpath of lightcurves on your computer for the given target name.
    Remark: These lightcurves have to be stored in the native `$ZTFDATA`/marshal/lightcurves/`name`
    """
    dir_ = target_lightcurves_directory(name)

    return {d: pandas.read_csv(dir_+d) for d in os.listdir( dir_ )}

# -------------- #
#  Downloading   #
# -------------- #
def download_lightcurve(name, dirout="default",
                            auth=None, verbose=False, **kwargs):
    """Download all spectra for a source in the marshal as a tar.gz file

    Parameters:
    -----------
    name: [str]
        Name of a target on the marshal.

    dirout: [str] -optional-
        Directory where the data should be stored.
        Additional options:
        - `dirout=None`: The spectra are not saved be returned
        - `dirout='default'`: The lightcurve will be saved in native target location
                              (`$ZTFDATA`/marshal/lightcurves/`name`)
                              lightcurve saved here can be recovered using `get_local_lightcurve`
                              * This is favored *

    auth: [str,str] -optional-
        Marshal [username, password]

    verbose: [bool] -optional-
        Prints to know what is going on.

    **kwargs goes to ztfquery.io.download_single_url()

    Returns
    -------
    None (or pandas.DataFrame)
    """
    # fileout is saved later to manage decompression

    response = io.download_single_url(MARSHAL_BASEURL+'print_lc.cgi',
                                   fileout=None,
                                   data={"name":name},
                                   auth=io._load_id_("marshal") if auth is None else auth,
                                   cookies="no_cookies", show_progress=False,
                                   **kwargs)

    # Convert the response into DataFrame
    data = response.text.split("
")[-1].replace(' ', '').replace('\n', '').split("
")
    dataframe = pandas.DataFrame(data=[d.split(",")[:8] for d in data[1:] if len(d)>0], columns=data[0].split(",")[:8])
    # returns it
    if dirout is None or dirout in ["None"]:
        return dataframe

    # Directory given, then dump data there:
    if dirout in ["default"]:
        dirout = target_lightcurves_directory(name)

    if verbose: print("Data will be stored here: %s"%dirout)
    if not os.path.exists(dirout):
        os.makedirs(dirout)

    dataframe.to_csv(dirout+"marshal_lightcurve_%s.csv"%name)

#############################
#                           #
#   Marshall Class          #
#                           #
#############################

class MarshalAccess( object ):
    """ Access the Marshal """
    def __init__(self, load_programs=False, **kwargs):
        """

        """
        if load_programs:
            self.load_user_programs( **kwargs )

    # -------------- #
    #  Main Methods  #
    # -------------- #
    #
    # DOWNLOADER
    #
    def download_spectra(self, name, dirout="default", auth=None,  **kwargs):
        """

        Method calling ztfquery.marshal.download_spectra()

        Parameters:
        -----------
        name: [str or list of]
            Name of a target on the marshal.

        dirout: [str] -optional-
            Directory where the data should be stored.
            Additional options:
            - `dirout=None`: The spectra are not saved be returned
            - `dirout='default'`: The spectra will be saved in native target location
                                  (`$ZTFDATA`/marshal/spectra/`name`)
                                 Spectra saved here can be recovered using `get_local_spectra`
                                 * This is favored *

        auth: [str,str] -optional-
            Marshal [username, password]

        **kwargs goes to ztfquery.io.download_single_url()

        Returns
        -------
        dict
        // {name: `return_of ztfquery.marshal.download_spectra()`}
        """
        return {name_: download_spectra(name, dirout=dirout, auth=auth, **kwargs) for name_ in np.atleast_1d(name)}

    def download_lightcurve(self, name, dirout="default", auth=None,  **kwargs):
        """

        Method calling ztfquery.marshal.download_lightcurve()

        Parameters:
        -----------
        name: [str or list of]
            Name of a target on the marshal.

        dirout: [str] -optional-
            Directory where the data should be stored.
            Additional options:
            - `dirout=None`: The spectra are not saved be returned
            - `dirout='default'`: The lightcurve will be saved in native target location
                              (`$ZTFDATA`/marshal/lightcurves/`name`)
                              lightcurve saved here can be recovered using `get_local_lightcurve`
                              * This is favored *

        auth: [str,str] -optional-
            Marshal [username, password]

        **kwargs goes to ztfquery.io.download_single_url()

        Returns
        -------
        dict
        // {name: `return_of ztfquery.marshal.download_lightcurve()`}
        """
        return {name_: download_lightcurve(name, dirout=dirout, auth=auth, **kwargs) for name_ in np.atleast_1d(name)}

    #
    # LOADER
    #
    def load_user_programs(self, auth=None):
        """ """
        if auth is None:
           auth = io._load_id_("marshal", askit=True)

        r = requests.post(MARSHAL_BASEURL+'list_programs.cgi',  auth=auth)
        r.raise_for_status() # raise a status if issue, like wrong auth

        self.program_data = pandas.DataFrame.from_dict(json.loads(r.text))

    def load_target_sources(self, program="*",
                            getredshift=True, getclassification=True,
                            setit=True, auth=None):
        """ download target source information and store them as a
            pandas.DataFrame as self.target_sources
            (or returns it, see setit parameter)

        Parameters
        ----------
        program: [str or list of] --optional--
            You want targets only associated to this program?
            e.g. program="Redshift Completeness Factor"
                 program=["AMPEL Test","Redshift Completeness Factor"]

            -> use program = None or program="*" for no program selection

        getredshift, getclassification: [bool, bool] -optional-
            If redshift and/or classification have been made in the marshal,
            do you want them ?

        setit: [bool] -optional-
            Do you want to set downloaded data to self.target_sources (`setit=True`, default)
            or would you prefer no directly get the pandas.DataFrame without
            touching to self.target_sources ? (`setit=False`)

        auth: [str,str] -optional-
            Marshal's [username, password]
            CAUTION: if you are requesting program(s), make sure the `auth`
                      matches that of your loaded program if already loaded.
                      Remark: If you did not load the user_program yet
                      (`self.load_user_programs()`),  they are authomatically matched.
        Returns
        -------
        None (or pandas.DataFrame if setit=False, see above)
        """

        r = requests.post(MARSHAL_BASEURL+'list_program_sources.cgi',
                       auth=io._load_id_("marshal", askit=True) if auth is None else auth,
                       data={'programidx': self._program_to_programidx_(program, auth=auth),
                             'getredshift': int(getredshift), 'getclassification': int(getclassification)})

        if setit:
            self.set_target_sources( pandas.DataFrame.from_dict(json.loads(r.text)) )
        else:
            return pandas.DataFrame.from_dict(json.loads(r.text))

    #
    # SETTER
    #
    def set_target_sources(self, target_source_dataframe):
        """ Provide a Pandas.DataFrame containing the target source information
            as obtained by `load_target_sources`
        """

        self.target_sources =  target_source_dataframe

    #
    # GETTER
    #
    def get_target_data(self, name):
        """ target_sources entry corresponding to the given name(s)

        Parameters
        ----------
        name: [str or list of]
            one or several target name(s) from `target_sources`

        Returns
        -------
        DataFrame Row(s)
        """
        return self.target_sources[self.target_sources["name"].isin(np.atleast_1d(name))]

    def get_target_coordinates(self, name):
        """ Target(s) coordinates ["ra", "dec"] in degree

        [simply  `self.get_target_data(name)[["ra","dec"]]` ]

        Parameters
        ----------
        name: [str or list of]
            one or several target name(s) from `target_sources`

        Returns
        -------
        Ra, Dec
        """
        return self.get_target_data(name)[["ra","dec"]]

    def get_target_redshift(self, name):
        """ Target(s) redshift as saved in the Marshal.
        # Caution, Redshift in the marshal might not be accurate. #

        [simply  `self.get_target_data(name)[["redshift"]]` ]

        Parameters
        ----------
        name: [str or list of]
            one or several target name(s) from `target_sources`

        Returns
        -------
        Redshift
        """
        return self.get_target_data(name)["redshift"]

    def get_target_classification(self, name):
        """

        [simply  `self.get_target_data(name)[["classification"]]` ]

        Parameters
        ----------
        name: [str or list of]
            one or several target name(s) from `target_sources`

        Returns
        -------
        classification
        // SN Ia, SN Ia 91bg-like, SN Ib, AGN, None, etc.
        """
        return self.get_target_data(name)["classification"]

    # -------------- #
    #  Internal      #
    # -------------- #
    def _program_to_programidx_(self, program, **kwargs):
        """ Returns the programidx corresponding to your program """

        if program is None or program in ['*','all']:
            programidx = "*"
        else:
            if not hasattr(self, "program_data"):
                self.load_user_programs(**kwargs)

            program  = np.atleast_1d(program)
            programidx = self.program_data[self.program_data["name"].isin(program)]["programidx"].values
            if len(programidx) == 0:
                raise ValueError("None of you program corresponds to "+", ".join(list(program)) +"\n"+
                                     "Your programs: "+", ".join(list(self.programs)))

        return programidx

    # =================== #
    #   Properties        #
    # =================== #
    # Programs
    @property
    def programs(self):
        """ """
        if not hasattr(self, "program_data"):
            raise AttributeError("You did not load the program_data. See load_user_programs()")
        return self.program_data["name"].values

    # Sources
    @property
    def nsources(self):
        """ """
        if not hasattr(self, "target_sources"):
            raise AttributeError("You did not load/set the target_sources. See {load,set}_target_sources()")
        return len(self.target_sources)


"""
Functionality related to provide the virtual duty phone for Baljan.

Incoming calls are first routed to the current staff on duty. If they are
busy the call will be routed to the other staff that is on duty the current
week. If both members on duty are busy, or if a call is made outside of
office hours, the call will be routed to a backup list stored in the database.
"""
import pytz
from datetime import date, datetime, time
from django.conf import settings

from cafesys.baljan import planning
from cafesys.baljan.models import Shift, IncomingCallFallback

tz = pytz.timezone(settings.TIME_ZONE)

# Mapping from office hours to shift indexes
DUTY_CALL_ROUTING = {
    (time(7, 0, 0, tzinfo=tz), time(12, 0, 0, tzinfo=tz)): 0,
    (time(12, 0, 0, tzinfo=tz), time(13, 0, 0, tzinfo=tz)): 1,
    (time(13, 0, 0, tzinfo=tz), time(18, 0, 0, tzinfo=tz)): 2,
}

def compile_incoming_call_response():
    """
    Compiles a response message to an incoming call. The algorithm for this
    response is found in the file header.
    """

    phone_numbers = []
    current_duty_phone_numbers = _get_current_duty_phone_numbers()

    # Check if we are within office hours
    if current_duty_phone_numbers is not None:
        _append(phone_numbers, current_duty_phone_numbers)
        _append(phone_numbers, _get_week_duty_phone_numbers())

    # Always append the fallback numbers
    _append(phone_numbers, _get_fallback_numbers())

    return _build_46elks_response(phone_numbers)

def _get_fallback_numbers():
    """Retrieves the list of fallback phone numbers from the database"""

    return [x.user.profile.mobile_phone for x in IncomingCallFallback.objects.all()]

def _get_current_duty_phone_numbers():
    """
    Returns the phone number for every staff on duty at the moment,
    or None if outside office hours.
    """

    current_time = datetime.now(tz).time()
    shifts_today = Shift.objects.filter(when=date.today())

    for time_range, shift_index in DUTY_CALL_ROUTING.items():
        if _time_in_range(time_range[0], time_range[1], current_time):
            on_callduty = shifts_today.filter(span=shift_index).first().on_callduty()
            return [x.profile.mobile_phone for x in on_callduty]

    return None

def _get_week_duty_phone_numbers():
    """Returns the phone number for every staff on duty this week"""

    plan = planning.BoardWeek.current_week()
    on_callduty = [item for sublist in plan.oncall() for item in sublist]

    return [x.profile.mobile_phone for x in on_callduty]

def _time_in_range(start, end, x):
    """Return true if x is in the range [start, end]"""

    if start <= end:
        return start <= x <= end
    else:
        return start <= x or x <= end

def _append(lst, element):
    """Appends the phone number of list of phone numbers to the given list lst"""

    if isinstance(element, list):
        for e in element:
            _append(lst, e)
    elif element:
        element = _format_phone(element)
        if element not in lst:
            lst.append(element)

def _format_phone(phone):
    """Makes sure that the number starts with an area code (needed by 46elks API)"""

    if phone[0] == '+':
        return phone
    else:
        return '+46' + phone[1:]

def _build_46elks_response(phone_numbers):
    """Builds a response message compatible with 46elks.com"""

    if phone_numbers:
        data = {
            'connect': phone_numbers[0],
            'callerid': '+46766860043'
        }

        busy = _build_46elks_response(phone_numbers[1:])
        if busy:
            data['timeout'] = '20'
            data['busy'] = busy
            data['failed'] = busy

        return data
    else:
        return {}


from PIL import Image, ImageDraw

try:
	rand = random.getrandbits(31) * 1
	rand1 = random.getrandbits(31) * 1
	rand2 = random.getrandbits(31) * 1
	upload = vk_api.VkUpload(vk_group)
	wget.download(event.obj.attachments[0]['photo']['sizes'][4]['url'], 'temp/'+str(rand)+'.png', bar=None)
	image_obj1 = Image.open('temp/'+str(rand)+'.png')
	image2 = image_obj1.crop([0,0,int(image_obj1.size[0]/2),int(image_obj1.size[1])])
	image2 = image2.transpose(Image.FLIP_LEFT_RIGHT)
	image_obj1.paste(image2,(int(image_obj1.size[0]/2),0))
	image_obj1.save('temp/'+str(rand1)+'.png')
	photo1 = upload.photo_messages('temp/'+str(rand1)+'.png')
	image_obj2 = Image.open('temp/'+str(rand)+'.png')
	image3 = image_obj2.transpose(Image.FLIP_LEFT_RIGHT)
	image3 = image3.crop([0,0,int(image_obj2.size[0]/2),int(image_obj2.size[1])])
	image3 = image3.transpose(Image.FLIP_LEFT_RIGHT)
	image_obj2 = image_obj2.transpose(Image.FLIP_LEFT_RIGHT)
	image_obj2.paste(image3,(int(image_obj2.size[0]/2),0))
	image_obj2.save('temp/'+str(rand2)+'.png')
	photo2 = upload.photo_messages('temp/'+str(rand2)+'.png')
	send(user, 'Вот фото:', 'photo{}_{},photo{}_{}'.format(photo1[0]['owner_id'], photo1[0]['id'],photo2[0]['owner_id'], photo2[0]['id']))
	os.remove('temp/'+str(rand)+'.png')
	os.remove('temp/'+str(rand1)+'.png')
	os.remove('temp/'+str(rand2)+'.png')
except KeyError:
	send(user, 'Отправьте фотографию.')


from flask import Flask, render_template, g
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager, current_user

# Define the WSGI application object
application = Flask(__name__)

# aws likes 'application' but I want to use 'app'
# this is a hack to make us both happy
app = application

# Configurations
app.config.from_object('config')

# Strip spacing in templates
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True

# Define the database object which is imported
# by modules and controllers
db = SQLAlchemy(app)

from main.models import User

# login manager to be used for authentication
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'auth.login'

# HTTP error handling
@app.errorhandler(404)
def not_found(error):
    return render_template('auth/404.html'), 404

from main.models import User
@login_manager.user_loader
def load_user(id):
    return User.query.get(int(id))

@app.before_request
def before_request():
    g.user = current_user

# Import a module / component using its blueprint handler variable (mod_auth)
from app.main.controllers import main
from app.main.auth_controllers import auth

# Register blueprint(s)
app.register_blueprint(main)
app.register_blueprint(auth)

# Build the database:
# This will create the database file using SQLAlchemy
db.create_all()


#
# Debugging code:
#
import algebra
import tools
tools.cls()
expr = algebra.Sum([1, algebra.Sum([2, 3])])
simplified = expr.simplify()
assert (list(simplified) == [1,2,3]), "Sum([1, Sum([2, 3])]) :: Unexpected simplified result."

expr2 = algebra.Product([algebra.Product(['a', 'b']),'c'])
simplified = expr2.simplify()
assert (list(simplified) == ['c','a','b']), "Product([algebra.Product(['a', 'b']),'c']) :: Unexpected simplified result."

# a . (c + d)
#Representation:
#Product([a, Sum([c,d])])
expr3 = algebra.Product(['a', algebra.Sum(['c','d'])])
result = expr3.simplify()
print( result )

# a . (c + d) + b . ( c + d)
# a.c + a.d + b.c + b.d
expr3 = algebra.Sum([algebra.Product(['a', algebra.Sum(['c','d'])]), algebra.Product(['b', algebra.Sum(['c','d'])])])
result = expr3.simplify()
print( result )

#(a + b) . (c + d)
#Representation:
# Product([Sum([a,b]), Sum([c,d])])
# a . (c + d) + b . (c + d)
#Representation:
# Sum([Product([a, Sum([c,d])]), Product([b, Sum([c,d])])])
# Sum([Product(['a', 'c']), Product(['a', 'd']), Product(['b', 'c']), Product(['b', 'd'])])
expr3 = algebra.Product([algebra.Sum(['a','b']), algebra.Sum(['c','d'])])
result = expr3.simplify()
print( result )

#1) (2 * (x + 1) * (y + 3))
#    (2 * x + 2 * 1) * (y + 3)
#2) ((2 * x * y) + (2 * x * 3) + (2 * 1 * y) + (2 * 1 * 3))
#Representation:
#1) Product([2, Sum(['x', 1]), Sum(['y', 3])])
#2) Sum([Product([2, 'x', 'y'], Product([2, 'x', 3]), Product([2, 1, 'y']), Product([2, 1, 3]))])
#Simplified:
#2) Sum([Product([2, 'x', 'y'], Product([2, 'x', 3]), Product([2, 'y']), Product([2, 3]))])
expr3 = algebra.Product([2, algebra.Sum(['x', 1]), algebra.Sum(['y', 3])])
result = expr3.simplify()
print( result )


from django.conf import settings
from django.core import serializers
from django.http import HttpResponse
from django.shortcuts import render, render_to_response
from gmap.utils import geolocate
from gmap.models import MapMarker

def showmap(request, address='', category=''):
    context = {}
    context['media_url'] = settings.MEDIA_URL

    if request.method == 'POST':
        address = request.POST.get('address', address)
        category = request.POST.get('category', category)
    if request.method == 'GET':
        address = request.GET.get('address', address)
        category = request.GET.get('category', category)

    if category:
        context['gmap_markers'] = MapMarker.objects.get(
                marker_type__category_name__iexact=category
                )
    else:
        context['gmap_markers'] = MapMarker.objects.all()

    if address:
        latlng = geolocate(address)
        if latlng:
            context['gmap_center_lat'] = latlng['latitude']
            context['gmap_center_lng'] = latlng['longitude']
        else:
            context['error'] = "Please try another address."

    return render(request, 'gmap.html', context)

def markers(request):
    data = serializers.serialize("json", MapMarker.objects.all(),use_natural_keys=True)
    return HttpResponse(data, mimetype='applicaton/javascript')

def gmap_search(request):
    context = {}
    return render(request, 'gmap_search.html', context)


from time import perf_counter
from numpy import zeros
from scipy import linalg
from laplaciana import laplaciana
import matplotlib.pyplot as plt

Ns = [10,20,50,100,200,500,1000,1500,2000,5000]

corridas = [1,2,3,4,5,6,7,8,9,10]

dts_inversion = [] #guardo los tiempos en que se invierte la matriz por corrida
mems = [] #guardo los bytes usados para invertir la matriz por corrida

txt = open("timing_inv_caso_3_double.txt","w")

for corrida in corridas:
    for N in Ns:

        t1= perf_counter()
        A = laplaciana(N)
        t2 = perf_counter()
        Am1 = linalg.inv(A,overwrite_a=True) #matriz inversa de A
        t3 = perf_counter()
        bytes_total = A.nbytes + Am1.nbytes

        dt_ensamblaje = t2 - t1
        dt_inversion = t3 - t2 #presentar tiempo de inversion en graf.

        dts_inversion.append(dt_inversion)
        mems.append(bytes_total)

        txt.write(f"N = {N}; Tiempo inversión = {dt_inversion} seg.; Uso de memoria = {bytes_total} bytes\n" )

        print ("N =",N)
        print (f"Uso memoria: {bytes_total} bytes")
        print (f"Tiempo ensamblaje: {dt_ensamblaje} seg")
        print (f"Tiempo inversion: {dt_inversion} seg")
        print ("----------------------------------------------------")

txt.close()

def separar_lista(lista,n):
    tiempos_grafico = []
    tiempos_grafico.append([lista[i*n:i*n+n] for i in range(n)])
    return tiempos_grafico

x1 = ["10","20","50","100","200","500","1000","2000","5000","10000","20000"]
y1 = ["0.1 ms","1 ms","10 ms","0.1 s","1 s","10 s","1 min","10 min"]
y2 = ["1 KB","10 KB","100 KB","1 MB","10 MB","100 MB","1 GB","10 GB"]

tiempo1 = [0.1/1000,1/1000,10/1000,0.1,1,10,60,60*10]
memoria1 = [1*10**3,10*10**3,100*10**3,1*10**6,10*10**6,100*10**6,1*10**9,10*10**9]
tamaño1 = [10,20,50,100,200,500,1000,2000,5000,10000,20000]

for i in range(len(corridas)):

    plt.subplot(2,1,1)
    plt.loglog(Ns,dts_inversion[i*10:i*10+10],"-o")
    plt.xticks(tamaño1,x1)
    plt.yticks(tiempo1,y1)
    plt.xlim(right=20000)
    plt.grid()
    plt.ylabel("Tiempo inversión (s)")

    plt.subplot(2,1,2)
    plt.loglog(Ns,mems[i*10:i*10+10],"-o")
    plt.xticks(tamaño1,x1)
    plt.yticks(memoria1,y2)
    plt.grid()
    plt.ylabel("Uso memoria")
    plt.xlabel("Tamaño matriz N")
    plt.xlim(right=20000)
    plt.hlines(y=10**8,xmin=0,xmax=2000,linestyle="--")

plt.show()


import abc
import os
import six

import mako.exceptions
import mako.template

from stevedore import extension

@six.add_metaclass(abc.ABCMeta)
class BaseFormatter(object):
    """Base class for the formatters."""
    def __init__(self):
        self.template_extension = ''
        self.templates = []
        self.templates_files = {}

    def _load_collectors(self, opts, providers):
        mgr = extension.ExtensionManager(
            namespace='cip.collectors',
            invoke_on_load=True,
            invoke_args=(opts, providers),
        )
        return dict((x.name, x.obj) for x in mgr)

    def _load_templates(self, template_dir):
        self.templates_files = {}
        for tpl in self.templates:
            template_file = os.path.join(
                template_dir,
                '%s.%s' % (tpl, self.template_extension))
            self.templates_files[tpl] = template_file

    def _format_template(self, template, info, extra={}):
        info = info.copy()
        info.update(extra)
        t = self.templates_files[template]
        tpl = mako.template.Template(filename=t)
        try:
            return tpl.render(attributes=info)
        except Exception:
            return mako.exceptions.text_error_template().render()

    def to_stdout(self, template):
        print(template)

    def format(self, opts, providers):
        available_collectors = self._load_collectors(opts, providers)
        self._load_templates(opts.template_dir)
        for tpl in self.templates:
            _collector = available_collectors[tpl]
            info = _collector.fetch()
            if info:
                extra_info = {
                    'middleware': opts.middleware,
                    'dynamic_provider': _collector.dynamic_provider}
                self.to_stdout(
                    self._format_template(tpl,
                                          info,
                                          extra_info).encode('utf-8'))


#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import logging
import optparse
import os
import sys
import time

ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
ACTIVATE_SCRIPT = os.path.join(ROOT_PATH, 'jenkins-env', 'bin', 'activate_this.py')
TREEHERDER_CONFIG_FILE = os.path.join(ROOT_PATH, '.jenkins.properties')

def main():
    parser = optparse.OptionParser()
    parser.add_option('--debug',
                      dest='debug',
                      action='store_true',
                      default=False)
    parser.add_option('--log-folder',
                      dest='log_folder',
                      default='log',
                      help='Folder to write notification log files into')
    parser.add_option('--log-level',
                      dest='log_level',
                      default='INFO',
                      help='Logging level, default: %default')
    parser.add_option('--authfile',
                      dest='authfile',
                      default='.authentication.ini',
                      help='Path to the authentiation file for Pulse Guardian and Jenkins')
    parser.add_option('--push-message',
                      dest='message',
                      help='Log file of a Pulse message to process for Jenkins')
    parser.add_option('--display-only',
                      dest='display_only',
                      action='store_true',
                      default=False,
                      help='Only display build properties and don\'t trigger jobs.')
    options, args = parser.parse_args()

    if not len(args):
        parser.error('A configuration file has to be passed in as first argument.')

    logging.Formatter.converter = time.gmtime
    logging.basicConfig(level=options.log_level,
                        format='%(asctime)s %(levelname)5s %(name)s: %(message)s',
                        datefmt='%Y-%m-%dT%H:%M:%SZ')
    logger = logging.getLogger('mozmill-ci')

    # Configure logging levels for sub modules. Set to ERROR by default.
    sub_log_level = logging.ERROR
    if options.log_level == logging.getLevelName(logging.DEBUG):
        sub_log_level = logging.DEBUG
    logging.getLogger('mozdownload').setLevel(sub_log_level)
    logging.getLogger('redo').setLevel(sub_log_level)
    logging.getLogger('requests').setLevel(sub_log_level)
    logging.getLogger('thclient').setLevel(sub_log_level)

    # Auto import the virtual environment so the script can directly
    # be called without having to source into first.
    try:
        execfile(ACTIVATE_SCRIPT, dict(__file__=ACTIVATE_SCRIPT))
        logger.info('Virtual environment activated successfully.')
    except IOError:
        logger.exception('Could not activate virtual environment at "{}"'.format(ACTIVATE_SCRIPT))
        sys.exit(1)

    from lib.automation import FirefoxAutomation

    FirefoxAutomation(configfile=args[0],
                      authfile=options.authfile,
                      treeherder_configfile=TREEHERDER_CONFIG_FILE,
                      debug=options.debug,
                      log_folder=options.log_folder,
                      logger=logger,
                      message=options.message,
                      display_only=options.display_only)

if __name__ == "__main__":
    main()


from rest_framework.decorators import api_view
from rest_framework.response import Response

from vehicles.models import Tool
from vehicles.serializers import ToolSerializer

@api_view(['GET'])
def list_tools(request):
    tools = [
        Tool('hammer', 'Mastercraft'),
        Tool('wrench', 'Husky'),
    ]

    serializer = ToolSerializer(tools, many=True)
    content = {
        'tools': serializer.data,
    }
    return Response(content)


from infra.system.core.meta.metaprocessor import MetaProcessor
import time
import datetime
import logging
from infra.system.helpers.artefact_schema_validator import ArtefactSchemaValidator
from infra.system.helpers.schema_validator import SchemaValidator
from infra.system.helpers.utils import getScriptDir
from infra.system.artefacts.artefacts import ARTEFACT_GOLANG_PROJECT_REPOSITORY_INFO, ARTEFACT_GOLANG_PROJECT_REPOSITORY_COMMIT

from gofed_lib.utils import dateToTimestamp
from gofed_lib.repositoryclientbuilder import RepositoryClientBuilder

class RepositoryDataExtractor(MetaProcessor):

	def __init__(self):
		self.input_validated = False

		self.repository = {}
		self.repository_directory = ''
		self.start_date = ''
		self.end_date = ''
		self.commit = ''

		self.client = None

		self.branches = []
		self.commits = {}

	def setData(self, data):
		self.input_validated = False
		if not self._validateInput(data):
			return False

		self.repository = data['repository']
		self.repository_directory = data['resource']

		self.client = RepositoryClientBuilder().buildWithLocalClient(self.repository, self.repository_directory)

		# TODO(jchaloup): check the date is in required format
		if 'start_date' in data:
			self.start_date = dateToTimestamp(data["start_date"])
		else:
			self.start_date = dateToTimestamp('1970-01-02')

		if 'end_date' in data:
			self.end_date = dateToTimestamp(data["end_date"])
		else:
			self.end_date = int(time.time() + 86400)

		if 'start_timestamp' in data:
			self.start_date = data["start_timestamp"]

		if 'end_timestamp' in data:
			self.end_date = data["end_timestamp"]

		# Check single commit only?
		self.commit = ""
		if 'commit' in data:
			self.commit = data["commit"]

		# Extract data from a single branch?
		self.branch = ""
		if 'branch' in data:
			self.branch = data["branch"]

		return True

	def _generateGolangProjectRepositoryInfo(self, branches):
		data = {
			"artefact": ARTEFACT_GOLANG_PROJECT_REPOSITORY_INFO,
			"repository": self.repository,
			"branches": []
		}

		for branch in branches:
			data["branches"].append({
				"branch": branch,
				"commits": branches[branch]
			})

		start_date = self.end_date
		end_date = self.start_date
		for branch in self.commits:
			for commit in self.commits[branch]:
				start_date = min(start_date, self.commits[branch][commit]["cdate"])
				end_date = max(end_date, self.commits[branch][commit]["cdate"])

		data["coverage"] = [{"start": start_date, "end": end_date}]

		return data

	def _generateGolangProjectRepositoryCommit(self, commit):
		data = {}

		data['artefact'] = ARTEFACT_GOLANG_PROJECT_REPOSITORY_COMMIT
		data['repository'] = self.repository
		data['commit'] = commit["hexsha"]

		# keep timestamps
		data['adate'] = commit["adate"]
		data['cdate'] = commit["cdate"]
		data['author'] = commit["author"]
		data['message'] = commit["message"]

		return data

	def getData(self):
		if not self.input_validated:
			return []

		if self.commit != "":
			return self._generateGolangProjectRepositoryCommit(self.commits[""])

		commits_data = {}
		branches = {}
		# TODO(jchaloup) this is quite redundant, make it better!!!
		for branch in self.commits:
			for commit in self.commits[branch]:
				try:
					branches[branch].append(commit)
				except KeyError:
					branches[branch] = [commit]

				if commit in commits_data:
					continue

				commits_data[commit] = self._generateGolangProjectRepositoryCommit(self.commits[branch][commit])

		# from all branches (up to master) filter out all commits that are already covered in master branch
		if "master" in branches:
			for branch in filter(lambda l: l != "master", branches.keys()):
				branches[branch] = list(set(branches[branch]) - set(branches["master"]))

		# TODO(jchaloup): move validation to unit-tests
		#for commit in commits_data:
		#	validator = ArtefactSchemaValidator(ARTEFACT_GOLANG_PROJECT_REPOSITORY_COMMIT)
		#	if not validator.validate(commits_data[commit]):
		#		logging.error('%s is not valid' % ARTEFACT_GOLANG_PROJECT_REPOSITORY_COMMIT)
		#		return {}

		info = self._generateGolangProjectRepositoryInfo(branches)
		validator = ArtefactSchemaValidator(ARTEFACT_GOLANG_PROJECT_REPOSITORY_INFO)
		if not validator.validate(info):
			logging.error('%s is not valid' % ARTEFACT_GOLANG_PROJECT_REPOSITORY_INFO)
			return {}

		repo_commits = []
		for commit in commits_data:
			repo_commits.append(commits_data[commit])

		return {
			"info": info,
			"commits": repo_commits,
			"branches": branches
		}

	def _validateInput(self, data):
		validator = SchemaValidator()
		schema = '%s/input_schema.json' % getScriptDir(__file__)
		self.input_validated = validator.validateFromFile(schema, data)
		return self.input_validated

	def execute(self):
		if self.commit != "":
			self.commits[""] = self.client.commit(self.commit)
			return True

		self.branches = self.client.branches()

		self.commits = {}

		# just a single branch
		if self.branch != "":
			if self.branch not in self.branches:
				raise ValueError("Requested branch '%s' not found" % self.branch)

			self.commits[self.branch] = self.client.commits(self.branch, since=self.start_date, to=self.end_date)
			return True

		# all branches
		for branch in self.branches:
			self.commits[branch] = self.client.commits(branch, since=self.start_date, to=self.end_date)

		return True


#!/usr/bin/env python

"""
Phil Adams http://philadams.net

As always, `-h` is your friend.

Given a .csv file of VERA+ statuses, locally archive all default-sized images.

TODO: hook logging back up to args.verbose and args.quiet
"""

import logging
import os
from StringIO import StringIO
import csv
import json
import time

from pprint import pprint

import requests
from PIL import Image

def main():
    import argparse

    # populate and parse command line options
    parser = argparse.ArgumentParser(description='Image archiving for VERA+')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('-v', '--verbose', action='count', default=0)
    group.add_argument('-q', '--quiet', action='store_true')
    group.add_argument('--outdir', help='dir in which to archive images')
    parser.add_argument('config', help='config params (json)')
    parser.add_argument('statuses', help='veraplus statuses (csv)')
    args = parser.parse_args()

    # load config params
    with open(args.config) as f:
        config = json.load(f)
        config['outdir'] = args.outdir or config.get('outdir', './gen')
        if not os.path.exists(config['outdir']):
            os.makedirs(config['outdir'])

    # logging config
    log_level = logging.WARNING  # default
    if args.verbose == 1:
        log_level = logging.INFO
    elif args.verbose >= 2:
        log_level = logging.DEBUG
    logging.basicConfig(level=logging.WARNING)  # global logging settings
    fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
    fmt_t = logging.Formatter('%(asctime)s - %(levelname)s:%(name)s:%(message)s')
    log_ch = logging.StreamHandler()
    log_ch.setLevel(logging.DEBUG)  # edit for debugging
    log_ch.setFormatter(fmt)
    log_fh = logging.FileHandler('%s/log.log' % config['outdir'])
    log_fh.setLevel(logging.WARNING)
    log_fh.setFormatter(fmt_t)
    imgdl_log = logging.getLogger('idl-tools.imagearchiver')
    imgdl_log.propagate = False
    imgdl_log.setLevel(logging.DEBUG)
    imgdl_log.addHandler(log_fh)
    imgdl_log.addHandler(log_ch)

    # load all the statuses into memory
    with open(args.statuses, 'rU') as f:
        statuses = [row for row in csv.reader(f)]

    fields = statuses.pop(0)  # header row
    for i, row in enumerate(statuses):
        if i % 25 == 0:
            imgdl_log.info('processing %d/%d' % (i+1, len(statuses)))

        status = dict(zip(fields, row))
        status['img_uri'] = '%s%s' % (config['urlbase'], status['image_path'])
        imgdir = '%s/%s' % (config['outdir'], status['user_id'])
        if not os.path.exists(imgdir):
            os.makedirs(imgdir)

        req = requests.get(status['img_uri'])
        time.sleep(0.1)
        if not req.ok:
            imgdl_log.error('sid%s img http req (%s)' % (status['status_id'],
                                                         status['img_uri']))
            continue
        try:
            img = Image.open(StringIO(req.content))
            img.save('%s/u%s_s%s.jpg' % (imgdir,
                                         status['user_id'],
                                         status['status_id']))
        except IOError as e:
            imgdl_log.error('sid%s img persist (%s)' % (status['status_id'],
                                                          status['img_uri']))
            continue

if '__main__' == __name__:
    main()


import os
import sys
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import accuracy_score
import numpy as np
import random
import pandas as pd
import collections
import pickle
from sklearn.ensemble import BaggingClassifier

#import io
#sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8')

current_dir = os.path.dirname(os.path.abspath(__file__))
top_dir = os.path.dirname(current_dir)
sys.path.append(top_dir)
data_dir = os.path.join(top_dir, 'data')
from other_funcs.nltk_funcs import preprocessing_word, tokenize_word, add_n_start
from other_funcs.cv import create_k_fold

IS_READ_FROM_TFIDF = False
IS_LDA = False
print ("IS_READ_FROM_TFIDF: ", IS_READ_FROM_TFIDF)

if IS_READ_FROM_TFIDF:
    pass
else:
    train_data_path = os.path.join(data_dir, 'train.csv')
    train_df = pd.read_csv(train_data_path)
    all_labels = train_df['author'].values
    all_texts = train_df['text'].values
    processed_texts = []

    # transform
    for text in all_texts:
        word_list = tokenize_word(text)
        for i, word in enumerate(word_list):
            word = word.lower()
            word_list[i] = preprocessing_word(word)

        word_list = add_n_start(word_list)
        processed_text = ' '.join(word_list)
        processed_texts.append(processed_text)

    X_Y = list(zip(processed_texts, all_labels))
    print ("Zip X and y successfully!")

# ----------------------------------------------------------------------------------------------------------------------
# LDA hyper parameters
# ----------------------------------------------------------------------------------------------------------------------
def LDA_n_topics(N):
    n_topics_list = list(range(10, 200))
    for i in range(N):
        n_topics = random.sample(n_topics_list, 1)[0]
        yield n_topics

# ----------------------------------------------------------------------------------------------------------------------

# ----------------------------------------------------------------------------------------------------------------------
# hyper parameters
# ----------------------------------------------------------------------------------------------------------------------

# MLP
def learning_rate_generator(N):
    for i in range(N):
        learning_rate = random.uniform(0.0001,0.001)
        yield learning_rate

def hidden_layer_1_size(N):
    for i in range(N):
        size = random.randint(500,1200)
        yield size

def hidden_layer_2_size(N):
    for i in range(N):
        size = random.randint(4, 7)
        yield size

def alpha_generator(N):
    for i in range(N):
        yield random.uniform(0.0001,0.001)

def early_stopping_generator(N):
    early_stopping_list = [True]
    for i in range(N):
        yield random.sample(early_stopping_list, 1)[0]

def validation_fraction_generator(N):
    for i in range(N):
        yield random.uniform(0.2, 0.4)

# TFIDF
def max_n_gram(N):
    max_n_gram_list = [2,3]
    for i in range(N):
        yield random.sample(max_n_gram_list, 1)[0]

def max_features_generator(N):
    for i in range(N):
        yield random.randint(30000, 50000)

def token_pattern_generator(N):
    token_pattern_list = [r"[\w']+|[.,!?;]"]
    #token_pattern_list = [r"[\w']+|[.,!?;]", r"(?u)\b\w\w+\b"]
    for i in range(N):
        yield random.sample(token_pattern_list, 1)[0]
# ----------------------------------------------------------------------------------------------------------------------

N = 15
hyper_parameter_df = []
hyper_parameter_name_list = ['LDA_n_topics','learning_rate_init','hidden_layer_1_size','hidden_layer_2_size',
                             'alpha','max_n_gram','early_stopping','validation_fraction','max_features',
                             'token_pattern',
                             'avg_accuracy']

for generator_tuple in zip(LDA_n_topics(N), learning_rate_generator(N), hidden_layer_1_size(N), hidden_layer_2_size(N),
                           alpha_generator(N), max_n_gram(N), early_stopping_generator(N),
                           validation_fraction_generator(N), max_features_generator(N), token_pattern_generator(N)):

    generator_tuple = list(generator_tuple)
    LDA_n_topics = generator_tuple[0]
    learning_rate_init = generator_tuple[1]
    hidden_layer_1_size = generator_tuple[2]
    hidden_layer_2_size = generator_tuple[3]
    alpha = generator_tuple[4]
    early_stopping = generator_tuple[6]
    if early_stopping:
        generator_tuple[6] = 1 # map True to 1
        validation_fraction = generator_tuple[7]
    else:
        generator_tuple[6] = 0 # map False to 0
        generator_tuple[7] = 0.0
        validation_fraction = 0.0

    # tfidf
    max_n_gram = generator_tuple[5]
    max_features = generator_tuple[8]
    token_pattern = generator_tuple[9]
    if token_pattern == r"[\w']+|[.,!?;]":
        generator_tuple[9] = 1 #mapping
    elif token_pattern == r"(?u)\b\w\w+\b":
        generator_tuple[9] = 2 #mapping
    else:
        raise Exception("Check token_pattern!!")

    # ----------------------------------------------------------------------------------------------------------------------
    # set MLP
    # ----------------------------------------------------------------------------------------------------------------------
    hidden_layer_sizes = (hidden_layer_1_size, hidden_layer_2_size)
    verbose = False
    tol = 1e-5
    max_iter = 1000
    random_state = 999
    mlp1 = MLPClassifier(hidden_layer_sizes=hidden_layer_sizes, learning_rate_init=learning_rate_init, verbose=verbose,
                         tol=tol, max_iter=max_iter, alpha=alpha, random_state=random_state, early_stopping=early_stopping,
                         validation_fraction=validation_fraction)
    # ----------------------------------------------------------------------------------------------------------------------

    # ----------------------------------------------------------------------------------------------------------------------
    # LDA
    # ----------------------------------------------------------------------------------------------------------------------
    lda = LatentDirichletAllocation(n_topics=LDA_n_topics, learning_method='online')
    # ----------------------------------------------------------------------------------------------------------------------

    # ----------------------------------------------------------------------------------------------------------------------
    # TFIDF
    # ----------------------------------------------------------------------------------------------------------------------
    tf_vectorizer = TfidfVectorizer(ngram_range=(1, max_n_gram), max_features=max_features, token_pattern=token_pattern)
    # ----------------------------------------------------------------------------------------------------------------------

    # CV

    cv_accuracy_list = []
    tfidf_k_fold_list =[]

    if IS_READ_FROM_TFIDF:
        tfidf_k_fold_tuple = pickle.load(open('tfidf_k_fold', 'rb'))
        for train, test in tfidf_k_fold_tuple:
            X_train, Y_train = train
            X_test, Y_test = test

            if IS_LDA:
                # convert to LDA
                lda.fit(X_train)
                X_train = lda.transform(X_train)
                X_test = lda.transform(X_test)
                print("LDA done!")

            # train by classifier
            mlp1.fit(X_train, Y_train)
            print("Training done!")

            # predict
            Y_test_pred = list(mlp1.predict(X_test))
            accuracy = accuracy_score(Y_test, Y_test_pred)
            cv_accuracy_list.append(accuracy)
            print("accuracy: {}".format(accuracy))
    else:
        for train, test in create_k_fold(X_Y, k_fold=5):
            # read train and test value
            X_train, Y_train = list(zip(*train))
            X_test, Y_test = list(zip(*test))

            # convert train X to tfidf
            tf_vectorizer.fit(X_train)
            X_train = tf_vectorizer.transform(X_train)
            X_test = tf_vectorizer.transform(X_test)
            print("Tfidf done!")

            # tfidf_k_fold_list.append(((X_train, Y_train), (X_test, Y_test)))
            # continue

            if IS_LDA:
                # convert to LDA
                #lda = LatentDirichletAllocation(n_topics=LDA_n_topics, learning_method='online')
                lda.fit(X_train)
                X_train = lda.transform(X_train)
                X_test = lda.transform(X_test)
                print("LDA done!")

            # train by classifier
            mlp1.fit(X_train, Y_train)
            print("Training done!")

            # predict
            Y_test_pred = list(mlp1.predict(X_test))
            accuracy = accuracy_score(Y_test, Y_test_pred)
            cv_accuracy_list.append(accuracy)
            print ("accuracy: {}".format(accuracy))

    avg_accuracy = np.average(cv_accuracy_list)
    hyper_parameter_value_list = list(generator_tuple) + [avg_accuracy]
    hyper_parameter_df.append(dict(zip(hyper_parameter_name_list, hyper_parameter_value_list)))

    print ("------------------------------------")
    print ("generator_tuple: ", generator_tuple)
    print ("avg_accuracy: {}".format(avg_accuracy))

    #pickle.dump(tuple(tfidf_k_fold_list), open('tfidf_k_fold', 'wb'))

    #sys.exit()

# LDA_n_topics = generator_tuple[0]
# learning_rate_init = generator_tuple[1]
# hidden_layer_1_size = generator_tuple[2]
# hidden_layer_2_size = generator_tuple[3]
# alpha = generator_tuple[4]
# early_stopping = generator_tuple[6]
# validation_fraction = generator_tuple[7]
# # tfidf
# max_n_gram = generator_tuple[5]
# max_features = generator_tuple[8]

hyper_parameter_df = pd.DataFrame(hyper_parameter_df, columns=hyper_parameter_name_list)
hyper_parameter_df = hyper_parameter_df.sort_values('avg_accuracy', ascending=False)
print (hyper_parameter_df)
hyper_parameter_df.to_csv('hyper_parameter_df.csv', index=False, )


import numpy as np
from keras.datasets import mnist
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import SimpleRNN, Activation, Dense
from keras.optimizers import Adam

a = np.zeros((70, 7))

for i in range(70):
    for j in range(7):
        if np.random.rand()>0.5:
            a[i][j] += 1

b = np.random.rand(70, 2)

data_array = np.hstack((a, b))
data_array = data_array[:, np.newaxis, :]
class_array = np.zeros((70, 6))
class_array_temp = np.random.randint(6, size=70)
for i in range(70):
    class_array[i][class_array_temp[i]] = 1
# 70 data, 60 train, 10 test
X_train = data_array[0:60, :, :]
X_test = data_array[60:70, :, :]
y_train = class_array[0:60, :]
y_test = class_array[60:70, :]

#build model
model = Sequential()

model.add(SimpleRNN(batch_input_shape=(None, 1, 9), units = 50))
model.add(Dense(units=6, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])
'''
for step in range(1, 51):

    loss = model.train_on_batch(X_train, y_train)

    # 每 500 批，顯示測試的準確率
    # 模型評估
    loss, accuracy = model.evaluate(X_test, y_test, batch_size=y_test.shape[0],
        verbose=False)
    print("test loss: {}  test accuracy: {}".format(loss,accuracy))
'''
X = X_test
predictions = model.predict_classes(X)
# get prediction result

accuracy = 0

for i in range(10):
    for j in range(6):
        if y_test[i][j] == 1:
            ans = j
    if ans == predictions[i]:
        accuracy += 1

print(accuracy/10)


from django import template

register = template.Library()

print('LOAD_CUSTOM_FILTERS')

@register.filter(name='get_key_value')
def get_key_value(d, key):
    for k, v in d:
        if k == key:
            return str(v)
    return ''

@register.filter(name='times')
def times(number):
    return range(number)


from django.contrib import admin
from .models import (Group,
	Facultet,
	Student,
	AcademicSubject,
	Teacher,
	HomeWork,
	DayOfWeek,
	Shedule)

# Register your models here.

@admin.register(Group)
class GroupAdmin(admin.ModelAdmin):
	""""""
	fields = ('group',)

@admin.register(Facultet)
class FacultetAdmin(admin.ModelAdmin):
	""""""
	fields = ('facultet',)

@admin.register(Student)
class StudentAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'user',
		'firstNameStudent',
		'secondNameStudent',
		'patronymicNameStudent',
		'groupOfStudent',
		'facultetOfStudent',
		'dateOfBirthOfStudent',
		)

@admin.register(AcademicSubject)
class AcademicSubjectAdmin(admin.ModelAdmin):
	""""""
	fields = ('nameOfAcademicSubject',)

@admin.register(Teacher)
class TeacherAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'firstNameTeacher',
		'secondNameTeacher',
		'patronymicNameTeacher',
		'teachAcademicSubject',
		'dateOfBirthOfTeacher',
		)

@admin.register(HomeWork)
class HomeWorkAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'academicSubject',
		'teacher',
		'group',
		'dateOfHomeWork',
		'textOfHomeWork',
		)

@admin.register(DayOfWeek)
class DayOfWeekAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'week',
		'dayOfWeek',
		'group',
		'firstLesson',
		'secondLesson',
		'thirdLesson',
		'fourthLesson',
		'fivethLesson',
		'sixthLesson',
		)

@admin.register(Shedule)
class SheduleAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'group',
		'week',
		'monday',
		'tuesday',
		'wednesday',
		'thursday',
		'friday',
		'saturday',
		)


import numpy as np
import matplotlib.pyplot as plt

# load iris data

columns=["sepal.length","sepal.width","petal.length","petal.width"]
name={'Setosa':0, 'Versicolor':1, 'Virginica':2}
X=np.loadtxt('iris.csv', delimiter=',', skiprows=1, converters={4:
	lambda x: name[x.decode().strip('"')]})
X1=X[X[:,-1]==0]
X2=X[X[:,-1]==1]
X3=X[X[:,-1]==2]

# plot

plt.title('IRIS - sepal.length')
plt.plot(X1[:,0],'ro-')
plt.plot(X2[:,0],'gs:')
plt.plot(X3[:,0],'b^--')
plt.legend(name.keys())
plt.xlabel('samples')
plt.ylabel('values')
plt.show()


# Copyright 2020 Lorna Authors. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import re
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.checkpoint as cp
from collections import OrderedDict

from .utils import densenet_params
from .utils import get_model_params
from .utils import bn_function_factory
from .utils import load_pretrained_weights

class DenseLayer(nn.Sequential):
    def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, memory_efficient=False):
        super(DenseLayer, self).__init__()
        self.add_module("norm1", nn.BatchNorm2d(num_input_features)),
        self.add_module("relu1", nn.ReLU(inplace=True)),
        self.add_module("conv1", nn.Conv2d(num_input_features, bn_size *
                                           growth_rate, kernel_size=1, stride=1,
                                           bias=False)),
        self.add_module("norm2", nn.BatchNorm2d(bn_size * growth_rate)),
        self.add_module("relu2", nn.ReLU(inplace=True)),
        self.add_module("conv2", nn.Conv2d(bn_size * growth_rate, growth_rate,
                                           kernel_size=3, stride=1, padding=1,
                                           bias=False)),
        self.drop_rate = drop_rate
        self.memory_efficient = memory_efficient

    def forward(self, *prev_features):
        bn_function = bn_function_factory(self.norm1, self.relu1, self.conv1)
        if self.memory_efficient and any(prev_feature.requires_grad for prev_feature in prev_features):
            bottleneck_output = cp.checkpoint(bn_function, *prev_features)
        else:
            bottleneck_output = bn_function(*prev_features)
        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))
        if self.drop_rate > 0:
            new_features = F.dropout(new_features, p=self.drop_rate,
                                     training=self.training)
        return new_features

class DenseBlock(nn.Module):
    def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, memory_efficient=False):
        super(DenseBlock, self).__init__()
        for i in range(num_layers):
            layer = DenseLayer(
                num_input_features + i * growth_rate,
                growth_rate=growth_rate,
                bn_size=bn_size,
                drop_rate=drop_rate,
                memory_efficient=memory_efficient,
            )
            self.add_module(f"denselayer{i + 1}", layer)

    def forward(self, init_features):
        features = [init_features]
        for name, layer in self.named_children():
            new_features = layer(*features)
            features.append(new_features)
        return torch.cat(features, 1)

class Transition(nn.Sequential):
    def __init__(self, num_input_features, num_output_features):
        super(Transition, self).__init__()
        self.add_module("norm", nn.BatchNorm2d(num_input_features))
        self.add_module("relu", nn.ReLU(inplace=True))
        self.add_module("conv", nn.Conv2d(num_input_features, num_output_features,
                                          kernel_size=1, stride=1, bias=False))
        self.add_module("pool", nn.AvgPool2d(kernel_size=2, stride=2))

class DenseNet(nn.Module):
    """Densenet-BC model class, based on
    `"Densely Connected Convolutional Networks" <https://arxiv.org/pdf/1608.06993.pdf>`_
    """

    def __init__(self, blocks_args=None, global_params=None):
        # growth_rate=32, block_config=(6, 12, 24, 16),
     #            num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000, memory_efficient=False):

        super().__init__()
        assert isinstance(blocks_args, list), "blocks_args should be a list"
        assert len(blocks_args) > 0, "block args must be greater than 0"
        self._global_params = global_params
        self._blocks_args = blocks_args

        # First convolution
        self.features = nn.Sequential(OrderedDict([
            ("conv0", nn.Conv2d(3, self._global_params.num_init_features, kernel_size=7, stride=2,
                                padding=3, bias=False)),
            ("norm0", nn.BatchNorm2d(self._global_params.num_init_features)),
            ("relu0", nn.ReLU(inplace=True)),
            ("pool0", nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),
        ]))

        # Each denseblock
        num_features = self._global_params.num_init_features
        for i, num_layers in enumerate(self._blocks_args):
            block = DenseBlock(
                num_layers=num_layers,
                num_input_features=num_features,
                bn_size=self._global_params.bn_size,
                growth_rate=self._global_params.growth_rate,
                drop_rate=self._global_params.drop_rate,
                memory_efficient=self._global_params.memory_efficient
            )
            self.features.add_module(f"denseblock{i + 1}", block)
            num_features = num_features + num_layers * self._global_params.growth_rate
            if i != len(self._blocks_args) - 1:
                trans = Transition(num_input_features=num_features,
                                   num_output_features=num_features // 2)
                self.features.add_module(f"transition{i + 1}", trans)
                num_features = num_features // 2

        # Final batch norm
        self.features.add_module("norm5", nn.BatchNorm2d(num_features))

        # Linear layer
        self.classifier = nn.Linear(num_features, self._global_params.num_classes)

        # Official init from torch repo.
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.constant_(m.bias, 0)

    def forward(self, x):
        features = self.features(x)
        out = F.relu(features, inplace=True)
        out = F.adaptive_avg_pool2d(out, (1, 1))
        out = torch.flatten(out, 1)
        out = self.classifier(out)
        return out

    @classmethod
    def from_name(cls, model_name, override_params=None):
        cls._check_model_name_is_valid(model_name)
        blocks_args, global_params = get_model_params(model_name, override_params)
        return cls(blocks_args, global_params)

    @classmethod
    def from_pretrained(cls, model_name, num_classes=1000):
        model = cls.from_name(model_name, override_params={"num_classes": num_classes})
        load_pretrained_weights(model, model_name, load_fc=(num_classes == 1000))
        return model

    @classmethod
    def get_image_size(cls, model_name):
        cls._check_model_name_is_valid(model_name)
        _, _, res = densenet_params(model_name)
        return res

    @classmethod
    def _check_model_name_is_valid(cls, model_name):
        """ Validates model name. None that pretrained weights are only available for
        the first four models (densenet{i} for i in 121,161,169,201) at the moment. """
        num_models = [121, 161, 169, 201]
        valid_models = ["densenet" + str(i) for i in num_models]
        if model_name not in valid_models:
            raise ValueError("model_name should be one of: " + ", ".join(valid_models))


try:
    import Image
except ImportError:       # pragma: no cover
    from PIL import Image # pragma: no cover

class Resizer(object):

    def __init__(self, expand=False, filter=Image.ANTIALIAS):
        self.expand = expand
        self.filter = filter

    def transformations(self, img, target_size):
        # should return a list of JSON-serializable commands
        # that are processed with transform method
        raise NotImplementedError

    def transform(self, img, transformation, params):
        # transformations MUST be idempotent
        if transformation == 'crop':
            return img.crop(params)
        elif transformation == 'resize':
            return img.resize(params, self.filter)
        else:
            raise NotImplementedError(transformation)

    def __call__(self, img, target_size):
        transforms = self.transformations(img, target_size)
        for transformation, params in transforms:
            img = self.transform(img, transformation, params)
        return img

class ResizeFit(Resizer):

    def transformations(self, img, target_size):
        sw, sh = img.size
        tw, th = target_size
        if not self.expand and sw<=tw and sh<=th:
            return []

        if sw*th>sh*tw:
            h = sh*tw//sw
            return [('resize', (tw, h))]
        else:
            w = sw*th//sh
            return [('resize', (w, th))]

class ResizeCrop(Resizer):

    def transformations(self, img, target_size):
        sw, sh = img.size
        tw, th = target_size
        if not self.expand and sw<=tw and sh<=th:
            return []

        transforms = []
        if sw*th>sh*tw:
            if sh!=th and (sh>th or self.expand):
                w = sw*th//sh
                transforms.append(('resize', (w, th)))
                sw, sh = w, th
            if sw>tw:
                wd = (sw-tw)//2
                transforms.append(('crop', (wd, 0, tw+wd, sh)))
        else:
            if sw!=tw and (sw>tw or self.expand):
                h = sh*tw//sw
                transforms.append(('resize', (tw, h)))
                sw, sh = tw, h
            if sh>th:
                hd = (sh-th)//2
                transforms.append(('crop', (0, hd, sw, th+hd)))
        return transforms

class ResizeMixed(Resizer):

    def __init__(self, hor_resize, vert_resize):
        self.hor_resize = hor_resize
        self.vert_resize = vert_resize

    def get_resizer(self, img, target_size):
        sw, sh = img.size
        if sw >= sh:
            return self.hor_resize
        else:
            return self.vert_resize

    def transformations(self, img, size):
        return self.get_resizer(img, size).transformations(img, size)

    #def transform(self, img, *args):
    # XXX is this method needed?
    #    return self.get_resizer(img).transform(img, *args)

    def __call__(self, img, size):
        return self.get_resizer(img, size)(img, size)

class ResizeFixedWidth(Resizer):
    def transformations(self, img, target_size):
        sw, sh = img.size
        tw, th = target_size
        if not self.expand and sw<=tw:
            return []
        h = sh*tw//sw
        return [('resize', (tw, h))]

class ResizeFixedHeight(Resizer):

    def transformations(self, img, target_size):
        sw, sh = img.size
        tw, th = target_size
        if not self.expand and  sh<=th:
            return []
        w = sw*th//sh
        return [('resize', (w, th))]