# -*- coding: utf-8 -*-

from unittest import TestCase
from unittest.mock import MagicMock, patch

import numpy as np
from copulas.univariate import Univariate
from sklearn.gaussian_process import GaussianProcessRegressor

from btb.tuning.metamodels.gaussian_process import (
    GaussianCopulaProcessMetaModel, GaussianProcessMetaModel)

class TestGaussianProcessMetaModel(TestCase):

    def test___init__(self):
        # run
        instance = GaussianProcessMetaModel()

        # assert
        assert instance._MODEL_KWARGS_DEFAULT == {'normalize_y': True}
        assert instance._MODEL_CLASS == GaussianProcessRegressor

    def test__predict(self):
        # setup
        instance = MagicMock()
        instance._model_instance.predict.return_value = [[1], [2]]

        # run
        result = GaussianProcessMetaModel._predict(instance, 1)

        # assert
        instance._model_instance.predict.assert_called_once_with(1, return_std=True)
        np.testing.assert_array_equal(result, np.array([[1, 2]]))

class TestGaussianCopulaProcessMetaModel(TestCase):

    def test___init__(self):
        # run
        instance = GaussianCopulaProcessMetaModel()
        # assert
        assert instance._MODEL_KWARGS_DEFAULT == {'normalize_y': True}
        assert instance._MODEL_CLASS == GaussianProcessRegressor

    def test__trasnform(self):
        # setup
        instance = MagicMock()
        trials = np.array([
            [0.1, 0.8, 0.2],
            [0.7, 0.6, 0.4],
            [0.4, 0.2, 0.3],
            [0.7, 0.8, 0.9],
            [0.2, 0.3, 0.4]
        ])

        distributions = []
        for column in trials.T:
            distribution = Univariate()
            distribution.fit(column)
            distributions.append(distribution)

        instance._distributions = distributions

        # run
        result = GaussianCopulaProcessMetaModel._transform(instance, trials)

        # assert
        expected_result = np.array([
            [-0.93051407, 0.72584367, -1.05823838],
            [0.78865964, 0.1155097, 0.3732126],
            [-0.04619363, -0.96388991, -0.49278658],
            [0.78865964, 0.72584367, 1.82607428],
            [-0.60472608, -0.66069297, 0.3732126]
        ])

        np.testing.assert_allclose(result, expected_result)

    @patch('btb.tuning.metamodels.gaussian_process.super')
    def test__fit(self, mock_super):
        # setup
        instance = GaussianCopulaProcessMetaModel()
        trials = np.array([
            [0.1, 0.8, 0.2],
            [0.7, 0.6, 0.4],
            [0.4, 0.2, 0.3],
            [0.7, 0.8, 0.9],
            [0.2, 0.3, 0.4]
        ])
        scores = np.array([0.1, 0.4, 0.6, 0.8, 0.9])

        # run
        instance._fit(trials, scores)

        # assert
        expected_trials = np.array([
            [-0.93051407, 0.72584367, -1.05823838],
            [0.78865964, 0.1155097, 0.3732126],
            [-0.04619363, -0.96388991, -0.49278658],
            [0.78865964, 0.72584367, 1.82607428],
            [-0.60472608, -0.66069297, 0.3732126]
        ])
        expected_scores = np.array([-1.15916747, -0.43719902, 0.0415104, 0.57965234, 0.87915571])

        mock_fit_call = mock_super.return_value._fit.call_args_list
        np.testing.assert_allclose(expected_trials, mock_fit_call[0][0][0])
        np.testing.assert_allclose(expected_scores, mock_fit_call[0][0][1], rtol=1e-06)

    def test__fit_inconsistent_numbers_of_samples(self):
        """Fitting the GP model with inconsistent number of samples raises ValueError."""
        # setup
        instance = GaussianCopulaProcessMetaModel()
        trials = np.array([
            [0.1, 0.8, 0.2],
            [0.7, 0.6, 0.4],
            [0.4, 0.2, 0.3],
            [0.7, 0.8, 0.9],
            [0.2, 0.3, 0.4]
        ])
        scores = np.array([0.1, 0.6, 0.8, 0.9])

        # run
        with self.assertRaises(ValueError):
            instance._fit(trials, scores)

    def test__predict_one_candidate(self):
        """Test the prediction of one candidate."""
        # setup
        instance = GaussianCopulaProcessMetaModel()
        trials = np.array([
            [0.1, 0.8, 0.2],
            [0.7, 0.6, 0.4],
            [0.4, 0.2, 0.3],
            [0.7, 0.8, 0.9],
            [0.2, 0.3, 0.4]
        ])
        scores = np.array([0.1, 0.4, 0.6, 0.8, 0.9])

        candidates = np.array([[0.2, 0.8, 0.4]])
        instance._fit(trials, scores)

        # run
        predicted_scores = instance._predict(candidates)

        # assert
        expected_scores = np.array([0.54393124])

        np.testing.assert_allclose(expected_scores, predicted_scores)

    def test__predict_candidate_gt_one(self):
        """Test the prediction of multiple candidates."""
        # setup
        instance = GaussianCopulaProcessMetaModel()
        trials = np.array([
            [0.1, 0.8, 0.2],
            [0.7, 0.6, 0.4],
            [0.4, 0.2, 0.3],
            [0.7, 0.8, 0.9],
            [0.2, 0.3, 0.4]
        ])
        scores = np.array([0.1, 0.4, 0.6, 0.8, 0.9])

        candidates = np.array([
            [0.2, 0.8, 0.4],
            [0.1, 0.8, 0.2]
        ])
        instance._fit(trials, scores)

        # run
        predicted_scores = instance._predict(candidates)

        # assert
        expected_scores = np.array([0.54393124, 0.1])

        np.testing.assert_allclose(expected_scores, predicted_scores)


from pandas import read_csv
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import cross_val_score
from  sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from keras.regularizers import l2
from sklearn.model_selection import train_test_split
from keras.constraints import maxnorm
import pandas as pd
from sklearn import svm

model_id = "drugbank_approved_joined_logP_SVR_rbf"
dataframe1 = read_csv("../ecfp-drugbank-approved.csv", delimiter = ',')
dataframe2 = read_csv("../maccskeys-drugbank-approved.csv", delimiter = ',')
dataframe  = pd.concat([dataframe1, dataframe2], axis=1)

dataset = dataframe.values
# split into input (X) and output (Y) variables
X = dataset[:,:]
Y = np.loadtxt("../drugbank_approved_logP.AlogP.value")
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)

# define wider model
model =  svm.SVR(kernel='rbf', C=10, gamma=0.2)

#hyperparameter; do it in the following pair

C=[1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]
kernel=['rbf', 'linear', 'poly','sigmoid']
gamma=[0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2.0]
#coef0 for poly and sigmoid; degree for poly
coef0=[]
degree=[1,3,5,7,9]
"""
example
svr_rbf = SVR(kernel='rbf', C=100, gamma=0.1, epsilon=.1)
svr_lin = SVR(kernel='linear', C=100, gamma='auto')
svr_poly = SVR(kernel='poly', C=100, gamma='auto', degree=3, epsilon=.1,
                       coef0=1)
linear: C
poly: C, degree, gamma, coef0
rbf: C, gamma,
sigmoid: C, gamma, coef0
"""
resultscsv='gridscores-'+str(model_id)+".csv"

import pickle
import scipy
from scipy import stats
nlayer=1
for i in range(1):
    estimators = svm.SVR(kernel='rbf',C=100, gamma=0.1 )

    history=estimators.fit(X_train, Y_train, )
    print(history.score(X_train, Y_train,))
    logFilePath = './log'+str(model_id)+'.txt'
    fobj = open(logFilePath, 'a')
    fobj.write('model id: ' + str(model_id)+'\n')
    fobj.write('x_train shape: ' + str(X_train.shape) + '\n')
    fobj.write('x_test shape: ' + str(X_test.shape)+'\n')
    fobj.write('training accuracy: ' + str(history.score(X_train, Y_train,)) + '\n')
    #fobj.write('model evaluation results: ' + str(score[0]) + '  ' +str(score[-1])+'\n')
    fobj.write('---------------------------------------------------------------------------\n')
    fobj.write('\n')
    fobj.close()

    prediction0=estimators.predict(X_train)
    print("train-pearson: ",scipy.stats.pearsonr(Y_train,prediction0))

    prediction=estimators.predict(X_test)
    print("test-pearson: ",scipy.stats.pearsonr(Y_test,prediction))
#hyperparameter; do it in the following pair

#save raw output
prediction0=estimators.predict(X_train)
predictedtxt0 = 'predicted_train_'+str(model_id)+'.txt'
truetxt0 = 'true_train_'+str(model_id)+'.txt'
np.savetxt(predictedtxt0, prediction0, fmt='%f')
np.savetxt(truetxt0, Y_train, fmt='%f')

predictedtxt = 'predicted_'+str(model_id)+'.txt'
truetxt = 'true_'+str(model_id)+'.txt'
np.savetxt(predictedtxt, prediction, fmt='%f')
np.savetxt(truetxt, Y_test, fmt='%f')

exit()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

pd.set_option('display.max_rows', 1000)
pd.set_option('display.max_columns', 1000)
# 每年38个处理的非传递性
path = 'C:/Users/97899/Desktop/N/'
path_D = {"huang/huang_compe.xls": ["rank_nested","rank_no_nested"]}
df_ex = pd.read_excel(path + "实验处理_ex.xls")
title = ["Competition ranking about Carex korshinskyi in nested",
         "Competition ranking about Carex korshinskyi in no nested",]
columns = ['顺序', '氮素', '频率', '刈割']
m = -1
for key in path_D.keys():
    for sheet in path_D[key]:

        df_Int = pd.read_excel(path + key, sheet_name=sheet)
        df_Int = pd.concat([df_Int, pd.DataFrame(columns=columns)])
        m=m+1
        '''Intransitivity and experiment deal'''

        for item in range(df_Int.shape[0]):
            for jtem in range(df_ex.shape[0]):
                if int(df_Int.iloc[item, 0]) == int(df_ex.iloc[jtem, 1]):
                    df_Int.loc[item, '顺序'] = df_ex.iloc[jtem, 1]
                    df_Int.loc[item, '氮素'] = df_ex.iloc[jtem, 2]
                    df_Int.loc[item, '频率'] = df_ex.iloc[jtem, 3]
                    df_Int.loc[item, '刈割'] = df_ex.iloc[jtem, 4]
        df_Int.drop([0, 19], inplace=True)
        df_Int.drop([2008.0], axis=1, inplace=True)

        '''Group'''
        gb1 = df_Int.groupby("频率")
        gx = {}
        for g1 in gb1:
            gb2 = g1[1].groupby("刈割")
            gx[g1[0]] = {}
            for g2 in gb2:
                gb3 = g2[1].groupby("氮素")
                gx[g1[0]][g2[0]] = []
                for g3 in gb3:
                    print(g3[0])
                    g0 = g3[1].drop(["顺序", "氮素", "刈割", "频率", "Unnamed: 0"], axis=1)
                    # print(g0)
                    mean = np.mean([i for i in g0.values[0] if i > 0])
                    if np.isnan(mean):
                        gx[g1[0]][g2[0]].append(0)
                    else:
                        gx[g1[0]][g2[0]].append(mean)
                    # print(gx[g1[0]][g2[0]])
        '''Figure'''

        result = []
        for key1 in gx.keys():
            for key2 in gx[key1].keys():
                sum_ = sum(gx[key1][key2])
                result.append([i for i in gx[key1][key2]])
        category_names = ['N=0', 'N=1', 'N=2', 'N=3', 'N=5', 'N=10', 'N=15', 'N=20', 'N=50']

        results = {
            'LF-NM': result[0],
            'LF-M': result[1],
            'HF-NM': result[2],
            'HF-M': result[3],
        }

        def survey(results, category_names):
            """
            Parameters
            ----------
            results : dict
                A mapping from question labels to a list of answers per category.
                It is assumed all lists contain the same number of entries and that
                it matches the length of *category_names*.
            category_names : list of str
                The category labels.
            """
            labels = list(results.keys())
            data = np.array(list(results.values()))
            data_cum = data.cumsum(axis=1)
            category_colors = plt.get_cmap('RdYlGn')(
                np.linspace(0.15, 0.85, data.shape[1]))

            fig, ax = plt.subplots(figsize=(9.2, 5))
            ax.invert_yaxis()
            ax.xaxis.set_visible(False)
            ax.set_xlim(0, np.sum(data, axis=1).max())

            for i, (colname, color) in enumerate(zip(category_names, category_colors)):
                widths = data[:, i]
                starts = data_cum[:, i] - widths
                ax.barh(labels, widths, left=starts, height=0.5,
                        label=colname, color=color)
                xcenters = starts + widths / 2
                print(labels)

                r, g, b, _ = color
                text_color = 'black' if 0 < r * g * b else "red"
                for y, (x, c) in enumerate(zip(xcenters, widths)):
                    if c == 0:
                        continue
                    else:
                        ax.text(x, y, str("%.2f" % c), ha='center', va='center',
                                color=text_color,fontsize=15)
            ax.legend(ncol=1, bbox_to_anchor=(1.1, 0.5), fontsize=13)
            ax.set_title(title[m],fontsize=20)

            return fig, ax

        # plt.title("Biomass")
        survey(results, category_names)
        plt.show()


from typing import List

class Solution:
    def splitArray(self, nums: List[int], m: int) -> int:

        prefix_sum = [nums[0]]
        M = m
        for n in nums[1:]:
            prefix_sum.append(prefix_sum[-1] + n)

        solution = [p for p in prefix_sum]
        new_solution = [0] * len(nums)

        import math
        for m in range(1, M):
            for n in range(1, len(nums)):
                min_value = math.inf
                for k in range(n-1, -1, -1):
                    x = solution[k]
                    y = prefix_sum[n] - prefix_sum[k]
                    max_value = max(x, y)
                    min_value = min(max_value, min_value)
                new_solution[n] = min_value
            solution = new_solution
            new_solution = [0] * len(nums)
        return solution[-1]

    # time limited exceeded
    def splitArraydfs(self, nums, k, p):
        import math
        if not nums[p:]:
            return -math.inf
        if k == 1:
            return sum(nums[p:])

        min_value = math.inf
        for i in range(p+1, len(nums) + 1):
            curr = sum(nums[p:i])
            child_sum = self.splitArraydfs(nums, k-1, i)
            min_value = min(max(curr, child_sum), min_value)
        return min_value

if __name__ == '__main__':
    nums = [7, 2, 5, 10, 8]
    #nums = [1, 2]
    print(Solution().splitArraydfs(nums, 4, 0))


#  我真诚地保证：
#  我自己独立地完成了整个程序从分析、设计到编码的所有工作。
#  如果在上述过程中，我遇到了什么困难而求教于人，那么，我将在程序实习报告中
#  详细地列举我所遇到的问题，以及别人给我的提示。
#  在此，我感谢 XXX, …, XXX对我的启发和帮助。下面的报告中，我还会具体地提到
#  他们在各个方法对我的帮助。
#  我的程序里中凡是引用到其他程序或文档之处，
#  例如教材、课堂笔记、网上的源代码以及其他参考书上的代码段,
#  我都已经在程序的注释里很清楚地注明了引用的出处。

#  我从未没抄袭过别人的程序，也没有盗用别人的程序，
#  不管是修改式的抄袭还是原封不动的抄袭。
#  我编写这个程序，从来没有想过要去破坏或妨碍其他计算机系统的正常运转。
#  <胡崟昊> 改成自己的名字

from flask import Flask, request, render_template
from search import *
app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def home():
    return render_template('index.html')

@app.route('/search', methods=['GET', 'POST'])
def search():
    # 2011-2018月份
    month_s = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
    # 2019月份
    month_2019 = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11']

    city = request.form['city']
    year = request.form['year']

    if int(year) == 2019:
        months = month_2019
    else:
        months = month_s
    # months = ['01']
    city_pinyin = get_city_pinyin(city)
    print(type(city_pinyin))
    temperature_list = get_date(city_pinyin, year, months)  # 日期，最高温度，最低温度
    # temperature_list = [[3, 2], [2, 1], [5, 5], [9, 6],
    #                     [8, 6], [8, 6], [7, 5], [7, 5],
    #                     [5, 5], [7, 6], [9, 6], [8, 5],
    #                     [8, 4], [6, 5], [7, 3], [4, 0],
    #                     [9, 0], [14, 4], [17, 8], [10, 2],
    #                     [8, 2], [12, 1], [14, 2], [16, 1],
    #                     [11, 3], [9, 0], [13, 3], [12, 6],
    #                     [15 6], [21, 7], [10, 3]]

    # encoding=utf-8
    return render_template('search.html', list=temperature_list, city=city, year1=str(year), year2=int(year))

if __name__ == '__main__':
    app.run(debug=True)


from datetime import datetime
from dimagi.utils.couch.database import get_db
from dateutil.relativedelta import relativedelta
from patient_group import PathfinderPatientGroup
from dimagi.utils.parsing import string_to_datetime

def get_patients_by_provider(domain, provider):
    return get_db().view('pathfinder/pathfinder_gov_reg_by_username', keys=[[domain, provider]], include_docs=True).all()

def retrieve_patient_group(user_ids, domain, year, month):
    """
    Given a set of user IDs, retrieve all patients matching that set.  Update them with followup and referral form info.
    """
    caseid_set = set()
    patients = {}
    year = int(year)
    month = int(month)
    form_now = datetime(year, month, 1)+relativedelta(months=1)-relativedelta(seconds=1)
    pyear = 0
    pmonth = 0
    for result in user_ids:
        rform = result['doc']['form'] #shortcut
        try: # strptime() can fail if the CHW enters weird data.
            regdate = string_to_datetime(rform['patient']['date_of_registration'])
            if regdate > form_now:
                continue
        except:
            pass
        p = dict()
        age_tmp = string_to_datetime(rform['patient']['date_of_birth'])
        p['age'] = int((form_now - age_tmp).days / 365.25)
        p.update(rform['case'])
        p.update(rform['patient'])
        caseid_set.add(p['case_id'] )
        p['ward'] = p['village']
        p['registered_this_month'] = True if (regdate.year == form_now.year and regdate.month == form_now.month) else False
        p['followup_this_month'] = 0
        p['referrals_made'] = 0
        for i in ['provider',
                  'registration_and_followup_hiv',
                  'hiv_status_during_registration',
                  'hiv_status_after_test',
                  'referrals_hiv',
                  'ctc',
                  'medication_given',
                  'services',
                  'referrals']:
            if not i in p:
                p[i] = ''
        patients[p['case_id']] = p
    update_patients_with_followups(domain, patients, caseid_set,year,month)
    update_patients_with_referrals(patients,caseid_set,year,month)
    ## I chose not to remove patients without followups this month -- sometimes there's a referral in a month with no followup.
    # map(lambda x: patients.__delitem__(x), filter(lambda y: not patients[y]['followup_this_month'], patients.keys()))
    gp =  PathfinderPatientGroup()
    gp += patients.values()
    return gp

def update_patients_with_followups(domain, patients, caseid_set, year, month):
    """
    Given a set of patients, update them with info from followup forms.
    """
    followup_keys = [[domain, x, year, month] for x in list(caseid_set)]
    followups = get_db().view('pathfinder/pathfinder_gov_followup_by_caseid', keys = followup_keys, include_docs=True).all()
    for f in followups:
        if not f['key'][1] in patients:
            continue
        p = patients[f['key'][1]]
        f = f['doc']['form']
        fp = f['patient']
        p['provider'] = f['meta']['userID']

        # The code below exists to hack around issues like people coming back to life after being dead,
        # multiple followups in a single month having different services, medication, and/or referral values,
        # and so on.

        if not p.has_key('registration_and_followup_hiv') or p['registration_and_followup_hiv'] == 'continuing' or p['registration_and_followup_hiv'] == '': # don't allow a value of 'dead' to change
            if fp.has_key('reg_followup'):
                p['registration_and_followup_hiv'] = fp['reg_followup']
        if p.has_key('followup_this_month'):
            p['followup_this_month'] += 1
        else:
            p['followup_this_month'] = 1
        if "medication_given" in fp and fp['medication_given']:
            p['medication_given'] += fp['medication_given'] + " "
        if "services_given" in fp and fp['services_given']:
            p['services'] += fp['services_given'] + " "
        if "any_referral" in fp and fp['any_referral'] == "yes":
            p['referrals'] += fp['referrals'] + " "
        p.update(fp)

def update_patients_with_referrals(patients, ids, year,month):
    """
    Given a set of patients, count how many completed referrals they had in that month.
    """
    refs = get_db().view('pathfinder/pathfinder_gov_referral', keys=[[x, int(year), int(month)] for x in list(ids)], include_docs=True).all()

    for p in patients: patients[p]['referrals_completed'] = 0
    for r in refs:
        if r['doc']['form']['client']['referral'] == 'yes':
            patients[r['doc']['form']['case']['case_id']]['referrals_completed'] += 1


'''
Created on Mar 31, 2012

@author: SFUser
'''
from xml.sax.handler import ContentHandler
from xml.sax import parse

class Dispatcher:
    def startElement(self, name, attrs):
        self.dispatch('start', name, attrs)

    def endElement(self, name):
        self.dispatch('end', name)

    def dispatch(self, prefix, name, attrs=None):
        mname = prefix + name.capitalize()
        dname = 'default' + prefix.capitalize()
        method = getattr(self, mname, None)
        if callable(method): args = ()
        else:
            method = getattr(self, dname, None)
            args = name,
        if prefix == 'start':args += attrs
        if callable(method): method(*args)

class TestHandler(Dispatcher, ContentHandler):
    def startElement(self, name, attrs):
        print(name, attrs.keys(), attrs.values())

parse("Nokia.xml", TestHandler())


import string
import sys,os
try:
  import xml.etree.cElementTree as ET
except ImportError:
  import xml.etree.ElementTree as ET

def getAllXml(path):
    txt = []
    cls = []
    for root, dirs, files in os.walk(path):
        for img_name in files:
            if not img_name.lower ().endswith (('.bmp', '.jpeg', '.jpg', '.png', '.tif', '.tiff')):
                continue
            imgpath = os.path.join (root, img_name)
            xml_name = img_name[0:img_name.find (".")]+".xml"
            xmlpath = os.path.join (root, xml_name)
            if (os.path.isfile (xmlpath) == True):
                tree = ET.parse (xmlpath)
                if (tree == None): continue
                gen = tree.getroot ()
                obj = gen.findall ("object")
                for item in obj:
                    str = ""
                    str += imgpath
                    name = item.find ("name").text
                    #name=name[name.find(" ")+1:len(name)]
                    if (name.find ("snail") != -1):
                        name = "snail"
                    elif (name.find ("coin") != -1):
                        name = "coin"
                    elif (name.find ("eel") != -1):
                        name = "eel"
                    elif (name.find ("female_sturgeon") != -1):
                        name = "female_sturgeon"
                    elif (name.find ("male_sturgeon") != -1):
                        name = "male_sturgeon"
                    elif (name.find ("female") != -1):
                        name = "female"
                    elif (name.find ("male") != -1):
                        name = "male"
                    elif (name.find ("carapace") != -1):
                        name = "carapace"
                    elif (name.find ("carb") != -1 or name.find ("crab") != -1):
                        name = "crab"
                    else:
                        print ("label error!,name is :", name)
                        continue
                    bndbox = item.find ("bndbox")
                    x1 = bndbox.find ("xmin").text
                    y1 = bndbox.find ("ymin").text
                    x2 = bndbox.find ("xmax").text
                    y2 = bndbox.find ("ymax").text
                    str += "," + (x1) + "," + (y1) + "," + (x2) + "," + (y2)
                    #str += ",crab"
                    str +=","+name
                    if(name not in cls):
                        cls.append(name)
                    txt.append (str)
            else:
                str = ""
                name = "eel"
                if (name not in cls):
                    cls.append (name)
                str += imgpath+",0,0,1,1,"+name
                txt.append(str)

    return txt,cls

if __name__ == '__main__':
    path="/home/sy/crab/unused/fish/"
    txt,cls=getAllXml(path)
    title="iscas_frcnn_test_fish"
    cnt = 1
    with open ("%s.csv" % title, "w") as f:
        for item in txt:
            cnt +=1
            print (item)
            f.write (item)
            f.write ("\n")
    f.close()
    print("count:",cnt)

    title = "iscas_frcnn_class_test_fish"
    id = 0
    with open ("%s.csv" % title, "w") as f:
        print('female_sturgeon,0',file = f)
        print('male_sturgeon,1',file = f)
        print('eel,2',file = f)
    f.close()


import sys, os, shutil
from mars_rover import Ui_MainWindow, QtWidgets
from PyQt5 import QtCore, QtGui, QtWidgets
import subprocess

class MyApp(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self):
        QtWidgets.QMainWindow.__init__(self)
        Ui_MainWindow.__init__(self)
        self.setupUi(self)
        self.btnUpdateGridSize.clicked.connect(self.set_new_grid)
        self.btnNumRoversUpdate.clicked.connect(self.create_rovers_list)
        self.comboRoverSelect.currentTextChanged.connect(self.load_selected_rover)
        self.btnMoveForward.clicked.connect(self.move_rover_forward)
        self.btnTurnRight.clicked.connect(self.turn_rover_right)
        self.btnTurnLeft.clicked.connect(self.turn_rover_left)
        self.scene = None
        self.rovers_dict = {}
        self.rover_coords = {}
        self.selected_rover = None
        self.set_direction = 'N'
        self.grid_x_dimension = None
        self.grid_y_dimension = None
        self.read = True
        self.rovers_list = []

        data = {"Rover Name": [], "X-Coord": [], "Y-Coord": [], "Direction": [] }
        self.table_data = data
        self.setTableData()

    def setTableData(self):
        """Set output table default layout"""
        horHeaders = []
        header = self.tableRoverInfo.horizontalHeader()
        header.setSectionResizeMode(0, QtWidgets.QHeaderView.Stretch)
        header.setSectionResizeMode(1, QtWidgets.QHeaderView.ResizeToContents)
        header.setSectionResizeMode(2, QtWidgets.QHeaderView.ResizeToContents)
        header.setSectionResizeMode(3, QtWidgets.QHeaderView.ResizeToContents)
        for n, key in enumerate((self.table_data.keys())):
            horHeaders.append(key)
            for m, item in enumerate(self.table_data[key]):
                newitem = QtWidgets.QTableWidgetItem(item)
                self.tableRoverInfo.setItem(m, n, newitem)
        self.tableRoverInfo.setHorizontalHeaderLabels(horHeaders)

    def set_new_grid(self):
        """Create new grid/coordinate field"""
        self.grid_x_dimension = self.spinXGrid.value()
        self.grid_y_dimension = self.spinYGrid.value()

        self.scene = QtWidgets.QGraphicsScene()
        self.graphGridView.setScene(self.scene)
        pen = QtGui.QPen(QtCore.Qt.green)

        side = 30

        for i in range(self.grid_x_dimension):
            for j in range(self.grid_y_dimension):
                grids = QtCore.QRectF(QtCore.QPointF(i*side, j*side), QtCore.QSizeF(side, side))
                self.scene.addRect(grids, pen)

    def create_rovers_list(self):
        """Create new rovers based on user input"""
        self.set_new_grid()
        self.comboRoverSelect.blockSignals(True)
        self.comboRoverSelect.setCurrentIndex(0)
        self.comboRoverSelect.blockSignals(False)
        x_grid_coord = self.grid_x_dimension
        y_grid_coord = self.grid_y_dimension
        num_of_rovers = self.spinNumOfRovers.value()
        self.tableRoverInfo.setRowCount(0)
        self.rovers_list = []
        for x in range(num_of_rovers):
            self.rovers_dict["rover" + str(x)] = QtWidgets.QGraphicsEllipseItem((30*1) - 38, (30*y_grid_coord) - 8, 15, 15)
            self.rovers_dict["rover" + str(x)].setPen(QtGui.QPen(QtGui.QColor("blue")))
            self.rover_coords["rover" + str(x)] = [0, 0, 'N']
            self.scene.addItem(self.rovers_dict["rover"+str(x)])
            self.rovers_list.append("rover" + str(x))
            rowPosition = self.tableRoverInfo.rowCount()
            self.tableRoverInfo.insertRow(rowPosition)
        self.comboRoverSelect.clear()
        rover_names_list = self.rovers_dict.keys()
        self.comboRoverSelect.addItem("[ Select Rover ]")
        self.comboRoverSelect.addItems(rover_names_list)
        # self.table_data = {"Rover Name": self.rovers_list, "X-Coord": [], "Y-Coord": [], "Direction": [] }
        self.reset_coordinates_table_data()
        self.setTableData()

    def reload_rovers(self):
        """Reload rovers list after user input"""
        self.set_new_grid()
        x_grid_coord = self.grid_x_dimension
        y_grid_coord = self.grid_y_dimension
        num_of_rovers = self.spinNumOfRovers.value()

        for x in range(num_of_rovers):
            rover_coords = self.rover_coords["rover" + str(x)]
            rover_y_coord = (y_grid_coord - rover_coords[1])
            rover_x_coord = rover_coords[0]
            self.rovers_dict["rover" + str(x)] = QtWidgets.QGraphicsEllipseItem((30*(rover_x_coord + 1)) - 38, (30*rover_y_coord) - 8, 15, 15)
            self.rovers_dict["rover" + str(x)].setPen(QtGui.QPen(QtGui.QColor("blue")))
            self.scene.addItem(self.rovers_dict["rover" + str(x)])

    def reset_coordinates_table_data(self):
        """Reset coordinates table after user input"""
        x_coords_list = []
        y_coords_list = []
        dir_list = []

        for x in self.rover_coords.keys():
            x_coords_list.append(str(self.rover_coords[x][0]))
            y_coords_list.append(str(self.rover_coords[x][1]))
            dir_list.append(self.rover_coords[x][2])
        self.table_data = {"Rover Name": self.rovers_list, "X-Coord": x_coords_list, "Y-Coord": y_coords_list, "Direction": dir_list }

    def load_selected_rover(self):
        self.selected_rover = self.comboRoverSelect.currentText()
        if self.selected_rover == "[ Select Rover ]":
            pass
        else:
            try:
                curr_coords = self.rover_coords[self.selected_rover]
                self.set_direction = curr_coords[2]
            except Exception:
                pass

    def check_rover_range(self, curr_coords):
        """Check if rover is in bounds before moving"""
        out_of_range = False
        curr_x_coord = curr_coords[0]
        curr_y_coord = curr_coords[1]
        curr_rover_dir = curr_coords[2]

        if curr_x_coord == 0 and self.set_direction == "W":
            out_of_range = True
        if curr_x_coord == self.grid_x_dimension and self.set_direction == "E":
            out_of_range = True
        if curr_y_coord == 0 and self.set_direction == "S":
            out_of_range = True
        if curr_y_coord == self.grid_y_dimension and self.set_direction == "N":
            out_of_range = True
        return out_of_range

    def initiate_rover_move(self, curr_coords):
        """Move rover in direction set by user"""
        curr_x_coord = curr_coords[0]
        curr_y_coord = curr_coords[1]
        curr_rover_dir = curr_coords[2]
        if self.set_direction == "N":
            new_y_coord = curr_y_coord + 1
            self.rover_coords[self.selected_rover] = [curr_x_coord, new_y_coord, self.set_direction]
            self.reload_rovers()
        elif self.set_direction == "W":
            new_x_coord = curr_x_coord - 1
            self.rover_coords[self.selected_rover] = [new_x_coord, curr_y_coord, self.set_direction]
            self.reload_rovers()
        elif self.set_direction == "S":
            new_y_coord = curr_y_coord - 1
            self.rover_coords[self.selected_rover] = [curr_x_coord, new_y_coord, self.set_direction]
            self.reload_rovers()
        elif self.set_direction == "E":
            new_x_coord = curr_x_coord + 1
            self.rover_coords[self.selected_rover] = [new_x_coord, curr_y_coord, self.set_direction]
            self.reload_rovers()
        self.reset_coordinates_table_data()
        self.setTableData()

    def move_rover_forward(self):
        """Call initiate move function if rover is in bounds"""
        curr_coords = self.rover_coords[self.selected_rover]

        if self.check_rover_range(curr_coords):
            msg = QtWidgets.QMessageBox()
            msg.setIcon(QtWidgets.QMessageBox.Critical)
            msg.setText("Error")
            msg.setInformativeText('Rover is out of range, Please change direction.')
            msg.setWindowTitle("Error")
            msg.exec_()
        else:
            self.initiate_rover_move(curr_coords)

    def check_selected_rover(self):
        """Check if combobox is set to a valid selection"""
        self.selected_rover = self.comboRoverSelect.currentText()
        if self.selected_rover == "[ Select Rover ]":
            return False
        else:
            return True
    def turn_rover_right(self):
        """Turn rover 90 degrees to the right"""
        curr_dir = self.set_direction
        if curr_dir == "N":
            self.set_direction = "E"
        elif curr_dir == "W":
            self.set_direction = "N"
        elif curr_dir == "S":
            self.set_direction = "W"
        elif curr_dir == "E":
            self.set_direction = "S"
        self.reset_coordinates_table_data()
        self.setTableData()

    def turn_rover_left(self):
        """Turn rover 90 degrees to the left"""
        curr_dir = self.set_direction
        if curr_dir == "N":
            self.set_direction = "W"
        elif curr_dir == "W":
            self.set_direction = "S"
        elif curr_dir == "S":
            self.set_direction = "E"
        elif curr_dir == "E":
            self.set_direction = "N"
        self.reset_coordinates_table_data()
        self.setTableData()

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    window = MyApp()
    window.show()
    sys.exit(app.exec_())


"""Module for asset category resource."""

from flask_restplus import Resource
from flask import request, jsonify

from api.models import AssetCategory
from api.models import Asset
from main import api
from api.middlewares.token_required import token_required
from api.utilities.model_serializers.asset_category import (
    AssetCategorySchema, UpdateAssetCategorySchema,
    EagerLoadAssetCategoryAttributesSchema)
from ..utilities.model_serializers.asset import AssetSchema
from api.utilities.model_serializers.attribute import (AttributeSchema,
                                                       UpdateAttributeSchema)
from api.middlewares.base_validator import ValidationError
from api.utilities.validators.validate_id import validate_id
from api.utilities.messages.error_messages import serialization_errors
from ..utilities.validators.validate_json_request import validate_json_request
from ..utilities.paginator import paginate, pagination_helper

@api.route('/asset-categories')
class AssetCategoryResource(Resource):
    """
    Resource class for peforming crud on the asset categories
    """

    @token_required
    @validate_json_request
    def post(self):
        """
        Creates asset categories and the corresponding attributes
        """

        request_data = request.get_json()

        asset_category_schema = AssetCategorySchema()
        asset_category = asset_category_schema.load_object_into_schema(
            request_data)

        attributes_data = request_data.get('attributes')

        if attributes_data:
            attributes_schema = AttributeSchema(
                many=True, exclude=['id', 'deleted'])

            attributes = attributes_schema.load_object_into_schema(
                attributes_data)

            asset_category.attributes = attributes
            attributes = attributes_schema.dump(attributes).data
        else:
            raise ValidationError({
                'message':
                serialization_errors['provide_attributes']
            })

        asset_category = asset_category.save()

        response = jsonify({
            "status": 'success',
            "data": {
                "name": asset_category.name,
                "customAttributes": attributes
            }
        })

        response.status_code = 201
        return response

@api.route('/asset-categories/stats')
class AssetCategoryStats(Resource):
    """
    Resource class for getting asset categories and
    their corresponding asset counts
    """

    @token_required
    def get(self):
        """
        Gets asset categories and the corresponding asset count
        """
        asset_categories = AssetCategory._query(request.args)

        asset_category_schema = AssetCategorySchema(
            many=True, exclude=['deleted'])

        return {
            'status': 'success',
            'data': asset_category_schema.dump(asset_categories).data
        }

@api.route('/asset-categories/<string:id>')
class AssetCategoryListResource(Resource):
    """Asset category list resource"""

    @token_required
    @validate_id
    def get(self, id):
        """
        Get a single asset category
        """

        single_category = AssetCategory.get_or_404(id)

        attributes_schema = AttributeSchema(
            many=True, exclude=['choices', 'id', 'deleted'])

        return {
            'status': 'success',
            'data': {
                'name':
                single_category.name,
                'customAttributes':
                attributes_schema.dump(single_category.attributes).data
            }
        }, 200

    @token_required
    @validate_json_request
    @validate_id
    def patch(self, id):
        """
        Updates asset categories and the corresponding attributes
        """

        request_data = request.get_json()

        asset_category_schema = UpdateAssetCategorySchema()
        asset_category_data = asset_category_schema.load_object_into_schema(
            request_data)

        asset_category_name = asset_category_data.get('name')

        asset_category = AssetCategory.get_or_404(id)

        if asset_category_name:
            asset_category._update(name=asset_category_name)

        attributes_data = request_data.get('attributes')
        attributes_schema = UpdateAttributeSchema(
            many=True, exclude=['deleted'])

        if attributes_data:
            attributes_schema_data = attributes_schema.load_object_into_schema(
                attributes_data)  # noqa
            for attribute in attributes_schema_data:
                attribute_result = asset_category.attributes.filter_by(
                    id=attribute.get('id')).first()
                if attribute_result:
                    attribute_result._update(**attribute)

        attributes = attributes_schema.dump(
            asset_category.attributes.all()).data

        response = jsonify({
            "status": 'success',
            "data": {
                "name": asset_category.name,
                "customAttributes": attributes
            }
        })

        response.status_code = 200
        return response

    @token_required
    @validate_id
    def delete(self, id):
        """
        Soft delete asset categories
        """

        single_category = AssetCategory.get_or_404(id)
        single_category.delete()

        return {
            'status': 'success',
            'message': 'category deleted successfully'
        }

@api.route('/asset-categories')
class AssetCategoryList(Resource):
    """
    Resource class for getting the list of asset categories and
    their corresponding asset counts
    """

    @token_required
    def get(self):
        """
        Gets list of asset categories and the corresponding asset count
        """
        asset_categories_schema = AssetCategorySchema(
            many=True, exclude=['deleted'])

        if request.args:
            include = request.args.get('include')
            if include and include.lower() == 'attributes':
                asset_categories = AssetCategory._query()
                eager_loaded_schema = EagerLoadAssetCategoryAttributesSchema(
                    many=True, exclude=['deleted'])
                data = eager_loaded_schema.dump(asset_categories).data
            else:
                asset_categories = AssetCategory._query(request.args)
                data = asset_categories_schema.dump(asset_categories).data
        else:
            asset_categories = AssetCategory._query()
            data = asset_categories_schema.dump(asset_categories).data

        return {'status': 'success', 'data': data}

@api.route('/asset-categories/<string:id>/assets')
class AssetCategoryAsset(Resource):
    """
    Resource class for getting the list of assets that belong to an asset category  # noqa
    """

    @token_required
    @validate_id
    def get(self, id):
        """
        Gets list of assets that belong to an asset category
        """

        asset_category = AssetCategory.get_or_404(id)
        assets = AssetSchema(
            many=True, exclude=['deleted_at']).dump(
                asset_category.assets.filter_by(deleted=False)).data

        return {
            'status':
            'success',
            'message':
            serialization_errors['asset_category_assets'].format(
                asset_category.name),  # noqa
            'data':
            assets
        }


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import matplotlib.pyplot as plt
import datetime as dt

directory_in_str = str("/home/xollad/Dropbox/laptop_time/log/")
# directory = os.fsencode(directory_in_str)
directory = directory_in_str

times = list()
dates = list()
days =list()
dateTime = list()

# get current month and year number
today = dt.datetime.today()
month_num = today.month
month_name = str( today.strftime("%B") )
year_num = str( today.year )
axix_label_x = month_name + "," + year_num
today_date_num=str(today.day)
print("%s,%s" %(today_date_num,axix_label_x))

# Read contents of files into
for root, dirs, filenames in os.walk(directory):
    for f in filenames:
        dates.append(f)
        fullpath = os.path.join(directory, f)
        with open(fullpath) as f:
            times.append(int(f.read()))

# reverse the list to make today stays as the last element
times.reverse()
dates.reverse()
maxY = max(times) + 50

# extract only day from the date
for date in dates:
    date=str(date)
    dateTime.append(dt.datetime.strptime(date, "%d-%m-%y").date())
    day,mon,year=date.split("-");
    days.append(int(day))

hours=times[-1]/60
mins=times[-1]%60
# print(times)
# print(days)
print("Today used %d hours and %d minutes " %(hours, mins))

#plot
barlist= plt.bar(days, times, align='center', alpha=0.5)
barlist[-1].set_color('g')
plt.xlabel('dates')
plt.ylabel('minutes')
plt.axis([1,31,0,maxY])
plt.title("Today used %d hours and %d minutes " %(hours, mins), fontweight='bold')
# plt.plot(dateTime, times)
plt.show()


import pytest

def test_nfs_common_is_installed(host):
    assert host.package("nfs-kernel-server").is_installed

@pytest.mark.parametrize("export", [
    "/home/dir1 host1.local(rw,sync,no_root_squash,no_subtree_check)",
    "/home/dir2/dir host2.local(rw,sync,no_root_squash,no_subtree_check)",
    "/home/dir1/test host3.local(rw,sync,no_root_squash,no_subtree_check)"
])
def test_nfs_exports(host, export):
    assert host.file("/etc/exports").contains(export)


import argparse
import os
from math import log10

import pytorch_ssim
from data_utils import load_training_data, load_val_data, do_transform
from loss import GeneratorLoss
from model import Generator, Discriminator

import pandas as pd
import torch.optim as optim
import torch.utils.data
import torchvision.utils as utils
from torch.autograd import Variable
from torch.utils.data import DataLoader
from tqdm import tqdm

parser = argparse.ArgumentParser(description='Train Super Resolution Models')
parser.add_argument('--crop_size', default=90, type=int, help='training images crop size')
parser.add_argument('--upscale', default=4, type=int, choices=[2, 4, 8],
                    help='upscale factor')
parser.add_argument('--epochs', default=100, type=int, help='train epoch number')

if __name__ == '__main__':
    opt = parser.parse_args()

    crop_size = opt.crop_size
    upscale = opt.upscale_factor
    epoch = opt.epochs

    train_set = load_training_data('DIV2K_train_HR', crop_size=crop_size, upscale=upscale)
    val_set = load_val_data('DIV2K_valid_HR', upscale=upscale)
    train_loader = DataLoader(dataset=train_set, num_workers=4, batch_size=64, shuffle=True)
    val_loader = DataLoader(dataset=val_set, num_workers=4, batch_size=1, shuffle=False)

    netG = Generator(upscale)
    print('# generator parameters:', sum(param.numel() for param in netG.parameters()))
    netD = Discriminator()
    print('# discriminator parameters:', sum(param.numel() for param in netD.parameters()))
    generator_loss = GeneratorLoss()

    if torch.cuda.is_available():
        netG.cuda()
        netD.cuda()
        generator_loss.cuda()

    optimizerG = optim.Adam(netG.parameters())
    optimizerD = optim.Adam(netD.parameters())

    results = {'d_loss': [], 'g_loss': [], 'd_score': [], 'g_score': [], 'psnr': [], 'ssim': []}

    for epoch in range(1, epoch + 1):
        trainning_results = {'batch_sizes': 0, 'd_loss': 0, 'g_loss': 0, 'd_score': 0, 'g_score': 0}

        netG.train()
        netD.train()
        for data, target in train_loader:
            g_update_first = True
            batch_size = data.size(0)
            trainning_results['batch_sizes'] += batch_size

            real_img = Variable(target)
            if torch.cuda.is_available():
                real_img = real_img.cuda()
            z = Variable(data)
            if torch.cuda.is_available():
                z = z.cuda()
            generated_img = netG(z)

            netD.zero_grad()
            real_out = netD(real_img).mean()
            generate_out = netD(generated_img).mean()
            d_loss = 1 - real_out + generate_out
            d_loss.backward(retain_graph=True)
            optimizerD.step()

            netG.zero_grad()
            g_loss = generator_loss(generate_out, generated_img, real_img)
            g_loss.backward()

            generated_img = netG(z)
            generate_out = netD(generated_img).mean()

            optimizerG.step()
            trainning_results['g_loss'] += g_loss.item() * batch_size
            trainning_results['d_loss'] += d_loss.item() * batch_size
            trainning_results['d_score'] += real_out.item() * batch_size
            trainning_results['g_score'] += generate_out.item() * batch_size

        netG.eval()
        out_path = 'training_results/SRF_' + str(upscale) + '/'
        if not os.path.exists(out_path):
            os.makedirs(out_path)

        with torch.no_grad():
            val_bar = tqdm(val_loader)
            valing_results = {'mse': 0, 'ssims': 0, 'psnr': 0, 'ssim': 0, 'batch_sizes': 0}
            val_images = []
            for val_lr, val_hr_restore, val_hr in val_bar:
                batch_size = val_lr.size(0)
                valing_results['batch_sizes'] += batch_size
                lr = val_lr
                hr = val_hr
                if torch.cuda.is_available():
                    lr = lr.cuda()
                    hr = hr.cuda()
                sr = netG(lr)

                batch_mse = ((sr - hr) ** 2).data.mean()
                valing_results['mse'] += batch_mse * batch_size
                batch_ssim = pytorch_ssim.ssim(sr, hr).item()
                valing_results['ssims'] += batch_ssim * batch_size
                valing_results['psnr'] = 10 * log10(1 / (valing_results['mse'] / valing_results['batch_sizes']))
                valing_results['ssim'] = valing_results['ssims'] / valing_results['batch_sizes']
                val_bar.set_description(
                    desc='[converting LR images to SR images] PSNR: %.4f dB SSIM: %.4f' % (
                        valing_results['psnr'], valing_results['ssim']))

                val_images.extend(
                    [do_transform()(val_hr_restore.squeeze(0)), do_transform()(hr.data.cpu().squeeze(0)),
                     do_transform()(sr.data.cpu().squeeze(0))])

            val_images = torch.stack(val_images)
            val_images = torch.chunk(val_images, val_images.size(0) // 15)
            val_save_bar = tqdm(val_images, desc='[saving training results]')
            index = 1
            for image in val_save_bar:
                image = utils.make_grid(image, nrow=3, padding=5)
                utils.save_image(image, out_path + 'epoch_%d_index_%d.png' % (epoch, index), padding=5)
                index += 1

        torch.save(netG.state_dict(), 'epochs/netG_epoch_%d_%d.pth' % (upscale, epoch))
        torch.save(netD.state_dict(), 'epochs/netD_epoch_%d_%d.pth' % (upscale, epoch))

        results['d_loss'].append(trainning_results['d_loss'] / trainning_results['batch_sizes'])
        results['g_loss'].append(trainning_results['g_loss'] / trainning_results['batch_sizes'])
        results['d_score'].append(trainning_results['d_score'] / trainning_results['batch_sizes'])
        results['g_score'].append(trainning_results['g_score'] / trainning_results['batch_sizes'])
        results['psnr'].append(valing_results['psnr'])
        results['ssim'].append(valing_results['ssim'])

        if epoch % 10 == 0 and epoch != 0:
            out_path = 'statistics/'
            data_frame = pd.DataFrame(
                data={'Loss_D': results['d_loss'], 'Loss_G': results['g_loss'], 'Score_D': results['d_score'],
                      'Score_G': results['g_score'], 'PSNR': results['psnr'], 'SSIM': results['ssim']},
                index=range(1, epoch + 1))
            data_frame.to_csv(out_path + 'srf_' + str(upscale) + '_train_results.csv', index_label='Epoch')


"""DIRAC RPC Client utilities."""
import logging
from contextlib import contextmanager
import copy
import rpyc

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

# This is not strictly necessary as all works without it when deep copying
# However it bypasses the standard deepcopy implementation
# which does type detection for unknown netref type and uses getattr('__deepcopy__')
# which causes an not found exception server side. This tidy's up server side log since we know
# netref<class='__builtin__.dict'> behaves as a dict
netref_dict = rpyc.core.netref.builtin_classes_cache[('dict', '__builtin__')]
copy._deepcopy_dispatch[netref_dict] = copy._deepcopy_dict
# Add the other basic collection types
netref_list = rpyc.core.netref.builtin_classes_cache[('list', '__builtin__')]
copy._deepcopy_dispatch[netref_list] = copy._deepcopy_list
netref_tuple = rpyc.core.netref.builtin_classes_cache[('tuple', '__builtin__')]
copy._deepcopy_dispatch[netref_tuple] = copy._deepcopy_tuple

# Used in Solid to list the DIRAC file catalogue
@contextmanager
def dirac_rpc_client(rpc_endpoint, host="localhost", port=18861):
    """RPC DIRAC RPC client context."""
    conn = rpyc.connect(host, port, config={"allow_public_attrs": True,
                                            "sync_request_timeout": 300})  # 5 mins
    try:
        yield conn.root.RPCClient(rpc_endpoint)
    finally:
        conn.close()

@contextmanager
def dirac_api_client(host="localhost", port=18861):
    """RPC DIRAC API client context."""
    conn = rpyc.connect(host, port, config={"allow_public_attrs": True,
                                            "sync_request_timeout": 300})
    try:
        yield conn.root.Dirac()
    finally:
        conn.close()

@contextmanager
def dirac_api_job_client(host="localhost", port=18861):
    conn = rpyc.connect(host, port, config={"allow_public_attrs": True,
                                            "sync_request_timeout": 300})
    try:
        yield conn.root.Dirac(), conn.root.Job
    finally:
        conn.close()


import tensorflow as tf
import os
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from preprocessing.time_series.df_to_dataset_synthetic import data_to_dataset_3D, split_input_target
import numpy as np

def split_dataset_into_seq(dataset, start_index, end_index, history_size, step):
  data = []
  start_index = start_index + history_size

  if end_index is None:
    end_index=len(dataset)

  for i in range(start_index, end_index):
    indices = range(i-history_size, i, step)
    data.append(dataset[indices])

  return np.array(data)

def df_to_data_regression(file_path, fname, col_name, index_name, history, step, TRAIN_SPLIT, VAL_SPLIT=0.5, VAL_SPLIT_cv=0.9, cv=False, max_samples=None):
  zip_path = tf.keras.utils.get_file(
      origin=file_path,
      fname=fname,
      extract=True)

  csv_path, _ = os.path.splitext(zip_path)
  df = pd.read_csv(csv_path)
  uni_data_df = df[col_name]
  uni_data_df.index = df[index_name]
  print('length of original continuous dataset: {}'.format(len(uni_data_df)))

  uni_data = uni_data_df.values
  if max_samples is not None:
    #index = list(np.random.randint(0, len(uni_data_df), size=max_samples))
    uni_data = uni_data[:max_samples, :]

  # normalization
  data_mean = uni_data.mean(axis=0)
  data_std = uni_data.std(axis=0)
  uni_data = (uni_data - data_mean) / data_std

  stats = (data_mean, data_std)

  data_in_seq = split_dataset_into_seq(uni_data, 0, None, history, step)

  if not cv:
    # split between validation dataset and test set:
    train_data, val_data = train_test_split(data_in_seq, train_size=TRAIN_SPLIT, shuffle=True)
    val_data, test_data = train_test_split(val_data, train_size=VAL_SPLIT, shuffle=True)

    # reshaping arrays to have a (future shape) of (B,S,1):
    if len(col_name) == 1:
      train_data = np.reshape(train_data, newshape=(train_data.shape[0], train_data.shape[1], 1))
      val_data = np.reshape(val_data, newshape=(val_data.shape[0], val_data.shape[1], 1))
      test_data = np.reshape(test_data, newshape=(test_data.shape[0], test_data.shape[1], 1))

    return (train_data, val_data, test_data), uni_data_df, stats

  else:
    train_val_data, test_data = train_test_split(data_in_seq, train_size=VAL_SPLIT_cv)
    kf = KFold(n_splits=5)
    list_train_data, list_val_data = [], []
    for train_index, val_index in kf.split(train_val_data):
      train_data = train_val_data[train_index, :, :]
      val_data = train_val_data[val_index, :, :]
      list_train_data.append(train_data)
      list_val_data.append(val_data)

    if len(col_name) == 1:
      list_train_data = [np.reshape(d, newshape=(d.shape[0], d.shape[1], 1)) for d in list_train_data]
      list_val_data = [np.reshape(d, newshape=(d.shape[0], d.shape[1], 1)) for d in list_val_data]
      test_data = np.reshape(test_data, newshape=(test_data.shape[0], test_data.shape[1], 1))

    return (list_train_data, list_val_data, test_data), uni_data_df, stats

if __name__ == '__main__':
    file_path = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/jena_climate_2009_2016.csv.zip'
    fname = 'jena_climate_2009_2016.csv.zip'
    col_name = ['p (mbar)', 'T (degC)', 'rh (%)', 'rho (g/m**3)']
    index_name = 'Date Time'
    # temperature recorded every 10 minutes.
    TRAIN_SPLIT = 0.7
    history = 6*24*4 + 6*4
    step = 6*4  # sample a temperature every 4 hours.
    cv = False

    (train_data, val_data, test_data), original_df, stats = df_to_data_regression(file_path=file_path,
                                                                                fname=fname,
                                                                                col_name=col_name,
                                                                                index_name=index_name,
                                                                                TRAIN_SPLIT=TRAIN_SPLIT,
                                                                                history=history,
                                                                                step=step,
                                                                                cv=cv,
                                                                                max_samples=20000)

    print(train_data[:5])

    BUFFER_SIZE = 5000
    BATCH_SIZE = 64

    train_dataset, val_dataset, test_dataset = data_to_dataset_3D(train_data=train_data,
                                                                  val_data=val_data,
                                                                  test_data=test_data,
                                                                   split_fn=split_input_target,
                                                                   BUFFER_SIZE=BUFFER_SIZE,
                                                                   BATCH_SIZE=BATCH_SIZE,
                                                                   cv=cv)


import paddle
import os
import sys
sys.path.insert(0, os.getcwd())
from ppgan.apps import Pixel2Style2PixelPredictor
import argparse

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--input_image", type=str, help="path to source image")

    parser.add_argument("--output_path",
                        type=str,
                        default='output_dir',
                        help="path to output image dir")

    parser.add_argument("--weight_path",
                        type=str,
                        default=None,
                        help="path to model checkpoint path")

    parser.add_argument("--model_type",
                        type=str,
                        default=None,
                        help="type of model for loading pretrained model")

    parser.add_argument("--seed",
                        type=int,
                        default=None,
                        help="sample random seed for model's image generation")

    parser.add_argument("--size",
                        type=int,
                        default=1024,
                        help="resolution of output image")

    parser.add_argument("--style_dim",
                        type=int,
                        default=512,
                        help="number of style dimension")

    parser.add_argument("--n_mlp",
                        type=int,
                        default=8,
                        help="number of mlp layer depth")

    parser.add_argument("--channel_multiplier",
                        type=int,
                        default=2,
                        help="number of channel multiplier")

    parser.add_argument("--cpu",
                        dest="cpu",
                        action="store_true",
                        help="cpu mode.")

    args = parser.parse_args()

    if args.cpu:
        paddle.set_device('cpu')

    predictor = Pixel2Style2PixelPredictor(
        output_path=args.output_path,
        weight_path=args.weight_path,
        model_type=args.model_type,
        seed=args.seed,
        size=args.size,
        style_dim=args.style_dim,
        n_mlp=args.n_mlp,
        channel_multiplier=args.channel_multiplier
    )
    predictor.run(args.input_image)


import streamlit as st
import pandas as pd

from ResultPlots import *

@st.cache(allow_output_mutation=True)
def getUserImages(user):
    print("Caching image data")
    arr = user.openUserNpz("im_plot.npz", "img")
    # res=plot_arrayimg(imgs[value, :, :])
    #Plot each satelital sanpshot of the particiant and cache it
    res = [plot_arrayimg(arr[v, :, :]) for v in range(arr.shape[0])]
    print(arr.shape[0])
    return res

def ResultTab(users):
    st.subheader("Participant List")

    row_array = []
    users_list = []
    #Create a list of particiapnts for internal computation
    for user, data in users.items():
        row_array.append([data.userid, data.name, data.lastname])
        users_list.append(data.userid)

    st.write("Search for user in the following bar by typing user name:")
    option = st.multiselect(
        '', users_list)
    #Show list of all participants
    df = pd.DataFrame(row_array, columns=['User', "Name", "Last Name"])
    if (len(option)) is not 0:
        #Show a table with only the selected participants
        search_df = df[df['User'].isin(option)]
        st.table(search_df)
    else:
        #If not selected participant show all of them
        st.table(df)
    #More than one selected user
    if (len(option)) > 1:
        st.subheader("Comaprison of users")
        # Generate RMS plot concatenating all selected users
        seqgroup = [users[opt].openUserNpz_serie("rms_plot.npz") for opt in option]
        res = plotMultSeq(seqgroup, type="linear")
        st.image(res)
        # Generate Frequency spectral error plot concatenating all selected users
        seqgroup = [users[opt].openUserNpz_serie("freq_plot.npz") for opt in option]
        res = plotMultSeq(seqgroup, type="frequence")
        st.image(res)
    #In the user selction bar there was only one user selected
    if (len(option)) == 1:
        cols = st.beta_columns(2)
        # for usr_ind in range(len(option)):
        cols[0].subheader("Results of %s" % (option[0]))
        #Generate RMS plot
        res1 = plotSingSeq(users[option[0]].openUserNpz_serie("rms_plot.npz"), type="linear")
        #Generate spectral power plot
        res2 = plotSingSeq(users[option[0]].openUserNpz_serie("freq_plot.npz"), type="frequence")
        cols[0].image(res1)
        cols[0].image(res2)
        #Generate images of satelital reconstructed snaptshots
        res = getUserImages(users[option[0]])
        cols[0].write("Move the slider to visualize the reconstructed satelital timelapse")
        #Slider creation
        value = cols[0].slider("Number of frames: "+str(len(res)), 0, len(res)-1, 0, 1)
        # res3 = plot_arrayimg(res[value])
        #Show the snapshot
        cols[0].image(res[value])


from mmcv.runner import HOOKS
from mmcv.runner.hooks.lr_updater import CosineAnnealingLrUpdaterHook, annealing_cos

@HOOKS.register_module()
class CosineAnnealingUntilEpochLrUpdaterHook(CosineAnnealingLrUpdaterHook):
    """The same LR updater as CosineAnnealing but with `last epoch` support.

    Args:
        last_epoch (int, optional): The number of the last epoch where LR
            updating stops. This value can be greater than total_epochs. If it
            is equal to -1 LR will update until the end of the training.
            Default: -1.
    """

    def __init__(self, last_epoch=-1, **kwargs):
        assert last_epoch != 0
        self.last_epoch = int(last_epoch)
        super().__init__(**kwargs)
        if last_epoch > 0:
            assert self.by_epoch, '"last_epoch" requires "by_epoch" LR updating'

    def get_lr(self, runner, base_lr):
        if self.last_epoch == -1:
            self.last_epoch = runner.max_epochs

        if self.by_epoch:
            progress = runner.epoch
            max_progress = self.last_epoch
        else:
            progress = runner.iter
            max_progress = runner.max_iters

        if self.min_lr_ratio is not None:
            target_lr = base_lr * self.min_lr_ratio
        else:
            target_lr = self.min_lr

        if self.by_epoch and self.last_epoch < progress:
            return target_lr

        return annealing_cos(base_lr, target_lr, progress / max_progress)


import io
import os
import sys
import time
import pytest
from dotenv import load_dotenv
import peracommon

def import_executable(name):
    import importlib.util
    import importlib.machinery

    spec = importlib.util.spec_from_loader(name, importlib.machinery.SourceFileLoader(name, f"{os.path.dirname(__file__)}/../../{name}"))
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    sys.modules[name] = module
    return module

peracruda = import_executable("peracruda")

test_folders = [entries for entries in os.listdir("tests/source_files/") if os.path.isdir(f"tests/source_files/{entries}")]
for fold in set(test_folders):
    if "baseboard.txt" not in os.listdir(f"tests/source_files/{fold}"):
        test_folders.remove(fold)

# @pytest.fixture(params=test_folders)
# def args(request, monkeypatch):
#     parser = peracruda.generate_parser()
#     args = parser.parse_args(["-f", f"tests/source_files/{request.param}"])
#     # the following function could prompt a menu requesting gpu location.
#     # Setting a value into stdin buffer just in case input is needed
#     cli_input = io.StringIO("b")
#     monkeypatch.setattr("sys.stdin", cli_input)
#     return args

# def test_get_gpu(args):  # checking that gpu_location is correctly read
#     # checking mutual exclusion
#     assert [args.cpu, args.gpu, args.motherboard].count(True) <= 1

@pytest.fixture(params=test_folders)
def args2(request):
    # noinspection DuplicatedCode
    parsers = {
        peracommon.ParserComponents.CPU,
        peracommon.ParserComponents.GPU,
        peracommon.ParserComponents.MOTHERBOARD,
        peracommon.ParserComponents.RAM,
        peracommon.ParserComponents.CASE,
        peracommon.ParserComponents.PSU,
    }

    path = f"tests/source_files/{request.param}"
    try:
        with open(os.path.join(path, "gpu_location.txt"), "r") as f:
            gpu_flag = f.readline()
            if gpu_flag == "cpu":
                where = peracommon.GpuLocation.CPU
            elif gpu_flag == "gpu":
                where = peracommon.GpuLocation.DISCRETE
            else:
                where = peracommon.GpuLocation.MOTHERBOARD
    except FileNotFoundError:
        where = peracommon.GpuLocation.NONE

    return path, parsers, where

@pytest.fixture(autouse=True)
def load_dotenv_for_upload():
    load_dotenv()

@pytest.mark.upload
def test_upload_pytarallo(args2, monkeypatch, capsys):  # testing pytarallo integration with peracotta
    def auto_bulk_id():  # testing straight upload with automatic bulk_id
        cli_input = io.StringIO(f"y\n\n")
        monkeypatch.setattr("sys.stdin", cli_input)
        peracruda.upload_to_tarallo(result)
        assert "all went fine" in capsys.readouterr().out.lower()

    def fixed_bulk_id():  # testing automatic upload with a fixed bulk_id
        identifier = f"{args2[0]}{time.time()}"
        cli_input = io.StringIO(f"y\n{identifier}\n")
        monkeypatch.setattr("sys.stdin", cli_input)
        peracruda.upload_to_tarallo(result)
        assert "all went fine" in capsys.readouterr().out.lower()
        return identifier

    def overwrite_bulk_id(
        identifier,
    ):  # testing the overwrite of a bulk_id with an upload
        cli_input = io.StringIO(f"y\n{identifier}\ny\n")
        monkeypatch.setattr("sys.stdin", cli_input)
        peracruda.upload_to_tarallo(result)
        output = capsys.readouterr().out.lower()
        assert "do you want to try overwriting" in output
        assert "all went fine" in output

    def change_identifier(
        old_identifier,
    ):  # testing an upload with a new identifier after a failure
        cli_input = io.StringIO(f"y\n{old_identifier}\nn\n{args2[0]}{time.time()}")
        monkeypatch.setattr("sys.stdin", cli_input)
        peracruda.upload_to_tarallo(result)
        output = capsys.readouterr().out.lower()
        assert "do you want to use another identifier" in output
        assert "all went fine" in output

    try:
        result = peracommon.call_parsers(args2[0], args2[1], args2[2])
        result = peracommon.split_products(result)
        result = peracommon.make_tree(result)

        auto_bulk_id()
        overwrite_id = fixed_bulk_id()
        overwrite_bulk_id(overwrite_id)
        change_identifier(overwrite_id)

    except EnvironmentError:
        # if this exception is raised probably bad config of peracotta/.env -> test must fail
        assert False, "Bad peracotta .env configuration"


#!/usr/bin/env python3.7
"""

"""

from __future__ import annotations
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
from typing import List, Dict, Tuple, Union

_g_foo: Dict[str, int] = None

class RequestHandler(SimpleHTTPRequestHandler):
	protocol_version = 'HTTP/1.1'

	def do_GET(self):
		if self.path == '/':
			self.directory = 'static'
			super().do_GET()
		elif self.path == '/favicon.ico':
			self.directory = 'static'
			super().do_GET()
		elif self.path.startswith('/static/'):
			self.directory = 'static'
			self.path = self.path[len('/static'):]
			super().do_GET()
		elif self.path == '/stream/':
			self.do_GET_stream()
		elif self.path.startswith('/foo/'):
			self.do_GET_foo()
		else:
			print('GET', self.path)
			raise NotImplementedError

	def do_GET_stream(self):
		self.send_response(200)
		self.send_header('Content-Type', 'application/octet-stream')
		self.end_headers()
		for _ in range(10):
			self.wfile.write(b'$6\r\nfoobar\r\n')

	def do_GET_foo(self):
		"GET /foo/:name"
		_, foo, name = self.path.split('/')
		assert _ == ''
		assert foo == 'foo'
		name = name.encode('utf-8')

		count = _g_foo.get(name, 0)

		content = b'Hello %s (from GET)!\r\nSeen %d times' % (name, count)

		_g_foo[name] = count + 1

		self.send('text/plain', content)

	def do_POST(self):
		length = self.headers['content-length']
		nbytes = int(length)
		data = self.rfile.read(nbytes)
		# throw away extra data? see Lib/http/server.py:1203-1205
		self.data = data

		if self.path == '/foo/':
			self.do_POST_foo()
		else:
			print('POST', self.path)
			raise NotImplementedError

	def do_POST_foo(self):
		"POST /foo/"
		_, foo, _2 = self.path.split('/')
		assert _ == ''
		assert foo == 'foo'
		assert _2 == ''

		name = self.data

		count = _g_foo.get(name, 0)

		content = b'Hello %s (from POST)!\r\nSeen %d times\r\n' % (name, count)

		_g_foo[name] = count + 1

		self.send('text/plain', name)

	def send(self, content_type, content):
		use_keep_alive = self._should_use_keep_alive()
		use_gzip = self._should_use_gzip()

		if use_gzip:
			import gzip
			content = gzip.compress(content)

		self.send_response(200)
		self.send_header('Content-Type', content_type)
		self.send_header('Content-Length', str(len(content)))
		if use_keep_alive:
			self.send_header('Connection', 'keep-alive')
		if use_gzip:
			self.send_header('Content-Encoding', 'gzip')
		self.end_headers()
		self.wfile.write(content)

	def _should_use_keep_alive(self):
		connection = self.headers['connection']
		if connection is None:
			return False
		if connection != 'keep-alive':
			return False
		return True

	def _should_use_gzip(self):
		accept_encoding = self.headers['accept-encoding']
		if accept_encoding is None:
			return False
		if 'gzip' not in accept_encoding:
			return False
		return True

def main(bind, port):
	foo = {}

	global _g_foo
	_g_foo = foo

	address = (bind, port)
	print(f'Listening on {address}')
	server = ThreadingHTTPServer(address, RequestHandler)
	server.serve_forever()

def cli():
	import argparse

	parser = argparse.ArgumentParser()
	parser.add_argument('--bind', default='')
	parser.add_argument('--port', type=int, default=8810)
	args = vars(parser.parse_args())

	main(**args)

if __name__ == '__main__':
	cli()


from django.conf.urls import patterns, include, url

#estas dos lineas son para habilitar el admin
from django.contrib import admin
admin.autodiscover()

# importamos serializers
from rest_framework import routers
from webapp.views import *

router = routers.DefaultRouter()

router.register(r'users', UserViewSet)
router.register(r'facultades', FacultadesViewSet)
router.register(r'escuelas', EscuelaViewSet)
router.register(r'denominaciones', DenominacionViewSet)
router.register(r'alumnos', AlumnoViewSet)
router.register(r'sisopes', SisopeViewSet)
router.register(r'SisopesVersiones', SisopesVersionesViewSet)
router.register(r'registros', RegistroViewSet)
router.register(r'marcas', MarcaViewSet)
router.register(r'estados', EstadoViewSet)
router.register(r'laptops', LaptopViewSet)

admin.autodiscover()

urlpatterns = patterns('',

    url(r'^api/', include(router.urls)),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),

	(r'^grappelli/', include('grappelli.urls')), # grappelli URLS
	(r'^admin/',  include(admin.site.urls)), # admin site
    # Examples:
    # url(r'^$', 'controlunt.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    # url(r'^admin/', include(admin.site.urls)),
  	url(r'^$', 'webapp.views.home', name='home'),

  	url(r'^alumnos/(?P<n_pag>.*)/$', 'webapp.views.alumnos_pag', name='alumnos_pag'),
    url(r'^laptops/(?P<n_pag>.*)/$', 'webapp.views.laptops_pag', name='laptops_pag'),

    url(r'^control/$', 'webapp.views.control', name='control'),
    url(r'^reportes/$', 'webapp.views.reportes', name='reportes'),
    url(r'^soporte/$', 'webapp.views.soporte', name='soporte'),

    #Urls de sistema de login
    url(r'^usuario/nuevo$','webapp.views.nuevoUsuario'),
    url(r'^ingresar/$','webapp.views.ingresar'),
    url(r'^cerrar/$', 'webapp.views.cerrar'),

    url(r'^edit/alumno/(?P<id_alumno>.*)/$', 'webapp.views.edit_alumno', name='edit_alumno'),
    url(r'^new/alumno/$', 'webapp.views.nuevo_alumno', name='nuevo_alumno'),
    url(r'^edit/laptop/(?P<id_laptop>.*)/$', 'webapp.views.edit_laptop', name='edit_laptop'),
    url(r'^new/laptop/$', 'webapp.views.nueva_laptop', name='nueva_laptop'),
)

from django.conf import settings
if settings.DEBUG:
    # static files (images, css, javascript, etc.)
    urlpatterns += patterns('',
        (r'^media/(?P<path>.*)$', 'django.views.static.serve', {
        'document_root': settings.MEDIA_ROOT}))


from django import forms

class SelectAJAX(forms.Select):
    '''Clase para extender el widget Select estandar con funcionalidades de filtrado por AJAX'''

    def __init__(self, bindings ,attrs=None, choices=()):
        '''
        Se necesitan los bindings para menejar la consulta, el resto depende de
        la clase padre (Select)

        bindings: Instancia con los datos para realizar la consulta sobre la vista
                  y actualizar los campos correspondientes.
        '''
        self.bindings = bindings
        super(SelectAJAX, self).__init__(attrs, choices)

    def render (self, name, value, attrs=None, choices=()):
        '''
        Los parametros dependen de la clase padre.
        la funcion genera contenido extra mediante los bindings para utilizar
        el contenido JS
        '''
        name_html=self.build_attrs(attrs, name=name).get('id', name)
        prefix= 'id_'
        if name_html.rfind('-') > 0:
            prefix= name_html[:name_html.rfind('-') + 1]
        onchange= []
        for b in self.bindings:
            onchange.append("populateSelect('%s', '%s', '%s', '%s', '%s', '%s_set', '%s');"%(
                b.url, name_html, prefix + b.detail_field_name, b.app_name, b.master_model_name, b.detail_model_name.lower(), b.detail_display_field))
        att={}
        if len(onchange)!=0:
            att['onChange']= ' '.join(onchange)
        att.update(attrs)
        return super(SelectAJAX,self).render(name, value, att, choices)


from __future__ import absolute_import
from flask import current_app

from flamaster.core.decorators import api_resource
from flamaster.core.resources import Resource
from flamaster.core.utils import jsonify_status_code

from werkzeug import import_string
from . import bp

@api_resource(bp, 'options', {'id': int})
class DeliveryResource(Resource):

    def get(self, id=None):
        resolved_options = {}
        for key, value in current_app.config['DELIVERY_OPTIONS'].iteritems():
            if 'default' not in value:
                resolved_options[key] = import_string(value['module'])

        items = [{key: Cls.name} for key, Cls in resolved_options.iteritems()]
        return self.__build_response(items)

    def put(self, id=None):
        raise NotImplementedError()

    def post(self):
        raise NotImplementedError()

    def delete(self, id=None):
        raise NotImplementedError()

    def __build_response(self, items):
        response = {
            'objects': items,
            'meta': None
        }
        return jsonify_status_code(response)


import os
import tensorflow as tf
import numpy as np

def _read_to_tensor(fname, output_height=256, output_width=256, normalize_data=False):
    '''Function to read images from given image file path, and provide resized images as tensors
        Inputs:
            fname - image file path
            output_height - required output image height
            output_width - required output image width
            normalize_data - if True, normalize data to be centered around 0 (mean 0, range 0 to 1)
        Output: Processed image tensors
    '''

    # Read the image as a tensor
    img_strings = tf.io.read_file(fname)
    imgs_decoded = tf.image.decode_jpeg(img_strings, channels=3)
    print(imgs_decoded.shape)
    # Resize the image
    output = tf.image.resize(imgs_decoded, [output_height, output_width])
    print(output.shape)
    # Normalize if required
    if normalize_data:
        output = (output - 128) / 128
    return output

def read_images(img_dir):
    '''Function to get all image directories, read images and masks in separate tensors
        Inputs:
            img_dir - file directory
        Outputs
            frame_tensors, masks_tensors, frame files list, mask files list
    '''

    # Get the file names list from provided directory
    # file_list = [f for f in os.listdir(img_dir) if os.path.isfile(os.path.join(img_dir, f))]

    file_list_frames = []

    for f in os.listdir(img_dir + "train_frames/train/"):
        file_list_frames.append(f)

    file_list_masks = []

    for f in os.listdir(img_dir + "train_masks/train/"):
        file_list_masks.append(f)

    # Separate frame and mask files lists, exclude unnecessary files
    frames_list = file_list_frames
    masks_list = file_list_masks

    print('{} frame files found in the provided directory.'.format(len(frames_list)))
    print('{} mask files found in the provided directory.'.format(len(masks_list)))

    # Create file paths from file names
    frames_paths = [os.path.join(img_dir + 'train_frames/train/', fname) for fname in frames_list]
    masks_paths = [os.path.join(img_dir + 'train_masks/train/', fname) for fname in masks_list]
    frames_paths.sort()
    masks_paths.sort()

    print(frames_paths)
    print(masks_paths)

    # Create dataset of tensors
    frame_data = tf.data.Dataset.from_tensor_slices(frames_paths)
    masks_data = tf.data.Dataset.from_tensor_slices(masks_paths)

    # Read images into the tensor dataset
    frame_tensors = frame_data.map(_read_to_tensor)
    masks_tensors = masks_data.map(_read_to_tensor)

    print('Completed importing {} frame images from the provided directory.'.format(len(frames_list)))
    print('Completed importing {} mask images from the provided directory.'.format(len(masks_list)))

    return frame_tensors, masks_tensors, frames_list, masks_list

def generate_image_folder_structure(frames, masks, frames_list, masks_list):
    '''Function to save images in the appropriate folder directories
        Inputs:
            frames - frame tensor dataset
            masks - mask tensor dataset
            frames_list - frame file paths
            masks_list - mask file paths
    '''
    # Create iterators for frames and masks
    frame_batches = tf.compat.v1.data.make_one_shot_iterator(
        frames)  # outside of TF Eager, we would use make_one_shot_iterator
    mask_batches = tf.compat.v1.data.make_one_shot_iterator(masks)
    print(frames_list)

    # Iterate over the train images while saving the frames and masks in appropriate folders
    dir_name = 'train'
    for file in zip(frames_list[:-round(0.2 * len(frames_list))], masks_list[:-round(0.2 * len(masks_list))]):
        # Convert tensors to numpy arrays
        frame = frame_batches.next().numpy().astype(np.uint8)
        mask = mask_batches.next().numpy().astype(np.uint8)

        # Convert numpy arrays to images
        frame = Image.fromarray(frame)
        mask = Image.fromarray(mask)

        # Save frames and masks to correct directories
        frame.save(DATA_PATH + '{}_frames/{}'.format(dir_name, dir_name) + '/' + file[0])
        mask.save(DATA_PATH + '{}_masks/{}'.format(dir_name, dir_name) + '/' + file[1])

    # Iterate over the val images while saving the frames and masks in appropriate folders
    dir_name = 'val'
    for file in zip(frames_list[-round(0.2 * len(frames_list)):], masks_list[-round(0.2 * len(masks_list)):]):
        # Convert tensors to numpy arrays
        frame = frame_batches.next().numpy().astype(np.uint8)
        mask = mask_batches.next().numpy().astype(np.uint8)

        # Convert numpy arrays to images
        frame = Image.fromarray(frame)
        mask = Image.fromarray(mask)

        # Save frames and masks to correct directories
        frame.save(DATA_PATH + '{}_frames/{}'.format(dir_name, dir_name) + '/' + file[0])
        mask.save(DATA_PATH + '{}_masks/{}'.format(dir_name, dir_name) + '/' + file[1])

    print("Saved {} frames to directory {}".format(len(frames_list), DATA_PATH))
    print("Saved {} masks to directory {}".format(len(masks_list), DATA_PATH))

def parse_code(l):
    '''Function to parse lines in a text file, returns separated elements (label codes and names in this case)
    '''

    #print(l)
    #print(l.strip().split("\t"))
    #print(l.strip().split("\t"))
    object_name, r,g,b = l.strip().split("\t")
    color_code = (r,g,b)
    r,g,b = [int(x) for x in color_code]
    #values = a.split(" ")
    #print(values)
    #values = [int(ele) for ele in values]

    return (r,g,b), object_name

def rgb_to_onehot(rgb_image, colormap):
    '''Function to one hot encode RGB mask labels
        Inputs:
            rgb_image - image matrix (eg. 256 x 256 x 3 dimension numpy ndarray)
            colormap - dictionary of color to label id
        Output: One hot encoded image of dimensions (height x width x num_classes) where num_classes = len(colormap)
    '''
    num_classes = len(colormap)
    shape = rgb_image.shape[:2]+(num_classes,)
    encoded_image = np.zeros( shape, dtype=np.int8 )
    for i, cls in enumerate(colormap):
        encoded_image[:,:,i] = np.all(rgb_image.reshape( (-1,3) ) == colormap[i], axis=1).reshape(shape[:2])
    return encoded_image

def onehot_to_rgb(onehot, colormap):
    '''Function to decode encoded mask labels
        Inputs:
            onehot - one hot encoded image matrix (height x width x num_classes)
            colormap - dictionary of color to label id
        Output: Decoded RGB image (height x width x 3)
    '''
    single_layer = np.argmax(onehot, axis=-1)
    output = np.zeros( onehot.shape[:2]+(3,) )
    for k in colormap.keys():
        output[single_layer==k] = colormap[k]
    return np.uint8(output)

import albumentations as A
from tensorflow.keras.utils import Sequence

def round_clip_0_1(x, **kwargs):
    return x.round().clip(0, 1)

# define heavy augmentations
def get_training_augmentation():
    train_transform = [

        A.HorizontalFlip(p=0.5),

        A.ShiftScaleRotate(scale_limit=0.5, rotate_limit=0, shift_limit=0.1, p=1, border_mode=0),

        A.PadIfNeeded(min_height=320, min_width=320, always_apply=True, border_mode=0),
        A.RandomCrop(height=320, width=320, always_apply=True),

        A.IAAAdditiveGaussianNoise(p=0.2),
        A.IAAPerspective(p=0.5),

        A.OneOf(
            [
                A.CLAHE(p=1),
                A.RandomBrightness(p=1),
                A.RandomGamma(p=1),
            ],
            p=0.9,
        ),

        A.OneOf(
            [
                A.IAASharpen(p=1),
                A.Blur(blur_limit=3, p=1),
                A.MotionBlur(blur_limit=3, p=1),
            ],
            p=0.9,
        ),

        A.OneOf(
            [
                A.RandomContrast(p=1),
                A.HueSaturationValue(p=1),
            ],
            p=0.9,
        ),
        A.Lambda(mask=round_clip_0_1)
    ]
    return A.Compose(train_transform)

# https://www.kaggle.com/khlevnov/imagedatagenerator-and-albumentations-without-pain
class AugmentDataGenerator(Sequence):
    def __init__(self, datagen, augment=None):
        self.datagen = datagen
        if augment is None:
            self.augment = A.Compose([])
        else:
            self.augment = augment

    def __len__(self):
        return len(self.datagen)

    def __getitem__(self, x):
        images, *rest = self.datagen[x]
        augmented = []
        for image in images:
            image = self.augment(image=image)['image']
            augmented.append(image)
        return (np.array(augmented), *rest)

def TrainAugmentGenerator(DATA_PATH, id2code, train_frames_datagen, train_masks_datagen,seed=1, batch_size=5):
    '''Train Image data generator
        Inputs:
            seed - seed provided to the flow_from_directory function to ensure aligned data flow
            batch_size - number of images to import at a time
        Output: Decoded RGB image (height x width x 3)
    '''
    train_image_generator = train_frames_datagen.flow_from_directory(
        DATA_PATH + 'train_frames/',
        batch_size=batch_size, seed=seed)

    train_mask_generator = train_masks_datagen.flow_from_directory(
        DATA_PATH + 'train_masks/',
        batch_size=batch_size, seed=seed)

    while True:
        X1i = train_image_generator.next()
        X2i = train_mask_generator.next()

        # One hot encoding RGB images
        mask_encoded = [rgb_to_onehot(X2i[0][x, :, :, :], id2code) for x in range(X2i[0].shape[0])]

        yield X1i[0], np.asarray(mask_encoded)

def ValAugmentGenerator(DATA_PATH, id2code, val_frames_datagen, val_masks_datagen, seed=1, batch_size=5):
    '''Validation Image data generator
        Inputs:
            seed - seed provided to the flow_from_directory function to ensure aligned data flow
            batch_size - number of images to import at a time
        Output: Decoded RGB image (height x width x 3)
    '''
    val_image_generator = val_frames_datagen.flow_from_directory(
        DATA_PATH + 'val_frames/',
        batch_size=batch_size, seed=seed)

    val_mask_generator = val_masks_datagen.flow_from_directory(
        DATA_PATH + 'val_masks/',
        batch_size=batch_size, seed=seed)

    while True:
        X1i = val_image_generator.next()
        X2i = val_mask_generator.next()

        # One hot encoding RGB images
        mask_encoded = [rgb_to_onehot(X2i[0][x, :, :, :], id2code) for x in range(X2i[0].shape[0])]

        yield X1i[0], np.asarray(mask_encoded) , X2i[0]
        #yield X1i[0], np.asarray(mask_encoded), X2i[0]


#!/usr/bin/env python
# coding=utf-8

# Author      :   Xionghui Chen
# Created     :   2017-11-11
# Modified    :   2017-11-14
# Version     :   1.0
import tensorflow as tf
import collections
import os

class CustomSession(object):

    ALREADY_INITIALIZED = set()
    VALUE_SETTERS = collections.OrderedDict()

    def __init__(self, name='default_session_name'):
        self.session_name = name
        self.sess = None

    def make_session(self, graph, percent=1.0, adaptive=False, interactive=False):
        """
        create session object

        Parameters
        ----------
        percent : float
            A value between 0 and 1 that indicates
            what fraction of the available GPU memory to pre-allocate for each process.

        adaptive : bool
            If true, the allocator does not pre-allocate the entire specified GPU memory region,
            instead starting small and growing as needed.

            NOTE: if adaptive set true, percent turn to useless

        Returns
        -------
        string
            a value in a string

        Raises
        ------
        KeyError
            when a key error
        """
        gpu_options = tf.GPUOptions(
            allow_growth=adaptive, per_process_gpu_memory_fraction=percent)
        tf_config = tf.ConfigProto(gpu_options=gpu_options)
        self.graph = graph
        if interactive:
            self.sess = tf.InteractiveSession(graph=graph, config=tf_config)
        else:
            self.sess = tf.Session(graph=graph, config=tf_config)
        return self.sess

    def run(self, *argv, **argkw):
        with self.graph.as_default():
            self.sess(argv, argkw)

    def make_cpu_session(self, graph, percent=1, adaptive=False, interactive=False):
        os.environ['CUDA_VISIBLE_DEVICES'] = ''
        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0)
        tf_config = tf.ConfigProto(
            gpu_options=gpu_options, device_count={'gpu': 0})
        # self.sess = tf.Session(config=tf.ConfigProto(device_count={'gpu': 0}))
        self.graph = graph
        if interactive:
            self.sess = tf.InteractiveSession(graph=graph, config=tf_config)
        else:
            self.sess = tf.Session(graph=graph, config=tf_config)
        return self.sess

    def initialize(self):
        """Initialize all the uninitialized variables in the global scope."""
        new_variables = set(tf.global_variables()) - self.ALREADY_INITIALIZED
        self.sess.run(tf.variables_initializer(new_variables))
        self.ALREADY_INITIALIZED.update(new_variables)

    def run(self, op, feed_dict=None):
        if feed_dict != None:
            self.sess.run(op, feed_dict=feed_dict)
        else:
            self.sess.run(op)

    def init_saver(self, max_to_keep=3):
        self.saver = tf.train.Saver(max_to_keep=max_to_keep)

    def load_state(self, fname):
        checkpoint = tf.train.get_checkpoint_state(fname)
        if checkpoint and checkpoint.model_checkpoint_path:
            print(checkpoint)
            self.saver.restore(self.sess, checkpoint.model_checkpoint_path)
            print("Successfully loaded:", checkpoint.model_checkpoint_path)
        else:
            print("Could not find old network weights")

    def save_state(self, fname, time_step):
        os.makedirs(os.path.dirname(fname), exist_ok=True)
        self.saver.save(self.sess, fname, global_step=time_step)

    def function(self, inputs, outputs, updates=None, givens=None):
        """Just like Theano function. Take a bunch of tensorflow placeholders and expressions
        computed based on those placeholders and produces f(inputs) -> outputs. Function f takes
        values to be fed to the input's placeholders and produces the values of the expressions
        in outputs.

        Input values can be passed in the same order as inputs or can be provided as kwargs based
        on placeholder name (passed to constructor or accessible via placeholder.op.name).

        [this function modify from: https://github.com/openai/baselines]

        Example:
            x = tf.placeholder(tf.int32, (), name="x")
            y = tf.placeholder(tf.int32, (), name="y")
            z = 3 * x + 2 * y
            lin = function([x, y], z, givens={y: 0})

            with single_threaded_session():
                initialize()

                assert lin(2) == 6
                assert lin(x=3) == 9
                assert lin(2, 2) == 10
                assert lin(x=2, y=3) == 12

        Parameters
        ----------
        inputs: [tf.placeholder or TfInput]
            list of input arguments
        outputs: [tf.Variable] or tf.Variable
            list of outputs or a single output to be returned from function. Returned
            value will also have the same shape.
        """
        if isinstance(outputs, list):
            return _Function(self.sess, inputs, outputs, updates, givens=givens)
        elif isinstance(outputs, (dict, collections.OrderedDict)):
            f = _Function(self.sess, inputs, outputs.values(), updates, givens)
            return lambda *args, **kwargs: type(outputs)(zip(outputs.keys(), f(*args, **kwargs)))
        else:
            f = _Function(self.sess, inputs, [outputs], updates, givens=givens)
            return lambda *args, **kwargs: f(*args, **kwargs)[0]

class _Function(object):
    '''
    [this function modify from: https://github.com/openai/baselines]

    '''

    def __init__(self, session, inputs, outputs, updates, givens, check_nan=False):
        # for inpt in inputs:
        #     if not issubclass(type(inpt), TfInput):
        #         assert len(inpt.op.inputs) == 0, "inputs should all be placeholders of baselines.common.TfInput"
        self.inputs = inputs
        updates = updates or []
        self.update_group = tf.group(*updates)
        self.outputs_update = list(outputs) + [self.update_group]
        self.givens = {} if givens is None else givens
        self.check_nan = check_nan
        self.session = session

    def _feed_input(self, feed_dict, inpt, value):
        # if issubclass(type(inpt), TfInput):
        #     feed_dict.update(inpt.make_feed_dict(value))
        # elif is_placeholder(inpt):
        feed_dict[inpt] = value

    def __call__(self, *args, **kwargs):
        assert len(args) <= len(self.inputs), "Too many arguments provided"
        feed_dict = {}
        # Update the args
        for inpt, value in zip(self.inputs, args):
            self._feed_input(feed_dict, inpt, value)
        # Update the kwargs
        kwargs_passed_inpt_names = set()
        for inpt in self.inputs[len(args):]:
            inpt_name = inpt.name.split(':')[0]
            inpt_name = inpt_name.split('/')[-1]
            assert inpt_name not in kwargs_passed_inpt_names, \
                "this function has two arguments with the same name \"{}\", so kwargs cannot be used.".format(
                    inpt_name)
            if inpt_name in kwargs:
                kwargs_passed_inpt_names.add(inpt_name)
                self._feed_input(feed_dict, inpt, kwargs.pop(inpt_name))
            else:
                assert inpt in self.givens, "Missing argument " + inpt_name
        assert len(kwargs) == 0, "Function got extra arguments " + \
            str(list(kwargs.keys()))
        # Update feed dict with givens.
        for inpt in self.givens:
            feed_dict[inpt] = feed_dict.get(inpt, self.givens[inpt])
        results = self.session.run(
            self.outputs_update, feed_dict=feed_dict)[:-1]
        if self.check_nan:
            if any(np.isnan(r).any() for r in results):
                raise RuntimeError("Nan detected")
        return results

def is_placeholder(x):
    return type(x) is tf.Tensor and len(x.op.inputs) == 0


# -*- coding: utf-8 -*-
'''
urls
'''
from django.conf.urls import include, url
from django.contrib import admin
from django.contrib.auth.views import login, logout
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.views.generic.base import TemplateView
from django.views.i18n import javascript_catalog

import views

admin.autodiscover()

urlpatterns = [
    url(r'^admin/',             include(admin.site.urls)),
    url(r'^admin/jsi18n',       javascript_catalog),    # hack to use admin form widgets
    url(r'^accounts/login/$',   login, name='login'),
    url(r'^logout$',            logout, name='logout'),
    url(r'^$',                  TemplateView.as_view(template_name='index.html'), name='index'),
    url(r'^about$',             TemplateView.as_view(template_name='about.html'), name='about'),
    url(r'^core/',              include('core.urls')),
    url(r'^bills/',             include('bills.urls')),
    url(r'^scan/',              include('scan.urls')),
    url(r'^contract/',          include('contract.urls')),
    url(r'^contrarch/',         include('contrarch.urls')),
    url(r'^report/',            include('reports.urls')),
    url(r'^chk/$',              views.chk, name='chk'),
    url(r'^cln/(?P<f>\d+)/$',   views.cln, name='cln'),
]

urlpatterns += staticfiles_urlpatterns()