"""Licores URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from authApp import views
from rest_framework_simplejwt.views import (TokenObtainPairView, TokenRefreshView)

urlpatterns = [
    path('admin/', admin.site.urls),
    path('login/', TokenObtainPairView.as_view()),
    path('refresh/', TokenRefreshView.as_view()),
    path('user/', views.UserCreateView.as_view()),
    path('user/<int:pk>/', views.UserDetailView.as_view()),
    path('ventas/', views.VentasCreateView.as_view()),
    path('ventas/<int:user>/<int:pk>/', views.VentasDetailView.as_view()),
    path('ventas/<int:user>/<int:producto>/', views.VentasProductosView.as_view()),
    path('ventas/remove/<int:user>/<int:pk>/', views.VentasDeleteView.as_view()),
    path('productos/', views.ProductosCreateView.as_view()),
    path('productos/remove/<pk>/', views.ProductosDeleteView.as_view()),

]


import os; os.environ['TF_CPP_MIN_LOG_LEVEL']='3'  # disable TF logging
import numpy as np
import keras
from keras.models import load_model

##########################################################
def test(x_test, y_test):
    model = load_model("./model/mnist_CNN_model.h5")

    score = model.evaluate(x_test, y_test, batch_size=100, verbose=1)
    print("\nAccurary of trained model against test data:", score[1])

    n_random_test = 5
    print("\nRandom test with %d data.." %n_random_test)
    idx = np.random.choice(x_test.shape[0], n_random_test)  # 5 indices
    v = model.predict(x_test[idx])
    y = model.predict_classes(x_test[idx])
    np.set_printoptions(precision=1)
    for k in range(n_random_test):
        i = idx[k]
        print("="*30 + "\nRandom test %d:" %i)
        print("actual: %d,  predicted: %d" %(np.argmax(y_test[i]), y[k]))
        # print("predicted softmax:", v[k])

##############################################
def run():
    mnist = keras.datasets.mnist.load_data()  # ~/.keras/datasets/mnist.npz
    (_, _), (x_test, y_test) = mnist
    x_test = x_test.reshape(-1, 28, 28, 1) / 255.0
    y_test = keras.utils.to_categorical(y_test)
    test(x_test, y_test)

run()


#!/home/noische/Enthought/Canopy_64bit/User/bin/python

import sys
import getopt
import optparse
import cPickle as pickle

import numpy as np
from numpy import arccos
from numpy.linalg import norm
import tqdm

import bgf
import bgftools as bt
import lammpstrj as lt
#from lammpstrj import *
import nutils as nu

from lammps_analyze_trj_2 import *

def count_layer(bgf_file, trj_file, ff_file='', out_file=''):
    '''counts number of O atoms in each layer.
    '''
    # variables
    result = dict()

    # 1. Load BGF
    mybgf = bgf.BgfFile(bgf_file)
    N_BGF_ATOMS = len(mybgf.a)
    r_vdw_C = 3.38383824/2

    # 2. Read LAMMPS Trajectory
    mytrj = lt.lammpstrj(trj_file)
    timesteps = mytrj.load()
    N_ATOMS = mytrj.natoms[0]
    if N_BGF_ATOMS != N_ATOMS:
        nu.die("Number of atoms in trajectory file does not match with BGF file.")

    # 4. Update coordinates from the snapshot
    for t in tqdm.tqdm(timesteps, ncols=120, desc="Calculating graphene z positions"):
        mybgf = get_timestep(mybgf, trj_file, t)
        avg_gra_z1 = bt.atoms_average(mybgf, 'atom.z', selection="'C_2G' in atom.ffType and atom.rNo == 1")
        avg_gra_z2 = bt.atoms_average(mybgf, 'atom.z', selection="'C_2G' in atom.ffType and atom.rNo == 2")
        dist = avg_gra_z2 - avg_gra_z1
        eff_dist = avg_gra_z2 - avg_gra_z1 - 2 * r_vdw_C
        result[t] = [avg_gra_z1, avg_gra_z2, dist, eff_dist]

    # 5. Analyze
    timesteps = sorted(result.keys())
    with open('z_position.test.dat', 'w') as f:
        output = ''
        for t in timesteps:
            output += "%d " % t
            output += "%8.3f %8.3f %8.3f %8.3f\n" % result[t]

        f.write(output)

    # 7. Done

if __name__ == "__main__":
    bgf_file = ""; trj_file = ""; ff_file = ""; z_ranges = '';

    usage = "%s -b bgf_file -t trj_file -f ff_file" % sys.argv[0]
    if len(sys.argv) < 2:
        print(usage)
        sys.exit(0)

    options, args = getopt.getopt(sys.argv[1:], 'hb:t:f:', ['help', 'bgf=', 'trj=', 'ff='])

    print(sys.argv)
    print("Requested options: " + str(options))

    for option, value in options:
        if option in ('-h', '--help'):
            print(usage)
            sys.exit(0)
        elif option in ('-b', '--bgf'):
            bgf_file = value
        elif option in ('-t', '--trj'):
            trj_file = value
        elif option in ('-f', '--ff'):
            ff_file = str(value).strip()
        elif option == NULL:
            print(usage)
            sys.exit(0)

    count_layer(bgf_file, trj_file, ff_file=ff_file)


"""
Admin controller

This file contains the logic behind our admin site. We register our models with the Django Admin system.
Through the admin interface, admins can view and edit most models.
Custom functionality for certain models is implemented below.

Models we have included:
Doctor -------- All doctors and their profile information
Hospital ------ All hospitals and their profile information
Nurse --------- All nurses and their profile information
Patient ------- All patients and their profile information
Message ------- All messages and their contents
UploadedFile -- All files and their contents.
Appointment --- All appointments and their contents.
LogEntry ------ Displays logs for all user actions. Read only
AdminLog ------ Displays logs for Admin actions. Read only
AdmissionLog -- Used to provide statistics on patient admissions. Read only
"""

from django.contrib import admin
from django.contrib.admin.models import LogEntry as AdminLogEntry

from .models import Appointment, Doctor, Hospital, Nurse, Patient, LogEntry, TestFile, Message, AdmissionLog, Test, \
    Prescription, MedInfo

admin.site.register(Doctor)
admin.site.register(Hospital)
admin.site.register(Nurse)
admin.site.register(Patient)
admin.site.register(Message)
admin.site.register(AdmissionLog)
admin.site.register(Prescription)
admin.site.register(Test)
admin.site.register(MedInfo)

class TestFileAdmin(admin.ModelAdmin):
    list_display = ('title', 'file', 'uploaded_at')

admin.site.register(TestFile, TestFileAdmin)

class AppointmentAdmin(admin.ModelAdmin):
    list_display = ('start_time', 'doctor', 'patient')

admin.site.register(Appointment, AppointmentAdmin)

# User Logs are read only. Styling included to allow filtering of all Log Entries
class LogAdmin(admin.ModelAdmin):
    readonly_fields = ('userMail', 'time', 'action')
    list_display = ('time', 'userMail', 'action')
    list_filter = ['time']
    search_fields = ['userMail', 'action']

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request):
        return False

    def save_model(self, request, obj, form, change):
        # Return nothing to make sure user can't update any data
        pass

    def get_actions(self, request):
        actions = super(LogAdmin, self).get_actions(request)
        del actions['delete_selected']
        return actions

admin.site.register(LogEntry, LogAdmin)

# Admin Logs are read only and cannot be added or deleted on command.
class AdminLogEntryAdmin(admin.ModelAdmin):
    readonly_fields = ('content_type',
                       'user',
                       'action_time',
                       'object_id',
                       'object_repr',
                       'action_flag',
                       'change_message'
                       )

    list_display = ('user', 'get_action')

    def get_action(self, obj):
        return str(obj)

    get_action.short_description = 'Action'

    def has_delete_permission(self, request, obj=None):
        return False

    def has_add_permission(self, request):
        return False

    def save_model(self, request, obj, form, change):
        # Return nothing to make sure user can't update any data
        pass

    def get_actions(self, request):
        actions = super(AdminLogEntryAdmin, self).get_actions(request)
        del actions['delete_selected']
        return actions

admin.site.register(AdminLogEntry, AdminLogEntryAdmin)


from marshmallow import Schema, fields, validates_schema

from willstores.errors import ValidationError

class PriceRangeSchema(Schema):
    min = fields.Float(required=True)
    max = fields.Float(required=True)
    currency = fields.String(required=True)

    @validates_schema(skip_on_field_errors=True)
    def validate_pricerange(self, data):
        valmin = data["min"]
        valmax = data["max"]
        currency = data["currency"]
        if valmin <= 0.0:
            raise ValidationError("Invalid min '%s', must be positive" % valmin)
        elif valmax <= 0.0:
            raise ValidationError("Invalid max '%s', must be positive" % valmax)
        elif valmax < valmin:
            raise ValidationError("Max '%s' cannot be lower than min '%s'" % (valmax, valmin))
        elif len(currency) != 3:
            raise ValidationError("Invalid currency '%s'. Must be an acronym with 3 chars." % currency)


# -*- coding: utf-8 -*-
import numpy as np

# NLP imports
from nltk.corpus import stopwords
stopwords=stopwords.words('german')

# modeling imports
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import fbeta_score, make_scorer

from utils import modeling as m
from utils import cleaning, transformers

import mlflow
from modeling.config import TRACKING_URI, EXPERIMENT_NAME#, TRACKING_URI_DEV
import logging

from time import time

# set logging
logger = logging.getLogger(__name__)
logging.basicConfig(format="%(asctime)s: %(message)s")
logging.getLogger("pyhive").setLevel(logging.CRITICAL)  # avoid excessive logs
logger.setLevel(logging.INFO)

if __name__ == "__main__":
    data = m.Posts()
    scorer = make_scorer(fbeta_score, beta=2)

    trans_os = {'translate':[0.9], 'oversample':[0.9]}

    TARGET_LABELS = ['label_discriminating', 'label_inappropriate',
        'label_sentimentnegative', 'label_needsmoderation']

    embedding_dict_glove = transformers.load_embedding_vectors(embedding_style='glove', file="./embeddings/glove_vectors.txt")
    embedding_dict_word2vec = transformers.load_embedding_vectors(embedding_style='word2vec', file="./embeddings/word2vec_vectors.txt")

    preps = {
            'norm': lambda x: cleaning.series_apply_chaining(x, [cleaning.normalize]),
            'glove': transformers.MeanEmbeddingVectorizer(embedding_dict=embedding_dict_glove).transform,
            'word2vec': transformers.MeanEmbeddingVectorizer(embedding_dict=embedding_dict_word2vec).transform,
            }
    vecs = {
            'count': CountVectorizer(),
           'tfidf': TfidfVectorizer(),
            }
    PRE_VEC_COMBINATIONS = [
                        ['glove', 'glove'],
                        ['word2vec', 'word2vec'],
                        ['norm', 'count'],
                        ['norm', 'tfidf'],
                    ]

    for method, strat in trans_os.items():
        for strategy in strat:
            print(method, strategy)
            for label in TARGET_LABELS:
                for c in PRE_VEC_COMBINATIONS:
                    mlflow_params=dict()
                    print(c)
                    constant_preprocessor=preps[c[0]]
                    if c[1] in ['count', 'tfidf']:
                        pipeline = Pipeline([
                            ("vectorizer", vecs[c[1]]),
                            ("clf", SVC()),
                            ])
                        param_grid = {
                            "vectorizer__ngram_range" : [(1,1), (1,2), (1,3)],
                            "vectorizer__stop_words" : [stopwords, None],
                            "vectorizer__min_df": [0.],
                            "vectorizer__max_df": [0.9],
                           "clf__C": [.5**0, .5**1, .5**2, .5**3, .5**4, .5**5],
                            "clf__kernel": ['linear', 'rbf'],
                        }
                        grid_search_params = param_grid.copy()
                        # MLFlow params have limited characters, therefore stopwords must not be given as list
                        grid_search_params["vectorizer__stop_words"] = ["NLTK-German", None]
                        mlflow_params["normalization"] = c[0]
                        mlflow_params["vectorizer"]    = c[1]
                    else:
                        pipeline = Pipeline([
                            ("clf", SVC()),
                            ])
                        param_grid = {
                            "clf__C": [.5**0, .5**1, .5**2, .5**3, .5**4, .5**5],
                            "clf__kernel": ['linear', 'rbf'],
                        }
                        grid_search_params = param_grid.copy()
                        mlflow_params["normalization"] = 'norm'
                        mlflow_params["vectorizer"]    = c[1]

                    gs = GridSearchCV(pipeline, param_grid, scoring=scorer, cv=5, verbose=1, n_jobs=-1)

                    mlflow_params["model"]=  "SupportVectorMachine"
                    mlflow_params["grid_search_params"]=  str(grid_search_params)[:249]
                    mlflow_tags = {
                        "cycle4": True,
                    }

                    IS_DEVELOPMENT = False

                    mlflow_logger = m.MLFlowLogger(
                        uri=TRACKING_URI,
                        experiment=EXPERIMENT_NAME,
                        is_dev=IS_DEVELOPMENT,
                        params=mlflow_params,
                        tags=mlflow_tags
                    )
                    training = m.Modeling(data, gs, mlflow_logger)

                    logger.info(f"-"*20)
                    logger.info(f"Target: {label}")
                    data.set_label(label=label)
                    data.set_balance_method(balance_method=method, sampling_strategy=strategy)
                    training.train(constant_preprocessor=constant_preprocessor)
                    training.evaluate(["train", "val"],constant_preprocessor=constant_preprocessor)
                    #if True:
                    with mlflow.start_run(run_name='svc_with_fbeta') as run:
                        mlflow_logger.log()


#!/usr/bin/python3 -tt
#
# This script reads files from the input_files directory:
# - UI labels file (get from /UI/app/lang/lang_en_US_labels.json from current branch in your platform/ui repo
# - database privileges information (run process_privileges.sql on the latest Abiquo DB to create process_privileges_sql.txt)
# - an extra text file (process_privileges_extratext.txt)
# It creates wiki_privileges.txt - a wiki storage format table for pasting into Privileges page of the wiki
# NB: check that privs_processed which is written to standard out is equal to
# the number of rows in privilege table in Abiquo DB
# select * from privilege;
#
import sys
import re
import json as simplejson
import os

def main():
    sql_total = 0
    privs_processed = 0
    rowing = {}
    rowing[1] = "
\n\n"
    rowing[3] = "Privileges	\n"
    rowing[4] = " 	\n"
    for x in range(5, 11):
        rowing [x] = rowing [4]
    rowing[11] = "
 \n"
    rowing[12] = "

GUI Label _________________
	\n"
    rowing[13] = "
Application Tag
	\n"
    rowing[14] = "\n
Privilege____________________________________
	\n"
    rowing[15] = "
Cloud Admin
	\n"
    rowing[16] = "
Ent Admin
	\n"
    rowing[17] = "
Ent User
	\n"
    rowing[18] = "
Outbound API
	\n"
    rowing[19] = "
Info

\n"
    extratext = {}
    input_gitdir = '../platform/ui/app/lang'
    input_subdir = 'input_files'
    output_subdir = 'output_files'
    extfile = 'process_privileges_extratext.txt'
    extlines = (extline.rstrip() for extline in open(os.path.join(input_subdir,extfile)))
    for ext_orig in extlines:
        #print (ext_orig)
        extlist = ext_orig.split("|")
        extkey = extlist[0]
        extkey = extkey.strip()
        exttext = extlist[1]
        exttext = exttext.strip()
        extratext[extkey] = exttext
    sqllabels = {}
    sqlroles = {}
    sqlfile = 'process_privileges_sql_2014-10-21.txt'
    sqllines = (sqlline.rstrip() for sqlline in open(os.path.join(input_subdir,sqlfile)))
    for sql_orig in sqllines:
        #print (sql_orig)
        sqllist = re.findall(r'([\w_]+)', sql_orig)
        sqllist_joinkey = sqllist[0] + "=" + sqllist[1]
        sqllabels[sqllist_joinkey] = sqllist[0]
        sqlroles[sqllist_joinkey] = sqllist[1]

    #orderlabels = {}
    #orderroles = {}
    #orderfile = 'order_brief.txt'
    #orderlines = (orderline.rstrip() for orderline in open(orderfile))
    #for order_orig in orderlines:
        #print (sql_orig)
    #    ordernum = re.search(r'([\d_]+)', order_orig)
    #    orderkey = re.search(r'([\w_]+)', order_orig)
    #    orderlist_joinkey = ordernum + "=" + orderkey[1]
    #    orderlabels[orderlist_joinkey] = ordernum
    #    orderroles[orderlist_joinkey] = orderkey

    grouporder = {1: 'home', 2: 'infrastructure', 3: 'virtualDatacenters', 4: 'virtualAppliances', 5: 'appsLibrary', 6: 'users', 7: 'systemConfiguration', 8: 'events', 9: 'pricing'}
    groupmatch = {'home': 'ENTERPRISE', 'infrastructure': 'PHYS', 'virtualDatacenters': 'VDC', 'virtualAppliances': 'VAPP','appsLibrary': 'APPLIB', 'users': 'USERS', 'systemConfiguration': 'SYSCONFIG', 'events': 'EVENTLOG', 'pricing': 'PRICING'}

    labelmatch = {'MANAGE_LOADBALANCERS': 'VDC', 'MANAGE_FIREWALLS': 'VDC', 'MANAGE_FLOATINGIPS': 'VDC', 'WORKFLOW_OVERRIDE': 'VAPP', 'MANAGE_HARD_DISKS': 'VAPP', 'ASSIGN_LOADBALANCERS': 'VAPP', 'ASSIGN_FIREWALLS': 'VAPP', 'APPLIB_VM_COST_CODE': 'PRICING'}

    json_data = open(os.path.join(input_gitdir,'lang_en_US_labels.json'))
    data = simplejson.load(json_data)
    privlabels = {}
    privnames = {}
    privdescs = {}
    privgroups = {}
    labelkeys = sorted(data.keys())
    for labelkey_orig in labelkeys:
        labelkey = labelkey_orig.split(".")
        pg = labelkey[0]
        if pg == "privilegegroup":
            pgk = labelkey[1]
            if pgk != "allprivileges":
                privgroups[pgk] = data[labelkey_orig]
                #print ("privilege group: ", labelkey)
        elif pg == "privilege":
            pd = labelkey[1]
            if pd == "description":
                pdk = labelkey[2]
                privdescs[pdk] = data[labelkey_orig]
                #print("privilege description: ", labelkey)
            elif pd != "details":
                privlabels[pd] = pd
                privnames[pd] = data[labelkey_orig]
                #print("privilege: ", labelkey)
    output_file_name = 'wiki_privileges_2014-10-21.txt'
    out_file = open(os.path.join(output_subdir,output_file_name), 'w')
    out_file.write ("
\n
\n")
    pgkeys = sorted(grouporder.keys())
    plkeys = sorted(privlabels.keys())
    for pgk in pgkeys:
        pgkordered = grouporder[pgk]
        current_group = groupmatch[pgkordered]
        privgroupindexed = privgroups[pgkordered]
        rowing[2] = privgroupindexed + "\n"
        #rowkeys = sorted(rowing.keys())
        #for rkey in rowkeys:
        for rkey in range(1, 20):
            out_file.write (rowing[rkey])

        for plk_orig in plkeys:
            if plk_orig in labelmatch.keys():
                plk = labelmatch[plk_orig]
            else:
                plk_split = plk_orig.split("_")
                plk = plk_split[0]

            if current_group == plk:
                output_row = "
 \n  \n " + privnames[plk_orig] + "	 \n  \n " + plk_orig + "	 \n  \n " + privdescs[plk_orig]
                out_file.write (output_row)
                privs_processed = privs_processed + 1
                #print (privnames[plk_orig],": ",privdescs[plk_orig])
                if plk_orig in extratext:
                    out_file.write (". " + extratext[plk_orig])
                out_file.write ("	\n")
                key_cloud_admin = plk_orig + "=CLOUD_ADMIN"
                key_ent_admin = plk_orig + "=ENTERPRISE_ADMIN"
                key_user = plk_orig + "=USER"
                key_muser = plk_orig + "=OUTBOUND_API_EVENTS"
                outroles = {}
                oca = " "
                oea = " "
                oeu = " "
                omu = " "
                if key_cloud_admin in sqlroles:
                    #print ("CLOUD_ADMIN")
                    oca = "X"
                if key_ent_admin in sqlroles:
                    #print ("ENT_ADMIN")
                    oea = "X"
                if key_user in sqlroles:
                    #print ("USER")
                    oeu = "X"
                if key_muser in sqlroles:
                    omu = "X"

                outroles[1] = "
 " + oca + "
 	\n"
                outroles[2] = "
 " + oea + "
 	\n"
                outroles[3] = "
 " + oeu + "
 	 \n"
                outroles[4] = "
 " + omu + "
 	 \n"
                outroles[5] = "
\n"
                #okeys = sorted(outroles.keys())
                #for okey in okeys:
                for okey in range (1, 6):
                    out_file.write (outroles[okey])
    out_file.write ("
\n
\n")
    print("privs_processed: ",privs_processed)
    json_data.close()
    out_file.close()

# Calls the main() function
if __name__ == '__main__':
  main()


# ch22_2.py
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys

# 搜尋條件
city = "台北市"
checkin_time = "2019-11-24"
checkout_time = "2019-11-25"
# 網址處理
url = "https://tw.hotels.com/"
# 擷取網頁
driverPath = 'D:\geckodriver\chromedriver.exe'
browser = webdriver.Chrome(executable_path=driverPath)
browser.implicitly_wait(10)                                 # 等待網頁載入
browser.get(url)                                            # 網頁下載至瀏覽器
time.sleep(3)
searchNode = browser.find_elements_by_xpath("//input[contains(@id,'q-destination')]")
checkInNode = browser.find_elements_by_xpath("//input[contains(@id,'q-localised-check-in')]")
checkOutNode = browser.find_elements_by_xpath("//input[contains(@id,'q-localised-check-out')]")

# 輸入搜尋關鍵資料
searchNode[0].clear()                                       # 清除城市欄位
searchNode[0].send_keys(city)                               # 輸入搜尋城市
time.sleep(3)
searchNode[0].send_keys(Keys.TAB)                           # 按Tab跳到check in欄位
checkInNode[0].clear()                                      # 清除check in欄位
checkInNode[0].send_keys(checkin_time)                      # 輸入check in時間
time.sleep(3)
searchNode[0].send_keys(Keys.TAB)                           # 按Tab跳至check out欄位
checkOutNode[0].clear()                                     # 清除check out欄位
checkOutNode[0].send_keys(checkout_time)                    # 輸入check out時間
time.sleep(3)
checkOutNode[0].send_keys(Keys.ENTER)                       # 表單按Enter表示輸入完成
time.sleep(3)

# 旅館名稱
hotelPath = "//ol[@class='listings infinite-scroll-enabled']//h3/a"
names = browser.find_elements_by_xpath(hotelPath)
# 旅館地址
addressPath = "//ol[@class='listings infinite-scroll-enabled']//span[@class='address']"
addresses = browser.find_elements_by_xpath(addressPath)
# 旅館星級評價
starPath = "//ol[@class='listings infinite-scroll-enabled']//span[@class='star-rating-text']"
stars = browser.find_elements_by_xpath(starPath)
# 旅館定價
listingpricePath = "//ol[@class='listings infinite-scroll-enabled']//del[@data-reason='DRR-448']"
lprices = browser.find_elements_by_xpath(listingpricePath)
# 計算旅館數量
num = len(names)
print('旅館數量 : ', num)
print('-' * 70)
# 列出系列旅館資訊
for name,address,star,lprice in zip(names,addresses,stars,lprices):
    print('旅館名稱 : ', name.text)
    print('旅館地址 : ', address.text)
    print('旅館星級 : ', star.text)
    print('旅館定價 : ', lprice.text)
    print('-' * 70)
    time.sleep(1)


"""
Tafra: the innards of a dataframe

Author
------
Derrick W. Turk
David S. Fulford

Notes
-----
Created on April 25, 2020
"""
__all__ = ['Tafra']

import sys
import re
import warnings
import pprint as pprint
from datetime import date, datetime
from itertools import chain
import dataclasses as dc

import numpy as np
from .protocol import Series, DataFrame  # just for mypy...

from typing import (Any, Callable, Dict, Mapping, List, Tuple, Optional, Union as _Union, Sequence,
                    NamedTuple, Sized, Iterable, Iterator, Type, KeysView, ValuesView, ItemsView)
from typing import cast
from typing_extensions import Protocol

from .formatter import ObjectFormatter

object_formatter = ObjectFormatter()
object_formatter['Decimal'] = lambda x: x.astype(float)

TAFRA_TYPE: Dict[str, Callable[[np.ndarray], np.ndarray]] = {
    'int': lambda x: x.astype(int),
    'float': lambda x: x.astype(float),
    'decimal': lambda x: x.astype(float),
    'bool': lambda x: x.astype(bool),
    'str': lambda x: x.astype(str),
    'date': lambda x: x.astype('datetime64'),
    'object': lambda x: x.astype(object),
}

NAMEDTUPLE_TYPE: Dict[str, Type[Any]] = {
    'int': int,
    'float': float,
    'decimal': float,
    'bool': bool,
    'str': str,
    'date': datetime,
    'object': str,
}

RECORD_TYPE: Dict[str, Callable[[Any], Any]] = {
    'int': lambda x: int(x),
    'float': lambda x: float(x),
    'bool': lambda x: bool(x),
    'str': lambda x: str(x),
    'date': lambda x: x.strftime(r'%Y-%m-%d'),
    'object': lambda x: str(x),
}

@dc.dataclass
class Tafra:
    """
    The innards of a dataframe.

    Parameters
    ----------
        _data: Dict[str, np.ndarray]
            The data used to build the :class:`Tafra`.

        _dtypes: Dict[str, str] = {}
            The dtypes of the ``_data``. If not given, will be inferred from
            the ``_data``.

    """

    Scalar = _Union[str, int, float, bool]
    _Mapping = _Union[
        Mapping[str, Any],
        Mapping[int, Any],
        Mapping[float, Any],
        Mapping[bool, Any],
    ]
    InitVar = _Union[
        Tuple[Scalar, Any],
        _Mapping,
        Iterator[Tuple[Scalar, Any]],
        Iterator[_Mapping]
    ]

    data: dc.InitVar[InitVar]
    dtypes: dc.InitVar[InitVar] = None

    _data: Dict[str, np.ndarray] = dc.field(init=False)
    _dtypes: Dict[str, str] = dc.field(init=False)

    def __post_init__(self, data: InitVar, dtypes: Optional[InitVar]) -> None:
        # TODO: enable this?
        # if isinstance(self._data, DataFrame):
        #     tf = self.from_dataframe(df=self._data)
        #     self._data = tf._data
        #     self._dtypes = tf._dtypes
        #     self._rows = tf._rows
        #     return

        rows: Optional[int] = None
        self._data = cast(Dict[str, np.ndarray], self._check_initvar(data))
        if dtypes is None or isinstance(dtypes, property):
            self._dtypes = {}
        else:
            self._dtypes = cast(Dict[str, str], self._check_initvar(dtypes))

        for column, value in self._data.items():
            value, modified = self._validate_value(value, check_rows=False)
            if modified:
                self._data[column] = value

            if rows is None:
                rows = len(value)
            elif rows != len(value):
                raise ValueError('`Tafra` must have consistent row counts.')

        if rows is None:
            raise ValueError('No data provided in constructor statement.')

        self._update_rows()
        self._coalesce_dtypes()
        self.update_dtypes_inplace(self._dtypes)

    def _check_initvar(self, values: InitVar) -> Dict[str, _Union[str, np.ndarray]]:
        if isinstance(values, Iterator):
            head = next(values)
            if isinstance(head, Dict):
                for _dict in values:
                    head.update(cast(Mapping[Any, Any], _dict))
                _values = head
            elif isinstance(head, Sequence) and len(head) == 2:
                # maybe an iterator of mapping? Cast and try it.
                _values = dict(chain(
                    cast(Iterable[Tuple[str, Any]], [head]),
                    cast(Iterator[Tuple[str, Any]], values)))
        else:
            _values = cast(Dict[str, _Union[str, np.ndarray]], values)

        if not isinstance(_values, Dict):
            raise TypeError(f'Must supply a `Dict` or `Mapping`, got `{type(_values)}`')

        # must copy first as mutating the dict changes next(iterator)
        columns = [c for c in _values.keys() if not isinstance(c, str)]
        for column in columns:
            _values[str(column)] = _values.pop(column)

        return _values

    def __getitem__(
            self,
            item: _Union[str, int, slice, Sequence[_Union[str, int, bool]], np.ndarray]) -> Any:
        # return type is actually _Union[np.ndarray, 'Tafra'] but mypy goes insane
        if isinstance(item, str):
            return self._data[item]

        elif isinstance(item, int):
            return self._slice(item)

        elif isinstance(item, slice):
            return self._slice(item)

        elif isinstance(item, np.ndarray):
            return self._ndindex(item)

        elif isinstance(item, Sequence):
            if isinstance(item[0], str):
                return self.select(cast(Sequence[str], item))
            else:
                return self._index(item)

        else:
            raise TypeError(f'Type {type(item)} not supported.')

    def __setitem__(self, item: str, value: _Union[np.ndarray, Sequence[Any], Any]) -> None:
        _value, _ = self._validate_value(value)
        self._data[item] = _value
        self._dtypes[item] = self._format_type(_value.dtype)

    def __len__(self) -> int:
        assert self._data is not None, 'Cannot construct a Tafra with no data.'
        return self._rows

    def __iter__(self) -> Iterator['Tafra']:
        for i in range(self._rows):
            yield self[i]

    def iterrows(self) -> Iterator['Tafra']:
        """
        Yield rows as :class:`Tafra`.

        Returns
        -------
            tafras: Iterator[Tafra]
                An iterator of :class:`Tafra`.
        """
        yield from self.__iter__()

    def itertuples(self, name: str = 'Tafra') -> Iterator[Tuple[Any, ...]]:
        """
        Yield rows as :class:`NamedTuple`.

        Parameters
        ----------
            name: str = 'Tafra'
                The name for the :class:`NamedTuple`.

        Returns
        -------
            tuples: Iterator[NamedTuple[Any, ...]]
                An iterator of :class:`NamedTuple`.
        """
        TafraNT = NamedTuple(name, **{  # type: ignore
            column: NAMEDTUPLE_TYPE[dtype] for column, dtype in self._dtypes.items()})

        for tf in self.__iter__():
            yield TafraNT(*(value.item() for value in tf._data.values()))

    def itercols(self) -> Iterator[Tuple[str, np.ndarray]]:
        """
        Yield columns as :class:`Tuple[str, np.ndarray]` .

        Returns
        -------
            tuples: Iterator[Tuple[str, np.ndarray]]
                An iterator of :class:`Tafra`.
        """
        for column, value in self._data.items():
            yield column, value

    def __str__(self) -> str:
        return self.__repr__()

    def __repr__(self) -> str:
        return f'Tafra(data={self._data}, dtypes={self._dtypes}, rows={self._rows})'

    def _update_rows(self) -> None:
        iter_values = iter(self._data.values())
        self._rows = len(next(iter_values))
        if not all(len(v) == self._rows for v in iter_values):
            raise TypeError('Uneven length of data.')

    def _slice(self, _slice: _Union[int, slice]) -> 'Tafra':
        """
        Use slice object to slice np.ndarray.

        Parameters
        ----------
            _slice: Union[int, slice]
                The ``slice`` object.

        Returns
        -------
            tafra: Tafra
                The sliced :class:`Tafra`.
        """
        return Tafra(
            {column: value[_slice]
                for column, value in self._data.items()},
            self._dtypes
        )

    def _index(self, index: Sequence[_Union[str, int, bool]]) -> 'Tafra':
        """
        Use numpy indexing to slice the data :class:`np.ndarray`.

        Parameters
        ----------
            index: Sequence[Union[str, int, bool]]

        """
        # TODO: just let numpy handle errors?
        # head = index[0]

        # if isinstance(index[0], str):
        #     if all(isinstance(item, str) for item in index):
                # return self.select(cast(Sequence[str], index))
        #     else:
        #         raise IndexError('Index `Sequence` does not contain all `str`.')

        # elif (isinstance(head, bool)
        #         and not(all(isinstance(item, bool) for item in index))):
        #     raise IndexError('Index `Sequence` does not contain all `bool`.')

        # elif (isinstance(head, int)   # type: ignore
        #         and not(all(isinstance(item, int) for item in index))):
        #     raise IndexError('Index `Sequence` does not contain all `int`.')

        return Tafra(
            {column: value[index]
                for column, value in self._data.items()},
            self._dtypes
        )

    def _ndindex(self, index: np.ndarray) -> 'Tafra':
        """
        Use :class:`numpy.ndarray` indexing to slice the data :class:`np.ndarray`.

        Parameters
        ----------
            index: np.ndarray

        """
        # TODO: just let numpy handle errors?
        # if not (index.dtype == np.int or index.dtype == np.bool):
        #     raise IndexError(
        #         f'Index array is of dtype={index.dtype}, '
        #         'must subtype of `np.int` or `np.bool`.')
        if index.ndim != 1:
            raise IndexError(f'Indexing np.ndarray must ndim == 1, got ndim == {index.ndim}')

        return Tafra(
            {column: value[index]
                for column, value in self._data.items()},
            self._dtypes
        )

    def _repr_pretty_(self, p: 'IPython.lib.pretty.RepresentationPrinter',  # type: ignore # noqa
                      cycle: bool) -> None:
        """
        A dunder method for IPython to pretty print.

        Parameters
        ----------
            p: IPython.lib.pretty.RepresentationPrinter
                IPython provides this class to handle the object representation.

            cycle: bool
                IPython has detected an infinite loop. Print an alternative
                represenation and return.

        Returns
        -------
            None
                Calls p.text and returns.
        """
        if cycle:
            p.text('Tafra(...)')
        else:
            p.text(self._pretty_format(lambda s: ' ' + pprint.pformat(s, indent=1)[1:].strip()))

    def _repr_html_(self) -> str:
        """
        a dunder moethod for Jupyter Notebook to print HTML.
        """
        return self.to_html()

    def _pretty_format(self, formatter: Callable[[object], str]) -> str:
        """
        Format _data and _dtypes for pretty printing.

        Parameters
        ----------
            formatter: Callabke[[object], str]
                A formatter that operates on the _data and _dtypes :class:`dict`.

        Returns
        -------
            string: str
                The formatted string for printing.
        """
        PATTERN = r'(, dtype=[a-z]+)(?=\))'

        return '\n'.join([
            'Tafra(data = {',
            f'{re.sub(PATTERN, "", formatter(self._data))},',
            'dtypes = {',
            f'{re.sub(PATTERN, "", formatter(self._dtypes))},',
            f'rows = {self._rows})'
        ])

    def pformat(self, indent: int = 1, width: int = 80, depth: Optional[int] = None,
                compact: bool = False) -> str:
        """
        Format for pretty printing. Parameters are passed to :class:`pprint.PrettyPrinter`.

        Parameters
        ----------
            indent: int
                Number of spaces to indent for each level of nesting.

            width: int
                Attempted maximum number of columns in the output.

            depth: Optional[int]
                The maximum depth to print out nested structures.

            compact: bool
                If true, several items will be combined in one line.

        Returns
        -------
            formatted string: str
                A formatted string for pretty printing.
        """
        return self._pretty_format(
            lambda s: indent * ' ' + pprint.pformat(
                s, indent, width, depth, compact=compact)[1:].strip())

    def pprint(self, indent: int = 1, width: int = 80, depth: Optional[int] = None,
               compact: bool = False) -> None:
        """
        Pretty print. Parameters are passed to :class:`pprint.PrettyPrinter`.

        Parameters
        ----------
            indent: int
                Number of spaces to indent for each level of nesting.

            width: int
                Attempted maximum number of columns in the output.

            depth: Optional[int]
                The maximum depth to print out nested structures.

            compact: bool
                If true, several items will be combined in one line.

        Returns
        -------
            None: None
        """
        print(self.pformat(indent, width, depth, compact=compact))

    @staticmethod
    def _html_thead(columns: Iterable[Any]) -> str:
        """
        Construct the table head of the HTML representation.

        Parameters
        ----------
            columns: Iterable[Any]
                An iterable of items with defined func:`__repr__` methods.

        Returns
        -------
            HTML: str
                The HTML table head.
        """
        return '
\n
\n{th}\n
\n
' \
            .format(th='\n'.join(f'{c}	' for c in columns))

    @staticmethod
    def _html_tr(row: Iterable[Any]) -> str:
        """
        Construct each table row of the HTML representation.

        Parameters
        ----------
            row: Iterable[Any]
                An iterable of items with defined func:`__repr__` methods.

        Returns
        -------
            HTML: str
                The HTML table row.
        """
        return '
\n{td}\n
' \
            .format(td='\n'.join(f'{td}	' for td in row))

    @staticmethod
    def _html_tbody(tr: Iterable[str]) -> str:
        """
        Construct the table body of the HTML representation.

        Parameters
        ----------
            tr: Iterable[str]
                An iterable of HTML table rows.

        Returns
        -------
            HTML: str
                The HTML table body.
        """
        return '
\n{tr}\n
' \
            .format(tr='\n'.join(tr))

    @staticmethod
    def _html_table(thead: str, tbody: str) -> str:
        """
        Construct the final table of the HTML representation.

        Parameters
        ----------
            thead: str
                An HTML representation of the table head.

            tbody: str
                An HTML representation of the table body.

        Returns
        -------
            HTML: str
                The HTML table.
        """
        return f'
\n{thead}\n{tbody}\n
'

    def to_html(self, n: int = 20) -> str:
        """
        Construct an HTML table representation of the :class:`Tafra` data.

        Parameters
        ----------
            n: int = 20
                Number of items to print.

        Returns
        -------
            HTML: str
                The HTML table representation.
        """
        thead = self._html_thead(chain([''], self._data.keys()))
        tr = chain(
            [self._html_tr(chain(
                ['dtype'],
                self._dtypes.values()
            ))],
            (self._html_tr(chain(
                [i],
                (v[i] for v in self._data.values())
            ))
                for i in range(min(n, self._rows)))
        )
        tbody = self._html_tbody(tr)
        return self._html_table(thead, tbody)

    def _validate_value(self, value: _Union[np.ndarray, Sequence[Any], Any],
                        check_rows: bool = True) -> Tuple[np.ndarray, bool]:
        """
        Validate values as an :class:`np.ndarray` of equal length to
        :attr:`rows` before assignment. Will attempt to create a
        :class:`np.ndarray` if ``value`` is not one already, and will check
        that :attr`np.ndarray.ndim` is ``1``.
        If :attr:`np.ndarray.ndim` ``> 1`` it will attempt :meth:`np.squeeze`
        on ``value``.

        Parameters
        ----------
            value: Union[np.ndarray, Sequence[Any], Any]
                The value to be assigned.

        Returns
        -------
            value: np.ndarray
                The validated value.
        """
        modified = False
        rows = self._rows if check_rows else 1

        if isinstance(value, np.ndarray) and len(value.shape) == 0:
            value = np.full(rows, value.item())
            modified = True

        elif isinstance(value, str) or not isinstance(value, Sized):
            value = np.full(rows, value)
            modified = True

        elif isinstance(value, Iterable):
            value = np.array(value)
            modified = True

        assert isinstance(value, np.ndarray), '`Tafra` only supports assigning `ndarray`.'

        if value.ndim > 1:
            sq_value = value.squeeze()
            if sq_value.ndim > 1:
                raise ValueError('`ndarray` or `np.squeeze(ndarray)` must have ndim == 1.')
            elif sq_value.ndim == 1:
                # if value was a single item, squeeze returns zero length item
                warnings.warn('`np.squeeze(ndarray)` applied to set ndim == 1.')
                warnings.resetwarnings()
                value = sq_value

            modified = True

        # special parsing of various object types
        if value.dtype == np.dtype(object):
            type_name = type(value[0]).__name__
            if type_name in object_formatter:
                value = object_formatter[type_name](value)
                modified = True

        assert value.ndim >= 1, '`Tafra` only supports assigning ndim >= 1.'

        if check_rows and len(value) != rows:
            raise ValueError(
                '`Tafra` must have consistent row counts.\n'
                f'This `Tafra` has {rows} rows. Assigned np.ndarray has {len(value)} rows.')

        return value, modified

    def _validate_columns(self, columns: Iterable[str]) -> None:
        """
        Validate that the column name(s) exists in :attr:`_data`.

        Parameters
        ----------
            columns: Iterable[str]
                The column names to validate.

        Returns
        -------
            None: None
        """
        for column in columns:
            if column not in self._data.keys():
                raise ValueError(f'Column {column} does not exist in `tafra`.')

    def _validate_dtypes(self, dtypes: Dict[str, Any]) -> Dict[str, str]:
        """
        Validate that the dtypes are defined as valid dtypes in
        :const:`TAFRA_TYPE` and the columns exists in :attr:`_data`.

        Parameters
        ----------
            dtypes: Dict[str, Any]
                The dtypes to validate.

        Returns
        -------
            dtypes: Dict[str, str]
                The validated types.
        """
        msg = ''
        _dtypes = {}

        self._validate_columns(dtypes.keys())

        for column, _dtype in dtypes.items():
            _dtypes[column] = self._format_type(_dtype)
            if _dtypes[column] not in TAFRA_TYPE:
                msg += f'`{_dtypes[column]}` is not a valid dtype for `{column}.`\n'

        if len(msg) > 0:
            # should be KeyError value Python 3.7.x has a bug with '\n'
            raise ValueError(msg)

        return _dtypes

    @staticmethod
    def _format_type(dtype: Any) -> str:
        """
        Parse a dtype into the internally used string representation.

        Parameters
        ----------
            dtype: Any
                The dtype to parse.

        Returns
        -------
            dtype: str
                The parsed dtype.
        """
        _dtype = str(dtype)

        if 'int' in _dtype:
            return 'int'
        if 'float' in _dtype:
            return 'float'
        if 'bool' in _dtype:
            return 'bool'
        if 'str' in _dtype:
            return 'str'
        if '<U' in _dtype:
            return 'str'
        if 'date' in _dtype:
            return 'date'
        if '<M' in _dtype:
            return 'date'
        if 'object' in _dtype:
            return 'object'
        if 'O' in _dtype:
            return 'object'

        return _dtype

    @staticmethod
    def _apply_type(dtype: str, array: np.ndarray) -> np.ndarray:
        """
        Apply the dtype to the :class:`np.ndarray`.

        Parameters
        ----------
            dtype: str
                The parsed dtype.

            array: np.ndaray
                The array to which the dtype is applied.

        Returns
        -------
            array:
                The array with updated dtype.
        """
        return TAFRA_TYPE[dtype](array)

    @classmethod
    def from_series(cls, s: Series, dtype: Optional[str] = None) -> 'Tafra':
        """
        Construct a :class:`Tafra` from a :class:`pd.DataFrame`.

        Parameters
        ----------
            df: pd.DataFrame
                The dataframe used to build the :class:`Tafra`.

            dtypes: Optional[Dict[str, str]] = None
                The dtypes of the columns.

        Returns
        -------
            tafra: Tafra
                The constructed :class:`Tafra`.
        """
        if dtype is None:
            dtype = s.dtype
        dtypes = {s.name: cls._format_type(dtype)}

        return cls(
            {s.name: cls._apply_type(dtypes[s.name], s.values)},
            dtypes
        )

    @classmethod
    def from_dataframe(cls, df: DataFrame, dtypes: Optional[Dict[str, str]] = None) -> 'Tafra':
        """
        Construct a :class:`Tafra` from a :class:`pd.DataFrame`.

        Parameters
        ----------
            df: pd.DataFrame
                The dataframe used to build the :class:`Tafra`.

            dtypes: Optional[Dict[str, str]] = None
                The dtypes of the columns.

        Returns
        -------
            tafra: Tafra
                The constructed :class:`Tafra`.
        """
        if dtypes is None:
            dtypes = {c: t for c, t in zip(df.columns, df.dtypes)}
        dtypes = {c: cls._format_type(t) for c, t in dtypes.items()}

        return cls(
            {c: cls._apply_type(dtypes[c], df[c].values) for c in df.columns},
            {c: dtypes[c] for c in df.columns}
        )

    @classmethod
    def as_tafra(cls, maybe_tafra: _Union['Tafra', DataFrame, Series, Dict[str, Any], Any]
                 ) -> Optional['Tafra']:
        """
        Returns the unmodified `tafra`` if already a `Tafra`, else construct
        a `Tafra` from known types or subtypes of :class:`DataFrame` or `dict`.
        Structural subtypes of :class:`DataFrame` or :class:`Series` are also
        valid, as are classes that have ``cls.__name__ == 'DataFrame'`` or
        ``cls.__name__ == 'Series'``.

        Parameters
        ----------
            maybe_tafra: Union['tafra', DataFrame]
                The object to ensure is a :class:`Tafra`.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra`, or None is ``maybe_tafra`` is an unknown
                type.
        """
        if isinstance(maybe_tafra, Tafra):
            return maybe_tafra

        elif isinstance(maybe_tafra, Series):
            return cls.from_series(maybe_tafra)  # pragma: no cover

        elif type(maybe_tafra).__name__ == 'Series':
            return cls.from_series(cast(Series, maybe_tafra))  # pragma: no cover

        elif isinstance(maybe_tafra, DataFrame):
            return cls.from_dataframe(maybe_tafra)  # pragma: no cover

        elif type(maybe_tafra).__name__ == 'DataFrame':
            return cls.from_dataframe(cast(DataFrame, maybe_tafra))  # pragma: no cover

        elif isinstance(maybe_tafra, dict):
            return cls(maybe_tafra)

        raise TypeError(f'Unknown type `{type(maybe_tafra)}` for conversion to `Tafra`')

    @property
    def columns(self) -> Tuple[str, ...]:
        """
        The names of the columns. Equivalent to `Tafra`.keys().

        Returns
        -------
            columns: Tuple[str, ...]
                The column names.
        """
        return tuple(self._data.keys())

    @columns.setter
    def columns(self, value: Any) -> None:
        raise ValueError('Assignment to `columns` is forbidden.')

    @property
    def rows(self) -> int:
        """
        The number of rows of the first item in :attr:`data`. The :func:`len()`
        of all items have been previously validated.

        Returns
        -------
            rows: int
                The number of rows of the :class:`Tafra`.
        """
        return self.__len__()

    @rows.setter
    def rows(self, value: Any) -> None:
        raise ValueError('Assignment to `rows` is forbidden.')

    @property  # type: ignore
    def data(self) -> Dict[str, np.ndarray]:
        """
        The :class:`Tafra` data.

        Returns
        -------
            data: Dict[str, np.ndarray]
                The data.
        """
        return self._data

    @data.setter
    def data(self, value: Any) -> None:
        raise ValueError('Assignment to `data` is forbidden.')

    @property  # type: ignore
    def dtypes(self) -> Dict[str, str]:
        """
        The :class:`Tafra` dtypes.

        Returns
        -------
            dtypes: Dict[str, str]
                The dtypes.
        """
        return self._dtypes

    @dtypes.setter
    def dtypes(self, value: Any) -> None:
        raise ValueError('Assignment to `dtypes` is forbidden.')

    @property
    def size(self) -> int:
        """
        The :class:`Tafra` size.

        Returns
        -------
            size: int
                The size.
        """
        return self.rows * len(self.columns)

    @size.setter
    def size(self, value: Any) -> None:
        raise ValueError('Assignment to `size` is forbidden.')

    @property
    def ndim(self) -> int:
        """
        The :class:`Tafra` number of dimensions.

        Returns
        -------
            ndim: int
                The number of dimensions.
        """
        return max(2, len(self.columns))

    @ndim.setter
    def ndim(self, value: Any) -> None:
        raise ValueError('Assignment to `ndim` is forbidden.')

    @property
    def shape(self) -> Tuple[int, int]:
        """
        The :class:`Tafra` shape.

        Returns
        -------
            shape: int
                The shape.
        """
        return self.rows, len(self.columns)

    @shape.setter
    def shape(self, value: Any) -> None:
        raise ValueError('Assignment to `shape` is forbidden.')

    def row_map(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Iterator[Any]:
        """
        Map a function over rows. To apply to specific columns, use
        :meth:`select` first. The function must operate on :class:`Tafra`.

        Parameters
        ----------
            fn: Callable[..., Any]
                The function to map.

            *args: Any
                Additional positional arguments to ``fn``.

            **kwargs: Any
                Additional keyword arguments to ``fn``.

        Returns
        -------
            iter_tf: Iterator[Any]
                An iterator to map the function.
        """
        for tf in self.__iter__():
            yield fn(tf, *args, **kwargs)

    def col_map(self, fn: Callable[..., Any], name: bool = True,
                *args: Any, **kwargs: Any) -> Iterator[_Union[Any, Tuple[str, Any]]]:
        """
        Map a function over columns. To apply to specific columns, use
        :meth:`select` first. The function must operate on :class:`Tuple[str, np.ndarray]`.

        Parameters
        ----------
            fn: Callable[..., Any]
                The function to map.

            name: bool
                Return the column name.

            *args: Any
                Additional positional arguments to ``fn``.

            **kwargs: Any
                Additional keyword arguments to ``fn``.

        Returns
        -------
            iter_tf: Iterator[Any]
                An iterator to map the function.
        """
        if name:
            for column, value in self.itercols():
                yield column, fn(value, *args, **kwargs)
            return

        for column, value in self.itercols():
            yield fn(value, *args, **kwargs)

    def head(self, n: int = 5) -> None:  # pragma: no cover
        """
        Display the head of the :class:`Tafra`.

        Parameters
        ----------
            n: int = 5
                The number of rows to display.

        Returns
        -------
            None: None
        """
        if _in_notebook():
            try:
                from IPython.display import display  # type: ignore # noqa
                display(self[:min(self._rows, n)])
                return
            except Exception as e:
                pass

        print(self[:min(self._rows, n)].pformat())

    def select(self, columns: Sequence[str]) -> 'Tafra':
        """
        Use column names to slice the :class:`Tafra` columns analogous to
        SQL SELECT.
        This does not copy the data. Call :meth:`copy` to obtain a copy of
        the sliced data.

        Parameters
        ----------
            columns: Sequence[str]
                The column names to slice from the :class:`Tafra`.

        Returns
        -------
            tafra: Tafra
                the :class:`Tafra` with the sliced columns.
        """
        self._validate_columns(columns)

        return Tafra(
            {column: value for column, value in self._data.items()
             if column in columns},
            {column: value for column, value in self._dtypes.items()
             if column in columns},
        )

    def keys(self) -> KeysView[str]:
        """
        Return the keys of :attr:`data`, i.e. like :meth:`dict.keys()`.

        Returns
        -------
            data keys: KeysView[str]
                The keys of the data property.
        """
        return self._data.keys()

    def values(self) -> ValuesView[np.ndarray]:
        """
        Return the values of :attr:`data`, i.e. like :meth:`dict.values()`.

        Returns
        -------
            data values: ValuesView[np.ndarray]
                The values of the data property.
        """
        return self._data.values()

    def items(self) -> ItemsView[str, np.ndarray]:
        """
        Return the items of :attr:`data`, i.e. like :meth:`dict.items()`.

        Returns
        -------
            items: ItemsView[str, np.ndarray]
                The data items.
        """
        return self._data.items()

    def get(self, key: str, default: Any = None) -> Any:
        """
        Return from the :meth:`get` function of :attr:`data`, i.e. like
        :meth:`dict.get()`.

        Parameters
        ----------
            key: str
                The key value in the data property.

            default: Any
                The default to return if the key does not exist.

        Returns
        -------
            value: Any
                The value for the key, or the default if the key does not
                exist.
        """
        return self._data.get(key, default)

    def update(self, other: 'Tafra') -> 'Tafra':
        """
        Update the data and dtypes of this :class:`Tafra` with another
        :class:`Tafra`. Length of rows must match, while data of different
        ``dtype`` will overwrite.

        Parameters
        ----------
            other: Tafra
                The other :class:`Tafra` from which to update.

        Returns
        -------
            None: None
        """
        tafra = self.copy()
        rows = tafra._rows

        for column, value in other._data.items():
            if len(value) != rows:
                raise ValueError(
                    'Other `Tafra` must have consistent row count. '
                    f'This `Tafra` has {rows} rows, other `Tafra` has {len(value)} rows.')
            tafra._data[column] = value

        tafra.update_dtypes_inplace(other._dtypes)
        return tafra

    def update_inplace(self, other: 'Tafra') -> None:
        """
        Update the data and dtypes of this :class:`Tafra` with another
        :class:`Tafra`. Length of rows must match, while data of different
        ``dtype`` will overwrite.

        Parameters
        ----------
            other: Tafra
                The other :class:`Tafra` from which to update.

        Returns
        -------
            None: None
        """
        rows = self._rows

        for column, value in other._data.items():
            if len(value) != rows:
                raise ValueError(
                    'Other `Tafra` must have consistent row count. '
                    f'This `Tafra` has {rows} rows, other `Tafra` has {len(value)} rows.')
            self._data[column] = value

        self.update_dtypes_inplace(other._dtypes)

    def _coalesce_dtypes(self) -> None:
        """
        Update :attr:`_dtypes` with missing keys that exist in :attr:`_data`.
        **Must be called if :attr:`data` or :attr`:_data` is directly
        modified!**

        Returns
        -------
            None: None
        """
        for column in self._data.keys():
            if column not in self._dtypes:
                self._dtypes[column] = self._format_type(self._data[column].dtype)

    def update_dtypes(self, dtypes: Dict[str, Any]) -> 'Tafra':
        """
        Apply new dtypes.

        Parameters
        ----------
            dtypes: Dict[str, Any]
                The dtypes to update. If ``None``, create from entries in
                :attr:`data`.

        Returns
        -------
            tafra: Optional[Tafra]
                The updated :class:`Tafra`.
        """
        tafra = self.copy()

        tafra._validate_columns(dtypes.keys())
        dtypes = tafra._validate_dtypes(dtypes)
        tafra._dtypes.update(dtypes)

        for column in tafra._dtypes.keys():
            if tafra._format_type(tafra._data[column].dtype) != tafra._dtypes[column]:
                tafra._data[column] = tafra._apply_type(tafra._dtypes[column], tafra._data[column])

        return tafra

    def update_dtypes_inplace(self, dtypes: Dict[str, Any]) -> None:
        """
        Apply new dtypes.

        Parameters
        ----------
            dtypes: Dict[str, Any]
                The dtypes to update. If ``None``, create from entries in
                :attr:`data`.

        Returns
        -------
            tafra: Optional[Tafra]
                The updated :class:`Tafra`.
        """
        self._validate_columns(dtypes.keys())
        dtypes = self._validate_dtypes(dtypes)
        self._dtypes.update(dtypes)

        for column in self._dtypes.keys():
            if self._format_type(self._data[column].dtype) != self._dtypes[column]:
                self._data[column] = self._apply_type(self._dtypes[column], self._data[column])

    def rename(self, renames: Dict[str, str]) -> 'Tafra':
        """
        Rename columns in the :class:`Tafra` from a :class:`dict`.

        Parameters
        ----------
            renames: Dict[str, str]
                The map from current names to new names.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with update names.
        """

        tafra = self.copy()
        for cur, new in renames.items():
            tafra._data[new] = tafra._data.pop(cur)
            tafra._dtypes[new] = tafra._dtypes.pop(cur)
        return tafra

    def rename_inplace(self, renames: Dict[str, str]) -> None:
        """
        In-place version.

        Rename columns in the :class:`Tafra` from a :class:`dict`.

        Parameters
        ----------
            renames: Dict[str, str]
                The map from current names to new names.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with update names.
        """
        self._validate_columns(renames.keys())

        for cur, new in renames.items():
            self._data[new] = self._data.pop(cur)
            self._dtypes[new] = self._dtypes.pop(cur)
        return None

    def delete(self, columns: Iterable[str]) -> 'Tafra':
        """
        Remove a column from :attr:`data` and :attr:`dtypes`.

        Parameters
        ----------
            column: str
                The column to remove.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with the deleted column.
        """
        if isinstance(columns, str):
            columns = [columns]

        self._validate_columns(columns)

        return Tafra(
            {column: value.copy() for column, value in self._data.items()
             if column not in columns},
            {column: value for column, value in self._dtypes.items()
             if column not in columns}
        )

    def delete_inplace(self, columns: Iterable[str]) -> None:
        """
        In-place version.

        Remove a column from :attr:`data` and :attr:`dtypes`.

        Parameters
        ----------
            column: str
                The column to remove.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with the deleted column.
        """
        if isinstance(columns, str):
            columns = [columns]

        self._validate_columns(columns)

        for column in columns:
            _ = self._data.pop(column, None)
            _ = self._dtypes.pop(column, None)

    def copy(self, order: str = 'C') -> 'Tafra':
        """
        Create a copy of a :class:`Tafra`.

        Parameters
        ----------
            order: str = 'C' {‘C’, ‘F’, ‘A’, ‘K’}
                Controls the memory layout of the copy. ‘C’ means C-order,
                ‘F’ means F-order, ‘A’ means ‘F’ if a is Fortran contiguous,
                ‘C’ otherwise. ‘K’ means match the layout of a as closely as
                possible.

        Returns
        -------
            tafra: Tafra
                A copied :class:`Tafra`.
        """
        return Tafra(
            {column: value.copy(order=order)
                for column, value in self._data.items()},
            self._dtypes.copy()
        )

    def coalesce(self, column: str,
                 fills: Iterable[_Union[None, str, int, float, bool, np.ndarray]]) -> np.ndarray:
        """
        Fill ``None`` values from ``fills``. Analogous to ``SQL COALESCE`` or
        :meth:`pd.fillna`.

        Parameters
        ----------
            column: str
                The column to coalesce.

            fills: Iterable[Union[str, int, float, bool, np.ndarray]:

        Returns
        -------
            data: np.ndarray
                The coalesced data.
        """
        #TODO: handle dtype?
        iter_fills = iter(fills)
        head = next(iter_fills)

        if column in self._data.keys():
            value = self._data[column].copy()
        else:
            value = np.empty(self._rows, np.asarray(head).dtype)

        for fill in chain([head], iter_fills):
            f = np.atleast_1d(fill)
            where_na = np.full(self._rows, False)
            where_na |= value == np.array([None])
            try:
                where_na |= np.isnan(value)
            except:
                pass
                # pass

            for w in where_na:
                if len(f) == 1:
                    value[where_na] = f
                else:
                    value[where_na] = f[where_na]

        return value

    def coalesce_inplace(self, column: str,
                         fills: Iterable[_Union[None, str, int, float, bool, np.ndarray]]) -> None:
        """
        In-place version.

        Fill ``None`` values from ``fills``. Analogous to ``SQL COALESCE`` or
        :meth:`pd.fillna`.

        Parameters
        ----------
            column: str
                The column to coalesce.

            fills: Iterable[Union[str, int, float, bool, np.ndarray]:

        Returns
        -------
            data: np.ndarray
                The coalesced data.
        """
        self._data[column] = self.coalesce(column, fills)
        self.update_dtypes_inplace({column: self._data[column].dtype})

    def union(self, other: 'Tafra') -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.LeftJoin`.
        """
        return Union().apply(self, other)

    def union_inplace(self, other: 'Tafra') -> None:
        """
        Helper function to implement :class:`tafra.groups.LeftJoin`.
        """
        Union().apply_inplace(self, other)

    @staticmethod
    def _cast_record(dtype: str, data: np.ndarray, cast_null: bool) -> Optional[float]:
        """
        Casts needed to generate records for database insert.
        Will cast ``np.nan`` to ``None``. Requires changing ``dtype`` to
        ``object``.

        Parameters
        ----------
            dtype: str
                The dtype of the data value.

            data: np.ndarray
                The data to have its values cast.

            cast_null: bool
                Perform the cast for ``np.nan``

        Returns
        -------
            value: Any
                The cast value.
        """
        value: Any = RECORD_TYPE[dtype](data.item())
        if cast_null and dtype == 'float' and np.isnan(data.item()):
            return None
        return value

    def to_records(self, columns: Optional[Sequence[str]] = None,
                   cast_null: bool = True) -> Iterator[Tuple[Any, ...]]:
        """
        Return a :class:`Iterator` of :class:`Tuple`, each being a record
        (i.e. row) and allowing heterogeneous typing. Useful for e.g. sending
        records back to a database.

        Parameters
        ----------
            columns: Optional[Sequence[str]] = None
                The columns to extract. If ``None``, extract all columns.

            cast_null: bool
                Cast ``np.nan`` to None. Necessary for :mod:``pyodbc``

        Returns
        -------
            records: Iterator[Tuple[Any, ...]]
        """

        if columns is None:
            columns = self.columns
        else:
            if isinstance(columns, str):
                columns = [columns]
            self._validate_columns(columns)

        for row in range(self._rows):
            yield tuple(self._cast_record(
                self._dtypes[c], self._data[c][[row]],
                cast_null
            ) for c in columns)
        return

    def to_list(self, columns: Optional[Sequence[str]] = None,
                inner: bool = False) -> _Union[List[np.ndarray], List[List[Any]]]:
        """
        Return a list of homogeneously typed columns (as np.ndarrays). If a
        generator is needed, use `Tafra.to_records()`. If `inner == True`
        each column will be cast from :class:`np.ndarray` to a :class:`List`.

        Parameters
        ----------
            columns: Optional[Sequence[str]] = None
                The columns to extract. If ``None``, extract all columns.

            inner: bool = False
                Cast all :class:`np.ndarray` to :class`List`.

        Returns
        -------
            list: Union[List[np.ndarray], List[List[Any]]]
        """
        if columns is None:
            if inner:
                return [list(v) for v in self._data.values()]
            return list(self._data.values())

        else:
            if isinstance(columns, str):
                columns = [columns]
            if inner:
                return [list(self._data[c]) for c in columns]
            return list(self._data[c] for c in columns)

    def to_array(self, columns: Optional[Sequence[str]] = None) -> np.ndarray:
        """
        Return an object array.

        Parameters
        ----------
            columns: Optional[Sequence[str]] = None
                The columns to extract. If ``None``, extract all columns.

        Returns
        -------
            array: np.ndarray
        """
        if columns is None:
            return np.array(list(self._data.values())).T

        else:
            if isinstance(columns, str):
                columns = [columns]
            return np.array(list(self._data[c] for c in columns)).T

    def group_by(self, group_by: Sequence[str], aggregation: 'InitAggregation' = {},
                 iter_fn: Mapping[str, Callable[[np.ndarray], Any]] = dict()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.GroupBy`.
        """
        return GroupBy(group_by, aggregation, iter_fn).apply(self)

    def transform(self, group_by: Sequence[str], aggregation: 'InitAggregation' = {},
                  iter_fn: Dict[str, Callable[[np.ndarray], Any]] = dict()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.Transform`.
        """
        return Transform(group_by, aggregation, iter_fn).apply(self)

    def iterate_by(self, group_by: Sequence[str]) -> Iterator['GroupDescription']:
        """
        Helper function to implement :class:`tafra.groups.IterateBy`.
        """
        yield from IterateBy(group_by).apply(self)

    def inner_join(self, right: 'Tafra', on: Sequence[Tuple[str, str, str]],
                   select: Sequence[str] = list()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.InnerJoin`.
        """
        return InnerJoin(on, select).apply(self, right)

    def left_join(self, right: 'Tafra', on: Sequence[Tuple[str, str, str]],
                  select: Sequence[str] = list()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.LeftJoin`.
        """
        return LeftJoin(on, select).apply(self, right)

    def cross_join(self, right: 'Tafra',
                   select: Sequence[str] = list()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.CrossJoin`.
        """
        return CrossJoin([], select).apply(self, right)

def _in_notebook() -> bool:  # pragma: no cover
    """
    Checks if running in a Jupyter Notebook.

    Returns
    -------
        in_notebook: bool
    """
    try:
        from IPython import get_ipython  # type: ignore
        if 'IPKernelApp' in get_ipython().config:  # pragma: no cover
            return True
    except Exception as e:
        pass
    return False

# Import here to resolve circular dependency
from .group import (GroupBy, Transform, IterateBy, InnerJoin, LeftJoin, CrossJoin, Union,
                    InitAggregation, GroupDescription)

Tafra.group_by.__doc__ += GroupBy.__doc__  # type: ignore
Tafra.transform.__doc__ += Transform.__doc__  # type: ignore
Tafra.iterate_by.__doc__ += IterateBy.__doc__  # type: ignore
Tafra.inner_join.__doc__ += InnerJoin.__doc__  # type: ignore
Tafra.left_join.__doc__ += LeftJoin.__doc__  # type: ignore
Tafra.cross_join.__doc__ += CrossJoin.__doc__  # type: ignore


#!/usr/bin/python

import pickle
import numpy
numpy.random.seed(42)
from sklearn import tree
import graphviz

### The words (features) and authors (labels), already largely processed.
### These files should have been created from the previous (Lesson 10)
### mini-project.
words_file = "C:/PythonFiles/ud120-projects-master/text_learning/your_word_data.pkl"
authors_file = "C:/PythonFiles/ud120-projects-master/text_learning/your_email_authors.pkl"
word_data = pickle.load( open(words_file, "rb"))
authors = pickle.load( open(authors_file, "rb") )

### test_size is the percentage of events assigned to the test set (the
### remainder go into training)
### feature matrices changed to dense representations for compatibility with
### classifier functions in versions 0.15.2 and earlier

#from sklearn import cross_validation
#features_train, features_test, labels_train, labels_test = cross_validation.train_test_split(word_data, authors, test_size=0.1, random_state=42)

#For Scikit-learn version 0.19
from sklearn.model_selection import train_test_split

features_train, features_test, labels_train, labels_test = train_test_split(word_data, authors, test_size=0.1, random_state=42)

from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
                             stop_words='english')
features_train = vectorizer.fit_transform(features_train)
features_test  = vectorizer.transform(features_test).toarray()

### a classic way to overfit is to use a small number
### of data points and a large number of features;
### train on only 150 events to put ourselves in this regime
features_train = features_train[:150].toarray()
labels_train   = labels_train[:150]

### your code goes here
treeEmail  = tree.DecisionTreeClassifier()
treeEmail = treeEmail.fit(features_train,labels_train)

#tree_data = tree.export_graphviz(treeEmail,out_file = None)
#graph = graphviz.Source(tree_data)
#graph.render("emailTree")
print("The accuracy of the Decision Tree is ", treeEmail.score(features_test, labels_test))

featureArray = treeEmail.feature_importances_

wordsList = vectorizer.get_feature_names()
for i in range(0,len(featureArray)):
	if featureArray[i] >= 0.2:
		print("The feature number is %d with importance %.2f" % (i,featureArray[i]))
		print("The word with the problem is " + wordsList[i])

#wordsList = vectorizer.get_feature_names()
print("The length of the word features is ", len(wordsList))
print("The length of the featureArray is ", len(featureArray))
maxWord = max(featureArray)


# -*- coding:utf-8 -*-
# Created by Hans-Thomas on 2011-05-13.
#=============================================================================
#   growler.py --- Growl notifier
#=============================================================================
from os.path import abspath, dirname, join

from gntp.notifier import GrowlNotifier

class Notifier(object):

    def __init__(self, app_name='autorestart'):
        self.growl = GrowlNotifier(
            applicationName = app_name,
            notifications = ['New Message'],
            defaultNotifications = ['New Message']
        )
        self.growl.register()

    def notify(self, title, description, sticky=False):
        self.growl.notify(
            noteType = 'New Message',
            title = title,
            description = description,
            sticky = sticky,
        )

#.............................................................................
#   growler.py


import torch
import torch.nn as nn
import torch.nn.functional as F

"""
Network design from paper:

Layer #     Layer type      Activation func.        # units

1           Input           Linear                  B
2           Hidden          g                       9R
3           Hidden          g                       6R
4           Hidden          g                       3R
5           Hidden          g                       R
6           Batch Norm      -                       R
7           Thresholding    RelU / LRelU            R
8           Enforce ASC     -                       R
9           Guassian Dropout

"""

class Autoencoder(nn.Module):
    def __init__(self, input_shape, n_endmembers):
        super(Autoencoder, self).__init__()

        # encoder
        self.encoder = NotImplemented

        # decoder
        self.decoder = nn.Sequential(
            nn.Linear(n_endmembers, input_shape, bias=False),
        )

    def forward(self, x):

        x = self.encoder(x)

        # Enforce that abundances sum to 1 (ASC)
        encoded = x / x.sum(dim=1, keepdim=True)

        out = self.decoder(encoded)

        return encoded, out

class Net1(Autoencoder):
    def __init__(self, input_shape, n_endmembers):
        super(Net1, self).__init__(input_shape, n_endmembers)

        # encoder
        self.encoder = nn.Sequential(
            nn.Linear(input_shape, n_endmembers),
            nn.Sigmoid(),
        )

class Net2(Autoencoder):
    def __init__(self, input_shape, n_l1, n_endmembers):
        super(Net2, self).__init__(input_shape, n_endmembers)

        # encoder
        self.encoder = nn.Sequential(
            nn.Linear(input_shape, n_l1),
            nn.ReLU(),
            nn.Linear(n_l1, n_endmembers),
            nn.ReLU()
        )

class Net3(Autoencoder):
    def __init__(self, input_shape, n_l1, n_l2, n_endmembers):
        super(Net3, self).__init__(input_shape, n_endmembers)

        # encoder
        self.encoder = nn.Sequential(
            nn.Linear(input_shape, n_l1),
            nn.ReLU(),
            nn.Linear(n_l1, n_l2),
            nn.ReLU(),
            nn.Linear(n_l2, n_endmembers),
            nn.ReLU()
        )

if __name__ == '__main__':

    model = Net1(300, 5)

    # model
    print(model)

    # params
    # for param_tensor in model.state_dict():
    #     print(param_tensor, "\t", model.state_dict()[param_tensor].size())


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 29 17:26:48 2019

@author: zl
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
#from models.other_layers.l2norm import L2Norm
import numpy as np

class Sub_Block(nn.Module):
    def __init__(self,layers_in, layers_out, stride=1):
        super(Sub_Block, self).__init__()
        self.x = nn.BatchNorm2d(layers_in)
        layers = nn.ModuleList()
        layers.append(nn.Conv2d(layers_in, layers_out, kernel_size=1, stride = stride, bias=False))
        layers.append(nn.ReLU())
        layers.append(nn.BatchNorm2d(layers_out))
        layers.append(nn.Conv2d(layers_out, layers_out, kernel_size=3, stride = stride,padding=1, bias=False))
        layers.append(nn.ReLU())
        layers.append(nn.BatchNorm2d(layers_out))
        layers.append(nn.Conv2d(layers_out, layers_in, kernel_size=1, stride = stride, bias=False))
        layers.append(nn.ReLU())

        self.y = layers
        self.act = nn.ReLU()
    def forward(self, x, phase='train'):
       # print('sub in ', x.shape)
       # if phase == 'eval':
       #     vis_list = []
        out_x = self.x(x)
      #  if phase == 'eval':
      #      vis_list.append(out_x)
        out_y = out_x
        for i in range(len(self.y)):
            out_y = (self.y[i])(out_y)
      #      if phase == 'eval':
      #          vis_list.append(out_y)
        #    print('out_y ', out_y.shape)
    #    print('out_x ', out_x.shape)

       # out = torch.add(out_x, out_y)
        out_y += out_x
      #  if phase == 'eval':
      #          vis_list.append(out_y)
        out_y = self.act(out_y)
      #  if phase == 'eval':
      #          vis_list.append(out_y)
     #   print('sub out ', out_y.shape)
     #   if phase == 'eval':
     #           return out_y, vis_list
        return out_y

class Branch_Model(nn.Module):
    def __init__(self):
        super(Branch_Model, self).__init__()
        self.layers = nn.ModuleList()
        self.layers.append(nn.Conv2d(1, 64, kernel_size=9, stride=2))
        self.layers.append(nn.ReLU())
        self.layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
        for _ in range(2):
            self.layers.append(nn.BatchNorm2d(64))
            self.layers.append(nn.Conv2d(64, 64, kernel_size=3))
        self.layers.append(nn.ReLU())
        self.layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
        self.layers.append(nn.BatchNorm2d(64))
        self.layers.append(nn.Conv2d(64, 128, kernel_size=1))
        self.layers.append(nn.ReLU())
        for _ in range(4):
            self.layers.append(Sub_Block(128, 64))

        self.layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
        self.layers.append(nn.BatchNorm2d(128))
        self.layers.append(nn.Conv2d(128, 256, kernel_size=1))
        self.layers.append(nn.ReLU())
        for _ in range(4):
            self.layers.append(Sub_Block(256, 64))

        self.layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
        self.layers.append(nn.BatchNorm2d(256))
        self.layers.append(nn.Conv2d(256, 384, kernel_size=1))
        self.layers.append(nn.ReLU())
        for _ in range(4):
            self.layers.append(Sub_Block(384, 96))

        self.layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
        self.layers.append(nn.BatchNorm2d(384))
        self.layers.append(nn.Conv2d(384, 512, kernel_size=1))
        self.layers.append(nn.ReLU())
        for _ in range(4):
            self.layers.append(Sub_Block(512, 128)  )

    def forward(self, x, phase='train'):
       # print('x ', x.shape)
        out = x

      #  if phase == 'eval':
      #      vis_list = []
        for i in range(len(self.layers)):
            out = (self.layers[i])(out)
       #     if phase == 'eval':
       #         vis_list.append(out)
       # print('x2 ', out.shape)
        out = F.max_pool2d(out, kernel_size=out.size()[2:])
      #  if phase == 'eval':
     #       vis_list.append(out)
      #      return out, vis_list
        return out
class Flatten(nn.Module):
    def forward(self, input):
        return input.view(input.size(0), -1)
class Header_Model(nn.Module):
    def __init__(self):
        super(Header_Model, self).__init__()
        self.layer_1 = nn.Sequential(nn.Conv2d(4, 32, kernel_size=(4, 1), padding=0), nn.ReLU())
        #self.layer_2 = nn.Sequential(nn.Conv2d(32, 32, kernel_size=(1, 32)), nn.Linear(32, 1))
        self.layer_2 = nn.Sequential(nn.Conv2d(509, 509, kernel_size=( 32, 1), padding=0))

        self.flatten = Flatten()

        self.dense = nn.Sequential(nn.Linear(509, 1, bias = True), nn.Sigmoid())#

    def forward(self, x, phase='train'):

       # print('x in head', x.shape)
        x_0 = x[:,0]
        x_1 = x[:,1]
        #print('x_0 ', x_0.shape)
        x1 = x_0 * x_1
      #  print('x1 ', x1.shape)
        x2 = x_0 + x_1
      #  print('x2 ', x2.shape)
        x3 = torch.abs(x_0 - x_1)
      #  print('x3 ', x3.shape)
        x4 = torch.mul(x3, x3)
      #  print('x4 ', x4.shape)
        out = torch.cat((x1, x2, x3, x4), 1)
      #  print('out1 ', out.shape)
        out = out.view(x.shape[0], 4, -1, 1)
     #   if phase == 'eval':
     #       vis_list = []
     #   print('out shape ', out.shape)
        out = self.layer_1(out)
      #  if phase == 'eval':
      #      vis_list.append(out)
     #   print('out1 shape ', out.shape)
        out = out.view(x.shape[0],-1, 32, 1)
     #   if phase == 'eval':
     #       vis_list.append(out)
     #   print('out2 shape ', out.shape)
        out = self.layer_2(out)
     #   if phase == 'eval':
     #       vis_list.append(out)
     #   print('out3 shape ', out.shape)
        out = self.flatten(out)
     #   print('out4 shape ', out.shape)
        out = self.dense(out)
     #   print('out5 shape ', out.shape)
     #   if phase == 'eval':
     #       return out, vis_list
        return out

class Whole_Model(nn.Module):
    def __init__(self):
        super(Whole_Model, self).__init__()
        self.branch_model = Branch_Model()
        self.header_model = Header_Model()

    def forward(self, x, phase='train'):
     #   if phase == 'eval':
     #       xa, vis_a = self.branch_model(x[:,0], phase = phase)
    #        xb, vis_b = self.branch_model(x[:,1], phase = phase)
     #   else:
       # print('in ', x.shape)
        xa = self.branch_model(x[:,0])
        xb = self.branch_model(x[:,1])
     #   print('xa -1 ', xa.shape)

        xa = xa.unsqueeze(1)
        xb = xb.unsqueeze(1)
        print('xa -2 ', xa.shape)

        x = torch.cat((xa,xb), 1)

        print('xa -3 ', x.shape)

     #   if phase == 'eval':
     #       x, vis_x = self.header_model(x, phase = phase)
     #   else:
        x = self.header_model(x)

    #    if phase == 'eval':
     #       vis = []
    #        vis.extend(vis_a)
    #        vis.extend(vis_b)
    #        vis.extend(vis_x)
    #        return x, vis
        return x

class ModelLoss(nn.Module):
    def __init__(self, use_gpu=True):
        super(ModelLoss, self).__init__()
        self.use_gpu = use_gpu
        self.cri = nn.MSELoss()

    def forward(self, predictions, targets):
        conf_data = predictions

        conf_t = torch.from_numpy( targets).type(torch.cuda.FloatTensor)
        #conf_t = torch.from_numpy( targets)
        if self.use_gpu:
            conf_t = conf_t.cuda()
            conf_data = conf_data.cuda()

        loss_c =self.cri(conf_data, conf_t)
     #   print('loss_c ', loss_c)
        return loss_c