import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
import numpy as np
import pandas as pd
import pickle
from sklearn import metrics, tree, svm
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import KFold,cross_val_score,train_test_split,LeaveOneOut
from sklearn.naive_bayes import MultinomialNB
import csv
from datetime import datetime
import json
import pickle
import numpy as np
import sys
import itertools
import json
import numpy as np
import pandas as pd
import pickle
from statistics import mean
from sklearn import metrics, tree, svm, preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import KFold,cross_val_score,train_test_split,LeaveOneOut
from sklearn.naive_bayes import MultinomialNB

# Changing questions into column headers for readability
READ_HEADERS = {
                'What Engineering program are you in?':'program',
                'Are you happy with what your program provides you with (i.e. courses, job opportunities, projects, etc.)':'happy',
                'What kind of problems do you enjoy solving?':'problem_type',
                'Would you consider yourself as being creative?':'creative',
                'What industry can you see yourself working in, in the future? Select all that apply.':'industry',
                'How comfortable are you working in the outdoors?':'outdoors',
                'What can you see yourself doing in the future?':'career',
                'How do you feel about group work? ':'group_work',
                'What high school class did you enjoy the most?':'liked_courses',
                'What high school class did you enjoy the least?':'disliked_courses',
                'How do you feel about computer programming?':'programming',
                'What type of club would you consider joining?':'join_clubs',
                'What type of club would you NOT want to join?':'not_clubs',
                'What project would you want to be a part of? ':'liked_projects',
                'What project do you find the least interesting?':'disliked_projects',
                'If you could only watch one TV show for the rest of your life, what would it be?':'tv_shows',
                'If engineering didn’t exist, what would you consider studying?':'alternate_degree',
                'How do you feel about working with expensive equipment?':'expensive_equipment',
                'How would you describe your drawing abilities?':'drawing',
                'You have an assignment to write an essay about anything you want. How does that make you feel?':'essay',
                "What is your gender?  **We're asking this to discover any gender biases in our questions**":'gender'
                }

READ_PROGRAMS = {
                'Mechanical Engineering':'mech',
                'Biomedical Engineering':'bmed',
                'Software Engineering':'swe',
                'Computer Engineering':'ce',
                'Mechatronics Engineering':'tron',
                'Civil Engineering':'cive',
                'Chemical Engineering':'chem',
                'Systems Design Engineering':'syde',
                'Management Engineering':'msci',
                'Electrical Engineering':'elec',
                'Nanotechnology Engineering':'nano',
                'Geological Engineering':'geo',
                'Environmental Engineering':'env',
                'Architectural Engineering':'arch-e',
                'Architecture':'arch'
                }

READ_PROBLEMS = {
                'Problems that are well defined.':'defined',
                'Problems that require some investigation.':'investigate',
                'Problems with very limited information.':'discover'
                }

READ_CREATIVE = {
                'I am somewhat creative.':'somewhat_creative',
                'I am not creative.' : 'not_creative',
                'I am very creative.' : 'creative'
                }

READ_INDUSTRY = {
                'Architecture (i.e. Designing a building taller than the CN Tower)':'architecture',
                'Automotive (i.e. Designing a new autonomous car)':'automotive',
                'Business (i.e. Starting your own consulting company)':'business',
                'Construction (i.e. Building a smart city)':'construction',
                'Health (i.e. Creating technology for minimally invasive surgeries)':'health',
                'Environment (i.e. Producing energy in sustainable ways)':'environment',
                'Manufacturing (i.e. Optimization or automation of manufacturing processes)':'manufacturing',
                'Technology (i.e. Working with cloud based software)':'technology'
                }

READ_OUTDOORS = {
                'Working outside would be okay, but only for short periods of time.':'limited',
                'I would rather work inside.':'indoors',
                'I love the outdoors and wish I could work outside every day.':'outdoors'
                }

READ_CAREERS = {
                'Building things with moving parts.':'moving_parts',
                'Designing or building sensor based technology.':'sensors',
                'Programming apps.':'programming',
                'Optimizing processes.':'optimizing',
                "Improving the way we use the world's resources.":'resources',
                'Designing buildings.':'buildings',
                'Making discoveries at the molecular level.':'molecules'
                }

READ_GROUPWORK = {
                'I occasionally like working with others.':'occasionally',
                'I enjoy working with others.':'yes',
                'I do not like working as part of a team. I would rather work alone.':'no'
                 }

READ_COURSES = {
                'Autoshop':'autoshop',
                'Biology':'biology',
                'Business':'business',
                'Chemistry':'chemistry',
                'Computer Science':'computer_science',
                'Geography':'geography',
                'History':'history',
                'Language Arts':'language_arts',
                'Math':'math',
                'Physics':'physics',
                'Visual Arts':'visual_arts'
                }

READ_PROGRAMMING = {
                    "I can code but it's not my favourite thing to do.":'partial',
                    "I code, I enjoy it and I'm good at it.":'complete',
                    'I don’t code and I have no desire to learn.':'no',
                    "I don't code but I am interested in trying it.":'interested'
                    }

READ_CLUBS = {
            'Art or design club (i.e. Fashion for Change)':'art/design',
            'Autoshop club (i.e. Autonomoose Autonomous Car Club)':'autoshop',
            'Business club (i.e. UW Finance Association)':'business',
            'Consulting club (i.e. DECA)':'consulting',
            'Environment club (i.e. UW Energy Network)':'environment',
            'Robotics club (i.e. UW Robotics Team)':'robotics',
            'Hacker club (i.e. UW Hacks)':'hacker_club',
            'Student council (i.e. Engineering Society)':'student_council',
            }

READ_PROJECTS = {
                'Prototyping a musical instrument for children with a disability.':'prototyping_instrument',
                'Designing a water treatment system for Mars.':'mars_water_treatment',
                'Programming a robot that can make you dinner.':'robot',
                'Building the world’s most powerful supercomputer.':'supercomputer',
                'Designing an Olympic village.':'olympic_village',
                'Creating a battery from recycled material.':'battery',
                'Optimizing the Uber Pool routes.':'uber_pool'
                }

READ_TV = {
            'Big Bang Theory':'big_bang_theory',
            'Breaking Bad':'breaking_bad',
            'Grey’s Anatomy':'greys_anatomy',
            'House Hunters':'house_hunters',
            'Myth Busters':'myth_busters',
            'Planet Earth':'planet_earth',
            'Silicon Valley':'silicon_valley'
            }

READ_ALTERNATE_DEGREE = {
            'Applied Science':'applied_science',
            'Business':'business',
            'Computer Science':'cs',
            'Economics':'econ',
            'English Literature':'lit',
            'Environmental Studies':'env',
            'Finance':'fin',
            'Geography':'geo',
            'Graphic Design':'design',
            'Health Studies':'health',
            'Marketing':'marketing',
            'Math':'math',
            'Political Science':'poli_sci',
            'Psychology':'psych',
            'Visual Arts':'visual_arts'
            }

READ_EQUIPMENT = {
            'That sounds cool!':'yes',
            "Could be cool, but I don't really care about fancy equipment or how much it costs.":'maybe',
            "That scares me and I don't want to touch it.":'no'
            }

READ_DRAWING = {
            'I am not the best, but I am not the worst.':'partial',
            'I am not very good.':'bad',
            'Really good, I can draw just about anything.':'good'
            }

READ_ESSAY = {
            'Excited! I can share my theories with the world.':'yes',
            'Annoyed. I would much rather be given a topic with clear instructions.':'no',
            'A bit apprehensive. I get overwhelmed with so many options.':'partial'
            }

INDEX_PROGRAM = {
                'mech': 9,
                'bmed': 2,
                'swe': 12,
                'tron': 14,
                'cive': 5,
                'chem': 4,
                'syde': 13,
                'msci': 10,
                'ce': 3,
                'elec': 6,
                'nano': 11,
                'geo': 8,
                'env': 7,
                'arch-e': 1,
                'arch': 0
                }
INV_INDEX_PROGRAM= {
            9:'mech',
            2:'bmed',
            12:'swe',
            14:'tron',
            5:'cive',
            4:'chem',
            13:'syde',
            10:'msci',
            3:'ce',
            6:'elec',
            11:'nano',
            8:'geo',
            7:'env',
            1:'arch-e',
            0:'arch'
            }
# Functions from data load

class NpEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        else:
            return super(NpEncoder, self).default(obj)

def get_clean_data(directory,drop_not_happy='H',drop_gender=True,data_balance=False):
    '''
    Should we drop "Are you happy with your program?"
    '''
    data = pd.read_csv(directory,dtype=str)

    # dropping PII + skill_test + timestamp + year + raw_industry
    data = data.drop(data.columns[[0,1,3,24]], axis=1)

    # renaming data for readability
    data = data.rename(index=str,columns = READ_HEADERS)
    data.program = data.program.map(READ_PROGRAMS)
    data.problem_type = data.problem_type.map(READ_PROBLEMS)
    data.creative = data.creative.map(READ_CREATIVE)
    data.outdoors = data.outdoors.map(READ_OUTDOORS)
    data.career = data.career.map(READ_CAREERS)
    data.group_work = data.group_work.map(READ_GROUPWORK)
    data.liked_courses = data.liked_courses.map(READ_COURSES)
    data.disliked_courses = data.disliked_courses.map(READ_COURSES)
    data.programming = data.programming.map(READ_PROGRAMMING)
    data.join_clubs = data.join_clubs.map(READ_CLUBS)
    data.not_clubs = data.not_clubs.map(READ_CLUBS)
    data.liked_projects = data.liked_projects.map(READ_PROJECTS)
    data.disliked_projects = data.disliked_projects.map(READ_PROJECTS)
    data.tv_shows = data.tv_shows.map(READ_TV)
    data.alternate_degree = data.alternate_degree.map(READ_ALTERNATE_DEGREE)
    data.expensive_equipment = data.expensive_equipment.map(READ_EQUIPMENT)
    data.drawing = data.drawing.map(READ_DRAWING)
    data.essay = data.essay.map(READ_ESSAY)
    try:
        data.new_programming = data.new_programming.map(READ_NEW_PROGRAMMING)
    except:
        print("new programming question not included")

    # Cleaning industry data
    data.index.name = 'id'
    industry_data = data["industry"].str.split(";", expand = True)
    industry_data = industry_data.replace(READ_INDUSTRY)
    binary_industry_data = np.array([np.arange(len(data))]*8).T
    binary_industry_data = pd.DataFrame(binary_industry_data, columns=READ_INDUSTRY.values())
    binary_industry_data.index.name = 'id'

    for col in binary_industry_data.columns:
        binary_industry_data[col].values[:] = '0'

    for index, row in industry_data.iterrows():
        for i in range(8):
            try:
                binary_industry_data.iloc[int(index), binary_industry_data.columns.get_loc(row[i])] = '1'
            except:
                error = "None_Type detected"

    data.index = data.index.map(int)
    binary_industry_data.index = binary_industry_data.index.map(int)
    data = (data.merge(binary_industry_data, left_on='id', right_on='id',how='left'))
    data = data.drop(['industry'], axis=1)

    # if drop where all values are unhapppy
    if drop_not_happy == 'H':
        data = data[data.happy == 'Yes']
        data = data.drop(['happy'], axis=1)
    if drop_not_happy == 'NH':
        data = data[data.happy == 'No']

    # drop gender data
    if drop_gender:
        data = data.drop(['gender'], axis=1)

    data.index = data.index.map(int)
    # data = data.sample(frac=1).reset_index(drop=True) ##Shuffle

    if data_balance != False:
        programs = list(READ_PROGRAMS.values())
        b_df = data.copy()
        b_df = b_df.head(0)

        for program in programs:
            temp_df = data.copy()[data.program==program]
            while len(temp_df) <= data_balance[program]:
                temp_df = temp_df.append(temp_df)
            temp_df = temp_df.head(data_balance[program])
            b_df = b_df.append(temp_df)
            b_df = b_df.reset_index(drop=True)
        data = b_df

    data.columns = data.columns.astype(str)
    return data

def transform_post_dict(post_dict):
    print("Transforming post_dict...")
    post_dict = json.dumps(dict(post_dict))
    post_dict = json.loads(post_dict)
    post_dict = dict(post_dict)
    industries = post_dict['industry']
    post_dict['architecture'] = '0'
    post_dict['automotive'] = '0'
    post_dict['business'] = '0'
    post_dict['construction'] = '0'
    post_dict['health'] = '0'
    post_dict['environment'] = '0'
    post_dict['manufacturing'] = '0'
    post_dict['technology'] = '0'
    for industry in industries:
        post_dict[industry] = '1'
    return dict(post_dict)

def get_label_encoded_data(directory,model_name,column_list,drop_not_happy='H',data_balance=False,drop_gender=True):
    df = get_clean_data(directory,drop_not_happy,data_balance=data_balance,drop_gender=drop_gender)
    if drop_gender:
        df = df[column_list]
    else:
        column_list.append('gender')
        df = df[column_list]

    col_list = list(df.columns)
    encoded_dict_list = []
    for col in col_list:
        keys = df[col].unique()
        le = preprocessing.LabelEncoder()
        le.fit(list(keys))
        df[col] = le.transform(list(df[col]))
        vals = df[col].unique()
        keys = list(le.inverse_transform(vals))
        cd = dict(zip(keys,vals))
        row = {str(col):cd}
        encoded_dict_list.append(row)
        with open('poc/quiz/exported_model_files/'+model_name+'_'+col+'_encoded_dictionary.json', 'w') as f:
            json.dump(str(row),f,cls=NpEncoder)

    with open('poc/quiz/exported_model_files/'+model_name+'_cols.txt', 'w') as f:
        for col in col_list:
            f.write(col)
            f.write('\n')

    return [df,encoded_dict_list]

def get_encoded_data(directory,model_name,drop_not_happy):
    df = get_clean_data(directory,drop_not_happy)
    df = df.drop(['happy'], axis=1)

    col_list = list(df.columns)
    encoded_dict_list = []
    for col in col_list:
        keys = df[col].unique()
        le = preprocessing.LabelEncoder()
        le.fit(list(keys))
        df[col] = le.transform(list(df[col]))
        vals = df[col].unique()
        keys = list(le.inverse_transform(vals))
        cd = dict(zip(keys,vals))
        row = {str(col):cd}
        encoded_dict_list.append(row)
        with open('poc/quiz/exported_model_files/'+model_name+'_'+col+'_encoded_dictionary.json', 'w') as f:
            json.dump(row,f,cls=NpEncoder)

    with open('poc/quiz/exported_model_files/'+model_name+'_cols.txt', 'w') as f:
        for col in col_list:
            f.write(col)
            f.write('\n')

    return [df,encoded_dict_list]

def get_encoded_dict(model_name):
    cols = []

    with open('poc/quiz/exported_model_files/'+model_name+'_cols.txt', 'r') as f:
        for line in f:
            # remove linebreak which is the last character of the string
            currentPlace = line[:-1]
            # add item to the list
            cols.append(currentPlace)
    encoded_dict = {}
    for col in cols:
        with open('poc/quiz/exported_model_files/'+model_name+'_'+col+'_encoded_dictionary.json', 'r') as f:
            row = json.loads(f.read())
        encoded_dict[col] = row
    return encoded_dict

def get_merged_encoded_data(directory,model_name,one_hot_encode,column_list,drop_not_happy='H',data_balance=False,drop_gender=True):
    df = get_label_encoded_data(directory,model_name,column_list,drop_not_happy,data_balance,drop_gender=drop_gender)[0]
    df = pd.get_dummies(df,columns=one_hot_encode)
    return df

def save_model(df,model,cat,model_name):
    df.to_csv ('poc/quiz/exported_model_files/'+model_name+'.csv', index = None, header=True)
    with open('poc/quiz/exported_model_files/'+model_name+'.pkl', 'wb') as fid:
        pickle.dump(model, fid,2)
    with open('poc/quiz/exported_model_files/'+model_name+'_cat', 'wb') as fid:
        pickle.dump(cat, fid,2)

def retrieve_prediction_labels(model,prediction):
    # returns a dictionary for each label and their probability in the prediction
    labels = model.classes_
    results = prediction[0]
    results_dict = {}
    for i in range(len(results)):
        results_dict[INV_INDEX_PROGRAM[labels[i]]] = np.round(results[i],4)
    return results_dict

# Define Parameters
MODEL_NAME = 'nb_ohe_f0_d0_b7_c36_v0'

d0 = 'poc/quiz/exported_model_files/d0.csv'

c36 = [
    'creative',
       'outdoors',
       'career',
       'group_work',
       'liked_courses',
       'disliked_courses',
       'join_clubs',
       'not_clubs',
       'liked_projects',
       'disliked_projects',
       'alternate_degree',
       'drawing',
       'essay',
       'architecture',
       'automotive',
       'business',
       'construction',
       'health',
       'environment',
       'manufacturing',
       'technology',
       'program'
]

ohe =  [
        'creative',
        'outdoors',
        'career',
        'group_work',
        'liked_courses',
        'disliked_courses',
        'join_clubs',
        'not_clubs',
        'liked_projects',
        'disliked_projects',
        'alternate_degree',
        'drawing',
        'essay'
        ]

b0 = False # this is only relevant when we want to use untreated data for code d0

b7 = {
    'mech': 100,
    'bmed': 100,
    'swe': 100,
    'tron': 100,
    'cive': 100,
    'chem': 100,
    'syde': 100,
    'msci': 100,
    'ce': 40,
    'elec': 100,
    'nano': 100,
    'geo': 100,
    'env': 100,
    'arch-e': 100,
    'arch': 100
    }

b1 = {
    'mech': 100,
    'bmed': 100,
    'swe': 100,
    'tron': 100,
    'cive': 100,
    'chem': 100,
    'syde': 100,
    'msci': 100,
    'ce': 100,
    'elec': 100,
    'nano': 100,
    'geo': 100,
    'env': 100,
    'arch-e': 100,
    'arch': 100
    }

v0 = 1

#model_name = 'model-type_encoding_directory_datastructure_column-set_version'
# experiment_model_name = 'dataSet_dataBalance_columnSet_dataBalanceMultiple'
experiment_model_name = 'd0_b7_c36_v0'
directory = d0
data_balance = b7
column_list = c36
data_balance_multiple = v0 # Ratio of other programs to program in binary classifier. 2 means double of other programs, 0.5 means half

def save_scores(scoring_dictionary,experiment_model_name):
    df = pd.DataFrame(scoring_dictionary)
    df = df.T
    df.to_csv("exported_model_files/scores/"+experiment_model_name+".csv", header=True)

def get_mclass_accuracy(temp_model_name,model,test_array,test_actual):
    test_pred = []
    for i in range(len(test_array)):
        test_pred.append(model.predict([test_array[i]]))
    accuracy = metrics.accuracy_score(test_pred,test_actual)
    return accuracy

def sort_probability_dict(p_df):
    ordered_probabilties = sorted(p_df.values(),reverse=True)
    ordered_programs = sorted(p_df, key=p_df.get,reverse=True)
    return [p_df, ordered_probabilties, ordered_programs]

def get_mclass_t3(temp_model_name,model,test_array,test_actual):
    t3_scores = []
    for i in range(len(test_array)):
        prediction = model.predict_proba([test_array[i]])
        probs = sort_probability_dict(retrieve_prediction_labels(model,prediction))[2][:3]
        n_probs = []
        for prob in probs:
            n_probs.append(INDEX_PROGRAM[prob])
        try:
            t3 = (1/(n_probs.index(test_actual[i])+1))
        except:
            t3 = 0
        t3_scores.append(t3)

    return mean(t3_scores)

def get_mclass_rr(temp_model_name,model,test_array,test_actual):
    rr_scores = []
    for i in range(len(test_array)):
        prediction = model.predict_proba([test_array[i]])
        probs = sort_probability_dict(retrieve_prediction_labels(model,prediction))[2]
        n_probs = []
        for prob in probs:
            n_probs.append(INDEX_PROGRAM[prob])
        try:
            rr = (1/(n_probs.index(test_actual[i])+1))
        except:
            rr = 0
        rr_scores.append(rr)
    return mean(rr_scores)

# Supporting Functions for RE-Building the model on the Heroku Server
print("building model...")
if 'le' in MODEL_NAME:
    # Building New model
    model_name = 'nb_le_f0_'+ experiment_model_name
    data = get_label_encoded_data(directory,model_name,column_list,'H',data_balance=data_balance)[0]

    x_df = data.drop(axis=1,columns=["program"])
    y_df = data["program"]

    X = np.array(x_df) # convert dataframe into np array
    Y = np.array(y_df) # convert dataframe into np array

    mnb = MultinomialNB()
    model = mnb.fit(X, Y) # fit the model using training data

    cat = data.drop('program',axis=1)
    cat = dict(zip(cat.columns,range(cat.shape[1])))

elif 'ohe' in MODEL_NAME:
    model_name = 'nb_ohe_f0_'+ experiment_model_name
    data = get_merged_encoded_data(directory,model_name,one_hot_encode=ohe,column_list = column_list,drop_not_happy='H',data_balance=data_balance)

    x_df = data.drop(axis=1,columns=["program"])
    y_df = data["program"]

    X = np.array(x_df) # convert dataframe into np array
    Y = np.array(y_df) # convert dataframe into np array

    mnb = MultinomialNB()
    model = mnb.fit(X, Y) # fit the model using training data

    cat = data.drop('program',axis=1)
    cat = dict(zip(cat.columns,range(cat.shape[1])))

save_model(data,model,cat,model_name)

# Scoring models
print("Scoring model")
model_name = model_name
temp_model_name = model_name

model_data = pd.read_csv('poc/quiz/exported_model_files/'+model_name+'.csv',dtype=str)
# Loading test data
if 'le' in model_name:
    test_data_t7 = get_label_encoded_data('poc/quiz/exported_model_files/t7.csv',model_name='t7',column_list=column_list,drop_not_happy='H',data_balance=False)[0]
elif 'ohe' in model_name:
    test_data_t7 = get_merged_encoded_data(directory = 'poc/quiz/exported_model_files/t7.csv',model_name ='t7',one_hot_encode=ohe,column_list = column_list,drop_not_happy='H',data_balance=False)

test_data_t7_temp = test_data_t7.copy()[list(model_data.columns)].head(210)

# Loading model files
pkl_file = open('poc/quiz/exported_model_files/'+model_name+'_cat', 'rb')
index_dict = pickle.load(pkl_file)
new_vector = np.zeros(len(index_dict))

pkl_file = open('poc/quiz/exported_model_files/'+model_name+'.pkl', 'rb')
model = pickle.load(pkl_file)

# Preparing Loading data
test_array = np.array(test_data_t7_temp.drop(axis=1,columns=["program"]))
test_actual = np.array(test_data_t7_temp["program"])

mclass_t3 = get_mclass_t3(temp_model_name,model,test_array,test_actual)
mclass_RR = get_mclass_rr(temp_model_name,model,test_array,test_actual)
mclass_accuracy = get_mclass_accuracy(temp_model_name,model,test_array,test_actual)

print("Model:  "+model_name)
print("t3:  "+str(mclass_t3))
print("RR:  "+str(mclass_RR))
print("Accuracy:  "+str(mclass_accuracy))


import json
import os
from pprint import pprint
from konlpy.tag import Okt
from tensorflow.keras.models import load_model
import time
import datetime
import numpy as np
import tensorflow as tf
import nltk
from tensorflow.keras import models
from tensorflow.keras import layers
from tensorflow.keras import optimizers
from tensorflow.keras import losses
from tensorflow.keras import metrics

class ReviewAi(object):
    def __init__(self):
        self.okt = Okt()
        self.x_train = None
        self.y_train = None
        self.path =  os.path.abspath('')
        self.fname =  str
        self.x_test = None
        self.y_test = None

    def create_docs(self):
        path = self.path
        fname = self.fname
        fname = '/com_dayoung_api/cop/rev/model/data/train_docs.json'
        with open(path + fname, encoding='utf-8') as f:
            train_docs = json.load(f)

        fname = '/com_dayoung_api/cop/rev/model/data/test_docs.json'
        with open(path + fname, encoding='utf-8') as f:
            test_docs = json.load(f)
        # print(train_docs[:10])
        return [train_docs, test_docs]

    @staticmethod
    def create_tokens(train_docs):
        tokens = [t for d in train_docs for t in d[0]]
        return tokens

    @staticmethod
    def create_nltk_text(tokens):
        return nltk.Text(tokens, name='NMSC')

    def tokenize(self, review):
        #형태소와 품사를 join
        return ['/'.join(t) for t in self.okt.pos(review, norm=True, stem=True)]

    @staticmethod
    def transfer_text_to_selected_words(text):
        selected_words = [f[0] for f in text.vocab().most_common(10000)]
        return selected_words

    def term_frequency(self, doc):
        ai = ReviewAi()
        docs = self.create_docs()
        train_docs = docs[0]
        tokens = ai.create_tokens(train_docs)
        text = ai.create_nltk_text(tokens)
        selected_words = self.transfer_text_to_selected_words(text)
        return [doc.count(word) for word in selected_words]

    def set_train(self):
        train_x = [self.term_frequency(d) for d, _ in train_docs]
        test_x = [self.term_frequency(d) for d, _ in test_docs]
        train_y = [c for _, c in train_docs]
        test_y = [c for _, c in test_docs]
        self.x_train = np.asarray(train_x).astype('float32')
        self.x_test = np.asarray(test_x).astype('float32')
        self.y_train = np.asarray(train_y).astype('float32')
        self.y_test = np.asarray(test_y).astype('float32')

    def model_save(self):

        FREQUENCY_COUNT = 10000;

        # 레이어 구성
        model = tf.keras.models.Sequential([
            tf.keras.layers.Dense(64, activation='relu', input_shape=(FREQUENCY_COUNT,)),
            tf.keras.layers.Dense(64, activation='relu'),
            tf.keras.layers.Dense(1, activation='sigmoid')
        ])

        # 학습 프로세스 설정
        model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.001),
            loss=tf.keras.losses.binary_crossentropy,
            metrics=[tf.keras.metrics.binary_accuracy]
            )
        model.fit(self.x_train, self.y_train, epochs=10, batch_size=512)

        model.save('another_model.h5')

    def model_load(self):
        path = self.path
        fname = self.fname

        fname = '/com_dayoung_api/cop/rev/model/data/review_model.h5'
        loaded_model = load_model(path + fname)
        return loaded_model

    def model_eval(self, model):
        result = model.evaluate(self.x_test, self.y_test)
        return result

    def predict_review(self, review):
        token = self.tokenize(review)
        tfq = self.term_frequency(token)
        data = np.expand_dims(np.asarray(tfq).astype('float32'), axis=0)
        loaded_model = self.model_load()
        score = float(loaded_model.predict(data))
        if(score > 0.5):
            print(f"{review} ==> {round(score*100)}% 확률로 긍정 리뷰입니다.")
        else:
            print(f"{review} ==> {round((1-score)*100)}% 확률로 부정 리뷰입니다.")

        return score

if __name__ == "__main__":
    ai = ReviewAi()
    docs = ai.create_docs()
    train_docs = docs[0]
    test_docs = docs[1]
    print(f'[ 훈련데이터 상위 10 Rows ] : {train_docs[:10]}')
    tokens = ai.create_tokens(train_docs)
    print(f'[ 전체 토큰의 개수 ] : {len(tokens)}')
    # docs = tokens
    # temp_tokens = ai.tokenize(train_docs)
    nltk_text =  ai.create_nltk_text(tokens)
    print(f'[ 중복을 제외한 토큰의 개수 ] {len(nltk_text.tokens)}')
    print(f'[ 출현 빈도가 높은 상위 토큰 10개 ] {nltk_text.vocab().most_common(10)}')

    # 출현 빈도 높은 상위 10000개 토큰 분류
    selected_words = ai.transfer_text_to_selected_words(nltk_text)
    print(f'[ 출현 빈도가 높은 상위 토큰 100개 ] {selected_words[:100]}')

    # term_freq_count = ai.term_frequency(train_docs)
    # ai.set_train()
    # ai.model_save()
    # model = ai.model_load()
    # answer = ai.model_eval(model)
    # print(f'[ 모델 평가한 답 ] {answer}')
    # print(f'================= Practice Result ====================')
    ai.predict_review("재미 정말 없어요 갖다 버리세요")
    ai.predict_review("이건 개망한 영화인데;; 누가보냐")
    ai.predict_review("좋았어 이거 또 보러 온다")
    ai.predict_review("너무 예쁜데요 ㅠㅠ 최고에요")
    ai.predict_review("하하하 쩔었다~~")
    ai.predict_review("21세기 최고의 영화다 진짜!!")
    ai.predict_review("너무 재밌어요 ㅋㅋ 진짜 쩐다 쩔어")
    ai.predict_review("수업 가기 귀찮다...")


import unittest
import dateutil.parser

import norman

class TestTimestampFormatRule(unittest.TestCase):

    def test_it_formats_timestamp_to_valid_format(self):

        rule = norman.TimestampFormatRule()

        timestamp = "4/1/11 11:00:00 PM"

        actual = rule.normalize(timestamp)
        expected = dateutil.parser.parse(timestamp).isoformat()

        self.assertEqual(expected, actual)


from django.conf.urls import patterns, include, url

from .views import (DashboardView, RecentActivityView, MeasurementRecentActivityView,
                    CommentRecentActivityView)

urlpatterns = patterns(
    '',
    url(r'^$', DashboardView.as_view(), name='dashboard'),
    url(r'^experiment/all/activity/(?P<experiment_pk>[\d]+)/$',
        RecentActivityView.as_view(),
        name='experiment-all-activity'),
    url(r'^experiment/measurement/activity/(?P<experiment_pk>[\d]+)/$',
        MeasurementRecentActivityView.as_view(),
        name='experiment-measurement-activity'),
    url(r'^experiment/comment/activity/(?P<experiment_pk>[\d]+)/$',
        CommentRecentActivityView.as_view(),
        name='experiment-comment-activity'),

    url(r'^all/activity/$',
        RecentActivityView.as_view(),
        name='all-activity'),
    url(r'^measurement/activity/$',
        MeasurementRecentActivityView.as_view(),
        name='measurement-activity'),
    url(r'^comment/activity/$',
        CommentRecentActivityView.as_view(),
        name='comment-activity'),

    url(r'^api/', include('dashboard.api.urls')),
)


import unittest
import json
from modules.positon_manage.position_set.position_create import *
from common.get_result_db import get_result_from_sql

positionType_0 = "0"
positionType_1 = "1"
positionName = readcfg.position_name
position_virtual_name = readcfg.position_virtual_name
position_timeZone_name = readcfg.position_timeZone_name
parentPositionId = readcfg.positionId_real1_Gary
positionId_Jenny = readcfg.positionId_real1_Jenny
positionId_wrong = readcfg.positionId_wrong
timeZone = readcfg.timeZone
remark = readcfg.remark
sql = readcfg.sql_position_create
sql_timeZone = readcfg.sql_position_create_timeZone
options = 0

class TestPositionCreate(unittest.TestCase):
    """
    创建位置
    """

    def test_position_create_01(self):
        """
        测试创建位置
        1、调用创建接口
        2、获取接口返回的position_id
        3、查询数据库中用户下最新的position_id
        4、判断是否相等
        """
        result = position_create(positionType_1, positionName, parentPositionId, timeZone, remark, options)
        position_id = json.loads(result.text)["result"]["positionId"]
        position_id_sql = get_result_from_sql(sql)[0]
        self.assertEqual(position_id, position_id_sql,
                         "创建的positionId：%s，数据库查询的positionId：%s" % (position_id, position_id_sql))

    def test_position_create_02(self):
        """测试创建位置名称已存在"""
        result = position_create(positionType_1, positionName, parentPositionId, timeZone, remark, options)
        self.assertIn('"code":703', result.text)

    def test_position_create_03(self):
        """测试创建位置时parentPositionId错误或不存在"""
        result = position_create(positionType_1, positionName, positionId_wrong, timeZone, remark, options)
        self.assertIn('"code":710', result.text)

    def test_position_create_04(self):
        """测试使用其他用户的位置为parentPositionId创建位置"""
        result = position_create_jenny(positionType_1, positionName, positionId_Jenny, timeZone, remark, options)
        self.assertIn('"code":710', result.text)

    def test_position_create_05(self):
        """创建虚拟位置（positionType=0）"""
        result = position_create(positionType_0, position_virtual_name, parentPositionId=None, timeZone=timeZone,
                                 remark=remark, options=options)
        position_id = json.loads(result.text)["result"]["positionId"]
        position_id_sql = get_result_from_sql(sql)[0]
        self.assertEqual(position_id, position_id_sql,
                         "创建的positionId：%s，数据库查询的positionId：%s" % (position_id, position_id_sql))

    def test_position_create_06(self):
        """测试positionType为空"""
        result = position_create("", positionName, parentPositionId=None, timeZone=timeZone, remark=remark,
                                 options=options)
        self.assertIn('"code":302', result.text)

    def test_position_create_07(self):
        """测试positionName为空"""
        result = position_create(positionType_1, "", parentPositionId=None, timeZone=timeZone, remark=remark,
                                 options=options)
        self.assertIn('"code":302', result.text)

    def test_position_create_08(self):
        """测试timeZone：-0.5"""
        result = position_create(positionType_1, position_timeZone_name, parentPositionId=None, timeZone="-0.5",
                                 remark=remark, options=options)
        time_zone_sql = get_result_from_sql(sql_timeZone)[0]
        self.assertEqual("-0.5", time_zone_sql)

    def test_position_create_09(self):
        """测试请求参数不传remark"""
        result = position_create(positionType_1, positionName + "1", parentPositionId, timeZone, remark=None,
                                 options=options)
        self.assertIn('"code":0', result.text)

    def test_position_create_10(self):
        """测试请求参数不传options"""
        result = position_create(positionType_1, positionName + "2", parentPositionId, timeZone, remark, options=None)
        self.assertIn('"code":0', result.text)

if __name__ == '__main__':
    unittest.main()


from django.conf.urls import url
from django.contrib.auth.decorators import login_required as auth
from . import views

urlpatterns = [

    url(r'^$', views.index, name='index'),
    url(r'^createarticle/$', auth(views.CreateArticle.as_view()), name='create_article'),
    url(r'^createdarticle/$', auth(views.created_article), name='created_article'),
    url(r'^editarticle/(?P<slug>.*)/$', auth(views.EditArticle.as_view()), name='edit_article'),
    url(r'^deletearticle/(?P<slug>.*)/$', auth(views.DeleteArticle.as_view()), name='delete_article'),
    url(r'^deletedarticle/(?P<slug>.*)/$', auth(views.deleted_article), name='deleted_article'),

    url(r'^lastposts/$', views.lastposts, name = 'lastposts'),
    url(r'^tags/(?P<tagsslug>.*)/$', views.tags, name='tags' ),
    url(r'^category/(?P<categoryslug>.*)/$', views.category, name='category' ),
    url(r'^archives/', views.archives, name='archives'),
    url(r"^archive/(\d+)/([1-9]|1[012])/$", views.monthly_archive , name='archive'),

    url(r'^(?P<year>\d{4})/(?P<month>\d{1,2})/(?P<day>\d{1,2})/(?P<slug>[-\w]*)/$', views.ArticleDetail.as_view(), name='article_detail'),

]


from sklearn.base import BaseEstimator, TransformerMixin

from TaxiFareModel.utils import haversine_vectorized
import pandas as pd

class DistanceTransformer(BaseEstimator, TransformerMixin):
    """Compute the haversine distance between two GPS points."""

    def __init__(self,
                 start_lat="pickup_latitude",
                 start_lon="pickup_longitude",
                 end_lat="dropoff_latitude",
                 end_lon="dropoff_longitude"):
        self.start_lat = start_lat
        self.start_lon = start_lon
        self.end_lat = end_lat
        self.end_lon = end_lon

    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        """Returns a copy of the DataFrame X with only one column: 'distance'"""
        X_ = X.copy()
        X_['distance'] = haversine_vectorized(X_,
             start_lat="pickup_latitude",
             start_lon="pickup_longitude",
             end_lat="dropoff_latitude",
             end_lon="dropoff_longitude")
        return X_[[
                'distance',
            ]]

class TimeFeaturesTransformer(BaseEstimator, TransformerMixin):
    """Compute the haversine distance between two GPS points."""
    def __init__(self, time_column, time_zone_name='America/New_York'):
        self.time_column = time_column
        self.time_zone_name = time_zone_name

    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        """Returns a copy of the DataFrame X with only four columns: 'dow', 'hour', 'month', 'year'"""
        X_ = X.copy()
        timezone_name = 'America/New_York'
        time_column = "pickup_datetime"
        X_.index = pd.to_datetime(X[time_column])
        X_.index = X_.index.tz_convert(timezone_name)
        X_["dow"] = X_.index.weekday
        X_["hour"] = X_.index.hour
        X_["month"] = X_.index.month
        X_["year"] = X_.index.year
        return X_[[
                'dow', 'hour', 'month', 'year'
            ]]


from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.status import HTTP_200_OK
from api.models.user import User, UserSerializer, UserAuth
from api.models.user_token import get_user_for_token

def users_usage():
    return {
                'methods' : {
                    'api/users' : {
                        'methods' : {
                            'GET' : 'Access to this Usage',
                            'POST' : 'Create the new user'
                        }
                    },
                    'api/users/auth' : {
                        'methods' : {
                            'POST' : 'Try to authenticate (get a token)'
                        }
                    },
                    'api/users/retrieve' : {
                        'methods' : {
                            'POST' : 'Access user\'s data according to the token (require a token)'
                        }
                    },

                }
            }

@api_view(['GET', 'POST'])
def users(request):
    if request.method == 'GET':
        return Response(users_usage(), HTTP_200_OK)
    elif request.method == 'POST':
        response = User().handle_create(request.data)
        return Response(response['data'], response['code'])

@api_view(['POST'])
def auth(request):
    data = request.data
    response = UserAuth().auth_process(data)
    return Response(response['data'], response['code'])

@api_view(['POST'])
def retrieve(request):
    data = request.data
    response = get_user_for_token(data)
    return Response(response['data'], response['code'])


from warp import DataProcessor, XPathExtractor

class TitleExtractor(XPathExtractor):

    def configure(self):
        self.id = 'title-extractor'
        self.version = 1
        self.output_format = 'application/json'
        self.invoked = False

    def extract_from_tree(self, html_tree, input_dict):
        t = html_tree.xpath("//h1/text()")[0]
        self.invoked = True
        return dict(title=t)

class TitleDecorator(DataProcessor):

    def configure(self):
        self.id = 'title-decorator'
        self.version = 1
        self.output_format = 'application/json'
        self.invoked = False

    def process(self, input_dict):
        t = "<b>%s</b>" % input_dict['title-extractor']['title']
        self.invoked = True
        return dict(title=t)


import asyncio
import json
import logging
from datetime import datetime, timedelta

import discord
from dateutil import parser
from tabulate import tabulate

import connector
from run_args import get_args

args = get_args()
log = logging.getLogger('__name__')

iconURL = 'https://raw.githubusercontent.com/Map-A-Droid/MAD/master/madmin/static/mad_banner_trans.png'

class MyClient(discord.Client):

    def __init__(self):
        discord.Client.__init__(self)

    async def on_ready(self):
        log.info(f"Logged in. Username: {self.user.name}. User ID:{self.user.id}")

    async def on_message(self, message):
        if message.content.startswith('!status'):
            with open('servers.json') as f:
                servers_config_json = json.load(f)
                for server in servers_config_json:
                    error_found = False
                    if message.channel.id == int(server['status_channel_id']):
                        server_name = server['name']
                        table_header = ['Origin', 'Route', 'Pos', 'Time']
                        table_contents = []

                        log.info(f"Status request received for {server_name}")
                        log.debug("Calling /get_status for current status")
                        device_status_response = connector.get_status(server)

                        for device in device_status_response or []:
                            table_before = tabulate(table_contents, headers=table_header)
                            routemanager = device.get('rmname', '') if device.get('rmname',
                                                                                'Not Defined') is not None else 'Not Defined'
                            origin = device.get('name', '') if device.get('rmname', '') is not None else ''
                            route_pos = device.get('routePos', '?') if device.get('routePos', '?') is not None else '?'
                            route_max = device.get('routeMax', '?') if device.get('routeMax', '?') is not None else '?'
                            lastProtoDateTime = device.get('lastProtoDateTime', '') if device.get('lastProtoDateTime',
                                                                                '') is not None else ''
                            number_front_chars = 6
                            number_end_chars = 5

                            try:
                                datetime_from_status_json = datetime.fromtimestamp(lastProtoDateTime)
                                formatted_device_last_proto_time = datetime_from_status_json.strftime("%H:%M")
                                latest_acceptable_datetime = (datetime.now() - timedelta(minutes=args.duration_before_alert))
                                log.debug(f"{origin} Last Proto Date Time: {datetime_from_status_json}")
                                log.debug(f"{origin} Last Acceptable Time: {latest_acceptable_datetime}")

                                if datetime_from_status_json < latest_acceptable_datetime:
                                    error_found = True

                            except Exception as e:
                                error_found = True
                                formatted_device_last_proto_time = 'Unkwn'

                            if args.trim_table_content:
                                if len(routemanager) > (number_front_chars + number_end_chars):
                                    routemanager = f"{routemanager[:number_front_chars]}..{routemanager[-number_end_chars:]}"
                                if len(origin) > (number_front_chars + number_end_chars):
                                    origin = f"{origin[:number_front_chars]}..{origin[-number_end_chars:]}"

                            table_contents.append([origin,
                                                   routemanager,
                                                   f"{route_pos}/{route_max}",
                                                   formatted_device_last_proto_time
                                                   ])

                            table_after = tabulate(table_contents, headers=table_header)

                            table_before_len = len(table_before)
                            table_after_len = len(table_after)

                            log.debug(f"{table_before_len} and after {table_after_len}")
                            log.debug("Error" + str(error_found))

                            color = 0xFF6E6E if error_found is True else 0x98FB98

                            if table_before_len > 2000:
                                log.error("Table before exceeds 2000 word count. How did this happened?")
                                return

                            if table_after_len > 2000:
                                log.info("Table size was greater than 2000. Commence table split.")
                                log.debug(table_before)

                                embed = discord.Embed(description='```' + table_before + '```', colour=color)
                                embed.set_thumbnail(url=iconURL)
                                embed.set_author(name=server['name'], url='', icon_url='')
                                await message.channel.send(embed=embed)

                                table_contents.clear()
                                table_contents.append([origin,
                                                       routemanager,
                                                       f"{route_pos}/{route_max}",
                                                       formatted_device_last_proto_time
                                                       ])

                        log.debug(f"Sending status table for {server_name}")
                        table_to_send = tabulate(table_contents, headers=table_header)

                        log.debug(table_to_send)

                        # TODO Status colours
                        embed = discord.Embed(description='```' + table_to_send + '```', colour=color)
                        embed.set_thumbnail(url=iconURL)
                        embed.set_author(name=server['name'], url='', icon_url='')
                        await message.channel.send(embed=embed)

def run():
    asyncio.set_event_loop(asyncio.new_event_loop())
    client = MyClient()
    client.run(args.discord_token)


from selenium import webdriver
import time
import os

path = r'D:\phantomjs-2.1.1-windows\bin\phantomjs.exe'
browser = webdriver.PhantomJS(path)

url = 'https://dianying.taobao.com/cinemaDetail.htm?cinemaId=24332&n_s=new'

browser.get(url)
time.sleep(1)
# browser.save_screenshot(r'ceshi.png'))

browser.save_screenshot(r'test1.png')

# 让browser执行简单的js代码，模拟滚动滚动到底部
js = 'document.body.scrollTop=10000'
browser.execute_script(js)

time.sleep(1)

browser.save_screenshot(r'test2.png')

for i in range(11):
    button = browser.find_element_by_class_name('sortbar-more')
    button.click()
    time.sleep(1)

browser.save_screenshot(r'ceshi.png')

# 获取网页的代码，保存到文件中
html = browser.page_source
print(html)

with open(r'tppyuanma.html', 'w', encoding='utf8') as fp:
    fp.write(html)

browser.quit()


import pandas as pd
import numpy as np
import logging
import collections
import theano

import beans.functions.sample
import beans.functions.scale
import beans.functions.knee
import beans.functions.project
import beans.functions.encode

class Development:

    def __init__(self, data: pd.DataFrame, labels: list, target: str):
        """

        :param data: The training data
        :param labels: The list of distinct labels in the target field
        :param target: The target field
        """

        self.data = data
        self.labels = labels
        self.target = target

        logging.basicConfig(level=logging.WARNING, format='%(message)s\n%(asctime)s.%(msecs)03d',
                            datefmt='%Y-%m-%d %H:%M:%S')
        self.logger = logging.getLogger(__name__)

    def sample_(self):
        """
        Over-samples, and re-samples, the training data via SVNSMOTE; ref. beans.functions.sample

        :return:
        """

        sample = beans.functions.sample.Sample()

        return sample.exc(blob=self.data, target=self.target)

    def scale_(self, blob: pd.DataFrame):
        """
        Scales the independent variables of blob

        :param blob:  The data
        :return:
        """

        focus = blob.copy().drop(columns=self.target)

        scale = beans.functions.scale.Scale()
        scaler = scale.exc(blob=focus)
        x_scaled = scale.apply(blob=focus, scaler=scaler)
        scaled = pd.concat((x_scaled, blob[self.target]), axis=1, ignore_index=False)

        return scaled, scaler

    def project_(self, blob: pd.DataFrame):
        """

        :param blob:  The data that will be projected
        :return:
        """

        knee = beans.functions.knee.Knee()
        n_components = knee.exc(blob=blob, target=self.target)
        self.logger.warning('The # of Kernel Principal Components will be set to {}'.format(n_components))

        project = beans.functions.project.Project()
        matrix = blob.drop(columns=self.target).to_numpy()
        projector = project.exc(matrix=matrix, n_components=n_components)
        projected = project.apply(matrix=matrix, vector=blob[self.target], projector=projector)

        return projected, projector

    def encode_(self, blob: pd.DataFrame):
        """
        One-hot-encodes the dependent variable

        :param blob: In general, this will be the DataFrame w.r.t. the final preprocessing step
                                              before structuring
        :return:
        """

        CategoricalData = collections.namedtuple(
            typename='CategoricalData', field_names=['fields', 'arrays'])

        fields = [self.target]
        arrays = [np.array(self.labels)]
        categories = CategoricalData._make((fields, arrays))

        encode = beans.functions.encode.Encode()
        bits = encode.bits(frame=blob, categories=categories)
        encoded = pd.concat((blob.drop(columns=fields), bits), axis=1, ignore_index=False)

        return encoded

    def structure(self, blob: pd.DataFrame):
        """
        Structures the training data according to what is expected by the neural network model

        :param blob:
        :return:
        """

        x_matrix = blob.drop(columns=self.labels).to_numpy()
        y_matrix = blob[self.labels].to_numpy()

        unity = np.ones((x_matrix.shape[0], 1))
        x_points = np.concatenate((unity, x_matrix), axis=1).astype(theano.config.floatX)
        y_points = y_matrix.astype(theano.config.floatX)

        return x_points, y_points


"""
    AN NGUYEN
    A simple Python Flask MVC webapp for Food Carts
    A user can be able to view, submit and delete a food cart location
"""
import flask
from flask.views import MethodView
from index import Index
from view import View
from add import Add

app = flask.Flask(__name__)       # our Flask app

app.add_url_rule('/view/',
                view_func=View.as_view('view'),
                methods=['GET'])

app.add_url_rule('/add/',
                view_func=Add.as_view('add'),
                methods=['GET', 'POST'])

app.add_url_rule('/',
                 view_func=Index.as_view('index'),
                 methods=["GET"])

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=True)


import os
from datetime import datetime
from typing import Any, Callable, Optional

from ..components import Indexer
from ..memory import SlidingWindowStringConcatMemory
from ..symbol import Symbol

class CodeFormatter:
    def __call__(self, value: str, *args: Any, **kwds: Any) -> Any:
        # extract code from chat conversations or ```<language>\n{code}\n``` blocks
        return Symbol(value).extract('Only extract code without ``` block markers or chat conversations')

class Conversation(SlidingWindowStringConcatMemory):
    def __init__(self, init:     Optional[str] = None,
                 file_link:      Optional[str] = None,
                 index_name:     str           = Indexer.DEFAULT,
                 auto_print:     bool          = True,
                 token_ratio:    float         = 0.6, *args, **kwargs):
        super().__init__(token_ratio)
        self.token_ratio = token_ratio
        self.auto_print  = auto_print
        self.file_link   = file_link
        if init is not None:
            self.store_system_message(init, *args, **kwargs)
        if file_link is not None:
            self.store_file(file_link, *args, **kwargs)
        self.indexer = Indexer(index_name=index_name)
        self._index  = self.indexer()

    def store_system_message(self, message: str, *args, **kwargs):
        val = f"[SYSTEM::INSTRUCTION] <<<\n{str(message)}\n>>>\n"
        self.store(val, *args, **kwargs)

    def store_file(self, file_path: str, *args, **kwargs):
        if not os.path.exists(file_path):
            return
        # read in file
        with open(file_path, 'r') as file:
            content = file.read()
        val = f"[DATA::{file_path}] <<<\n{str(content)}\n>>>\n"
        self.store(val, *args, **kwargs)

    def commit(self, formatter: Optional[Callable] = None):
        if self.file_link is not None:
            # if file extension is .py, then format code
            format_ = formatter
            formatter = CodeFormatter() if format_ is None and self.file_link.endswith('.py') else formatter
            val = self.value
            if formatter is not None:
                val = formatter(val)
            # if file does not exist, create it
            with open(self.file_link, 'w') as file:
                file.write(str(val))

    def save(self, path: str, replace: bool = False) -> Symbol:
        return Symbol(self._memory).save(path, replace=replace)

    def index(self, file_path: str):
        return self._index(file_path)

    def forward(self, query: str, *args, **kwargs):
        query = self._to_symbol(query)
        # get timestamp in string format
        timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S:%f")
        # append to string to memory
        val = str(f"[USER::{timestamp}] <<<\n{str(query)}\n>>>\n")
        self.store(val, *args, **kwargs)
        history = Symbol(f'[HISTORY] <<<\n{self._memory}\n>>>\n')
        if 'payload' in kwargs:
            history =  f'{history}\n{kwargs["payload"]}'
            del kwargs['payload']
        res = self.recall(query, payload=history, *args, **kwargs)
        self.value = res.value # save last response
        val = str(f"[ASSISTANT::{timestamp}] <<<\n{str(res)}\n>>>\n")
        self.store(val, *args, **kwargs)
        if self.auto_print:
            print(res)
        return res

    def __repr__(self):
            """Get the representation of the Symbol object as a string.

            Returns:
                str: The representation of the Symbol object.
            """
            return str(self.value)


import logging
import os
import re
import sys

import time
from argparse import ArgumentParser

import warnings

import tqdm

from chunk_generator_context import chunk

warnings.simplefilter(action='ignore', category=FutureWarning)

from simpletransformers.t5 import T5Model, T5Args

logging.basicConfig(level=logging.INFO)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)

if __name__ == '__main__':
    parser = ArgumentParser(description='')
    parser.add_argument('data_path', help='')
    parser.add_argument('save_path', help='')
    parser.add_argument('--model_type', default="mt5", help='Name the model')
    parser.add_argument('--model_name', default="google/mt5-base", help='Name the model')

    parser.add_argument('--batch_size', default=32, type=int, help='mini batch size')
    parser.add_argument('--max_seq_length', default=256, type=int, help='max_seq_length')
    parser.add_argument('--max_length', default=256, type=int, help='')
    parser.add_argument('--num_beams', default=1, type=int, help='')
    parser.add_argument('--repetition_penalty', default=1.0, type=float, help='')

    parser.add_argument('--chunk', default=20, type=int, help='chunk size')
    parser.add_argument('--stride', default=5, type=int, help='stride size')

    parser.add_argument('--top_k', default=50, type=int, help='top_k')
    parser.add_argument('--top_p', default=0.95, type=float, help='top_p')
    parser.add_argument('--no_sample', action='store_true', help='sample')

    cache_dir = os.getenv('SCRATCH', '.') + '/cache/'
    args = parser.parse_args()

    # Configure the model
    model_args = T5Args()

    model_args.fp16 = False  # dont work with True
    model_args.max_length = args.max_length  # 20
    model_args.num_beams = args.num_beams  # 1
    model_args.repetition_penalty = args.repetition_penalty  # 1.0

    model_args.train_batch_size = args.batch_size
    model_args.eval_batch_size = args.batch_size
    model_args.use_multiprocessed_decoding = False

    model_args.max_seq_length = args.max_seq_length

    # gramformer
    model_args.do_sample = not args.no_sample
    model_args.top_k = args.top_k
    model_args.top_p = args.top_p

    time_of_run = time.time()

    model = T5Model(args.model_type, args.model_name, args=model_args)
    # print(model.args)

    # TODO: predict

    # Make predictions with the model
    # to_predict = [
    #     'ocr: Uradowany, Neron złożył dzięki niebiosom, uniesiony wdzięcznością za opiekę bogów.',
    #     'ocr: Przybywszy do Rzymu nie zwołał ani ludu ani senatu.',
    # ]
    #
    # preds = model.predict(to_predict)
    #
    # print(preds)

    try:
        f = open(args.save_path)
        lines=f.readlines()
        lines=lines[:-2]
        f.close()
    except UnicodeDecodeError:
        print('Unicode error - delete last line')
        sys.exit()
    except:
        lines=[]

    print('Lines:', len(lines))

    f=open(args.save_path, 'w')
    f.writelines(lines)
    f.flush()

    for i, hyp_line in enumerate(tqdm.tqdm(open(args.data_path))):
        if i<len(lines): continue

        hyp_line = hyp_line[:-1].replace('-\\n', '').replace('\\n', '\n\\n').replace('\\\\', '\\')
        # doc_id, page_no, year, hyp = hyp_line.split('\t')
        hyp = hyp_line.split('\t')[-1]

        hyp = re.sub(r'(\\n| )+', ' ', hyp)
        hyp = hyp.split()

        xs=[]
        for context_left, start, end, context_right in chunk(hyp, args.chunk, args.stride):
            if args.stride > 0:
                x = ' '.join(hyp[context_left:start]).strip() + ' <left_context> ' + ' '.join(
                    hyp[start:end]).strip() + ' <right_context> ' + ' '.join(
                    hyp[end:context_right]).strip()
            else:
                x = ' '.join(hyp[start:end]).strip()

            x = re.sub(' +', ' ', x)
            xs.append(x)

        # print([f'ocr: {x}' for x in xs])
        preds = model.predict([f'ocr: {x}' for x in xs])

        f.write(' '.join(preds))
        f.write('\n')
        f.flush()


from random import seed, randint

seed(23)

import simpy

class EV:
    def __init__(self, env):
        self.env = env
        self.drive_proc = env.process(self.drive(env))
        self.bat_ctrl_proc = env.process(self.bat_ctrl(env))
        self.bat_ctrl_reactivate = env.event()
        self.bat_ctrl_sleep = env.event()

    def drive(self, env):
        """驾驶进程"""
        while True:
            # 驾驶 20-40 分钟
            print("开始驾驶 时间: ", env.now)
            yield env.timeout(randint(20, 40))
            print("停止驾驶 时间: ", env.now)

            # 停车 1-6 小时
            print("开始停车 时间: ", env.now)
            self.bat_ctrl_reactivate.succeed()  # 激活充电事件
            self.bat_ctrl_reactivate = env.event()
            yield env.timeout(randint(60, 360)) & self.bat_ctrl_sleep  # 停车时间和充电程序同时都满足
            print("结束停车 时间:", env.now)

    def bat_ctrl(self, env):
        """电池充电进程"""
        while True:
            print("充电程序休眠 时间:", env.now)
            yield self.bat_ctrl_reactivate  # 休眠直到充电事件被激活
            print("充电程序激活 时间:", env.now)
            yield env.timeout(randint(30, 90))
            print("充电程序结束 时间:", env.now)
            self.bat_ctrl_sleep.succeed()
            self.bat_ctrl_sleep = env.event()

def main():
    env = simpy.Environment()
    ev = EV(env)
    env.run(until=600)

if __name__ == '__main__':
    main()


"""
Routes and views for the bottle application.
"""

from bottle import route, view
from datetime import datetime

# kilroy added this for static content
from bottle import static_file

@route('/')
@route('/home')
@view('index')
def home():
    """Renders the home page."""
    return dict(
        year=datetime.now().year
    )

@route('/contact')
@view('contact')
def contact():
    """Renders the contact page."""
    return dict(
        title='Contact',
        message='You can contact me at rob 5 at uw dot edu.',
        year=datetime.now().year
    )

@route('/about')
@view('about')
def about():
    """Renders the about page."""
    return dict(
        title='About',
        message='djargon (on the Bottle web framework (Python)) is documentation plus some API testing on Azure.',
        year=datetime.now().year
    )

# This is copycat of '/about' and so on; it doesn't render a static page (which is what I want at the moment)
# @route('/doc')
# @view('doc')
# def doc():
#     """Renders the simple doc page."""
#     return dict(
#         title='Doc',
#         message='Simple documentation page.',
#         year=datetime.now().year
#     )

# kilroy: documentation.html at ./static/html is the older version; but this was a pain to build out of
#   OneNote; took several tedious steps.

@route('/api')
@view('api')
def api_no_args():
    """Renders the api page as documentation"""
    return dict(title='API', message='Format = djargon.a.n/api/command=hydrograph^start=date^end=date^station=stnID^interval=time_interval', year=datetime.now().year)

# kilroy: Major problem is the parse character '&' works on localhost but not on Azure.
#   '+' fails as well but in a different way; karet '^' seems to work.
@route('/api/<command>')
@view('api')
def api(command):
    """Interprets an API call."""
    # kilroy debug: return dict(title='API', message=command, year=datetime.now().year)

    failed = 0
    args = command.split('&')
    thisCmd = 'null'

    for c in args:
        thisArgPair = c.split('=')
        if thisArgPair[0] == 'command':
            thisCmd = thisArgPair[1]
            break

    if thisCmd == 'hydrograph':
        startFound = False
        endFound = False
        stationFound = False
        intervalFound = False
        goodMsg = str(len(args)) + ' args; '
        for i in range(len(args)):
            thisQual = args[i].split('=')
            if len(thisQual) != 2:
                failed = 3
                break
            if thisQual[0] == 'start':
                startFound = True
                goodMsg += 'start ' + thisQual[1] + ', '
            if thisQual[0] == 'end':
                endFound = True
                goodMsg += 'end ' + thisQual[1] + ', '
            if thisQual[0] == 'station':
                stationFound = True
                goodMsg += 'Pardon me boys: Is this the ' + thisQual[1] + ' station?, '
            if thisQual[0] == 'interval':
                intervalFound = True
                goodMsg += 'interval ' + thisQual[1]
        if not startFound: failed = 3
        if not endFound: failed = 4
        if not stationFound: failed = 5
        if not intervalFound: failed = 6
    elif thisCmd == 'music':
        goodMsg = 'You have selected the song How High The Moon. The Sarah Vaughan cover on YouTube is recommended.'
    else:
        failed = 2

    if failed > 0: return dict(title='API', message='Dammit qualifier parse failed with error = ' + str(failed), year=datetime.now().year)
    else:
        return dict(title='API', message=goodMsg, year=datetime.now().year)

# kilroy: in this case we look for a filename fitting the regular expression *\.png where
#   I put hockney.png in the appropriate folder. So url/images/hockney.png or url/image/hockney.png works...
#   but without suitable machinery (kilroy) it is just vestigial.
@route('/images/<filename:re:.*\.png>')
@route('/image/<filename:re:.*\.png>')
def send_image(filename):
    return static_file(filename, root='./static/images', mimetype='image/png')

# kilroy added these routes to bind to static pdf files
#   The better way to do this is with a /pdf/Name approach but I'm in a hurry at the moment.
@route('/pdf/<static_pdf_file>')
def pdf_chooser(static_pdf_file):
    theText = str(static_pdf_file);
    theBase = theText.split('.')[0]
    allowedBases = ['Doc01_PythonWebFrameworkBasics', \
                    'Doc02_Djargon_on_Azure_Add_PDF_Button', \
                    'Doc03_AzureAPIBasics', \
                    'Doc04_AzureContentDeliveryNetworkBasics', \
                    'Doc05_JupyterNotebookBasics', \
                    'Doc06_MapBasics', \
                    'Doc07_BDSUserInterfaceBasics', \
                    'Doc08_BDSQueryBasics', \
                    'Doc09_BDSMetadataBasics', \
                    'Doc10_BDSMATLABFunctionsInMETemplates', \
                    'Doc11_BDSEEMPARAFAC', \
                    'Doc12_BDSDataEgress', \
                    'Doc20_PythonBasics', \
                    'Doc30_DIF_API',\
                    'Doc40_AWS_IOT_Arduino_Yun',\
                    'Doc41_AWSCloudFormationNetworkClusterBasics',\
                    'Doc42_HIPAA_on_AWS',\
                    'Doc43_Jupyter_on_AWS',\
                    'Doc44_Rosetta_on_AWS',\
                    'Doc45_IAM_on_AWS',\
                    'Doc46_EC2_Resources_on_AWS',\
                    'Doc70_OMC_Charter',\
                    'Doc71_OMC_Problems',\
                    'Doc72_OMC_Dialogs',\
                    'Doc73_OMC_FirstHandout',\
                    'Doc74_OMC_ConsentForm',\
                    'Doc75_OMC_TeamProjects',\
                    'Doc76_OMC_Journal',\
                    'Doc77_OMC_Constitution'\
                    ]
    if theBase in allowedBases:
        return static_file(theBase + '.pdf', root='./static/pdf/')


import tensorflow as tf
from tensorflow import keras
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

housing = fetch_california_housing()
X_train_full, X_test, y_train_full, y_test = train_test_split(
    housing.data, housing.target.reshape(-1,1), random_state=42
)
X_train, X_valid, y_train, y_valid = train_test_split(
    X_train_full, y_train_full, random_state=42
)

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_valid_scaled = scaler.transform(X_valid)
X_test_scaled = scaler.transform(X_test)

input_shape = X_train.shape[1:]

model = keras.models.Sequential([
    keras.layers.Dense(30, activation="selu", kernel_initializer="lecun_normal",input_shape=input_shape),
    keras.layers.Dense(1)
])
# def huber_fn(y_true, y_pred):
#     error = y_true-y_pred
#     is_small_error = tf.abs(error)<1
#     squared_loss = tf.square(error) / 2
#     linear_loss = tf.abs(error)-0.5
#     return tf.where(is_small_error,squared_loss,linear_loss)

from tensorflow.keras.losses import Huber
huber=Huber()
model.compile(loss=huber,optimizer="nadam",metrics=["accuracy"])
history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid_scaled,y_valid))

def create_huber(threshold=1.0):
    def huber_fn(y_true, y_pred):
        error = y_true-y_pred
        is_small_error = tf.abs(error) < threshold
        squared_loss = tf.square(error) / 2
        linear_loss = threshold * tf.abs(error)-threshold**2/2
        return tf.where(is_small_error, squared_loss, linear_loss)
    return huber_fn

class HuberLoss(keras.losses.Loss):
    def __init__(self, threshold = 1.0, **kwargs):
        self.threshold = threshold
        super().__init__(**kwargs)
    def call(self, y_true, y_pred):
        error = y_true - y_pred
        is_small_error = tf.abs(error) < self.threshold
        squared_loss = tf.square(error)
        linear_loss = self.threshold * tf.abs(error) - self.threshold**2/2
        return tf.where(is_small_error, squared_loss, linear_loss)
    def get_config(self):
        base_config = super().get_config()
        return {**base_config, "threshold": self.threshold}

model.compile(loss=create_huber(2.0), optimizer="nadam")

class HuberMetric(keras.metrics.Metric):
    def __init__(self, threshold=1.0, **kwargs):
        super().__init__(**kwargs)
        self.threshold = threshold
        self.huber_fn = create_huber(threshold)
        self.total = self.add_weight("total", initializer="zeros")
        self.count = self.add_weight("count", initializer="zeros")
    def update_state(self, y_true, y_pred, sample_weight=None):
        metric = self.huber_fn(y_true, y_pred)
        self.total.assign_add(tf.reduce_sum(metric))
        self.total.assign_add(tf.cast(tf.size(y_true), tf.float32))
    def result(self):
        return self.total/self.count
    def get_config(self):
        base_config = super().get_config()
        return {**base_config, "threshold": self.threshold}

class MyDense(keras.layers.layer):
    def __init__(self, units, activation=None, **kwargs):
        super().__init__(*kwargs)
        self.units = units
        self.activation = keras.activations.get(activation)

    def build(self, batch_input_shape):
        self.kernel = self.add_weight(
            name="kernel", shape=[batch_input_shape[-1], self.units],
            initializer = "glorot_normal")
        self.bias = self.add_weight(
            name="bias", shape=[self.units], initializer="zeros")
        super().build(batch_input_shape)

    def call(self, X):
        return self.activation(X @ self.kernel + self.bias)

    def compute_output_shape(self, batch_input_shape):
        return tf.TensorShape(batch_input_shape.as_lists()[:-1] + [self.units])
    def get_config(self):
        base_config = super().get_config()
        return {**base_config, "units": self.units, "activation": keras.activations.serialize(self.activation)}

class MyMultiLayer(tf.keras.layers.Layer):
    def call(self, X):
        X1, X2 = X
        return [X1+X2, X1*X2, X1/X2]   # outputs of 3
    def compute_output_shape(self, batch_input_shape):
        b1, b2 = batch_input_shape
        return [b1, b1, b1]  # shape of these 3 outputs

class MyGaussianNoise(keras.layers.Layer): # which does exactly the same as tf.keras.layers.GaussianNoise
    def __init__(self, stddev, **kwargs):
        super().__init__(**kwargs)
        self.stddev = stddev
    def call(self, X, training=None):
        if training:
            noise = tf.random.normal(tf.shape(X), steddev=self.stddev)
            return X + noise
        else:
            return X
    def compute_output_shape(self, batch_input_shape):
        return batch_input_shape

class ResidualBlock(tf.keras.layers.Layer):
    def __init__(self, n_layers, n_neurons, **kwargs):
        super().__init__(**kwargs)
        self.hidden = [tf.keras.layers.Dense(n_neurons, activation="elu", kernel_initializer="he_normal")
                       for _ in range(n_layers)]
        def call(self, inputs):
            Z = inputs
            for layer in self.hidden:
                Z = layer(Z)
            return inputs + Z

class ResidualRegressor(keras.Model):
    def __init__(self, output_dim, **kwargs):
        super().__init__(**kwargs)
        self.hidden1 = keras.layers.Dense(30, activation="elu", kernel_initializer="he_normal")
        self.block1 = ResidualBlock(2,30)
        self.block2 = ResidualBlock(2,30)
        self.out = tf.keras.layers.Dense(output_dim)

    def call(self, inputs):
        Z = self.hidden1(inputs)
        for _ in range(1+3):
            Z = self.block1(Z)
        Z = self.block2(Z)
        return self.out(Z)

class ReconstructingRegressor(keras.Model):
    def __init__(self, output_dim, **kwargs):
        super().__init__(kwargs)
        self.hidden = [tf.keras.layers.Dense(30, activation="selu", kernel_initializer="lecun_normal")
                       for _ in range(5)]
        self.out = tf.keras.layers.Dense(output_dim)

    def build(self, batch_input_shape):
        n_inputs = batch_input_shape[-1]
        self.reconstruct = tf.keras.layers.Dense(n_inputs)
        super().build(batch_input_shape)

    def call(self, inputs):
        Z = inputs
        for layer in self.hidden:
            Z = layer(Z)
        reconstruction = self.reconstruct(Z)
        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))
        self.add_loss(0.05*recon_loss)
        return self.out(Z)

#######################################################################
### computing gradients using autodiff
#######################################################################
def f(w1,w2):
    return 3*w1**2 + 2*w1*w2

w1, w2 = tf.Variable(5.), tf.Variable(3.)
with tf.GradientTape(persistent=True) as tape:
    z = f(w1,w2)

dz_dw1 = tape.gradient(z, w1)
dz_dw2 = tape.gradient(z, w2)
print(dz_dw1)
print(dz_dw2)


from PySide import QtCore, QtGui
from shiboken import wrapInstance
import maya.OpenMayaUI as omui
from functools import partial
import pymel.core as pm

import petfactory.util.verify as pet_verify

def maya_main_window():
    main_window_ptr = omui.MQtUtil.mainWindow()
    return wrapInstance(long(main_window_ptr), QtGui.QWidget)

class RemapMaterialWidget(QtGui.QWidget):

    def __init__(self, parent=None):

        self.curr_mat_list = None
        self.import_mat_list = None
        self.namespace = 'lookdev'

        super(RemapMaterialWidget, self).__init__(parent)
        self.setWindowFlags(QtCore.Qt.Tool)

        self.resize(400,400)
        self.setWindowTitle("Import materials")

        # layout
        main_vbox = QtGui.QVBoxLayout()
        main_vbox.setContentsMargins(5,5,5,5)
        self.setLayout(main_vbox)

        # models
        self.mat_model = QtGui.QStandardItemModel()
        self.mat_model.setHorizontalHeaderLabels(['Current Material', 'Imported Material'])

        # tableviews
        # curr mat tableview
        self.mat_tableview = QtGui.QTableView()
        self.mat_tableview.setModel(self.mat_model)
        mat_header = self.mat_tableview.horizontalHeader()
        mat_header.setStretchLastSection(True)
        main_vbox.addWidget(self.mat_tableview)
        self.mat_tableview.setColumnWidth(0, 200)

        self.mat_tableview.setModel(self.mat_model)

        self.add_current_mat_button = QtGui.QPushButton('1. List materials')
        self.add_current_mat_button.clicked.connect(self.add_mat)
        main_vbox.addWidget(self.add_current_mat_button)

        self.import_mat_button = QtGui.QPushButton('2. Import materials')
        self.import_mat_button.clicked.connect(self.import_mat)
        main_vbox.addWidget(self.import_mat_button)

        self.remap_mat_button = QtGui.QPushButton('3. Remap materials')
        self.remap_mat_button.clicked.connect(self.remap_mat)
        main_vbox.addWidget(self.remap_mat_button)

        self.remove_namespace_button = QtGui.QPushButton('4. remove namespace')
        self.remove_namespace_button.clicked.connect(self.remove_namespace)
        main_vbox.addWidget(self.remove_namespace_button)

    def list_object_from_material(self, node):

        # get the shading group
        sg_list = node.listConnections(type='shadingEngine')

        if len(sg_list) < 1:
            return (None, None)

        mesh_list = []
        face_list = []

        # loop through the sg looking for members
        for sg in sg_list:

            member_list = sg.members(flatten=True)

            # loop through the members, check it the mat is assigned to faces or meshes
            for member in member_list:

                if type(member) == pm.nodetypes.Mesh:
                    mesh_list.append(member)

                elif type(member) == pm.general.MeshFace:
                    face_list.append(member)

        return(mesh_list, face_list)

    def add_mat(self):

        self.mat_model.removeRows(0, self.mat_model.rowCount())

        self.curr_mat_list = [m for m in pm.ls(mat=True) if m.name() not in ['lambert1', 'particleCloud1']]

        for curr_mat in self.curr_mat_list:
            curr_mat_item = QtGui.QStandardItem(curr_mat.name())
            self.mat_model.appendRow(curr_mat_item)

    def import_mat(self):

        if self.curr_mat_list is None:
            pm.warning('Add the current materials')
            return

        file_name, selected_filter = QtGui.QFileDialog.getOpenFileName(None, 'Import Materials', None, 'Maya (*.mb *.ma)')

        if not file_name:
            return

        pm.importFile(file_name, namespace=self.namespace)

        mat_list = [m for m in pm.ls(mat=True) if m.name() not in ['lambert1', 'particleCloud1']]
        mat_set_after_import = set(mat_list)
        mat_set_before_import = set(self.curr_mat_list)
        diff_set = mat_set_after_import.difference(mat_set_before_import)

        mat_dict = {}
        for node in diff_set:
            split_name = node.name().split(':')[-1]
            mat_dict[split_name] = node

        num_rows = self.mat_model.rowCount()

        for row in range(num_rows):
            curr_item = self.mat_model.item(row)
            new_mat = mat_dict.get(curr_item.text())

            if new_mat is not None:
                new_item = QtGui.QStandardItem(new_mat.name())
                self.mat_model.setItem(row, 1, new_item)

    def remap_mat(self):

        num_rows = self.mat_model.rowCount()

        for row in range(num_rows):

            curr_mat_item = self.mat_model.item(row)
            if curr_mat_item:
                curr_mat_node = pet_verify.to_pynode(curr_mat_item.text())

            new_mat_item = self.mat_model.item(row, 1)
            if new_mat_item:
                new_mat_node = pet_verify.to_pynode(new_mat_item.text())

            if curr_mat_item is None or new_mat_node is None:
                continue

            mesh_list, face_list = self.list_object_from_material(curr_mat_node)

            new_mat_sg_list = new_mat_node.listConnections(type='shadingEngine')
            # we get an error if we have nultiple shading groups... look into this

            # if we do not have a shading group create it
            if len(new_mat_sg_list) < 1:
                sg = pm.sets(renderable=True, noSurfaceShader=True, empty=True, name='{0}SG'.format(new_mat_unicode))
                new_mat_node.outColor >> sg.surfaceShader
                new_mat_sg = sg

            else:
                new_mat_sg = new_mat_sg_list[0]

            if mesh_list is not None:
                if len(mesh_list) > 0:
                    print(mesh_list)
                    pm.sets(new_mat_sg, forceElement=mesh_list)

    def remove_namespace(self):

        pm.namespace(force=True, mv=(':{0}'.format(self.namespace), ':'))

        # remove the namespace if it exists
        if pm.namespace(exists=self.namespace):

            try:
                pm.namespace(rm=self.namespace)

            except RuntimeError as e:
                pm.warning('Could not remove namespace {0}'.format(self.namespace), e)

def show():
    win = RemapMaterialWidget(parent=maya_main_window())
    win.show()
    return win

'''
try:
    win.close()

except NameError:
    pass

win = show()
win.move(100,150)
'''