#TODO:10)avaldada browseri addon, mis laseks tabis avatud lehekohta käivaid kommentaare lugeda ja kommentaari lisada
#TODO:6)Teha nee, et savestaks  kommentaarid samasse faili olenemata protokollist (http, https, ftp jne.).
#TODO:8)Määrama kommentaaride kuvamisele jäjekorra.
#TODO:9)Panna oma lehel reklaame ja sellega raha teenida.
#TODO:14)teha nii, et kui ühelt iplt on palju kommentaare, siis nende nägemiseks peab klikkama, et näita kõiki.
#TODO:15)tausta värvid õigeks teha.
from flask import Flask, request, redirect, Response,render_template
from datetime import datetime
from re import match
import os
from random import choice

app = Flask(__name__)

HTMLi_algus=["""

        <title>Universaalne kommentaarium</title>

Universaalne Kommentaarium2
english
            <form method="GET">
                See veebiteenus võimaldab teil kommenteerida mistahes uudist mistahes veebiväljaandes või mistahes muud internetilehekülge.
Sisestage lehekülje, mida kommenteerida soovite, URL:<input type="text" name="kommenteeritava_URL">.
                Kuva all kommenteeritav lehekülg<input type="checkbox" name="kuva" checked>.
                <input type="submit" value="kommenteerima">
            </form>""","""

        <title>Universal commentary</title>

Universal commentary
eesti keel
            <form amethod="GET">
                This page offers you opportunity to comment any news in web or any other webpage.
Enter URL of the page you want to comment:<input type="text" name="URL_being_commented">.
                Show the page below<input type="checkbox" name="show" value="show" checked>.
                <input type="submit" value="to commenting">
            </form>"""]
parameeter_kommenteeriava_URL=["kommenteeritava_URL", "URL_being_commented"]
parameeter_kommentaar=["kommentaar","comment"]
parameeter_sisu_hinnang=["sisu","sisu"]
parameeter_kajastuse_hinnang=["kajastus","kajastus"]
kommentaare_ei_ole=["SELLE LEHE KOHTA KOMMENTAARE VEEL EI OLE","HERE ARE NO COMMENTS ABOUT THIS PAGE YET"]
browser_ei_toeta=["TEIE BROWSER EI TOETA HTML TAGi iframe´i","YOUR WEB BROWSER DOES NOT SUPORT HTML iframe TAG"]
HTMLi_postitamise_osa_1=['''
            <form method="POST">
                <textarea style="width:100%;height:120;resize:vertical" name="kommentaar" placeholder="Sinu kommentaar"></textarea>
                Meeldib uudise sisu? <input type="radio" name="sisu" value="meeldib"> Meeldib<input type="radio" name="sisu" value="ei meeldi"> Ei meeldi

                Meeldib kajastus ja juuresolevad kommentaarid? <input type="radio" name="kajastus" value="meeldib"> Meeldib<input type="radio" name="kajastus" value="ei meeldi"> Ei meeldi

                Postitad kommentaari ja annad hinnangu lehe ''','''
            <form method="POST">
                <textarea style="width:100%;height:120;resize:vertical" name="comment" placeholder="Your comment"></textarea>
                Do you like the news? <input type="radio" name="sisu" value="meeldib">Like<input type="radio" name="sisu" value="ei meeldi">Dislike

                Do you like presentation of the news? <input type="radio" name="kajastus" value="meeldib">Like<input type="radio" name="kajastus" value="ei meeldi">Dislike

                You are posting a comment and giving rating about page ''']
HTMLi_postitamise_osa_2=[''' kohta. <input type="submit" value="Postita''',''' . <input type="submit" value="Post''']
aeg=["aeg:","time:"]
hiljuti_kommenteeritud_lehed_tekst=["hiljuti kommenteeritud lehed","recently commented pages"]
parameeter_meeldib=["meeldib","liked"]
parameeter_ei_meeldi=["ei meeldi","disliked"]
parameeter_sisu=["sisu","content"]
parameeter_kajastus=["kajastus","presentation"]
parameeter_lehe=["Lehe ","There are no comments about page "]
parameeter_kohta=[' kohta pole kommentaare. Kuvatud kommentaarid on lehe ',"sown comments are about "]
parameeter_kohta_2=[" kohta.
","
"]

viited={"static/reklaamid/ekre":"https://ekre.ee/",
        "static/reklaamid/uueduudised":"https://uueduudised.ee/",
        "static/reklaamid/SÄ":"https://ekre.ee/noored/",
        "static/reklaamid/SÄ2":"https://www.facebook.com/Sinine2ratus/",
        "static/reklaamid/SÄ3":"https://steamcommunity.com/groups/SinineAratus#members",
        "static/reklaamid/SÄ4":"https://sininearatus.ee/yritused/"}
kuva_reklaami=True
betale_lubatud_ipd=set()

def vii_URL_oigele_kujule(url):
    if not match(r"^(?:f|ht)tps?://", url):
        url="http://"+url
    www_asukoht=match(r"^(?:f|ht)tps?://www.", url)
    if www_asukoht:#kui aadressis on www. algus
        return ((url[:www_asukoht.end()-4]+url[www_asukoht.end():]).lower())#eemaldab URList "www."i
    return url.lower()

def pole_faile(url):
    for kommentaari_fail in sorted(os.listdir(url)):#kui ei sisalda ühtegi faili.
        if os.path.isfile(url+"/"+kommentaari_fail):
            return False
    return True

def vali_suvaline_fail(p="static/reklaamid"):
    try:
        p+="/"+(choice(os.listdir(os.path.join(os.path.dirname(__file__), p))))
        if os.path.isfile(os.path.join(os.path.dirname(__file__), p)):
            return (p,viited[p[:p.rfind("/")]])
        else:
            return vali_suvaline_fail(p)
    except IndexError:
        return vali_suvaline_fail()

@app.route('/')#suunab eestikeelsele pealehele
def ainult_domeen():
    return(redirect("/et"))

@app.route('/et', methods=['GET', 'POST'])
def pealeht_et():
    return pealeht(0)

@app.route('/eng', methods=['GET', 'POST'])
def pealeht_eng():
    return pealeht(1)

def pealeht(keel):
    if parameeter_kommenteeriava_URL[keel] in request.args:#Juhul kui päringus on sees URL, mida klient kommenteerida tahab. näiteks "http://objektiiv.ee/"
        kommentaaride_kaust=vii_URL_oigele_kujule(request.args[parameeter_kommenteeriava_URL[keel]])
        if parameeter_kommentaar[keel] in request.form:#juhul kui päringus on kommentaar, mida klient postitada tahab.
            hiljuti_kommenteeritud_lehtede_fail=open("hiljuti kommenteeritud lehed")
            hiljuti_kommenteeritud_lehed="".join(hiljuti_kommenteeritud_lehtede_fail.readlines()[1:])
            hiljuti_kommenteeritud_lehtede_fail.close()
            hiljuti_kommenteeritud_lehtede_fail=open("hiljuti kommenteeritud lehed","w")
            hiljuti_kommenteeritud_lehtede_fail.write(hiljuti_kommenteeritud_lehed+"\n"+kommentaaride_kaust.replace("<","&lt").replace("\r","").replace("\n"," "))#paneb lehe hiljuti kommenteeritud lehtede hulka
            hiljuti_kommenteeritud_lehtede_fail.close()
            hinnang=0# kui sisu hindamata: 0,1,2 ;kui sisu meeldib: 6,7,8; kui sisu ei meeldi 3,4,5
            # kui kajastus hindamata: 0,3,6 ;kui kajastus meeldib: 2,5,8; kui kajastus ei meeldi 1,4,7
            if parameeter_sisu_hinnang[keel] in request.form:
                if request.form[parameeter_sisu_hinnang[keel]]=="meeldib":
                    hinnang=6
                elif request.form[parameeter_sisu_hinnang[keel]]=="ei meeldi":
                    hinnang=3
                    # print(request.form[parameeter_sisu_hinnang[keel]])
            if parameeter_kajastuse_hinnang[keel] in request.form:
                if request.form[parameeter_kajastuse_hinnang[keel]]=="meeldib":
                    hinnang+=2
                elif request.form[parameeter_kajastuse_hinnang[keel]]=="ei meeldi":
                    hinnang+=1
                    #print(request.form[parameeter_kajastuse_hinnang[keel]])
            if os.path.exists(kommentaaride_kaust):
                if os.path.isfile(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))):#on vaja faili enne kommentaari reavahetus vaja panna.
                    if hinnang:#!=0
                        vana_hinnang=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))).readline()#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                        if vana_hinnang==hinnang:
                            #print("ei muuda hinnangut.")
                            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                            f.write("\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                        else:
                            #print("uus hinnang on",hinnang,"vana sisu:",vana_hinnang)
                            vana_sisu=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))).read()[1:]
                            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"w")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                            f.write(str(hinnang)+"\n"+vana_sisu[1:]+"\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                            del vana_sisu
                    else:
                        f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                        f.write("\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                    #print("fail olemas")
                else:
                    f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                    f.write(str(hinnang)+"\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
                    #print("faili pole")
            else:#kui selle lehe kohta veel kommentaare ei ole.#ei ole vaja faili algusesse reavahetust.
                os.makedirs(kommentaaride_kaust)#teeb vastava kausta
                f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
                f.write(str(hinnang)+"\n"+str(datetime.now())+request.form[parameeter_kommentaar[keel]].replace("<","&lt").replace("\r","").replace("\n"," "))
            f.close()
        html=HTMLi_algus[keel]+HTMLi_postitamise_osa_1[keel]+kommentaaride_kaust+HTMLi_postitamise_osa_2[keel]+"""!">
            </form>"""
        vähendatud_URL=kommentaaride_kaust
        on_algne_url=True
        while not os.path.exists(vähendatud_URL) or pole_faile(vähendatud_URL):#alamprogramm pole_faile on juhuks kui kaust olemas, aga selle sees on on ainult kaustad mitte failid(kommentaarid)
            #print("url:",vähendatud_URL)
            if vähendatud_URL.count("/")<3:
                html+='
'+"
"+kommentaare_ei_ole[keel]+"
"
                html_2=""
                break
            vähendatud_URL=vähendatud_URL[:vähendatud_URL.rfind("/")]
            on_algne_url=False
        else:
            if on_algne_url:
                html+='
'
            else:
                html+='
\n\t\t\t
'+parameeter_lehe[keel]+kommentaaride_kaust+parameeter_kohta[keel]+vähendatud_URL+parameeter_kohta_2[keel]#Lisab lehele märke, et kommentaarid pole täpse URLi kohta.
            sisu_meeldimisi=0
            sisu_mitte_meeldimisi=0
            kajastuse_meeldimisi=0
            kajastuse_mitte_meeldimisi=0
            html_2=""
            for kommentaari_fail in sorted(os.listdir(vähendatud_URL)):#loeb failidest kommentaare, et need html'i panna, mis kliendile saadetakse.
                if os.path.isfile(vähendatud_URL+"/"+kommentaari_fail):
                    html_2+="""

ip:"""+kommentaari_fail+"
"
                    fail=open(vähendatud_URL+"/"+kommentaari_fail)
                    #LIKEDE LOENDAMINE
                    hinnang=int(next(fail))
                    if hinnang//3==2:
                        sisu_meeldimisi+=1
                        html_2+='
'+parameeter_sisu[keel]+':

'+parameeter_meeldib[keel]+'
'
                    elif hinnang//3==1:
                        sisu_mitte_meeldimisi+=1
                        html_2+='
'+parameeter_sisu[keel]+':

'+parameeter_ei_meeldi[keel]+'
'
                    if hinnang%3==2:
                        kajastuse_meeldimisi+=1
                        html_2+='
'+parameeter_kajastus[keel]+':

'+parameeter_meeldib[keel]+'
'
                    elif hinnang%3==1:
                        kajastuse_mitte_meeldimisi+=1
                        html_2+='
'+parameeter_kajastus[keel]+':

'+parameeter_ei_meeldi[keel]+'
'
                    html_2+="\n\t\t\t\t\t
"
                    for kommentaar in fail:#paneb tühjade postituste ajad samale reale.
                        html_2+=aeg[keel]+kommentaar[:19]
                        if kommentaar[26:].strip("\n"):
                            html_2+="""
\n\t\t\t\t\t
"""+kommentaar[26:].strip("\n")+"
\n\t\t\t\t\t
"#paneb tarbetu realõpu kui [26:]. jätab viimase kommentaari viimase tähe ära kui [26:-1]
                        else:
                            html_2+=" ; "
                    fail.close()
                    html_2=html_2[:-5]+"

\n"
            if on_algne_url:
                try:
                    html_2+='\t\t\t\t
'+str(sisu_mitte_meeldimisi)+'
'
                    html+='\n\n'+str(sisu_meeldimisi)+'
'
                    #print("sisu keskmine hinnang:",sisu_meeldimisi)
                except ZeroDivisionError:
                    #print("sisu pole hinnatud")
                    try:
                        #print("kajastuse keskmine hinnang:",kajastuse_meeldimisi/(kajastuse_meeldimisi+kajastuse_mitte_meeldimisi))
                        html_2+='\t\t\t\t
'+str(kajastuse_mitte_meeldimisi)+'
'
                        html += '\n\n'+str(kajastuse_meeldimisi) + '
'
                    except ZeroDivisionError:
                        #print("kajastust pole hinnatud")
                        html_2+='\t\t\t\t

'
        html_2+='''

'''
        if "kuva" in request.args:
            html_2+='
'+browser_ei_toeta[keel]+'
'#TODO:4)panna iframes´is tehtud clickid muutma kommenteeritavat URLi
        return (html+html_2+'''

''')
    html=HTMLi_algus[keel]+"""

"""+hiljuti_kommenteeritud_lehed_tekst[keel]+":
"
    for i in open("hiljuti kommenteeritud lehed"):
        i=i.strip()
        html+="\n
"+i+"
"
    if False:#request.headers.get('X-Real-IP') in betale_lubatud_ipd:#kuva_reklaami:
        path,url=vali_suvaline_fail()
        html+="""

reklaam

        """
    else:
        html+="""

        """#pealeht
    print(html)
    return html+"""

"""

@app.route("/et/info")
def info_et():
    return """

            <title>Info universaalse kommentaariumi kohta</title>

Info universaalse kommentaariumi kohta
english

Lehe www.kommentaarium.ee loomise eesmärk on pakkuda võimalust kommenteerida teisi internetilehekülgi(eelkõige mõeldud uudiste kommenteerimiseks). Teatavasti mitmed internetiportaalid ei paku endapoolt võimalust enda uudiste kommenteerimiseks, mõnedes neist peab kommenteerimiseks sisse logima, mõnedes kustutatakse anonüümsed kommentaarid automaatselt mingi aja möödudes(postimehes näiteks) ja mõnedes väidetavalt kustutakse kommentaare lähtuvalt nendes väljendatavast poliitilisest vaatest.

        """
@app.route("/eng/info")
def info_eng():
    return """

            <title>About Universal commentary</title>

About Universal commentary
eesti keel

Purpous of creating webpage www.kommentaarium.ee is to offer oppurtunity to comment other webpages(especialy news). Many news portals do not offer commentary on theyr own page and on some you have to login to comment.

        """

@app.route("/et/plugin/info")#/info võib URLi lopust ära võtta.
def plugina_info_et():
    return """

        <title>Universaalse kommentaariumi plugina kohta</title>

Universaalse kommentaariumi plugina kohta
english

plugin võimaldab www.kommentaarid.ee teenust mugavamini kasutada.

"""
@app.route("/en/plugin/info")#/info võib URLi lopust ära võtta.
def plugina_info_en():
    return """

        <title>About add-on of www.kommentaarid.ee</title>

About add-on of www.kommentaarid.ee
eesti keel

The add-on offers you a interface to post and read comments about pages, that you are visiting.

"""

@app.route('/reklaam')
def miski():
    return render_template('reklaam/reklaam.html')

@app.route('/en/derivation')
def suvaline_pilt():
    return "\n"

@app.route('/portfelio')
def portfelio():
    return render_template('portfoolio/e-portfoolio.html')

"""@app.route('/betale',methods=["GET","POST"])
def render_static():
    global betale_lubatud_ipd
    if request.headers.get('X-Real-IP') in betale_lubatud_ipd:
        return(redirect("/"))
    if "salasona" in request.form:
        if request.form["salasona"]=="cdlijwdedwq":
            betale_lubatud_ipd.add(request.headers.get('X-Real-IP'))
            return (redirect("/"))
        else:
            return "vale salasõna"
    return render_template('/betale vorm.html')"""

@app.route("/plugin",methods=["GET"])
def plugina_data():
    if b"\n" in request.data:
        kommentaaride_kaust=vii_URL_oigele_kujule(request.data.split(b"\n",1)[0].decode("utf-8"))
        if os.path.exists(kommentaaride_kaust):
            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')),"a")#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serveris on vaja str funktsiooni kasutatda?
            if os.path.isfile(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP'))):#on vaja faili enne kommentaari reavahetus vaja panna.
                f.write("\n"+str(datetime.now())+request.data.split(b"\n",1)[1].decode("utf-8").replace("<", "&lt").replace("\r","").replace("\n"," "))
            else:
                f.write(str(datetime.now())+request.data.split(b"\n",1)[1].decode("utf-8").replace("<", "&lt").replace("\r","").replace("\n"," "))
        else:#kui selle lehe kohta veel kommentaare ei ole.#ei ole vaja faili algusesse reavahetust.
            os.makedirs(kommentaaride_kaust)#teeb vastava kausta
            f=open(kommentaaride_kaust+"/"+str(request.headers.get('X-Real-IP')))#request.environ["REMOTE_ADDR"],jsonify({'ip': request.remote_addr},request.remote_addr,request.headers.get('X-Real-IP')#kas serceris on vaja str funktsiooni kasutatda?
            f.write(str(datetime.now())+request.data.split(b"\n",1)[1].decode("utf-8").replace("<", "&lt").replace("\r","").replace("\n"," "))
        f.close()
        return "1"
    else:
        kommentaarid=""
        kommentaaride_kaust=vii_URL_oigele_kujule(request.data.decode("utf-8"))
        if os.path.exists(kommentaaride_kaust):#kui selle lehe kohta on kommentaare.
            for kommentaari_fail in os.listdir(kommentaaride_kaust):#loeb failidest kommentaare, et need html'i panna, mis kliendile saadetakse.
                if os.path.isfile(kommentaaride_kaust+"/"+kommentaari_fail):
                    fail=open(kommentaaride_kaust+"/"+kommentaari_fail)
                    kommentaarid+=kommentaari_fail+"\n"+fail.read()+"\n\n"
                    fail.close()
        return Response(kommentaarid[:-1], mimetype='text/plain')

@app.route("/sissejuhatuse-projekt",methods=["GET","POST"])
def galerii():
    return render_template('index.html')
@app.route("/sissejuhatuse-projekt2",methods=["GET","POST"])
def galerii2():
    return render_template('galerii2.html')

if __name__ == '__main__':
    app.run()


# -*- coding: utf-8 -*-

import json
import torch
import argparse
from network import Network
from shakespeare_dataset import encode_text, decode_text, create_one_hot_matrix
from pathlib import Path
from torch import nn

##############################
##############################
## PARAMETERS
##############################
parser = argparse.ArgumentParser(description='Generate sonnet starting from a given text')

parser.add_argument('--seed', type=str, default='o, romeo, romeo!', help='Initial context')
parser.add_argument('--length', type=str, default='500', help='Number of produced characters')

##############################
##############################
##############################

##############################
##############################
## SAMPLING FROM SOFTMAX
##############################

torch.manual_seed(3)

def sample(net_out, temperature):
    EPSILON = 10e-16 # to avoid taking the log of zero
    out_temp = (net_out+EPSILON)/temperature
    preds = nn.functional.softmax(out_temp, dim=1)
    preds_tens = torch.as_tensor(preds).float()
    probas = torch.multinomial(preds_tens, 1)
    return probas.item()

##############################
##############################
##############################

if __name__ == '__main__':

    ### Parse input arguments
    args = parser.parse_args()

    #%% Load training parameters
    model_dir = Path('model')
    print ('Loading model from: %s' % model_dir)
    training_args = json.load(open(model_dir / 'training_args_OPT.json'))

    #%% Load encoder and decoder dictionaries
    number_to_char = json.load(open(model_dir / 'number_to_char_OPT.json'))
    char_to_number = json.load(open(model_dir / 'char_to_number_OPT.json'))

    #%% Initialize network
    net = Network(input_size=training_args['alphabet_len'],
                  hidden_units=training_args['hidden_units'],
                  layers_num=training_args['layers_num'])

    #%% Load network trained parameters
    net.load_state_dict(torch.load(model_dir / 'net_params_OPT.pth', map_location='cpu'))
    net.eval() # Evaluation mode (e.g. disable dropout)

    #%% Define temperature
    temperature = 0.1

    #%% Find initial state of the RNN
    with torch.no_grad():
        # Encode seed
        seed_encoded = encode_text(char_to_number, args.seed)
        # One hot matrix
        seed_onehot = create_one_hot_matrix(seed_encoded, training_args['alphabet_len'])
        # To tensor
        seed_onehot = torch.tensor(seed_onehot).float()
        # Add batch axis
        seed_onehot = seed_onehot.unsqueeze(0)
        # Forward pass
        net_out, net_state = net(seed_onehot)
        # Sample from softmax last output index
        next_char_encoded = sample(net_out[:, -1, :], temperature)
        # Print the seed letters
        print(args.seed, end='', flush=True)
        print(number_to_char[str(next_char_encoded)])

    #%% Generate sonnet
    new_line_count = 0
    tot_char_count = 0
    while True:
        with torch.no_grad(): # No need to track the gradients
            # The new network input is the one hot encoding of the last chosen letter
            net_input = create_one_hot_matrix([next_char_encoded], training_args['alphabet_len'])
            net_input = torch.tensor(net_input).float()
            net_input = net_input.unsqueeze(0)
            # Forward pass
            net_out, net_state = net(net_input, net_state)
            # Sample from softmax last output index
            next_char_encoded = sample(net_out[:, -1, :], temperature)
            # Decode the letter
            next_char = number_to_char[str(next_char_encoded)]
            print(next_char, end='', flush=True)
            # Count total letters
            tot_char_count += 1
            # Break if n letters
            if tot_char_count > int(args.length):
                break

sequence modeling with RNN/code/trained_model.py


# saddle node bifurcation diagram 1 (dx/dt = r+x^2)

import numpy as np
import matplotlib.pyplot as plt

# Data for plotting
x = np.arange(-2.5, 2.5, 0.01)
f1 = -0.5*x-x**3/3

# Note that using plt.subplots below is equivalent to using
# fig = plt.figure and then ax = fig.add_subplot(111)
fig, ax = plt.subplots()
ax.plot(x, f1)

ax.grid()

plt.axvline(0,color='black')
plt.axhline(0,color='black')

ax.set_xlabel("$x$",fontsize=16)
ax.set_ylabel("$U(x)$",fontsize=16)
ax.yaxis.labelpad=-2
#ax.set_xticks([-2,-1,0,1,2])

#ax.set_yticks([-1.0,-0.5,0,0.5,1.0])
#ax.set_xlim([-1.1,1.1])

#ax.plot(-1,-0.666,'bo',markersize=15)
#ax.plot(1,0.666,'bo',fillstyle='none',markersize=15)

ax.set_title("$r>0$",fontsize=16)

fig.savefig("1d-bif-snex2.pdf")
plt.show()


from asyncio import get_event_loop
from logging import Logger, basicConfig, getLogger
from typing import Coroutine, Optional

from ..base import Data

basicConfig(level=Data.LOGGER)
log: Logger = getLogger("dispatch")

class Listener:
    """
    A class representing how events become dispatched and listened to.

    :ivar asyncio.AbstractEventLoop loop: The coroutine event loop established on.
    :ivar dict events: A list of events being dispatched.
    """

    def __init__(self) -> None:
        self.loop = get_event_loop()
        self.events = {}

    def dispatch(self, name: str, *args, **kwargs) -> None:
        r"""
        Dispatches an event given out by the gateway.

        :param name: The name of the event to dispatch.
        :type name: str
        :param \*args: Multiple arguments of the coroutine.
        :type \*args: typing.list[typing.Any]
        :param \**kwargs: Keyword-only arguments of the coroutine.
        :type \**kwargs: dict
        :return: None
        """
        for event in self.events.get(name, []):
            self.loop.create_task(event(*args, **kwargs))
            log.debug(f"DISPATCH: {event}")

    def register(self, coro: Coroutine, name: Optional[str] = None) -> None:
        """
        Registers a given coroutine as an event to be listened to.
        If the name of the event is not given, it will then be
        determined by the coroutine's name.

        i.e. : async def on_guild_create -> "ON_GUILD_CREATE" dispatch.

        :param coro: The coroutine to register as an event.
        :type coro: typing.Coroutine
        :param name: The name to associate the coroutine with. Defaults to None.
        :type name: typing.Optional[str]
        :return: None
        """
        _name: str = coro.__name__ if name is None else name
        event = self.events.get(_name, [])
        event.append(coro)

        self.events[_name] = event
        log.debug(f"REGISTER: {self.events[_name]}")


from PIL import Image, ImageFont, ImageDraw
import numpy as np
import grpc
import tensorflow as tf
from tensorflow_serving.apis import predict_pb2,prediction_service_pb2_grpc
import os
from grpc._cython.cygrpc import CompressionAlgorithm,CompressionLevel
import colorsys

def Client(object):
    def __init__(self,address,class_names,size=416,options=[
            ('grpc.max_receive_message_length', -1),
            ('grpc.default_compression_algorithm', CompressionAlgorithm.gzip),
            ('grpc.default_compression_level', CompressionLevel.high)
        ]):
        channel = grpc.insecure_channel(address, options)
        self.stub=prediction_service_pb2_grpc.PredictionServiceStub(channel)
        self.size=size
        self.class_names=class_names

    def predict(self,image):
        scale = self.size / max(image.size)
        if scale < 1:
            image = image.resize((int(line * scale) for line in image.size),
                                 Image.BILINEAR)
        image_data = np.array(image, dtype='uint8')
        image_data = np.expand_dims(image_data, 0)
        request = predict_pb2.PredictRequest()
        request.model_spec.name = 'detection'
        request.model_spec.signature_name = 'serving_default'
        request.inputs['predict_image:0'].CopyFrom(
            tf.make_tensor_proto(image_data))
        result = self.stub.Predict(request)
        out_classes = np.array(result.outputs['yolo/classes:0'].int_val)
        out_classes = [self.class_names[c] for c in out_classes]
        out_scores = np.array(result.outputs['yolo/scores:0'].float_val)
        out_boxes = np.array(result.outputs['yolo/boxes:0'].int_val).reshape(-1,4)
        return out_classes,out_scores,out_boxes


import pandas as pd
import tia.bbg.datamgr as dm
import numpy as np
import scipy as sp
##from googlefinance.client import get_price_data, get_prices_data, get_prices_time_data
##from pandas_datareader import data
##from alpha_vantage.timeseries import TimeSeries
import datetime
import math
import fetch_data_from_bbg

root_bbg = "YCGT00"
end_bbg  = " Index"
country_codes = {}
country_codes["US"] = "YCGT0025 Index"
country_codes["GER"] = "YCGT0016 Index"
country_codes["JPY"] = "YCGT0018 Index"
country_codes["UK"] = "YCGT0022 Index"
countries    = ["US", "UK", "GER", "JPY"]
def inv_index(my_map):
        return {v:k for k,v in my_map.items()}

def get_bbg_data(tickers, field):
        mgr   = dm.BbgDataManager()
        sids  = mgr[tickers]
        today = datetime.date.today()
        df    = sids.get_historical(field, today-datetime.timedelta(days = 200), today)
        return df

## file On the run yields.xlsx contains the tickers
def get_field_from_bbg(excel_file, bbg_field):
    pd_tick = df      = pd.read_excel(excel_file)
    tickers = df['Ticker'].tolist()
    return get_bbg_data(tickers, bbg_field)

def get_tickers_for_countries(countries):
        return [country_codes[country] for country in countries]

def get_historical_otr_bonds(countries, str_date):
        bbg_tickers = get_tickers_for_countries(countries)
        return fetch_data_from_bbg.main(bbg_tickers, ["CURVE_TENOR_RATES"], ["CURVE_DATE"], [str_date])

def get_columns_from_tuple_list(tuple_list):
        index_to_country = inv_index(country_codes)
        columns          = []

        for tupl in tuple_list:
                columns = columns + [index_to_country[tupl[0]] + tupl[1]]
        return sorted(list(set(columns)))
'''
def get_elements_from_data(tuple_list):
        unique_elements = []
        for tupl in tuple_list:
                unique_elements = unique_elements + [tupl[5]]
        return (sorted(list(set(unique_elements))))
'''

def create_date_time_range(numdays):
        base = datetime.date.today()
        date_list = [base-datetime.timedelta(days = x) for x in range(0, numdays)]
        return date_list

def add_element_from_bbg_to_df(tuple_list, bbg_date, which_df, df):
        index_to_country = inv_index(country_codes)
        for tupl in tuple_list:
                column = index_to_country[tupl[0]] + tupl[1]
                bbg   = from_string_to_date(bbg_date)
                val    = tupl[which_df]
                df.at[bbg, column] = val

        return df

def from_date_to_bbg(date):
        start = str(date.year)
        if(date.month <10):
                start = start + "0" + str(date.month)
        else:
                start = start + str(date.month)
        if (date.day<10):
                start = start + "0" + str(date.day)
        else:
                start = start + str(date.day)
        return start

def from_string_to_date(date_str):
        year  = int(date_str[:4])
        month = int(date_str[4:6])
        day   = int(date_str[-2:])

        return datetime.date(year, month, day)

def add_all_historical_elements(bbg_data_list, df_ask,df_mid, df_bid ):
        for element in bbg_data_list:
                tuple_list = get_historical_otr_bonds(countries, element)
                add_element_from_bbg_to_df(tuple_list, element,2,df_ask)
                add_element_from_bbg_to_df(tuple_list, element,3,df_mid)
                add_element_from_bbg_to_df(tuple_list, element,4,df_bid)
        return df_ask, df_mid, df_bid

date_ix      = sorted(create_date_time_range(20))
bbg_api_date = [from_date_to_bbg(norm_date) for norm_date in date_ix]
tuple_list   = get_historical_otr_bonds(countries, bbg_api_date[-1])
columns      = get_columns_from_tuple_list(tuple_list)
df_ask       = pd.DataFrame(index = date_ix, columns = columns)
df_mid       = pd.DataFrame(index = date_ix, columns = columns)
df_bid       = pd.DataFrame(index = date_ix, columns = columns)
df_ask, df_mid, df_bid = add_all_historical_elements(bbg_api_date,df_ask, df_mid, df_bid)
print (df_ask)

def return_bid_ask_mid():
##          return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
        return df_ask, df_mid, df_bid
##print (tuple_list)
##print (get_elements_from_data(tuple_list))
##print (get_columns_from_tuple_list(tuple_list))


from ops.forms import ServiceForm, AffectedForm, ServiceVehicleForm, ArrestForm, ArrestPaymentForm
from django.shortcuts import render_to_response, redirect
from django.template.context import RequestContext
from django.contrib.auth.decorators import login_required
from django.forms.formsets import formset_factory
from ops.models import ServiceVehicle, ServiceAffected, Service, ArrestPayment
from personal.models import Firefighter
from common.models import BasePerson, TelephoneNumber, PersonTelephoneNumber
from django.contrib import messages
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.exceptions import ValidationError
from django.db import transaction
import json
from datetime import datetime

@login_required
def list_services(request):
    services_qs = Service.objects.all()
    paginator = Paginator(services_qs, 15)
    page = request.GET.get('page')
    try:
        services = paginator.page(page)
    except PageNotAnInteger:
        services = paginator.page(1)
    except EmptyPage:
        services = paginator.page(paginator.num_pages)

    return render_to_response("list_services.html", RequestContext(request, {"services": services, "paginator": paginator}))

@login_required
def view_service(request, service_id):
    service = Service.objects.get(id=service_id)
    return render_to_response("view_service.html", RequestContext(request, {"service": service}))

@login_required
@transaction.commit_on_success
def insert_service(request):
    params = {}
    AffectedFormSet = formset_factory(AffectedForm, extra=0)
    VehiclesFormSet = formset_factory(ServiceVehicleForm, extra=1)
    firefighter = request.user.get_profile()
    service_form = ServiceForm()
    crew_dict = {}

    if request.method == 'POST':
        data = request.POST.copy()
        data['time'] = data['time'][0:-2]+":"+data['time'][-2:]

        service_form = ServiceForm(data)
        affected_formset = AffectedFormSet(data, prefix='affected')
        vehicles_formset = VehiclesFormSet(data, prefix='vehicles')
        if service_form.is_valid() and affected_formset.is_valid() and vehicles_formset.is_valid():
            service = service_form.save()
            service.created_by = firefighter
            service.save()

            for vf in vehicles_formset.forms:
                if "lead" in vf.cleaned_data:
                    s_vehicle = ServiceVehicle()
                    s_vehicle.service = service
                    s_vehicle.lead = Firefighter.objects.get(id=vf.cleaned_data['lead'])
                    s_vehicle.vehicle = vf.cleaned_data['vehicle']
                    if vf.cleaned_data['driver']:
                        s_vehicle.driver = Firefighter.objects.get(id=vf.cleaned_data['driver'])

                    s_vehicle.save()

                    for crew_id in vf.cleaned_data['crew_ids'].split(","):
                        if crew_id != "":
                            s_vehicle.crew.add(Firefighter.objects.get(id=crew_id))

            for af in affected_formset.forms:
                if 'first_name' in af.cleaned_data:
                    if af.cleaned_data["id_document"]:
                        person, _ = BasePerson.objects.get_or_create(id_document=af.cleaned_data["id_document"])
                    else:
                        person = BasePerson()
                    person.first_name = af.cleaned_data['first_name']
                    person.first_name_2 = af.cleaned_data['first_name_2']
                    person.last_name = af.cleaned_data['last_name']
                    person.last_name_2 = af.cleaned_data['last_name_2']
                    person.gender = af.cleaned_data['gender']
                    if af.cleaned_data["primary_email"]:
                        person.primary_email = af.cleaned_data["primary_email"]

                    person.save()
                    if af.cleaned_data["phone_code"] and af.cleaned_data["phone_number"]:
                        telephone = TelephoneNumber(code=af.cleaned_data["phone_code"],
                                                    number=af.cleaned_data["phone_number"])
                        telephone.save()
                        PersonTelephoneNumber(person=person,type='O',
                                              telephone_number=telephone).save()

                    s_affected = ServiceAffected(person_affected=person,
                                                 notes=af.cleaned_data["notes"],
                                                 type=af.cleaned_data["type"])
                    s_affected.save()
                    service.affected.add(s_affected)
            messages.success(request, u'El servicio fue guardado exitosamente')
            return redirect(list_services)
        else:
            crew_ids_str = ""
            for k, v in data.iteritems():
                if "crew_ids" in k and v!="":
                    crew_ids_str = crew_ids_str+","+v
            crew_ids = [x for x in crew_ids_str.split(",") if x!='']

#            import ipdb;ipdb.set_trace()
            crew = Firefighter.objects.filter(id__in=crew_ids)
            for member in crew:
                crew_dict[member.id] = str(member)
    else:
        affected_formset = AffectedFormSet(prefix='affected')
        vehicles_formset = VehiclesFormSet(prefix='vehicles')

    params['form'] = service_form
    params['affected'] = affected_formset
    params['vehicles'] =  vehicles_formset
    params['media'] = service_form.media
    params['ff'] = firefighter
    params['crew_data'] = json.dumps(crew_dict)

    return render_to_response("insert_service.html", RequestContext(request, params))

@login_required
@transaction.commit_on_success
def insert_arrest(request):
    params = {}
    arrest_form = ArrestForm()
    firefighter = request.user.get_profile()

    if request.method == 'POST':
        data = request.POST.copy()
        arrest_form = ArrestForm(data)
        if arrest_form.is_valid():
            arrest = arrest_form.save(commit=False)
            arrest.created_by = firefighter
            arrest.arrested = Firefighter.objects.get(id=arrest_form.cleaned_data['arrested'])
            arrest.save()
            messages.success(request, u'El arresto fue guardado exitosamente')
            return redirect(insert_arrest)

    params['arrest_form'] = arrest_form
    return render_to_response("insert_arrest.html", RequestContext(request, params))

@login_required
@transaction.commit_on_success
def insert_arrest_payment(request):
    params = {}
    arrest_payment_form = ArrestPaymentForm()
    firefighter = request.user.get_profile()

    if request.method == 'POST':
        data = request.POST.copy()
        data['start_time_time'] = data['start_time_time'][0:-2]+":"+data['start_time_time'][-2:]
        data['end_time_time'] = data['end_time_time'][0:-2]+":"+data['end_time_time'][-2:]

        arrest_payment_form = ArrestPaymentForm(data)
        if arrest_payment_form.is_valid():
            arrest_payment = ArrestPayment()
            arrest_payment.created_by = firefighter
            arrest_payment.start_time = datetime.combine(arrest_payment_form.cleaned_data['start_time_date'], arrest_payment_form.cleaned_data['start_time_time'])
            arrest_payment.end_time = datetime.combine(arrest_payment_form.cleaned_data['end_time_date'], arrest_payment_form.cleaned_data['end_time_time'])
            arrest_payment.payer = Firefighter.objects.get(id=arrest_payment_form.cleaned_data['payer'])
            try:
                arrest_payment.full_clean()
                arrest_payment.save()
                messages.success(request, u'El pago arresto fue guardado exitosamente')
                return redirect(insert_arrest_payment)
            except ValidationError as e:
                messages.error(request, e.message_dict["__all__"][0])

    params['arrest_payment_form'] = arrest_payment_form
    return render_to_response("insert_arrest_payment.html", RequestContext(request, params))


import pickle
import numpy as np
import scipy
from scipy.cluster.hierarchy import linkage, dendrogram
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from matplotlib.offsetbox import TextArea, DrawingArea, OffsetImage, AnnotationBbox
from typing import List, Any
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import floyd_warshall

# Load pADELM results
basin_labels, basin_minima, basin_minima_energies, item_basin_barriers = pickle.load(open('adelm_7/ADELM_dispatch.pkl', 'rb'))

item_count = [0 for _ in range(len(basin_minima))]

for basin_label in basin_labels:
  item_count[basin_label] += 1

# collect barriers between basins
n_basin = len(basin_minima)
barriers = np.zeros([n_basin, n_basin]) + np.inf
barrier_indices = np.arange(len(barriers))

for i_item in range(len(item_basin_barriers)):
  i_basin = basin_labels[i_item]
  for j_basin in range(len(item_basin_barriers[i_item])):
    barriers[i_basin, j_basin] = min(item_basin_barriers[i_item][j_basin], barriers[i_basin, j_basin])
    if i_basin == j_basin:
      barriers[i_basin, j_basin] = basin_minima_energies[i_basin]

# remove disconnected basins
for _ in range(10):
  disconnected_basins = np.isinf(barriers).sum(1) == len(barriers)-1
  barriers = barriers[~disconnected_basins][:,~disconnected_basins]
  barrier_indices = barrier_indices[~disconnected_basins]

sparse_barrier = csr_matrix(barriers.shape)
for i in range(len(barriers)):
  for j in range(len(barriers)):
    if not np.isinf(barriers[i,j]):
      sparse_barrier[i,j] = barriers[i,j]

shortest_distance, shortest_path = floyd_warshall(sparse_barrier, directed=False, return_predecessors=True, unweighted=True)
disconnected_basins = (shortest_path == -9999).sum(0) == len(shortest_path)-1
disconnected_basins[26] = True
barriers = barriers[~disconnected_basins][:,~disconnected_basins]
barrier_indices = barrier_indices[~disconnected_basins]

for i_basin in range(len(barriers)):
  for j_basin in range(len(barriers)):
    barriers[i_basin, j_basin] = min(barriers[i_basin, j_basin], barriers[j_basin, i_basin])

# collect minimum energy barriers between basins
def consolidate(barriers):
  # input: N x N mat
  sparse_barrier = csr_matrix(barriers.shape)
  for i in range(len(barriers)):
    for j in range(len(barriers)):
      if not np.isinf(barriers[i,j]):
        sparse_barrier[i,j] = barriers[i,j]
  shortest_distance, shortest_path = floyd_warshall(sparse_barrier, directed=False, return_predecessors=True, unweighted=True)
  min_barrier = barriers.copy()
  for i in range(len(barriers)):
    for j in range(len(barriers)):
      path = []
      while j != i:
        path.append(j)
        j = shortest_path[i,j]
      path.append(i)
      path.reverse()
      __min_barrier = -np.inf
      for cur_id in range(len(path)-1):
        next_id = cur_id + 1
        __min_barrier = max(__min_barrier, barriers[path[cur_id], path[next_id]])
        min_barrier[i, path[next_id]] = min(__min_barrier, min_barrier[i, path[next_id]])
  return min_barrier

barriers = consolidate(barriers)
barriers_backup = barriers.copy()

# prepare datastructure for visualization
# output: Z: N x 4: left-child, right-child, height, num-children
Z = []
total_examples = len(barriers)
for _iter in range(total_examples-1):
  # find next closest pair
  min_bar_idx = (barriers + np.eye(len(barriers)) * 1000 ).argmin()
  i, j = min_bar_idx // len(barriers), min_bar_idx % len(barriers)
  # find which one is the new cluster minima
  i_energy = barriers[i,i]
  j_energy = barriers[j,j]
  B = barriers[i,j]
  minima_energy = min(i_energy, j_energy)
  minima_idx = i
  if i_energy > j_energy:
    minima_idx = j
  # append new cluster to barriers and set the two selected entries to inf
  barriers = np.concatenate([barriers, barriers[[i,j]].min(0, keepdims=True)], axis=0)
  barriers = np.concatenate([barriers, barriers[:,[i,j]].min(1, keepdims=True)], axis=1)
  barriers[i,:] = np.inf
  barriers[j,:] = np.inf
  barriers[:,i] = np.inf
  barriers[:,j] = np.inf
  if i < total_examples:
    N = 1
  else:
    N = Z[i-total_examples][3]
  if j < total_examples:
    N += 1
  else:
    N += Z[j-total_examples][3]
  Z.append([i,j,B,N])

barriers = barriers_backup.copy()

def collect_instances(cluster_id):
  if cluster_id < total_examples:
    return [cluster_id]
  left, right, energy, num_nodes = Z[cluster_id - total_examples]
  result = collect_instances(int(left)) + collect_instances(int(right))
  assert len(result) == num_nodes
  return result

def get_example_energy(i):
  return barriers[i,i]

# draw image
R = dendrogram(Z, leaf_font_size=12, show_leaf_counts=True, no_plot=True)

xs = np.array(R['icoord'])
ys = np.array(R['dcoord'])
fig = plt.figure(figsize=(16,9))
ax = fig.add_subplot(111)
minima = []
i_leaf = 0
min_y = 0

for i in range(len(R['ivl'])-1):
  x = xs[i]
  y = ys[i]
  y0 = False
  y3 = False
  if y[0] == 0:
    i_leaf = int((x[0] - 5) / 10)
    y[0] = get_example_energy(R['leaves'][i_leaf])
    minima.append(y[0])
    y0 = True
  if y[3] == 0:
    i_leaf = int((x[3] - 5) / 10)
    y[3] = get_example_energy(R['leaves'][i_leaf])
    minima.append(y[3])
    y3 = True
  _ = ax.plot(x, y, c='k')
  if y0:
    _ = ax.scatter(x[0], y[0], s=item_count[barrier_indices[R['leaves'][i_leaf]]] * 10, facecolor='white', edgecolor='black')
  if y3:
    _ = ax.scatter(x[3], y[3], s=item_count[barrier_indices[R['leaves'][i_leaf]]] * 10, facecolor='white', edgecolor='black')

ax.spines['bottom'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.axes.get_xaxis().set_visible(False)
plt.show()
# fig.savefig('dendrogram_no_text.svg')

# pickle.dump([Z, R], open('dendrogram_5p_7.pkl', 'wb'))
# plt.show()


#!/usr/bin/env python
# coding: utf-8

# #  Job Hunt Skills
# ## Using Python, Pandas, Plotly, Requests, and Beautiful Soup
# ### branched from Jeff Hale's Data Science Skills Code
# ### 2020 October 20

# Import necessary libraries
import pandas as pd
import numpy as np
from bs4 import BeautifulSoup as bs
import requests
import matplotlib.pyplot as plt
import datetime

# # Objective
# Show prioritization of skills for certain jobs
# This will be based on regularly scraped data from online job listings.
# Initially start with Indeed, Dice, Monster, SimplyHired.
# Glassdoor and # AngelCo , LinkedIn with dynamic content will be a later iteration
#
# A legal footnote about scraping:
# Basically, LinkedIn's claim that HiQ Labs scraping action constituted hacking was ruled incorrect based on the CFAA (Computer Fraud and Abuse Act) given the information scraped is basically public and HiQ (or any scraper) did not create any substantial circumvention of any authorized access (the job listing is not even protected by username/password).
# https://cdn.ca9.uscourts.gov/datastore/opinions/2019/09/09/17-16783.pdf
#

# ## Scrape the data from Online Job Sites
# The header to pass and the search terms to iterate through with "<job title>" are below.
# Add other search terms if you like.

ERR_VALUE = 0
WEB_TIMEOUT = 10

DEBUG = 16
ERROR = 8
WARN  = 4
INFO  = 2
QUIET = 0

verbose = DEBUG

def verbose(level, msg):
    """
        control how much console information is shown
        improve sophistication later
    """
    if (level & DEBUG):
        print(msg)
    elif (level & INFO):
        print(msg)
    return

def SaveData(filename, save_list, file_header="LIST"):
    """
     IMPORTANT: for convenience the first skill entry is NULL STRING by design for generic skill query
    """
    tdf = pd.DataFrame({file_header:save_list})
    tdf = tdf.fillna('deliberately_null')
    verbose(INFO, "\nSaving ... " + filename)
    tdf.to_csv(filename, index=False, header=True)
    return

def LoadData(filename):
    """
    Load CSV and return a list e.g. of skills or jobs
    >>> listOfSkills = LoadData("Skill_2020-10-20")
    """
    tdf = pd.read_csv(filename)
    verbose(INFO, "\nLoading ... "+filename)
    tdf = tdf.apply(lambda x:x.str.replace('deliberately_null', '').astype(object), axis=1)
    tdf = tdf.fillna('')
    # assume significant list is always first column
    return list(tdf[tdf.columns[0]])

# -----------------------------------------------------------------------------------------

# do extra study on User-Agent and other header information relevant to web requests
header = {
  "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36",
  "X-Requested-With": "XMLHttpRequest"
}

# # Indeed
def ScrapeIndeed(job_titles, search_terms, location):
    """
        Specialized Scrape Logic specific to Indeed.com job listing site format
        returns count per skills/search_terms psoted for job_titles in location

        for Exception cases e.g. text not found if timedout connection or search failed
        we append ERR_VALUE to list, to make sure the list from search terms are of equal length
        and we can even manually back fetch data later
    """
    verbose(INFO,"\n### Analyzing Indeed.com ###")
    for job in job_titles:
        indeed_list=[]
        for term in search_terms:
            url = f'https://www.indeed.com/jobs?q=%22{job}%22+%22{term}%22&l={location}'
            verbose(DEBUG, "\tSearching skill [{t}] for Job=[{j}]...".format(t=term,j=job))
            try:
                r = requests.get(url, headers=header, timeout=WEB_TIMEOUT)
                soup = bs(r.text, 'html.parser')
                count_str = soup.find('div', id="searchCountPages").get_text()
                numb = count_str.split()
                # replace all commas in the number string to make it ok to convert to integer
                nresults = numb[-2].replace(",","")
                indeed_list.append(int(nresults))
            except Exception as e:
                indeed_list.append(ERR_VALUE)  # always add to list so each search term has an entry even when not found
                verbose(INFO, f'error: {e}')
    return indeed_list

# # Monster
def ScrapeMonster(job_titles, search_terms, location):
    verbose(INFO, "\n### Anayzing Monster.com ###")
    for job in job_titles:
        monster_list = []
        for term in search_terms:
            # Monster.com assumes USA location - let us do location here some other time
            url = f'https://www.monster.com/jobs/search/?q=__22{job}__22-__22{term}__22'
            verbose(INFO, "\tSearching skill [{}] for Job=[{}]...".format(term, job))
            try:
                r = requests.get(url, headers=header, timeout=WEB_TIMEOUT)
                soup = bs(r.text, 'html.parser')
                count_str = soup.find('h2', class_="figure").get_text()
                numb = count_str.split()
                # cannot just convert to int count_str due to fancy formats e.g. with comma etc.
                monster_count = numb[0].replace("(", "")
                monster_list.append(int(monster_count))
            except Exception as e:
                monster_list.append(ERR_VALUE) # always add to list so each search term has an entry even when not found
                verbose(INFO, f'error: {e}')
    return monster_list

# # SimplyHired

def ScrapeSimplyHired(job_titles, search_terms, location):
    verbose(INFO, "\n### Analyzing SimplyHired.com ###")
    for job in job_titles:
        simply_list = []
        for term in search_terms:
            url = f'https://www.simplyhired.com/search?q=%22{job}%22+%22{term}%22&l={location}'
            verbose(DEBUG, "\tSearching skill=[{}] for Job=[{}]...".format(term, job))
            try:
                r = requests.get(url, headers=header, timeout=WEB_TIMEOUT)
                soup = bs(r.text, 'html.parser')
                count_str = soup.find('span', class_="CategoryPath-total").get_text()
                # each job site has its own fancy formats for job count e.g. with comma etc.
                count_str = count_str.replace(",","")
                simply_list.append(int(count_str))
            except Exception as e:
                simply_list.append(ERR_VALUE) # always add to list so each search term has an entry even when not found
                verbose(INFO, f'error: {e}')
    return simply_list

def ScrapeSites(job_titles, search_terms, location, output_file="data.csv"):
    """
        Assemble the dataframe tabulation of jobs skills and sites and write out to file
    """
    df = pd.DataFrame(index=search_terms)
    # Changes to special characters, e.g. C# and C++ as these get escaped in query strings**
    # ** change C# as ASCII C%23 and C++  as ASCII C%2B%2B will be universally ok*
    # two styles of replacing special characters for C# and C++
    search_terms= [term.replace("+","%2B") for term in search_terms]
    search_terms = list(map(lambda st : str.replace(st, "#","%23"),search_terms))

    df['Indeed'] =  ScrapeIndeed(job_titles, search_terms, location)
    df['Monster'] = ScrapeMonster(job_titles, search_terms, location)
    df['SimplyHired'] = ScrapeSimplyHired(job_titles, search_terms, location)
    # temporarily assume one job_tile and use that as heading for skills index
    df.rename(index={'':job_titles[0]}, inplace=True)
    verbose(INFO, "Number of Skills:{} Catalogued :{} rows of {} entries".format(len(search_terms), len(df), df.size))
    # This process below is not necessary if integer conversion is done during each search
    #   The advantage below is it is a more efficient batch conversion assuming we only worry about comma
    #   df = df.apply(lambda x:x.str.replace(',', '').astype(int32), axis=1)
    df.head()
    df.to_csv(output_file)
    return df

def ScrapeJobs():
    # data.csv format: include in filename context of data
    today = datetime.date.today()
    # improve this later - temporarily look for Data Scientist job
    job_header = "Position"
    job_file = f'Jobs_{today}.csv'
    skill_header = "Skills"
    skill_file = f'{skill_header}_{today}.csv'
    search_terms = LoadData(skill_file)

    # job_titles = LoadData(job_file)
    # use hard coded job and location first
    job_titles=["Data Scientist"]
    job = job_titles[0]
    location = "United States"
    data_file = f'{job}_{location}_{today}.csv'
    ScrapeSites(job_titles, search_terms, location, data_file)
    return

def main():
    # ----- Assume these files were saved originally
    # SaveData(skill_file, search_terms, skill_header)
    # SaveData(job_file, job_titles, job_header)
    # --------------------------
    verbose( INFO, "\nLet us go job hunting...")
    ScrapeJobs()

if __name__ == "__main__" : main()