#! /usr/bin/env python
import sys
import os
# sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + "/../../")

import datetime, traceback, shutil, inspect
import logging
import logging.config
from PIL import Image
from textwrap import wrap
from time import strftime, sleep, time
from subprocess import check_call, CalledProcessError, DEVNULL

from conf.serverconf import KEGG_DATA_DIR, CLIENT_TMP_DIR, DOWNLOAD_DELAY_1, DOWNLOAD_DELAY_2, MAX_TRIES_1, MAX_TRIES_2
from scripts.downloadReactome import *

VERSION=0.12

#------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------
#---  MAIN FUNCTIONS                                                                   ----
#------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------

def download_command(inputfile=None, specie=None, kegg=0, mapping=0, common=0, retry=0, reactome = 0, updateReactome = 0):
    """
    Download the information for given species
    Usage: AdminTools.py download <options>
    Examples:
              ./DBManager.py download --specie=mmu --kegg=1 --mapping=1 --common=0 --reactome=0 --updateReactome=0

    Keyword arguments:
        from_file -- a file containing a list of a list of species IDs (one per line), followed by (tabulated)
                            + [0,1]: download Kegg data for the specie
                            + [0,1]: download Mapping data,  where [0 = FALSE,1 = TRUE]
                            default ""
                            e.g.
                                mmu   1   0
                                hsa   1   1
                                ...
        specie    -- a valid KEGG specie code e.g. mmu, hsa

        kegg      -- (optional) download the KEGG data for the given specie. Default=0
        mapping   -- (optional) download the Mapping data for the given specie. Default=0
        common    -- (optional) 1 if Pathways info (classification, PNG images...) should be downloaded, 0 to keep from previous version. Default=0
        retry     -- (optional) 1 to retry the installation of ERRONEOUS SPECIES from previous version, 0 to ignore them. Default=0
    """
    if inputfile is None and specie is None:
        print("Organisms not specified, please type ./DBManager.py download -h for help")
        exit(-1)

    #**************************************************************************
    #STEP 1. READ CONFIGURATION AND PARSE INPUT FILES
    #**************************************************************************
    readConfigurationFile()
    download_dir = KEGG_DATA_DIR + "download/"
    downloadLog= download_dir + "download.log"
    currentStep = 0
    SPECIES_DOWNLOAD = None
    DOWNLOADED_SPECIES = []
    FAILED_SPECIES = []

    #Create the log files
    os.system("touch " + downloadLog)
    summary = open(download_dir + 'summary.log', 'w')

    if inputfile is None:
        n=3 #Download mapping and kegg
        if mapping == 0:
            n-=1 #Do not download mapping
        if kegg == 0:
            n-=2 #Do not download kegg
        SPECIES_DOWNLOAD= {specie : n} #THE IDS FOR THE SPECIES TO UPDATE
    #Check install options
    else:
        SPECIES_DOWNLOAD= readFile(inputfile) #THE IDS FOR THE SPECIES TO UPDATE

    log("######################################################################" )
    log("### PAINTOMICS 4.0 - DATABASE KEGG DOWNLOADER ")
    log("### v."+ str(VERSION))
    log("######################################################################" )
    log("")
    log("Download log is at: " + downloadLog)
    log("")
    log("STEP " + str(currentStep) + ". READ CONFIGURATION AND PARSE INPUT FILES..." )
    log("       - " + str(len(SPECIES_DOWNLOAD.keys())) + " new organisms will be downloaded." )
    log("")

    if((common==None and confirm(prompt='Download common KEGG information (pathway names, classifications, PNG images,...)?', resp=False)) or (common== "1")):
        common = True

    #********************************************************************************
    #STEP 2. IF WE CHOSE TO DOWNLOAD THE GENERAL DATA (PATHWAYS CLASSIFICATION, ETC.) -> GO TO 2.A
    #        OTHERWISE --> GO TO 2.B
    #********************************************************************************
    currentStep+=1

    #STEP 2.A Download common data
    if common:
        datadir = os.path.join(download_dir, "common/")
        try:
            #STEP 2.A.0 INITIALIZE THE NEW DIRECTORY
            if os.path.isdir(datadir):
                shutil.rmtree(datadir)
            os.mkdir(datadir)
            # Add the flag file "DOWNLOADING"
            version = open(datadir + "DOWNLOADING",'w')
            version.write("# DOWNLOAD STARTS:" + strftime("%Y%m%d %H%M"))
            version.close()
            log('')
            log("New data will be stored at " + datadir)
            log("STEP " + str(currentStep) + ". DOWNLOAD THE COMMON KEGG INFORMATION")

            if updateReactome:
            	checkExists = False
            else:
            	checkExists = True

            if reactome:
                reactomeURL = "https://reactome.org/download/current/"
                downloadFile( URL=reactomeURL,
                              fileName="ReactomePathwaysRelation.txt",
                              outputName=datadir + "ReactomePathwaysRelation.list",
                              delay=DOWNLOAD_DELAY_1,
                              maxTries=MAX_TRIES_1,
                              checkIfExists=checkExists )

                downloadFile( URL=reactomeURL,
                              fileName="UniProt2Reactome_PE_All_Levels.txt",
                              outputName=datadir + "UniProt2Reactome_PE_All_Levels.txt",
                              delay=DOWNLOAD_DELAY_1,
                              maxTries=MAX_TRIES_1,
                              checkIfExists=checkExists )

                downloadFile( URL=reactomeURL,
                              fileName="ChEBI2Reactome_PE_All_Levels.txt",
                              outputName=datadir + "ChEBI2Reactome_PE_All_Levels.txt",
                              delay=DOWNLOAD_DELAY_1,
                              maxTries=MAX_TRIES_1,
                              checkIfExists=checkExists )

                downloadFile( URL=reactomeURL,
                              fileName="Ensembl2Reactome_PE_All_Levels.txt",
                              outputName=datadir + "Ensembl2Reactome_PE_All_Levels.txt",
                              delay=DOWNLOAD_DELAY_1,
                              maxTries=MAX_TRIES_1,
                              checkIfExists=checkExists )

                downloadFile( URL=reactomeURL,
                              fileName="NCBI2Reactome_PE_All_Levels.txt",
                              outputName=datadir + "NCBI2Reactome_PE_All_Levels.txt",
                              delay=DOWNLOAD_DELAY_1,
                              maxTries=MAX_TRIES_1,
                              checkIfExists=checkExists )

                downloadFile( URL="rest.kegg.jp/conv/compound/",
                              fileName="chebi",
                              outputName=datadir + "kegg2chebi",
                              delay=DOWNLOAD_DELAY_1,
                              maxTries=MAX_TRIES_1,
                              checkIfExists=True )

            #STEP 2.A.1 DOWNLOAD THE DATA FILES
            downloadKEGGFile("              * LIST OF ORGANISMS", downloadLog, "http://rest.kegg.jp/list/organism", datadir, "organisms_all.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
            downloadKEGGFile("              * PATHWAYS CLASSIFICATION", downloadLog,  "http://rest.kegg.jp/get/br:br08901", datadir, "pathways_classification.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
            downloadKEGGFile("              * LIST OF REFERENCE PATHWAYS", downloadLog,  "http://rest.kegg.jp/list/pathway", datadir, "pathways_all.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
            downloadKEGGFile("              * LIST OF COMPOUND NAMES", downloadLog,  "http://rest.kegg.jp/list/compound", datadir, "compounds_all.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
            downloadKEGGFile("              * PATHWAY to COMPOUND TABLE", downloadLog,  "http://rest.kegg.jp/link/pathway/compound", datadir, "pathway2compound.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)

            #STEP 2.A.2 DOWNLOAD THE PNG IMAGES
            pathways = readFile(datadir + "pathways_all.list", {"forced": True, "forcedColumn": 0})
            total = len(pathways.keys())
            log("             DETECTED " + str(total) + " REFERENCE PATHWAYS " + calculateAproxTime(total, DOWNLOAD_DELAY_2 + 3))
            os.mkdir(datadir + "png/")
            os.mkdir(datadir + "png/thumbnails")

            i=1
            for pathway in pathways.keys():
                pathway = pathway.replace("path:","")
                downloadKEGGFile("                     - " + pathway + " [" + str(i) + "/" + str(total) + "]", downloadLog,  "http://rest.kegg.jp/get/"+ pathway+"/image", datadir + "png/", pathway + ".png",  DOWNLOAD_DELAY_2, MAX_TRIES_1)
                generateThumbnail(datadir + "png/" +  pathway + ".png")
                i+=1

            #STEP 2.A.3 REMOVE THE DOWNLOADING FLAG AND ADD THE VERSION FILE
            os.remove(datadir + "DOWNLOADING")
            version = open(datadir + "VERSION",'w')
            version.write("# DOWNLOAD DATE:\t" + strftime("%Y%m%d %H%M"))
            version.close()
            DOWNLOADED_SPECIES.append("common")
            summary.write('\tDOWNLOAD\tSUCCESS\tcommon\n')
            log("DOWNLOAD COMMON KEGG INFORMATION... SUCCESS\n")

        except Exception as e:
            if os.path.isdir(download_dir + "error/common"):
                shutil.rmtree(download_dir + "error/common")
            if os.path.isdir(datadir):
                shutil.move(datadir, download_dir + "error/")

            log("        FAILED WHILE DOWNLOADING/COPYING COMMON KEGG INFORMATION. UNABLE TO CONTINUE. ABORTING!!")
            summary.write('FAILED WHILE DOWNLOADING/COPYING COMMON INFORMATION. UNABLE TO CONTINUE')
            errorlog(e)
            summary.close()
            exit(1)

    currentStep+=1

    #**************************************************************************
    #STEP 2B. GET DATA FOR "TO UPDATE" SPECIES
    #**************************************************************************
    log('')
    log("STEP " + str(currentStep) + ". DOWNLOADING THE INFORMATION FOR THE SELECTED ORGANISMS")
    specie_code_list = SPECIES_DOWNLOAD.keys()
    total =  str(len(specie_code_list))
    log("       - " + str(total) + " new organisms will be downloaded." )

    step=0
    for specie in specie_code_list:
        if specie[0] == "#":
            log("    IGNORING " + specie[1:] + "...")
            continue

        datadir = os.path.join(download_dir, specie + "/")
        try:
            #STEP 2.B.0 INITIALIZE THE NEW DIRECTORY
            if os.path.isdir(datadir):
                shutil.rmtree(datadir)
            os.mkdir(datadir)

            # Add the flag file "DOWNLOADING"
            version = open(datadir + "DOWNLOADING",'w')
            version.write("# DOWNLOAD STARTS:" + strftime("%Y%m%d %H%M"))
            version.close()

            step+=1
            log("")
            log("New data will be stored at " + datadir)
            log("        DOWLOADING  " + specie + "...")

            kegg_errors = "";
            mapping_errors = "";

            if reactome:
                currentStep += 1
                log( "STEP " + str( currentStep ) + " Download Reactome Files..." )
                downloadReactome( specie )

            # STEP 2.B.1 IF USER SPECIFIED THAT KEGG DATA SHOULD BE DOWNLOADED, DOWNLOAD THE KEGG DATA, OTHERWISE COPY PREVIOUS DATA (IF EXISTS)
            # 2 = updateKegg, 3 = updateKegg && updateMapping
            if(SPECIES_DOWNLOAD[specie] > 1 or (not os.path.exists(KEGG_DATA_DIR + "current/" + specie))):
                os.mkdir(datadir + "kgml")
                kegg_errors = getSpecieKeggData(specie, downloadLog, datadir, str(step)+ "/" + total)
            else:
                log("COPYING PREVIOUS KEGG DATA FOR " + specie + "...")
                shutil.rmtree(datadir)
                shutil.copytree(KEGG_DATA_DIR + "current/" + specie, datadir, symlinks=True)# COPYT THE ENTIRE DIRECTORY
                shutil.rmtree(datadir + "mapping")
                # Add the flag file "DOWNLOADING"
                version = open(datadir + "DOWNLOADING", 'w')
                version.write("# DOWNLOAD STARTS:" + strftime("%Y%m%d %H%M"))
                version.close()
                if os.path.isfile(datadir + "VERSION"):
                    os.remove(datadir + "VERSION")

            # STEP 2.B.2 IF SELECTED, GET THE MAPPING DATA, OTHERWISE COPY PREVIOUS DATA
            # 1=updateMapping, 3 = updateKegg && updateMapping

            if(SPECIES_DOWNLOAD[specie] == 1 or SPECIES_DOWNLOAD[specie] == 3 or (not os.path.exists(KEGG_DATA_DIR + "species/" + specie + "/mapping/"))):
                log("DOWNLOADING MAPPING DATA...")
                os.mkdir(datadir + "mapping")
                mapping_errors = getSpecieMappingData(specie, downloadLog, datadir + "mapping/", str(step)+ "/" + total, ROOT_DIRECTORY + "AdminTools/scripts/")
            else:
                log("COPYING PREVIOUS MAPPING DATA...")
                shutil.copytree(KEGG_DATA_DIR + "current/" + specie + "/mapping", datadir + "mapping", symlinks=True)# COPYT THE ENTIRE DIRECTORY

            #IF SOMETHING WENT WRONG DURING THE DOWNLOAD BUT THE PROCESS CONTINUED (TOLERANCE)
            if kegg_errors != "" or mapping_errors != "":
                log("Errors detected during the download for organism " + specie)
                log("  - Errors during KEGG data download: " + kegg_errors)
                log("  - Errors during MAPPING data download: " + mapping_errors)
                log("The organism will be moved to the erroneous directory but could be valid for installation.")
                raise Exception("Errors detected during the download for organism " + specie + ". Aborting.")

            # STEP 2.B.3 REMOVE THE DOWNLOADING FLAG AND ADD THE VERSION FILE
            os.remove(datadir + "DOWNLOADING")
            version = open(datadir + "VERSION",'w')
            version.write("# DOWNLOAD DATE:\t" + strftime("%Y%m%d %H%M"))
            version.close()

            DOWNLOADED_SPECIES.append(specie)
            summary.write(specie + '\tDOWNLOAD\tSUCCESS\t' + str(SPECIES_DOWNLOAD[specie]) + '\n')
            log("DOWNLOAD  "+ str(SPECIES_DOWNLOAD[specie]) + " " + specie + "...SUCCESS\n")

        except Exception as e:
            if os.path.isdir(download_dir + "error/" + specie):
                shutil.rmtree(download_dir + "error/" + specie)
            if os.path.isdir(datadir):
                shutil.move(datadir, download_dir + "error/")
            summary.write(specie + '\tDOWNLOAD\tERROR\t' + str(SPECIES_DOWNLOAD[specie]) + '\n')
            log("DOWNLOAD  "+ str(SPECIES_DOWNLOAD[specie]) + " " + specie + "...ERROR\n")
            errorlog(e)
            FAILED_SPECIES.append(specie)

    currentStep+=1

    #**************************************************************************
    #STEP 6. CLOSING LOG FILES, GENERATING VERSION FILE
    #**************************************************************************
    log('')
    log("STEP " + str(currentStep) + ". CLOSING LOG FILES, GENERATING VERSION FILE")
    summary.close()

    version = open(download_dir + 'VERSION','w')
    version.write("# CREATION DATE:\t" + strftime("%Y%m%d %H%M")+"\n\n")
    version.write("######################################################################\n")
    version.write("#### THIS FILE WAS CREATED USING PAINTOMICS DATABASE GENERATOR    ####\n")
    version.write("######################################################################\n\n")
    version.write("----------------------------------------------------------------------\n\n")
    version.write("# DOWNLOADED SPECIES\n\n")
    version.write("\n".join(wrap("\t".join(DOWNLOADED_SPECIES), 40)))
    version.write("\n\n")
    version.write("----------------------------------------------------------------------\n\n")
    version.write("# ERRONEOUS SPECIES\n\n")
    version.write("\n".join(wrap("\t".join(FAILED_SPECIES), 40)))
    version.write("\n\n")
    version.write("----------------------------------------------------------------------\n\n")
    version.close()

    if len(FAILED_SPECIES) > 0 and len(DOWNLOADED_SPECIES) > 0 :
        exit(2)
    elif len(FAILED_SPECIES) > 0:
        exit(1)
    else:
        exit(0)

def install_command(inputfile=None, specie=None, common=0, hub=1):
    """
    Install the information for given species
    Usage: AdminTools.py install <options>
    Examples:
              ./DBManager.py install --specie=mmu --common=0 --hub=1

    Keyword arguments:
        from_file -- a file containing a list of a list of species IDs (one per line) to be installed
        specie    -- a valid KEGG specie code e.g. mmu, hsa
        common    -- (optional) 1 if Pathways info (classification, PNG images...) should be reinstalled, 0 to keep from previous version. Default=0
    """
    if inputfile == None and specie == None:
        print("Organisms not specified, please type ./DBManager.py install -h for help")
        exit(-1)

    readConfigurationFile()

    #**************************************************************************
    #STEP 1. READ CONFIGURATION AND PARSE INPUT FILES
    #**************************************************************************
    currentDataDir = os.path.join(KEGG_DATA_DIR, "current/")
    downloadDir = os.path.join(KEGG_DATA_DIR, "download/")
    oldDataDir = os.path.join(KEGG_DATA_DIR, "old/")
    if not os.path.exists(oldDataDir):
        os.mkdir(oldDataDir)

    # Add hub analysis result dir
    hubDir = os.path.join(downloadDir, specie.lower() + "/hubData/")

    if not os.path.exists(hubDir):
        os.makedirs(hubDir)

    installLog= currentDataDir + "install.log"
    summary = open(currentDataDir + 'summary.log','a')
    currentStep = 1;

    log("######################################################################" )
    log("### PAINTOMICS 4.0 - DATABASE INSTALLER ")
    log("### v." + str(VERSION))
    log("######################################################################" )
    log("")
    log("Installation log is at: " + installLog)
    log("")
    log("STEP " + str(currentStep) + ". READ CONFIGURATION AND PARSE INPUT FILES..." )

    INSTALLED_SPECIES = []
    ERRONEOUS_SPECIES = []

    SPECIES_INSTALL = None
    if inputfile != None:
        SPECIES_INSTALL= readFile(inputfile) #THE IDS FOR THE SPECIES TO UPDATE
    else:
        SPECIES_INSTALL= {specie : 1} #THE IDS FOR THE SPECIES TO UPDATE

    INSTALLED_PREVIOUS = getCurrentInstalledSpecies() #THE PREVIOUSLY DOWNLOADED SPECIES

    sleep(2)
    log("       - " + str(len(SPECIES_INSTALL.keys())) + " new organisms will be installed." )
    # log("       - " + str(len(DOWNLOADED_PREVIOUS)) + " organisms were downloaded on previous executions." )
    log("       - " + str(len(INSTALLED_PREVIOUS)) + " organisms were installed on previous executions." )
    # log("       - " + str(len(ERRONEOUS_PREVIOUS)) + " organisms failed during the installation on previous executions." )
    log("")

    if((common==None and confirm(prompt='Do you want to install the common KEGG information (compound names, pathway names, ...)?', resp=False)) or (common== 1)):
        common = True

    #**************************************************************************
    #STEP 2. INSTALLING KEGG GLOBAL/HUB DATA
    #**************************************************************************

    #********************************************************************************
    #STEP 2.A.1 IF WE CHOOSED TO install THE hub analysis data
    #********************************************************************************
    try:
        if hub:
            log("STEP EXTRA: INSTALLING HUB ANALYSIS INFORMATION...")
            check_call(
                [
                    ROOT_DIRECTORY + "AdminTools/scripts/hubAnalysisInstall.R",
                    '--organism="' + specie + '"',
                    '--scriptDir="' + ROOT_DIRECTORY + "AdminTools/scripts/" + '"',
                    '--outputDir="' + hubDir + '"'
                ],stderr=STDOUT,stdout=DEVNULL
            )
    except Exception as e:
        log("        FAILED WHILE INSTALLING HUB ANALYSIS INFORMATION. UNABLE TO CONTINUE. ABORTING!!")
        summary.write('FAILED WHILE INSTALLING HUB ANALYSIS INFORMATION. UNABLE TO CONTINUE. ABORTING!!')
        errorlog(e)
        summary.close()
        exit(1)

    #********************************************************************************
    #STEP 2.A.1 IF WE CHOOSED TO DONWLOAD THE GENERAL DATA (PATHWAYS CLASSIFICATION, ETC.)
    #********************************************************************************
    try:
        if common:
            log("STEP " + str(currentStep) + ". INSTALLING COMMON KEGG INFORMATION")
            replaceNewVersionData(downloadDir, currentDataDir, "common", oldDataDir)
            installCommonData(currentDataDir + "common/", ROOT_DIRECTORY + "AdminTools/scripts/")
            currentStep+=1
    except Exception as e:
        #TODO: RESTORE
        restorePreviousVersionData(oldDataDir, currentDataDir, "common", downloadDir + "error/")
        installCommonData(currentDataDir + "common/", ROOT_DIRECTORY + "AdminTools/scripts/")
        log("        FAILED WHILE INSTALLING COMMON INFORMATION. UNABLE TO CONTINUE. ABORTING!!")
        summary.write('FAILED WHILE INSTALLING COMMON INFORMATION. UNABLE TO CONTINUE. ABORTING!!')
        errorlog(e)
        summary.close()
        exit(1)

    #**************************************************************************
    #STEP 3. INSTALLING THE PROVIDED SPECIES
    #**************************************************************************
    log("")
    log("STEP " + str(currentStep) + ". INSTALLING NEW ORGANISMS")
    speciesAux = SPECIES_INSTALL.keys()
    total =  str(len(speciesAux))
    log("       " + str(total) + " new organisms will be installed." )

    step=0
    for specie in speciesAux:
        if specie[0] == "#":
            log("    IGNORING " + specie[1:] + "...")
            continue

        step+=1
        dirNameAux = os.path.join(currentDataDir, specie + "/")
        log("        INSTALLING  " + specie + "...")

        try:
            replaceNewVersionData(downloadDir, currentDataDir, specie, oldDataDir)
            installSpecieData(specie, installLog, dirNameAux, str(step)+ "/" + total, ROOT_DIRECTORY + "AdminTools/scripts/")
            INSTALLED_SPECIES.append(specie)
            summary.write(specie + '\tINSTALL\tSUCCESS\t' + str(SPECIES_INSTALL[specie]) + '\n')
            log("INSTALL  "+ str(SPECIES_INSTALL[specie]) + " " + specie + "...SUCCESS\n")
        except Exception as e:
            restorePreviousVersionData(oldDataDir, currentDataDir, specie, downloadDir + "error/")
            installSpecieData(specie, installLog, dirNameAux, str(step)+ "/" + total, ROOT_DIRECTORY + "AdminTools/scripts/")
            summary.write(specie + '\tINSTALL\tERROR\t' + str(SPECIES_INSTALL[specie]) + '\n')
            log("INSTALL  "+ str(SPECIES_INSTALL[specie]) + " " + specie + "...ERROR\n")
            errorlog(e)
            ERRONEOUS_SPECIES.append(specie)

    step=0

    currentStep+=1

    #**************************************************************************
    #STEP 6. CREATE THE species.json FILE
    #**************************************************************************
    log("")
    log("STEP " + str(currentStep) + ". CREATING THE species.json FILE")
    generateAvailableSpeciesFile(INSTALLED_PREVIOUS + INSTALLED_SPECIES, currentDataDir + "common/organisms_all.list", currentDataDir + "species.json")

    currentStep+=1

    #**************************************************************************
    #STEP 7. CLOSING LOG FILES, GENERATING VERSION FILE
    #**************************************************************************
    log("")
    log("STEP " + str(currentStep) + ". CLOSING LOG FILES, GENERATING VERSION FILE")
    summary.close()

    version = open(currentDataDir + 'VERSION','a')

    version.write("\n\n")
    version.write("----------------------------------------------------------------------\n\n")
    version.write("# INSTALLED SPECIES\n\n")
    version.write("\n".join(wrap("\t".join(INSTALLED_SPECIES), 40)))
    version.write("\n\n")
    version.write("----------------------------------------------------------------------\n\n")
    version.write("# FAILED SPECIES (INSTALL)\n\n")
    version.write("\n".join(wrap("\t".join(ERRONEOUS_SPECIES), 40)))
    version.write("\n\n")
    version.write("----------------------------------------------------------------------\n\n")
    version.close()

def restore_command(remove=1, force=0):
    """
    Restores the last version of the database to previous version

    Keyword arguments:
        remove  -- 1 remove the current database directory, 0 keep the directory
        force   -- 1 force remove the current database directory and use previous version, 0  to prompt
    """

    readConfigurationFile()

    realPath = os.path.realpath(KEGG_DATA_DIR + "/last")
    currentFile = os.path.basename(realPath)

    older= '19870729_0534'
    limit =  datetime.datetime.strptime(currentFile, "%Y%m%d_%H%M")
    for i in next(os.walk(KEGG_DATA_DIR))[1]:
        if i == "last" or i == "tmp" or i == "TEST_DATA":
            continue

        aux = datetime.datetime.strptime(i, "%Y%m%d_%H%M")
        aux2= datetime.datetime.strptime(older, "%Y%m%d_%H%M")
        if (aux > aux2) and (aux < limit):
            older=i

    if older != '19870729_0534':
        if(force==0 and not confirm(prompt='Restore to previous directory ' + older + '?', resp=False)):
            exit(1)

        if os.path.exists(KEGG_DATA_DIR + "/last"):
            os.remove(KEGG_DATA_DIR + "/last")
        os.symlink(KEGG_DATA_DIR + older, KEGG_DATA_DIR + "last")

        if remove == 1:
            if(force==1 or confirm(prompt='Remove previous directory '+realPath+'?', resp=False)):
                import shutil
                shutil.rmtree(realPath)

def findnew_command():
    """Find new available species in KEGG."""
    readConfigurationFile()

def findolder_command(nDays):
    """Find installed species older than nDays."""
    readConfigurationFile()

#------------------------------------------------------------------------------------------
#---  AUXILIAR FUNCTIONS                                                               ----
#------------------------------------------------------------------------------------------

def replaceNewVersionData(origin, destination, dirname, backup_dir):
    if os.path.isdir(origin + dirname) and os.path.isdir(destination) and os.path.isdir(backup_dir):
        #Remove previous old data
        if os.path.isdir(backup_dir + dirname):
            shutil.rmtree(backup_dir + dirname)
        #Move the current data to old dir
        if os.path.isdir(destination + dirname):
            shutil.move(destination + dirname, backup_dir)
        #Move the new data to current dir
        shutil.move(origin + dirname, destination)

def restorePreviousVersionData(origin, destination, dirname, backup_dir):
    replaceNewVersionData(origin, destination, dirname, backup_dir)

def downloadKEGGFile(message, logFile, URL, dirName, fileName, delay, maxTries):
    log(message)

    nTry = 1
    while nTry <= maxTries:
        wait(delay)
        try:
            check_call(["wget", "-O", dirName + fileName, "-a", logFile, URL])
            return True
        except Exception as e:
            nTry+=1
            errorlog("FAIL! Trying again... " + str(nTry) + " of " + str(maxTries))
    raise Exception('Unable to retrieve ' + fileName + " from " + URL)

def getSpecieKeggData(specie, downloadLog, dirName, step):
    start_time = time()
    kegg_errors = ""

    log("            FETCHING KEGG DATA FOR " + specie + " (" + step + ")...")
    version = open(dirName + 'KEGG_VERSION','w')
    version.write("# CREATION DATE:\t" + strftime("%Y%m%d %H%M"))
    version.close()

    downloadKEGGFile("              * GENE to PATHWAY TABLE",  downloadLog, "http://rest.kegg.jp/link/pathway/"+ specie, dirName, "gene2pathway.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
    downloadKEGGFile("              * PATHWAYS LIST", downloadLog,  "http://rest.kegg.jp/list/pathway/"+ specie, dirName, "pathways.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
    #downloadKEGGFile("              * PATHWAY to GENE TABLE", downloadLog,  "http://rest.kegg.jp/link/" + specie + "/pathway", dirName, "pathway2gene.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)

    #CREATE THE pathway2gene.list File
    check_call("cat " + dirName + "/gene2pathway.list | awk '{print $2\"\t\"$1}' | sort > " + dirName + "/pathway2gene.list", shell=True)

    #GET THE PATHWAYS LIST AND DOWNLOAD THE IMAGES
    if os.path.isfile(dirName + "pathways.list"):
        if not os.path.isdir(dirName + "kgml/"):
            os.mkdir(dirName + "kgml/")

        pathways=readFile(dirName + "pathways.list", {"forced":True, "forcedColumn": 0})
        total=len(pathways.keys())
        log("                DETECTED " + str(total) + " PATHWAYS FOR " + specie + " " + calculateAproxTime(total, DOWNLOAD_DELAY_2 + 3))

        error_tolerance= int(total * 0.05) #we tolerate that a 5% of the pathways fail on download
        i=1
        for pathway in pathways.keys():
            try:
                pathway = pathway.replace("path:","")
                downloadKEGGFile("                     - " + pathway + " [" + str(i) + "/" + str(total) + "]", downloadLog,  "http://rest.kegg.jp/get/"+ pathway+"/kgml", dirName + "kgml/", pathway + ".kgml",  DOWNLOAD_DELAY_2, MAX_TRIES_1)
            except Exception as e:
                error_tolerance-=1;
                kegg_errors+=" " + pathway
                if error_tolerance == 0:
                    raise Exception("Too many errors while downloading the KGML files for organism " + specie + ": " + kegg_errors)
                log("                       Failed!! The download process will continue...")
            i+=1
    else:
        raise Exception('Unable to retrieve ' + dirName + "pathways.list")

    wait(DOWNLOAD_DELAY_2)

    log("        DOWNLOADED IN " + str((time() - start_time)) + " seconds ---")

    return kegg_errors

def getSpecieMappingData(specie, downloadLog, dirName, step, scriptsDir):
    start_time = time()
    mapping_errors = ""

    log("            FETCHING MAPPING DATA FOR " + specie + " (" + step + ")...")
    version = open(dirName + 'MAP_VERSION','w')
    version.write("# CREATION DATE:\t" + strftime("%Y%m%d %H%M"))
    version.close()

    downloadLogFile = open(downloadLog, 'a')
    try:
        if os.path.isfile(scriptsDir + specie + "_resources/download_others.py"):
            log("     * RETRIEVING EXTERNAL MAPPING DATA")
            try:
                check_call(["python3", scriptsDir + specie + "_resources/download_others.py", specie, ROOT_DIRECTORY + "AdminTools/", dirName], stdout=downloadLogFile, stderr=downloadLogFile)
            except CalledProcessError as exc:
                raise Exception("Error while calling " + scriptsDir + specie + "_resources/download_others.py" +": Exit status " + str(exc.returncode) + ". Output is available at " + downloadLog)

        #we tolerate that some of the files fail on download
        error_tolerance= 3
        try:
            downloadKEGGFile("             * KEGG TO NCBI GeneID", downloadLog,  "http://rest.kegg.jp/conv/"+ specie +"/ncbi-geneid", dirName, "ncbi-geneid2kegg.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
        except Exception as e:
            error_tolerance-=1;
            mapping_errors+=" ncbi-geneid"
            if error_tolerance == 0:
                raise Exception("Too many errors while downloading the KEGG mapping files for organism " + specie + ": " + mapping_errors)
            log("                       Failed!! The download process will continue...")

        try:
            downloadKEGGFile("             * KEGG TO Uniprot", downloadLog,  "http://rest.kegg.jp/conv/"+ specie +"/uniprot", dirName, "uniprot2kegg.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
        except Exception as e:
            error_tolerance-=1;
            mapping_errors+=" uniprot2kegg"
            if error_tolerance == 0:
                raise Exception("Too many errors while downloading the KEGG mapping files for organism " + specie + ": " + mapping_errors)
            log("                       Failed!! The download process will continue...")

        try:
            downloadKEGGFile("             * KEGG TO Gene Symbol", downloadLog,  "http://rest.kegg.jp/list/"+ specie, dirName, "kegg2genesymbol.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)
        except Exception as e:
            error_tolerance-=1;
            mapping_errors+=" kegg2genesymbol"
            if error_tolerance == 0:
                raise Exception("Too many errors while downloading the KEGG mapping files for organism " + specie + ": " + mapping_errors)
            log("                       Failed!! The download process will continue...")

        #downloadKEGGFile("             * KEGG TO NCBI GI", downloadLog,  "http://rest.kegg.jp/conv/"+ specie +"/ncbi-gi", dirName, "ncbi-gi2kegg.list",  DOWNLOAD_DELAY_1, MAX_TRIES_1)

        log("            DOWNLOADED IN " + str(int((time() - start_time)/60)) + " minutes ---")
    except Exception as ex:
        raise ex
    finally:
        downloadLogFile.close()
    return mapping_errors

def installSpecieData(specie, downloadLog, dirName, step, scriptsDir):
    start_time = time()
    log("                 INSTALLING MAPPING DATA FOR " + specie + " (" + step + ")...")
    downloadLogFile = open(downloadLog, 'a')

    try:
        if os.path.isfile(scriptsDir + specie + "_resources/build_database.py"):
            log("       * PROCESSING AND INSTALLING CUSTOM AND KEGG DATA ")
            try:
                check_call(["python3", scriptsDir + specie + "_resources/build_database.py", specie, ROOT_DIRECTORY + "AdminTools/", dirName, downloadLog], stdout=downloadLogFile, stderr=downloadLogFile)
            except CalledProcessError as exc:
                errorlog(traceback.extract_stack())
                raise Exception("Error while calling " + scriptsDir + specie + "_resources/build_database.py" +": Exit status " + str(exc.returncode) + ". Output is available at " + downloadLog)
        else:
            log("       * PROCESSING AND INSTALLING DEFAULT KEGG DATA ")
            try:
                check_call(["python3", scriptsDir  +  "default/build_database.py", specie, ROOT_DIRECTORY + "AdminTools/", dirName, downloadLog], stdout=downloadLogFile, stderr=downloadLogFile)
            except CalledProcessError as exc:
                errorlog(traceback.extract_stack())
                raise Exception("Error while calling " + scriptsDir + "default/build_database.py" +": Exit status " + str(exc.returncode) + ". Output is available at " + downloadLog)

        log("        INSTALLED IN " + str(int((time() - start_time))) + " seconds ---")
    except Exception as ex:
        raise ex
    finally:
        downloadLogFile.close()
    return True

def installCommonData(dirName, scriptsDir):
    log("            INSTALLING COMMON DATA... ")

    try:
        import imp
        COMMON_BUILD_DB_TOOLS = imp.load_source('common_build_database', scriptsDir + "common_build_database.py")
        COMMON_BUILD_DB_TOOLS.processKEGGCommonData(dirName, ROOT_DIRECTORY)
    except Exception as ex:
        raise ex
    return True

def getCurrentInstalledSpecies():
    # ****************************************************************
    # Step 1.GET THE LIST OF INSTALLED SPECIES (DATABASES and SPECIES.JSON)
    # ****************************************************************
    organisms_names = {}
    import csv
    from conf.serverconf import MONGODB_HOST, MONGODB_PORT

    with open(KEGG_DATA_DIR + 'current/common/organisms_all.list') as organisms_all:
        reader = csv.reader(organisms_all, delimiter='\t')
        for row in reader:
            organisms_names[row[1]] = row[2]
    organisms_all.close()

    installedSpecies = []
    from pymongo import MongoClient

    client = MongoClient(MONGODB_HOST, MONGODB_PORT)
    databases = client.list_database_names()

    # ****************************************************************
    # Step 2.FOR EACH INSTALLED DATABASE GET THE INFORMATION
    # ****************************************************************
    databaseList = []
    common_info_date = ""

    for database in databases:
        if not "-paintomics" in database:
            continue
        elif "global-paintomics" == database:
            db = client[database]
            common_info_date = db.versions.find({"name": "COMMON"})[0].get("date")
        else:
            # Step 2.1 GET THE SPECIE CODE
            organism_code = database.replace("-paintomics", "")
            # Step 2.2 GET THE SPECIE NAME
            organism_name = organisms_names.get(organism_code, "Unknown specie")

            # Step 2.3 GET THE SPECIE VERSIONS
            db = client[database]
            kegg_date = db.versions.find({"name": "KEGG"})[0].get("date")
            mapping_date = db.versions.find({"name": "MAPPING"})[0].get("date")
            acceptedIDs = db.versions.find({"name": "ACCEPTED_IDS"})

            try:
                if acceptedIDs.count() > 0:
                    acceptedIDs = acceptedIDs[0].get("ids")
                else:
                    acceptedIDs = ""
            except Exception as ex:
                acceptedIDs = ""

            # Step 2.4 Check if the organism has non installed data available
            if os.path.isfile(KEGG_DATA_DIR + 'download/' + organism_code + '/VERSION'):
                downloaded = True
            elif os.path.isfile(KEGG_DATA_DIR + 'download/' + organism_code + '/DOWNLOADING'):
                downloaded = "downloading"
            else:
                downloaded = False
                # Erroneous download not removed --> remove
                if os.path.isdir(KEGG_DATA_DIR + 'download/' + organism_code):
                    shutil.rmtree(KEGG_DATA_DIR + 'download/' + organism_code)

            databaseList.append({
                "organism_name": organism_name,
                "organism_code": organism_code,
                "kegg_date": kegg_date,
                "mapping_date": mapping_date,
                "acceptedIDs": acceptedIDs,
                "downloaded": downloaded
            })

    client.close()
    return databaseList

def generateAvailableSpeciesFile(VALID_SPECIES, species_file, installed_species_file):
    try:
        #rootName, "organisms_all.list",
        import csv
        species={}
        with open(species_file, "r") as csvfile:
            rows = csv.reader(csvfile, delimiter='\t')
            #FILL THE TABLE specie_code -> specie_name
            for row in rows:
                species[row[1]] = row[2]
        csvfile.close()

        listAux = []
        for specie in VALID_SPECIES:
            if isinstance(specie, dict):
                listAux.append(specie.get("organism_code"))
            else:
                listAux.append(specie)

        VALID_SPECIES = listAux
        VALID_SPECIES.sort()
        VALID_SPECIES = set(VALID_SPECIES)

        total= len(VALID_SPECIES)

        file_content = '{"success": true, "species": [\n'
        for i, specieCode in enumerate(VALID_SPECIES):
                name = species.get(specieCode, "")
                if name!="":
                    name= '\t{"name": "' + name + '", "value": "' + specieCode + '"}'
                    if i < total-1:
                        name+=","
                    file_content += name + '\n'
                else:
                    #continue
                    errorlog("Error while writting specie files" + specieCode)
                    raise Exception()
    except Exception as ex:
        errorlog(traceback.extract_stack())
        raise Exception("Error while writting specie " + specieCode)

    if os.path.isfile(installed_species_file):
        shutil.copy(installed_species_file, installed_species_file + "_prev")

    output_file = open(installed_species_file, 'w')
    output_file.write(file_content)
    output_file.write(']}')
    output_file.close()

#------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------
#---  MORE AUXILIAR FUNCTIONS                                                          ----
#------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------

def generateThumbnail(imagePath):
    from PIL import ImageFile
    ImageFile.LOAD_TRUNCATED_IMAGES = True
    destination = imagePath.replace("png/", "png/thumbnails/").replace(".png", "_thumb.png")
    thumb = Image.open(imagePath)
    #Generate the thumbnail
    s = thumb.size
    n = min(s)
    thumb = thumb.crop((s[0]/2 - n/4, s[1]/2-n/4, s[0]/2 + n/4, s[1]/2 + n/4))
    thumb.thumbnail((300,300))
    thumb.save(destination)

def readConfigurationFile():
    global ROOT_DIRECTORY
    ROOT_DIRECTORY = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../") + "/"
    #PREPARE LOGGING
    logging.config.fileConfig(ROOT_DIRECTORY + 'conf/logging.cfg')

def readFile(path, options=None):
    data = {}
    forced=False
    forcedColumn=0

    if(options != None):
        options.get("forced", False)
        options.get("forcedColumn", 0)

    if os.path.isfile(path):
        with open(path, 'rU') as inputDataFile:
            import csv
            for line in csv.reader(inputDataFile, delimiter="\t"):
                if len(line) == 3 and forced == False: #IF IT IS UPDATE FILE
                    data[line[0]] = 0 # 0= DO NOTHING (DEFAULT ACTION), 1 = updateMapping, 2 = updateKegg, 3 = updateKegg && updateMapping

                    if(line[2] == "1"):#updateMapping = 1
                        data[line[0]] += 1
                    if(line[1] == "1"):#updateKegg = 1
                        data[line[0]] += 2
                    if(data[line[0]] == 0): #IF DO NOTHING, WE REMOVE THE SPECIE FROM THE LIST
                        del data[line[0]]
                else:
                    data[line[forcedColumn]] = 3 #updateKegg = 1, updateMapping = 1

        inputDataFile.close()
    return data

def confirm(prompt=None, resp=False):
    """
    prompts for yes or no response from the user. Returns True for yes and
    False for no.
    'resp' should be set to the default value assumed by the caller when
    user simply types ENTER.
    >>> confirm(prompt='Create Directory?', resp=True)
    Create Directory? [y]|n:
    True
    >>> confirm(prompt='Create Directory?', resp=False)
    Create Directory? [n]|y:
    False
    >>>
    Create Directory? [n]|y: y
    True
    """

    if prompt is None:
        prompt = 'Confirm'

    if resp:
        prompt = '%s [%s]|%s: ' % (prompt, 'y', 'n')
    else:
        prompt = '%s [%s]|%s: ' % (prompt, 'n', 'y')

    while True:
        ans = input(prompt)
        if not ans:
            return resp
        if ans not in ['y', 'Y', 'n', 'N']:
            print('please enter y or n.')
            continue
        if ans == 'y' or ans == 'Y':
            return True
        if ans == 'n' or ans == 'N':
            return False

def diff(a, b):
        b = set(b)
        return [aa for aa in a if aa not in b]

def log(msg):
    frame,filename,line_number,function_name,lines,index=inspect.getouterframes(
        inspect.currentframe())[1]
    line=lines[0]
    indentation_level=line.find(line.lstrip())
    logging.info('{i} {m}'.format(
        i=' '*indentation_level,
        m=msg
        ))

def errorlog(msg):
    frame,filename,line_number,function_name,lines,index=inspect.getouterframes(
        inspect.currentframe())[1]
    line=lines[0]
    indentation_level=line.find(line.lstrip())
    logging.error('{i} {m}'.format(
        i=' '*indentation_level,
        m=msg
        ))

def wait(nSeconds):
    #log(message)
    sleep(nSeconds)

def calculateAproxTime(nElems, delay):
    return "[" + str(int(nElems * delay /60) ) + " min aprox.]"

#------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------
#---  RUN APP
#------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------
if __name__ == '__main__':
    import scriptine
    scriptine.run()


from builder.converttmx import ConvertTmx
import unittest
import polib
import tempfile
from os import path

class TestConvertTmx(unittest.TestCase):
    def _get_files(self, filename):
        tmx_file = path.dirname(path.realpath(__file__))
        tmx_file += "/data/conversions/tmx/{0}".format(filename)

        tmpfile = tempfile.NamedTemporaryFile()
        po_filename = tmpfile.name + ".po"

        return tmx_file, po_filename

    def test_conversion(self):
        tmx_file, po_filename = self._get_files("test.tmx")
        convertTmx = ConvertTmx(tmx_file, po_filename)
        convertTmx.convert()

        entries = polib.pofile(po_filename)

        self.assertEqual(len(entries), 2)
        self.assertEqual(entries[0].msgid, "Modern visual refresh")
        self.assertEqual(entries[0].msgstr, "Una renovació visual moderna")
        self.assertEqual(
            entries[0].tcomment,
            "id: appstores:fx_androidwhatsnewandroid_57lang:a-clean-modern-visual-refresh",
        )

        self.assertEqual(entries[1].msgid, "As You Like It")
        self.assertEqual(entries[1].msgstr, "Com vulgueu")
        self.assertEqual(entries[1].tcomment, "")

    def test_conversion_omegat(self):
        tmx_file, po_filename = self._get_files("omegat.tmx")
        convertTmx = ConvertTmx(tmx_file, po_filename)
        convertTmx.convert()

        entries = polib.pofile(po_filename)

        self.assertEqual(len(entries), 1)
        self.assertEqual(entries[0].msgid, '"Aligner" aligner utility')
        self.assertEqual(entries[0].msgstr, 'Alineador de textos "Aligner"')
        self.assertEqual(entries[0].tcomment, "")

if __name__ == "__main__":
    unittest.main()


from django.conf import settings
from django.conf.urls import patterns, url, include
from rest_framework.routers import DefaultRouter
from views import *

router = DefaultRouter()
router.register(r'coins/sets', CoinSetViewSet)
router.register(r'banknotes/sets', BanknoteSetViewSet)
'''
router.register(r'mints', viewsets.MintViewSet)
router.register(r'countries', viewsets.CountryViewSet)
router.register(r'currencies', viewsets.CurrencyViewSet)
router.register(r'collections', viewsets.CollectionViewSet)
'''

urlpatterns = patterns(
    'coins.views',
    url(r'^', include(router.urls)),
    url(r'^user/token', DeviceTokenView.as_view()),
    url(r'^user/login', AuthTokenView.as_view())
)

if ('rest_framework.authentication.SessionAuthentication'
    in settings.REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] and
    'rest_framework.renderers.BrowsableAPIRenderer'
        in settings.REST_FRAMEWORK['DEFAULT_RENDERER_CLASSES']):
    urlpatterns += patterns(
        '',
        url(r'^auth/', include(
            'rest_framework.urls',
            namespace='rest_framework'))
    )


from prettytable import PrettyTable
from prettytable.colortable import ColorTable, Themes

table = PrettyTable()

table.add_column("Pokemon Name", ["Pikachu", "Squirtle", "Charmander"])
table.add_column("Type", ["Electric", "Water", "Fire"])

table.align = "l"
print(table)

# Costum table with colors
color_table = ColorTable(theme=Themes.OCEAN)

color_table.add_column("Student Name", ["Eka", "Budi", "Siti"])
color_table.add_column("Age", [16, 15, 17])

print(color_table)

Day 16 - Object Oriented Programming/practice_using_python_packages.py


import json
import math
from collections import defaultdict

from google.protobuf.wrappers_pb2 import BoolValue, UInt64Value, StringValue, DoubleValue

from connectors.integrations.datadog import DatadogConnector
from connectors.integrations.newrelic import NewRelicConnector

from event.clickhouse.models import Events
from event.engine.metric_engine import process_metric_expressions
from event.workflows.workflow_utils import get_metric_key_context, get_event_metric_expr_query, \
    get_transaction_metric_expr_query
from protos.event.entity_pb2 import WorkflowView

class Node:

    def __init__(self, event_type_name, event_type_id):
        self.name = event_type_name
        self.id = event_type_id
        self.has_in_connections = False
        self.has_out_connections = False
        self.metrics = []
        self.status = WorkflowView.NodeStatus.GOOD
        self.workflow_node_config = None

    def mark_in_connection(self):
        self.has_in_connections = True

    def mark_out_connection(self):
        self.has_out_connections = True

    def add_metric(self, metric):
        self.metrics.append(metric)

    def set_status(self, status):
        self.status = status

    def get_proto(self):
        conn_types = []

        if self.has_in_connections:
            conn_types.append(WorkflowView.NodeConnectionType.IN)
        if self.has_out_connections:
            conn_types.append(WorkflowView.NodeConnectionType.OUT)

        return WorkflowView.Node(
            node_id=UInt64Value(value=self.id), label=StringValue(value=self.name),
            status=self.status,
            connection_types=conn_types,
            metrics=self.metrics,
            workflow_node_config=StringValue(value=json.dumps(self.workflow_node_config)),
        )

class Edge:

    def __init__(self, start: Node, end: Node):
        self.start = start
        self.end = end
        start.mark_out_connection()
        end.mark_in_connection()

    def __str__(self):
        return "{} - {} (Count: {})".format(self.start.name, self.end.name, self.counter)

    def print(self):
        print("{} -> {} ({}, {})".format(self.start.name, self.end.name, len(self.owners), self.owners[:1]))

    def get_proto(self):
        return WorkflowView.Edge(
            start_node_id=UInt64Value(value=self.start.id),
            end_node_id=UInt64Value(value=self.end.id),
            metrics=[],
        )

class WorkflowBuilder:

    def __init__(self, dtr, account):
        self.nodes = []
        self.edges = []
        self.dtr = dtr
        self.account = account
        self.dd_connector = None
        self.nr_connector = None

    def get_metric_from_datadog(self, metric_expr, compare_to=None):
        metric_value, delta, delta_type = None, None, None

        self.dd_connector = DatadogConnector(self.dtr.to_tr(), self.account.id)
        metric_value_json = self.dd_connector.fetch_metric(metric_expr)
        metric_value_compare = metric_value_json.get('value')
        if metric_value_compare is not None:
            metric_value = "{} {}".format(metric_value_json.get('value'), metric_value_json.get('unit'))
        else:
            metric_value = None

        if compare_to:
            prev_dtr = self.dtr.get_prev_dtr(compare_to)
            prev_metric_value_json = self.dd_connector.fetch_metric(metric_expr, prev_dtr.to_tr())
            prev_metric_value_compare = prev_metric_value_json.get('value')
            if metric_value_compare and not prev_metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.UP
            elif prev_metric_value_compare and not metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.DOWN
            elif not prev_metric_value_compare and not metric_value_compare:
                delta = None
                delta_type = None
            else:
                delta = abs(round((metric_value_compare / prev_metric_value_compare) - 1.0, 2)) * 100
                if metric_value_compare >= prev_metric_value_compare:
                    delta_type = WorkflowView.DeltaType.UP
                else:
                    delta_type = WorkflowView.DeltaType.DOWN

        return metric_value, delta, delta_type

    def get_metric_from_drdroid(self, metric_expr, compare_to=None):
        metric_value, delta, delta_type = None, None, None

        aggregation_type = metric_expr.split(':')[0]
        aggregation_field = metric_expr.split(':')[1].split('{')[0]
        filters = list({'lhs': x.split(':')[0], 'rhs': x.split(':')[1]} for x in metric_expr.split('{')[1].split('}')[0].split(','))

        aggregation_object = aggregation_field.split('.')[0]
        metric_expr_query = None
        if aggregation_object == 'event':
            metric_expr_query = get_event_metric_expr_query(aggregation_type, aggregation_field, filters, self.account.id, self.dtr)
        if aggregation_object == 'transaction':
            metric_expr_query = get_transaction_metric_expr_query(aggregation_type, aggregation_field, filters, self.account.id, self.dtr)

        if metric_expr_query:
            qs = Events.objects.raw(metric_expr_query)
            metric_value = list(qs)[0].id
            metric_value = round(metric_value, 2)

            if compare_to:
                prev_dtr = self.dtr.get_prev_dtr(compare_to)
                prev_metric_expr_query = None
                if aggregation_object == 'event':
                    prev_metric_expr_query = get_event_metric_expr_query(aggregation_type, aggregation_field, filters,
                                                                    self.account.id, prev_dtr)
                if aggregation_object == 'transaction':
                    prev_metric_expr_query = get_transaction_metric_expr_query(aggregation_type, aggregation_field, filters,
                                                                          self.account.id, prev_dtr)

                prev_qs = Events.objects.raw(prev_metric_expr_query)
                prev_metric_value = list(prev_qs)[0].id
                if metric_value and not prev_metric_value:
                    delta = 100
                    delta_type = WorkflowView.DeltaType.UP
                elif prev_metric_value and not metric_value:
                    delta = 100
                    delta_type = WorkflowView.DeltaType.DOWN
                elif not prev_metric_value and not metric_value:
                    delta = None
                    delta_type = None
                else:
                    delta = abs(round((metric_value / prev_metric_value) - 1.0, 2)) * 100
                    if metric_value >= prev_metric_value:
                        delta_type = WorkflowView.DeltaType.UP
                    else:
                        delta_type = WorkflowView.DeltaType.DOWN

        return metric_value, delta, delta_type

    def process_workflow_config(self, workflow_config):

        compare_to = workflow_config.get('compare_to')
        nodes = workflow_config.get('nodes')

        for idx, node in enumerate(nodes):
            name = node.get('name')
            metrics = node.get('metrics')
            wb_node = Node(name, idx)

            for metric in metrics:
                metric_name = metric.get('name')
                metric_source = metric.get('source')
                metric_expr = metric.get('expr')

                metric_value, delta, delta_type = None, None, None

                if metric_source == 'Datadog':
                    metric_value, delta, delta_type = self.get_metric_from_datadog(metric_expr, compare_to)
                if metric_source == 'DrDroid':
                    metric_value, delta, delta_type = self.get_metric_from_drdroid(metric_expr, compare_to)
                if metric_source == 'Newrelic':
                    metric_value, delta, delta_type = self.get_metric_from_newrelic(metric_expr, compare_to)

                metric_status = WorkflowView.MetricStatus.GREEN
                if delta_type:
                    metric_context = get_metric_key_context(metric_name)

                    if metric_context == 'up' and delta_type == WorkflowView.DeltaType.DOWN:
                        metric_status = WorkflowView.MetricStatus.RED
                    if metric_context == 'down' and delta_type == WorkflowView.DeltaType.UP:
                        metric_status = WorkflowView.MetricStatus.RED

                if metric_status == WorkflowView.MetricStatus.RED and delta and delta > 15.0:
                    wb_node.set_status(WorkflowView.NodeStatus.BAD)

                wb_node.add_metric(WorkflowView.Node.NodeMetric(name=StringValue(value=metric_name),
                                                                      value=StringValue(value=str(metric_value)),
                                                                      metric_status=metric_status,
                                                                      delta=DoubleValue(value=delta),
                                                                      delta_type=delta_type
                                                                    ))
                wb_node.workflow_node_config = {"config": node, "compare_to": compare_to}

            self.nodes.append(wb_node)
            if idx > 0:
                edge = Edge(self.nodes[idx - 1], wb_node)
                self.edges.append(edge)
                self.nodes[idx - 1].mark_out_connection()
                self.nodes[idx].mark_in_connection()

    def get_workflow_view(self):
        data = []

        for node in self.nodes:
            data.append(
                node.get_proto()
            )

        links = []
        for edge in self.edges:
            links.append(
                edge.get_proto()
            )

        return WorkflowView(
            nodes=data,
            edges=links
        )

    def get_metric_from_newrelic(self, metric_expr, compare_to):
        metric_value, delta, delta_type = None, None, None

        self.nr_connector = NewRelicConnector(self.dtr.to_tr(), self.account.id)
        metric_value_json = self.nr_connector.fetch_metric(metric_expr)
        metric_value_compare = metric_value_json.get('value')
        if metric_value_compare is not None:
            metric_value = "{} {}".format(metric_value_json.get('value'), metric_value_json.get('unit'))
        else:
            metric_value = None

        if compare_to:
            prev_dtr = self.dtr.get_prev_dtr(compare_to)
            prev_metric_value_json = self.nr_connector.fetch_metric(metric_expr, prev_dtr.to_tr())
            prev_metric_value_compare = prev_metric_value_json.get('value')
            if metric_value_compare and not prev_metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.UP
            elif prev_metric_value_compare and not metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.DOWN
            elif not prev_metric_value_compare and not metric_value_compare:
                delta = None
                delta_type = None
            else:
                delta = abs(round(((metric_value_compare / prev_metric_value_compare) - 1.0), 2)) * 100
                if metric_value_compare >= prev_metric_value_compare:
                    delta_type = WorkflowView.DeltaType.UP
                else:
                    delta_type = WorkflowView.DeltaType.DOWN

        return metric_value, delta, delta_type


# -*- coding: utf-8 -*-
"""
Created on Wed Sep  7 08:45:00 2022
@author: sahakyz
"""

# First install data that comes with the textbook
# Type in the right lower window $pip install wooldridge

#%%
import wooldridge as woo

#%%
# After that we need our standard packages

import pandas as pd # data import and manipulation, pandas stands for Python Data Analysis Library
import numpy as np # math operations
import statsmodels.formula.api as smf #regressions
import seaborn as sns # library for plotting /easy version
import matplotlib.pyplot as plt # library for plotting
from scipy import stats #library for statistical functions

#%%
# Load Data wage2.csv file

wage2df = woo.dataWoo('wage2')

#%%

#%%
'Drop missing observations'
wage2df.dropna(inplace=True) #Drop NAN observations
wage2df.reset_index(drop=True, inplace=True) #Reset index

#%%

# Examine the data'
print(wage2df[:5]) #show me the first 5 observations

#%%
print(wage2df.head()) #show me the first (5) observations

print(wage2df.tail()) #show me the last (5) observations

#%%
#Simple plot
plt.plot(wage2df.educ, wage2df.wage, 'ro')
plt.xlabel("Education")
plt.ylabel("Wage Rate")
plt.title("Hourly Wage Rate vs Years of Schooling")
plt.show()

#%%
'Create a new variable and assign to your DataFrame'

wage2df['lnwage']=np.log(wage2df.wage) # log(wage)

#%%
wage2df['educ2']=np.square(wage2df.educ) # educ**2

#%%
# Plot Data with regression line going through it
sns.regplot(x='educ', y='wage', data=wage2df).set_title('Return on Education')

#%%
#Run regression
reg = smf.ols('wage ~ educ', data=wage2df, missing='drop').fit()
print(reg.summary2())

#%%
#Get fitted (predicted) values of y and print them
'wagehat =b0+b1*educ'

print(reg.fittedvalues)

#%%
'Get Fitted values/predicted values and assign them to yhat variable in your dataframe'
wage2df['wagehat']=reg.fittedvalues

#%%
'Get residuals and assign to your datatframe'
wage2df['uhat']=reg.resid

#%%
# Residual Plot
plt.scatter(wage2df.wagehat, wage2df.uhat)
plt.show()

#%%
#Or use sns package
#Residual Plot
sns.residplot(x='educ', y='wage', \
            data=wage2df).set_title('Residuals vs Years of Schooling')

#%%
#Creating new dataframe where wage >900 and save as a new datafrae

wage2df900=wage2df[wage2df['wage']>900]

#%%
#Get the number of observations in this dataframe
print(wage2df900.shape[0])

#%%
# Report your results in for hoemwork like this
#Question 2 part a
print('The number of observation that have \
      wage rate above 900 is', wage2df900.shape[0])

#%%
#You can also use count() function
print(wage2df900.count())

#%%
#Report as this
print(wage2df900.count()[1])

#%%
#Describe wage column in wage2 dataframe
print(wage2df['wage'].describe())

#%%

print(wage2df['wage'].describe()[2])

#%%
print('The standard deviation of wage rate in our suample is', wage2df['wage'].describe()[2])


import os
import glob
import unicodedata
import sentencepiece as spm

from transformers import AlbertTokenizer, AlbertTokenizerFast
from cai_common.defaults import cai_s3

from .cai_tokenizer_mixin import CAITokenizerMixin

DATA_BASE_PATH = os.environ.get('CAI_DATA_BASE_PATH', None)
if DATA_BASE_PATH in {'', '***unset***'}:
    DATA_BASE_PATH = None
if DATA_BASE_PATH is None:
    DATA_BASE_PATH = cai_s3

VOCAB_FILES_NAMES = {"vocab_file": "spm_model.model"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "olive-tiny":
            os.path.join(DATA_BASE_PATH, "champion_models/tokenizer-olive/olive_tiny/spm_model.model"),
        "olive-small":
            os.path.join(DATA_BASE_PATH, "champion_models/tokenizer-olive/olive_small/spm_model.model"),
        "olive-large":
            os.path.join(DATA_BASE_PATH, "champion_models/tokenizer-olive/olive_big/spm_model.model"),
    }
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "olive-tiny": 512,
    "olive-small": 512,
    "olive-large": 512,
    "tibert-unigram-large": 512
}

SPIECE_UNDERLINE = "▁"

def _tokenize_shim(self, super, text, **kwargs):    # pylint: disable=redefined-builtin
    # The AlBERT tokenizer deliberately splits on the control tokens in the text to introduce white space between
    #   them. So [MASK]de dmar ro/ becomes ['[MASK]', '_de', 'dmar', 'ra', 'o', '/']. This doesn't work for
    #   Tibetan, we want it to be ['_', '[MASK]', 'de', 'dmar', 'ra', 'o', '/'] otherwise we have a huge mismatch
    #   between the training and test. So we replace the control tokens with non-Tibetan special text, tokenize as
    #   per AlBERT, then replace the special text with the control tokens back. Everything then flows correctly
    #   through the rest of the AlBERT tokenizer.
    #
    # We also introduce the option to only add the underline between sections. This can improve performance on
    #   unmasking short sentences. Likely this is happening because of a mismatch between the training and test
    #   data, which means the short sentence part of the BERT pre-training data generator is actually important.
    #   In Tibetan there is no obvious way to do this without a sentence segmentation model, so we leave this as is
    #   until we are able to make a reliable sentence segmenter as part of the part-of-speech tagging experiments.
    #   When we have a reliable segmenter we can augment the training set with short individual sentences that have
    #   the proper SentencePiece underscores in front of them and polish the transformer model.
    if ' ' in text and self.underline_between_sections:
        sections = [self.tokenize(section, **kwargs) for section in text.split(' ')]
        res = []
        for section in sections + [[]]:
            if len(section) > 0:
                res.extend(section + [SPIECE_UNDERLINE])
        if len(res) > 0:
            res = res[:-1]  # Remove the final SentencePiece underline
        return res
    if self.tsheg_pretokenization:
        for mark in self.intersyllabic_marks:
            # Break up the intersyllabic marks with a non-Tibetan token. This will force SentencePiece's BPE to
            #   not join across the out-of-vocabulary token.
            text = text.replace(mark, mark + 'a')
    special_token_map = dict(
        [(str(idx), control_token) for idx, control_token in enumerate(list(self.unique_no_split_tokens))])
    for replacement, control_token in special_token_map.items():
        text = text.replace(control_token, replacement)
    res = [
        special_token_map[token] if token in special_token_map else token
        for token in super.tokenize(text, **kwargs)]
    if self.underline_between_sections:
        final_res = []
        for token in res:
            if token == SPIECE_UNDERLINE:
                continue
            if token[0] == SPIECE_UNDERLINE:
                token = token[1:]
            final_res.append(token)
    else:
        final_res = res
    if self.tsheg_pretokenization:
        # Now remove the non-Tibetan tokens to get a strictly intersyllabic tokenization.
        final_res = [token for token in final_res if not token == 'a']
    return final_res

def concat_docstrings(*docstr):
    def docstring_decorator(fn):
        fn.__doc__ = "".join(docstr) + (fn.__doc__ if fn.__doc__ is not None else "")
        return fn

    return docstring_decorator

class CAITokenizerBaseMixin:
    _args_docstring = """
    Args:
        vocab_file (:obj:`string`):
            `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a .spm extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`):
            Whether to lowercase the input when tokenizing.
        remove_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
            Whether to strip the text when tokenizing (removing excess spaces before and after the string).
        keep_accents (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Whether to keep accents when tokenizing.
        bos_token (:obj:`string`, `optional`, defaults to "[CLS]"):
            The beginning of sequence token that was used during pre-training. Can be used a sequence classifier token.
            .. note::
                When building a sequence using special tokens, this is not the token that is used for the beginning
                of sequence. The token used is the :obj:`cls_token`.
        eos_token (:obj:`string`, `optional`, defaults to "[SEP]"):
            The end of sequence token.
            .. note::
                When building a sequence using special tokens, this is not the token that is used for the end
                of sequence. The token used is the :obj:`sep_token`.
        unk_token (:obj:`string`, `optional`, defaults to "<unk>"):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (:obj:`string`, `optional`, defaults to "[SEP]"):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
            for sequence classification or for a text and a question for question answering.
            It is also used as the last token of a sequence built with special tokens.
        pad_token (:obj:`string`, `optional`, defaults to "<pad>"):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (:obj:`string`, `optional`, defaults to "[CLS]"):
            The classifier token which is used when doing sequence classification (classification of the whole
            sequence instead of per-token classification). It is the first token of the sequence when built with
            special tokens.
        mask_token (:obj:`string`, `optional`, defaults to "[MASK]"):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        stochastic_tokenization (:obj:`bool`, `optional`, defaults to :obj:`True`):
            Whether to perform stochastic tokenization in SentencePiece or use the Viterbi tokens.
        stochastic_encoding_nbest (:obj:`int`, `optional`, defaults to :obj:`True`):
            The n-best paramater for the stochastic encoding in SentencePiece.
        stochastic_encoding_alpha  (:obj:`float`, `optional`, defaults to :obj:`True`):
            The alpha paramater for the stochastic encoding in SentencePiece.
        tsheg_pretokenization (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Pre-tokenize along the intersyllabic marks, primarily the tsheg. Useful for token classification tasks like
            named-entity recognition and part-of-speech tagging.
        underline_between_sections (:obj:`bool`, `optional`, defaults to :obj:`False`):
            Whether to put the SentencePiece underline only between sections. Note that, if there is only one section,
            there will be no underline. If you're getting bad results for the unmasking of the first token in a section
            then try setting this to True.
        eor_token_id (:obj:`int`, `optional`, defaults to :obj:`-1`):
            The token ID of the register split token, usually [eor]. The SiameseEncoderModel class splits on these IDs
            during preprocessing in the forward method.
    """

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    intersyllabic_marks = [unicodedata.lookup(c) for c in [
        'TIBETAN MARK INITIAL YIG MGO MDUN MA',
        'TIBETAN MARK CLOSING YIG MGO SGAB MA',
        'TIBETAN MARK SBRUL SHAD',
        'TIBETAN MARK INTERSYLLABIC TSHEG',
        'TIBETAN MARK DELIMITER TSHEG BSTAR',
        'TIBETAN MARK SHAD',
        'TIBETAN MARK RIN CHEN SPUNGS SHAD',
        'TIBETAN MARK GTER TSHEG',
        'TIBETAN SIGN RDEL DKAR GSUM',
        'TIBETAN KU RU KHA']]

    _print_tokenizer = False
    stochastic_encoding_nbest = 64
    stochastic_encoding_alpha = 0.1
    stochastic_tokenization = False
    tsheg_pretokenization = False
    underline_between_sections = False

    def __init__(self):
        self.keep_accents = True

    def enable_siamese(self, eor_token='[eor]'):
        self.eor_token = eor_token
        self.add_tokens(eor_token)
        self.eor_token_id = self.convert_tokens_to_ids(eor_token)

    @staticmethod
    def train(training_data_glob,
              model_file,
              model_type='bpe',
              vocab_size=10000):
        """Train the SentencePiece tokenizer and save to a file.

        Args:
            training_data_glob (:obj:`string`):
                A glob for the training data to use to train SentencePiece. For example:
                '../../tibert_data/training/tibetan_sections/spm_train.*.txt'
            model_file (:obj:`string`):
                `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a .spm extension) that
                will contain the vocabulary necessary to instantiate a tokenizer.
            model_type (:obj:`string`, `optional`, defaults to :obj:`bpe`):
                What SentencePiece model type to train. Can be 'bpe' or 'unigram'.
            vocab_sze (:obj:`int`, `optional`, defaults to :obj:`10000`):
                SentencePiece target vocabulary size.
        """

        # Settings are from https://github.com/google-research/albert/blob/master/README.md
        spm.SentencePieceTrainer.Train(
            input=glob.glob(training_data_glob),
            model_prefix=model_file,
            model_type=model_type,
            vocab_size=vocab_size,
            character_coverage=1,
            pad_id=0,
            unk_id=1,
            eos_id=-1,
            bos_id=-1,
            control_symbols="[CLS],[SEP],[MASK]",
            user_defined_symbols="(,),\",-,.,–,£,€")

@concat_docstrings(
    """Constructs the Tibert tokenizer. Very similar to the ALBERT tokenizer. Based on `SentencePiece
    <https://github.com/google/sentencepiece>`.

    This version of the tokenizer has the fast SentencePiece implementation from Hugging Face as the backend. It does
    not support stochastic tokenization.

    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizerFast` which contains most of the methods.
    Users should refer to the superclass for more information regarding methods.

    """,
    CAITokenizerBaseMixin._args_docstring
)
class CAITokenizerFast(CAITokenizerBaseMixin, AlbertTokenizerFast, CAITokenizerMixin):
    def __init__(self,
                 vocab_file,
                 do_lower_case=True,
                 remove_space=True,
                 keep_accents=True,
                 bos_token='[CLS]',
                 eos_token='[SEP]',
                 unk_token='<unk>',
                 sep_token='[SEP]',
                 pad_token='<pad>',
                 cls_token='[CLS]',
                 mask_token='[MASK]',
                 **kwargs):
        AlbertTokenizerFast.__init__(
            self,
            vocab_file=vocab_file,
            do_lower_case=do_lower_case,
            remove_space=remove_space,
            keep_accents=keep_accents,
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            **kwargs)
        CAITokenizerBaseMixin.__init__(self)

    def tokenize(self, text, **kwargs):
        if self.stochastic_tokenization:
            raise NotImplementedError("The fast version of CAITokenizer does not implement stochastic tokenization. "
                                      "Use the CAITokenizerSlow class for this.")
        return _tokenize_shim(self, super(), text, **kwargs)

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs):
        kwargs.setdefault("keep_accents", True)
        return super().from_pretrained(pretrained_model_name_or_path, *init_inputs, **kwargs)

@concat_docstrings(
    """Constructs the Tibert tokenizer. Very similar to the ALBERT tokenizer. Based on `SentencePiece
    <https://github.com/google/sentencepiece>`.

    This version of the tokenizer has the slow SentencePiece implementation from Hugging Face as the backend, which
    itself relies on Google's implementation of SentencePiece. This version supports stochastic tokenization, this is
    really the only reason to use this class.

    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the methods. Users
    should refer to the superclass for more information regarding methods.

    """,
    CAITokenizerBaseMixin._args_docstring,
    """
    Attributes:
        sp_model (:obj:`SentencePieceProcessor`):
            The `SentencePiece` processor that is used for every conversion (string, tokens and IDs).
    """
)
class CAITokenizerSlow(CAITokenizerBaseMixin, AlbertTokenizer, CAITokenizerMixin):
    def __init__(self,
                 vocab_file,
                 do_lower_case=True,
                 remove_space=True,
                 keep_accents=False,
                 bos_token='[CLS]',
                 eos_token='[SEP]',
                 unk_token='<unk>',
                 sep_token='[SEP]',
                 pad_token='<pad>',
                 cls_token='[CLS]',
                 mask_token='[MASK]',
                 **kwargs):
        AlbertTokenizer.__init__(
            self,
            vocab_file=vocab_file,
            do_lower_case=do_lower_case,
            remove_space=remove_space,
            keep_accents=keep_accents,
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            **kwargs)
        CAITokenizerBaseMixin.__init__(self)

    def _tokenize(self, text):
        # A copy of the _tokenize function from the original but with nbest and alpha exposed.
        if self._print_tokenizer:
            print(f"Tokenizing {text} with sample={self.stochastic_tokenization}")
        text = self.preprocess_text(text)

        if self.stochastic_tokenization:
            pieces = self.sp_model.SampleEncodeAsPieces(
                text,
                self.stochastic_encoding_nbest,
                self.stochastic_encoding_alpha)
        else:
            pieces = self.sp_model.EncodeAsPieces(text)
        new_pieces = []
        for piece in pieces:
            if len(piece) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
                cur_pieces = self.sp_model.EncodeAsPieces(piece[:-1].replace(SPIECE_UNDERLINE, ""))
                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                    if len(cur_pieces[0]) == 1:
                        cur_pieces = cur_pieces[1:]
                    else:
                        cur_pieces[0] = cur_pieces[0][1:]
                cur_pieces.append(piece[-1])
                new_pieces.extend(cur_pieces)
            else:
                new_pieces.append(piece)

        return new_pieces

    def tokenize(self, text, **kwargs):
        return _tokenize_shim(self, super(), text, **kwargs)

CAITokenizer = CAITokenizerFast

if __name__ == "__main__":
    training_data = os.path.join(DATA_BASE_PATH, "processed_datasets/dictionary-tokenizer-training/spm_train.txt")
    spm_tokenizers = os.path.join(DATA_BASE_PATH, "champion_models/tokenizer-olive")

    CAITokenizer.train(
        training_data,
        os.path.join(spm_tokenizers, 'olive_tiny'),
        model_type='bpe',
        vocab_size=1000)

    CAITokenizer.train(
        training_data,
        os.path.join(spm_tokenizers, 'olive_small'),
        model_type='bpe',
        vocab_size=5000)

    CAITokenizer.train(
        training_data,
        os.path.join(spm_tokenizers, 'olive_big'),
        model_type='bpe',
        vocab_size=10000)


from webrecorder.apiutils import wr_api_spec
from webrecorder.basecontroller import BaseController
from bottle import request, static_file, HTTPResponse, response
import os
import json
import sys
import tarfile
import re
import logging
import requests
import shutil
import traceback

# ============================================================================
class BehaviorMgr(BaseController):
    def __init__(self, *args, **kwargs):
        super(BehaviorMgr, self).__init__(*args, **kwargs)
        self.behaviors_api = os.environ.get('BEHAVIOR_API', 'http://behaviors:3030')
        self.behaviors_root = os.environ.get('BEHAVIORS_DIR')

        self.behaviors_tarfile = os.environ.get('BEHAVIORS_TARFILE')

        self.default_behavior = {}
        self.behaviors = {}

        if self.behaviors_root:
            self.behaviors_data = os.path.join(self.behaviors_root, 'dist')
            self.behaviors_metadata = os.path.join(self.behaviors_data, 'behaviorMetadata.json')

            try:
                self.unpack_behaviors()
                self.load()
                self.init_routes()
            except Exception as e:
                traceback.print_exc()

    def unpack_behaviors(self):
        if not self.behaviors_tarfile:
            logging.info('No Behaviors Tarfile specified, skipping unpack')

        os.makedirs(self.behaviors_data, exist_ok=True)

        current_tarfile = os.path.join(self.behaviors_root, os.path.basename(self.behaviors_tarfile))

        try:
            if os.path.isfile(current_tarfile) and os.path.getmtime(current_tarfile) > os.path.getmtime(self.behaviors_tarfile):
                logging.info('Already have latest behaviors, not unpacking')
                return
        except Exception as e:
            print(e)

        logging.info('Unpacking behaviors {0} -> {1}'.format(self.behaviors_tarfile, self.behaviors_root))
        tar = tarfile.open(self.behaviors_tarfile, 'r')
        tar.extractall(self.behaviors_root)
        tar.close()

        shutil.copyfile(self.behaviors_tarfile, current_tarfile)

    def load(self):
        logging.info('Loading behaviors from {0}'.format(self.behaviors_metadata))

        with open(self.behaviors_metadata, 'rt', encoding='utf-8') as fh:
            metadata = fh.read()
            metadata = json.loads(metadata)
            self.default_behavior = metadata['defaultBehavior']
            self.behaviors = {}

            for behavior in metadata['behaviors']:
                rx = re.compile(behavior['match']['regex'])
                self.behaviors[behavior['name']] = (behavior, rx)

    def find_match(self, url):
        if not url:
            return None

        for behavior, rx in self.behaviors.values():
            if rx.match(url):
                return behavior

        return None

    def init_routes(self):
        wr_api_spec.set_curr_tag('Behaviors')

        @self.app.get('/api/v1/behavior/info-list')
        def info_list():
            # for non-desktop proxy mode, proxy to behaviors server
            if not self.behaviors_root:
                query = dict(request.query)
                res = requests.get(self.behaviors_api + '/info-list', params=query)
                response.content_type = 'application/json; charset=utf-8'
                return res.content

            url = request.query.getunicode('url')
            behavior = self.find_match(url)
            behavior_list = []
            if behavior:
                behavior_list.append(behavior)

            behavior_list.append(self.default_behavior)
            return {'behaviors': behavior_list}

        @self.app.get('/api/v1/behavior/behavior')
        def get_behavior():
            # for non-desktop proxy mode, proxy to behaviors server
            if not self.behaviors_root:
                query = dict(request.query)
                res = requests.get(self.behaviors_api + '/behavior', params=query)
                response.content_type = 'application/json; charset=utf-8'
                return res.content

            url = request.query.getunicode('url')
            name = request.query.getunicode('name')
            behavior = None
            if name:
                res = self.behaviors.get(name)
                if res:
                    behavior = res[0]
            elif url:
                behavior = self.find_match(url)

            if not behavior:
                behavior = self.default_behavior

            res = static_file(behavior['fileName'], root=self.behaviors_data)
            return res

        wr_api_spec.set_curr_tag(None)


import webapp2

import validators

form = """
<form method="post">
    What is your birthday?

    Month
    <input type="text" name="month" value="{month}">

    Day
    <input type="text" name="day" value="{day}">

    Year
    <input type="text" name="year" value="{year}">

 {error}

    <input type="submit">
</form>
"""

class MainPage(webapp2.RequestHandler):
    def write_form(self, error="", month="", day="", year=""):
        self.response.write(form.format(error=validators.esc(error),
                                        month=validators.esc(month),
                                        day=validators.esc(day),
                                        year=validators.esc(year)))

    def get(self):
        self.write_form()

    def post(self):
        user_month = self.request.get('month')
        user_day = self.request.get('day')
        user_year = self.request.get('year')

        month = validators.valid_month(user_month)
        day = validators.valid_day(user_day)
        year = validators.valid_year(user_year)

        if not (month and day and year):
            self.write_form("That's an error buddy!",
                            user_month,
                            user_day,
                            user_year)
        else:
            self.redirect("/thanks")

class ThanksHandler(webapp2.RequestHandler):
    def get(self):
        self.response.out.write("Thanks!")

app = webapp2.WSGIApplication([('/', MainPage),
                               ("/thanks", ThanksHandler)],
                               debug=True)


#!/usr/bin/env python3
#

from pathlib import Path

before = "Copyright (C) 2012-2019 The VPaint Developers"

after = "Copyright (C) 2012-2019 The VPaint Developers"

if __name__ == "__main__":

    path = Path(".")
    for file in path.glob("src/**/*"):
        if file.suffix in [".h", ".cpp", ".pro"]:
            s1 = file.read_text()
            s2 = s1.replace(before, after)
            if s1 != s2:
                file.write_text(s2)
                print(file)


#!/usr/bin/env python

import simpy
from hwsim_utils import *
from scapy.all import *

env = simpy.Environment()
period = 1
pipe = simpy.Store(env)
bus_width = 32

pkt = Ether()/IP()/TCP()/'hello there pretty world!!!'
tuser = Tuser(len(pkt), 0b00000001, 0b00000100)
pkt_list = [(pkt, tuser)]*3

master = AXI_S_master(env, period, pipe, bus_width, pkt_list)
slave = AXI_S_slave(env, period, pipe, bus_width)

env.run(until=20)


import logging
from .state import State
from .parsers import UseParser
from functools import reduce

log = logging.getLogger("findingslist")

class FindingsList:
    class NotAUseExpression(Exception):
        pass

    def __init__(self, persist_path, resume_path, initial, fresh=False):
        self.current_findings = initial
        self.persist = State(persist_path, fresh)
        self.resume = State(resume_path, fresh)
        self.persist.load()
        self.previous_findings = self.persist.data
        self.ignore_existing = False

    def get(self, key, attributes=None, unique=True):
        """Return entries from the findings list.

        attributes -- If given, return each entry only with the given attributes.
        unique -- If True, return only unique elements.

        Note that even if only unique entries are added to the list,
        using the attributes argument may lead to duplicate entries,
        since it removes keys from each element that can be used to
        distinguish the elements.

        """
        if not key in self.current_findings:
            raise Exception(
                f"Unknown findings key: {key}; existing keys are: {', '.join(self.current_findings.keys())}"
            )
        out = self.current_findings[key]
        if attributes is not None:
            out = [{k: el[k] for k in attributes} for el in out]
        if unique:
            uni_out = []
            for x in out:
                if x not in uni_out:
                    uni_out.append(x)
            out = uni_out
        return out

    def set(self, key, value):
        if not self.ignore_existing and key in self.current_findings:
            raise Exception(
                f"Storing findings in key {key} would overwrite existing findings."
            )
        self.current_findings[key] = value

    def update(self, data):
        for key, value in data.items():
            self.set(key, value)

    def get_previous(self, key, default):
        return self.previous_findings.get(key, default)

    def save_persist(self):
        self.persist.data = self.current_findings
        self.persist.save()

    def save_resume(self, step):
        self.resume.data[step] = self.current_findings
        self.resume.data["_step"] = step
        self.resume.save()

    def load_resume(self, step=None):
        self.resume.load()
        if step is None:
            step = self.resume.data["_step"]
        log.debug(f"Loading findings list resume data, step={step}")
        self.current_findings = self.resume.data[step]
        self.ignore_existing = True
        return step

    def get_common_and_missing_items(self, key1, key2):
        """Return items that are in findings with key1 and with key2; and
        items that are in findings with key1 but not in those with
        key2; and a boolean indicating if the lists are the same.

        """
        common_attrs = self.find_common_attributes(key1, key2)
        items1 = self.get(key1, common_attrs)
        items2 = self.get(key2, common_attrs)
        common_items = [item for item in items1 if item in items2]
        missing_items = [item for item in items1 if item not in items2]
        equals = len(common_items) == len(items1) == len(items2)

        return common_items, missing_items, equals

    def get_added_items(self, key):
        """Return items that appear in the current findings list with the
        given key, but not in the previous one.

        """
        current = self.get(key)
        previous = self.get_previous(key, [])
        added = [item for item in current if item not in previous]
        return added

    def find_common_attributes(self, *keys):
        """Find one or more attributes that each entry in each of the
        lists has in common. This function compares only the first
        member of each list and assumes that all further elements have
        the same attributes. This is enforced in other places in the
        code.

        """

        common_attrs = None
        for k in keys:
            all_items = self.get(k)
            if len(all_items) == 0:
                return None
            attrs = set(all_items[0].keys())
            if common_attrs is None:
                common_attrs = attrs
            else:
                common_attrs &= attrs

        if len(common_attrs) == 0:
            raise Exception(
                f"Unable to compare the findings lists {keys} (no common attributes)."
            )
        return common_attrs


#!/usr/bin/python
#
# renderpage.py
# Build the podcasting home page, including:
#
#   * State of icecast server (if we're live)
#   * All available podcasts rendered from
#     showindex_multi.json and mustache templates.
#
# J. Adams < jna@retina.net >
# 8/5/2014
#
# TODO: Don't touch the destination file if our state hasn't changed.
#       Better error handling - right now we have none.

from pytz import timezone
from datetime import tzinfo, timedelta, datetime
import json
import pystache
import requests
import rfc822
import time

# In the US, DST starts at 2am (standard time) on the first Sunday in April.
# and ends at 2am (DST time; 1am standard time) on the last Sunday of Oct.
# which is the first Sunday on or after Oct 25.
DSTSTART = datetime(1, 4, 1, 2)
DSTEND = datetime(1, 10, 25, 1)
ZERO = timedelta(0)
HOUR = timedelta(hours=1)

def first_sunday_on_or_after(dt):
  days_to_go = 6 - dt.weekday()
  if days_to_go:
    dt += timedelta(days_to_go)
  return dt

class UTC(tzinfo):
  """UTC"""
  def utcoffset(self, dt):
    return ZERO
  def tzname(self, dt):
    return "UTC"
  def dst(self, dt):
    return ZERO

class USTimeZone(tzinfo):
  def __init__(self, hours, reprname, stdname, dstname):
    self.stdoffset = timedelta(hours=hours)
    self.reprname = reprname
    self.stdname = stdname
    self.dstname = dstname

  def __repr__(self):
    return self.reprname

  def tzname(self, dt):
    if self.dst(dt):
      return self.dstname
    else:
      return self.stdname

  def utcoffset(self, dt):
    return self.stdoffset + self.dst(dt)

  def dst(self, dt):
    if dt is None or dt.tzinfo is None:
      # An exception may be sensible here, in one or both cases.
      # It depends on how you want to treat them.  The default
      # fromutc() implementation (called by the default astimezone()
      # implementation) passes a datetime with dt.tzinfo is self.
      return ZERO
    assert dt.tzinfo is self

    # Find first Sunday in April & the last in October.
    start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year))
    end = first_sunday_on_or_after(DSTEND.replace(year=dt.year))

    # Can't compare naive to aware objects, so strip the timezone from
    # dt first.
    if start <= dt.replace(tzinfo=None) < end:
      return HOUR
    else:
      return ZERO

class Index(object):
  """ methods for the index.mustache page """
  def is_live(self):
      """ right now we only broadcast under /wg, see if it is up """
      r = requests.get('http://podcast.retina.net:8000/')
      if r.status_code == 200:
          if r.text.find("Mount Point /wg") > -1:
              return True
      else:
          return False

  def shows(self):
    fp = open("showindex.json","r")
    shows = json.load(fp)

    # Convert the GMT times into something more friendly for HTML reading
    # incoming times are always in RSS format, we want them in US/Pacific
    # (Also, what the fuck year is this. Why am I writing this code.)
    utc = UTC()
    Pacific = USTimeZone(-8, "Pacific",  "PST", "PDT")
    for show in shows['shows']:
      n = 0
      for e in show['show']['episodes']:
        parsed_date = e['date']
        d = datetime.strptime( e['date'], "%a, %d %b %Y %H:%M:%S GMT" )
        aware = d.replace(tzinfo=utc)
        show['show']['episodes'][n]['date'] = aware.astimezone(Pacific).strftime("%A, %B %e, %Y")
        n = n + 1

    return shows['shows']

class Rss(object):
  """ methods for rendering the rss index """
  def shows(self):
    fp = open("showindex.json","r")
    shows = json.load(fp)

    # this RSS feed is only the first show (wg) episode list.
    return shows['shows'][0]['show']['episodes']

# set up the renderer
renderer = pystache.Renderer(search_dirs="/retina/podcast/html/templates")

# render the index page
page = Index()
parsed = renderer.render(page)
fout = open("index.html","w")
print >>fout, parsed
fout.close()

# render the RSS feed
page = Rss()
parsed = renderer.render(page)
fout = open("wg/wicked_grounds.rss","w")
print >>fout, parsed
fout.close()