import subprocess
import cmd
import os
import sys
import platform
import readline
import logging
import rlcompleter
import time
import frida

if 'libedit' in readline.__doc__:
    readline.parse_and_bind("bind ^I rl_complete")
else:
    readline.parse_and_bind("tab: complete")

RED   = "\033[1;31m"
BLUE  = "\033[1;34m"
CYAN  = "\033[1;36m"
WHITE = "\033[1;37m"
YELLOW= "\033[1;33m"
GREEN = "\033[0;32m"
RESET = "\033[0;0m"
BOLD    = "\033[;1m"
REVERSE = "\033[;7m"

class parser(cmd.Cmd):

    prompt = BLUE+'apkutils>'+RESET
    INSTALL = False
    device = None
    package = None
    permissions = None
    activities = None
    services = None
    receivers = None
    providers = None
    deeplinks = None
    deeplinks_= []
    strings = []
    filters = []

    # classes = []
    packages = []

    def do_search(self, line):
        found = False
        try:
            what = line.split(' ')[0]
            print(RED+'Searching Activities:'+RESET)
            for module in self.activities:
                if what.lower() in module.lower():
                    print('[+] '+module[:str(module.lower()).find(what.lower())]+GREEN+what+RESET+module[str(module.lower()).find(what.lower())+len(what.lower()):])
                    print("[Original String: {}]\n".format(module))
                    found = True
            if not found:
                print('No Activities found containing: {} !'.format(what))
            found = False

            print(RED+'Searching Services:'+RESET)
            for module in self.services:
                if what.lower() in module.lower():
                    print('[+] '+module[:str(module.lower()).find(what.lower())]+GREEN+what+RESET+module[str(module.lower()).find(what.lower())+len(what.lower()):])
                    print("[Original String: {}]\n".format(module))
                    found = True
            if not found:
                print('No Services found containing: {} !'.format(what))
            found = False

            print(RED+'Searching Receivers:'+RESET)
            for module in self.receivers:
                if what.lower() in module.lower():
                    print('[+] '+module[:str(module.lower()).find(what.lower())]+GREEN+what+RESET+module[str(module.lower()).find(what.lower())+len(what.lower()):])
                    print("[Original String: {}]\n".format(module))
                    found = True
            if not found:
                print('No Receivers found containing: {} !'.format(what))
            found = False

            print(RED+'Searching Providers:'+RESET)
            for module in self.providers:
                if what.lower() in module.lower():
                    print('[+] '+module[:str(module.lower()).find(what.lower())]+GREEN+what+RESET+module[str(module.lower()).find(what.lower())+len(what.lower()):])
                    print("[Original String: {}]\n".format(module))
                    found = True
            if not found:
                print('No Providers found containing: {} !'.format(what))
            found = False

            print(RED+'Searching Strings.xml:'+RESET)
            for module in self.strings:
                if what.lower() in module.lower():
                    print('[+] '+module[:str(module.lower()).find(what.lower())]+GREEN+what+RESET+module[str(module.lower()).find(what.lower())+len(what.lower()):])
                    print("[Original String: {}]\n".format(module))
                    found = True
            if not found:
                print('No Strings found containing: {} !'.format(what))
            found = False

        except Exception as e:
            print(e)

    def do_deeplink(self,line):
        try:
            if len(line.split()) > 1 and '--poc' in line.split()[1]:
                print("[+] Creating POC")
                poc = ''+''+'DEEPLINK POC'
                f = open("poc.html",'w')
                f.write(poc)
                f.close()
                print("[+] POC created")

            else:
                output=os.popen("adb -s {} shell am start -W -a android.intent.action.VIEW -d {}".format(self.device.id,line.split(' ')[0])).read()
                print(output)
        except Exception as e:
            print(e)

    def complete_deeplink(self, text, line, begidx, endidx):

        if not text:
            completions = self.deeplinks_[:]
        else:
            completions = [ f
                            for f in self.deeplinks_
                            if f.startswith(text)
                            ]
        return completions

    def printDeepLinksMap(self):
        a = 0
        print(GREEN+'\n'+'-'*40+'DeepLinks Map'+'-'*40+':'+RESET)
        try:
            for key in self.deeplinks:
                print(BLUE+'Deeplinks that trigger: '+GREEN+key+RESET)
                for value in self.deeplinks[key]:
                    self.deeplinks_.append(value)
                    print('\t|-> '+RED+value+RESET)
                    a = a+1
            print(GREEN+'-'*40+'Total Deeplinks:{}'.format(a)+'-'*40+'|')
        except Exception as e:
            print(e)

    def do_trace(self,line):

        opsys = platform.system()
        script = self.create_script(opsys,line)

        if not 'Error' in script:

            if 'Darwin' in opsys:
                subprocess.run("""osascript -e 'tell application "Terminal" to do script "{}" ' """.format(script), shell=True)
            elif 'Linux' in opsys:
                subprocess.run("""x-terminal-emulator -e {}""".format(script))
            elif 'Windows' in opsys:
                subprocess.call('start /wait {}'.format(script), shell=True)

    def do_pull(self, line):
        try:
            base_apk = os.popen("adb -s {} shell pm path {} | grep apk".format(self.device.id,line.split(' ')[0])).read()
            base_apk = base_apk[ base_apk.find(':')+1:]
            print("Extracting: "+base_apk)
            output = os.popen("adb -s {} pull {}".format(self.device.id,base_apk)).read()
            print(output)
        except Exception as e:
            print(e)

    def complete_pull(self, text, line, begidx, endidx):
        self.init_packages()
        if not text:
            completions = self.packages[:]
            self.packages = []
        else:
            completions = [ f
                            for f in self.packages
                            if f.startswith(text)
                            ]
            self.packages = []
        return completions

    def create_script(self,opsys,line):

        switch = line.split(' ')[0].strip()
        valid = True

        if '-j' in switch:
            param1 = line.split(' ')[1]+ '*!*'
            param = """frida-trace -D {} {} -j '{}' """.format(self.device.id,self.package,param1)
        elif '-n' in switch:
            param1 = line.split(' ')[1]+ '*'
            param = """frida-trace -D {} -i '{}' {}""".format(self.device.id,param1,self.package)
        elif '-a' in switch:
            param1 = line.split(' ')[1].strip()
            param = """frida-trace -D {} -I '{}' {}""".format(self.device.id,param1,self.package)
        else:
            print('[E] Invalid command, run help for options!')
            valid = False

        if valid:
            path = os.getcwd()

            if 'Windows' in opsys:
                script = path + '/script.bat'
                with open(script,'w') as file:
                    file.write(param)
            else:
                script = path + '/script.sh'
                with open(script,'w') as file:
                    file.write(param)
                os.chmod(script, 0o775)
        else:
            script = 'Error'

        return script

    def do_notify(self,line):
        try:
            output=os.popen("adb -s {} shell am broadcast  -a com.medusa.NOTIFY --es subject {} --es body {}".format(self.device.id,line.split(' ')[0],line.split(' ')[1])).read()
            print(output)
        except Exception as e:
            print(e)
            return

    def init_packages(self):
        for line1 in os.popen('adb -s {} shell pm list packages -3'.format(self.device.id)):
            self.packages.append(line1.split(':')[1].strip('\n'))

    def do_uninstall(self,package):
        try:
            output=os.popen("adb -s {} uninstall {}".format(self.device.id,package.split(' ')[0])).read()
            print(output)
        except Exception as e:
            print(e)
            return

    def do_kill(self,package):

        try:
            print(package)
            output=os.popen("adb -s {} shell  am force-stop {}".format(self.device.id,package.split(' ')[0])).read()
            print(output)
        except Exception as e:
            print(e)
            return

    def do_spawn(self,package):
        try:
            print('[+] Starting {}'.format(package))
            os.popen("adb -s {} shell  monkey -p {} -c 'android.intent.category.LAUNCHER 1'".format(self.device.id,package.split(' ')[0])).read()
            print('[+] {} started'.format(package))
        except Exception as e:
            print(e)
            return

    def complete_spawn(self, text, line, begidx, endidx):

        self.init_packages()
        if not text:
            completions = self.packages[:]
            self.packages = []
        else:
            completions = [ f
                            for f in self.packages
                            if f.startswith(text)
                            ]
            self.packages = []
        return completions

    def complete_kill(self, text, line, begidx, endidx):

        self.init_packages()
        if not text:
            completions = self.packages[:]
            self.packages = []
        else:
            completions = [ f
                            for f in self.packages
                            if f.startswith(text)
                            ]
            self.packages = []
        return completions

    def complete_uninstall(self, text, line, begidx, endidx):

        self.init_packages()
        if not text:
            completions = self.packages[:]
            self.packages = []
        else:
            completions = [ f
                            for f in self.packages
                            if f.startswith(text)
                            ]
            self.packages = []
        return completions

    def do_broadcast(self,line):
        try:
            output=os.popen("adb -s {} shell 'am broadcast -a {}'".format(self.device.id,line.split(' ')[0])).read()
            print(output)
        except Exception as e:
            print(e)
            return

    def complete_broadcast(self, text, line, begidx, endidx):

        if not text:
            completions = self.filters[:]
        else:
            completions = [ f
                            for f in self.filters
                            if f.startswith(text)
                            ]
        return completions

    def print_list(self,lst):
        print(GREEN)
        for item in lst:
            if type(item) is str:
                print('\t\t'+item)
        print(RESET)

    def print_strings(self,lst):

        for item in lst:
            print(GREEN+'KEY:' +RESET+'{}'.format(item.split('=')[0],) + GREEN+'\t,VAL:'+RESET+' {}'.format(item.split('=')[1] ))

    def do_show(self,line):
        what = line.split(' ')[0]
        if 'permissions' in what:
            self.print_list(self.permissions)
        elif 'activities' in what:
            self.print_list(self.activities)
        elif 'services' in what:
            self.print_list(self.services)
        elif 'receivers' in what:
            self.print_list(self.receivers)
        elif 'filters' in what:
            self.print_list(self.filters)
        elif 'providers' in what:
            self.print_list(self.providers)
        elif 'strings' in what:
            self.print_strings(self.strings)
        else:
            print('[i] Usage: show [permissions, activities, services, receivers, filters, providers, strings]')

    def complete_show(self, text, line, begidx, endidx):
        components = ['permissions', 'activities', 'services', 'receivers', 'filters','providers', 'strings']
        if not text:
            completions = components[:]
        else:
            completions = [ f
                            for f in components
                            if f.startswith(text)
                            ]
        return completions

    def do_adb(self,line):
        print("[i] Type 'exit' to return ")
        cmd = ''
        while cmd != 'exit':
            cmd = input(GREEN+'{}:adb:'.format(self.device.id)+RESET)
            if cmd != 'exit':
                subprocess.run('adb -s {} {}'.format(self.device.id,cmd), shell=True)

    def do_proxy(self,line):

        command = line.split(' ')[0]
        try:
            if 'get' in command:
                self.print_proxy()
            elif 'reset' in command:
                os.popen("adb -s {} shell settings put global http_proxy :0".format(self.device.id))
                os.popen("adb -s {} shell iptables -t nat -F".format(self.device.id))
                time.sleep(2)
                self.print_proxy()
            elif 'set' in command:
                switch = ip = line.split(' ')[1].split(':')[0]
                if '-t' in switch:
                    ip = line.split(' ')[2].split(':')[0]
                    port = line.split(' ')[2].split(':')[1]
                    self.transproxy(ip,port)
                else:
                    ip = line.split(' ')[1].split(':')[0]
                    port = line.split(' ')[1].split(':')[1]
                    os.popen("adb -s {} shell settings put global http_proxy {}:{}".format(self.device.id,ip,port))
                    time.sleep(2)
                    self.print_proxy()
            else:
                print('[!] Usage: proxy [set,get,reset] [<ip>:<port>] [-t]')
        except Exception as e:
            print(e)

    def transproxy(self,ip,port):
        try:
            print('[i] Pushing transproxy script !')
            os.popen("adb -s {} push utils/transproxy.sh /data/local/tmp/transproxy.sh".format(self.device.id))
            print('[i] Executing script')
            os.popen("adb -s {} shell 'chmod +x /data/local/tmp/transproxy.sh; /data/local/tmp/transproxy.sh {} {}; rm /data/local/tmp/transproxy.sh'".format(self.device.id,ip,port))
            self.print_proxy()
        except Exception as e:
            print(e)

    def do_installBurpCert(self,line):
        try:
            a = ''
            while a != 'y' and a !='x':
                a = input("""[!] Make sure that burp is running on 127.0.0.1:8080\n\nType 'y' to continue or 'x' to cancel:""")

            if a == 'y':
                os.popen("chmod +x utils/installBurpCert.sh; utils/installBurpCert.sh {}".format(self.device.id))
                os.popen("adb -s {} shell am broadcast -a com.medusa.INSTALL_CERTIFICATE -n com.medusa.agent/.Receiver".format(self.device.id))
                time.sleep(1)
                print()
        except Exception as e:
            print('')

    def complete_proxy(self, text, line, begidx, endidx):
        proxy_cmd = ['set','get','reset']
        if not text:
            completions = proxy_cmd[:]
        else:
            completions = [ f
                            for f in proxy_cmd
                            if f.startswith(text)
                            ]
        return completions

    def print_proxy(self):

        settings = os.popen("adb -s {} shell settings get global http_proxy".format(self.device.id)).read()
        print (WHITE+"--------------Global proxy settings-----------------:"+RESET)
        print ('Current proxy: {}'.format(settings))
        print (WHITE+"--------------IP tables settings--------------------:"+RESET)
        output = subprocess.run("adb -s {} shell iptables -t nat -L".format(self.device.id), shell=True)
        print(output)

    def do_screencap(self, line):
        try:
            if '-o' in line.split(' ')[0]:
                os.popen("adb -s {} exec-out screencap -p > {}".format(self.device.id,line.split(' ')[1]))
                print('[!] Screencap saved successfully to {}'.format(line.split(' ')[1]))
            else:
                print('[!] Usage: screencap -o filename.png')
        except Exception as e:
            print(e)
            print('[!] Usage: screencap -o filename.png')

    def do_jdwp(self,line):
        try:
            pid = os.popen("adb -s {} shell pidof {}".format(self.device.id,line.split(' ')[0])).read()
            output = os.popen("adb -s {} forward tcp:6667 jdwp:{}".format(self.device.id,pid)).read()
            print(output)

            subprocess.run('jdb -attach localhost:6667', shell=True)

        except Exception as e:
            print(e)
            print('[!] Usage: jdwp package_name')

    def complete_jdwp(self, text, line, begidx, endidx):

        self.init_packages()
        if not text:
            completions = self.packages[:]
            self.packages = []
        else:
            completions = [ f
                            for f in self.packages
                            if f.startswith(text)
                            ]
            self.packages = []
        return completions

    def do_start(self,line):
        try:
            output=os.popen("adb -s {} shell 'am start -n {}/{}'".format(self.device.id,self.package,line.split(' ')[0])).read()
            print(output)
        except Exception as e:
            print(e)
            return

    def complete_start(self, text, line, begidx, endidx):
        if not text:
            completions = self.activities[:]
        else:
            completions = [ f
                            for f in self.activities
                            if f.startswith(text)
                            ]
        return completions

    def do_startsrv(self,line):
        try:
            output=os.popen("adb -s {} shell 'am startservice -n {}/{}'".format(self.device.id,self.package,line.split(' ')[0])).read()
            print(output)
        except Exception as e:
            print(e)
            return

    def complete_startsrv(self, text, line, begidx, endidx):
        if not text:
            completions = self.services[:]
        else:
            completions = [ f
                            for f in self.services
                            if f.startswith(text)
                            ]
        return completions

    def do_stopsrv(self,line):
        try:
            output=os.popen("adb -s {} shell 'am stopservice -n {}/{}'".format(self.device.id,self.package,line.split(' ')[0])).read()
            print(output)
        except Exception as e:
            print(e)
            return

    def complete_stopsrv(self, text, line, begidx, endidx):
        if not text:
            completions = self.services[:]
        else:
            completions = [ f
                            for f in self.services
                            if f.startswith(text)
                            ]
        return completions

    def do_type(self,line):

        print("Type 'exit' to quit")

        while 'exit' not in line:
            line = input(':')
            os.popen("adb -s {} shell input text {}".format(self.device.id,line))

    def do_clear(self,line):
        os.system('clear')

    def do_shell(self,line):
        shell = os.environ['SHELL']
        subprocess.run('{}'.format(shell), shell=True)

    def do_exit(self,line):
        print('[i] Cleaning working directory: ')
        try:
            if os.path.isfile('./manifest.xml'):
                ask = input('\n[!] do you want to delete the manifest file ? (yes/no) ')
                if 'yes' in ask:
                    os.remove('./manifest.xml')

            if os.path.isfile('./strings.xml'):
                ask = input('\n[!] do you want to delete the strings.xml file ? (yes/no) ')
                if 'yes' in ask:
                    os.remove('./strings.xml')

            if os.path.isfile('./script.sh'):
                ask = input('\n[!] do you want to delete the trace script file ? (yes/no) ')
                if 'yes' in ask:
                    os.remove('./script.sh')

            if os.path.isfile('./script.bat'):
                ask = input('\n[!] do you want to delete the trace script file ? (yes/no) ')
                if 'yes' in ask:
                    os.remove('./script.bat')

            if self.INSTALL == True:
                uninstall = input("[!] Do you want to uninstall the apk ? (yes/no)")
                if 'yes' in uninstall:
                    subprocess.run('adb -s {} uninstall {}'.format(self.device.id,self.package),shell=True)

            if os.path.exists("__handlers__/"):
                uninstall = input("[!] Do you want to delete the __handlers__ folder? (yes/no)")
                if 'yes' in uninstall:
                    os.system("rm -r __handlers__/")

        except Exception as e:
            print(e)

        print('Bye !!')
        exit()

    def do_installagent(self,line):
        try:
            subprocess.run('adb -s {} install -g {}'.format(self.device.id, os.getcwd()+'/dependencies/agent.apk'),shell=True)
        except Exception as e:
            print(e)

    def do_help(self,line):
        if line != '':
            print('\n'+BLUE+self.display_tag(line,'Help')+RESET)
        else:
            print("""Available commands:

                    [+] TRACE Functions using frida-trace:
                    ---------------------

                    - trace -j com.myapp.name*          : Trace all the functions of the com.myapp.name* class
                    - trace -n foo                      : Trace a native function
                    - trace -a library.so               : Trace the functions of the library.so

                    Will spawn a new frida-trace instance with the given options

                    ===========================================================================================

                    [+] MANIFEST PARSER:
                    ---------------------

                    - show permissions          : Print the application's permissions
                    - show activities           : Print a list with the application's activities
                    - show services             : Print a list with the application's services
                    - show receivers            : Print a list with the application's receivers
                    - show providers            : Print a list with the application's content providers
                    - show filters              : Print broadcast filters
                    - show strings              : print application's strings
                    - search [keyword]          : Search components containing the given keyword
                    ===========================================================================================

                    [+] TRIGERS:
                    ---------------------

                    - start      [tab]          : Start and activity
                    - deeplink   [tab]          : Trigger a deeplink
                                                  Add the --poc to create an html poc file
                    - startsrv   [tab]          : Start a service
                    - stopsrv    [tab]          : Stop a service
                    - broadcast  [tab]          : Broadcast an intent
                    - spawn      [tab]          : Spawn an application

                    ===========================================================================================

                    [+] UTILITIES:
                    ---------------------
                    - pull       [tab]              : extract apk from the device
                    - installagent                  : Install the Medusa apk
                    - installBurpCert               : Install Burp Certificate
                    - notify subject body           : Display a notification to the phone's notification bar
                    e.g. notify test foo            (Requires the medusa agent to be installed and run)

                    - jdwp  package_name            : Open a jdb session with the debugger attached to the package
                                                    (Requires the --patch option)

                    - adb [cmd]                     : Send an adb command to the connected device
                    - clear                         : Clears the screen
                    - kill [tab]                    : Kill an app by the package name
                    - type                          : Type text to send to the device
                    - screencap -o filename         : Takes a device screenshot and saves it as 'filaname'
                    - shell                         : Opens an interactive shell
                    - proxy set [-t] <ip>:<port>    : Sets a global proxy at a given ip and port
                                                    ('-t' for transparent)
                    - proxy get                     : Displays proxy settings of the device
                    - proxy reset                   : Resets proxy settings
                    - uninstall [tab]               : Uninstals a packages from the device

                            """)


from Ui import Ui
from Data import Data
from Parse import Parse
import PySimpleGUI as gui
from Analysis import Analysis

import numpy as np

CANCEL = "Cancel"
EXIT = "Exit"

TIMEOUT_VALUE = 30
VALUE_START = 0
VALUE_END = 1000

if __name__ == "__main__":

    ui = Ui()
    window = ui.get_ui()
    progress_bar = ui.get_progress_bar()

    while True:

        event, values = window.Read(close=False)

        if event in (gui.WIN_CLOSED, EXIT):
            break

        company = values[0]

        data = Data()

        parse = Parse(company=company)
        articles_of_interest = parse.get_articles_of_interest()
        number_of_articles = parse.get_len_articles_of_interest()
        step = VALUE_END / number_of_articles
        index = VALUE_START

        analysis = Analysis(company=company, data=articles_of_interest, size=number_of_articles)

        for article in np.arange(VALUE_START, VALUE_END, step):
            analysis.analyse(index)
            index += 1
            progress_bar.UpdateBar(article)

        progress_bar.UpdateBar(VALUE_END)

        analysis.show_analysis()

        print("\nYou can key in another company's name to get started with a new analysis.")


from flask import Blueprint, render_template, abort, request, redirect, url_for, g

from .. import MediaArchive, populate_id, id_to_uuid, id_to_md5
from pagination_from_request import pagination_from_request
from regex_converter import RegexConverter
from media import MediumStatus, MediumSearchability, MediumProtection
from accounts.views import require_sign_in, require_global_group

def initialize(config, media, accounts, access_log):
	g.media_archive = MediaArchive(config, media, accounts, access_log)

media_archive = Blueprint(
	'media_archive',
	__name__,
	template_folder='templates',
	static_folder='static',
	static_url_path='/static'
)

@media_archive.route('/upload', methods=['GET', 'POST'])
@require_global_group('contributor')
def upload(api=False):
	manager = False
	contributors = []
	api_uris = {}
	if g.media_archive.accounts.has_global_group(g.media_archive.accounts.current_user, 'manager'):
		manager = True
		contributors = g.media_archive.get_contributors()
		api_uris = g.media_archive.get_api_uris()

	fields = {
		'groups': [],

		'generate_summaries': g.media_archive.config['upload_defaults']['generate_summaries'],
		'owner_id': g.media_archive.accounts.current_user.id,
		'status': 'allowed',
		'searchability': g.media_archive.config['upload_defaults']['searchability'],
		'protection': g.media_archive.config['upload_defaults']['protection'],
		'creation_date': '',

		'author_tag': g.media_archive.config['upload_defaults']['author_tag'],
		'filename_tag': g.media_archive.config['upload_defaults']['filename_tag'],
		'tags': '',

		'upload_uri': '',
	}

	errors = []
	if 'POST' != request.method:
		if api:
			abort(405)
		return render_template(
			'upload_media.html',
			errors=errors,
			manager=manager,
			contributors=contributors,
			api_uris=api_uris,
			fields=fields,
			groups=g.media_archive.config['requirable_groups'],
			medium=None,
		)

	errors, medium = g.media_archive.upload_from_request()

	if 0 == len(errors):
		g.media_archive.add_log(
			'create_medium',
			g.media_archive.accounts.current_user.uuid,
			id_to_uuid(medium.id)
		)
		# delay and re-fetch to get accurate uris
		import time

		time.sleep(0.5)
		medium = g.media_archive.get_medium(medium.md5)

		if 'create_medium' in g.media_archive.callbacks:
			for f in g.media_archive.callbacks['create_medium']:
				f(medium)

		if api:
			from statuspages import success
			return success({
				'id': medium.id,
				'view_uri': url_for('media_archive.view_medium', medium_id=medium.id, _external=True),
				'thumbnail': (
					render_template('medium_thumbnail.html', medium=medium)
					.replace('\n', '')
					.replace('\r', '')
					.replace('\t', '')
				),
			})

		return redirect(url_for('media_archive.view_medium', medium_id=medium.id), 302)

	if api:
		if 'medium_already_exists' in errors:
			abort(409, {
				'errors': errors,
				'view_uri': url_for('media_archive.view_medium', medium_id=medium.id, _external=True),
			})
		abort(400, {
			'errors': errors,
		})

	for field in ['generate_summaries', 'author_tag', 'filename_tag']:
		fields[field] = (field in request.form)

	form_fields = [
		'owner_id',
		'searchability',
		'protection',
		'creation_date',
		'tags',
		'upload_uri',
	]
	for field in form_fields:
		if field in request.form:
			fields[field] = request.form[field]

	for group in g.media_archive.accounts.users.available_groups:
		field = 'groups[' + group + ']'
		if field in request.form:
			fields['groups'].append(group)

	return render_template(
		'upload_media.html',
		errors=errors,
		manager=manager,
		contributors=contributors,
		api_uris=api_uris,
		fields=fields,
		groups=g.media_archive.config['requirable_groups'],
		medium=medium,
	)

@media_archive.route('/help')
def help():
	return render_template('media_help.html')

def populate_medium_protected_property(medium):
	medium.protected = False
	if (
			g.media_archive.accounts.current_user
			and g.media_archive.accounts.current_user.uuid == medium.owner_uuid
		):
		return
	if (
			(
					0 < int.from_bytes(medium.group_bits, 'big')
					or MediumProtection.NONE != medium.protection
				)
			and (
				not g.media_archive.accounts.current_user
				or not g.media_archive.accounts.current_user_has_permissions(
						medium.group_bits,
						'global'
					)
				)
		):
			medium.protected = True

def search(
		current_endpoint,
		overrides={},
		search_field=True,
		hide_tags_this_page=False,
		manage=False,
		omit_future=True,
		medium_id=None,
		slideshow_endpoint=None,
		header=None,
		**kwargs,
	):
	tags_query = ''
	if 'tags' in request.args:
		tags_query = request.args['tags']

	tags = g.media_archive.tag_string_to_list(tags_query)
	limited_tags = []
	for tag in tags:
		if g.media_archive.config['maximum_search_tags'] == len(limited_tags):
			break
		limited_tags.append(tag)
	tags = limited_tags

	if 'sort:random' in tags:
		tags.remove('sort:random')
		import random, string
		seed = ''.join(random.choices(string.ascii_letters + string.digits, k=6)).upper()
		tags.append('sort:random:' + seed)
		return redirect(url_for(current_endpoint, tags='#'.join(tags)), 302)

	if 0 < len(overrides):
		if 'tags' in overrides:
			tags = overrides['tags']
		else:
			if 'add_tags' in overrides:
				tags += overrides['add_tags']
			if 'remove_tags' in overrides:
				for tag in overrides['remove_tags']:
					if tag in tags:
						tags.remove(tag)

	if omit_future:
		from datetime import datetime, timezone
		import time
		current_datetime_atom = datetime.fromtimestamp(
			time.time(),
			timezone.utc
		).strftime('%Y-%m-%dT%H:%M:%S.%f%z')
		tags.append('created before:' + current_datetime_atom)

	filter = {}
	if tags:
		filter = g.media_archive.parse_search_tags(tags, manage)

	pagination = {
		'page': 0,
		'sort': 'creation',
		'order': 'desc',
		'perpage': 32,
	}
	if 'page' in request.args:
		pagination['page'] = int(request.args['page'])
	if 'sort' in filter:
		pagination['sort'] = filter['sort']
	if 'order' in filter:
		pagination['order'] = filter['order']
	if 'perpage' in filter:
		pagination['perpage'] = int(filter['perpage'])

	if 0 < len(overrides) and 'filter' in overrides:
		for key, value in overrides['filter'].items():
			filter[key] = value

	if 'export' in request.args:
		if not g.media_archive.accounts.current_user_has_global_group('manager'):
			if 'confirm' in request.args:
				#TODO store export request filter somewhere
				#print('export request by ' + g.media_archive.accounts.current_user.id + ' with filter:')
				#print(filter)
				g.media_archive.add_log(
					'export_media_request',
					g.media_archive.accounts.current_user.uuid
				)
				return redirect(url_for(current_endpoint, tags=tags_query, **kwargs), 302)
			return render_template('export_media_request.html')

		total_media = g.media_archive.media.count_media(filter=filter)
		bytesize = g.media_archive.media.media_size(filter=filter)

		if 'confirm' not in request.args:
			return render_template('export_media.html', total_media=total_media, bytesize=bytesize)

		g.media_archive.add_log(
			'export_media',
			g.media_archive.accounts.current_user.uuid
		)
		return g.media_archive.export_media(filter)

	# slideshow mode for this search
	if not slideshow_endpoint:
		slideshow_endpoint = current_endpoint
	if medium_id:
		# make sure specified medium is real
		medium = g.media_archive.require_medium(id_to_md5(medium_id))
		# get adjacent media
		prev_medium_id, next_medium_id = g.media_archive.get_adjacent_media(
			medium,
			filter=filter,
			sort=pagination['sort']
		)
		adjacent_media = (prev_medium_id, next_medium_id, slideshow_endpoint)
		return view_medium(medium_id, adjacent_media, tags=tags_query, **kwargs)

	media = g.media_archive.search_media(filter=filter, **pagination)
	total_media = g.media_archive.media.count_media(filter=filter)

	g.media_archive.populate_media_covers(media)

	contributors = []
	api_uris = {}
	if manage:
		contributors = g.media_archive.get_contributors()
		api_uris = g.media_archive.get_api_uris()

	# mark protected media if not managing
	else:
		for medium in media:
			populate_medium_protected_property(medium)

	tags_this_page = []
	if not hide_tags_this_page:
		for medium in media:
			if 0 < len(medium.tags):
				for tag in medium.tags:
					if tag not in tags_this_page:
						tags_this_page.append(tag)

	tag_suggestion_lists = g.media_archive.get_tag_suggestion_lists(manage=manage, search=True)

	import math
	import re

	return render_template(
		'search_media.html',
		media=media,
		tags_query=tags_query,
		header=header,
		search_field=search_field,
		tag_suggestion_lists=tag_suggestion_lists,
		manage=manage,
		tags_this_page=tags_this_page,
		pagination=pagination,
		total_media=total_media,
		total_pages=math.ceil(total_media / pagination['perpage']),
		current_endpoint=current_endpoint,
		slideshow_endpoint=slideshow_endpoint,
		groups=g.media_archive.config['requirable_groups'],
		contributors=contributors,
		api_uris=api_uris,
		re=re,
		kwargs=kwargs,
	)

@media_archive.route('/tags')
def manage_tags():
	search = {
		'tag': '',
	}
	if 'tag' in request.args:
		search['tag'] = request.args['tag']

	filter = {}
	escape = lambda value: (
		value
			.replace('\\', '\\\\')
			.replace('_', '\_')
			.replace('%', '\%')
			.replace('-', '\-')
	)
	if search['tag']:
		filter['tags'] = '%' + escape(search['tag']) + '%'

	pagination = pagination_from_request('count', 'desc', 0, 32)

	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		g.media_archive.accounts.require_global_group('contributor')
		filter['owner_uuid'] = g.media_archive.accounts.current_user.uuid

	total_tags = g.media_archive.media.count_tags(filter=filter, group=True)
	tags = g.media_archive.media.search_tags(filter=filter, **pagination, group=True)

	import math

	return render_template(
		'tags_list.html',
		tags=tags,
		search=search,
		pagination=pagination,
		total_tags=total_tags,
		total_pages=math.ceil(total_tags / pagination['perpage']),
	)

@media_archive.route('/tags/<mode>', methods=['GET', 'POST'])
def edit_tag(mode):
	owner_uuid = None
	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		g.media_archive.accounts.require_global_group('contributor')
		owner_uuid = g.media_archive.accounts.current_user.uuid

	if mode not in ['remove', 'replace', 'accompany']:
		abort(400)

	tag = ''
	if 'tag' in request.args:
		tag = request.args['tag']

	tag2 = ''
	for key in ['replacement', 'accompaniment']:
		if key in request.args:
			tag2 = request.args[key]

	if 'POST' != request.method:
		return render_template(
			'edit_tag.html',
			mode=mode,
			tag=tag,
			tag2=tag2,
		)

	if 'tag' in request.form and request.form['tag']:
		if 'remove' == mode:
			g.media_archive.media.remove_tag(tag, owner_uuid)
		elif 'tag2' in request.form and request.form['tag2']:
			if 'replace' == mode:
				g.media_archive.media.replace_tag(
					request.form['tag'],
					request.form['tag2'],
					owner_uuid
				)
			elif 'accompany' == mode:
				filter = {
					'with_tags': request.form['tag'],
				}
				if owner_uuid:
					filter['owner_uuids'] = owner_uuid
				media = g.media_archive.search_media(filter=filter)
				if media:
					g.media_archive.media.add_tags(media, request.form['tag2'])

	return redirect(url_for('media_archive.manage_tags'), 302)

@media_archive.route('/manage/<medium_id>')
def manage_slideshow(medium_id):
	return manage_media(medium_id=medium_id)

@media_archive.route('/manage')
def manage_media(medium_id=None):
	filter = {}

	if not g.media_archive.accounts.has_global_group(g.media_archive.accounts.current_user, 'manager'):
		# contributor manage self
		g.media_archive.accounts.require_global_group('contributor')
		filter['owner_uuids'] = g.accounts.current_user.uuid

	return search(
		'media_archive.manage_media',
		overrides={'filter': filter},
		manage=True,
		omit_future=False,
		medium_id=medium_id,
		slideshow_endpoint='media_archive.manage_slideshow',
	)

@media_archive.route('/import', methods=['GET', 'POST'])
@require_sign_in
def import_media():
	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		abort(403, {'message': 'import_requires_manager'})

	if 'POST' == request.method:
		if 'file_upload' not in request.files:
			abort(400, {'message': 'missing_import_file'})

		if 'confirm' not in request.form:
			info, data, other_files = g.media_archive.analyze_import_file(request.files['file_upload'])
			#TODO get users for any media uploaders or owners in data
			users = {}
			return render_template(
				'import_media.html',
				info=info,
				data=data,
				other_files=other_files,
				users=users,
			)

		# do actual import
		g.media_archive.add_log(
			'import_media',
			g.media_archive.accounts.current_user.uuid
		)
		g.media_archive.import_media(
			request.files['file_upload'],
			('generate_summaries' in request.form)
		)
		return redirect(url_for('media_archive.manage_media'), 302)

	return render_template('import_media.html')

def api_medium_ids_to_list():
	if 'medium_ids' not in request.form:
		abort(400)
	medium_ids = request.form['medium_ids'].split(',')
	if not medium_ids:
		abort(400)
	return medium_ids

@media_archive.route('/api/media/fetch/<medium_filename>', methods=['GET', 'POST'])
def api_fetch_medium(medium_filename):
	if (
			'POST' == request.method
			and 'response_type' in request.form
			and 'json' == request.form['response_type']
		):
		g.json_request = True

	import os

	filename_pieces = medium_filename.split('.')
	if 2 > len(filename_pieces):
		#TODO maybe actually feed 404 image fetch svg back?
		abort(404, {'message': 'medium_not_found'})

	medium_id = filename_pieces[0]
	medium = g.media_archive.require_medium(id_to_md5(medium_id))

	# trying to access nonprotected medium through protected medium path
	if MediumProtection.NONE == medium.protection:
		#TODO maybe actually feed 400 image fetch svg back?
		abort(400)

	e = check_protection(medium)
	if e:
		if hasattr(g, 'json_request') and g.json_request:
			raise
		#TODO maybe actually feed protected/premium image fetch svg back?
		return e

	if 2 == len(filename_pieces):
		medium_extension = filename_pieces[1]
		# reencode
		if (
				'video' == medium.category
				and 'webm' == medium_extension
				and not is_websafe_video(medium.mime)
			):
			media_path = os.path.join(g.media_archive.config['summaries_path'], 'protected')
		else:
			media_path = os.path.join(g.media_archive.config['media_path'], 'protected')
	else:
		media_path = os.path.join(g.media_archive.config['summaries_path'], 'protected')

	if not os.path.exists(os.path.join(media_path, medium_filename)):
		#TODO maybe actually feed 404 image fetch svg back?
		abort(404, {'message': 'medium_not_found'})

	g.no_store = True

	from flask import send_from_directory

	return send_from_directory(media_path, medium_filename, mimetype=medium.mime, conditional=True)

@media_archive.route('/api/media/edit', methods=['POST'])
def api_edit_medium():
	g.json_request = True
	medium_ids = api_medium_ids_to_list()
	if 1 < len(medium_ids):
		abort(400)
	return edit_medium(medium_ids[0], True)

@media_archive.route('/api/media/remove', methods=['POST'])
def api_remove_medium():
	g.json_request = True
	medium_ids = api_medium_ids_to_list()
	if 1 < len(medium_ids):
		abort(400)
	return remove_medium(medium_ids[0], True)

@media_archive.route('/api/media/build', methods=['POST'])
def api_build_medium():
	g.json_request = True
	medium_ids = api_medium_ids_to_list()
	if 1 < len(medium_ids):
		abort(400)
	return generate_summaries(medium_ids[0], True)

@media_archive.route('/api/media/set', methods=['POST'])
def api_generate_set():
	g.json_request = True
	g.media_archive.accounts.require_sign_in();
	medium_ids = api_medium_ids_to_list()
	media = g.media_archive.search_media(filter={'ids': medium_ids})

	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		for medium in media:
			if medium.owner_id != g.media_archive.accounts.current_user.id:
				abort(403)

	import uuid
	from .. import uuid_to_id

	set_tag = 'set:' + uuid_to_id(uuid.uuid4())

	g.media_archive.media.add_tags(media, set_tag)
	g.media_archive.media.touch_media(media)

	#TODO the media module should modify media in place so another fetch isn't necessary
	media = g.media_archive.search_media(filter={'ids': medium_ids})
	rendered = {}
	for medium in media:
		tags_string = '#'.join(medium.tags)
		if tags_string:
			tags_string = '#' + tags_string
		rendered[medium.id] = {
			'set_tiles': (
				render_template('medium_set_tiles.html', medium=medium)
				.replace('\n', '')
				.replace('\r', '')
				.replace('\t', '')
			),
			'tags': tags_string,
		}

	from statuspages import success
	return success({
		'media': rendered
	})

def api_modify_tags(mode):
	g.json_request = True
	g.media_archive.accounts.require_sign_in();
	medium_ids = api_medium_ids_to_list()
	media = g.media_archive.search_media(filter={'ids': medium_ids})

	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		for medium in media:
			if medium.owner_id != g.media_archive.accounts.current_user.id:
				abort(403)

	tags_list = g.media_archive.tag_string_to_list(request.form['tags'])
	if 'add' == mode:
		g.media_archive.media.add_tags(media, tags_list)
	elif 'remove' == mode:
		g.media_archive.media.remove_tags(media, tags_list)
	elif 'set' == mode:
		g.media_archive.media.remove_tags(media)
		g.media_archive.media.add_tags(media, tags_list)
	else:
		abort(400)

	g.media_archive.media.touch_media(media)

	#TODO the media module should modify media in place so another fetch isn't necessary
	media = g.media_archive.search_media(filter={'ids': medium_ids})
	rendered = {}
	for medium in media:
		tags_string = '#'.join(medium.tags)
		if tags_string:
			tags_string = '#' + tags_string
		rendered[medium.id] = {
			'tags': tags_string
		}

	g.media_archive.build_tag_suggestion_lists()

	from statuspages import success
	return success({
		'media': rendered
	})

@media_archive.route('/api/tags/add', methods=['POST'])
def api_add_tags():
	return api_modify_tags('add')

@media_archive.route('/api/tags/remove', methods=['POST'])
def api_remove_tags():
	return api_modify_tags('remove')

@media_archive.route('/api/tags/set', methods=['POST'])
def api_set_tags():
	return api_modify_tags('set')

@media_archive.route('/api/media/upload', methods=['POST'])
def api_upload_medium():
	g.json_request = True
	return upload(True)

@media_archive.route('/file/media/<medium_filename>')
def medium_file(medium_filename):
	import os

	summary_file = os.path.join(
		g.media_archive.config['summaries_path'],
		'nonprotected',
		medium_filename
	)
	medium_file = os.path.join(
		g.media_archive.config['media_path'],
		'nonprotected',
		medium_filename
	)
	if os.path.exists(summary_file):
		directory = os.path.join(
			g.media_archive.config['summaries_path'],
			'nonprotected'
		)
	elif os.path.exists(medium_file):
		directory = os.path.join(
			g.media_archive.config['media_path'],
			'nonprotected'
		)
	else:
		abort(404, {'message': 'medium_not_found'})

	from flask import send_from_directory

	return send_from_directory(directory, medium_filename, conditional=True)

@media_archive.route('/file/tags/<tags_filename>')
def tags_file(tags_filename):
	import os

	tags_file = os.path.join(
		g.media_archive.config['tags_path'],
		tags_filename
	)
	if not os.path.exists(tags_file):
		abort(404, {'message': 'tags_not_found'})

	from flask import send_from_directory

	return send_from_directory(
		g.media_archive.config['tags_path'],
		tags_filename,
		conditional=True
	)

def check_protection(medium, slideshow=None, require_groups=False, **kwargs):
	import werkzeug.exceptions

	from statuspages import PaymentRequired

	protected = False
	groups = []
	try:
		g.media_archive.require_access(medium, require_groups)
	except werkzeug.exceptions.Unauthorized as e:
		protected = True
		if 'groups' in e.description:
			groups = e.description['groups']
	except werkzeug.exceptions.Forbidden as e:
		protected = True
		if 'groups' in e.description:
			groups = e.description['groups']
	except PaymentRequired as e:
		if 'groups' in e.description:
			groups = e.description['groups']
		return render_template(
			'premium_media.html',
			groups=groups,
			slideshow=slideshow,
			kwargs=kwargs
		)
	if protected:
		return render_template(
			'protected_media.html',
			groups=groups,
			slideshow=slideshow,
			kwargs=kwargs
		)

	return None

@media_archive.route('/' + "<regex('([a-zA-Z0-9_\-]+)'):medium_id>", methods=['GET', 'POST'])
def view_medium(medium_id, slideshow=None, **kwargs):
	medium = g.media_archive.require_medium(id_to_md5(medium_id))

	e = check_protection(medium, slideshow, require_groups=True, **kwargs)
	if e:
		return e

	edit_medium = False
	edit_tags = False
	tag_suggestion_lists = []

	if (
			g.media_archive.accounts.current_user
			and (
				medium.owner_uuid == g.media_archive.accounts.current_user.uuid
				or g.media_archive.accounts.current_user_has_global_group('manager')
			)
		):
		edit_medium = True
		edit_tags = True
		tag_suggestion_lists = g.media_archive.get_tag_suggestion_lists(manage=True, search=False)

	if edit_tags and 'POST' == request.method:
		tags = g.media_archive.tag_string_to_list(request.form['tags'])
		g.media_archive.media.remove_tags(medium)
		if 0 < len(tags):
			#TODO clearing tags to force tag addition works
			#TODO but probably should be handled inside the media module
			medium.tags = []
			g.media_archive.media.add_tags(medium, tags)
		g.media_archive.build_tag_suggestion_lists()
		if 'tags' in request.args and request.args['tags']:
			return redirect(
				url_for(
					'media_archive.view_medium',
					medium_id=medium_id,
					tags=request.args['tags']
				),
				302
			)
		return redirect(
			url_for(
				'media_archive.view_medium',
				medium_id=medium_id
			),
			302
		)

	next_medium_id = ''
	prev_medium_id = ''
	sets = {}
	visible_tags = 0
	for tag in medium.tags:
		if 'next:' == tag[:5]:
			next_medium_id = tag[5:]
			continue
		if 'prev:' == tag[:5]:
			prev_medium_id = tag[5:]
			continue
		if 'set:' == tag[:4]:
			set_name = tag[4:]
			colon_pos = set_name.find(':')
			if 0 < colon_pos:
				set_name = set_name[:colon_pos]
			sets[set_name] = []
		if (
				'filename:' == tag[:9]
				or 'set:' == tag[:4]
				or 'cover:' == tag[:6]
				or 'mirror:' == tag[:7]
				or 'superior of:' == tag[:12]
				or 'inferior of:' == tag[:12]
				or 'next:' == tag[:5]
				or 'prev:' == tag[:5]
				or 'suppress:' == tag[:9]
			):
			continue
		visible_tags += 1

	# get set media
	if sets:
		escape = lambda value: (
			value
				.replace('\\', '\\\\')
				.replace('_', '\_')
				.replace('%', '\%')
				.replace('-', '\-')
		)
		for set_name in sets:
			media = g.media_archive.search_media(
				filter={
					'with_tags_like': 'set:' + escape(set_name) + '%',
				},
				sort='creation_time',
				order='asc',
			)
			unordered = []
			ordered = {}
			g.media_archive.populate_media_covers(media)
			for set_medium in media:
				populate_medium_protected_property(set_medium)
				for tag in set_medium.tags:
					if 'set:' != tag[:4]:
						continue
					current_set_name = tag[4:]
					colon_pos = current_set_name.find(':')
					order = None
					if 0 < colon_pos:
						current_set_name = current_set_name[:colon_pos]
						order = int(current_set_name[colon_pos:])
					if set_name != current_set_name:
						continue
					if None == order:
						unordered.append(set_medium)
					else:
						ordered[order] = set_medium
					break
			sorted_ordered = []
			for key, value in sorted(ordered.items()):
				sorted_ordered.append(value)
			sets[set_name] = sorted_ordered + unordered
		medium.sets = sets

	navigation_endpoint = 'media_archive.view_medium'
	if slideshow:
		prev_medium_id, next_medium_id, navigation_endpoint = slideshow

	g.media_archive.populate_media_covers(medium)

	return render_template(
		'view_medium.html',
		medium=medium,
		edit_medium=edit_medium,
		edit_tags=edit_tags,
		tag_suggestion_lists=tag_suggestion_lists,
		search_endpoint=g.media_archive.config['search_endpoint'],
		visible_tags=visible_tags,
		next_medium_id=next_medium_id,
		prev_medium_id=prev_medium_id,
		navigation_endpoint=navigation_endpoint,
		kwargs=kwargs,
	)

@media_archive.route('/' + "<regex('([a-zA-Z0-9_\-]+)'):medium_id>/edit", methods=['GET', 'POST'])
@require_sign_in
def edit_medium(medium_id, api=False):
	manager = False
	contributors = []
	if g.media_archive.accounts.has_global_group(g.media_archive.accounts.current_user, 'manager'):
		manager = True
		contributors = g.media_archive.get_contributors()

	medium = g.media_archive.get_medium(id_to_md5(medium_id))

	if not manager:
		if not medium or MediumStatus.ALLOWED != medium.status:
			abort(403, {'message': 'manager_required'})

		g.media_archive.accounts.require_global_group('contributor')
		if medium.owner_uuid != g.media_archive.accounts.current_user.uuid:
			abort(403, {'message': 'not_medium_owner'})
	elif not medium:
		abort(404, {'message': 'medium_not_found'})

	fields = {
		'groups': [],

		'owner_id': medium.owner_id,
		'status': medium.status.name.lower(),
		'searchability': medium.searchability.name.lower(),
		'protection': medium.protection.name.lower(),
		'creation_time': medium.creation_time,
		'creation_date': medium.creation_time.strftime('%Y-%m-%dT%H:%M:%S.%f%z'),

		'file_uri': '',
	}

	# populate uploader
	if medium.uploader_uuid:
		medium.uploader = g.media_archive.accounts.get_user(medium.uploader_uuid)
		print(medium.uploader)

	for group in g.media_archive.accounts.users.available_groups:
		if g.media_archive.accounts.users.contains_all_group_bits(
				medium.group_bits,
				g.media_archive.accounts.users.group_name_to_bit(group)
			):
			fields['groups'].append(group)

	errors = []
	if 'POST' != request.method:
		if api:
			abort(405)
		return render_template(
			'edit_medium.html',
			errors=errors,
			manager=manager,
			contributors=contributors,
			fields=fields,
			groups=g.media_archive.config['requirable_groups'],
			medium=medium,
		)

	updates = {}

	# manager can set medium owner directly and modify status
	if manager:
		if 'status' in request.form:
			status = request.form['status'].upper()
			if status != medium.status.name:
				updates['status'] = status
		if 'owner_id' in request.form and request.form['owner_id'] != medium.owner_id:
			owner = g.media_archive.accounts.get_user(id_to_uuid(request.form['owner_id']))
			if not owner:
				errors.append('owner_not_found')
				return errors, None
			#TODO only allow contributors to be assigned as owner?
			#self.accounts.users.populate_user_permissions(owner)
			#if not self.accounts.users.has_global_group(owner, 'contributor'):
				#TODO warning for owner not set
			#else:
			updates['owner_uuid'] = owner.uuid

	if 'searchability' in request.form:
		#TODO add exception handling here for ValueErrors
		updates['searchability'] = request.form['searchability'].upper()

	place_files = False
	if 'protection' in request.form:
		protection = request.form['protection'].upper()
		if protection != medium.protection.name:
			#TODO add exception handling here for ValueErrors
			updates['protection'] = protection
			place_files = True

	if 'creation_date' in request.form:
		import dateutil.parser
		try:
			creation_time = dateutil.parser.parse(request.form['creation_date'])
		except ValueError:
			#TODO warning for creation time not set
			pass
		else:
			updates['creation_time'] = creation_time.timestamp()

	if 'groups' in request.form:
		fields['groups'] = []
		for group in g.media_archive.accounts.users.available_groups:
			field = 'groups[' + group + ']'
			if field in request.form:
				fields['groups'].append(group)
		updates['group_bits'] = 0
		if 0 < len(fields['groups']):
			updates['group_bits'] = int.from_bytes(
				g.media_archive.accounts.users.combine_groups(names=fields['groups']),
				'big'
			)

	if 0 < len(updates):
		g.media_archive.media.update_medium(medium, **updates)
		#TODO update medium values in-place to avoid extra fetch
		medium = g.media_archive.get_medium(medium.md5)

	if place_files:
		g.media_archive.place_medium_file(medium)
		g.media_archive.place_medium_summaries(medium)
		#TODO update medium uris in-place to avoid extra fetch
		medium = g.media_archive.get_medium(medium.md5)

	if (
			'file_upload' in request.files
			or (
				'file_uri' in request.form
				and request.form['file_uri']
			)
		):
		errors, new_medium = g.media_archive.upload_from_request()
		if errors:
			#TODO display replacement failed message?
			pass
		else:
			g.media_archive.add_log(
				'replace_medium',
				g.media_archive.accounts.current_user.uuid,
				id_to_uuid(new_medium.id)
			)
			#TODO check if old medium had summaries before attempting to generate summaries
			g.media_archive.generate_medium_summaries(new_medium)
			# move tags to new medium
			g.media_archive.media.move_tags(medium, new_medium)
			g.media_archive.media.update_medium(
				new_medium,
				owner_uuid=medium.owner_uuid,
				status=medium.status,
				searchability=medium.searchability,
				protection=medium.protection,
				group_bits=medium.group_bits
			)
			# remove old medium
			g.media_archive.remove_medium(medium)
			# place replacement medium files
			g.media_archive.place_medium_file(new_medium)
			g.media_archive.place_medium_summaries(new_medium)
			# fetch fresh replacement medium
			medium = g.media_archive.get_medium(new_medium.md5)

	if 0 == len(errors):
		g.media_archive.media.touch_media(medium)
		g.media_archive.add_log(
			'edit_medium',
			g.media_archive.accounts.current_user.uuid,
			id_to_uuid(medium.id)
		)
		if api:
			# delay and re-fetch to get accurate thumbnail
			import time
			time.sleep(0.5)
			medium = g.media_archive.get_medium(medium.md5)

			from statuspages import success
			#TODO return only renders of changed data?
			return success({
				'media': {
					medium.id: {
						'thumbnail': (
							render_template('medium_thumbnail.html', medium=medium)
							.replace('\n', '')
							.replace('\r', '')
							.replace('\t', '')
						),
						#'group_tiles': render_template('group_tiles.html', groups=fields['groups'])
					}
				}
			})

		return redirect(url_for('media_archive.edit_medium', medium_id=medium.id), 302)

	if api:
		abort(400, errors)

	form_fields = [
		'owner_id',
		'searchability',
		'protection',
		'creation_date',
		'upload_uri',
	]
	for field in form_fields:
		if field in request.form:
			fields[field] = request.form[field]

	#TODO get creation datetime from request.form['creation_date'] string

	return render_template(
		'edit_medium.html',
		errors=errors,
		manager=manager,
		contributors=contributors,
		fields=fields,
		groups=g.media_archive.config['requirable_groups'],
		medium=medium,
	)

@media_archive.route('/' + "<regex('([a-zA-Z0-9_\-]+)'):medium_id>/build")
@require_sign_in
def generate_summaries(medium_id, api=False):
	medium = g.media_archive.require_medium(id_to_md5(medium_id))

	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		if MediumStatus.ALLOWED != medium.status:
			abort(403)
		if medium.owner_uuid != g.media_archive.accounts.current_user.uuid:
			abort(403)

	#TODO confirmation?

	g.media_archive.generate_medium_summaries(medium)

	if api:
		medium = g.media_archive.get_medium(medium.md5)

		from statuspages import success
		return success({
			'media': {
				medium.id: {
					'thumbnail': (
						render_template('medium_thumbnail.html', medium=medium)
						.replace('\n', '')
						.replace('\r', '')
						.replace('\t', '')
					),
				}
			}
		})

	return redirect(url_for('media_archive.edit_medium', medium_id=medium.id), 302)

@media_archive.route('/' + "<regex('([a-zA-Z0-9_\-]+)'):medium_id>/remove/summaries")
@require_sign_in
def remove_summaries(medium_id):
	medium = g.media_archive.require_medium(id_to_md5(medium_id))

	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		if MediumStatus.ALLOWED != medium.status:
			abort(403)
		if medium.owner_uuid != g.media_archive.accounts.current_user.uuid:
			abort(403)

	#TODO confirmation
	#TODO	if 'confirm' not in request.args:
	#TODO		return render_template('confirm_remove_medium_summaries.html')

	g.media_archive.remove_medium_summaries(medium)

	return redirect(url_for('media_archive.edit_medium', medium_id=medium.id), 302)

@media_archive.route('/' + "<regex('([a-zA-Z0-9_\-]+)'):medium_id>/remove/original")
@require_sign_in
def remove_file(medium_id):
	medium = g.media_archive.require_medium(id_to_md5(medium_id))

	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		if MediumStatus.ALLOWED != medium.status:
			abort(403)
		if medium.owner_uuid != g.media_archive.accounts.current_user.uuid:
			abort(403)

	#TODO confirmation
	#TODO	if 'confirm' not in request.args:
	#TODO		return render_template('confirm_remove_medium_file.html')

	g.media_archive.remove_medium_file(medium)
	g.media_archive.media.touch_media(medium)

	return redirect(url_for('media_archive.edit_medium', medium_id=medium.id), 302)

@media_archive.route('/' + "<regex('([a-zA-Z0-9_\-]+)'):medium_id>/remove")
@require_sign_in
def remove_medium(medium_id, api=False):
	medium = g.media_archive.require_medium(id_to_md5(medium_id))

	if not g.media_archive.accounts.current_user_has_global_group('manager'):
		if MediumStatus.ALLOWED != medium.status:
			g.media_archive.add_log(
				'attempt_non_allowed_medium_remove',
				g.media_archive.accounts.current_user.uuid,
				id_to_uuid(medium.id)
			)
			abort(404, {'message': 'medium_not_found'})
		if medium.owner_uuid != g.media_archive.accounts.current_user.uuid:
			g.media_archive.add_log(
				'attempt_non_owner_medium_remove',
				g.media_archive.accounts.current_user.uuid,
				id_to_uuid(medium.id)
			)
			abort(403)

	#TODO	if not api and 'confirm' not in request.args:
	#TODO		return render_template('confirm_remove_medium.html')

	if 'remove_medium' in g.media_archive.callbacks:
		for f in g.media_archive.callbacks['remove_medium']:
			f(medium)

	g.media_archive.remove_medium(medium)
	g.media_archive.add_log(
		'remove_medium',
		g.media_archive.accounts.current_user.uuid,
		id_to_uuid(medium.id)
	)

	if api:
		from statuspages import success
		return success({
			'media': {
				medium.id: {
					'remove': True,
				}
			},
		})

	return redirect(url_for('media_archive.manage_media'), 302)


from django.shortcuts import render,  redirect
from ..login_register.models import User
from .models import Travel
from django.urls import reverse
from django.contrib import messages

def index(request):
    user = User.objects.get(id=request.session['user'])
    trips = Travel.objects.filter(users=user).order_by('date_from','date_to')
    trips_others = Travel.objects.exclude(users=user).order_by('date_from','date_to')
    context = {
        'user' : user,
        'trips' : trips,
        'others' : trips_others
    }
    return render(request, 'travel/index.html', context)

def new(request):
    return render(request, 'travel/new.html')

def create(request):
    if request.method == 'POST':
        user = User.objects.get(id=request.session['user'])
        result = Travel.objects.validate_travel(request.POST, user)
        if result[0]:
            messages.success(request, result[1])
            return redirect(reverse('travel:index'))
        print_messages(request, result[1])
    return redirect(reverse('travel:new'))

def join(request, trip_id):
    Join = User.objects.get(id=request.session['user'])
    trip = Travel.objects.get(id=trip_id)
    trip.users.add(Join)
    messages.success(request, "Joined!")
    return redirect(reverse('travel:index'))

def cancel(request, trip_id):
    Cancel = User.objects.get(id=request.session['user'])
    trip = Travel.objects.get(id=trip_id)
    trip.users.remove(Cancel)
    messages.success(request, "cancelled!")
    return redirect(reverse('travel:index'))

def delete(request, trip_id):
    Travel.objects.get(id=trip_id).delete()
    messages.success(request, "deleted")
    return redirect(reverse('travel:index'))

def show(request, trip_id):
    trip = Travel.objects.get(id=trip_id)
    other_users = trip.users.exclude(id=trip.user.id).order_by('name')
    context = {
        'trip' : trip,
        'others' : other_users
    }
    return render(request, 'travel/show.html', context)

def print_messages(request, error_list):
     for error in error_list:
        messages.error(request, error)


import requests
import time
from bs4 import BeautifulSoup

def get_urls(page):
    #returns a list of non-duplicated urls from the page.
    r = requests.get(page)
    html = r.content
    soup = BeautifulSoup(html, 'html.parser')
    links = soup.find_all('a')
    urls_all = [link.get('href') for link in links if link.get('href') and link.get('href')[0:4]=='http']
    urls = list(set(urls_all)) #removing duplicates
    return urls

def process_list(urls):
    #processes a list of urls passed as argument and returns a list of urls found.
    urls_to_check = []
    for url in urls:
        urls = get_urls(url)
        urls_to_check += urls
    return urls_to_check

def check_urls(urls):
    results = []
    for url in urls:
        print("Now Processing: " + str(url) + "\n")
        try:
            r = requests.get(url)
            if r.history:
                history_status_codes = [str(h.status_code) for h in r.history]
                result = (r.status_code, r.history, url, 'Success. Redirect to ' + r.url)
            elif r.status_code == 200:
                result = (r.status_code, r.history, url, 'Success. No Redirect')
            else:
                result = (r.status_code, r.history, url, 'ERROR')
        except Exception as e:
            result = (0, [], url, e)
        results.append(result)
    return results

if __name__ == "__main__":
    file_name = 'links_check.txt' #name of the text file where results are written.
    site = 'https://www.ghirardelli.com'
    urls_list = []
    urls_list.append(site) #starting point is just the url of the site.
    urls_to_check = [] #this is where I store all the urls which will have to be checked.
    depth = 1 #I use this variable to control how many iterations I will do. i.e. how deep in the tree I will go to check the links.
    i = 0
    while i < depth:
        urls = process_list(urls_list)
        urls_to_check += urls
        urls_list = urls
        i += 1
    urls_to_check = list(set(urls_to_check)) #removing duplicates

    try:
        f = open(file_name, 'w')
    except OSError:
        print('ERROR opening the file where to write results.')

    for result in urls_to_check:
        f.write(str(result) + '\n')

    time_start = time.time()
    for result in check_urls(urls_to_check):
        f.write(str(result) + '\n')
    time_end = time.time()
    print("Execution time - Version 1: " + str(time_end - time_start ))

    f.close()


"""
Crawler for Empregos.com.br based on ATSSpider

scrapy crawl empregos -a extract=1 -a url="http://www.empregos.com.br/vagas"

Sample URLS:
    http://www.empregos.com.br/vagas/modal/3806075?4847345
"""

from re import compile
from scrapy.http import Request
from scrapy.selector import Selector

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import Prefix, NormalizedJoin

REF_REGEX = compile(r'/(\d+)?(\d+)')
LOCATION_RE = compile(r'([^-]+)')
DATE_RE = compile(r'Publicada em (\S+)')

class Empregos(ATSSpider):

    name = 'empregos'
    allowed_domains = ['empregos.com.br']

    def parse(self, response):
        """
            GET request method to find out all the jobs.
        """
        sel = Selector(response)
        for result in sel.xpath('//div[@class="vaga"]'):
            meta= {}
            meta['title'] = result.xpath('.//a[contains(@id, "tituloVaga")]/text()').extract()
            meta['company'] = result.xpath('.//div[contains(@id, "dvItemsVerdes")]//a/text()').extract()
            url = result.xpath('.//a[contains(@id, "tituloVaga")]/@href').extract()
            if url:
                yield Request(callback=self.parse_job_callback(), url=url[0], meta=meta)

        next_page = sel.xpath('//a[@class="proximo"]/@href').extract()
        if next_page:
            yield Request(callback=self.parse, url=next_page[0])

    def parse_job(self, response):
        """
            Extract all required information.
            If response url is same as domain, loader object will returns the items.
        """
        sel = Selector(response)
        loader = BrightcorpItemLoader(selector=sel)
        for key in ['title', 'company', 'date']:
            loader.add_value(key, response.meta.get(key))

        loader.add_xpath('title', '//div[contains(@class, "titulo-outros")]/h1/text()')
        loader.add_xpath('company', '//div[contains(@class, "titulo-outros")]/p/a//text()')
        loader.add_xpath('date', '//div[contains(@class, "titulo-outros")]/p//text()', re=DATE_RE)

        loader.add_xpath('description', '(//tr[contains(@id, "Descricao")])[1]/td[@class="col-dir"]/p/text()')
        loader.add_xpath('qualifications', '(//tr[contains(@id, "trQualificacao")])[1]/td[@class="col-dir"]/p/text()')
        loader.add_xpath('educationrequirements', '(//tr[contains(@id, "trFormacao")])[1]/td[@class="col-dir"]/p/text()')
        loader.add_xpath('location', '(//tr[contains(@id, "trLocalTrabalho")])[1]/td[@class="col-dir"]/p/text()[1]')
        loader.add_xpath('location', '//div[contains(@class, "titulo-outros")]/p/strong//text()', re=LOCATION_RE)

        loader.add_value('referencenumber', response.url, NormalizedJoin('-'), Prefix('empregos-'), re=REF_REGEX)
        loader.add_value('url', response.url)
        loader.add_value('apply_url', response.url)
        yield loader.load_item()


from django.urls import path

from blog import views
from .views import (ListPostView, DetailPostView, PostUpdateView, PostDeleteView, PostCreateView,)

urlpatterns = [
    path('', ListPostView.as_view(), name='list_post'),
    path('<int:pk>', DetailPostView.as_view(), name='post_detail'),
    path('<int:pk>/edit', PostUpdateView.as_view(),name='update_post'),
    path('<int:pk>/delete>', PostDeleteView.as_view(),name='delete_post'),
    path('new/',PostCreateView.as_view(),name='new_post'),
    path('comment/<int:pk>/add', views.comment_create_view,name='new_comment')
]


from youtube_dl import YoutubeDL

def get_audio(music_url,name_user):
    try:
        """
            Get info music
        """
        video_info =YoutubeDL().extract_info(
        url= f"{music_url}",download=False)

        """
            Create name for music file and folder with name user
        """
        filename = f"{name_user}/{video_info['title']}.mp3"

        options={
        'format': 'bestaudio/best',
            'outtmpl':filename,
                'postprocessors': [{
                    'key': 'FFmpegExtractAudio',
                    'preferredcodec': 'mp3',
                    'preferredquality': '192',
                }],
            }

        with YoutubeDL(options) as ydl:
            ydl.download([video_info["webpage_url"]])

        return filename
    except:
        return "fail"


from django.contrib import admin
from django.urls import path
# from django.conf.urls import url

from student import views

urlpatterns = [
    path('admin/', admin.site.urls),
    path('getstudents/', views.getAllStudents, name='getstudents'),
    path('currency/', views.getCurrency, name='currency'),
    path('addstudent/', views.addStudent, name='add'),
    path('updatestudent/', views.updateStudent, name='update'),
    path('deletestudent/', views.deleteStudent, name='delete')
]


import torch
from sklearn.metrics.pairwise import euclidean_distances, cosine_distances
from sklearn.neighbors import NearestNeighbors

import matplotlib.pyplot as plt
from sklearn.decomposition import PCA, FastICA
from sklearn.cluster import KMeans
from sklearn.manifold import TSNE
import os
import numpy as np
from nltk.corpus import stopwords
import ast

from mpl_toolkits.mplot3d import Axes3D

punctuation = ['.', ',', '!', '?', '[SEP]', ')', '(', '"', "'", '-', '–']

def reduce(x, method, dims):
    if method == "pca":
        pca = PCA(n_components=2)
        reduced = pca.fit_transform(x)

        return reduced.transpose()
    if method == "ica":
        ica = FastICA(n_components=dims, random_state=0)
        reduced = ica.fit_transform(x)

        return reduced.transpose()
    if method == "tsne":
        tsne = TSNE(n_components=dims)
        reduced = tsne.fit_transform(x)

        return reduced.transpose()

def euclidean(x):
    return euclidean_distances(x.numpy())

def cosine(x):
    return cosine_distances(x.numpy())

def get_color_for_token(i, sample, sentence_colored=False):
    if sentence_colored:
        num_colors = sample['tokens'].count('.') + 1
        cm = plt.get_cmap('gist_rainbow')
        slice = sample['tokens'][:i]
        token_sen = slice.count('.') + slice.count('[SEP]')
        return [cm(token_sen / num_colors)]

    sp_colors = ['darkgreen', 'greenyellow', 'limegreen']
    question_colors = ['cyan', 'blue', 'dodgerblue']

    prediction_ind = sample["pred_index"] if "pred_index" in sample else range(0, 0)
    sp_parts = sample["sp_parts"] if "sp_parts" in sample else [range(0, 0)]
    question_parts = sample["question_parts"] if "question_parts" in sample else [range(0, 0)]

    col = '0.5'

    for j, col_range in enumerate(sp_parts):
        if i in col_range:
            col = sp_colors[j]

    for k, col_range in enumerate(question_parts):
        if i in col_range:
            col = question_colors[k]

    if i in prediction_ind:
        col = 'red'

    return col

def save_and_close_plot(plot_path, title):
    if not os.path.exists(plot_path):
        os.makedirs(plot_path)

    plt.savefig(os.path.join(plot_path, title) + ".svg")

    # close plot
    plt.clf()

def plot(sample, x, reduce_method, title, path, plot_3d, sentence_colored=False):
    if plot_3d:
        plot3d(sample, x, reduce_method, title, path, sentence_colored)
    else:
        plot2d(sample, x, reduce_method, title, path, sentence_colored)

def token_features_for_demo(token_index, token, highlighted_tokens, question_range, sup_ranges, pred_range):
    marker = "o"
    col = "gray"
    size = 8

    for sup_range in sup_ranges:
        if token_index in sup_range:
            col = "darkcyan"

    if token_index in question_range:
        marker = "*"
        col = "orangered"

    for highlighted_token in highlighted_tokens:
        if isinstance(highlighted_token, str):
            if token == highlighted_token:
                size = 10
                break
        elif isinstance(highlighted_token, int):
            if token_index == highlighted_token:
                size = 10
                break

    if token_index in pred_range:
        marker = "d"
        col = "crimson"

    return col, size, marker

def plot_for_demo(sample, x, title, path, highlighted_tokens, sup_ranges, pred_range, reduce_method="pca"):
    tokens = sample["tokens"]

    question_range = get_question_range(tokens)

    coords = reduce(x, reduce_method, 2)

    for i, val in enumerate(coords[0]):
        x = coords[0][i]
        y = coords[1][i]

        token = tokens[i]

        col, size, marker = token_features_for_demo(i, token, highlighted_tokens, question_range, sup_ranges,
                                                    pred_range)

        if token == "[SEP]" or token == "[CLS]":
            continue

        plt.scatter(x, y, c=col, marker=marker)

        if i < len(tokens):
            plt.text(x + 0.1, y + 0.2, token, fontsize=size)

    plt.xlabel("PC 1")
    plt.ylabel("PC 2")

    plt.title(title)
    save_and_close_plot(os.path.join(path, 'paper_demo'), title)

def plot2d(sample, x, reduce_method, title, path, sentence_colored=False):
    tokens = sample["tokens"]

    coords = reduce(x, reduce_method, 2)

    for i, val in enumerate(coords[0]):
        x = coords[0][i]
        y = coords[1][i]

        col = get_color_for_token(i, sample, sentence_colored)

        plt.scatter(x, y, c=col)

        if i < len(tokens):
            plt.text(x + 0.1, y + 0.1, tokens[i], fontsize=6)
        else:
            plt.text(x + 0.1, y + 0.1, "PAD", fontsize=6)

    plt.title(title)
    save_and_close_plot(os.path.join(path, 'colored_plots'), title)

def plot3d(sample, x, reduce_method, title, path, sentence_colored=False):
    tokens = sample["tokens"]

    coords = reduce(x, reduce_method, 3)

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    for i, val in enumerate(coords[0]):
        x = coords[0][i]
        y = coords[1][i]
        z = coords[2][i]

        col = get_color_for_token(i, sample, sentence_colored)

        # plt.scatter(x, y, c=col)
        ax.scatter([x], [y], [z], c=col)

        if i < len(tokens):
            ax.text(x + 0.1, y + 0.1, z + 0.1, tokens[i], fontsize=6)
        else:
            ax.text(x + 0.1, y + 0.1, z + 0.1, "PAD", fontsize=6)

    ax.set_title(title)
    save_and_close_plot(os.path.join(path, 'colored_plots', '3D'), title)

def plot_kmeans(sample, x, reduce_method, title, path):
    colors = ['yellow', 'red', 'gray', 'orange', 'pink', 'blue', 'purple', 'green', 'cyan']
    tokens = sample["tokens"]

    coords = reduce(x, reduce_method, 2)

    kmeans = KMeans(n_clusters=9)
    kmeans.fit(x)

    for i, val in enumerate(coords[0]):
        x = coords[0][i]
        y = coords[1][i]

        plt.scatter(x, y, c=colors[kmeans.labels_[i]])

        if i < len(tokens):
            plt.text(x + 0.001, y + 0.001, tokens[i], fontsize=6)
        else:
            plt.text(x + 0.001, y + 0.001, "PAD", fontsize=6)

    plt.title(title)
    save_and_close_plot(os.path.join(path, 'kmeans'), title)

def build_kmeans(sample, x, title):
    tokens = sample["tokens"]

    kmeans = KMeans(n_clusters=10)
    kmeans.fit(x)

    clusters = {}

    summary = ""

    for i, label in enumerate(kmeans.labels_):
        if label not in clusters:
            clusters[label] = []
        word = tokens[i]

        if 0 < i < tokens.index("[SEP]"):
            word = "*{}*".format(word)
        clusters[label].append(word)

    summary += title + "\n"

    for cluster in clusters.values():
        summary += str(cluster) + "\n"

    return summary

def plot_parts(sample, x, reduce_method, title, path):
    x = x[:, 50:100]
    plot(sample, x, reduce_method, title, os.path.join(path, '50-100'))

def track_changes(tokens, layers, cut_index, cut_index_question):
    stopWords = set(stopwords.words('english'))

    distance_diffs = []

    for l in range(len(layers) - 1):
        first = layers[l][0][:cut_index]
        second = layers[l + 1][0][:cut_index]

        distances1 = euclidean(first)[:cut_index_question]
        distances2 = euclidean(second)[:cut_index_question]
        diff = distances2 - distances1

        distance_diffs.append(diff)

    for i in range(cut_index_question):

        info = str(tokens[i]) + " <-"

        for j in range(len(layers) - 1):

            diff = distance_diffs[j][i]

            k_min = np.argsort(diff)

            info += " ("

            cnt = 0
            for k in k_min:

                diff_token = tokens[k]

                if diff_token.lower() not in stopWords and diff_token not in punctuation:
                    distance_loss = str(diff[k])
                    info += str(diff_token)

                    if cnt > 2:
                        info += ")"
                        break
                    else:
                        info += ", "
                        cnt += 1
        print(info)

def track_neighbors(tokens, pred_ind, layers, cut_index, cut_index_question):
    # stopWords = set(stopwords.words('english'))

    for l in range(len(layers)):
        print("-------- LAYER " + str(l) + " --------")
        vectors = layers[l][0][:cut_index]

        nbrs = NearestNeighbors(n_neighbors=15, algorithm='ball_tree').fit(vectors)
        distances, indices = nbrs.kneighbors(vectors)

        q_neighbors = indices[:cut_index_question]
        pred_neighbors = indices[pred_ind[0]:pred_ind[-1] + 1]
        for n in pred_neighbors:
            print_neighbors(tokens, n)

        for m in q_neighbors:
            print_neighbors(tokens, m)

def print_neighbors(tokens, neighbors):
    token_i = neighbors[0]
    info = str(tokens[token_i]) + ": ("
    for i in range(1, len(neighbors)):
        info += str(tokens[neighbors[i]]) + ", "
    info += ")"
    print(info)

def load_representations(path):
    inputs_file = os.path.join(path, "inputs.pkl")
    embedding_file = os.path.join(path, "embedding.pkl")
    layers_file = os.path.join(path, "encoded_layers.pkl")

    inputs = torch.load(inputs_file, map_location='cpu')
    embedding = torch.load(embedding_file, map_location='cpu')[0]
    layers = torch.load(layers_file, map_location='cpu')

    return inputs, embedding, layers

def load_old_representations(path):
    inputs_file = os.path.join(path, "inputs.pkl")
    embedding_file = os.path.join(path, "embedding.pkl")
    layers_file = os.path.join(path, "encoded_layers.pkl")

    inputs = torch.load(inputs_file, map_location='cpu')[0]
    embedding = torch.load(embedding_file, map_location='cpu')[0]
    layers = torch.load(layers_file, map_location='cpu')[0]

    return inputs, embedding, layers

def get_cut_index(inputs, tokens):
    if len((inputs == 0).nonzero()) != 0 and len(tokens) != 384:
        cut_index = (inputs == 0).nonzero()[0][1].item()
    else:
        cut_index = len(tokens)

    print(cut_index)
    print(len(tokens))

    return cut_index

def get_question_range(tokens):
    return range(1, tokens.index("[SEP]"))

def get_context_range(tokens):
    return range(tokens.index("[SEP]"), len(tokens) - 1)

def parse_token_file(path):
    with open(path, encoding='utf-8') as f:
        tokens = ast.literal_eval(f.read())
        return tokens

def parse_prediction_index_range(path):
    with open(path, encoding='utf-8') as f:
        preds = ast.literal_eval(f.read())

        for entry in preds.values():
            r = range(int(entry[0]["start"]), int(entry[0]["end"] + 1))
            return r

def get_sentence_ranges(sample):
    tokens = sample["tokens"]

    start_i = tokens.index("[SEP]")

    dot_indices = [i for i, x in enumerate(tokens) if x == "."]
    sentence_ranges = []

    prev_dot_index = start_i
    for i, dot_index in enumerate(dot_indices):
        if prev_dot_index + 1 < dot_index:
            sentence_ranges.append(range(prev_dot_index + 1, dot_index))
        prev_dot_index = dot_index

    for r in sentence_ranges:
        print(tokens[r[0]:r[-1] + 1])
    return sentence_ranges


from twilio.rest import Client
import config
import scraper

# use api keys from config.py
account_sid = config.account_sid
auth_token = config.auth_token

def text_me(session, uniquenumber, last, current):
    client = Client(account_sid, auth_token)
    client.messages.create(
        to=config.tonumber,
        from_=config.fromnumber,
        body="CLASS: " + uniquenumber +
             "\nTITLE: " + scraper.get_name(session, uniquenumber) +
             "\nWITH: " + scraper.getInfoAboutClass(session, uniquenumber, "Instructor") +
             " \nWAS: " + last +
             " \nIS NOW: " + current
    )

def text_me_str(str):
    client = Client(account_sid, auth_token)
    client.messages.create(
        to=config.tonumber,
        from_=config.fromnumber,
        body=str

    )


from utils.utils import read_text_file

content = read_text_file('inputs/day1_p1.txt')
total_fuel = 0
revised_total = 0
for value in content:
    fuel = (int(value)//3)-2
    total_fuel += fuel
    while fuel > 0:
        revised_total += fuel
        fuel = (int(fuel)//3)-2

print('Part One: ' + str(total_fuel))
print('Part Two: ' + str(revised_total))


import matplotlib.pyplot as plt
import numpy as np
import csv

a = []
b = []
x = []
y = []
theta1 = []
theta = []
time = []
time1 = []
t1 = []
t2 = []
t = []
r = []
r1 = []

with open('PlanetsVV_2_0.000_merc.txt','r') as f:
	lines = f.readlines()
	time1 = [line.split()[0] for line in lines]
	a = [line.split()[3] for line in lines]
	b = [line.split()[4] for line in lines]

with open('PlanetsVV_2_0.000.txt','r') as f:
	lines = f.readlines()
	time = [line.split()[0] for line in lines]
	x = [line.split()[3] for line in lines]
	y = [line.split()[4] for line in lines]

x1 = a[:]
y1 = b[:]
t2 = time1[:]

for i in range(len(x1)-1):
	r1.append(np.sqrt(float(x1[i])*float(x1[i])+float(y1[i])*float(y1[i])))
	r.append(np.sqrt(float(x[i])*float(x[i])+float(y[i])*float(y[i])))

for i in range(len(x1)-1):
	if r1[i] < r1[i-1] and r1[i] < r1[i+1]:
		theta1.append(np.arctan(float(y1[i])/float(x1[i])))
		t1.append(t2[i])
		theta.append(np.arctan(float(y[i])/float(x[i])))
		t.append(time[i])

plt.plot(t1,theta1,label="Relativistic Effect")
plt.plot(t,theta,label="Newtonian")
plt.xlabel('Time(Years)')
plt.ylabel('Perihelion Angle')
#plt.title('Plotted planets!')
plt.legend()
plt.show()


import pickle
import socket
import struct

import cv2

HOST = ''
PORT = 8089

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#print('Socket created')

s.bind((HOST, PORT))
#print('Socket bind complete')
s.listen(10)
#print('Socket now listening')

conn, addr = s.accept()

data = b'' ### CHANGED
payload_size = struct.calcsize("L")

video = cv2.VideoCapture(1, cv2.CAP_DSHOW)

while True:
    try:
        ####################
        ##sending server stream
        ####################
        ret,ser=video.read()
        serverdata = pickle.dumps(ser)
        message_size = struct.pack("L", len(serverdata))
        conn.sendall(message_size + serverdata)

        ####################
        ##rcv client stream
        ####################

        # Retrieve message size
        while len(data) < payload_size:
            data += conn.recv(4096)

        packed_msg_size = data[:payload_size]
        data = data[payload_size:]
        msg_size = struct.unpack("L", packed_msg_size)[0] ### CHANGED

        # Retrieve all data based on message size
        while len(data) < msg_size:
            data += conn.recv(4096)

        frame_data = data[:msg_size]
        data = data[msg_size:]

        # Extract frame
        frame = pickle.loads(frame_data)

        # Display
        cv2.imshow('sender_frame', frame)
        if cv2.waitKey(1) == 13:
            print("you closed")
            conn.shutdown(2)
            conn.close()
            cv2.destroyAllWindows()
            break
    except:
        cv2.destroyAllWindows()
        print("other person closed")
        break

video.release()


import torch
from torch.utils.data import DataLoader
from helper_scripts.datasets import MNIST
from helper_scripts.models import MLP
from helper_scripts.train import train

if __name__ == "__main__":
    assert torch.cuda.is_available(), "Script requires a CUDA-cabable GPU to run, found none."
    print(f"Training on {torch.cuda.device_count()} GPUs")

    trainset, _ = MNIST("./data")
    batch_size = 128
    trainloader = DataLoader(trainset, batch_size, shuffle=True, num_workers=8)

    # only additional passage w.r.t. regular model training
    net = torch.nn.DataParallel(MLP())
    # can specify also
    # DataParallel(model, device_ids=[0,1], output_device=[1])
    # this specifies that only GPUs 0 and 1 will be used
    # and that the output of the model will be found on GPU 1
    # defaults values are
    # device_ids = list(range(torch.cuda.device_count()))
    # output_device = device_ids[0]
    net.to("cuda:0")

    optimizer = torch.optim.Adam(net.parameters())
    loss_fn = torch.nn.CrossEntropyLoss()

    num_epochs = 3

    train(net, optimizer, loss_fn, num_epochs, trainloader, "cuda:0")


#!/usr/bin/env python3

import os
import os.path as osp
from ogb.nodeproppred import Evaluator, PygNodePropPredDataset
from quiver.cuda_sampler import CudaNeighborSampler
from quiver.cuda_loader import CudaNeighborLoader
from quiver.profile_utils import StopWatch
import torch
import multiprocessing as mp

def async_sampler(data, train_idx, w):
    train_loader = CudaNeighborSampler(data.edge_index,
                                       node_idx=train_idx,
                                       mode='sync',
                                       sizes=[15, 10, 5],
                                       batch_size=1024,
                                       shuffle=True)
    w.tick('create train_loader')

    for i in range(10):
        for batch_size, n_id, adjs in train_loader:
            pass
        w.tick('one round')
    w.tick('sample')

def async_loader(data, train_idx, w):
    train_loader = CudaNeighborLoader(
        (data.edge_index, [15, 10, 5], train_idx), 1024, 1)
    w.tick('create train_loader')
    count = 0
    for i in range(10):
        for batch_size, n_id, adjs in train_loader:
            count += 1
            if count == 1:
                w.tick('init generator')
        w.tick('one round')
        if i == 9:
            train_loader.close()
            break
        train_loader.reset()
        w.tick('reset')
    w.tick('sample')

if __name__ == '__main__':
    mp.set_start_method('spawn')
    w = StopWatch('main')
    home = os.getenv('HOME')
    data_dir = osp.join(home, '.pyg')
    root = osp.join(data_dir, 'data', 'products')
    dataset = PygNodePropPredDataset('ogbn-products', root)
    split_idx = dataset.get_idx_split()
    evaluator = Evaluator(name='ogbn-products')
    data = dataset[0]

    train_idx = split_idx['train']

    w.tick('load data')

    async_loader(data, train_idx, w)

    del w


import argparse
from flask import Flask, jsonify, request
from flask import render_template, send_from_directory
import os
import re
import joblib
import socket
import json
import numpy as np
import pandas as pd
from datetime import datetime as dt
import time

## import model specific functions and variables
from helper.fetchlib import fetch_ts, fetch_data, engineer_features

from helper.modeltools import model_train, model_load, model_predict

from helper.modeltools import models_load, models_train, _models_train

from helper.modeltools import _plot_learning_curve, _plot_feature_importance, _make_compare_plot

MODEL_DIR = "models"
MODEL_VERSION = 0.1
MODEL_VERSION_NOTE = "supervised learing model for time-series"
data_dir = os.path.join("data","cs-train")

app = Flask(__name__)
print('Starting API...')
print('Loading models...')
all_data, all_models = models_load(data_dir=data_dir, MODEL_DIR=MODEL_DIR)
print('Models loading completed...')

@app.route('/retrain', methods=['GET'])
def retrain():
    run_start = time.time()
    try:
        models_train(data_dir, MODEL_DIR=MODEL_DIR, MODEL_VERSION=MODEL_VERSION, test=False)
    except:
            return jsonify(msg='model_train error')
    try:
        tmp_all_data, tmp_all_models = models_load(data_dir=data_dir, MODEL_DIR=MODEL_DIR)
    except:
            return jsonify(msg='model_load error')
    all_data = tmp_all_data
    all_models = tmp_all_models

    m, s = divmod(time.time()-run_start,60)
    h, m = divmod(m, 60)
    return jsonify(modelTrain = 'OK', modelReload = 'OK',runningTime = "%d:%02d:%02d"%(h, m, s), modelsLoaded = ",".join(all_models.keys()))

@app.route('/reloadmodel', methods=['GET'])
def reloadmodel():
    run_start = time.time()

    try:
        tmp_all_data, tmp_all_models = models_load(data_dir=data_dir, MODEL_DIR=MODEL_DIR)
    except:
            return jsonify(msg='model_load error')
    all_data = tmp_all_data
    all_models = tmp_all_models

    m, s = divmod(time.time()-run_start,60)
    h, m = divmod(m, 60)
    return jsonify(modelReload='OK',runningTime = "%d:%02d:%02d"%(h, m, s), modelsLoaded = ",".join(all_models.keys()))

@app.route('/predict', methods=['GET'])
def predict():
    """
    basic predict function for the API

    country='all'
    year='2019'
    month='04'
    day='06'
    2017-11-29    2019-05-31
    """
    run_start = time.time()

    country = request.args.get('country', default = 'all', type = str)
    date = request.args.get('date', default = '2019-05-06', type = str)
    startDate = dt.strptime("2017-11-29", "%Y-%m-%d")
    endDate = dt.strptime("2019-05-31", "%Y-%m-%d")
    try:
        predictDate = dt.strptime(date, "%Y-%m-%d")
    except:
        return jsonify(errMsg = 'Error Date, date should be in range 2017-11-29  -  2019-05-31')

    if predictDate >=startDate and predictDate < endDate:
        dataSplit = date.split('-')
        year = dataSplit[0]
        month = dataSplit[1]
        day = dataSplit[2]
        try:
            result = model_predict(country, year, month, day, all_models=all_models, all_data=all_data, data_dir=data_dir, MODEL_DIR=MODEL_DIR, MODEL_VERSION=MODEL_VERSION)
        except:
            return jsonify(msg='model_predict error')
        #{'y_pred': array([134258.56976667]), 'y_proba': None}
        m, s = divmod(time.time()-run_start,60)
        h, m = divmod(m, 60)

        return  jsonify(status='OK', date = date, country = country, y_pred = result['y_pred'][0], runningTime = "%d:%02d:%02d"%(h, m, s),)
    else:
        return jsonify(errMsg = 'Error Date, date should be in range 2017-11-29  -  2019-05-31')

@app.route('/logs/<filename>', methods=['GET'])
def logs(filename):
    """
    API endpoint to get logs
    """

    if not re.search(".log",filename):
        print("ERROR: API (log): file requested was not a log file: {}".format(filename))
        return jsonify(msg = "ERROR: API (log): file requested was not a log file: {}".format(filename))

    log_dir = os.path.join(".","logs")
    if not os.path.isdir(log_dir):
        print("ERROR: API (log): cannot find log dir")
        return jsonify(msg = "ERROR: API (log): cannot find log dir")

    file_path = os.path.join(log_dir, filename)
    if not os.path.exists(file_path):
        print("ERROR: API (log): file requested could not be found: {}".format(filename))
        return jsonify(msg = "ERROR: API (log): file requested could not be found: {}".format(filename))

    return send_from_directory(log_dir, filename, as_attachment=True)

if __name__ == '__main__':

    ## parse arguments for debug mode
    ap = argparse.ArgumentParser()
    ap.add_argument("-d", "--debug", action="store_true", help="debug flask")
    args = vars(ap.parse_args())

    if args["debug"]:
        app.run(debug=True, port=8080)
    else:
        app.run(host='0.0.0.0', threaded=True, port=8080)
        print('API runing on http://localhost:8080')


from nj.models import *
import json
from django.http import HttpResponse
import os
from profkom import settings
import importlib.machinery
from urllib.parse import unquote

def get_module(name):
    module_name = name + '.api'
    path = settings.BASE_DIR + '/nj/tabs/' + name + '/api.py'
    loader = importlib.machinery.SourceFileLoader(module_name, path)
    mod = loader.load_module(module_name)
    return mod

def get_tab(request):
    name = request['name']
    path = settings.BASE_DIR + '/nj/tabs/' + name

    try:
        content_file = open(path + '/content.html', 'rb')
        content_content = content_file.read().decode('utf8')
        content_file.close()
    except:
        content_content = ''

    try:
        pass
    except:
        content_content = ''

    try:
        styles_file = open(path + '/styles.css')
        styles_content = styles_file.read()
        styles_file.close()
    except:
        styles_content = ''

    try:
        client_file = open(path + '/client.js')
        client_content = client_file.read()
        client_file.close()
    except:
        client_content = ''

    response = {
        'content': content_content,
        'styles': styles_content,
        'client': client_content
    }

    try:
        mod = get_module(name)
        methods = []
        for method in dir(mod):
            if method.startswith('__'):
                continue
            methods.append(method)

        response['methods'] = methods;
    except:
        response['methods'] = []

    return response

def tab_api(request):
    method = request['method']
    mod = get_module(request['name'])
    method = getattr(mod, method)
    response = method(request.get('args', {}))
    return response


import configparser
import tkinter as tk
import tweepy
from tweepy import OAuthHandler

master = tk.Tk()
master.title('timelineFetch')

def fetchtimeline(event=None):
	config = configparser.ConfigParser()

	config.read('../config.ini')

	if 'api.twitter.com' in config:
		consumer_key = config['api.twitter.com']['consumer_key']
		consumer_secret = config['api.twitter.com']['consumer_secret']
		access_token = config['api.twitter.com']['access_token']
		access_secret = config['api.twitter.com']['access_secret']

	auth = OAuthHandler(consumer_key, consumer_secret)
	auth.set_access_token(access_token, access_secret)

	api = tweepy.API(auth)

	twitterid = inputid.get()

	root = tk.Tk()
	root.title('Results')

	tweeter = tk.Label(root, text="Search results for: " + twitterid + "\n-------------------------")
	tweeter.pack()

	for status in tweepy.Cursor(api.user_timeline, id=twitterid).items(10):
		w = tk.Label(root, text=status.text + "\nRetweets: " + str(status.retweet_count) + "   Favorites: " + str(status.favorite_count) + "\n-------------------------------------------")
		w.pack()

tk.Label(master, text='TwitterID').grid(row=0)

inputid = tk.StringVar()
e1 = tk.Entry(master, textvariable=inputid)
e1.grid(row=0, column=1)
e1.bind('<Return>', fetchtimeline)
e1.focus_set()

tk.Button(master, text='Fetch', command=fetchtimeline).grid(row=0, column=2)

master.mainloop()