# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

"""
See documentation for usage
https://github.com/CGCookie/blender-addon-updater

"""

import urllib.request
import urllib
import os
import json
import zipfile
import shutil
import asyncio
import threading
import time
import fnmatch
from datetime import datetime, timedelta

# blender imports, used in limited cases
import bpy
import addon_utils

# -----------------------------------------------------------------------------
# Define error messages/notices & hard coded globals
# -----------------------------------------------------------------------------

# currently not used
DEFAULT_TIMEOUT = 10
DEFAULT_PER_PAGE = 30

# -----------------------------------------------------------------------------
# The main class
# -----------------------------------------------------------------------------

class Singleton_updater(object):
    """
	This is the singleton class to reference a copy from,
	it is the shared module level class
	"""

    def __init__(self):

        self._engine = GithubEngine()
        self._user = None
        self._repo = None
        self._website = None
        self._current_version = None
        self._tags = []
        self._tag_latest = None
        self._tag_names = []
        self._latest_release = None
        self._include_branches = False
        self._include_branch_list = ['master']
        self._include_branch_autocheck = False
        self._manual_only = False
        self._version_min_update = None
        self._version_max_update = None

        # by default, backup current addon if new is being loaded
        self._backup_current = True
        self._backup_ignore_patterns = None

        # set patterns for what files to overwrite on update
        self._overwrite_patterns = ["*.py", "*.pyc"]
        self._remove_pre_update_patterns = []

        # by default, don't auto enable/disable the addon on update
        # as it is slightly less stable/won't always fully reload module
        self._auto_reload_post_update = False

        # settings relating to frequency and whether to enable auto background check
        self._check_interval_enable = False
        self._check_interval_months = 0
        self._check_interval_days = 7
        self._check_interval_hours = 0
        self._check_interval_minutes = 0

        # runtime variables, initial conditions
        self._verbose = False
        self._fake_install = False
        self._async_checking = False  # only true when async daemon started
        self._update_ready = None
        self._update_link = None
        self._update_version = None
        self._source_zip = None
        self._check_thread = None
        self._skip_tag = None

        # get from module data
        self._addon = __package__.lower()
        self._addon_package = __package__  # must not change
        self._updater_path = os.path.join(os.path.dirname(__file__),
                                          self._addon + "_updater")
        self._addon_root = os.path.dirname(__file__)
        self._json = {}
        self._error = None
        self._error_msg = None
        self._prefiltered_tag_count = 0

        # to verify a valid import, in place of placeholder import
        self.invalidupdater = False

    # -------------------------------------------------------------------------
    # Getters and setters
    # -------------------------------------------------------------------------

    @property
    def engine(self):
        return self._engine.name

    @engine.setter
    def engine(self, value):
        if value.lower() == "github":
            self._engine = GithubEngine()
        elif value.lower() == "gitlab":
            self._engine = GitlabEngine()
        elif value.lower() == "bitbucket":
            self._engine = BitbucketEngine()
        else:
            raise ValueError("Invalid engine selection")

    @property
    def private_token(self):
        return self._engine.token

    @private_token.setter
    def private_token(self, value):
        if value == None:
            self._engine.token = None
        else:
            self._engine.token = str(value)

    @property
    def addon(self):
        return self._addon

    @addon.setter
    def addon(self, value):
        self._addon = str(value)

    @property
    def verbose(self):
        return self._verbose

    @verbose.setter
    def verbose(self, value):
        try:
            self._verbose = bool(value)
            if self._verbose == True:
                print(self._addon + " updater verbose is enabled")
        except:
            raise ValueError("Verbose must be a boolean value")

    @property
    def include_branches(self):
        return self._include_branches

    @include_branches.setter
    def include_branches(self, value):
        try:
            self._include_branches = bool(value)
        except:
            raise ValueError("include_branches must be a boolean value")

    @property
    def include_branch_list(self):
        return self._include_branch_list

    @include_branch_list.setter
    def include_branch_list(self, value):
        try:
            if value == None:
                self._include_branch_list = ['master']
            elif type(value) != type(['master']) or value == []:
                raise ValueError("include_branch_list should be a list of valid branches")
            else:
                self._include_branch_list = value
        except:
            raise ValueError("include_branch_list should be a list of valid branches")

    @property
    def overwrite_patterns(self):
        return self._overwrite_patterns

    @overwrite_patterns.setter
    def overwrite_patterns(self, value):
        if value == None:
            self._overwrite_patterns = ["*.py", "*.pyc"]
        elif type(value) != type(['']):
            raise ValueError("overwrite_patterns needs to be in a list format")
        else:
            self._overwrite_patterns = value

    @property
    def remove_pre_update_patterns(self):
        return self._remove_pre_update_patterns

    @remove_pre_update_patterns.setter
    def remove_pre_update_patterns(self, value):
        if value == None:
            self._remove_pre_update_patterns = []
        elif type(value) != type(['']):
            raise ValueError("remove_pre_update_patterns needs to be in a list format")
        else:
            self._remove_pre_update_patterns = value

    # not currently used
    @property
    def include_branch_autocheck(self):
        return self._include_branch_autocheck

    @include_branch_autocheck.setter
    def include_branch_autocheck(self, value):
        try:
            self._include_branch_autocheck = bool(value)
        except:
            raise ValueError("include_branch_autocheck must be a boolean value")

    @property
    def manual_only(self):
        return self._manual_only

    @manual_only.setter
    def manual_only(self, value):
        try:
            self._manual_only = bool(value)
        except:
            raise ValueError("manual_only must be a boolean value")

    @property
    def auto_reload_post_update(self):
        return self._auto_reload_post_update

    @auto_reload_post_update.setter
    def auto_reload_post_update(self, value):
        try:
            self._auto_reload_post_update = bool(value)
        except:
            raise ValueError("Must be a boolean value")

    @property
    def fake_install(self):
        return self._fake_install

    @fake_install.setter
    def fake_install(self, value):
        if type(value) != type(False):
            raise ValueError("fake_install must be a boolean value")
        self._fake_install = bool(value)

    @property
    def user(self):
        return self._user

    @user.setter
    def user(self, value):
        try:
            self._user = str(value)
        except:
            raise ValueError("User must be a string value")

    @property
    def json(self):
        if self._json == {}:
            self.set_updater_json()
        return self._json

    @property
    def repo(self):
        return self._repo

    @repo.setter
    def repo(self, value):
        try:
            self._repo = str(value)
        except:
            raise ValueError("User must be a string")

    @property
    def website(self):
        return self._website

    @website.setter
    def website(self, value):
        if self.check_is_url(value) == False:
            raise ValueError("Not a valid URL: " + value)
        self._website = value

    @property
    def async_checking(self):
        return self._async_checking

    @property
    def api_url(self):
        return self._engine.api_url

    @api_url.setter
    def api_url(self, value):
        if self.check_is_url(value) == False:
            raise ValueError("Not a valid URL: " + value)
        self._engine.api_url = value

    @property
    def stage_path(self):
        return self._updater_path

    @stage_path.setter
    def stage_path(self, value):
        if value == None:
            if self._verbose: print("Aborting assigning stage_path, it's null")
            return
        elif value != None and not os.path.exists(value):
            try:
                os.makedirs(value)
            except:
                if self._verbose: print("Error trying to staging path")
                return
        self._updater_path = value

    @property
    def tags(self):
        if self._tags == []:
            return []
        tag_names = []
        for tag in self._tags:
            tag_names.append(tag["name"])

        return tag_names

    @property
    def tag_latest(self):
        if self._tag_latest == None:
            return None
        return self._tag_latest["name"]

    @property
    def latest_release(self):
        if self._releases_latest == None:
            return None
        return self._latest_release

    @property
    def current_version(self):
        return self._current_version

    @property
    def update_ready(self):
        return self._update_ready

    @property
    def update_version(self):
        return self._update_version

    @property
    def update_link(self):
        return self._update_link

    @current_version.setter
    def current_version(self, tuple_values):
        if tuple_values == None:
            self._current_version = None
            return
        elif type(tuple_values) is not tuple:
            try:
                tuple(tuple_values)
            except:
                raise ValueError(
                    "Not a tuple! current_version must be a tuple of integers")
        for i in tuple_values:
            if type(i) is not int:
                raise ValueError(
                    "Not an integer! current_version must be a tuple of integers")
        self._current_version = tuple(tuple_values)

    def set_check_interval(self, enable=False, months=0, days=14, hours=0, minutes=0):
        # enabled = False, default initially will not check against frequency
        # if enabled, default is then 2 weeks

        if type(enable) is not bool:
            raise ValueError("Enable must be a boolean value")
        if type(months) is not int:
            raise ValueError("Months must be an integer value")
        if type(days) is not int:
            raise ValueError("Days must be an integer value")
        if type(hours) is not int:
            raise ValueError("Hours must be an integer value")
        if type(minutes) is not int:
            raise ValueError("Minutes must be an integer value")

        if enable == False:
            self._check_interval_enable = False
        else:
            self._check_interval_enable = True

        self._check_interval_months = months
        self._check_interval_days = days
        self._check_interval_hours = hours
        self._check_interval_minutes = minutes

    @property
    def check_interval(self):
        return (self._check_interval_enable,
                self._check_interval_months,
                self._check_interval_days,
                self._check_interval_hours,
                self._check_interval_minutes)

    @property
    def error(self):
        return self._error

    @property
    def error_msg(self):
        return self._error_msg

    @property
    def version_min_update(self):
        return self._version_min_update

    @version_min_update.setter
    def version_min_update(self, value):
        if value == None:
            self._version_min_update = None
            return
        if type(value) != type((1, 2, 3)):
            raise ValueError("Version minimum must be a tuple")
        else:
            # potentially check entries are integers
            self._version_min_update = value

    @property
    def version_max_update(self):
        return self._version_max_update

    @version_max_update.setter
    def version_max_update(self, value):
        if value == None:
            self._version_max_update = None
            return
        if type(value) != type((1, 2, 3)):
            raise ValueError("Version maximum must be a tuple")
        else:
            # potentially check entries are integers
            self._version_max_update = value

    @property
    def backup_current(self):
        return self._backup_current

    @backup_current.setter
    def backup_current(self, value):
        if value == None:
            self._backup_current = False
            return
        else:
            self._backup_current = value

    @property
    def backup_ignore_patterns(self):
        return self._backup_ignore_patterns

    @backup_ignore_patterns.setter
    def backup_ignore_patterns(self, value):
        if value == None:
            self._backup_ignore_patterns = None
            return
        elif type(value) != type(['list']):
            raise ValueError("Backup pattern must be in list format")
        else:
            self._backup_ignore_patterns = value

    # -------------------------------------------------------------------------
    # Parameter validation related functions
    # -------------------------------------------------------------------------

    def check_is_url(self, url):
        if not ("http://" in url or "https://" in url):
            return False
        if "." not in url:
            return False
        return True

    def get_tag_names(self):
        tag_names = []
        self.get_tags(self)
        for tag in self._tags:
            tag_names.append(tag["name"])
        return tag_names

    # declare how the class gets printed

    def __repr__(self):
        return "<Module updater from {a}>".format(a=__file__)

    def __str__(self):
        return "Updater, with user: {a}, repository: {b}, url: {c}".format(
            a=self._user,
            b=self._repo, c=self.form_repo_url())

    # -------------------------------------------------------------------------
    # API-related functions
    # -------------------------------------------------------------------------

    def form_repo_url(self):
        return self._engine.form_repo_url(self)

    def form_tags_url(self):
        return self._engine.form_tags_url(self)

    def form_branch_url(self, branch):
        return self._engine.form_branch_url(branch, self)

    def get_tags(self):
        request = self.form_tags_url()
        if self._verbose: print("Getting tags from server")

        # get all tags, internet call
        all_tags = self._engine.parse_tags(self.get_api(request), self)
        self._prefiltered_tag_count = len(all_tags)

        # pre-process to skip tags
        if self.skip_tag != None:
            self._tags = [tg for tg in all_tags if self.skip_tag(self, tg) == False]
        else:
            self._tags = all_tags

        # get additional branches too, if needed, and place in front
        # does NO checking here whether branch is valid
        if self._include_branches == True:
            temp_branches = self._include_branch_list.copy()
            temp_branches.reverse()
            for branch in temp_branches:
                request = self.form_branch_url(branch)
                include = {
                    "name": branch.title(),
                    "zipball_url": request
                }
                self._tags = [include] + self._tags  # append to front

        if self._tags == None:
            # some error occurred
            self._tag_latest = None
            self._tags = []
            return
        elif self._prefiltered_tag_count == 0 and self._include_branches == False:
            self._tag_latest = None
            self._error = "No releases found"
            self._error_msg = "No releases or tags found on this repository"
            if self._verbose: print("No releases or tags found on this repository")
        elif self._prefiltered_tag_count == 0 and self._include_branches == True:
            self._tag_latest = self._tags[0]
            if self._verbose:
                branch = self._include_branch_list[0]
                print("{} branch found, no releases".format(branch), self._tags[0])
        elif len(self._tags) == 0 and self._prefiltered_tag_count > 0:
            self._tag_latest = None
            self._error = "No releases available"
            self._error_msg = "No versions found within compatible version range"
            if self._verbose: print("No versions found within compatible version range")
        else:
            if self._include_branches == False:
                self._tag_latest = self._tags[0]
                if self._verbose: print("Most recent tag found:", self._tags[0]['name'])
            else:
                # don't return branch if in list
                n = len(self._include_branch_list)
                self._tag_latest = self._tags[n]  # guaranteed at least len()=n+1
                if self._verbose: print("Most recent tag found:", self._tags[n]['name'])

    # all API calls to base url
    def get_raw(self, url):
        # print("Raw request:", url)
        request = urllib.request.Request(url)

        # setup private request headers if appropriate
        if self._engine.token != None:
            if self._engine.name == "gitlab":
                request.add_header('PRIVATE-TOKEN', self._engine.token)
            else:
                if self._verbose: print("Tokens not setup for engine yet")

        # run the request
        try:
            result = urllib.request.urlopen(request)
        except urllib.error.HTTPError as e:
            self._error = "HTTP error"
            self._error_msg = str(e.code)
            self._update_ready = None
        except urllib.error.URLError as e:
            self._error = "URL error, check internet connection"
            self._error_msg = str(e.reason)
            self._update_ready = None
            return None
        else:
            result_string = result.read()
            result.close()
            return result_string.decode()

    # result of all api calls, decoded into json format
    def get_api(self, url):
        # return the json version
        get = None
        get = self.get_raw(url)
        if get != None:
            try:
                return json.JSONDecoder().decode(get)
            except Exception as e:
                self._error = "API response has invalid JSON format"
                self._error_msg = str(e.reason)
                self._update_ready = None
                return None
        else:
            return None

    # create a working directory and download the new files
    def stage_repository(self, url):

        local = os.path.join(self._updater_path, "update_staging")
        error = None

        # make/clear the staging folder
        # ensure the folder is always "clean"
        if self._verbose: print("Preparing staging folder for download:\n", local)
        if os.path.isdir(local) == True:
            try:
                shutil.rmtree(local)
                os.makedirs(local)
            except:
                error = "failed to remove existing staging directory"
        else:
            try:
                os.makedirs(local)
            except:
                error = "failed to create staging directory"

        if error != None:
            if self._verbose: print("Error: Aborting update, " + error)
            self._error = "Update aborted, staging path error"
            self._error_msg = "Error: {}".format(error)
            return False

        if self._backup_current == True:
            self.create_backup()
        if self._verbose: print("Now retrieving the new source zip")

        self._source_zip = os.path.join(local, "source.zip")

        if self._verbose: print("Starting download update zip")
        try:
            request = urllib.request.Request(url)

            # setup private token if appropriate
            if self._engine.token != None:
                if self._engine.name == "gitlab":
                    request.add_header('PRIVATE-TOKEN', self._engine.token)
                else:
                    if self._verbose: print("Tokens not setup for selected engine yet")
            self.urlretrieve(urllib.request.urlopen(request), self._source_zip)
            # add additional checks on file size being non-zero
            if self._verbose: print("Successfully downloaded update zip")
            return True
        except Exception as e:
            self._error = "Error retrieving download, bad link?"
            self._error_msg = "Error: {}".format(e)
            if self._verbose:
                print("Error retrieving download, bad link?")
                print("Error: {}".format(e))
            return False

    def create_backup(self):
        if self._verbose: print("Backing up current addon folder")
        local = os.path.join(self._updater_path, "backup")
        tempdest = os.path.join(self._addon_root,
                                os.pardir,
                                self._addon + "_updater_backup_temp")

        if os.path.isdir(local) == True:
            shutil.rmtree(local)
        if self._verbose: print("Backup destination path: ", local)

        # make the copy
        if self._backup_ignore_patterns != None:
            shutil.copytree(
                self._addon_root, tempdest,
                ignore=shutil.ignore_patterns(*self._backup_ignore_patterns))
        else:
            shutil.copytree(self._addon_root, tempdest)
        shutil.move(tempdest, local)

        # save the date for future ref
        now = datetime.now()
        self._json["backup_date"] = "{m}-{d}-{yr}".format(
            m=now.strftime("%B"), d=now.day, yr=now.year)
        self.save_updater_json()

    def restore_backup(self):
        if self._verbose: print("Restoring backup")

        if self._verbose: print("Backing up current addon folder")
        backuploc = os.path.join(self._updater_path, "backup")
        tempdest = os.path.join(self._addon_root,
                                os.pardir,
                                self._addon + "_updater_backup_temp")
        tempdest = os.path.abspath(tempdest)

        # make the copy
        shutil.move(backuploc, tempdest)
        shutil.rmtree(self._addon_root)
        os.rename(tempdest, self._addon_root)

        self._json["backup_date"] = ""
        self._json["just_restored"] = True
        self._json["just_updated"] = True
        self.save_updater_json()

        self.reload_addon()

    def unpack_staged_zip(self, clean=False):

        if os.path.isfile(self._source_zip) == False:
            if self._verbose: print("Error, update zip not found")
            return -1

        # clear the existing source folder in case previous files remain
        try:
            shutil.rmtree(os.path.join(self._updater_path, "source"))
            os.makedirs(os.path.join(self._updater_path, "source"))
            if self._verbose: print("Source folder cleared and recreated")
        except:
            pass

        if self._verbose: print("Begin extracting source")
        if zipfile.is_zipfile(self._source_zip):
            with zipfile.ZipFile(self._source_zip) as zf:
                # extractall is no longer a security hazard, below is safe
                zf.extractall(os.path.join(self._updater_path, "source"))
        else:
            if self._verbose:
                print("Not a zip file, future add support for just .py files")
            raise ValueError("Resulting file is not a zip")
        if self._verbose: print("Extracted source")

        # either directly in root of zip, or one folder level deep
        unpath = os.path.join(self._updater_path, "source")
        if os.path.isfile(os.path.join(unpath, "__init__.py")) == False:
            dirlist = os.listdir(unpath)
            if len(dirlist) > 0:
                unpath = os.path.join(unpath, dirlist[0])

            # smarter check for additional sub folders for a single folder
            # containing __init__.py
            if os.path.isfile(os.path.join(unpath, "__init__.py")) == False:
                if self._verbose:
                    print("not a valid addon found")
                    print("Paths:")
                    print(dirlist)

                raise ValueError("__init__ file not found in new source")

        # now commence merging in the two locations:
        # note this MAY not be accurate, as updater files could be placed elsewhere
        origpath = os.path.dirname(__file__)

        # merge code with running addon directory, using blender default behavior
        # plus any modifiers indicated by user (e.g. force remove/keep)
        self.deepMergeDirectory(origpath, unpath, clean)

        # Now save the json state
        #  Change to True, to trigger the handler on other side
        #  if allowing reloading within same blender instance
        self._json["just_updated"] = True
        self.save_updater_json()
        self.reload_addon()
        self._update_ready = False

    # merge folder 'merger' into folder 'base' without deleting existing
    def deepMergeDirectory(self, base, merger, clean=False):
        if not os.path.exists(base):
            if self._verbose: print("Base path does not exist")
            return -1
        elif not os.path.exists(merger):
            if self._verbose: print("Merger path does not exist")
            return -1

        # paths to be aware of and not overwrite/remove/etc
        staging_path = os.path.join(self._updater_path, "update_staging")
        backup_path = os.path.join(self._updater_path, "backup")
        json_path = os.path.join(self._updater_path, "updater_status.json")

        # If clean install is enabled, clear existing files ahead of time
        # note: will not delete the update.json, update folder, staging, or staging
        # but will delete all other folders/files in addon directory
        error = None
        if clean == True:
            try:
                # implement clearing of all folders/files, except the
                # updater folder and updater json
                # Careful, this deletes entire subdirectories recursively...
                # make sure that base is not a high level shared folder, but
                # is dedicated just to the addon itself
                if self._verbose: print("clean=True, clearing addon folder to fresh install state")

                # remove root files and folders (except update folder)
                files = [f for f in os.listdir(base) if os.path.isfile(os.path.join(base, f))]
                folders = [f for f in os.listdir(base) if os.path.isdir(os.path.join(base, f))]

                for f in files:
                    os.remove(os.path.join(base, f))
                    print("Clean removing file {}".format(os.path.join(base, f)))
                for f in folders:
                    if os.path.join(base, f) == self._updater_path: continue
                    shutil.rmtree(os.path.join(base, f))
                    print("Clean removing folder and contents {}".format(os.path.join(base, f)))

            except error:
                error = "failed to create clean existing addon folder"
                print(error, str(e))

        # Walk through the base addon folder for rules on pre-removing
        # but avoid removing/altering backup and updater file
        for path, dirs, files in os.walk(base):
            # prune ie skip updater folder
            dirs[:] = [d for d in dirs if os.path.join(path, d) not in [self._updater_path]]
            for file in files:
                for ptrn in self.remove_pre_update_patterns:
                    if fnmatch.filter([file], ptrn):
                        try:
                            fl = os.path.join(path, file)
                            os.remove(fl)
                            if self._verbose: print("Pre-removed file " + file)
                        except OSError:
                            print("Failed to pre-remove " + file)

        # Walk through the temp addon sub folder for replacements
        # this implements the overwrite rules, which apply after
        # the above pre-removal rules. This also performs the
        # actual file copying/replacements
        for path, dirs, files in os.walk(merger):
            # verify this structure works to prune updater sub folder overwriting
            dirs[:] = [d for d in dirs if os.path.join(path, d) not in [self._updater_path]]
            relPath = os.path.relpath(path, merger)
            destPath = os.path.join(base, relPath)
            if not os.path.exists(destPath):
                os.makedirs(destPath)
            for file in files:
                # bring in additional logic around copying/replacing
                # Blender default: overwrite .py's, don't overwrite the rest
                destFile = os.path.join(destPath, file)
                srcFile = os.path.join(path, file)

                # decide whether to replace if file already exists, and copy new over
                if os.path.isfile(destFile):
                    # otherwise, check each file to see if matches an overwrite pattern
                    replaced = False
                    for ptrn in self._overwrite_patterns:
                        if fnmatch.filter([destFile], ptrn):
                            replaced = True
                            break
                    if replaced:
                        os.remove(destFile)
                        os.rename(srcFile, destFile)
                        if self._verbose: print("Overwrote file " + os.path.basename(destFile))
                    else:
                        if self._verbose: print(
                            "Pattern not matched to " + os.path.basename(destFile) + ", not overwritten")
                else:
                    # file did not previously exist, simply move it over
                    os.rename(srcFile, destFile)
                    if self._verbose: print("New file " + os.path.basename(destFile))

        # now remove the temp staging folder and downloaded zip
        try:
            shutil.rmtree(staging_path)
        except:
            error = "Error: Failed to remove existing staging directory, consider manually removing " + staging_path
            if self._verbose: print(error)

    def reload_addon(self):
        # if post_update false, skip this function
        # else, unload/reload addon & trigger popup
        if self._auto_reload_post_update == False:
            print("Restart blender to reload addon and complete update")
            return

        if self._verbose: print("Reloading addon...")
        addon_utils.modules(refresh=True)
        bpy.utils.refresh_script_paths()

        # not allowed in restricted context, such as register module
        # toggle to refresh
        bpy.ops.wm.addon_disable(module=self._addon_package)
        bpy.ops.wm.addon_refresh()
        bpy.ops.wm.addon_enable(module=self._addon_package)

    # -------------------------------------------------------------------------
    # Other non-api functions and setups
    # -------------------------------------------------------------------------

    def clear_state(self):
        self._update_ready = None
        self._update_link = None
        self._update_version = None
        self._source_zip = None
        self._error = None
        self._error_msg = None

    # custom urlretrieve implementation
    def urlretrieve(self, urlfile, filepath):
        chunk = 1024 * 8
        f = open(filepath, "wb")
        while 1:
            data = urlfile.read(chunk)
            if not data:
                # print("done.")
                break
            f.write(data)
        # print("Read %s bytes"%len(data))
        f.close()

    def version_tuple_from_text(self, text):
        if text == None: return ()

        # should go through string and remove all non-integers,
        # and for any given break split into a different section
        segments = []
        tmp = ''
        for l in str(text):
            if l.isdigit() == False:
                if len(tmp) > 0:
                    segments.append(int(tmp))
                    tmp = ''
            else:
                tmp += l
        if len(tmp) > 0:
            segments.append(int(tmp))

        if len(segments) == 0:
            if self._verbose: print("No version strings found text: ", text)
            if self._include_branches == False:
                return ()
            else:
                return (text)
        return tuple(segments)

    # called for running check in a background thread
    def check_for_update_async(self, callback=None):

        if self._json != None and "update_ready" in self._json:
            if self._json["update_ready"] == True:
                self._update_ready = True
                self._update_link = self._json["version_text"]["link"]
                self._update_version = str(self._json["version_text"]["version"])
                # cached update
                callback(True)
                return

        # do the check
        if self._check_interval_enable == False:
            return
        elif self._async_checking == True:
            if self._verbose: print("Skipping async check, already started")
            return  # already running the bg thread
        elif self._update_ready == None:
            self.start_async_check_update(False, callback)

    def check_for_update_now(self, callback=None):

        self._error = None
        self._error_msg = None

        if self._verbose:
            print("Check update pressed, first getting current status")
        if self._async_checking == True:
            if self._verbose: print("Skipping async check, already started")
            return  # already running the bg thread
        elif self._update_ready == None:
            self.start_async_check_update(True, callback)
        else:
            self._update_ready = None
            self.start_async_check_update(True, callback)

    # this function is not async, will always return in sequential fashion
    # but should have a parent which calls it in another thread
    def check_for_update(self, now=False):
        if self._verbose: print("Checking for update function")

        # clear the errors if any
        self._error = None
        self._error_msg = None

        # avoid running again in, just return past result if found
        # but if force now check, then still do it
        if self._update_ready != None and now == False:
            return (self._update_ready, self._update_version, self._update_link)

        if self._current_version == None:
            raise ValueError("current_version not yet defined")
        if self._repo == None:
            raise ValueError("repo not yet defined")
        if self._user == None:
            raise ValueError("username not yet defined")

        self.set_updater_json()  # self._json

        if now == False and self.past_interval_timestamp() == False:
            if self._verbose:
                print("Aborting check for updated, check interval not reached")
            return (False, None, None)

        # check if using tags or releases
        # note that if called the first time, this will pull tags from online
        if self._fake_install == True:
            if self._verbose:
                print("fake_install = True, setting fake version as ready")
            self._update_ready = True
            self._update_version = "(999,999,999)"
            self._update_link = "http://127.0.0.1"

            return (self._update_ready, self._update_version, self._update_link)

        # primary internet call
        self.get_tags()  # sets self._tags and self._tag_latest

        self._json["last_check"] = str(datetime.now())
        self.save_updater_json()

        # can be () or ('master') in addition to branches, and version tag
        new_version = self.version_tuple_from_text(self.tag_latest)

        if len(self._tags) == 0:
            self._update_ready = False
            self._update_version = None
            self._update_link = None
            return (False, None, None)
        elif self._include_branches == False:
            link = self._tags[0]["zipball_url"]  # potentially other sources
        else:
            n = len(self._include_branch_list)
            if len(self._tags) == n:
                # effectively means no tags found on repo
                # so provide the first one as default
                link = self._tags[0]["zipball_url"]  # potentially other sources
            else:
                link = self._tags[n]["zipball_url"]  # potentially other sources

        if new_version == ():
            self._update_ready = False
            self._update_version = None
            self._update_link = None
            return (False, None, None)
        elif str(new_version).lower() in self._include_branch_list:
            # handle situation where master/whichever branch is included
            # however, this code effectively is not triggered now
            # as new_version will only be tag names, not branch names
            if self._include_branch_autocheck == False:
                # don't offer update as ready,
                # but set the link for the default
                # branch for installing
                self._update_ready = True
                self._update_version = new_version
                self._update_link = link
                self.save_updater_json()
                return (True, new_version, link)
            else:
                raise ValueError("include_branch_autocheck: NOT YET DEVELOPED")
            # bypass releases and look at timestamp of last update
            # from a branch compared to now, see if commit values
            # match or not.

        else:
            # situation where branches not included

            if new_version > self._current_version:
                self._update_ready = True
                self._update_version = new_version
                self._update_link = link
                self.save_updater_json()
                return (True, new_version, link)

        # elif new_version != self._current_version:
        # 	self._update_ready = False
        # 	self._update_version = new_version
        # 	self._update_link = link
        # 	self.save_updater_json()
        # 	return (True, new_version, link)

        # if no update, set ready to False from None
        self._update_ready = False
        self._update_version = None
        self._update_link = None
        return (False, None, None)

    def set_tag(self, name):
        tg = None
        for tag in self._tags:
            if name == tag["name"]:
                tg = tag
                break
        if tg == None:
            raise ValueError("Version tag not found: " + revert_tag)
        new_version = self.version_tuple_from_text(self.tag_latest)
        self._update_version = new_version
        self._update_link = tg["zipball_url"]

    def run_update(self, force=False, revert_tag=None, clean=False, callback=None):
        # revert_tag: could e.g. get from drop down list
        # different versions of the addon to revert back to
        # clean: not used, but in future could use to totally refresh addon
        self._json["update_ready"] = False
        self._json["ignore"] = False  # clear ignore flag
        self._json["version_text"] = {}

        if revert_tag != None:
            self.set_tag(revert_tag)
            self._update_ready = True

        # clear the errors if any
        self._error = None
        self._error_msg = None

        if self._verbose: print("Running update")

        if self._fake_install == True:
            # change to True, to trigger the reload/"update installed" handler
            if self._verbose:
                print("fake_install=True")
                print("Just reloading and running any handler triggers")
            self._json["just_updated"] = True
            self.save_updater_json()
            if self._backup_current == True:
                self.create_backup()
            self.reload_addon()
            self._update_ready = False
            res = True  # fake "success" zip download flag

        elif force == False:
            if self._update_ready != True:
                if self._verbose: print("Update stopped, new version not ready")
                return "Update stopped, new version not ready"
            elif self._update_link == None:
                # this shouldn't happen if update is ready
                if self._verbose: print("Update stopped, update link unavailable")
                return "Update stopped, update link unavailable"

            if self._verbose and revert_tag == None:
                print("Staging update")
            elif self._verbose:
                print("Staging install")

            res = self.stage_repository(self._update_link)
            if res != True:
                print("Error in staging repository: " + str(res))
                if callback != None: callback(self._error_msg)
                return self._error_msg
            self.unpack_staged_zip(clean)

        else:
            if self._update_link == None:
                if self._verbose: print("Update stopped, could not get link")
                return "Update stopped, could not get link"
            if self._verbose: print("Forcing update")

            res = self.stage_repository(self._update_link)
            if res != True:
                print("Error in staging repository: " + str(res))
                if callback != None: callback(self._error_msg)
                return self._error_msg
            self.unpack_staged_zip(clean)
        # would need to compare against other versions held in tags

        # run the front-end's callback if provided
        if callback != None: callback()

        # return something meaningful, 0 means it worked
        return 0

    def past_interval_timestamp(self):
        if self._check_interval_enable == False:
            return True  # ie this exact feature is disabled

        if "last_check" not in self._json or self._json["last_check"] == "":
            return True
        else:
            now = datetime.now()
            last_check = datetime.strptime(self._json["last_check"],
                                           "%Y-%m-%d %H:%M:%S.%f")
            next_check = last_check
            offset = timedelta(
                days=self._check_interval_days + 30 * self._check_interval_months,
                hours=self._check_interval_hours,
                minutes=self._check_interval_minutes
            )

            delta = (now - offset) - last_check
            if delta.total_seconds() > 0:
                if self._verbose:
                    print("{} Updater: Time to check for updates!".format(self._addon))
                return True
            else:
                if self._verbose:
                    print("{} Updater: Determined it's not yet time to check for updates".format(self._addon))
                return False

    def set_updater_json(self):
        if self._updater_path == None:
            raise ValueError("updater_path is not defined")
        elif os.path.isdir(self._updater_path) == False:
            os.makedirs(self._updater_path)

        jpath = os.path.join(self._updater_path, "updater_status.json")
        if os.path.isfile(jpath):
            with open(jpath) as data_file:
                self._json = json.load(data_file)
                if self._verbose: print("{} Updater: Read in json settings from file".format(self._addon))
        else:
            # set data structure
            self._json = {
                "last_check": "",
                "backup_date": "",
                "update_ready": False,
                "ignore": False,
                "just_restored": False,
                "just_updated": False,
                "version_text": {}
            }
            self.save_updater_json()

    def save_updater_json(self):
        # first save the state
        if self._update_ready == True:
            if type(self._update_version) == type((0, 0, 0)):
                self._json["update_ready"] = True
                self._json["version_text"]["link"] = self._update_link
                self._json["version_text"]["version"] = self._update_version
            else:
                self._json["update_ready"] = False
                self._json["version_text"] = {}
        else:
            self._json["update_ready"] = False
            self._json["version_text"] = {}

        jpath = os.path.join(self._updater_path, "updater_status.json")
        outf = open(jpath, 'w')
        data_out = json.dumps(self._json, indent=4)
        outf.write(data_out)
        outf.close()
        if self._verbose:
            print(self._addon + ": Wrote out updater json settings to file, with the contents:")
            print(self._json)

    def json_reset_postupdate(self):
        self._json["just_updated"] = False
        self._json["update_ready"] = False
        self._json["version_text"] = {}
        self.save_updater_json()

    def json_reset_restore(self):
        self._json["just_restored"] = False
        self._json["update_ready"] = False
        self._json["version_text"] = {}
        self.save_updater_json()
        self._update_ready = None  # reset so you could check update again

    def ignore_update(self):
        self._json["ignore"] = True
        self.save_updater_json()

    # -------------------------------------------------------------------------
    # ASYNC stuff
    # -------------------------------------------------------------------------

    def start_async_check_update(self, now=False, callback=None):
        if self._async_checking == True:
            return
        if self._verbose: print("{} updater: Starting background checking thread".format(self._addon))
        check_thread = threading.Thread(target=self.async_check_update,
                                        args=(now, callback,))
        check_thread.daemon = True
        self._check_thread = check_thread
        check_thread.start()

        return True

    def async_check_update(self, now, callback=None):
        self._async_checking = True
        if self._verbose: print("{} BG thread: Checking for update now in background".format(self._addon))
        # time.sleep(3)  # to test background, in case internet too fast to tell
        # try:
        self.check_for_update(now=now)
        # except Exception as exception:
        # 	print("Checking for update error:")
        # 	print(exception)
        # 	self._update_ready = False
        # 	self._update_version = None
        # 	self._update_link = None
        # 	self._error = "Error occurred"
        # 	self._error_msg = "Encountered an error while checking for updates"

        if self._verbose:
            print("{} BG thread: Finished checking for update, doing callback".format(self._addon))
        if callback != None: callback(self._update_ready)
        self._async_checking = False
        self._check_thread = None

    def stop_async_check_update(self):
        if self._check_thread != None:
            try:
                if self._verbose: print("Thread will end in normal course.")
            # however, "There is no direct kill method on a thread object."
            # better to let it run its course
            # self._check_thread.stop()
            except:
                pass
        self._async_checking = False
        self._error = None
        self._error_msg = None

# -----------------------------------------------------------------------------
# Updater Engines
# -----------------------------------------------------------------------------

class BitbucketEngine(object):

    def __init__(self):
        self.api_url = 'https://api.bitbucket.org'
        self.token = None
        self.name = "bitbucket"

    def form_repo_url(self, updater):
        return self.api_url + "/2.0/repositories/" + updater.user + "/" + updater.repo

    def form_tags_url(self, updater):
        return self.form_repo_url(updater) + "/refs/tags?sort=-name"

    def form_branch_url(self, branch, updater):
        return self.get_zip_url(branch, updater)

    def get_zip_url(self, name, updater):
        return "https://bitbucket.org/{user}/{repo}/get/{name}.zip".format(
            user=updater.user,
            repo=updater.repo,
            name=name)

    def parse_tags(self, response, updater):
        if response == None:
            return []
        return [{"name": tag["name"], "zipball_url": self.get_zip_url(tag["name"], updater)} for tag in
                response["values"]]

class GithubEngine(object):

    def __init__(self):
        self.api_url = 'https://api.github.com'
        self.token = None
        self.name = "github"

    def form_repo_url(self, updater):
        return "{}{}{}{}{}".format(self.api_url, "/repos/", updater.user,
                                   "/", updater.repo)

    def form_tags_url(self, updater):
        return "{}{}".format(self.form_repo_url(updater), "/tags")

    def form_branch_list_url(self, updater):
        return "{}{}".format(self.form_repo_url(updater), "/branches")

    def form_branch_url(self, branch, updater):
        return "{}{}{}".format(self.form_repo_url(updater),
                               "/zipball/", branch)

    def parse_tags(self, response, updater):
        return response

class GitlabEngine(object):

    def __init__(self):
        self.api_url = 'https://gitlab.com'
        self.token = None
        self.name = "gitlab"

    def form_repo_url(self, updater):
        return "{}{}{}".format(self.api_url, "/api/v3/projects/", updater.repo)

    def form_tags_url(self, updater):
        return "{}{}".format(self.form_repo_url(updater), "/repository/tags")

    def form_branch_list_url(self, updater):
        # does not validate branch name.
        return "{}{}".format(
            self.form_repo_url(updater),
            "/repository/branches")

    def form_branch_url(self, branch, updater):
        # Could clash with tag names and if it does, it will
        # download TAG zip instead of branch zip to get
        # direct path, would need.
        return "{}{}{}".format(
            self.form_repo_url(updater),
            "/repository/archive.zip?sha=",
            branch)

    def get_zip_url(self, sha, updater):
        return "{base}/repository/archive.zip?sha:{sha}".format(
            base=self.form_repo_url(updater),
            sha=sha)

    # def get_commit_zip(self, id, updater):
    # 	return self.form_repo_url(updater)+"/repository/archive.zip?sha:"+id

    def parse_tags(self, response, updater):
        if response == None:
            return []
        return [{"name": tag["name"], "zipball_url": self.get_zip_url(tag["commit"]["id"], updater)} for tag in
                response]

# -----------------------------------------------------------------------------
# The module-shared class instance,
# should be what's imported to other files
# -----------------------------------------------------------------------------

Updater = Singleton_updater()


from mock import patch

from bluebottle.payments_logger.adapters import PaymentLogAdapter
from bluebottle.payments.services import PaymentService
from bluebottle.payments_docdata.adapters import DocdataPaymentAdapter
from bluebottle.payments_logger.models import PaymentLogEntry
from bluebottle.test.factory_models.orders import OrderFactory
from bluebottle.test.factory_models.payments import OrderPaymentFactory
from bluebottle.test.utils import BluebottleTestCase, FsmTestMixin

class TestPaymentLogger(BluebottleTestCase, FsmTestMixin):
    @patch('bluebottle.payments_docdata.adapters.gateway.DocdataClient')
    def setUp(self, mock_client):
        super(TestPaymentLogger, self).setUp()

        # Mock response to creating the payment at docdata
        instance = mock_client.return_value
        instance.create.return_value = {'order_key': 123, 'order_id': 123}

        self.order = OrderFactory.create(total=35)
        self.order_payment = OrderPaymentFactory.create(
            payment_method='docdataIdeal', order=self.order,
            integration_data={'default_pm': 'ideal'})
        self.service = PaymentService(self.order_payment)

    def test_create_payment_create_log(self):
        """
        This test will start the process of creating a new payment and tests if
        a log associated has been created
        """
        # Get the number of logs in the table
        last_log = PaymentLogEntry.objects.all().order_by('-timestamp')[:1][0]

        # The latest entry should be for the payment associated with this test
        self.assertEqual(last_log.payment_id, self.order_payment.payment.id)

    @patch.object(DocdataPaymentAdapter, '_store_payment_transaction')
    @patch.object(DocdataPaymentAdapter, '_fetch_status')
    def test_check_authorized_status_logged(self, mock_fetch_status,
                                            mock_transaction):
        # Mock the status check with docdata
        mock_fetch_status.return_value = self.create_status_response(
            'AUTHORIZED',
            totals={'totalAcquirerApproved': '1000', 'totalRegistered': '1000'}
        )
        self.service.check_payment_status()

        last_log = PaymentLogEntry.objects.all().order_by('-timestamp')[:1][0]

        # Check that the status change was logged
        self.assertEqual(last_log.payment_id, self.order_payment.payment.id)
        self.assertEqual(last_log.message,
                         'DocdataPayment object - a new payment status authorized')
        self.assertEqual(last_log.level, 'INFO')

class TestPaymentLoggerAdapter(BluebottleTestCase):
    @patch('bluebottle.payments_docdata.adapters.gateway.DocdataClient')
    def setUp(self, mock_client):
        super(TestPaymentLoggerAdapter, self).setUp()

        # Mock response to creating the payment at docdata
        instance = mock_client.return_value
        instance.create.return_value = {'order_key': 123, 'order_id': 123}

        self.order = OrderFactory.create()
        self.order_payment = OrderPaymentFactory.create(
            payment_method='docdata', order=self.order,
            integration_data={'default_pm': 'ideal'})
        self.service = PaymentService(self.order_payment)

        PaymentLogEntry.objects.all().delete()

    def test_payment_log_adapter(self):
        """
        Tests the adapter creating different log messages
        """
        payment = self.order_payment.payment
        payment_logger = PaymentLogAdapter()

        payment_logger.log(payment=payment,
                           level='ERROR',
                           message='Test Error log')
        payment_logger.log(payment=payment,
                           level='INFO',
                           message='Test Info log')
        payment_logger.log(payment=payment,
                           level='WARN',
                           message='Test Warn log')

        self.assertEqual(3, PaymentLogEntry.objects.all().count())


# 使用方式类似于AC自动机:
# KMP(pattern)：构造函数, pattern为模式串.
# Match(s,start): 返回模式串在s中出现的所有位置.
# Move(pos, char): 从当前状态pos沿着char移动到下一个状态, 如果不存在则移动到fail指针指向的状态.
# IsMatched(pos): 判断当前状态pos是否为匹配状态.
# Period(i): 求字符串 s 的前缀 s[:i+1] 的最短周期(0<=i<n). 如果不存在周期, 返回0.

# https://www.ruanyifeng.com/blog/2013/05/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm.html

from typing import List, Optional

class KMP:
    """单模式串匹配"""

    @staticmethod
    def getNext(pattern: str) -> List[int]:
        next = [0] * len(pattern)
        j = 0
        for i in range(1, len(pattern)):
            while j and pattern[i] != pattern[j]:
                j = next[j - 1]
            if pattern[i] == pattern[j]:
                j += 1
            next[i] = j
        return next

    __slots__ = ("next", "_pattern")

    def __init__(self, pattern: str):
        self._pattern = pattern
        self.next = self.getNext(pattern)

    def match(self, s: str, start=0) -> List[int]:
        res = []
        pos = 0
        for i in range(start, len(s)):
            pos = self.move(pos, s[i])
            if self.isMatched(pos):
                res.append(i - len(self._pattern) + 1)
                pos = 0
        return res

    def move(self, pos: int, input_: str) -> int:
        assert 0 <= pos < len(self._pattern)
        while pos and input_ != self._pattern[pos]:
            pos = self.next[pos - 1]  # rollback
        if input_ == self._pattern[pos]:
            pos += 1
        return pos

    def isMatched(self, pos: int) -> bool:
        return pos == len(self._pattern)

    def period(self, i: Optional[int] = None) -> int:
        """
        求字符串 s 的前缀 s[:i+1] 的最短周期(0<=i<n)
        如果不存在周期, 返回0.
        """
        if i is None:
            i = len(self._pattern) - 1
        assert 0 <= i < len(self._pattern)
        res = (i + 1) - self.next[i]
        if res and (i + 1) > res and (i + 1) % res == 0:
            return res
        return 0

def getNext(needle: str) -> List[int]:
    """kmp O(n)求 `needle`串的 `next`数组
    `next[i]`表示`[:i+1]`这一段字符串中最长公共前后缀(不含这一段字符串本身,即真前后缀)的长度
    """
    next = [0] * len(needle)
    j = 0
    for i in range(1, len(needle)):
        while j and needle[i] != needle[j]:  # 1. fallback后前进：匹配不成功j往右走
            j = next[j - 1]
        if needle[i] == needle[j]:  # 2. 匹配：匹配成功j往右走一步
            j += 1
        next[i] = j
    return next

if __name__ == "__main__":
    next = getNext("aabaabaabaab")  # 模式串的next数组
    assert next == [0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

    kmp = KMP("aab")
    assert kmp.match("aabaabaabaab") == [0, 3, 6, 9]
    assert kmp.match("aabaabaabaab", 1) == [3, 6, 9]

    pos = 0
    nextPos = kmp.move(pos, "a")
    assert nextPos == 1
    nextPos = kmp.move(nextPos, "a")
    assert nextPos == 2
    nextPos = kmp.move(nextPos, "b")
    assert kmp.isMatched(nextPos)


from django.urls import path
from . import views
urlpatterns = [
    path('', views.index, name="index"),
    path('subjects', views.subjects, name='subjects'),
    path('subjects/<int:topics_id>', views.topics, name="topics"),
    path('subjects/ans/<int:topic_id>', views.topic, name="topic"),
    path('new_question/<subject_id>', views.new_topic, name="new_topic"),
    path('new_answer/<topic_id>', views.new_entry, name="new_entry"),
    path('edit_answer/<edit_id>', views.edit_entry, name="edit_entry"),
    path('delete_answer/<edit_id>', views.delete_entry, name="delete_entry"),
]


from pyfiglet import figlet_format
from termcolor import colored
import requests
from random import choice

URL = "https://icanhazdadjoke.com/search"

def print_heading(heading):
    heading = colored(figlet_format(heading), color="magenta")
    print(heading)

def find_joke(term):
    response = requests.get(
        URL,
        headers={
            "Accept": "application/json"
        },
        params={
            "term": term
        }
    )
    results = response.json()["results"]
    print_joke(results, term)

def print_joke(results, term):
    if not results:
        return print(f"Sorry, I don't have any jokes about {term}! Please try again.")
    else:
        print("I've got one joke about fruit. Here it is:")
        print(choice(results)["joke"])

print_heading("Dad Joke 3000")
topic = input("Let me tell you a joke! Give me a topic: ")
find_joke(topic)


from utils.database_connection_utils import dbc_Utils

class sql:

    @classmethod
    def select_by_id(cls,table):
        connect = dbc_Utils.get_connect()
        cursor = dbc_Utils.get_cursor(connect)

        sql = "SELECT id FROM %s" % table
        cursor.execute(sql)

        return cursor.fetchone[0]

    @classmethod
    def delete_by_id(cls,table,id):
        connect = dbc_Utils.get_connect()
        cursor = dbc_Utils.get_cursor(connect)

        sql = "DELETE FROM %s WHERE id = %s" % (table,id)
        cursor.execute(sql)
        connect.commit()

        return cursor.rowcount


import cv2
import os
import numpy as np

def read_images_color(file_list):
    ims = []
    for file in file_list:
        im = cv2.imread(file)
        im = np.array([im])
        im = np.swapaxes(im, 0, 1)
        im = np.swapaxes(im, 1, 2)
        ims.append(im)
    return np.array(ims, np.float32)

def read_images_gray(file_list):
    ims = []
    for file in file_list:
        im = cv2.imread(file)
        im = np.array([cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)])
        im = np.swapaxes(im, 0, 1)
        im = np.swapaxes(im, 1, 2)
        ims.append(im)
    return np.array(ims, np.float32)

def get_file_list(root):
    file_list = []
    for file in os.listdir(root):
        file_list.append(os.path.join(root, file))
    return file_list

def save_image(files, root):
    if not os.path.exists(root):
        os.mkdir(root)

    m = np.min(files)
    l = np.max(files) - m
    files = (files - m) / l

    i = 0
    for im in files:
        cv2.imwrite(os.path.join(root, str(i) + '.png'), im)
        i += 1


from confluent_kafka import Producer
from typing import Sequence

from snuba import settings, util
from snuba.consumer import ConsumerWorker
from snuba.consumers.snapshot_worker import SnapshotAwareWorker
from snuba.datasets.factory import enforce_table_writer, get_dataset
from snuba.snapshots import SnapshotId
from snuba.stateful_consumer.control_protocol import TransactionData
from snuba.utils.streams.batching import BatchingConsumer
from snuba.utils.streams.kafka import KafkaConsumer, KafkaConsumerWithCommitLog, KafkaMessage, TransportError, build_kafka_consumer_configuration

class ConsumerBuilder:
    """
    Simplifies the initialization of a batching consumer by merging
    parameters that generally come from the command line with defaults
    that come from the dataset class and defaults that come from the
    settings file.
    """

    def __init__(
        self,
        dataset_name: str,
        raw_topic: str,
        replacements_topic: str,
        max_batch_size: int,
        max_batch_time_ms: int,
        bootstrap_servers: Sequence[str],
        group_id: str,
        commit_log_topic: str,
        auto_offset_reset: str,
        queued_max_messages_kbytes: int,
        queued_min_messages: int,
        dogstatsd_host: str,
        dogstatsd_port: int
    ) -> None:
        self.dataset = get_dataset(dataset_name)
        self.dataset_name = dataset_name
        if not bootstrap_servers:
            self.bootstrap_servers = settings.DEFAULT_DATASET_BROKERS.get(
                dataset_name,
                settings.DEFAULT_BROKERS,
            )
        else:
            self.bootstrap_servers = bootstrap_servers

        stream_loader = enforce_table_writer(self.dataset).get_stream_loader()
        self.raw_topic = raw_topic or stream_loader.get_default_topic_spec().topic_name
        default_replacement_topic_name = stream_loader.get_replacement_topic_spec().topic_name \
            if stream_loader.get_replacement_topic_spec() \
            else None
        self.replacements_topic = replacements_topic or default_replacement_topic_name
        default_commit_log_topic_name = stream_loader.get_commit_log_topic_spec().topic_name \
            if stream_loader.get_commit_log_topic_spec() \
            else None
        self.commit_log_topic = commit_log_topic or default_commit_log_topic_name

        # XXX: This can result in a producer being built in cases where it's
        # not actually required.
        self.producer = Producer({
            'bootstrap.servers': ','.join(self.bootstrap_servers),
            'partitioner': 'consistent',
            'message.max.bytes': 50000000,  # 50MB, default is 1MB
        })

        self.metrics = util.create_metrics(
            dogstatsd_host, dogstatsd_port, 'snuba.consumer',
            tags={
                "group": group_id,
                "dataset": self.dataset_name,
            }
        )

        self.max_batch_size = max_batch_size
        self.max_batch_time_ms = max_batch_time_ms
        self.group_id = group_id
        self.auto_offset_reset = auto_offset_reset
        self.queued_max_messages_kbytes = queued_max_messages_kbytes
        self.queued_min_messages = queued_min_messages

    def __build_consumer(self, worker: ConsumerWorker) -> BatchingConsumer:
        configuration = build_kafka_consumer_configuration(
            bootstrap_servers=self.bootstrap_servers,
            group_id=self.group_id,
            auto_offset_reset=self.auto_offset_reset,
            queued_max_messages_kbytes=self.queued_max_messages_kbytes,
            queued_min_messages=self.queued_min_messages,
        )

        if self.commit_log_topic is None:
            consumer = KafkaConsumer(configuration)
        else:
            consumer = KafkaConsumerWithCommitLog(
                configuration,
                self.producer,
                self.commit_log_topic,
            )

        return BatchingConsumer(
            consumer,
            self.raw_topic,
            worker=worker,
            max_batch_size=self.max_batch_size,
            max_batch_time=self.max_batch_time_ms,
            metrics=self.metrics,
            recoverable_errors=[TransportError],
        )

    def build_base_consumer(self) -> BatchingConsumer:
        """
        Builds the consumer with a ConsumerWorker.
        """
        return self.__build_consumer(
            ConsumerWorker(
                self.dataset,
                producer=self.producer,
                replacements_topic=self.replacements_topic,
                metrics=self.metrics
            )
        )

    def build_snapshot_aware_consumer(
        self,
        snapshot_id: SnapshotId,
        transaction_data: TransactionData,
    ) -> BatchingConsumer:
        """
        Builds the consumer with a ConsumerWorker able to handle snapshots.
        """
        worker = SnapshotAwareWorker(
            dataset=self.dataset,
            producer=self.producer,
            snapshot_id=snapshot_id,
            transaction_data=transaction_data,
            metrics=self.metrics,
            replacements_topic=self.replacements_topic,
        )
        return self.__build_consumer(worker)


#!flask/bin/python
from __future__ import print_function
from flask import Flask, jsonify, abort, request, make_response, url_for
from flask import render_template, redirect
from datetime import datetime, timedelta
import boto3
import requests
import json
import ast
#Enter AWS Credentials
AWS_KEY=""
AWS_SECRET=""
REGION="us-east-2"

# Get the table
dynamodb = boto3.resource('dynamodb', aws_access_key_id=AWS_KEY,
                            aws_secret_access_key=AWS_SECRET,
                            region_name=REGION)

app = Flask(__name__, static_url_path="")

####Globals
numCluster = 6;

@app.route('/', methods=['GET'])
def home_page():

    return render_template('index.html')

#Weather Routes
@app.route('/getPastWeather', methods=['GET'])
#Get past weather data for the past 30 days
def get_past_weather():
  past30Days = []
  for i in range(0, 5): #Change the '5' to '30' to get weather for past 30 days
    #Get date i days ago
    date = str(datetime.now() - timedelta(days=i))[0:10]
    date = datetime.strptime(date, '%Y-%m-%d').strftime('%m/%d/%Y')

    #Pass date to wunderground API
    apiDate = "history_" + datetime.strptime(date, '%m/%d/%Y').strftime('%Y%m%d')

    result = requests.get('http://api.wunderground.com/api/29c156d422e6a0ff/' + apiDate + '/q/IL/Chicago.json').json()

    #Populate list
    weatherData = {}
    weatherData['Date'] = date
    weatherData['PrecipTotal'] = result['history']['dailysummary'][0]['precipi']
    weatherData['Tavg'] = result['history']['dailysummary'][0]['meantempi']
    weatherData['Tmax'] = result['history']['dailysummary'][0]['maxtempi']
    weatherData['Tmin'] = result['history']['dailysummary'][0]['mintempi']

    past30Days.append(weatherData)
  return render_template('predictive.html', pastWeather = past30Days)

@app.route('/getForecast', methods=['GET'])
#Get weather forecast for next five days including current day
def get_forecast():
  result = requests.get('https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20weather.forecast%20where%20woeid%20in%20(select%20woeid%20from%20geo.places(1)%20where%20text%3D%22chicago%2C%20IL%22)&format=json&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys').json()

  forecast = []
  for i in range(0,5):
  	date = str(datetime.now() + timedelta(days=i))[0:10]
  	date = datetime.strptime(date, '%Y-%m-%d').strftime('%m/%d/%Y')

  	day = {}
  	day['Date'] = date
  	day['Tmin'] = result['query']['results']['channel']['item']['forecast'][i]['low']
  	day['Tmax'] = result['query']['results']['channel']['item']['forecast'][i]['high']
  	day['Tavg'] = str((int(day['Tmin']) + int(day['Tmax']))/2)
  	forecast.append(day)
  return render_template('predictive.html', forecast = forecast)

@app.route('/weatherdata', methods=['GET'])
def get_data_weather():
  #Get Appropriate data from Dynamo
  weatherTable = dynamodb.Table('ChicagoWeather')

  response = weatherTable.scan()

  items = response['Items']

  weatherdataList = []
  dateString = []
  if len(items) > 0:
    for item in items:
      data = {}
      #split date into Y-M-D for chart
      dateString = str(item["Date"]).split('/')
      for i in range(0, len(dateString)):
        if len(dateString[i]) < 2:
          dateString[i] = '0' + dateString[i]
      data["Date"] = int(dateString[2] + dateString[0] + dateString[1])  #for sorting x-axis
      data["Month"] = int(dateString[0])
      data["Day"] = int(dateString[1])
      data["Year"] = int('20' + dateString[2])

      #other data
      data["Depart"] = float(item["Depart"])
      data["Heat"] = int(item["Heat"])
      data["PrecipTotal"] = float(item["PrecipTotal"])
      data["Tavg"] = int(item["Tavg"])
      data["Tmax"] = int(item["Tmax"])
      data["Tmin"] = int(item["Tmin"])
      weatherdataList.append(data)
    return jsonify(sorted(weatherdataList, key=lambda k: k["Date"]))
  else:
    return jsonify({'Date': 0, 'Year': 0, 'Month': 0, 'Day': 0, 'Depart': 0, 'Heat': 0, 'PrecipTotal': 0, 'Tavg': 0, 'Tmax': 0, 'Tmin': 0})

@app.route('/weatherdata/<dataField>', methods=['GET'])
def get_field_data_weather(dataField):
  #Get Appropriate data from Dynamo
  weatherTable = dynamodb.Table('ChicagoWeather')

  response = weatherTable.scan()

  items = response['Items']

  weatherdataList = []
  dateString = []
  if len(items) > 0:
    for item in items:
      data = {}
      #split date into Y-M-D for chart
      dateString = str(item["Date"]).split('/')
      for i in range(0, len(dateString)):
        if len(dateString[i]) < 2:
          dateString[i] = '0' + dateString[i]
      data["Date"] = int(dateString[2] + dateString[0] + dateString[1])  #for sorting x-axis
      data["Month"] = int(dateString[0])
      data["Day"] = int(dateString[1])
      data["Year"] = int('20' + dateString[2])

      #other data
      data[dataField] = float(item[dataField])
      weatherdataList.append(data)
    return jsonify(sorted(weatherdataList, key=lambda k: k["Date"]))
  else:
    return jsonify({'Date': 0, 'Year': 0, 'Month': 0, 'Day': 0, dataField: 0})

@app.route('/weather', methods=['GET'])
def weather_page():
  return render_template('weather.html')

#Crime Routes
@app.route('/crimedata', methods=['GET'])
def get_data_crime():
  #Get Appropriate data from Dynamo
  crimeTable = dynamodb.Table('ChicagoCrime')

  response = crimeTable.scan()

  items = response['Items']

  crimedataList = []
  dateString = []
  if len(items) > 0:
    for item in items:
      data = {}
      #split date into Y-M-D for chart
      dateString = str(item["Date"]).split('/')
      for i in range(0, len(dateString)):
        if len(dateString[i]) < 2:
          dateString[i] = '0' + dateString[i]
      data["Date"] = int(dateString[2] + dateString[0] + dateString[1])  #for sorting x-axis
      data["Month"] = int(dateString[0])
      data["Day"] = int(dateString[1])
      data["Year"] = int('20' + dateString[2])

      #other data
      keys = item.keys()
      for key in keys:
        if key == 'Date':
          pass
        else:
          data[key] = int(item[key])
      crimedataList.append(data)
    return jsonify(sorted(crimedataList, key=lambda k: k["Date"]))

@app.route('/crimedata/<dataField>', methods=['GET'])
def get_field_data_crime(dataField):
  #Get Appropriate data from Dynamo

  #Handle invalid characters in url, i.e. '/', ',', ' '
  dataFieldNew = dataField.replace('Location_', 'Location: ')
  dataFieldNew = dataFieldNew.replace('Type_', 'Type: ')
  dataFieldNew = dataFieldNew.replace('_s_', ' / ')
  dataFieldNew = dataFieldNew.replace('_c_', ', ')
  dataFieldNew = dataFieldNew.replace('_p1_', '(')
  dataFieldNew = dataFieldNew.replace('_p2_', ')')
  dataFieldNew = dataFieldNew.replace('_x_', '.')
  dataFieldNew = dataFieldNew.replace('_h_', '-')
  dataFieldNew = dataFieldNew.replace('__', '/')
  dataFieldNew = dataFieldNew.replace('_', ' ')

  crimeTable = dynamodb.Table('ChicagoCrime')

  response = crimeTable.scan()

  items = response['Items']

  crimedataList = []
  dateString = []
  if len(items) > 0:
    for item in items:
      data = {}
      #split date into Y-M-D for chart
      dateString = str(item["Date"]).split('/')
      for i in range(0, len(dateString)):
        if len(dateString[i]) < 2:
          dateString[i] = '0' + dateString[i]
      data["Date"] = int(dateString[2] + dateString[0] + dateString[1])  #for sorting x-axis
      data["Month"] = int(dateString[0])
      data["Day"] = int(dateString[1])
      data["Year"] = int('20' + dateString[2])

      #other data
      data[dataField] = float(item[dataFieldNew])
      crimedataList.append(data)
    return jsonify(sorted(crimedataList, key=lambda k: k["Date"]))
  else:
    return jsonify({'Date': 0, 'Year': 0, 'Month': 0, 'Day': 0, dataField: 0})

#Public Areas/Others Crimes Page
@app.route('/crime', methods=['GET'])
def crimedata_page():
  return render_template('crime.html')

#Residential Crimes Page
@app.route('/crimeResidential', methods=['GET'])
def crimeResidentialdata_page():
  return render_template('crimeResidential.html')

#Stores/Restaurants Crimes Page
@app.route('/crimeStores', methods=['GET'])
def crimeStoresdata_page():
  return render_template('crimeStores.html')

#Crime Types Page 1
@app.route('/crimeTypes1', methods=['GET'])
def crimeTypes1data_page():
  return render_template('crimeTypes1.html')

#Crime Types Page 2
@app.route('/crimeTypes2', methods=['GET'])
def crimeTypes2data_page():
  return render_template('crimeTypes2.html')

#Crime Types Page 3
@app.route('/crimeTypes3', methods=['GET'])
def crimeTypes3data_page():
  return render_template('crimeTypes3.html')

#Crime Types Page 4
@app.route('/crimeTypes4', methods=['GET'])
def crimeTypes4data_page():
  return render_template('crimeTypes4.html')

#Total Crimes Chart
@app.route('/crimeTotal', methods=['GET'])
def crimeTotaldata_page():
  return render_template('crimeTotal.html')

#Predictive Routes
@app.route('/kmeansClusters', methods = ['GET'])
def kmeansClusters():
  global numCluster
  numCluster = request.args.get('quantity')
  return render_template('predictive.html')

@app.route('/predictivedata', methods=['GET'])
def get_data_predictive():
  #print(numCluster)
  #Get Appropriate data from Spark Flask App
  r = requests.get('http://0.0.0.0:8081/kmeans/' + str(numCluster))
  kmeansData = ast.literal_eval(r.text)
  r = requests.get('http://0.0.0.0:8081/linearreg')
  linRegData = ast.literal_eval(r.text)
  r = requests.get('http://0.0.0.0:8081/corr')
  corrData = ast.literal_eval(r.text)
  r = requests.get('http://0.0.0.0:8081/gbtreg')
  gbtRegData = ast.literal_eval(r.text)
  r = requests.get('http://0.0.0.0:8081/rfreg')
  rfRegData = ast.literal_eval(r.text)
  r = requests.get('http://0.0.0.0:8081/dectreereg')
  dectreeRegData = ast.literal_eval(r.text)

  print(linRegData)
  result = jsonify({
          'LinRegData': linRegData,
          'kmeansData': kmeansData,
          'corrData': corrData,
          'gbtRegData': gbtRegData,
          'rfRegData': rfRegData,
          'dectreeRegData': dectreeRegData
          })
  return result

@app.route('/predictive', methods=['GET'])
def predictive_page():
  return render_template('predictive.html')

#addional routes may be needed for interactivity

if __name__ == '__main__':
    app.run(debug=True, port=8000)


from maya import cmds

import pyblish.api
import colorbleed.api
import colorbleed.maya.action
import colorbleed.maya.lib as lib

class ValidateRenderNoDefaultCameras(pyblish.api.InstancePlugin):
    """Ensure no default (startup) cameras are to be rendered."""

    order = colorbleed.api.ValidateContentsOrder
    hosts = ['maya']
    families = ['colorbleed.renderlayer']
    label = "No Default Cameras Renderable"
    actions = [colorbleed.maya.action.SelectInvalidAction]

    @staticmethod
    def get_invalid(instance):

        layer = instance.data["setMembers"]

        # Collect default cameras
        cameras = cmds.ls(type='camera', long=True)
        defaults = [cam for cam in cameras if
                    cmds.camera(cam, query=True, startupCamera=True)]

        invalid = []
        with lib.renderlayer(layer):
            for cam in defaults:
                if cmds.getAttr(cam + ".renderable"):
                    invalid.append(cam)

        return invalid

    def process(self, instance):
        """Process all the cameras in the instance"""
        invalid = self.get_invalid(instance)
        if invalid:
            raise RuntimeError("Renderable default cameras "
                               "found: {0}".format(invalid))


import cv2
import tensorflow as tf
import numpy as np
sess = tf.Session('', tf.Graph())
cam = cv2.VideoCapture(0)
with sess.graph.as_default():
	# Read meta graph and checkpoint to restore session
	saver = tf.train.import_meta_graph("cnn_files\model.ckpt-10012.meta")
	saver.restore(sess,"cnn_files\model.ckpt-10012")
	while True:
		_, frame = cam.read()
		image = cv2.resize(frame, (32, 32), interpolation=cv2.INTER_CUBIC)
		image = tf.expand_dims(image, axis=0)
		image = tf.cast(image, tf.float32, name="image")
		image = sess.run(image)
		# Start the queue runners. If they are not started the program will hang
		coord = tf.train.Coordinator()
		threads =[]
		for qr in sess.graph.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
			threads.extend(qr.create_threads(sess, coord=coord, daemon=True,
											 start=True))

		# In the graph created above, feed "is_training" and "imgs" placeholders
		# Feeding them will disconnect the path from queue runners to the graph
		# and enable a path from the placeholder instead. The "img" placeholder will be
		# fed with the image that was read above.
		logits = sess.run('softmax_linear/softmax_linear:0',
						  feed_dict={'is_training:0': False, 'imgs:0':image})
		#Print classification results
		print(logits)


import concurrent.futures
import json

import requests
from tqdm import tqdm

raw_text = "Slicing can be best visualized by considering the index to be between the elements as shown below. So if we want to access a range, we need two indices that will slice that portion from the list."
data = raw_text.split()
DATA = [w.upper() for w in data]

# Retrieve a single page and report the URL and contents
def get_lower(word):
    with requests.get('http://localhost:5000/api/lowercase/{}'.format(word)) as response:
        json_data = json.loads(response.text)
        return json_data['task']

if __name__ == '__main__':
    result = []
    data_list  = tqdm(DATA)
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        future_to_url = {executor.submit(get_lower, w): w for w in DATA}
        for future in concurrent.futures.as_completed(future_to_url):
            url = future_to_url[future]
            try:
                data = future.result()
            except Exception as exc:
                print('%r generated an exception: %s' % (url, exc))
            else:
                print('%r page is %d bytes' % (url, len(data)))


  # -*- coding: utf-8 -*-

from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _
from vds.models import Host
import libvirt_func

def snapshot(request, host_id):
    """

    Snapshot block

    """

    if not request.user.is_authenticated():
        return HttpResponseRedirect('/login')

    host = Host.objects.get(id=host_id)
    conn = libvirt_func.libvirt_conn(host)

    if type(conn) == dict:
        return HttpResponseRedirect('/overview/%s/' % host_id)
    else:
        all_vm = libvirt_func.vds_get_node(conn)
        all_vm_snap = libvirt_func.snapshots_get_node(conn)

        conn.close()

    if all_vm_snap:
        return HttpResponseRedirect('/snapshot/%s/%s/' % (host_id, all_vm_snap.keys()[0]))

    return render_to_response('snapshot.html', locals(), context_instance=RequestContext(request))

def dom_snapshot(request, host_id, vname):
    """

    Snapshot block

    """

    if not request.user.is_authenticated():
        return HttpResponseRedirect('/login')

    snapshot_page = True
    host = Host.objects.get(id=host_id)
    conn = libvirt_func.libvirt_conn(host)

    if type(conn) == dict:
        return HttpResponseRedirect('/overview/%s/' % host_id)
    else:
        dom = conn.lookupByName(vname)
        all_vm = libvirt_func.vds_get_node(conn)
        all_vm_snap = libvirt_func.snapshots_get_node(conn)
        vm_snapshot = libvirt_func.snapshots_get_vds(dom)

        if request.method == 'POST':
            if 'delete' in request.POST:
                snap_name = request.POST.get('name', '')
                libvirt_func.snapshot_delete(dom, snap_name)
                return HttpResponseRedirect('/snapshot/%s/%s/' % (host_id, vname))
            if 'revert' in request.POST:
                snap_name = request.POST.get('name', '')
                libvirt_func.snapshot_revert(dom, snap_name)
                message = _("Successful revert snapshot: ")
                message = message + snap_name

    return render_to_response('snapshot.html', locals(), context_instance=RequestContext(request))


# Copyright (c) 2015 App Annie Inc. All rights reserved.
from nose.plugins.attrib import attr

from tests.qa.base import BaseSeleniumTestCase
from tests.qa.cases.intelligence.user_admin.email_delegation.email_delegation_mixin import UserRequestAccessMixin
from tests.qa.constants.cmt_constants import JOB_FUNC
from tests.qa.constants.constants import POTENTIAL_SUPER_ADMIN, CSM_EMAIL, POTENTIAL_DOMAIN, INTRoles, \
    PASSWORD, DEFAULT_DOMAIN, SMOKE
from tests.qa.constants.paywall_constants import PotentialIntUser
from tests.qa.mixins.email_checker_mixin import EmailCheckerMixin
from tests.qa.mixins.user_account_mixin import UserAccountMixin
from tests.qa.pages.intelligence.int_user_management import INTUserMgmt
from tests.qa.pages.intelligence.top_chart_page import INTTopPage
from tests.qa.pages.user_admin.login_page import LoginPage
from tests.qa.services.intelligence.cmt_service import INTAccessRequestService
from tests.qa.utils import basic_utils, email_utility

class RequestAccessManage(BaseSeleniumTestCase, UserAccountMixin, EmailCheckerMixin, UserRequestAccessMixin):

    def setUp(self):
        BaseSeleniumTestCase.setUp(self)
        self.super_admin = POTENTIAL_SUPER_ADMIN
        self.reset_super_admin_email_delegation_btn()
        self.user_mgmt = INTUserMgmt(self.selenium)
        self.admin_role = INTRoles.admin
        self.generated_admin_emails = []

    def reset_super_admin_email_delegation_btn(self):
        LoginPage(self.selenium).login(self.super_admin, PASSWORD)
        user_mgmt = INTUserMgmt(self.selenium).goto()
        user_mgmt.switch_on_receive_mail_btn(user_mgmt.get_row_number_by_email(self.super_admin))

    def tearDown(self):
        self.reset_super_admin_email_delegation_btn()
        for email in self.generated_admin_emails:
            INTUserMgmt(self.selenium).remove_intelligence_user(email)
        BaseSeleniumTestCase.tearDown(self)

    def _invite_admin(self):
        result = {'admin_mail': basic_utils.get_random_email(POTENTIAL_DOMAIN)}
        result['invite_url'] = self.user_mgmt.add_intelligence_user(result['admin_mail'], self.admin_role)
        result['user_info'] = self.user_mgmt.get_invited_user_info_by_email(result['admin_mail'])
        self.generated_admin_emails.append(result['admin_mail'])
        return result

    @attr(SMOKE)
    def c37323_super_admin_manage_user_request_test(self):
        user_email = self._request_intelligence_access()
        email_data_dict = email_utility.get_email_data_str_for_email_list([CSM_EMAIL, self.super_admin])
        self.check_mail_dicts(
            email_data_dict, CSM_EMAIL, [self.super_admin], PotentialIntUser.REQUEST_MAIL_BODY)

    def c37324_super_admin_and_admin_manage_user_request_test(self):
        admin_info = self._invite_admin()
        self.assertEqual(admin_info['user_info'][2], INTRoles.get_description(self.admin_role))
        self.register_invited_intelligence_user(admin_info['admin_mail'], admin_info['invite_url'])
        LoginPage(self.selenium).login(self.super_admin, PASSWORD)
        self.user_mgmt.goto()
        row_number = self.user_mgmt.get_row_number_by_email(admin_info['admin_mail'])
        self.user_mgmt.switch_on_receive_mail_btn(row_number)
        user_email = self._request_intelligence_access()
        email_data_dict = email_utility.get_email_data_str_for_email_list(
            [CSM_EMAIL, self.super_admin, admin_info['admin_mail']])
        self.check_mail_dicts(
            email_data_dict, CSM_EMAIL, [self.super_admin, admin_info['admin_mail']],
            PotentialIntUser.REQUEST_MAIL_BODY)

    def c37325_two_admin_manage_user_request_test(self):
        admin_info1 = self._invite_admin()
        self.assertEqual(admin_info1['user_info'][2], INTRoles.get_description(self.admin_role))
        admin_info2 = self._invite_admin()
        self.assertEqual(admin_info2['user_info'][2], INTRoles.get_description(self.admin_role))
        self.register_invited_intelligence_user(admin_info1['admin_mail'], admin_info1['invite_url'])
        self.register_invited_intelligence_user(admin_info2['admin_mail'], admin_info2['invite_url'])
        LoginPage(self.selenium).login(self.super_admin, PASSWORD)
        self.user_mgmt.goto()
        self.user_mgmt.switch_off_receive_mail_btn(self.user_mgmt.get_row_number_by_email(self.super_admin))
        self.user_mgmt.switch_on_receive_mail_btn(self.user_mgmt.get_row_number_by_email(admin_info1['admin_mail']))
        self.user_mgmt.switch_on_receive_mail_btn(self.user_mgmt.get_row_number_by_email(admin_info2['admin_mail']))
        user_email = self._request_intelligence_access()
        email_data_dict = email_utility.get_email_data_str_for_email_list(
            [CSM_EMAIL, admin_info1['admin_mail'], admin_info2['admin_mail']])
        self.check_mail_dicts(
            email_data_dict, CSM_EMAIL, [admin_info1['admin_mail'], admin_info2['admin_mail']],
            PotentialIntUser.REQUEST_MAIL_BODY)

    def c37326_admin_manage_user_request_test(self):
        admin_info = self._invite_admin()
        self.assertEqual(admin_info['user_info'][2], INTRoles.get_description(self.admin_role))
        self.user_mgmt.switch_off_receive_mail_btn(self.user_mgmt.get_row_number_by_email(self.super_admin))
        self.register_invited_intelligence_user(admin_info['admin_mail'], admin_info['invite_url'])
        self.user_mgmt.goto()
        self.user_mgmt.switch_on_receive_mail_btn(self.user_mgmt.get_row_number_by_email(admin_info['admin_mail']))
        user_email = self._request_intelligence_access()
        email_data_dict = email_utility.get_email_data_str_for_email_list([CSM_EMAIL, admin_info['admin_mail']])
        self.check_mail_dicts(
            email_data_dict, CSM_EMAIL, [admin_info['admin_mail']], PotentialIntUser.REQUEST_MAIL_BODY)


from fastText import load_model
import torch
import numpy as np
import sentencepiece as spm

sp = spm.SentencePieceProcessor()
dirname = "/home/pragma/development/deeplearning/"

def main():
	words, embedding = createEmbedding()
	mytext = "Cengiz Han’ın büyük amacası Kutua, bu hakarete Çin üzerine ve Tatarlara bir dizi saldırı düzenleyerek cevap verdi ve bu akınlar sonunda “Moğol Herkülü” unvanını kazandı."
	# mytext = input("Text 1: ")
	myothertext = "Makarna mutfaklarımızda çoğunlukla olan, severek tüketilen bir lezzettir. Haşlanmış ve soslu makarna tariflerine artık sıkılanlar için kolay ve en lezzetli şekliyle beşamelli fırında makarna tarifimi sizlerle paylaşıyorum."
	# myothertext = input("Text 2: ")
	print("Text 1: " + mytext)
	print("Text 1: " + myothertext)
	print("Cos sim (mean): " + str(findSim(mytext, myothertext, words, embedding, "mean")))
	print("Cos sim (sum): " + str(findSim(mytext, myothertext, words, embedding, "sum")))

def findSim(text1, text2, words, embedding, method):
	text1v = findSentenceVec(text1, words, embedding, method)
	text2v = findSentenceVec(text2, words, embedding, method)
	return torch.nn.functional.cosine_similarity(text1v, text2v)

def findSentenceVec(text, words, embedding, method):
	sp.Load(dirname + "sentencepiecepython/sentences.model")
	text = text.lower()
	text = sp.EncodeAsPieces(text.encode('utf-8'))
	ids = [words[word] for word in text]
	vs = embedding(torch.LongTensor(ids))
	if method == "sum":
		return torch.sum(vs, dim = 0, keepdim = True)
	elif method == "mean":
		return torch.mean(vs, dim = 0, keepdim = True)
	return None

def createEmbedding():
	f = load_model(dirname + "fasttextresult/result-sp-10e.bin")
	words = f.get_words()
	print(str(len(words)) + " " + str(f.get_dimension()))

	mywords = {}
	mylist = []

	i = 0
	for w in words:
		mywords[w] = i
		mylist.append(f.get_word_vector(w))
		i = i + 1
	matrix = np.array(mylist)
	tensor = torch.from_numpy(matrix)
	embedding = torch.nn.Embedding.from_pretrained(tensor, freeze = True)
	return mywords, embedding

if __name__ == "__main__":
	main()


# Project Crime - Dhivya Sivaramakrishnan, Mangesh Bhangare

from pyspark import SparkConf, SparkContext
from pyspark.sql import SQLContext, DataFrame, Row
from pyspark.mllib.clustering import KMeans, KMeansModel
from numpy import array
from math import sqrt

import sys

conf = SparkConf().setAppName('K-Means test')
sc = SparkContext(conf=conf)
assert sc.version >= '1.5.1'
sqlContext = SQLContext(sc)

# Read the input parquet
input_crime = sys.argv[1]

# Read the parquet data and convert to RDD
parquet_crime = sqlContext.read.parquet(input_crime)
parquet_crime.registerTempTable("crime_table")
crime_table = sqlContext.sql("SELECT * FROM crime_table")
crime_rdd = crime_table.map(lambda line: str(line.Year) + "," + str(line.Latitude) + ","
                                       + str(line.Longitude) + "," + str(line.Crime_Frequency))

# K-means does multiple runs to find the optimal cluster center, so cache the input to K-means
cluster_input = crime_rdd.map(lambda line: array([float(x) for x in line.split(',')])).cache()

# Perform K-means clustering
clusters = KMeans.train(cluster_input, 20, maxIterations=5,
        runs=5, initializationMode="random")

# Compute root mean squared error and change cluster centers
def squared_error(point):
    center = clusters.centers[clusters.predict(point)]
    return sqrt(sum([x**2 for x in (point - center)]))

error = cluster_input.map(lambda point: squared_error(point)).reduce(lambda x, y: x + y)
print("Squared error for a cluster = " + str(error))

clusters.save(sc, "myModel_crime/crime")
sameModel = KMeansModel.load(sc, "myModel_crime/crime")


'''
Description: Runs program to simulate Cart Pole dynamics with
controller designed by non-algorithmic pole placement

Input: none

Output: Animation of Simulated System

Uses: GallowayMath7203MiniProject02Functions.py,
Python Version 3.7.3, Matplotlib Version 3.0.3, Numpy Version 1.16.2,
Control Version 0.8.3

Author: Josh Galloway
Version: 1.0
Date: 11 April 2020
'''
import numpy as np
from os import _exit
from control import ctrb,place
from matplotlib.pyplot import show,title
from GallowayMath7203MiniProject02Functions import simulateControl,buildAnimation

if __name__ == "__main__":

    # Initial Condition
    X0 = [-0.75,0,0.1,0]

    # System Parameters
    M,m,L,g,F = 1,0.1,0.5,-9.81,0

    print('\n=========================================================')
    print('This Program Runs a Simulation for a Cart Pole System\n'+
           'with controller designed by non-algorithmic pole placement')
    print('x = {:0.2f} initial position of the cart\n'.format(X0[0]) +
          'dx/dt = {:0.2f} intiial velocity of cart\n'.format(X0[1]) +
          'theta = {:0.2f} initial angle of pole with respect to vertical\n'.format(X0[2]) +
          'd theta/dt = {:0.2f} initial angular velocity of pole\n'.format(X0[3]) +
          'g = {:0.2f} m/s^2 acceleration due to gravity\n'.format(g) +
          'M = {:0.2f} kg mass of the cart\n'.format(M) +
          'm = {:0.2f} kg mass of the pole\n'.format(m) +
          'L = {:0.2f} meters length of the pole\n'.format(L) +
          'F = {:0.2f} N force applied to the cart'.format(F))
    print('=========================================================')

    '''Build '''
    A = np.matrix([[0,1,0,0],[0,0,m*g/M,0],[0,0,0,1],[0,0,-g*(M+m)/L/M,0]])
    B = np.matrix([[0],[1/M],[0],[-1/L/M]])

    '''Get Controllabiltiy Matrix Using Library'''
    CM = ctrb(A,B)

    # calculation of controllability matrix by hand
    CM2 = np.hstack((B,A*B,A*A*B,A*A*A*B))

    print('\nControllabiltiy Matrix from Library\n',CM)

    print('\nControllabiltiy Matrix by hand\n',CM2)

    '''Controlability Requires Controlability matrix be of full rank'''
    print('\nControllability Matrix Rank:\n',np.linalg.matrix_rank(CM))

    '''Eigenvalues of A, positive is unstable, has one unstable eigen value and one stable'''
    print('\nA matrix eigenvalues:\n',np.linalg.eigvals(A))

    '''Design a control law u = -Kx, by pole placement'''
    k1 = [-0.6,-0.8,-1,-1.2]
    print('\nPlace Poles at:\n',k1)

    K = place(A,B,k1)
    print('\nResulting K matrix:\n',K)

    '''Designed system has response X dot = (A - BK)x'''
    print('\nCheck Designed system has response eig(A - BK):\n',np.linalg.eigvals(A-B*K))

    # ## Simulate Non-algorithmic Pole Placement Controller

    '''now simulate with F = -K*(X - X_desired)'''
    '''Simulate'''
    t = np.linspace(0,15,15*24)
    x_pend,y_pend,x_cart,X = simulateControl(X0,t,K)

    amin = buildAnimation(t,x_pend,y_pend,x_cart)
    title('Cart Pole Simulation,\n Non-Algorithmic Controller Design',
          fontweight='bold')

    print()
    print('Close Simulation Display Window or Terminal to End...')

    show()

    _exit(0)

MATH 7203 Numerical Analysis 1/Project 2/CommandLineScript/RunMe_SimulationNonAlgorithmicControl.py