import graphviz
from collections import deque
from Regex import RegexToPostfix

class State:
    def __init__(self, label=None, arrow=None):
        self.label = label
        self.arrow = arrow if arrow else []

class NFA:
    def __init__(self, start, accept):
        self.start = start
        self.accept = accept

class PostfixToNFA:
    def __init__(self, postfix):
        self.postfix = postfix
        self.states_count = 0

    def new_state(self, label=None, arrow=None):
        self.states_count += 1
        return State("q{}".format(self.states_count), arrow)

    def thompsons_algorithm(self):
        stack = deque()

        for char in self.postfix:
            if char.isalnum() or char == 'ε' or char == '_':
                s1 = self.new_state()
                s2 = self.new_state()
                s1.arrow.append((char, s2))
                nfa = NFA(s1, s2)
                stack.append(nfa)
            elif char == '|':
                nfa2 = stack.pop()
                nfa1 = stack.pop()
                s1 = self.new_state(
                    arrow=[('ε', nfa1.start), ('ε', nfa2.start)])
                s2 = self.new_state()
                nfa1.accept.arrow.append(('ε', s2))
                nfa2.accept.arrow.append(('ε', s2))
                nfa = NFA(s1, s2)
                stack.append(nfa)
            elif char == '$':
                nfa2 = stack.pop()
                nfa1 = stack.pop()
                nfa1.accept.arrow.append(('ε', nfa2.start))
                nfa = NFA(nfa1.start, nfa2.accept)
                stack.append(nfa)
            elif char == '*':
                nfa1 = stack.pop()
                s1 = self.new_state(arrow=[('ε', nfa1.start)])
                s2 = self.new_state()
                nfa1.accept.arrow.append(('ε', s2))
                s1.arrow.append(('ε', s2))
                nfa = NFA(s1, nfa1.accept)
                stack.append(nfa)
            elif char == '?':
                nfa1 = stack.pop()
                s1 = self.new_state(arrow=[('ε', nfa1.start)])
                s2 = self.new_state()
                nfa1.accept.arrow.append(('ε', s2))
                s1.arrow.append(('ε', s2))
                nfa = NFA(s1, s2)
                stack.append(nfa)

        return stack.pop()

    def draw_nfa(self, nfa):
        graph = graphviz.Digraph(format='png', engine='dot')
        graph.graph_attr['rankdir'] = 'LR'
        graph.graph_attr['splines'] = 'polyline'

        visited = set()

        def visit(state):
            if state in visited:
                return
            visited.add(state)

            for label, next_state in state.arrow:
                graph.edge(state.label, next_state.label, label=label)
                visit(next_state)

        visit(nfa.start)
        graph.node(nfa.accept.label, shape='doublecircle')
        graph.render('./images/nfa', view=True)

    def print_states(self, nfa):
        print("Initial State:", nfa.start.label)
        print("Acceptance State:", nfa.accept.label)

        def visit(state, visited=None):
            if visited is None:
                visited = set()
            if state in visited:
                return
            visited.add(state)

            for label, next_state in state.arrow:
                visit(next_state, visited)

            return visited

        all_states = visit(nfa.start)
        non_accept_states = all_states - {nfa.start, nfa.accept}
        print("Non-Acceptance States:",
              ', '.join(s.label for s in non_accept_states))

    def get_all_states(self, nfa):
        visited = set()

        def visit(state):
            if state in visited:
                return
            visited.add(state)

            for label, next_state in state.arrow:
                visit(next_state)

            return visited

        return visit(nfa.start)

    def print_transition_table(self, nfa):
        all_states = self.get_all_states(nfa)
        symbols = {
            transition[0] for state in all_states for transition in state.arrow if transition[0] != 'ε'}

        print("NFA Transition Table:")
        print("State", end="\t")
        for symbol in sorted(symbols):
            print(symbol, end="\t")
        print("ε")

        for state in sorted(all_states, key=lambda s: int(s.label[1:])):
            print(state.label, end="\t")
            for symbol in sorted(symbols):
                next_states = [
                    arrow[1].label for arrow in state.arrow if arrow[0] == symbol]
                print(",".join(next_states) if next_states else "-", end="\t")
            epsilon_transitions = [
                arrow[1].label for arrow in state.arrow if arrow[0] == 'ε']
            print(",".join(epsilon_transitions)
                  if epsilon_transitions else "-")

    def nfa_to_dict(self, nfa):
        all_states = self.get_all_states(nfa)
        nfa_dict = {}

        for state in all_states:
            transitions = {}
            for symbol, next_state in state.arrow:
                if symbol not in transitions:
                    transitions[symbol] = []
                transitions[symbol].append(next_state.label)
            nfa_dict[state.label] = transitions

        return nfa_dict

    def get_start_state(self, nfa):
        return nfa.start.label

    def get_final_state(self, nfa):
        return nfa.accept.label

    def get_accept_states(self, nfa):
        return [nfa.accept.label]

if __name__ == "__main__":
    regex = "(a*|b*)$c"
    cadena = 'aabc'
    r2p = RegexToPostfix(regex)
    postfix = r2p.to_postfix()
    print("Postfix expression:", postfix)
    p2n = PostfixToNFA(postfix)
    nfa = p2n.thompsons_algorithm()
    p2n.print_states(nfa)
    p2n.print_transition_table(nfa)
    p2n.draw_nfa(nfa)
    print("NFA Dictionary:")
    print(p2n.nfa_to_dict(nfa))
    print("Start State:", p2n.get_start_state(nfa))
    print("Final State:", p2n.get_final_state(nfa))
    print("Accept States:", p2n.get_accept_states(nfa))


"""
Views.
By quantumg.
"""

from django.http import HttpRequest
from django.shortcuts import render

from LMACGalleryWebProject.core.generalSiteTools import mainMenuHelper, pageTitleHelper

def lilCheckerMainView(request: HttpRequest):
    """
    Lil checker main view.

    :param request: (HttpRequest) Request.

    :return: No return.
    """
    return render(request, 'lilCheckerMain.html', {
        'overall': {
            'mainMenu': mainMenuHelper('lil-checker'),
            'errorMessage': '',
            'title': pageTitleHelper('LIL Checker')
        }
    })


import os
from datetime import datetime
from time import sleep
from typing import Any, Iterable, List, Optional, Type

import click

from tutor import config as tutor_config
from tutor import env as tutor_env
from tutor import exceptions, fmt, hooks
from tutor import interactive as interactive_config
from tutor import serialize, utils
from tutor.commands import jobs
from tutor.commands.config import save as config_save_command
from tutor.commands.context import BaseTaskContext
from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES
from tutor.commands.upgrade.k8s import upgrade_from
from tutor.tasks import BaseTaskRunner
from tutor.types import Config, get_typed

class K8sClients:
    _instance = None

    def __init__(self) -> None:
        # Loading the kubernetes module here to avoid import overhead
        # pylint: disable=import-outside-toplevel
        from kubernetes import client, config

        if os.path.exists(
            os.path.expanduser(config.kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
        ):
            # found the kubeconfig file, let's load it!
            config.load_kube_config()
        elif (
            config.incluster_config.SERVICE_HOST_ENV_NAME in os.environ
            and config.incluster_config.SERVICE_PORT_ENV_NAME in os.environ
        ):
            # We are running inside a cluster, let's load the in-cluster configuration.
            config.load_incluster_config()
        else:
            raise exceptions.TutorError(
                f"there is no Kubernetes configuration file located in {config.kube_config.KUBE_CONFIG_DEFAULT_LOCATION}, and the service environment variables {config.incluster_config.SERVICE_HOST_ENV_NAME} and {config.incluster_config.SERVICE_PORT_ENV_NAME} have not been defined. To connect to a cluster, please configure your host appropriately."
            )

        self._batch_api = None
        self._core_api = None
        self._client = client

    @classmethod
    def instance(cls: Type["K8sClients"]) -> "K8sClients":
        if cls._instance is None:
            cls._instance = cls()
        return cls._instance

    @property
    def batch_api(self):  # type: ignore
        if self._batch_api is None:
            self._batch_api = self._client.BatchV1Api()
        return self._batch_api

    @property
    def core_api(self):  # type: ignore
        if self._core_api is None:
            self._core_api = self._client.CoreV1Api()
        return self._core_api

class K8sTaskRunner(BaseTaskRunner):
    """
    Run tasks (bash commands) in Kubernetes-managed services.

    Note: a single Tutor "task" correspond to a Kubernetes "job":
    https://kubernetes.io/docs/concepts/workloads/controllers/job/
    A Tutor "job" is composed of multiple Tutor tasks run in different services.

    In Kubernetes, each task that is expected to run in a "myservice" container will
    trigger the "myservice-job" Kubernetes job. This job definition must be present in
    the "k8s/jobs.yml" template.
    """

    def run_task(self, service: str, command: str) -> int:
        canonical_job_name = f"{service}-job"
        all_jobs = list(self._load_jobs())
        job = self._find_job(canonical_job_name, all_jobs)
        # Create a unique job name to make it deduplicate jobs and make it easier to
        # find later. Logs of older jobs will remain available for some time.
        job_name = canonical_job_name + "-" + datetime.now().strftime("%Y%m%d%H%M%S")

        # Wait until all other jobs are completed
        while True:
            active_jobs = self.active_job_names()
            if not active_jobs:
                break
            fmt.echo_info(
                f"Waiting for active jobs to terminate: {' '.join(active_jobs)}"
            )
            sleep(5)

        # Configure job
        job["metadata"]["name"] = job_name
        job["metadata"].setdefault("labels", {})
        job["metadata"]["labels"]["app.kubernetes.io/name"] = job_name
        # Define k8s entrypoint/args
        shell_command = ["sh", "-e", "-c"]
        if job["spec"]["template"]["spec"]["containers"][0].get("command") == []:
            # In some cases, we need to bypass the container entrypoint.
            # Unfortunately, AFAIK, there is no way to do so in K8s manifests. So we mark
            # some jobs with "command: []". For these jobs, the entrypoint becomes "sh -e -c".
            # We do not do this for every job, because some (most) entrypoints are actually useful.
            job["spec"]["template"]["spec"]["containers"][0]["command"] = shell_command
            container_args = [command]
        else:
            container_args = shell_command + [command]
        job["spec"]["template"]["spec"]["containers"][0]["args"] = container_args
        job["spec"]["backoffLimit"] = 1
        job["spec"]["ttlSecondsAfterFinished"] = 3600

        with open(
            tutor_env.pathjoin(self.root, "k8s", "jobs.yml"), "w", encoding="utf-8"
        ) as job_file:
            serialize.dump_all(all_jobs, job_file)

        # We cannot use the k8s API to create the job: configMap and volume names need
        # to be found with the right suffixes.
        kubectl_apply(
            self.root,
            "--selector",
            f"app.kubernetes.io/name={job_name}",
        )

        message = (
            "Job {job_name} is running. To view the logs from this job, run:\n\n"
            """    kubectl logs --namespace={namespace} --follow $(kubectl get --namespace={namespace} pods """
            """--selector=job-name={job_name} -o=jsonpath="{{.items[0].metadata.name}}")\n\n"""
            "Waiting for job completion..."
        ).format(job_name=job_name, namespace=k8s_namespace(self.config))
        fmt.echo_info(message)

        # Wait for completion
        field_selector = f"metadata.name={job_name}"
        while True:
            namespaced_jobs = K8sClients.instance().batch_api.list_namespaced_job(
                k8s_namespace(self.config), field_selector=field_selector
            )
            if not namespaced_jobs.items:
                continue
            job = namespaced_jobs.items[0]
            if not job.status.active:
                if job.status.succeeded:
                    fmt.echo_info(f"Job {job_name} successful.")
                    break
                if job.status.failed:
                    raise exceptions.TutorError(
                        f"Job {job_name} failed. View the job logs to debug this issue."
                    )
            sleep(5)
        return 0

    def load_job(self, name: str) -> Any:
        """
        Find a given job definition in the rendered k8s/jobs.yml template.
        """
        return self._find_job(name, self._load_jobs())

    def _find_job(self, name: str, all_jobs: Iterable[Any]) -> Any:
        """
        Find the matching job definition in the in the list of jobs provided.

        Returns the found job's manifest.
        """
        for job in all_jobs:
            job_name = job["metadata"]["name"]
            if not isinstance(job_name, str):
                raise exceptions.TutorError(
                    f"Invalid job name: '{job_name}'. Expected str."
                )
            if job_name == name:
                return job
        raise exceptions.TutorError(f"Could not find job '{name}'")

    def _load_jobs(self) -> Iterable[Any]:
        manifests = self.render("k8s", "jobs.yml")
        for manifest in serialize.load_all(manifests):
            if manifest["kind"] == "Job":
                yield manifest

    def active_job_names(self) -> List[str]:
        """
        Return a list of active job names
        Docs:
        https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#list-job-v1-batch

        This is necessary to make sure that we don't run the same job multiple times at
        the same time.
        """
        api = K8sClients.instance().batch_api
        return [
            job.metadata.name
            for job in api.list_namespaced_job(k8s_namespace(self.config)).items
            if job.status.active
        ]

class K8sContext(BaseTaskContext):
    def job_runner(self, config: Config) -> K8sTaskRunner:
        return K8sTaskRunner(self.root, config)

@click.group(help="Run Open edX on Kubernetes")
@click.pass_context
def k8s(context: click.Context) -> None:
    context.obj = K8sContext(context.obj.root)

@click.command(help="Configure and run Open edX from scratch")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.pass_context
def launch(context: click.Context, non_interactive: bool) -> None:
    run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root)
    if run_upgrade_from_release is not None:
        click.echo(fmt.title("Upgrading from an older release"))
        context.invoke(
            upgrade,
            from_release=tutor_env.get_env_release(context.obj.root),
        )

    click.echo(fmt.title("Interactive platform configuration"))
    config = tutor_config.load_minimal(context.obj.root)
    if not non_interactive:
        interactive_config.ask_questions(config, run_for_prod=True)
    tutor_config.save_config_file(context.obj.root, config)
    config = tutor_config.load_full(context.obj.root)
    tutor_env.save(context.obj.root, config)

    if run_upgrade_from_release and not non_interactive:
        question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}.

If you run custom Docker images, you must rebuild and push them to your private repository now by running the following
commands in a different shell:

    tutor images build all # add your custom images here
    tutor images push all

Press enter when you are ready to continue"""
        click.confirm(
            fmt.question(question), default=True, abort=True, prompt_suffix=" "
        )

    click.echo(fmt.title("Starting the platform"))
    context.invoke(start)

    click.echo(fmt.title("Database creation and migrations"))
    context.invoke(init, limit=None)

    config = tutor_config.load(context.obj.root)
    fmt.echo_info(
        """Your Open edX platform is ready and can be accessed at the following urls:

    {http}://{lms_host}
    {http}://{cms_host}
    """.format(
            http="https" if config["ENABLE_HTTPS"] else "http",
            lms_host=config["LMS_HOST"],
            cms_host=config["CMS_HOST"],
        )
    )

@click.command(
    short_help="Run all configured Open edX resources",
    help=(
        "Run all configured Open edX resources. You may limit this command to "
        "some resources by passing name arguments."
    ),
)
@click.argument("names", metavar="name", nargs=-1)
@click.pass_obj
def start(context: K8sContext, names: List[str]) -> None:
    config = tutor_config.load(context.root)
    # Create namespace, if necessary
    # Note that this step should not be run for some users, in particular those
    # who do not have permission to edit the namespace.
    try:
        utils.kubectl("get", "namespaces", k8s_namespace(config))
        fmt.echo_info("Namespace already exists: skipping creation.")
    except exceptions.TutorError:
        fmt.echo_info("Namespace does not exist: now creating it...")
        kubectl_apply(
            context.root,
            "--wait",
            "--selector",
            "app.kubernetes.io/component=namespace",
        )

    names = names or ["all"]
    for name in names:
        if name == "all":
            # Create volumes
            kubectl_apply(
                context.root,
                "--wait",
                "--selector",
                "app.kubernetes.io/component=volume",
            )
            # Create everything else except jobs
            kubectl_apply(
                context.root,
                "--selector",
                "app.kubernetes.io/component notin (job,volume,namespace)",
            )
        else:
            kubectl_apply(
                context.root,
                "--selector",
                f"app.kubernetes.io/name={name}",
            )

@click.command(
    short_help="Stop a running platform",
    help=(
        "Stop a running platform by deleting all resources, except for volumes. "
        "You may limit this command to some resources by passing name arguments."
    ),
)
@click.argument("names", metavar="name", nargs=-1)
@click.pass_obj
def stop(context: K8sContext, names: List[str]) -> None:
    config = tutor_config.load(context.root)
    names = names or ["all"]
    for name in names:
        if name == "all":
            delete_resources(config)
        else:
            delete_resources(config, name=name)

def delete_resources(
    config: Config, resources: Optional[List[str]] = None, name: Optional[str] = None
) -> None:
    """
    Delete resources by type and name.

    The load balancer is never deleted.
    """
    resources = resources or ["deployments", "services", "configmaps", "jobs"]
    not_lb_selector = "app.kubernetes.io/component!=loadbalancer"
    name_selector = [f"app.kubernetes.io/name={name}"] if name else []
    utils.kubectl(
        "delete",
        *resource_selector(config, not_lb_selector, *name_selector),
        ",".join(resources),
    )

@click.command(help="Reboot an existing platform")
@click.pass_context
def reboot(context: click.Context) -> None:
    context.invoke(stop)
    context.invoke(start)

@click.command(help="Completely delete an existing platform")
@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation")
@click.pass_obj
def delete(context: K8sContext, yes: bool) -> None:
    if not yes:
        click.confirm(
            "Are you sure you want to delete the platform? All data will be removed.",
            abort=True,
        )
    utils.kubectl(
        "delete",
        "-k",
        tutor_env.pathjoin(context.root),
        "--ignore-not-found=true",
        "--wait",
    )

@jobs.do_group
@click.pass_obj
def do(context: K8sContext) -> None:
    """
    Run a custom job in the right container(s).

    We make sure that some essential containers (databases, proxy) are up before we
    launch the jobs.
    """

    @hooks.Actions.DO_JOB.add()
    def _start_base_deployments(_job_name: str, *_args: Any, **_kwargs: Any) -> None:
        """
        We add this logic to an action callback because we do not want to trigger it
        whenever we run `tutor k8s do <job> --help`.
        """
        config = tutor_config.load(context.root)
        wait_for_deployment_ready(config, "caddy")
        for name in ["elasticsearch", "mysql", "mongodb"]:
            if tutor_config.is_service_activated(config, name):
                wait_for_deployment_ready(config, name)

@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_context
def init(context: click.Context, limit: Optional[str]) -> None:
    context.invoke(do.commands["init"], limit=limit)

@click.command(help="Scale the number of replicas of a given deployment")
@click.argument("deployment")
@click.argument("replicas", type=int)
@click.pass_obj
def scale(context: K8sContext, deployment: str, replicas: int) -> None:
    config = tutor_config.load(context.root)
    utils.kubectl(
        "scale",
        # Note that we don't use the full resource selector because selectors
        # are not compatible with the deployment/<name> argument.
        *resource_namespace_selector(
            config,
        ),
        f"--replicas={replicas}",
        f"deployment/{deployment}",
    )

@click.command(
    name="exec",
    help="Execute a command in a pod of the given application",
    context_settings={"ignore_unknown_options": True},
)
@click.argument("service")
@click.argument("args", nargs=-1, required=True)
@click.pass_obj
def exec_command(context: K8sContext, service: str, args: List[str]) -> None:
    config = tutor_config.load(context.root)
    kubectl_exec(config, service, args)

@click.command(help="View output from containers")
@click.option("-c", "--container", help="Print the logs of this specific container")
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service")
@click.pass_obj
def logs(
    context: K8sContext, container: str, follow: bool, tail: bool, service: str
) -> None:
    config = tutor_config.load(context.root)

    command = ["logs"]
    selectors = ["app.kubernetes.io/name=" + service] if service else []
    command += resource_selector(config, *selectors)

    if container:
        command += ["-c", container]
    if follow:
        command += ["--follow"]
    if tail is not None:
        command += ["--tail", str(tail)]

    utils.kubectl(*command)

@click.command(help="Wait for a pod to become ready")
@click.argument("name")
@click.pass_obj
def wait(context: K8sContext, name: str) -> None:
    config = tutor_config.load(context.root)
    wait_for_deployment_ready(config, name)

@click.command(
    short_help="Perform release-specific upgrade tasks",
    help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `launch`.",
)
@click.option(
    "--from",
    "from_release",
    type=click.Choice(OPENEDX_RELEASE_NAMES),
)
@click.pass_context
def upgrade(context: click.Context, from_release: Optional[str]) -> None:
    if from_release is None:
        from_release = tutor_env.get_env_release(context.obj.root)
    if from_release is None:
        fmt.echo_info("Your environment is already up-to-date")
    else:
        fmt.echo_alert(
            "This command only performs a partial upgrade of your Open edX platform. "
            "To perform a full upgrade, you should run `tutor k8s launch`."
        )
        upgrade_from(context, from_release)
    # We update the environment to update the version
    context.invoke(config_save_command)

@click.command(
    short_help="Direct interface to `kubectl apply`.",
    help=(
        "Direct interface to `kubnectl-apply`. This is a wrapper around `kubectl apply`. A;; options and"
        " arguments passed to this command will be forwarded as-is to `kubectl apply`."
    ),
    context_settings={"ignore_unknown_options": True},
    name="apply",
)
@click.argument("args", nargs=-1)
@click.pass_obj
def apply_command(context: K8sContext, args: List[str]) -> None:
    kubectl_apply(context.root, *args)

def kubectl_apply(root: str, *args: str) -> None:
    utils.kubectl("apply", "--kustomize", tutor_env.pathjoin(root), *args)

@click.command(help="Print status information for all k8s resources")
@click.pass_obj
def status(context: K8sContext) -> int:
    config = tutor_config.load(context.root)
    return utils.kubectl("get", "all", *resource_namespace_selector(config))

def kubectl_exec(config: Config, service: str, command: List[str]) -> int:
    selector = f"app.kubernetes.io/name={service}"
    pods = K8sClients.instance().core_api.list_namespaced_pod(
        namespace=k8s_namespace(config), label_selector=selector
    )
    if not pods.items:
        raise exceptions.TutorError(
            f"Could not find an active pod for the {service} service"
        )
    pod_name = pods.items[0].metadata.name

    # Run command
    return utils.kubectl(
        "exec",
        "--stdin",
        "--tty",
        "--namespace",
        k8s_namespace(config),
        pod_name,
        "--",
        *command,
    )

def wait_for_deployment_ready(config: Config, service: str) -> None:
    fmt.echo_info(f"Waiting for a {service} deployment to be ready...")
    utils.kubectl(
        "wait",
        *resource_selector(config, f"app.kubernetes.io/name={service}"),
        "--for=condition=Available=True",
        "--timeout=600s",
        "deployment",
    )

def resource_selector(config: Config, *selectors: str) -> List[str]:
    """
    Convenient utility to filter the resources that belong to this project.
    """
    selector = ",".join(
        ["app.kubernetes.io/instance=openedx-" + get_typed(config, "ID", str)]
        + list(selectors)
    )
    return resource_namespace_selector(config) + ["--selector=" + selector]

def resource_namespace_selector(config: Config) -> List[str]:
    """
    Convenient utility to filter the resources that belong to this project namespace.
    """
    return ["--namespace", k8s_namespace(config)]

def k8s_namespace(config: Config) -> str:
    return get_typed(config, "K8S_NAMESPACE", str)

k8s.add_command(launch)
k8s.add_command(start)
k8s.add_command(stop)
k8s.add_command(reboot)
k8s.add_command(delete)
k8s.add_command(init)
k8s.add_command(scale)
k8s.add_command(exec_command)
k8s.add_command(logs)
k8s.add_command(wait)
k8s.add_command(upgrade)
k8s.add_command(apply_command)
k8s.add_command(status)

@hooks.Actions.PLUGINS_LOADED.add()
def _add_k8s_do_commands() -> None:
    jobs.add_job_commands(do)
    k8s.add_command(do)

{'client': 'kubernetes.client', 'config': 'kubernetes.config'}.instance


import ctypes

from cuda import cuda
import numpy as np

from cutlass import DataType
from cutlass.backend.epilogue import EpilogueFunctorBase
import cutlass.backend.evt.backend
from cutlass.backend.frontend import TensorFrontend

class EpilogueFunctorVisitor(EpilogueFunctorBase):
    """
    Apply an epilogue functor described by the epilogue EVT

    :param cc: compute capability
    :param visitor_frontend: user-provide visitor frontend

    """
    def __init__(self, cc: int, visitor, element_compute=DataType.f32) -> None:
        # Type of Emitter based on CC
        self.emit_cls = getattr(cutlass.backend.evt.backend, f"Sm{cc}Emitter")

        # Visitor Types
        self.visitor = visitor
        self.graph = visitor.dag_ir

        # Data types
        self.element_epilogue = element_compute # element compute
        self.element_output = self.graph.get_node_meta('D').underlying_impl.element

        # Epilogue Thread Type
        epilogue_thread_type = self.visitor.epilogue_thread_type
        if cc == 90:
            self.arg_c_type = self.visitor.arg_c_type
            self.arg_d_type = self.visitor.arg_d_type
        output_names = self.visitor.return_names
        reduction_names = self.visitor.reduction_names

        # Epilogue stages specialized for sm80 kernel
        if cc == 80:
            if hasattr(self.visitor, "epilogue_stages"):
                self.epilogue_stages = self.visitor.epilogue_stages
                assert self.epilogue_stages <= 2, "Only supports Stages <=2 in SM80 Epilogue"

        # Epilogue Argument Type
        class _Arguments(ctypes.Structure):
            """
            Concepts:
            class _EpilogueArguments(ctypes.Structure):
                _fields_ = [
                    ("epilogue", _Arguments), <- this class
                    ("ptr_C", ctypes.c_void_p),
                    ("stride_C", StrideBatched_),
                    ("ptr_D", ctypes.c_void_p),
                    ("stride_D", StrideBatched_)
                ]
            """
            _fields_ = [
                ("output_op", epilogue_thread_type)
            ]

            def __init__(self, kwargs: dict) -> None:
                # The user-input kwargs is a dict of (name: tensors)
                # We first convert all of them to device pointers
                ptr_kwargs = {}
                for key in kwargs.keys():
                    is_output = key in output_names and key not in reduction_names
                    ptr_kwargs[key] = self.get_tensor_ptr(key, kwargs, is_output)
                # Initialize the thread arguments
                self.output_op = epilogue_thread_type(ptr_kwargs)

            def get_tensor_ptr(self, tensor_name, kwargs, is_output=False):
                """
                Helper function for extracting device pointer
                """
                # Skip the special tensors
                if cc == 90:
                    if tensor_name in ["C", "D"]:
                        return 0
                if tensor_name not in kwargs.keys():
                    raise ValueError(f"Tensor {tensor_name} is not provided.")
                tensor = kwargs[tensor_name]

                # For float scalar constant, directly return the value
                if isinstance(tensor, float):
                    return tensor

                # The tensor frontend returns a device buffer for np.ndarray
                # and device ptr for other frontends
                buffer_or_ptr = TensorFrontend.argument(tensor, is_output)
                if isinstance(tensor, np.ndarray):
                    # Remember the host tensor for later synchronization
                    setattr(self, f"{tensor_name}_buffer", buffer_or_ptr)
                    setattr(self, f"{tensor_name}_host", tensor)
                    return int(buffer_or_ptr.ptr)
                else:
                    return int(buffer_or_ptr)

            def sync(self):
                """
                Synchronize the results from device to host
                """
                for name in output_names:
                    if hasattr(self, f"{name}_host"):
                        host_tensor = getattr(self, f"{name}_host")
                        tensor_ptr = getattr(self, f"{name}_buffer").ptr
                        (err,) = cuda.cuMemcpyDtoH(
                            host_tensor,
                            tensor_ptr,
                            host_tensor.size * host_tensor.itemsize,
                        )
                        if err != cuda.CUresult.CUDA_SUCCESS:
                            raise RuntimeError("CUDA Error %s" % str(err))

        self.epilogue_type = _Arguments

    def emit(self, operation):
        """
        Emit the C++ code
        """
        emitter = self.emit_cls(operation, self.graph)
        return emitter.emit()

    def get_smem_size(self, tile_description):
        """
        Get the shared memory size in bytes
        """
        return self.visitor.get_smem_size(tile_description)


from flask import Flask, json, jsonify, render_template ,request ,make_response

import geopy.distance as ps
from numpy.lib.function_base import append
import pandas as pd
import gspread
from oauth2client.service_account import ServiceAccountCredentials

scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
creds = ServiceAccountCredentials.from_json_keyfile_name('client_secret.json', scope)
client = gspread.authorize(creds)
sheet = client.open("product").sheet1
data = sheet.get_all_records()
listdata = pd.DataFrame(data)
print(listdata)

app = Flask(__name__)

def create_flex(name , tel , job):
    flex_json = {
        "type": "bubble",
        "header": {
            "type": "box",
            "layout": "vertical",
            "contents": [
            {
                "type": "box",
                "layout": "horizontal",
                "contents": [
                {
                    "type": "image",
                    "url": "https://scdn.line-apps.com/n/channel_devcenter/img/flexsnapshot/clip/clip4.jpg",
                    "size": "full",
                    "aspectMode": "cover",
                    "aspectRatio": "150:196",
                    "gravity": "center",
                    "flex": 1
                },
                {
                    "type": "box",
                    "layout": "vertical",
                    "contents": [
                    {
                        "type": "image",
                        "url": "https://scdn.line-apps.com/n/channel_devcenter/img/flexsnapshot/clip/clip5.jpg",
                        "size": "full",
                        "aspectMode": "cover",
                        "aspectRatio": "150:98",
                        "gravity": "center"
                    },
                    {
                        "type": "image",
                        "url": "https://scdn.line-apps.com/n/channel_devcenter/img/flexsnapshot/clip/clip6.jpg",
                        "size": "full",
                        "aspectMode": "cover",
                        "aspectRatio": "150:98",
                        "gravity": "center"
                    }
                    ],
                    "flex": 1
                },
                {
                    "type": "box",
                    "layout": "horizontal",
                    "contents": [
                    {
                        "type": "text",
                        "text": "NEW",
                        "size": "xs",
                        "color": "#ffffff",
                        "align": "center",
                        "gravity": "center"
                    }
                    ],
                    "backgroundColor": "#EC3D44",
                    "paddingAll": "2px",
                    "paddingStart": "4px",
                    "paddingEnd": "4px",
                    "flex": 0,
                    "position": "absolute",
                    "offsetStart": "18px",
                    "offsetTop": "18px",
                    "cornerRadius": "100px",
                    "width": "48px",
                    "height": "25px"
                }
                ]
            }
            ],
            "paddingAll": "0px"
        },
        "body": {
            "type": "box",
            "layout": "vertical",
            "contents": [
            {
                "type": "box",
                "layout": "vertical",
                "contents": [
                {
                    "type": "box",
                    "layout": "vertical",
                    "contents": [
                    {
                        "type": "text",
                        "contents": [],
                        "size": "xl",
                        "wrap": True,
                        "text": "ชื่อ " + str(name) + " : หน้าที่ " + str(job) ,
                        "color": "#ffffff",
                        "weight": "bold"
                    }
                    ],
                    "spacing": "sm"
                }
                ]
            }
            ],
            "paddingAll": "20px",
            "backgroundColor": "#464F69"
        }
    }
    return flex_json
    #เทสปกติผ่าน api
    #return flex_json

    #สำหรับบอทต้อง return

def formatObject(flex_json):
    return [{
            "type": "flex",
            "altText": "Show Carousel",
            "contents": {
                "type": "carousel",
                "contents": flex_json
            }
        }]

def flex_all_order(name ):
    jsonData = {
        "type": "bubble",
        "hero": {
            "type": "image",
            "url": "https://scdn.line-apps.com/n/channel_devcenter/img/fx/01_1_cafe.png",
            "size": "full",
            "aspectRatio": "20:13",
            "aspectMode": "cover",
            "action": {
            "type": "uri",
            "uri": "http://linecorp.com/"
            }
        },
        "body": {
            "type": "box",
            "layout": "vertical",
            "contents": [
            {
                "type": "text",
                "text": "ชื่อ "+ str(name),
                "weight": "bold",
                "size": "xl"
            },
            {
                "type": "box",
                "layout": "baseline",
                "margin": "md",
                "contents": [
                {
                    "type": "icon",
                    "size": "sm",
                    "url": "https://scdn.line-apps.com/n/channel_devcenter/img/fx/review_gold_star_28.png"
                },
                {
                    "type": "icon",
                    "size": "sm",
                    "url": "https://scdn.line-apps.com/n/channel_devcenter/img/fx/review_gold_star_28.png"
                },
                {
                    "type": "icon",
                    "size": "sm",
                    "url": "https://scdn.line-apps.com/n/channel_devcenter/img/fx/review_gold_star_28.png"
                },
                {
                    "type": "icon",
                    "size": "sm",
                    "url": "https://scdn.line-apps.com/n/channel_devcenter/img/fx/review_gold_star_28.png"
                },
                {
                    "type": "icon",
                    "size": "sm",
                    "url": "https://scdn.line-apps.com/n/channel_devcenter/img/fx/review_gray_star_28.png"
                },
                {
                    "type": "text",
                    "text": "4.0",
                    "size": "sm",
                    "color": "#999999",
                    "margin": "md",
                    "flex": 0
                }
                ]
            },
            {
                "type": "box",
                "layout": "vertical",
                "margin": "lg",
                "spacing": "sm",
                "contents": [
                {
                    "type": "box",
                    "layout": "baseline",
                    "spacing": "sm",
                    "contents": [
                    {
                        "type": "text",
                        "text": "Place",
                        "color": "#aaaaaa",
                        "size": "sm",
                        "flex": 1
                    },
                    {
                        "type": "text",
                        "text": "test",
                        "wrap": True,
                        "color": "#666666",
                        "size": "sm",
                        "flex": 5
                    }
                    ]
                }
                ]
            }
            ]
        },
        "footer": {
            "type": "box",
            "layout": "vertical",
            "spacing": "sm",
            "contents": [
            {
                "type": "button",
                "style": "link",
                "height": "sm",
                "action": {
                "type": "uri",
                "label": "CALL",
                "uri": "https://linecorp.com"
                }
            },
            {
                "type": "button",
                "style": "link",
                "height": "sm",
                "action": {
                "type": "uri",
                "label": "WEBSITE",
                "uri": "https://linecorp.com"
                }
            },
            {
                "type": "spacer",
                "size": "sm"
            }
            ],
            "flex": 0
        }
    }
    return jsonData

@app.route("/")
def home():
    return render_template('index.html')

@app.route("/get_product")
def get_product():
    data = sheet.get_all_records()
    listdata = pd.DataFrame(data)
    print(listdata)

    flex_message = []
    for i in range( len(listdata)):
        item = listdata.iloc[i]
        print(item['ชื่อ'])
        jsonData = flex_all_order(item['ชื่อ'])
        flex_message.append(jsonData)
    print(flex_message)
    response= make_response(
                jsonify({"line_payload": formatObject(flex_message) }),
                200,
    )
    response.headers["Response-Type"] = "object"
    return response

@app.route("/get_profile")
def get_profile():
    name = "test tanup"
    jsonData = create_flex(name)
    response= make_response(
                jsonify({"line_payload": jsonData}),200,
    )
    response.headers["Response-Type"] = "object"
    return response  #? สำหรับ API แบบที่ 2

@app.route("/insert_member")
def insert_member():
    name = request.args.get("name")
    tel = request.args.get("tel")
    job = request.args.get("job")
    add_data = [name, int(tel),str(job)]
    data = sheet.get_all_records()
    listdata = pd.DataFrame(data)
    index = int(len(listdata) + 2)
    sheet.insert_row(add_data , index)
    return "ok"

@app.route("/get_order")
def get_order():
    name = request.args.get("name")
    data = sheet.get_all_records()
    listdata = pd.DataFrame(data)
    data = listdata[listdata['ชื่อ'] == name ]
    if len(data) == 0 :
        return "no data"
    jsonData = create_flex(
        data.iloc[0]["ชื่อ"] ,
        data.iloc[0]["เบอร์โทร"] ,
        data.iloc[0]["หน้าที่"] ,
    )
    response= make_response(
                jsonify({"line_payload": formatObject([jsonData]) }),200,
    )
    response.headers["Response-Type"] = "object"
    return response

    # jsonData = {
    #     "name":data.iloc[0]['ชื่อ'],
    #     "tel":str(data.iloc[0]['เบอร์โทร']),
    #     "order":str(data.iloc[0]['หน้าที่'])
    # }
    # return jsonify(jsonData)

# @app.route("/insert_order")
# def insert_order():
#     name = request.args.get("name")
#     tel = request.args.get("tel")
#     order = request.args.get("order")
#     add_data = [name, str(tel),str(order)]
#     data = sheet.get_all_records()
#     listdata = pd.DataFrame(data)
#     index = int(len(listdata) + 2)
#     sheet.insert_row(add_data , index)
#     return "ok"

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5000, threaded=True)


# LaplaceLine.py: Solve Laplace’s eqn, 3D matplot, close shell to quit

from numpy import *
import matplotlib.pylab as p;
from mpl_toolkits.mplot3d import Axes3D

#p.xkcd()                  # for fun, uncomment for xkcd atyle plots
print("Initializing")
Nmax = 100;               # grid points Nx, Ny
Niter = 70;               # sweeps/iterations to perform
V = zeros((Nmax, Nmax), float)

print( "Working hard, wait for the figure while I count to 60")
for k in range(0, Nmax-1): V[k,0] = 100.0  # line at 100V

for iter in range(Niter):  # iterations over algorithm
    if iter%10 == 0: print(iter)
    for i in range(1, Nmax-2):
        for j in range(1,Nmax-2): V[i,j] = 0.25*(V[i+1,j]+V[i-1,j]+V[i,j+1]+V[i,j-1])
x = range(0, Nmax-1, 2); y = range(0, 50, 2)    # plot every other point
X, Y = p.meshgrid(x,y)

def functz(V):    # Function returns V(x, y)
    z=V[X,Y]
    return z

Z = functz(V)
fig = p.figure()  # Create figure
ax = Axes3D(fig)  # plot axes
ax.plot_wireframe(X, Y, Z, color = 'r')  # red wireframe
ax.set_xlabel('X')                       # label axes
ax.set_ylabel('Y')
ax.set_zlabel('Potential')
print("Control-C to exit")
p.show()                                 # display fig


import logging
import os
from pathlib import Path
import re
import sys
import time
from typing import Type, Any, Iterable, Optional, \
                   Tuple, List, Dict, Callable
import warnings

from bs4 import BeautifulSoup
from gensim.models import Word2Vec, KeyedVectors
from keras.callbacks import EarlyStopping
from keras.layers import Dense, Embedding, Flatten
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D
from keras.models import Sequential
from keras.optimizers import Adam, SGD
import nltk.corpus
import nltk.tokenize
from nltk.corpus import wordnet
from nltk.stem import WordNetLemmatizer
import numpy as np
import pandas as pd
# from scipy.sparse import csc_matrix
import sklearn
from sklearn.cluster import MiniBatchKMeans
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB, BernoulliNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import roc_auc_score

# import cProfile

def _get_sklearn_version() -> Tuple[int, int, int]:
    """
    Returns the version of scikit-learn as a tuple.
    """
    (release, major, minor) = sklearn.__version__.split('.')
    return int(release), int(major), int(minor)

if _get_sklearn_version() >= (0, 19, 0):
    from sklearn.model_selection import StratifiedKFold
    from sklearn.model_selection import StratifiedShuffleSplit
else:  # We have an old version of sklearn...
    from sklearn.cross_validation import StratifiedKFold
    from sklearn.cross_validation import StratifiedShuffleSplit

def _get_current_file_dir() -> Path:
    """Returns the directory of the script."""
    try:
        return Path(os.path.realpath(__file__)).parent
    except(NameError):
        return Path(os.getcwd())

# Project root directory, i.e. the github repo directory.
_PROJECT_ROOT = _get_current_file_dir() / '..'

# Default configuration options.
# WARNING: Please, avoid changing it. Use a local `conf.py` in the project's
# root directory.
_DEFAULT_CONFIG = {
    # Options that specify where to read the data from.
    'in': {
        'labeled':   str(_PROJECT_ROOT / 'data' / 'labeledTrainData.tsv'),
        'unlabeled': str(_PROJECT_ROOT / 'data' / 'unlabeledTrainData.tsv'),
        'test':      str(_PROJECT_ROOT / 'data' / 'testData.tsv'),
        'clean':     str(_PROJECT_ROOT / 'data' / 'cleanReviews.tsv'),
    },
    # Options that specify where to write the results to.
    'out': {
        'result':       str(_PROJECT_ROOT / 'results' / 'Prediction.csv'),
        'wrong_result': str(_PROJECT_ROOT / 'results' / 'FalsePrediction.csv'),
    },
    # High-level algorithm specific options.
    'run': {
        # Type of the run, one of {'optimization', 'submission'}
        'type':             'submission',
        # How many splits to use in the StratifiedKFold
        'number_splits':    3,
        # When preprocessing the reviews, should we remove the stopwords?
        'remove_stopwords': True,
        # Should we cache the preprocessed reviews?
        'cache_clean':      True,
        # After the running the StratifiedKFold on the 90%, should we test the
        # result on the remaining 10?
        'test_10':          False,
        # Random seed used for the 90-10 split.
        'random':           54,
        # How many percent of the data should be left out for testing? I.e.
        # what is "10" in the 90-10 split.
        'alpha':            0.1,
    },
    # Type of the vectorizer, one of {'word2vec', 'bagofwords'}
    'vectorizer': 'word2vec',
    # Type of the classifier to use, one of
    # {'random-forest', 'logistic-regression', 'feed-forward', 'convolutional'}
    'classifier': 'convolutional',
    # Options specific to the bagofwords vectorizer.
    'bagofwords': {},
    # Options specific to the word2vec vectorizer.
    'word2vec': {
        # File name where to save/read the model to/from.
        'model_file': str(_PROJECT_ROOT / 'data'
                          / 'GoogleNews-vectors-negative300.bin'),
        # File name where to save/read the "compact" version to/from.
        'cache_file': str(_PROJECT_ROOT / 'data'
                          / 'GoogleNews-vectors-negative300.compact.bin'),
        # Retrain the model every time?
        'retrain':    False,
        # Averaging strategy to use, one of {'average', 'k-means', 'dummy'}
        # NOTE: Random Forest, Logistic Regression, and Feed Forward NN work
        # with 'average' and 'k-means'; Convolutional NN works __only__ with
        # 'dummy'.
        'strategy':   'dummy',
    },
    # Options specific to the random forest classifier.
    'random-forest': {
        'n_estimators': 1000,
        'n_jobs':       (-1),
        'max_depth':    10,
        'max_features': 'log2',
    },
    # Options specific to the logistic regression classifier.
    'logistic-regression': {
        # TODO: I might've messed this part up. Someone with a good
        # understanding of the required arguments, please check!
        #                                                Tom
        'penalty':           'l2',
        'dual':              True,
        'tol':               0.0001,
        'C':                 1,
        'fit_intercept':     True,
        'intercept_scaling': 1.0,
        'class_weight':      None,
        'random_state':      None,
    },
    # Options specific to Andre's convolutional NN classifier.
    'convolutional': {
        'n_units':      250,
        'n_filters':    90,
        'n_epochs':     4,
        'verbose':      2,
        'kernel_size':  6,
        'batch_size':   128,
        'use_bias':     True,
        'loss':         'binary_crossentropy',
        'optimizer':    Adam(lr=0.0021),
        'metrics':      ['accuracy'],
    },
    # Options specific to feed forward neural network classifier.
    'feed-forward': {
        'n_hidden_units1': 16,
        'n_hidden_units2': 10,
        'batch_size':      32,
        'n_epochs':        200,
        'loss':            'binary_crossentropy',
        'optimizer':       SGD(lr=0.01),
        'metrics':         ['accuracy'],
    },
    # Dummy entry for dummy averager :)
    'dummy': {},
    # Options specific to the "average" averaging strategy.
    'average': {},
    # Options specific to the "k-means" averaging strategy.
    'k-means': {
        'number_clusters_frac': 0.2,  # NOTE: This argument is required!
        'warn_on_missing': False,
        # 'max_iter':             100,
        # 'n_jobs':               4,
    },
}

# If you have a local conf.py which defines the configuration options, it will
# be used. Otherwise, this falls back to defaults.
# NOTE: See _DEFAULT_CONFIG for the format of the configuratio options.
try:
    from conf import conf
    print("Imported local config file.", file=sys.stderr)
except ImportError:
    conf = _DEFAULT_CONFIG
    print("Used the default config.", file=sys.stderr)

# TODO: Fix this.
# Turn off warnings about Beautiful Soup (Johanna has checked all of them
# manually)
warnings.filterwarnings("ignore", category=UserWarning, module='bs4')

def _read_data_from(path: str) -> pd.DataFrame:
    logging.info('Reading data from {!r}...'.format(path))
    return pd.read_csv(path, header=0, delimiter='\t', quoting=3)

class LemmatizationWithPOSTagger(object):
    def __init__(self):
        self.lemmatizer = nltk.WordNetLemmatizer()

    def get_wordnet_pos(self, treebank_tag):
        """
        Convert Part of Speech Tags to comply with the expectations of wordnet
        """
        if treebank_tag.startswith('J'):
            return wordnet.ADJ
        elif treebank_tag.startswith('V'):
            return wordnet.VERB
        elif treebank_tag.startswith('N'):
            return wordnet.NOUN
        elif treebank_tag.startswith('R'):
            return wordnet.ADV
        else:
            # As default pos in lemmatization is Noun
            return wordnet.NOUN

    def pos_tag(self, tokens):
        """
        Method to obtain POS-Tags

        """
        # find the pos tagginf for each tokens [('What', 'WP'), ('can', 'MD'), ('I', 'PRP') ....
        pos_tokens = [nltk.pos_tag(token) for token in tokens]

        # TODO: The following looks ugly...
        # lemmatization using pos tagg
        # convert into feature set of [('What', 'What', ['WP']), ('can', 'can', ['MD']), ... ie [original WORD, Lemmatized word, POS tag]
        pos_tokens = [ [(word, self.lemmatizer.lemmatize(word,self.get_wordnet_pos(pos_tag)), [pos_tag]) for (word,pos_tag) in pos] for pos in pos_tokens]
        return pos_tokens

class ReviewPreprocessor(object):
    """
    :py:class:`ReviewPreprocessor` is an utility class for processing raw HTML
    text into segments for further learning.
    """

    _stopwords = set(nltk.corpus.stopwords.words('english'))
    _tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

    @staticmethod
    def _striphtml(text: str) -> str:
        text = re.sub(r'(www.|http[s]?:\/)(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
                      r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
                      r'', text)
        text = BeautifulSoup(text, 'html.parser').get_text()
        return text

    @staticmethod
    def _2wordlist(text: str, remove_stopwords: bool = False) -> List[str]:
        text = re.sub('[^a-zA-Z]', ' ', text)
        words = text.lower().split()
        if remove_stopwords:
            words = [w for w in words if w not in
                     ReviewPreprocessor._stopwords]
        # TODO: This is ugly... Is it even correct?
        words = [words]
        lemma_pos_token = LemmatizationWithPOSTagger().pos_tag(words)
        words = [[x[1] for x in el] for el in lemma_pos_token]
        words = words[0]
        return words

    @staticmethod
    def review2wordlist(review: str, remove_stopwords: bool = False) \
            -> List[str]:
        """
        Given a review, parses it as HTML, removes all URLs and
        non-alphabetical characters, and optionally removes the stopwords. The
        review is then split into lowercase words and the resulting list is
        returned.

        :param str review:            The review as raw HTML.
        :param bool remove_stopwords: Whether to remove the stopwords.
        :return:                      Review split into words.
        :rtype:                       List[str]
        """
        return ReviewPreprocessor._2wordlist(
            ReviewPreprocessor._striphtml(review),
            remove_stopwords=remove_stopwords)

    @staticmethod
    def review2sentences(review: str, remove_stopwords: bool = False) \
            -> Iterable[List[str]]:
        """
        Given a review, splits it into sentences where each sentence
        is a list of words.

        :param str review:            The review as raw HTML.
        :param bool remove_stopwords: Whether to remove the stopwords.
        :return:                      Review split into sentences.
        :rtype:                       Iterable[List[str]].
        """
        raw_sentences = ReviewPreprocessor._tokenizer.tokenize(
            ReviewPreprocessor._striphtml(review))
        return map(
            lambda x: ReviewPreprocessor._2wordlist(x, remove_stopwords),
            filter(lambda x: x, raw_sentences))

# TODO: This function should probably also do the stemming, see
# https://github.com/mlp2018/BagofWords/issues/7.
def clean_up_reviews(reviews: Iterable[str],
                     remove_stopwords: bool = True,
                     clean_file: Optional[str] = None) -> np.ndarray:
    """
    Given an list of reviews, either loads pre-computed clean reviews from
    ``clean_file`` or applies :py:func:`ReviewPreprocessor.review2wordlist` to
    each review and converts it into an :py:class:`numpy.ndarray`. Also, if
    ``clean_file`` is not ``None`` clean reviews are saved to it.

    :param reviews:               Reviews to clean up.
    :type reviews:                Iterable[str]
    :param bool remove_stopwords: Whether to remove the stopwords.
    :param clean_file:            File where clean reviews are stored.
    :type clean_file:             Optional[str]
    :return:                      Iterable of clean reviews.
    :rtype:                       Iterable[str]
    """
    if clean_file is not None and Path(clean_file).exists():
        return _read_data_from(clean_file)['review'].values
    logging.info('Cleaning and parsing the reviews...')
    review = np.array(list(map(
        lambda x: ' '.join(
            ReviewPreprocessor.review2wordlist(x, remove_stopwords)),
        reviews)), object)
    # This assert is important to ensure we're doing things in a
    # memory-efficient way!
    assert review.dtype == np.object
    if clean_file is not None:
        logging.info('Saving clean data to file {!r}...'
                     .format(repr(clean_file)))
        pd.DataFrame(data={"review": review}) \
            .to_csv(clean_file, index=False, quoting=3)
    return review

def reviews2sentences(reviews: Iterable[str],
                      remove_stopwords: bool = True) -> Iterable[List[str]]:
    """
    Given a list of reviews, cleans them up, splits into sentences, and returns
    a list of *all* the sentences.

    :param reviews:               Reviews to clean up.
    :type reviews:                Iterable[str]
    :param bool remove_stopwords: Whether to remove the stopwords.
    :return:                      Iterable of sentences, where each sentence is
                                  a list of words.
    :rtype:                       Iterable[List[str]]
    """
    logging.info('Splittings reviews into sentences...')

    class R2SIter:
        def __init__(self, xs, remove):
            self.reviews = xs
            self.remove_stopwords = remove

        def __iter__(self):
            for review in self.reviews:
                for sentence in ReviewPreprocessor.review2sentences(
                        review, self.remove_stopwords):
                    yield sentence
    return R2SIter(reviews, remove_stopwords)

def submission_run(reviews: np.ndarray,
                   sentiments: np.ndarray,
                   test_reviews: np.ndarray,
                   ids: np.ndarray,
                   mk_vectorizer: Callable[[], Any],
                   mk_classifier: Callable[[], Any],
                   prediction_file: str) -> np.ndarray:
    """
    :param reviews:         Array of raw reviews texts.
    :type reviews:          'numpy.ndarray' of shape ``(N,)`` of type ``str``
    :param sentiments:      Array of review sentiments.
    :type sentiments:       'numpy.ndarray' of shape ``(N,)`` of type ``bool``
    :param test_reviews:    Array of test review texts.
    :type test_reviews:     'numpy.ndarray' of shape ``(N,)`` of type ``str``
    :param ids:             Array of review identifiers.
    :type ids:              'numpy.ndarray' of shape ``(N,)`` of type ``str``
    :param mk_vectorizer:   Factory function to create a new vectorizer.
    :type mk_vectorizer:    Callable[[], Vectorizer]
    :param mk_classifier:   Factory function to create a new classifier.
    :type mk_classifier:    Callable[[], Classifier]
    :param prediction_file: File where the predicted sentiments are saved to.
    :type prediction_file:  str
    """
    _, prediction = run_one_fold(
        (reviews, sentiments), test_reviews, mk_vectorizer(),
        mk_classifier())
    logging.info('Saving all predicted sentiments to {!r}...'
                 .format(prediction_file))
    pd.DataFrame(data={'id': ids, 'sentiment': prediction}) \
        .to_csv(prediction_file, index=False, quoting=3)

def bookkeeping(reviews: Type[np.ndarray],
                wrong_index: Type[np.ndarray],
                wrong_prediction_file: str):
    logging.info('Saving wrong predictions to {!r}...'
                 .format(wrong_prediction_file))
    pd.DataFrame(data={'review': reviews[wrong_index]}) \
        .to_csv(wrong_prediction_file, index=False, quoting=3, escapechar='\\')

def run_one_fold(train_data: Tuple[np.ndarray, np.ndarray],
                 test_data: Tuple[np.ndarray, np.ndarray],
                 vectorizer: Any, classifier: Any) \
        -> Tuple[float, np.ndarray]:
    """
    Given some data to train on and some data to test on, runs the whole
    feature extraction + classification procedure, computes the ROC AUC score
    of the predictions and returns it along with raw predictions.

    :param train_data: ``(array of reviews, array of sentiments)`` on which the
                       model will be trained.
    :param test_data:  ``(array of reviews, array of sentiments)`` on which the
                       accuracy of the model will be computed.
    :param vectorizer: Vectorizer to use.
    :param classifier: Classifier to use.
    :return:           ``(score, predictions)`` tuple.
    """
    score = None
    (train_reviews, train_labels) = train_data
    if isinstance(test_data, tuple):
        (test_reviews, test_labels) = test_data
    else:
        test_reviews = test_data
    logging.info('Transforming training data...')
    train_features = vectorizer.fit_transform(train_reviews)
    logging.info('Transforming test data...')
    test_features = vectorizer.transform(test_reviews)
    logging.info('Fitting...')
    classifier = classifier.fit(train_features, train_labels)
    logging.info('Predicting test labels...')
    prediction = classifier.predict(test_features)
    if isinstance(test_data, tuple):
        score = roc_auc_score(test_labels, prediction)
        logging.info('ROC AUC for this fold is {}.'.format(score))
    return score, prediction

def split_90_10(data: Tuple[np.ndarray, np.ndarray],
                alpha: float, seed: int = 42) \
        -> Tuple[Tuple[np.ndarray, np.ndarray],
                 Tuple[np.ndarray, np.ndarray]]:
    """
    Despite the very descriptive name this function does the ``1-alpha`` -
    ``alpha`` split rather than the ``90%`` - ``10%`` one.

    :param data:  The data to split.
    :param alpha: Percentage of data to use for testing.
    :param seed:  Random seed to use for splitting.
    :return: ``((reviews to train on, sentiments to train on),
                (reviews to test on, sentiments to test on))``.
    """
    assert alpha > 0 and alpha < 1
    reviews, labels = data
    train_index = None
    test_index = None
    if _get_sklearn_version() >= (0, 19, 0):
        train_index, test_index = StratifiedShuffleSplit(
            n_splits=1, test_size=alpha,
            random_state=seed).split(reviews, labels).__iter__().__next__()
    else:
        train_index, test_index = StratifiedShuffleSplit(
            labels, n_iter=1, test_size=alpha,
            random_state=seed).__iter__().__next__()
    x_train = reviews[train_index]
    x_test = reviews[test_index]
    y_train = labels[train_index]
    y_test = labels[test_index]
    return (x_train, y_train), (x_test, y_test)

def run_a_couple_of_folds(data: Tuple[np.ndarray, np.ndarray],
                          mk_vectorizer: Callable[[], Any],
                          mk_classifier: Callable[[], Any],
                          number_splits: int) \
        -> Tuple[float, float, np.ndarray]:
    # TODO: Write docs for this function.
    reviews, labels = data
    predictions = np.zeros(labels.shape, dtype=np.bool_)
    scores = np.zeros((number_splits,), dtype=np.float32)

    def go(idx, train_index, test_index):
        logging.info('Processing fold number {}...'.format(idx + 1))
        scores[idx], predictions[test_index] = run_one_fold(
            (reviews[train_index], labels[train_index]),
            (reviews[test_index], labels[test_index]),
            mk_vectorizer(), mk_classifier())

    if _get_sklearn_version() >= (0, 19, 0):
        skf = StratifiedKFold(n_splits=number_splits, shuffle=False)
        for idx, (train_index, test_index) \
                in enumerate(skf.split(reviews, labels)):
            go(idx, train_index, test_index)
    else:
        skf = StratifiedKFold(labels, n_folds=number_splits, shuffle=False)
        for idx, (train_index, test_index) in enumerate(skf):
            go(idx, train_index, test_index)

    score, score_std = np.mean(scores), np.std(scores)
    logging.info(('Overall accuracy on left-out data from k-fold '
                  'cross-validation was {}').format(score))
    wrong_index = np.where(predictions != labels)
    return score, score_std, wrong_index

def optimization_run(reviews: np.ndarray,
                     sentiments: np.ndarray,
                     mk_vectorizer: Callable[[], Any],
                     mk_classifier: Callable[[], Any],
                     conf: Dict[str, Any],
                     wrong_prediction_file: str) -> np.ndarray:
    """
    This is basically the main function.

    :param ids: Array of review identifiers.
    :type ids: 'numpy.ndarray' of shape ``(N,)``
    :param reviews: Array of raw reviews texts.
    :type reviews: 'numpy.ndarray' of shape ``(N,)``
    :param sentiments: Array of review sentiments.
    :type sentiments: 'numpy.ndarray' of shape ``(N,)``
    :param mk_vectorizer: Factory function to create a new vectorizer.
    :type mk_vectorizer: Callable[[], Vectorizer]
    :type mk_classifier: Callable[[], Classifier]
    :param n_splits
    :param test_on_10
    """
    data_90, data_10 = split_90_10((reviews, sentiments),
                                   conf['alpha'], conf['random'])
    reviews_90, _ = data_90
    reviews_10, labels_10 = data_10
    _, _, wrong_idx = run_a_couple_of_folds(
        data_90, mk_vectorizer, mk_classifier, conf['number_splits'])
    wrong_reviews = reviews_90[wrong_idx]
    if conf['test_10']:
        logging.info('Training on 90% and testing on 10%...')
        _, prediction_10 = run_one_fold(
            data_90, data_10, mk_vectorizer(), mk_classifier())
        wrong_idx_10 = np.where(prediction_10 != labels_10)
        wrong_reviews = np.concatenate(
            (wrong_reviews, reviews_10[wrong_idx_10]), axis=0)
    logging.info('Saving wrong predictions to {!r}...'
                 .format(wrong_prediction_file))
    pd.DataFrame(data={'review': wrong_reviews}) \
        .to_csv(wrong_prediction_file, index=False, quoting=3, escapechar='\\')

class SimpleAverager(object):
    """
    This class implements the simplest strategy for reducing a list of
    word2vec vectors into a single one -- averaging.
    """

    def __init__(self) -> None:
        pass

    @staticmethod
    def _make_avg_feature_vector(words: str,
                                 known_words: KeyedVectors,
                                 average_vector: Optional[np.ndarray] = None) \
            -> np.ndarray:
        """
        Given a list of words, returns the their average.

        :param str words:      Words to average.
        :param known_words:    Dictionary mapping known words to vectors.
        :type known_words:     Dict[str, np.ndarray]
        :param average_vector: Pre-allocated zero-initialised vector. It will
                               contain the average of ``words``.
        :type average_vector:  np.ndarray
        :return:               The average of ``words``.
        :rtype:                np.ndarray
        """
        word_count = sum(
            1 for _ in map(lambda x: np.add(average_vector, known_words[x],
                                            out=average_vector),
                           filter(lambda x: x in known_words, words.split()))
        )
        return np.divide(average_vector, float(word_count), out=average_vector)

    def transform(self, reviews: np.ndarray, model: KeyedVectors) \
            -> np.ndarray:
        # TODO: Docs
        (_, number_features) = model.syn0.shape
        (number_reviews,) = reviews.shape
        feature_vectors = np.zeros(
            (number_reviews, number_features), dtype='float32')
        for (i, (review, vector)) in enumerate(zip(reviews, feature_vectors)):
            if i % 1000 == 0:
                logging.info('PROGRESS: At review #{} of {}...'
                             .format(i, number_reviews))
            SimpleAverager._make_avg_feature_vector(
                review, model, vector)
        return feature_vectors

    def fit_transform(self, reviews: np.ndarray, model: KeyedVectors)\
            -> np.ndarray:
        """
        :py:class:`SimpleAverager` has no state, and :py:func:`fit_transform`
        thus simply calls :py:func:`transform`.
        """
        return self.transform(reviews, model)

class KMeansAverager(object):
    """
    This class implements the 'k-means' averaging strategy for reducing a list
    of word2vec vectors to a single one.
    """

    def __init__(self, number_clusters_frac: float,
                 warn_on_missing: bool, **kwargs: dict) -> None:
        self.number_clusters_frac = number_clusters_frac
        self.warn_on_missing = warn_on_missing
        self.kmeans_args = kwargs
        self.kmeans = None
        self.word2centroid = None

    @staticmethod
    def _make_bag_of_centroids(words: Iterable[str],
                               word2centroid: Dict[str, int],
                               bag_of_centroids: np.ndarray,
                               warn_on_missing: bool = False) \
            -> np.ndarray:
        """
        Given a stream of words and a mapping of words to centroid indices,
        converts these words to "bag of centroids" representation.

        :param words:            A list of words to convert.
        :type words:             Iterable[str]
        :param word2centroid:    A ``word → cluster index`` map.
        :type word2centroid:     Dict[str, int]
        :param bag_of_centroids: Output array.
        :type bag_of_centroids:  np.ndarray
        :param warn_on_missing:  Whether to issue warnings is some words are
                                 not in the `word2centroid` dictionary.
        :type warn_on_missing:   bool
        :return:                 Bag of centroids representation of ``words``.
        :rtype:                  np.ndarray
        """
        for word in words:
            i = word2centroid.get(word)
            if i is not None:
                bag_of_centroids[i] += 1
            elif warn_on_missing:
                warnings.warn(('While creating a bag of centroids: {!r} is '
                               'not in the word→index map.').format(word))
        return bag_of_centroids

    # NOTE: This uses the third option from here
    # https://github.com/mlp2018/BagofWords/issues/16. Is this the right
    # way to go?
    def transform(self, reviews: np.ndarray,
                  model: KeyedVectors) -> np.ndarray:
        """
        Given a list of reviews and a word2vec mapping, transforms all reviews
        to the bag of centroids representation.

        :param reviews: Reviews to transform.
        :type reviews:  np.ndarray
        :param model:   ``word→vector`` mapping.
        :type model:    Dict[str, np.ndarray]
        :return:        Array of reviews in the "bag of centroids"
                        representation.
        """
        (num_reviews,) = reviews.shape
        logging.info('Creating bags of centroids...')
        bags = np.zeros((num_reviews, self.kmeans.n_clusters), dtype='float32')
        for (review, bag) in zip(reviews, bags):
            KMeansAverager._make_bag_of_centroids(
                review.split(), self.word2centroid, bag, self.warn_on_missing)
        return bags

    def fit_transform(self, reviews: np.ndarray,
                      model: KeyedVectors) -> np.ndarray:
        """
        Given a list of reviews and a word2vec mapping, runs k-means clustering
        on the vector representation of reviews and converts them to the bag of
        centroids representation.

        :param reviews: Reviews to transform.
        :type reviews:  np.ndarray
        :param model:   ``word→vector`` mapping.
        :type model:    Dict[str, np.ndarray]
        :return:        Array of reviews in the "bag of centroids"
                        representation.
        """
        (num_reviews,) = reviews.shape
        num_clusters = int(self.number_clusters_frac * num_reviews)
        self.kmeans = MiniBatchKMeans(
            n_clusters=num_clusters, **self.kmeans_args)
        vectors = model.syn0
        logging.info('Running k-means + labeling...')
        start = time.time()
        cluster_indices = self.kmeans.fit_predict(vectors)
        end = time.time()
        logging.info('Done with k-means clustering in {:.0f} seconds!'
                     .format(end - start))
        logging.info('Creating word→index map...')
        self.word2centroid = dict(zip(model.index2word, cluster_indices))
        return self.transform(reviews, model)

class DummyAverager(object):
    """
    Averager that does not average :)
    """

    def __init__(self):
        pass

    def transform(self, reviews, model):
        return reviews, model

    def fit_transform(self, reviews, model):
        return reviews, model

class Word2VecVectorizer(object):
    """
    This class implements conversion of reviews to feature vectors using
    Word2Vec model.
    """

    _make_averager_fn = {
        'average': SimpleAverager,
        'k-means': KMeansAverager,
        'dummy':   DummyAverager,
    }

    def __init__(self,
                 strategy: str,
                 model_file: str,
                 model_args: Optional[Dict[str, Any]] = None,
                 averager_args: Optional[Dict[str, Any]] = None,
                 cache_file: Optional[str] = None,
                 retrain: bool = False,
                 train_data: Optional[Iterable[str]] = None) -> None:
        self.averager = \
            Word2VecVectorizer._make_averager_fn[strategy](**averager_args)
        self.model = None
        self.cache_file = cache_file
        self.is_compact = False
        # First of all, retrain the model if we're passed some training data.
        if train_data is not None:
            self.model = Word2VecVectorizer._retrain(
                train_data, model_file, **model_args)
        else:
            # If we already have our model cached, let's just load it.
            if cache_file is not None and Path(cache_file).exists():
                self.model = KeyedVectors.load_word2vec_format(
                    cache_file, binary=True)
                self.is_compact = True
            else:
                # Well, let's load the model and construct remove all
                # unnecessary stuff afterwards...
                # TODO: This always loads in binary format, because _retrain
                # uses binary format for saving. Not sure if that's OK.
                self.model = KeyedVectors.load_word2vec_format(
                    model_file, binary=True)

    @staticmethod
    def _retrain(train_data: Iterable[str], model_file: str, **model_args) \
            -> Word2Vec:
        # Touch the model_file _before_ we start training so that if the path
        # is wrong we don't wait hours for nothing...
        Path(model_file).touch()

        logging.info('Training Word2Vec model...')
        start = time.time()
        model = Word2Vec(reviews2sentences(train_data), **model_args).wv
        stop = time.time()
        logging.info('Done training in {:.0f} seconds!'
                     .format(stop - start))
        logging.info('Saving KeyedVectors model to {!r}...'
                     .format(model_file))
        # TODO: This always saves in binary format. I'm not sure that's what we
        # want.
        #                                                          Tom
        model.save_word2vec_format(model_file, binary=True)
        return model

    @staticmethod
    def _compress_keyed_vectors(model: KeyedVectors, reviews: np.ndarray,
                                cache_file: Optional[str] = None) -> None:
        required_words = set(word for review in reviews
                             for word in review.split())
        for (i, word) in enumerate(model.index2word):
            assert model.vocab[word].index == i

        # This is a really shady piece :)
        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # First of all, extract all the words we really need
        index2word = [w for w in model.index2word if w in required_words]

        def update_index(item, index):
            item.index = index
            return item
        # In the first approximation, we want
        # vocab = dict((w, model.vocab[w]) for w in index2word)
        # i.e. to keep the data for words which are in index2word. The problem
        # is that Vocab type has a field index which stores word's index in the
        # syn0 vector and index2word list. We, obbiously, break this indexing
        # by rebuilding index2word from scratch. So here we manually
        # reconstruct correct indexing.
        vocab = dict((w, update_index(model.vocab[w], i))
                     for (i, w) in enumerate(index2word))
        # Now, extract correct indices as an np.ndarray
        indices = np.fromiter(
            map(lambda x: x.index, vocab.values()), dtype=int)
        # Only keep feature vectors for the words we really need.
        vectors = model.syn0[indices]
        # Do the actual update
        model.index2word = index2word
        model.vocab = vocab
        model.syn0 = vectors
        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # end shady code...

        if cache_file is not None:
            model.save_word2vec_format(cache_file, binary=True)
        return model

    def fit_transform(self, reviews):
        if not self.is_compact:
            self.model = Word2VecVectorizer._compress_keyed_vectors(
                self.model, reviews, self.cache_file)
            self.is_compact = True
        return self.averager.fit_transform(reviews, self.model)

    def transform(self, reviews):
        if not self.is_compact:
            self.model = Word2VecVectorizer._compress_keyed_vectors(
                self.model, reviews, self.cache_file)
            self.is_compact = True
        return self.averager.transform(reviews, self.model)

def _make_vectorizer(conf):
    vectorizer_str = conf['vectorizer']
    if vectorizer_str == 'bagofwords':
        return CountVectorizer(**conf[vectorizer_str])
    elif vectorizer_str == 'word2vec':
        train_data = \
            _read_data_from(conf['in']['unlabeled'])['review'] \
            if conf['word2vec']['retrain'] else None
        return Word2VecVectorizer(
            **conf['word2vec'],
            averager_args=conf[conf['word2vec']['strategy']],
            train_data=train_data)
    else:
        raise Exception("Unknown vectorizer type.")

def _make_classifier(conf):
    _fn = {
        'logistic-regression':    LogisticRegression,
        'naive-bayes-bagofwords': MultinomialNB,
        'naive-bayes-word2vec':   BernoulliNB,
        'random-forest':          RandomForestClassifier,
        'convolutional':          ConvNNClassifier,
        'feed-forward':           FeedForwardNNClassifier,
    }
    return _fn[conf['classifier']](**conf[conf['classifier']])

class ConvNNClassifier(object):
    def __init__(self, n_units, n_filters, kernel_size,
                 use_bias, n_epochs, batch_size, loss,
                 optimizer, metrics, verbose):
        self.n_units = n_units
        self.n_filters = n_filters
        self.kernel_size = kernel_size
        self.use_bias = use_bias
        self.loss = loss
        self.optimizer = optimizer
        self.metrics = metrics
        self.batch_size = batch_size
        self.n_epochs = n_epochs
        self.verbose = verbose
        self.callbacks = [EarlyStopping(patience=1)]
        self.max_words = 500
        self.model = None
        self.history = None

    def _construct(self, word2vec):
        (n_words, n_features) = word2vec.vectors.shape
        model = Sequential()
        layer = Embedding(input_dim=n_words,
                          output_dim=n_features,
                          input_length=self.max_words,
                          weights=[word2vec.vectors],
                          trainable=False)
        model.add(layer)
        model.add(Conv1D(filters=self.n_filters,
                         kernel_size=self.kernel_size,
                         padding='same',
                         activation='relu',
                         use_bias=self.use_bias))
        model.add(MaxPooling1D(pool_size=self.max_words, strides=2))
        model.add(Flatten())
        model.add(Dense(self.n_units,
                        activation='relu',
                        use_bias=self.use_bias))
        model.add(Dense(1, activation='sigmoid', use_bias=self.use_bias))
        model.compile(
            loss=self.loss, optimizer=self.optimizer, metrics=self.metrics)
        logging.info(model.summary())
        return model

    @staticmethod
    def _review_to_tensor(review, word2vec, max_words):
        tensor = np.full((max_words,), fill_value=max_words)  # ?????
        for (i, word) in enumerate(filter(lambda x: x in word2vec,
                                          review.split())):
            # TODO: The following `if` is ungly, fix it please!
            if i >= max_words:
                break
            if word in word2vec:
                tensor[i] = word2vec.vocab[word].index
        return tensor

    def fit(self, train_data, labels):
        reviews, word2vec = train_data
        logging.info('Converting reviews to tensors...')
        reviews = np.array(list(map(
            lambda x: ConvNNClassifier._review_to_tensor(x, word2vec,
                                                         self.max_words),
            reviews)))
        logging.info('Done!')
        self.model = self._construct(word2vec)
        self.history = self.model.fit(x=reviews, y=labels,
                                      validation_data=None,
                                      epochs=self.n_epochs,
                                      batch_size=self.batch_size,
                                      verbose=self.verbose,
                                      callbacks=self.callbacks)
        return self

    def predict(self, train_data):
        reviews, word2vec = train_data
        (n_reviews,) = reviews.shape
        logging.info('Converting reviews to tensors...')
        reviews = np.array(list(map(
            lambda x: ConvNNClassifier._review_to_tensor(x, word2vec,
                                                         self.max_words),
            reviews)))
        logging.info('Done!')
        return self.model.predict(reviews).reshape((n_reviews,))

class FeedForwardNNClassifier(object):

    def __init__(self, n_hidden_units1, n_hidden_units2, batch_size,
                 n_epochs, optimizer, loss, metrics):
        self.n_hidden_units1 = n_hidden_units1
        self.n_hidden_units2 = n_hidden_units2
        self.batch_size = batch_size
        self.n_epochs = n_epochs
        self.optimizer = optimizer
        self.loss = loss
        self.metrics = metrics
        self.model = None
        self.history = None

    def _construct(self, n_features):
        model = Sequential()
        model.add(Dense(self.n_hidden_units1,
                        input_dim=n_features,
                        activation='relu',
                        kernel_initializer='uniform'))
        model.add(Dense(self.n_hidden_units2,
                        activation='relu',
                        kernel_initializer='uniform'))
        model.add(Dense(1, activation='sigmoid'))
        model.compile(loss=self.loss, optimizer=self.optimizer,
                      metrics=self.metrics)
        logging.info(model.summary())
        return model

    def fit(self, reviews, sentiments):
        (_, n_features) = reviews.shape
        self.model = self._construct(n_features)
        self.history = self.model.fit(x=reviews, y=sentiments,
                                      epochs=self.n_epochs,
                                      batch_size=self.batch_size)
        return self

    def predict(self, reviews):
        (n_reviews, _) = reviews.shape
        return self.model.predict(reviews).reshape((n_reviews,))

def main():
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
                        level=logging.INFO)
    if conf['run']['type'] == 'optimization':
        train_data = _read_data_from(conf['in']['labeled'])
        reviews = clean_up_reviews(train_data['review'],
                                   conf['run']['remove_stopwords'],
                                   conf['in']['clean'])
        sentiments = np.array(train_data['sentiment'], dtype=np.bool_)

        def mk_vectorizer():
            return _make_vectorizer(conf)

        def mk_classifier():
            return _make_classifier(conf)

        optimization_run(reviews, sentiments,
                         mk_vectorizer, mk_classifier,
                         conf['run'],
                         conf['out']['wrong_result'])
    elif conf['run']['type'] == 'submission':
        train_data = _read_data_from(conf['in']['labeled'])
        test_data = _read_data_from(conf['in']['test'])
        ids = np.array(test_data['id'], dtype=np.unicode_)
        reviews = clean_up_reviews(train_data['review'],
                                   conf['run']['remove_stopwords'],
                                   conf['in']['clean'])
        sentiments = np.array(train_data['sentiment'], dtype=np.bool_)
        test_reviews = clean_up_reviews(test_data['review'],
                                        conf['run']['remove_stopwords'])

        def mk_vectorizer():
            return _make_vectorizer(conf)

        def mk_classifier():
            return _make_classifier(conf)

        submission_run(reviews, sentiments,
                       test_reviews,
                       ids,
                       mk_vectorizer, mk_classifier,
                       conf['out']['result'])
    else:
        raise NotImplementedError()

if __name__ == '__main__':
    # cProfile.run('main()')
    main()


from django import template

from categories.models import Category

register = template.Library()

@register.inclusion_tag('categories/categories_menu.html')
def list_categories(category):
    """
    Create a dictionary with all the categories
    :param category: this is a Category object
    :return: a dictionary with the categories
    """
    categories = Category.objects.all()
    return {'categories': categories}


import requests

from scrapy import Spider, Request, log
from scrapy.selector import Selector

from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import NoSuchElementException

from utils.sql import get_session
from utils.sql.models.course_unit import CourseUnit
from utils.sql.models.course_video import CourseVideo

from scraper.edx_bot.spiders import EdXLoggerIn
from scraper.edx_bot.spiders.general_course_content import GeneralCourseContent

from scraper.edx_bot.items import CourseVideoItem

class VideoTranscripts(Spider):
    name = 'video_transcripts'
    allowed_domains = ['edx.org']
    session = None

    def start_requests(self):
        self.session = get_session()
        self.edx_logger = EdXLoggerIn()

        driver = self.edx_logger.driver
        driver.maximize_window()

        for video_id in self.session.query(CourseVideo.id).filter(\
            CourseVideo.transcript == None):

            # Get a unit corresponding to that video. Though there may
            # be multiple units that host that video, the transcripts
            # should be similar across all... therefore the first unit
            # should allow us to get the transcript.
            unit_href = self.session.query(CourseUnit.href).filter(\
                CourseUnit.videos.any(CourseVideo.id == video_id)).first()

            yield Request(
                url = unit_href[0],
                meta = {'video_id':video_id[0]},
                cookies = driver.get_cookies(),
                callback = self.fetch_transcript
            )

    def fetch_transcript(self, response):
        driver = self.edx_logger.driver
        driver.get(response.url)

        sequence_el = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located(
                (By.XPATH, '//*[@id="seq_content"]')))

        for module in sequence_el.find_elements_by_xpath('.//div/div/div'):
            data_type = module.get_attribute('data-block-type')

            if data_type == 'video':
                try:
                    for download_button in module.find_elements_by_class_name('video-download-button'):
                        sub_element = download_button.find_element_by_xpath('.//a')
                        if sub_element.text == 'Download transcript':
                            transcript_href = sub_element.get_attribute('href')

                            msg = "Got transcript url=%s" % (transcript_href)
                            log.msg(msg, level=log.INFO)

                            return self.parse_transcript(transcript_href, \
                                driver.get_cookies(), meta = response.meta)

                except NoSuchElementException:
                    msg = "No txt transcript found for video in unit with url=%s" \
                        % (response.url)
                    log.msg(msg, level=log.INFO)

        return None

    def parse_transcript(self, transcript_href, driver_cookies, meta):
        cookies = {d['name']:d['value'] for d in driver_cookies}
        r = requests.get(transcript_href, cookies=cookies)

        return CourseVideoItem(
            _id = meta['video_id'],
            transcript = r.text
        )

    def closed(self, reason):
        self.session.close()
        self.edx_logger.close()


import tensorflow as tf
from tensorflow import keras

import numpy as np

print(tf.__version__)

# Load in the built-in housing dataset
boston_housing = keras.datasets.boston_housing

(train_data, train_labels), (test_data, test_labels) = boston_housing.load_data()

# Shuffle the training set
order = np.argsort(np.random.random(train_labels.shape))
train_data = train_data[order]
train_labels = train_labels[order]

# Use pandas library to display the first several rows of the dataset in a table
import pandas as pd

column_names = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD',
                'TAX', 'PTRATIO', 'B', 'LSTAT']

df = pd.DataFrame(train_data, columns=column_names)
print(df.head())

# Test data is *not* used when calculating the mean and std.
# Normalizing data - (subtract the mean of the feature and divide by standard deviation)
# Makes training far easier.

mean = train_data.mean(axis=0)
std = train_data.std(axis=0)
# Saving copy of original test_data to display in table at the end of run.
original_test_data = test_data
train_data = (train_data - mean) / std
test_data = (test_data - mean) / std

# Seuquential model with two densely connected hidden layers.
# Output layer returns single, continuous value.

def build_model():
  model = keras.Sequential([
    keras.layers.Dense(64, activation=tf.nn.relu,
                       input_shape=(train_data.shape[1],)),
    keras.layers.Dense(64, activation=tf.nn.relu),
    keras.layers.Dense(1)
  ])

  optimizer = tf.train.RMSPropOptimizer(0.001)

  model.compile(loss='mse',
                optimizer=optimizer,
                metrics=['mae'])
  return model

# model = build_model()
# model.summary()

# Display training progress by printing a single dot for each completed epoch.
class PrintDot(keras.callbacks.Callback):
  def on_epoch_end(self,epoch,logs):
    if epoch % 100 == 0: print('')
    print('.', end='')

EPOCHS = 500

# Store training stats
# history = model.fit(train_data, train_labels, epochs=EPOCHS,
#                     validation_split=0.2, verbose=0,
#                     callbacks=[PrintDot()])

import matplotlib as mpl
mpl.use('TkAgg')
import matplotlib.pyplot as plt

# Graph to display model's improvement.
def plot_history(history):
  plt.figure()
  plt.xlabel('Epoch')
  plt.ylabel('Mean Abs Error [1000$]')
  plt.plot(history.epoch, np.array(history.history['mean_absolute_error']),
           label='Train Loss')
  plt.plot(history.epoch, np.array(history.history['val_mean_absolute_error']),
           label = 'Val loss')
  plt.legend()
  plt.ylim([0,5])
  plt.show()

# plot_history(history)

model = build_model()

# The patience parameter is the amount of epochs to check for improvement.
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)

history = model.fit(train_data, train_labels, epochs=EPOCHS,
                    validation_split=0.2, verbose=0,
                    callbacks=[early_stop, PrintDot()])

plot_history(history)

# Evaluate the model using the testing data and get the total loss as well as
# Mean Absolute Error - which is a common regression metric. In this case,
# it represents the average difference in prediction price to actual price.
[loss, mae] = model.evaluate(test_data, test_labels, verbose=0)

print("\nTesting set Mean Abs Error: ${:7.2f}".format(mae * 1000))

test_predictions = model.predict(test_data).flatten()

# Live coding done below:


from mnist import MNIST
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from torch import optim
from torchvision import transforms
from torch.utils.data import DataLoader
from torch import nn
from torchvision.utils import save_image

# from torchvision.datasets import MNIST

mndata = MNIST('./../data/')
images, labels = mndata.load_training()

img_transform = transforms.Compose([
                transforms.ToTensor()
                ])

num_epochs = 100
batch_size = 64
learning_rate = 1e-3
IMG_SIZE = 784
IMG_WIDTH = 28
IMG_HEIGHT = 28

dataset = torch.tensor(images)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

def to_img(x):
    x = x.clamp(0, 1)
    x = x.view(x.size(0), 1, 28, 28)
    return x

class AutoEncoder(nn.Module):
	"""
	"""
	def __init__(self, latent_num=16):
		"""
		TODO: doconvolution
		"""
		super(AutoEncoder, self).__init__()

		self.fc1 = nn.Linear(IMG_SIZE, 256)
		self.fc1.weight.data.normal_(0.0, 0.05)

		self.fc2 = nn.Linear(256, 64)
		self.fc2.weight.data.normal_(0.0, 0.05)

		self.fc3 = nn.Linear(64, 256)
		self.fc3.weight.data.normal_(0.0, 0.05)

		self.fc4 = nn.Linear(256, IMG_SIZE)
		self.fc4.weight.data.normal_(0.0, 0.05)

	def forward(self, x):
		h1 = F.relu(self.fc1(x))  # IMG_SIZE -> 518
		h2 = F.relu(self.fc2(h1)) # 518 -> 256
		h3 = F.relu(self.fc3(h2)) # 256 -> 128
		h4 = F.relu(self.fc4(h3)) # 128 -> 256
		output = h4
		# output = F.sigmoid(h6)
		return output

# ref: http://kvfrans.com/variational-autoencoders-explained/
# 	1) encoder loss = mean square error from original image and decoder image
# 	2) decoder loss = KL divergence
encoder_loss = nn.MSELoss(size_average=True)

def loss_function(output, x):
		"""
		"""
		mse = encoder_loss(output, x)
		return mse

# way to construct DNN network
# 	1) topology
# 	2) loss function
#		3) optimizer
# 	4) forward
# 	5) free zero grad of variable
# 	6) backward

model = AutoEncoder()
optimizer = optim.Adam(model.parameters(), lr=1e-3)

for epoch in range(num_epochs):
	train_loss = 0
	for batch_idx, data in enumerate(dataloader):
		img = data.view(data.size(0), -1)
		img = Variable(img.float())
		# free zero grad
		optimizer.zero_grad()
		output = model(img)
		# backward
		loss = loss_function(output, img)
		loss.backward()
		train_loss += loss.data[0]
		optimizer.step()
		if batch_idx % 100 == 0:
			print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch,
                batch_idx * len(img),
                len(dataloader.dataset), 100. * batch_idx / len(dataloader),
                loss.data[0] / len(img)))
	print('====> Epoch: {} Average loss: {:.4f}'.format(
      	epoch, train_loss / len(dataloader.dataset)))
	if epoch % 10 == 0:
		save = to_img(output.cpu().data)
		save_image(save, './vae_img/image_{}.png'.format(epoch))

torch.save(model.state_dict(), './vae.pth')


import calendar
from datetime import date, datetime

class CalendarUtil:
    def __init__(self, year=date.today().year, month=date.today().month):
        self.year = year
        self.month = month
        self.weekdays = [
            'Segunda-Feira',
            'Terça-Feira',
            'Quarta-Feira',
            'Quinta-Feira',
            'Sexta-Feira',
            'Sábado',
            'Domingo'
        ]
        self.months = [
            'Janeiro',
            'Fevereiro',
            'Março',
            'Abril',
            'Maio',
            'Junho',
            'Julho',
            'Agosto',
            'Setembro',
            'Outubro',
            'Novembro',
            'Dezembro,'
        ]

    def get_current_month_range(self):
        _, day_count = calendar.monthrange(self.year, self.month)

        return [i+1 for i in range(day_count)]

    def get_weekday(self, day):
        date_obj = date(self.year, self.month, day)
        return (date_obj.weekday(), self.weekdays[date_obj.weekday()])

if __name__ == '__main__':
    print(CalendarUtil().get_current_month_range())