from datetime import datetime

import pandas
import pytest

from db import CONNECTION, db
from query_enriched_data_views import create_enriched_user_data, create_enriched_offerer_data
from stock_queries import create_stocks_booking_view
from tests.utils import create_user, create_product, create_offerer, create_venue, create_offer, \
    create_stock, create_booking, create_payment, create_payment_status, clean_views, clean_database
from view_queries import create_enriched_stock_view

class ViewQueriesTest:
    @pytest.fixture(autouse=True)
    def setup_method(self, app):
        yield
        clean_database(app)
        clean_views()

    class CreateEnrichedStockViewTest:
        def test_should_return_all_values(self, app):
            # Given
            create_user(app, id=1)
            create_user(app, id=2, email='other@test.com')
            create_product(app, id=1, product_type='EventType.CINEMA')
            create_product(app, id=2, product_type='ThingType.LIVRE_EDITION')
            create_offerer(app, id=3)
            create_venue(app, offerer_id=3, id=1, siret=None, postal_code=None, city=None, departement_code=None,
                         is_virtual=True)
            create_offer(app, venue_id=1, product_id=1, id=3, product_type='EventType.CINEMA', name="Test")
            create_stock(app, offer_id=3, id=1, date_created='2019-11-01', available=10,
                         booking_limit_datetime='2019-11-23', beginning_datetime='2019-11-24')
            create_offer(app, venue_id=1, product_id=2, id=2, product_type='ThingType.LIVRE_EDITION', name="Test bis")
            create_stock(app, offer_id=2, id=2, date_created='2019-10-01', available=12)
            create_booking(app, user_id=1, stock_id=1, id=4, quantity=2)
            create_payment(app, booking_id=4, id=1)
            create_payment_status(app, payment_id=1, id=1, date='2019-01-01', status='PENDING')

            with app.app_context():
                create_stocks_booking_view()

            expected_stocks_details = pandas.DataFrame(
                index=pandas.Index(data=[1, 2], name='stock_id'),
                data={"Identifiant de l'offre": [3, 2],
                      "Nom de l'offre": ["Test", "Test bis"],
                      "offerer_id": [3, 3],
                      "Type d'offre": ["EventType.CINEMA", 'ThingType.LIVRE_EDITION'],
                      "Département": [None, None],
                      "Date de création du stock": [datetime(2019, 11, 1), datetime(2019, 10, 1)],
                      "Date limite de réservation": [datetime(2019, 11, 23), pandas.NaT],
                      "Date de début de l'évènement": [datetime(2019, 11, 24), pandas.NaT],
                      "Stock disponible brut de réservations": [10, 12],
                      "Nombre total de réservations": [2, 0],
                      "Nombre de réservations annulées": [0, 0],
                      "Nombre de réservations ayant un paiement": [2, 0]
                      }
            )

            # When
            with app.app_context():
                create_enriched_stock_view()

            # Then
            stocks_details = pandas.read_sql_table('enriched_stock_data', CONNECTION, index_col='stock_id')
            pandas.testing.assert_frame_equal(stocks_details, expected_stocks_details)

    class CreateEnrichedUserViewTest:
        def test_should_create_enriched_user_data_view_with_columns(self, app):
            # When
            with app.app_context():
                create_enriched_user_data()

            # Then
            expected_columns = ["Vague d'expérimentation", "Département", "Date d'activation",
                                "Date de remplissage du typeform",
                                "Date de première connexion", "Date de première réservation",
                                "Date de deuxième réservation",
                                "Date de première réservation dans 3 catégories différentes",
                                "Date de dernière recommandation",
                                "Nombre de réservations totales", "Nombre de réservations non annulées",
                                "Ancienneté en jours",
                                "Montant réél dépensé", "Montant théorique dépensé", "Dépenses numériques",
                                "Dépenses physiques"]

            beneficiary_users_details = pandas.read_sql_table('enriched_user_data', CONNECTION, index_col='user_id')
            for expected_column_in_view in expected_columns:
                assert expected_column_in_view in beneficiary_users_details.columns

    class CreateEnrichedOffererViewTest:
        def test_should_create_enriched_offerer_data_view_with_columns(self, app):
            # When
            with app.app_context():
                create_enriched_offerer_data()

            # Then
            expected_columns = ["Date de création", "Date de création du premier stock",
                                "Date de première réservation", "Nombre d’offres",
                                "Nombre de réservations non annulées"]

            offerers_details = pandas.read_sql_table('enriched_offerer_data', CONNECTION, index_col='offerer_id')
            for expected_column_in_view in expected_columns:
                assert expected_column_in_view in offerers_details.columns


from pathlib import Path
import pytest
import yaml

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from shopping_cart import data_model as m
from shopping_cart.store.operations import populate

directory = Path(__file__).parent

@pytest.fixture(name="session")
def make_store_session():
    """
    Make a new empty store database session

    """
    engine = create_engine('sqlite://')
    session = sessionmaker(bind=engine)()
    m.Base.metadata.create_all(engine)
    yield session
    session.close()

@pytest.fixture(name="product")
def make_product():
    """
    Pytest fixture to generate a product instance

    """

    def _generate(
            product_id: int,
            name: str,
            price: float
    ):
        return m.Product(
            id=product_id,
            name=name,
            unit_price=price
        )

    return _generate

@pytest.fixture(name="store")
def make_store_database(session):
    """
    Pytest fixture to add list of products
    to the store database from a mock store configuration

    """

    with open(directory / "mock_data/store.yaml") as f:
        product_dict = yaml.safe_load(f)

    populate(
        product_dict,
        session
    )

    return session


# Copyright 2019-2020 The Kale Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import warnings

from typing import Any, Dict

import nbformat as nb

from kale.config import Field
from kale import Pipeline, Step, PipelineConfig
from kale.static_analysis import dependencies, ast

# fixme: Change the name of this key to `kale_metadata`
KALE_NB_METADATA_KEY = 'kubeflow_notebook'

SKIP_TAG = r'^skip$'
IMPORT_TAG = r'^imports$'
FUNCTIONS_TAG = r'^functions$'
PREV_TAG = r'^prev:[_a-z]([_a-z0-9]*)?$'
# `step` has the same functionality as `block` and is
# supposed to be the new name
STEP_TAG = r'^step:([_a-z]([_a-z0-9]*)?)?$'
# Extension may end up with 'block:' as a tag. We handle
# that as if it was empty.
# TODO: Deprecate `block` tag in future release
BLOCK_TAG = r'^block:([_a-z]([_a-z0-9]*)?)?$'
PIPELINE_PARAMETERS_TAG = r'^pipeline-parameters$'
PIPELINE_METRICS_TAG = r'^pipeline-metrics$'
# Annotations map to actual pod annotations that can be set via KFP SDK
_segment = "[a-zA-Z0-9]+([a-zA-Z0-9-_.]*[a-zA-Z0-9])?"
K8S_ANNOTATION_KEY = "%s([/]%s)?" % (_segment, _segment)
ANNOTATION_TAG = r'^annotation:%s:(.*)$' % K8S_ANNOTATION_KEY
LABEL_TAG = r'^label:%s:(.*)$' % K8S_ANNOTATION_KEY
# Limits map to K8s limits, like CPU, Mem, GPU, ...
# E.g.: limit:nvidia.com/gpu:2
LIMITS_TAG = r'^limit:([_a-z-\.\/]+):([_a-zA-Z0-9\.]+)$'

_TAGS_LANGUAGE = [SKIP_TAG,
                  IMPORT_TAG,
                  FUNCTIONS_TAG,
                  PREV_TAG,
                  BLOCK_TAG,
                  STEP_TAG,
                  PIPELINE_PARAMETERS_TAG,
                  PIPELINE_METRICS_TAG,
                  ANNOTATION_TAG,
                  LABEL_TAG,
                  LIMITS_TAG]
# These tags are applied to every step of the pipeline
_STEPS_DEFAULTS_LANGUAGE = [ANNOTATION_TAG,
                            LABEL_TAG,
                            LIMITS_TAG]

def get_annotation_or_label_from_tag(tag_parts):
    """Get the key and value from an annotation or label tag.

    Args:
        tag_parts: annotation or label notebook tag

    Returns (tuple): key (annotation or label name), values
    """
    # Since value can be anything, merge together everything that's left.
    return tag_parts[0], "".join(tag_parts[1:])

def get_limit_from_tag(tag_parts):
    """Get the key and value from a notebook limit tag.

    Args:
        tag_parts: annotation or label notebook tag

    Returns (tuple): key (limit name), values
    """
    return tag_parts.pop(0), tag_parts.pop(0)

class NotebookConfig(PipelineConfig):
    """Config store for a notebook.

    This config extends the base pipeline config to take into account some
    small differences in the handling of a notebook.
    """
    notebook_path = Field(type=str, required=True)
    # FIXME: Probably this can be removed. The labextension passes both
    #  'experiment_name' and 'experiment', but the latter is not used in the
    #  backend.
    experiment = Field(type=dict)
    # Used in the UI to keep per-notebook state of the volumes snapshot toggle
    snapshot_volumes = Field(type=bool, default=False)
    # override from PipelineConfig: set the default value to False
    autosnapshot = Field(type=bool, default=False)

    @property
    def source_path(self):
        """Get the path to the source notebook."""
        return self.notebook_path

    def _preprocess(self, kwargs):
        kwargs["steps_defaults"] = self._parse_steps_defaults(
            kwargs.get("steps_defaults"))

    def _parse_steps_defaults(self, steps_defaults):
        """Parse common step configuration defined in the metadata."""
        result = dict()

        if not isinstance(steps_defaults, list):
            return steps_defaults

        for c in steps_defaults:
            if any(re.match(_c, c)
                   for _c in _STEPS_DEFAULTS_LANGUAGE) is False:
                raise ValueError("Unrecognized common step configuration:"
                                 " {}".format(c))

            parts = c.split(":")

            conf_type = parts.pop(0)
            if conf_type in ["annotation", "label"]:
                result_key = "{}s".format(conf_type)
                if result_key not in result:
                    result[result_key] = dict()
                key, value = get_annotation_or_label_from_tag(
                    parts)
                result[result_key][key] = value

            if conf_type == "limit":
                if "limits" not in result:
                    result["limits"] = dict()
                key, value = get_limit_from_tag(parts)
                result["limits"][key] = value
        return result

class NotebookProcessor:
    """Convert a Notebook to a Pipeline object."""

    def __init__(self,
                 nb_path: str,
                 nb_metadata_overrides: Dict[str, Any] = None,
                 skip_validation: bool = False):
        """Instantiate a new NotebookProcessor.

        Args:
            nb_path: Path to source notebook
            nb_metadata_overrides: Override notebook config settings
            skip_validation: Set to True in order to skip the notebook's
                metadata validation. This is useful in case the
                NotebookProcessor is used to parse a part of the notebook
                (e.g., retrieve pipeline metrics) and the notebook config (for
                pipeline generation) might still be invalid.
        """
        self.nb_path = os.path.expanduser(nb_path)
        self.notebook = self._read_notebook()

        nb_metadata = self.notebook.metadata.get(KALE_NB_METADATA_KEY, dict())

        # fixme: needed?
        nb_metadata.update({"notebook_path": nb_path})
        if nb_metadata_overrides:
            nb_metadata.update(nb_metadata_overrides)
        # validate and populate defaults
        # FIXME: Maybe improve this by implementing a "skip_validation" flag
        #  in the config class
        self.config = None
        if not skip_validation:
            self.config = NotebookConfig(**nb_metadata)
        self.pipeline = Pipeline(self.config)

    def _read_notebook(self):
        if not os.path.exists(self.nb_path):
            raise ValueError("NotebookProcessor could not find a notebook at"
                             " path %s" % self.nb_path)
        return nb.read(self.nb_path, as_version=nb.NO_CONVERT)

    def to_pipeline(self):
        """Convert an annotated Notebook to a Pipeline object."""
        (pipeline_parameters_source,
         pipeline_metrics_source,
         imports_and_functions) = self.parse_notebook()

        self.parse_pipeline_parameters(pipeline_parameters_source)
        self.pipeline.set_volume_pipeline_parameters()

        # get a list of variables that need to be logged as pipeline metrics
        pipeline_metrics = ast.parse_metrics_print_statements(
            pipeline_metrics_source)

        # run static analysis over the source code
        dependencies.dependencies_detection(
            self.pipeline,
            imports_and_functions=imports_and_functions
        )
        dependencies.assign_metrics(self.pipeline, pipeline_metrics)

        # if there are multiple DAG leaves, add an empty step at the end of the
        # pipeline for final snapshot
        leaf_steps = self.pipeline.get_leaf_steps()
        if self.config.autosnapshot and len(leaf_steps) > 1:
            _name = "final_auto_snapshot"
            self.pipeline.add_step(Step(name=_name, source=[]))
            # add a link from all the last steps of the pipeline to
            # the final auto snapshot one.
            for step in leaf_steps:
                self.pipeline.add_edge(step.name, _name)

        # FIXME: Move this to a base class Processor, to be executed by default
        #  after `to_pipeline`, so that it is agnostic to the type of
        #  processor.
        for step in self.pipeline.steps:
            step.config.update(self.pipeline.config.steps_defaults)

        # TODO: Additional action required:
        #  Run a static analysis over every step to check that pipeline
        #  parameters are not assigned with new values.
        return self.pipeline

    def parse_pipeline_parameters(self, source: str):
        """Get pipeline parameters from source code."""
        pipeline_parameters = ast.parse_assignments_expressions(source)
        for name, (v_type, v_value) in pipeline_parameters.items():
            pipeline_parameters[name] = ast.PipelineParam(v_type, v_value)
        self.pipeline.pipeline_parameters = pipeline_parameters

    def parse_notebook(self):
        """Creates a NetworkX graph based on the input notebook's tags.

        Cell's source code are embedded into the graph as node attributes.
        """
        # will be assigned at the end of each for loop
        prev_step_name = None

        # All the code cells that have to be pre-pended to every pipeline step
        # (i.e., imports and functions) are merged here
        imports_block = list()
        functions_block = list()

        # Variables that will become pipeline parameters
        pipeline_parameters = list()
        # Variables that will become pipeline metrics
        pipeline_metrics = list()

        for c in self.notebook.cells:
            if c.cell_type != "code":
                continue

            tags = self.parse_cell_metadata(c.metadata)

            if len(tags['step_names']) > 1:
                raise NotImplementedError("Kale does not yet support multiple"
                                          " step names in a single notebook"
                                          " cell. One notebook cell was found"
                                          " with %s  step names"
                                          % tags['step_names'])

            step_name = (tags['step_names'][0]
                         if 0 < len(tags['step_names'])
                         else None)

            if step_name == 'skip':
                # when the cell is skipped, don't store `skip` as the previous
                # active cell
                continue
            if step_name == 'pipeline-parameters':
                pipeline_parameters.append(c.source)
                prev_step_name = step_name
                continue
            if step_name == 'imports':
                imports_block.append(c.source)
                prev_step_name = step_name
                continue
            if step_name == 'functions':
                functions_block.append(c.source)
                prev_step_name = step_name
                continue
            if step_name == 'pipeline-metrics':
                pipeline_metrics.append(c.source)
                prev_step_name = step_name
                continue

            # if none of the above apply, then we are parsing a code cell with
            # a block names and (possibly) some dependencies

            # if the cell was not tagged with a step name,
            # add the code to the previous cell
            if not step_name:
                if prev_step_name == 'imports':
                    imports_block.append(c.source)
                elif prev_step_name == 'functions':
                    functions_block.append(c.source)
                elif prev_step_name == 'pipeline-parameters':
                    pipeline_parameters.append(c.source)
                elif prev_step_name == 'pipeline-metrics':
                    pipeline_metrics.append(c.source)
                # current_block might be None in case the first cells of the
                # notebooks have not been tagged.
                elif prev_step_name:
                    # this notebook cell will be merged to a previous one that
                    # specified a step name
                    self.pipeline.get_step(prev_step_name).merge_code(c.source)
            else:
                # in this branch we are sure that we are reading a code cell
                # with a step tag, so we must not allow for pipeline-metrics
                if prev_step_name == 'pipeline-metrics':
                    raise ValueError("Tag pipeline-metrics must be placed on a"
                                     " cell at the end of the Notebook."
                                     " Pipeline metrics should be considered"
                                     " as a result of the pipeline execution"
                                     " and not of single steps.")
                # add node to DAG, adding tags and source code of notebook cell
                if step_name not in self.pipeline.nodes:
                    step = Step(name=step_name, source=[c.source],
                                ins=set(), outs=set(),
                                limits=tags.get("limits", {}),
                                labels=tags.get("labels", {}),
                                annotations=tags.get("annotations", {}))
                    self.pipeline.add_step(step)
                    for _prev_step in tags['prev_steps']:
                        if _prev_step not in self.pipeline.nodes:
                            raise ValueError("Step %s does not exist. It was "
                                             "defined as previous step of %s"
                                             % (
                                                 _prev_step,
                                                 tags['step_names']))
                        self.pipeline.add_edge(_prev_step, step_name)
                else:
                    self.pipeline.get_step(step_name).merge_code(c.source)

                prev_step_name = step_name

        # Prepend any `imports` and `functions` cells to every Pipeline step
        for step in self.pipeline.steps:
            step.source = imports_block + functions_block + step.source

        # merge together pipeline parameters
        pipeline_parameters = '\n'.join(pipeline_parameters)
        # merge together pipeline metrics
        pipeline_metrics = '\n'.join(pipeline_metrics)

        imports_and_functions = "\n".join(imports_block + functions_block)
        return pipeline_parameters, pipeline_metrics, imports_and_functions

    def parse_cell_metadata(self, metadata):
        """Parse a notebook's cell's metadata field.

        The Kale UI writes specific tags inside the 'tags' field, as a list
        of string tags. Supported tags are defined by _TAGS_LANGUAGE.

        Args:
            metadata (dict): a dict containing a notebook's cell's metadata

        Returns (dict): parsed tags based on Kale tagging language

        """
        parsed_tags = dict()

        # `step_names` is a list because a notebook cell might be assigned to
        # more than one Pipeline step.
        parsed_tags['step_names'] = list()
        parsed_tags['prev_steps'] = list()
        # define intermediate variables so that dicts are not added to a steps
        # when they are empty
        cell_annotations = dict()
        cell_labels = dict()
        cell_limits = dict()

        # the notebook cell was not tagged
        if 'tags' not in metadata or len(metadata['tags']) == 0:
            return parsed_tags

        for t in metadata['tags']:
            if not isinstance(t, str):
                raise ValueError("Tags must be string. Found tag %s of type %s"
                                 % (t, type(t)))
            # Check that the tag is defined by the Kale tagging language
            if any(re.match(_t, t) for _t in _TAGS_LANGUAGE) is False:
                raise ValueError("Unrecognized tag: {}".format(t))

            # Special tags have a specific effect on the cell they belong to.
            # Specifically:
            #  - skip: ignore the notebook cell
            #  - pipeline-parameters: use the cell to populate Pipeline
            #       parameters. The cell must contain only assignment
            #       expressions
            #  - pipeline-metrics: use the cell to populate Pipeline metrics.
            #       The cell must contain only variable names
            #  - imports: the code of the corresponding cell(s) will be
            #       prepended to every Pipeline step
            #  - functions: same as imports, but the corresponding code is
            #       placed **after** `imports`
            special_tags = ['skip', 'pipeline-parameters', 'pipeline-metrics',
                            'imports', 'functions']
            if t in special_tags:
                parsed_tags['step_names'] = [t]
                return parsed_tags

            # now only `block|step` and `prev` tags remain to be parsed.
            tag_parts = t.split(':')
            tag_name = tag_parts.pop(0)

            if tag_name == "annotation":
                key, value = get_annotation_or_label_from_tag(tag_parts)
                cell_annotations.update({key: value})

            if tag_name == "label":
                key, value = get_annotation_or_label_from_tag(tag_parts)
                cell_labels.update({key: value})

            if tag_name == "limit":
                key, value = get_limit_from_tag(tag_parts)
                cell_limits.update({key: value})

            # name of the future Pipeline step
            # TODO: Deprecate `block` in future release
            if tag_name in ["block", "step"]:
                if tag_name == "block":
                    warnings.warn("`block` tag will be deprecated in a future"
                                  " version, use `step` tag instead",
                                  DeprecationWarning)
                step_name = tag_parts.pop(0)
                parsed_tags['step_names'].append(step_name)
            # name(s) of the father Pipeline step(s)
            if tag_name == "prev":
                prev_step_name = tag_parts.pop(0)
                parsed_tags['prev_steps'].append(prev_step_name)

        if not parsed_tags['step_names'] and parsed_tags['prev_steps']:
            raise ValueError(
                "A cell can not provide `prev` annotations without "
                "providing a `block` or `step` annotation as well")

        if cell_annotations:
            if not parsed_tags['step_names']:
                raise ValueError(
                    "A cell can not provide Pod annotations in a cell"
                    " that does not declare a step name.")
            parsed_tags['annotations'] = cell_annotations

        if cell_limits:
            if not parsed_tags['step_names']:
                raise ValueError(
                    "A cell can not provide Pod resource limits in a"
                    " cell that does not declare a step name.")
            parsed_tags['limits'] = cell_limits
        return parsed_tags

    def get_pipeline_parameters_source(self):
        """Get just pipeline parameters cells from the notebook.

        Returns (str): pipeline parameters source code
        """
        return self._get_reserved_tag_source(PIPELINE_PARAMETERS_TAG)

    def get_pipeline_metrics_source(self):
        """Get just pipeline metrics cells from the notebook.

        Returns (str): pipeline metrics source code
        """
        # check that the pipeline metrics tag is only assigned to cells at
        # the end of the notebook
        detected = False
        tags = _TAGS_LANGUAGE[:]
        tags.remove(PIPELINE_METRICS_TAG)

        for c in self.notebook.cells:
            # parse only source code cells
            if c.cell_type != "code":
                continue

            # if we see a pipeline-metrics tag, set the flag
            if (('tags' in c.metadata
                 and len(c.metadata['tags']) > 0
                 and any(re.match(PIPELINE_METRICS_TAG, t)
                         for t in c.metadata['tags']))):
                detected = True
                continue

            # if we have the flag set and we detect any other tag from the tags
            # language, then raise error
            if (detected
                and 'tags' in c.metadata
                and len(c.metadata['tags']) > 0
                and any([any(re.match(tag, t) for t in c.metadata['tags'])
                         for tag in tags])):
                raise ValueError(
                    "Tag pipeline-metrics tag must be placed on a "
                    "cell at the end of the Notebook."
                    " Pipeline metrics should be considered as a"
                    " result of the pipeline execution and not of"
                    " single steps.")
        return self._get_reserved_tag_source(PIPELINE_METRICS_TAG)

    def _get_reserved_tag_source(self, search_tag):
        """Get just the specific tag's source code.

        When searching for tag x, will return all cells that are tagged with x
        and, if untagged, follow cells with tag x. The result is a multiline
        string containing all the python code associated to x.

        Note: This is designed for 'special' tags, as the STEP_TAG is excluded
              from the match.

        Args:
            search_tag (str): the target tag

        Returns: the unified code of all the cells belonging to `search_tag`
        """
        detected = False
        source = ''

        language = _TAGS_LANGUAGE[:]
        language.remove(search_tag)

        for c in self.notebook.cells:
            # parse only source code cells
            if c.cell_type != "code":
                continue
            # in case the previous cell was a `search_tag` cell and this
            # cell is not any other tag of the tag language:
            if (detected
                and (('tags' not in c.metadata
                      or len(c.metadata['tags']) == 0)
                     or all([not any(re.match(tag, t)
                                     for t in c.metadata['tags'])
                            for tag in language]))):
                source += '\n' + c.source
            elif (('tags' in c.metadata
                   and len(c.metadata['tags']) > 0
                   and any(re.match(search_tag, t)
                           for t in c.metadata['tags']))):
                source += '\n' + c.source
                detected = True
            else:
                detected = False
        return source.strip()


import pygame
from Settings import Settings
from Ship import Ship
from Bullet import Bullet
from pygame.sprite import Group

class Game():
    def __init__(self):
        """Initialize game."""

        self.settings = Settings()
        self.initScreen()

        self.ship = Ship(self.screen)

        self.bullets = Group()

        # Startup app sound
        pygame.mixer.music.load('sounds/start.mp3')
        pygame.mixer.music.play(0)

    def draw(self):
        # Draw Background & a rectangle
        self.screen.fill(self.settings.bg_color)
        pygame.draw.rect(self.screen, (60, 60, 180), self.small_screen_rect )

        # Draw Ship
        self.ship.draw()

        for bullet in self.bullets:
            bullet.draw()

        # Make the most recently Drawn screen visible.
        pygame.display.flip()

    def update(self):
        # Draw Background
        self.ship.update()
        for bullet in self.bullets:
            bullet.update()

        for bullet in self.bullets.copy():  # .copy() is used in order to use bullets.remove above
            if bullet.rect.bottom <= 0:
                self.bullets.remove(bullet)

    def processKeyDown(self, key):
        if key == pygame.K_RIGHT:
            self.ship.moveRight(True)
        if key == pygame.K_LEFT:
            self.ship.moveLeft(True)
        if key == pygame.K_SPACE:
            new_bullet = Bullet(self.settings, self.screen, self.ship)
            self.bullets.add(new_bullet)

    def processKeyUp(self, key):
        if key == pygame.K_RIGHT:
            self.ship.moveRight(False)
        if key == pygame.K_LEFT:
            self.ship.moveLeft(False)

    def initScreen(self):
        self.screen = pygame.display.set_mode((self.settings.screen_width, self.settings.screen_height))
        self.screen_rect = self.screen.get_rect()
        self.small_screen_rect = pygame.Rect(30, 30, self.settings.screen_width - 60, self.settings.screen_height - 60 )


#!/bin/python
import gzip
import argparse
import re
import numpy as np
from itertools import islice

#-----------------------------------------------------------------------
# The objective of this function is to demultiplex Illumina reads that are
# dual-indexed. The script can identify low quality score reads and
# index hopping. Script will produce R1 and R2 fastq files for each known
# index, unknow records, and index hopped records.
#-----------------------------------------------------------------------

#Set input variables
def get_args():
    parser = argparse.ArgumentParser(description="Generate a per base \
    distribution of quality scores for a file")
    parser.add_argument("-f1", "--R1_file", help="R1 File name", type=str)
    parser.add_argument("-f2", "--R2_file", help="R2 File name", type=str)
    parser.add_argument("-f3", "--R3_file", help="R3 File name", type=str)
    parser.add_argument("-f4", "--R4_file", help="R4 File name", type=str)
    parser.add_argument("-i", "--index_file", help="index file name", type=str)
    parser.add_argument("-sq", "--seq_cutoff", help="minimum average sequence quality score cut off", type=int)
    parser.add_argument("-iq", "--index_cutoff", help="minimum average index quality score cut off", type=int)
    return parser.parse_args()

args = get_args()
#Given Files
R1_seq_file = args.R1_file
R1_index_file = args.R2_file
R2_index_file = args.R3_file
R2_seq_file = args.R4_file
index_file = args.index_file

#Minimum quality score input values
seq_qual_score = args.seq_cutoff # minimum sequence quality score cut off
index_qual_score = args.index_cutoff # minimum index quality score cut off

#  Will be used to check qscore.
def convert_phred(letter):
    '''Converts a single character into a phred score'''
    if letter != 'N':
        val = ord(letter)
        val = val - 33
    else:
        val = 0
    return val

#Used in reverse complement function
def reverse(string):
    '''Reverse a string. Used in reverse_complement'''
    str = ""
    for char in string:
        str = char + str
    return str

# Dictionary used in reverse_complement function, defined dictionary outside
# of function to maximize efficiency.
complement_dict = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A', 'N': 'N'}

def reverse_complement(sequence):
    '''Finds the reverse complement of a given string'''
    temp_string = str()
    for character in sequence:
        temp_string += complement_dict[character]
    return reverse(temp_string) #Returning the reverse of the complement

def write_record(file_handle, record_array, barcode1, barcode2):
    '''Writes a record to the corresponding file'''
    #File names stored as the value of the dictionary
    header_line = record_array[0] + "_" + barcode1 + "_" + barcode2
    file_handle.write(header_line + '\n')
    file_handle.write(record_array[1] + '\n')
    file_handle.write(record_array[2] + '\n')
    file_handle.write(record_array[3] + '\n')

def check_index_quality(index_string, seq):
    '''Checks if the index q-scores are above a specified value'''
    sum_q_array = []
    bool_val = False
    length = len(index_string)
    for x in range(length):
        sum_q_array.append(0)
    counter = 0
    for letter in index_string:
        sum_q_array[counter] += (convert_phred(letter))
        counter+=1
    numpy_array = np.array(sum_q_array)
    mean = numpy_array.mean()
    if mean >= index_qual_score: #Makes sure that the quality score is above a average of 28
        bool_val = True
    if 'N' in seq:
        bool_val = False
    return bool_val

# This function checks the sequence quality score of every record in order to
# ensure only high quality reads are processed. The minimum quality score can be
# passed in through argparse.
def check_seq_quality(seq):
    '''Checks if the sequences average q-score is above a specific value'''
    sum_q_array = []
    length = len(seq)
    for x in range(length):
        sum_q_array.append(0)
    bool_val = False
    counter = 0
    for letter in seq:
        sum_q_array[counter] += (convert_phred(letter))
        counter+=1
    numpy_array = np.array(sum_q_array)
    mean = numpy_array.mean()
    if mean >= seq_qual_score: #Makes sure that the average quality score is 35 or above
        bool_val = True
    return bool_val

# This function checks if the index is in the given index list
def valid_barcode(barcode, barcode_dict):
    '''Checks to see if the sequenced barcode is in the given barcode list'''
    bool_val = False
    if barcode in barcode_dict:
        bool_val = True
    return bool_val

def read_record(file_handle):
    '''Reads a record from a file and outputs it as a list'''
    rec_array = []
    record_object = islice(file_handle, 4) #islice returns an iterable object
    rec_list = list(record_object)
    for line in rec_list:
        line=line.strip()
        rec_array.append(line)
    return(rec_array)

#Dictionaries used to store the indexes
R1_dict = dict()
R2_dict = dict()

#Dictionaries used to store the file handles
R1_fh = dict()
R2_fh = dict()

#Dictionary to keep track of the counts of each index
index_count_dict = dict()

#Read index file, create dictionary with key being indexes and value being file name
with open("/projects/bgmp/wyatte/Bi622/indexes.txt") as fh:
    fh.readline()
    for line in fh:
        line = line.strip().split()
        index = line[4]
        R1_dict[index] = index + "_R1.fq"
        R2_dict[index] = index + "_R2.fq"

# OPEN ALL WRITE FILES
#Open every record file for R1 files
for key in R1_dict.keys():
    file_handle_R1 =(key + "_R1_file")
    file_handle_R1 = open(R1_dict[key], "a")
    R1_fh[key] = file_handle_R1
#Open every record file for R2 files
for key in R2_dict.keys():
    file_handle_R2 =(key + "_R2_file")
    file_handle_R2 = open(R2_dict[key], "a")
    R2_fh[key] = file_handle_R2
Unknown_R1_fh=open("Unknown_R1.fq", "a")
Unknown_R2_fh=open("Unknown_R2.fq", "a")
Hopped_R1_fh=open("Index_Hopped_R1.fq","a")
Hopped_R2_fh=open("Index_Hopped_R2.fq","a")

# Files to output Unknown & Index Hopped reads
Unknown_R1 = "Unknown_R1.fq"
Unknown_R2 = "Unknown_R2.fq"
Hopped_R1 = "Index_Hopped_R1.fq"
Hopped_R2 = "Index_Hopped_R2.fq"

#Body of code
with gzip.open(R1_seq_file, 'rt') as R1_seq, gzip.open(R1_index_file, 'rt') as R1_index, gzip.open(R2_index_file, 'rt') as R2_index, gzip.open(R2_seq_file, 'rt') as R2_seq:
    line = "Hi"
    read_pairs_val = 0
    index_hopped_val = 0
    unknown_pairs_val = 0
    total_recs = 0

    while True:
        seq_R1_qlty = False #Used to check if the quality of the sequence is above standard
        seq_R2_qlty = False
        index_R1_qlty = False #Used to check if the quality of the index is above standard
        index_R2_qlty = False

        #Checks to see if the indexes are valid
        Valid_Index_R1 = False
        Valid_Index_R2 = False

        #Arrays to store a record from each file
        R1_seq_array = []
        R1_index_array = []
        R2_seq_array = []
        R2_index_array = []

        #Read in 4 lines from each file at a time
        R1_seq_array = read_record(R1_seq)
        R1_index_array = read_record(R1_index)
        R2_index_array= read_record(R2_index)
        R2_seq_array = read_record(R2_seq)

        #Checks to see if its the end of the file by checking the length of the records read in
        if (len(R1_seq_array) + len(R1_index_array) + len(R2_seq_array) + len(R2_index_array)) < 16:
            break

        # Checking to see if the quality scores for the indexes and sequences are good
        seq_R1_qlty = check_seq_quality(R1_seq_array[3])
        seq_R2_qlty = check_seq_quality(R2_seq_array[3])
        index_R1_qlty = check_index_quality(R1_index_array[3], R1_index_array[1])
        index_R2_qlty = check_index_quality(R2_index_array[3], R2_index_array[1])

        #Increase the of records to keep track of the total
        total_recs += 1

        # Checking to see if the indexes are valid
        Valid_Index_R1 = valid_barcode(R1_index_array[1], R1_dict)
        Valid_Index_R2 = valid_barcode(reverse_complement(R2_index_array[1]), R2_dict)

        #Chunk of code that filters through the records and prints it to the right file
        if (seq_R1_qlty & seq_R2_qlty & index_R1_qlty & index_R2_qlty & Valid_Index_R1 & Valid_Index_R2):
            #If the quality of all 2 sequences and 2 indexes are good and R2 is the reverse complement of R1
            if R1_index_array[1] == reverse_complement(R2_index_array[1]):
                read_pairs_val += 1
                write_record(R1_fh[R1_index_array[1]], R1_seq_array, R1_index_array[1], R2_index_array[1])
                write_record(R2_fh[reverse_complement(R2_index_array[1])], R2_seq_array, R2_index_array[1], R1_index_array[1])
                #Increment dictionary to keep count of each index
                if R1_index_array[1] in index_count_dict.keys():
                    index_count_dict[R1_index_array[1]] += 1
                else:
                    index_count_dict.setdefault(R1_index_array[1],1)

            #Index hopped occurs when R2 is not the reverse complement of R1
            else:
                index_hopped_val += 1
                write_record(Hopped_R1_fh, R1_seq_array, R1_index_array[1], R2_index_array[1])
                write_record(Hopped_R2_fh, R2_seq_array, R2_index_array[1], R1_index_array[1])

        # If a sequence or index does not pass the quality check, both records are written to the unknown file
        else:
            unknown_pairs_val += 1
            write_record(Unknown_R1_fh, R1_seq_array, R1_index_array[1], R2_index_array[1])
            write_record(Unknown_R2_fh, R2_seq_array, R2_index_array[1], R1_index_array[1])

    #print summary report
    print("Total number of records processed: " + str(total_recs))
    print("Number of read pairs with properly matched indexes: " + str(read_pairs_val))
    print("Number of read pairs with index hopping observed: " + str(index_hopped_val))
    print("Percent of records where index hopping was observed: " + str((index_hopped_val/total_recs)*100) + "%")
    print("Number of read pairs with unknown indexes: " + str(unknown_pairs_val))
    print("Percent of records with unknown indexes: " + str((unknown_pairs_val/total_recs)*100) + "%")
    print("-------------------------------------------------------------")
    print("INDEX | Count of Index Reads | Percentage of Index Reads")
    for key in index_count_dict.keys():
        print(str(key) + " | Count: " + str(index_count_dict[key]) + " | " + str((index_count_dict[key]/total_recs)*100) + "%")

    #Closing Files
    R1_seq.close()
    R1_index.close()
    R2_seq.close()
    R2_index.close()

# CLOSE ALL WRITE FILES
#Close every record file for R1 files
for key in R1_fh.keys():
    R1_fh[key].close()
#Close every record file for R2 files
for key in R2_dict.keys():
    R2_fh[key].close()
Unknown_R1_fh.close()
Unknown_R2_fh.close()
Hopped_R1_fh.close()
Hopped_R2_fh.close()


from utils import *
import numpy as np
import torch
import argparse

parser = argparse.ArgumentParser(description='mechRNN')
parser.add_argument('--epoch', type=int, default=4, help='number of epochs')
parser.add_argument('--lr', type=float, default=0.05, help='learning rate')
parser.add_argument('--delta_t', type=float, default=0.1, help='time step of simulation')
parser.add_argument('--t_train', type=float, default=100, help='length of train simulation')
# parser.add_argument('--n_train_points', type=float, default=None, help='total number of train+testing data points. Default is to have this setting inactive (i.e None)')
parser.add_argument('--t_test', type=float, default=30, help='length of test simulation')
parser.add_argument('--t_test_synch', type=float, default=10, help='length of test simulation')
parser.add_argument('--savedir', type=str, default='default_output_dT', help='parent dir of output')
parser.add_argument('--model_solver', default=lorenz63, help='ode function')
parser.add_argument('--drive_system', type=str2bool, default=False, help='whether to force the system with a time-dependent driver')
parser.add_argument('--n_tests', type=int, default=1, help='number of independent testing sets to use')
parser.add_argument('--n_experiments', type=int, default=1, help='number of sim/fitting experiments to do')
parser.add_argument('--random_state_inits', type=str2bool, default=True, help='whether to randomly initialize initial conditions for simulated trajectory')
parser.add_argument('--compute_kl', type=str2bool, default=False, help='whether to compute KL divergence between test set density and predicted density')
parser.add_argument('--continue_trajectory', type=str2bool, default=False, help='if true, ignore n_tests and synch length. Instead, simply continue train trajectory into the test trajectory.')
parser.add_argument('--mxstep', type=int, default=0, help='max number of solver steps. 0 lets solver use defaults.')
FLAGS = parser.parse_args()

def main():
	(a,b,c) = [10, 28, 8/3]
	if FLAGS.random_state_inits:
		# sampel uniform initial conditions
		(xmin, xmax) = (-10,10)
		(ymin, ymax) = (-20,30)
		(zmin, zmax) = (10,40)

		xrand = xmin+(xmax-xmin)*np.random.random()
		yrand = ymin+(ymax-ymin)*np.random.random()
		zrand = zmin+(zmax-zmin)*np.random.random()
		my_state_inits = [[xrand, yrand, zrand]]
	else:
		my_state_inits = [[-5, 0, 30]]

	if FLAGS.continue_trajectory:
		FLAGS.n_tests = 1
		FLAGS.t_test_synch = 0
	else:
		if FLAGS.t_test_synch < FLAGS.delta_t:
			raise ValueError('t_test_synch (synch-length) must be larger than delta_t step size')

	lr = FLAGS.lr # learning rate
	delta_t = FLAGS.delta_t

	# tspan_train = np.arange(0,FLAGS.t_train,delta_t)  #np.arange(0,10000,delta_t)
	sim_model = FLAGS.model_solver
	rnn_sim_model = FLAGS.model_solver

	drive_system = FLAGS.drive_system #False

	n_epochs = FLAGS.epoch #1

	# train_frac = FLAGS.train_frac #0.9995
	i = 0
	for state_init in my_state_inits:
		i += 1
		init_output_dir = FLAGS.savedir + '_output' + str(i)

		for t_train in np.random.permutation([10, 50, 100, 500]):
			all_dirs = []
			tspan_train = np.arange(0,t_train,delta_t)
			tspan_test = np.arange(0,(FLAGS.t_test_synch+FLAGS.t_test),delta_t)  #np.arange(0,10000,delta_t)
			ntsynch = int(FLAGS.t_test_synch/delta_t)
			sim_model_params = {'state_names': ['x','y','z'], 'state_init':state_init, 'delta_t':delta_t, 'smaller_delta_t': min(delta_t, delta_t), 'ode_params':(a, b, c), 'time_avg_norm':0.529, 'mxstep':FLAGS.mxstep}
			rnn_model_params = {'state_names': ['x','y','z'], 'state_init':state_init, 'delta_t':delta_t, 'smaller_delta_t': min(delta_t, delta_t), 'ode_params':(a, b, c), 'time_avg_norm':0.529, 'mxstep':FLAGS.mxstep}

			output_dir = init_output_dir + '/Ndata_{0}'.format(t_train)

			# simulate clean and noisy data
			(input_data_train, y_clean_train, y_noisy_train,
			y_clean_test_vec, y_noisy_test_vec, x_test_vec) = make_RNN_data2(
				sim_model, tspan_train, tspan_test, sim_model_params,
				noise_frac=0.05, output_dir=output_dir, drive_system=False,
				n_test_sets=FLAGS.n_tests,
				f_get_state_inits=get_lorenz_inits,
				continue_trajectory=FLAGS.continue_trajectory)

			###### do train/test split #######
			# n_train = int(np.floor(train_frac*len(y_clean)))
			# y_clean_train = y_clean[:n_train]
			# y_clean_test = y_clean[n_train:]
			# y_noisy_train = y_noisy[:n_train]
			# y_noisy_test = y_noisy[n_train:]
			# x_train = input_data[:, :n_train]
			# x_test = input_data[:, n_train:]
			# y_list = [y_clean_train, y_noisy_train, y_clean_test, y_noisy_test]

			####### collect normalization information from TRAINING SET ONLY ######
			normz_info_clean = {}
			normz_info_clean['Ymax'] = np.max(y_clean_train,axis=0)
			normz_info_clean['Ymin'] = np.min(y_clean_train,axis=0)
			normz_info_clean['Ymean'] = np.mean(y_clean_train)
			normz_info_clean['Ysd'] = np.std(y_clean_train)
			# normz_info_clean['Xmean'] = np.mean(x_train)
			# normz_info_clean['Xsd'] = np.std(x_train)

			normz_info_noisy = {}
			normz_info_noisy['Ymax'] = np.max(y_noisy_train,axis=0)
			normz_info_noisy['Ymin'] = np.min(y_noisy_train,axis=0)
			normz_info_noisy['Ymean'] = np.mean(y_noisy_train)
			normz_info_noisy['Ysd'] = np.std(y_noisy_train)
			# normz_info_noisy['Xmean'] = np.mean(x_train)
			# normz_info_noisy['Xsd'] = np.std(x_train)

			normz_info = normz_info_clean
			y_clean_train_norm = f_normalize_minmax(normz_info,y_clean_train)
			y_noisy_train_norm = f_normalize_minmax(normz_info,y_noisy_train)
			y_clean_test_vec_norm = np.copy(y_clean_test_vec[:,ntsynch:,:])
			y_noisy_test_vec_norm = np.copy(y_noisy_test_vec[:,ntsynch:,:])
			# initialize testSynch vec with correct dimensions
			if FLAGS.continue_trajectory:
				y_clean_testSynch_vec_norm = np.copy(y_clean_test_vec[:,0,None])
				y_noisy_testSynch_vec_norm = np.copy(y_noisy_test_vec[:,0,None])
			else:
				y_clean_testSynch_vec_norm = np.copy(y_clean_test_vec[:,:ntsynch,:])
				y_noisy_testSynch_vec_norm = np.copy(y_noisy_test_vec[:,:ntsynch,:])
			for k in range(FLAGS.n_tests):
				y_clean_test_vec_norm[k,:,:] = f_normalize_minmax(normz_info, y_clean_test_vec[k,ntsynch:,:])
				y_noisy_test_vec_norm[k,:,:] = f_normalize_minmax(normz_info, y_noisy_test_vec[k,ntsynch:,:])
				if FLAGS.continue_trajectory:
					# in this case, just use the last training element as the synch element
					y_clean_testSynch_vec_norm[k,:,:] = y_clean_train_norm[-1,:]
					y_noisy_testSynch_vec_norm[k,:,:] = y_noisy_train_norm[-1,:]
				else:
					y_clean_testSynch_vec_norm[k,:,:] = f_normalize_minmax(normz_info, y_clean_test_vec[k,:ntsynch,:])
					y_noisy_testSynch_vec_norm[k,:,:] = f_normalize_minmax(normz_info, y_noisy_test_vec[k,:ntsynch,:])

			########## NOW start running RNN fits ############
			for n in range(FLAGS.n_experiments):
				for hidden_size in [50]:
					#### run vanilla RNN ####
					forward = forward_chaos_pureML
					# train on clean data
					run_output_dir = output_dir + '/iter{0}'.format(n) + '/vanillaRNN_clean_hs{0}'.format(hidden_size)
					all_dirs.append(run_output_dir)
					if not os.path.exists(run_output_dir+'/rnn_fit_ode_TEST_{0}.png'.format(FLAGS.n_tests-1)):
						# torch.manual_seed(0)
						train_chaosRNN(forward,
							y_clean_train_norm, y_clean_train_norm,
							y_clean_test_vec_norm, y_noisy_test_vec_norm,
							y_clean_testSynch_vec_norm, y_clean_testSynch_vec_norm,
							rnn_model_params, hidden_size, n_epochs, lr,
							run_output_dir, normz_info, rnn_sim_model,
							stack_hidden=False, stack_output=False,
							compute_kl=FLAGS.compute_kl)

				#### run mechRNN w/ BAD parameter ###
				forward = forward_chaos_hybrid_full

				for eps_badness in [0.05]:
					rnn_BAD_model_params = {'state_names': ['x','y','z'], 'state_init':state_init, 'delta_t':delta_t, 'smaller_delta_t': min(delta_t, delta_t), 'ode_params':(a, b*(1+eps_badness), c), 'time_avg_norm':0.529, 'mxstep':FLAGS.mxstep}

					# train on clean data
					run_output_dir = output_dir + '/iter{0}'.format(n) + '/mechRNN_epsBadness{0}_clean_hs{1}'.format(eps_badness, hidden_size)
					all_dirs.append(run_output_dir)

					if not os.path.exists(run_output_dir+'/rnn_fit_ode_TEST_{0}.png'.format(FLAGS.n_tests-1)):
						# torch.manual_seed(0)
						train_chaosRNN(forward,
							y_clean_train_norm, y_clean_train_norm,
							y_clean_test_vec_norm, y_noisy_test_vec_norm,
							y_clean_testSynch_vec_norm, y_clean_testSynch_vec_norm,
							rnn_BAD_model_params, hidden_size, n_epochs, lr,
							run_output_dir, normz_info_clean, rnn_sim_model,
							compute_kl=FLAGS.compute_kl)

					# GP ONLY
					for gp_style in [1,2,3,4]:
						run_output_dir = output_dir + '/iter{0}'.format(n) + '/hybridGPR{2}_epsBadness{0}_clean_hs{1}'.format(eps_badness, hidden_size, gp_style)
						all_dirs.append(run_output_dir)
						if not os.path.exists(run_output_dir+'/fit_ode_TEST_{0}.png'.format(FLAGS.n_tests-1)):
							# torch.manual_seed(0)
							train_chaosRNN(forward,
								y_clean_train_norm, y_clean_train_norm,
								y_clean_test_vec_norm, y_noisy_test_vec_norm,
								y_clean_testSynch_vec_norm, y_clean_testSynch_vec_norm,
								rnn_BAD_model_params, hidden_size, n_epochs, lr,
								run_output_dir, normz_info_clean, rnn_sim_model,
								compute_kl=FLAGS.compute_kl, gp_only=True, gp_style=gp_style)

			# plot comparative training errors
			my_dirs = [d for d in all_dirs if "clean" in d]
			compare_fits(my_dirs, output_fname=output_dir+'/model_comparisons_clean')
			# extract_epsilon_performance(my_dirs, output_fname=output_dir+'/epsilon_comparison_clean')
			# compare_fits([d for d in all_dirs if "noisy" in d], output_fname=output_dir+'/model_comparisons_noisy')

if __name__ == '__main__':
	main()


from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton
from aiogram.utils.callback_data import CallbackData

files_call = CallbackData('file_btns', 'format_file', 'link')

def get_formats(formats_lst: list, link: str):
    markup = InlineKeyboardMarkup()

    for elem in formats_lst:
        markup.insert(
            InlineKeyboardButton(
                text=elem,
                callback_data=files_call.new(format_file=elem, link=link)
            )
        )
    return markup

languages_call = CallbackData('languages', 'language', 'link', 'abbr')

def get_language(languages_lst: list, link: str, abbr_lst):
    markup = InlineKeyboardMarkup()

    for elem in range(len(languages_lst)):
        lang = languages_lst[elem]
        abbr = abbr_lst[elem]
        markup.insert(
            InlineKeyboardButton(
                text=lang,
                callback_data=languages_call.new(language=elem, link=link, abbr=abbr)
            )
        )
    return markup


import numpy as np
import keras.models
from scipy.misc import imread, imresize

json_file = open('model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = keras.models.model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("model.h5")
print("Loaded Model from disk")

# compile and evaluate loaded model
loaded_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# loss,accuracy = model.evaluate(X_test,y_test)
# print('loss:', loss)
# print('accuracy:', accuracy)
x = imread('output.png', mode='L')
x = np.invert(x)
x = imresize(x, (28, 28))
import matplotlib.pyplot as plt

plt.imshow(x)
plt.show()
# imshow(x)
x = x.reshape(1, 28, 28, 1)

out = loaded_model.predict(x)
print(out)
print(np.argmax(out, axis=1))


import serial, sys

# open serial
ser = serial.Serial('/dev/ttyUSB0', 115200)

# get data
data = ser.readline()

# convert to decimal
data = int(data, 16)

# convert celsius to fahrenheit
data = 9.0/5.0 * data + 32

# store
data = str(data)
f = open('temp_data', 'w')
f.write(data)

f.close()
ser.close()


from django.shortcuts import render
from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from events.forms import *
from events.models import *
from django.core.context_processors import csrf
import datetime
from django.core.paginator import Paginator
from django.core.paginator import PageNotAnInteger
from django.core.paginator import EmptyPage
from django.shortcuts import get_object_or_404
from django.template import RequestContext
from django.shortcuts import HttpResponseRedirect

@login_required(login_url='/accounts/login/')
def add_event(request):
    args = {}
    args.update(csrf(request))
    user = request.user
    args['user'] = user
    if request.method == 'POST':
        form = CreateEventForm(request.POST)
        args['form'] = form
        if form.is_valid():
            event = form.save(commit=False)
            event.owner = request.user
            if form.cleaned_data['participation']:
                event.participants.add(user)
            event.save()
            return HttpResponseRedirect('/events/event/' + event.slug)
    else:
        args['form'] = CreateEventForm()

    return render_to_response('add_event.html', args)

def events_ordered_by_creation_date(request):
    dbase = Event
    user = request.user
    all_events = dbase.objects.order_by('-creation_date')
    paginator = Paginator(all_events, 10)
    page = request.GET.get('page')
    order = 'creation'
    try:
        all_events_pagination = paginator.page(page)
    except PageNotAnInteger:
        all_events_pagination = paginator.page(1)
    except EmptyPage:
        all_events_pagination = paginator.page(paginator.num_pages)

    return render_to_response('all_events.html', {'pagination': all_events_pagination, 'user': user, 'order': order})

def events_ordered_by_event_date(request):
    dbase = Event
    user = request.user
    all_events = dbase.objects.order_by('date', 'time')
    paginator = Paginator(all_events, 10)
    page = request.GET.get('page')
    order = 'date'
    try:
        all_events_pagination = paginator.page(page)
    except PageNotAnInteger:
        all_events_pagination = paginator.page(1)
    except EmptyPage:
        all_events_pagination = paginator.page(paginator.num_pages)

    return render_to_response('all_events.html', {'pagination': all_events_pagination, 'user': user, 'order': order})

def event_details(request, slug):
    args = {}
    event = get_object_or_404(Event, slug=slug)
    args['event'] = event
    participants = event.participants.all()
    args['participants'] = participants
    database = Comment
    comment = database.objects.all().filter(event=event)
    args['comment'] = comment
    args['button'] = 'Dodaj'
    if request.user.is_authenticated():
        args.update(csrf(request))
        if request.method == 'POST':
            comment_form = CommentForm(request.POST)
            args['comment_form'] = comment_form
            if comment_form.is_valid():
                comment_form = comment_form.save(commit=False)
                comment_form.author = request.user
                comment_form.event = event
                comment_form.save()
        else:
            args['comment_form'] = CommentForm()

    return render_to_response('event_detail.html', args, context_instance=RequestContext(request))

@login_required(login_url='/accounts/login/')
def event_edit(request, slug):
    args = {}
    args.update(csrf(request))
    user = request.user
    args['user'] = user
    event = get_object_or_404(Event, slug=slug)
    args['event'] = event
    if request.POST:
        form = CreateEventForm(request.POST, instance=event)
        if form.is_valid():
            edited = form.save(commit=False)
            if form.cleaned_data['participation']:
                event.participants.add(user)
            edited.save()
            return HttpResponseRedirect('/events/event/' + slug)
    else:
        form = CreateEventForm(instance=event)
    args['form'] = form
    return render_to_response('edit_event.html', args, context_instance=RequestContext(request))

@login_required(login_url='/accounts/login/')
def event_join(request, slug):
    user = request.user
    event = get_object_or_404(Event, slug=slug)
    event.participants.add(user)
    event.save()

    return HttpResponseRedirect('/events/event/' + slug)

@login_required(login_url='/accounts/login/')
def event_leave(request, slug):
    user = request.user
    event = get_object_or_404(Event, slug=slug)
    event.participants.remove(user)
    event.save()

    return HttpResponseRedirect('/events/event/' + slug)

@login_required(login_url='/accounts/login/')
def event_delete(request, slug):
    event = get_object_or_404(Event, slug=slug)
    event.delete()

    return HttpResponseRedirect('/events/all_events/')

# comment_edit shows same page as event_detail, but changes form only to edit chosen comment
@login_required(login_url='/accounts/login/')
def comment_edit(request, id, slug):
    args = {}
    event = get_object_or_404(Event, slug=slug)
    args['event'] = event
    participants = event.participants.all()
    args['participants'] = participants
    database = Comment
    comment = database.objects.all().filter(event=event)
    args['comment'] = comment
    editable_comment = database.objects.get(pk=id)
    args['button'] = 'Edytuj'
    if request.user.is_authenticated():
        args.update(csrf(request))
        if request.method == 'POST':
            comment_form = CommentForm(request.POST, instance=editable_comment)
            args['comment_form'] = comment_form
            if comment_form.is_valid():
                comment_form.save()

                return HttpResponseRedirect('/events/event/' + event.slug)
        else:
            args['comment_form'] = CommentForm(instance=editable_comment)

    return render_to_response('event_detail.html', args,
                              context_instance=RequestContext(request))

def comment_delete(request, id):
    comment = Comment.objects.get(pk=id)
    comment.delete()

    return HttpResponseRedirect('/events/event/' + comment.event.slug)


# pylint: disable=W0614,W0401
'''
One off program to initialize and set up the Customer database as shown in the
video lesson/code
'''

import logging
from customer_db_model import *

logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)

LOGGER.info('One off program to set up the customer database')

DATABASE.create_tables([Customer])

DATABASE.close()


# -*- coding: utf-8 -*-
import argparse, json, sys, re, os, datetime, struct, copy, pylab

kAuPerAngstrom = 1.8897259885789
kPsecPerAu = 2.418884326505e-5  # Time

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('eigenvalue_vs_ipratio_paths', metavar='JSON', type=str, nargs='+',
                        help='')
    parser.add_argument('--xmin', metavar='XMIN', dest='x_min', type=float, default=None,
                        help='')
    parser.add_argument('--xmax', metavar='XMAX', dest='x_max', type=float, default=None,
                        help='')
    parser.add_argument('--ymin', metavar='YMIN', dest='y_min', type=float, default=None,
                        help='')
    parser.add_argument('--ymax', metavar='YMAX', dest='y_max', type=float, default=None,
                        help='')
    parser.add_argument('-t', metavar='TITLE', dest='title', type=str, default='',
                        help='')
    parser.add_argument('-s', metavar='TIME_START', dest='time_start', type=float, default=None,
                        help='')  # in ps.
    parser.add_argument('-e', metavar='TIME_END', dest='time_end', type=float, default=None,
                        help='')  # in ps.
    parser.add_argument('-o', metavar='OUT', dest='fig_path', type=str, default=None,
                        help='')
    parser.add_argument('-l', metavar='LABELS', dest='labels', type=str, default=None,
                        help='')
    args = parser.parse_args()

    if args.title == '':
        title = args.eigenvalue_vs_ipratio_paths[0]
    else:
        title = args.title

    if args.fig_path is None:
        fig_path = re.sub(r'(_eigenvalue_vs_ipratio)?\.[^.]+$', '', args.eigenvalue_vs_ipratio_paths[0]) + \
                   '_eigenvalue_vs_ipratio.png'
    else:
        fig_path = args.fig_path

    if args.labels is None:
        labels = ['sample %d' % i for i in range(len(args.eigenvalue_vs_ipratio_paths))]
    else:
        labels = args.labels.split(',')

    vsss = []
    for path in args.eigenvalue_vs_ipratio_paths:
        with open(path, 'r') as fp:
            vsss.append(json.load(fp))

    header = re.sub('\.[^.]+$', '', fig_path)

    min_eigenvalue = 1e100
    min_ipratio = 1e100
    max_eigenvalue = -1e100
    max_ipratio = -1e100
    for vss in vsss:
        for vs in vss:
            min_eigenvalue = min(min_eigenvalue, min(vs['eigenvalues']))
            min_ipratio = min(min_ipratio, min(vs['ipratios']))
            max_eigenvalue = max(max_eigenvalue, max(vs['eigenvalues']))
            max_ipratio = max(max_ipratio, max(vs['ipratios']))

    if args.x_min is None:
        x_min = min_eigenvalue
    else:
        x_min = args.x_min
    if args.x_max is None:
        x_max = max_eigenvalue
    else:
        x_max = args.x_max
    if args.y_min is None:
        y_min = 0.
    else:
        y_min = args.y_min
    if args.y_max is None:
        y_max = 1. / min_ipratio
    else:
        y_max = args.y_max

    for step in range(len(vsss[0])):
        output_filename = '%s_%06d.png' % (header, step)
        pylab.title(title + ' %.4f [ps]' % (vsss[0][step]['time'] * kPsecPerAu))
        pylab.xlabel('Eigenvalue [au]')
        pylab.ylabel('Pratio')
        pylab.grid()
        pylab.xlim(x_min, x_max)
        pylab.ylim(y_min, y_max)
        for sample in range(len(vsss)):
            pratios = [1. / ipratio for ipratio in vsss[sample][step]['ipratios']]
            pylab.plot(vsss[sample][step]['eigenvalues'], pratios, '+-', label=labels[sample])
        pylab.legend(loc='upper left')
        pylab.savefig(output_filename)
        pylab.clf()


from feedReader.models import SiteInfo
import feedparser
import hashlib
from feedReader.mongoFunctions import Mongo
from time import mktime
from datetime import datetime
from django.utils.html import strip_tags
import time
from bs4 import BeautifulSoup
#from urllib2 import Request, urlopen
import urllib3
import re

class ParsingFuncs:

    '''
    Contains all functions to get and parse the feeds
    '''

    def __init__(self):
        '''
        Initialise mongodb connection
        '''
        self.mongo = Mongo()

    def fetchFeeds(self):
        '''
        Fetches all the entries in the table siteInfo and fetches its feeds ..
        Stores to the database
        '''
        siteList = SiteInfo.objects.all()
        for site in siteList:
            modifiedStr = self.createLastModifiedStr(site.lastModified, site.etag)
            if modifiedStr is not None:
                feeds = feedparser.parse(site.feedUrl, modifiedStr)
            else:
                feeds = feedparser.parse(site.feedUrl)
            # find the last modified date. This value will be in feed.updated, feed.last_modified
            lastModified = self.findLastModifiedDate(feeds.feed)
            try:
                etag = feeds.feed.etag
            except:
                etag = None

            feedsHash = self.md5Feeds(feeds)  # calculating the hash of entire feeds
            if(site.feedHash == feedsHash):
                continue  # if no change in feeds ignore it.
            site.feedHash = feedsHash  # if changed save it in db
            if etag is None:
                site.lastModified = lastModified
            else:
                site.etag = etag
            site.save()
            for entry in feeds.entries:
                dt = datetime.fromtimestamp(mktime(entry.published_parsed))  # the format of published_parsed is not..
                entry['published_parsed'] = dt                              # compactible with mongodb
                try:
                    mediaContnet = entry['media_content']
                except:
                    mediaContnet = None
                try:
                    content = entry['content'][0]['value']
                except:
                    content = None
                entry['image_link'] = self.getImage(media_content=mediaContnet,
                                                    summary=entry['summary'],
                                                    content=content,
                                                    link=entry['link'])
                self.mongo.insertFeeds(entry, site.id)

    def allFeeds(self,user_id, lastDate=None):
        if lastDate is not None:
            return self.mongo.selectFeeds(user_id=user_id,dateOfLastItem=lastDate)
        return self.mongo.selectFeeds(user_id=user_id)

    def md5Feeds(self, feed):
        '''
        find md5 of feed
        '''
        md5 = hashlib.md5(str(feed).encode('utf-8'))
        return md5.hexdigest()

    def selectFeedById(self, id):
        return self.mongo.selectFeedById(id)

    def getSiteTitle(self, siteId):
        siteObject = SiteInfo.objects.filter(id=siteId)
        for site in siteObject:
            return site.title

    def getSummary(self, summary):
        summary1000wds = strip_tags(summary)
        summary1000wds = summary1000wds[:300] + "..."
        return summary1000wds

    def getFullPost(self, summaryDetail):
        post = strip_tags(summaryDetail)
        return post

    def createLastModifiedStr(self, last_modified=None, etag=None):
        modiStr = None
        if etag is not None:
            modiStr = "etag = " + str(etag)
        if last_modified is not None:
            modiStr = "modified = " + str(last_modified.utctimetuple())
        return modiStr

    def findLastModifiedDate(self, feed):
        try:
            last_modified = datetime.fromtimestamp(mktime(feed.updated_parsed))  # if updated date is present
        except:
            try:
                last_modified = datetime.fromtimestamp(mktime(feed.date_parsed))  # if date field is present
            except:
                try:
                    # if published_parsed is present
                    last_modified = datetime.fromtimestamp(mktime(feed.published_parsed))
                except:
                    structTime = time.localtime()
                    last_modified = datetime(*structTime[:6])
        return last_modified

    def getFullPostURLOpen(self, link, summary):
        http = urllib3.PoolManager()
        #req = Request(link, headers={'User-Agent': "ireadr"})
        try:
            page = http.request('GET', link)
            page = page.data
            #page = urlopen(req)
        except:
            page = None
        if page is not None:
            soup = BeautifulSoup(page)
            summary = summary[:25]
            # modify this like check match for entire summary.
            #if not found find a substring of length 50 and check again
            # if again not found then reduce the length and try again
            element = soup.find(text=re.compile(summary))
            post = element.findParent('div')
            return post
        return None

    def findImgsrcFromHtml(self, content):
        soup = BeautifulSoup(content)
        img_links = soup.findAll('img')
        if len(img_links) > 0:
            for link in img_links:
                try:
                    if link['height'] == '1' or link['width'] == '1':
                        continue
                    else:
                        return (link['src'])
                except:
                    return (link['src'])
        return None

    def getImage(self, media_content=None, summary=None, content=None, link=None):
        if media_content is not None:
            return (media_content[0]['url'])
        # if media_content is None
        if summary is not None:
            return (self.findImgsrcFromHtml(summary))
        # if no matching image is found in summary
        if content is not None:
            return (self.findImgsrcFromHtml(content))
        # if image is not found in content then fetch the original page and extract the image
        if link is not None:
            post = self.getFullPostURLOpen(link, summary)
            if post is not None:
                return(self.findImgsrcFromHtml(post))
        return None


import datetime
from odoo import models, fields, http, api
from odoo.http import request
from odoo.addons.website.models.ir_http import sitemap_qs2dom
from odoo.osv.expression import FALSE_DOMAIN
SITEMAP_CACHE_TIME = datetime.timedelta(hours=12)
LOC_PER_SITEMAP = 45000

class Website(models.Model):
    _inherit = "website"

    @api.multi
    def enumerate_pages_2(self, query_string=None, force=False):
        """ Available pages in the website/CMS. This is mostly used for links
            generation and can be overridden by modules setting up new HTML
            controllers for dynamic pages (e.g. blog).
            By default, returns template views marked as pages.
            :param str query_string: a (user-provided) string, fetches pages
                                     matching the string
            :returns: a list of mappings with two keys: ``name`` is the displayable
                      name of the resource (page), ``url`` is the absolute URL
                      of the same.
            :rtype: list({name: str, url: str})
        """

        router = request.httprequest.app.get_db_router(request.db)
        # Force enumeration to be performed as public user
        url_set = set()

        sitemap_endpoint_done = set()

        for rule in router.iter_rules():
            if 'sitemap' in rule.endpoint.routing:
                if rule.endpoint in sitemap_endpoint_done:
                    continue
                sitemap_endpoint_done.add(rule.endpoint)

                func = rule.endpoint.routing['sitemap']
                if func is False:
                    continue
                for loc in func(self.env, rule, query_string):
                    new_loc = self.check_loc(loc)
                    if new_loc:
                        yield new_loc
                continue

            if not self.rule_is_enumerable(rule):
                continue

            converters = rule._converters or {}
            if query_string and not converters and (query_string not in rule.build([{}], append_unknown=False)[1]):
                continue
            values = [{}]
            # converters with a domain are processed after the other ones
            convitems = sorted(
                converters.items(),
                key=lambda x: (hasattr(x[1], 'domain') and (x[1].domain != '[]'), rule._trace.index((True, x[0]))))

            for (i, (name, converter)) in enumerate(convitems):
                newval = []
                for val in values:
                    query = i == len(convitems) - 1 and query_string
                    if query:
                        r = "".join([x[1] for x in rule._trace[1:] if not x[0]])  # remove model converter from route
                        query = sitemap_qs2dom(query, r, self.env[converter.model]._rec_name)
                        if query == FALSE_DOMAIN:
                            continue
                    for value_dict in converter.generate(uid=self.env.uid, dom=query, args=val):
                        newval.append(val.copy())
                        value_dict[name] = value_dict['loc']
                        del value_dict['loc']
                        newval[-1].update(value_dict)
                values = newval

            for value in values:
                domain_part, url = rule.build(value, append_unknown=False)
                if not query_string or query_string.lower() in url.lower():
                    page = {'loc': url}
                    for key, val in value.items():
                        if key.startswith('__'):
                            page[key[2:]] = val
                    if url in ('/sitemap.xml',):
                        continue
                    if url in url_set:
                        continue
                    url_set.add(url)
                    new_loc = self.check_loc(page)
                    if new_loc:
                        yield new_loc

        # '/' already has a http.route & is in the routing_map so it will already have an entry in the xml
        domain = [('url', '!=', '/')]
        if not force:
            domain += [('website_indexed', '=', True)]
            # is_visible
            domain += [('website_published', '=', True), '|', ('date_publish', '=', False),
                       ('date_publish', '<=', fields.Datetime.now())]

        if query_string:
            domain += [('url', 'like', query_string)]

        pages = self.get_website_pages(domain)

        for page in pages:

            record = {'loc': page['url'], 'id': page['id'], 'name': page['name']}
            if page.view_id and page.view_id.priority != 16:
                record['__priority'] = min(round(page.view_id.priority / 32.0, 1), 1)
            if page['write_date']:
                record['__lastmod'] = page['write_date'].date()

            new_loc = self.check_loc(record)
            if new_loc:
                yield new_loc

    def check_loc(self, data):
        # Exclude
        if self and self.sitemap_exclude_urls:
            urls = self.sitemap_exclude_urls.split('\n')
            urls = [x.strip() for x in urls if x.strip()]
            if data['loc'] in urls:
                return False
        return data