# -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2012 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""BigMLer - A Higher Level API to BigML's API

# Basic usage
python bigmler.py \
    --train data/iris.csv \
    --test data/test_iris.csv
    --no-test-header

# Create an 10-model ensemble using bagging
python bigmler.py
    --train train.csv \
    --output submission.csv \
    --objective 0 \
    --types types.txt \
    --name 'Iris Ensemble' \
    --number_of_models 10 \
    --sample_rate 0.75 \
    --replacement \
    --tag my_ensemble

# Make predictions using models tagged with my_ensemble
python bigmler.py \
    --model_tag my_ensemble \
    --test test.csv
    --no-test-header

"""
from __future__ import absolute_import

import sys
import os
import datetime
import csv
import re
import shlex

try:
    import simplejson as json
except ImportError:
    import json

import bigml.api
import bigmler.utils as u

from bigml.model import Model
from bigml.multimodel import MultiModel
from bigml.multivote import COMBINATION_WEIGHTS, COMBINER_MAP, PLURALITY
from bigml.fields import Fields

from bigml.util import (localize, console_log, get_csv_delimiter,
                        get_predictions_file_name)

from bigmler.options import create_parser
from bigmler.defaults import get_user_defaults
from bigmler.defaults import DEFAULTS_FILE

MAX_MODELS = 10
EVALUATE_SAMPLE_RATE = 0.8
SEED = "BigML, Machine Learning made easy"
# Date and time in format SunNov0412_120510 to name and tag resources
NOW = datetime.datetime.now().strftime("%a%b%d%y_%H%M%S")
COMMAND_LOG = ".bigmler"
DIRS_LOG = ".bigmler_dir_stack"
SESSIONS_LOG = "bigmler_sessions"
LOG_FILES = [COMMAND_LOG, DIRS_LOG, u.NEW_DIRS_LOG]

def remote_predict(models, headers, output_path, number_of_tests, resume,
                   verbosity, test_reader, exclude, fields, api,
                   prediction_file, method, tags, objective_field,
                   session_file, test_set_header, log, debug):
    """Retrieve predictions remotely, combine them and save predictions to file

    """

    predictions_files = []
    prediction_args = {
        "tags": tags
    }
    for model in models:
        if not isinstance(model, basestring) and 'resource' in model:
            model = model['resource']
        predictions_file = get_predictions_file_name(model,
                                                     output_path)
        predictions_files.append(predictions_file)
        if (not resume or
            not u.checkpoint(u.are_predictions_created, predictions_file,
                             number_of_tests, debug=debug)):
            message = u.dated("Creating remote predictions.\n")
            u.log_message(message, log_file=session_file,
                          console=verbosity)

            predictions_file = csv.writer(open(predictions_file, 'w', 0))
            for row in test_reader:
                for index in exclude:
                    del row[index]
                input_data = fields.pair(row, headers, objective_field)
                prediction = api.create_prediction(model, input_data,
                                                   by_name=test_set_header,
                                                   wait_time=0,
                                                   args=prediction_args)
                u.log_message("%s\n" % prediction['resource'], log_file=log)
                prediction_row = u.prediction_to_row(prediction)
                predictions_file.writerow(prediction_row)
    u.combine_votes(predictions_files,
                    Model(models[0]).to_prediction,
                    prediction_file, method)

def local_predict(models, headers, test_reader, exclude, fields, method,
                  objective_field, output, test_set_header):
    """Get local predictions, combine them and save predictions to file

    """
    local_model = MultiModel(models)
    for row in test_reader:
        for index in exclude:
            del row[index]
        input_data = fields.pair(row, headers, objective_field)
        prediction = local_model.predict(input_data,
                                         by_name=test_set_header,
                                         method=method)
        u.write_prediction(prediction, output)

def local_batch_predict(models, headers, test_reader, exclude, fields, resume,
                        output_path, max_models, number_of_tests, api, output,
                        verbosity, method, objective_field, session_file,
                        debug):
    """Get local predictions form partial Multimodel, combine and save to file

    """
    def draw_progress_bar(current, total):
        """Draws a text based progress report.

        """
        pct = 100 - ((total - current) * 100) / (total)
        console_log("Predicted on %s out of %s models [%s%%]" % (
            localize(current), localize(total), pct))

    models_total = len(models)
    models_splits = [models[index:(index + max_models)] for index
                     in range(0, models_total, max_models)]
    input_data_list = []
    for row in test_reader:
        for index in exclude:
            del row[index]
        input_data_list.append(fields.pair(row, headers,
                                           objective_field))
    total_votes = []
    models_count = 0
    for models_split in models_splits:
        if resume:
            for model in models_split:
                pred_file = get_predictions_file_name(model,
                                                      output_path)
                u.checkpoint(u.are_predictions_created,
                             pred_file,
                             number_of_tests, debug=debug)
        complete_models = []
        for index in range(len(models_split)):
            complete_models.append(api.check_resource(
                models_split[index], api.get_model))

        local_model = MultiModel(complete_models)
        local_model.batch_predict(input_data_list,
                                  output_path, reuse=True)
        votes = local_model.batch_votes(output_path)
        models_count += max_models
        if models_count > models_total:
            models_count = models_total
        if verbosity:
            draw_progress_bar(models_count, models_total)
        if total_votes:
            for index in range(0, len(votes)):
                predictions = total_votes[index].predictions
                predictions.extend(votes[index].predictions)
        else:
            total_votes = votes
    message = u.dated("Combining predictions.\n")
    u.log_message(message, log_file=session_file, console=verbosity)
    for multivote in total_votes:
        u.write_prediction(multivote.combine(method), output)

def predict(test_set, test_set_header, models, fields, output,
            objective_field, remote=False, api=None, log=None,
            max_models=MAX_MODELS, method=0, resume=False,
            tags=None, verbosity=1, session_file=None, debug=False):
    """Computes a prediction for each entry in the `test_set`.

       Predictions can be computed remotely, locally using MultiModels built
       on all the models or locally using MultiModels on subgroups of models.
       Chosing a max_batch_models value not bigger than the number_of_models
       flag will lead to the last case, where memory usage is bounded and each
       model predictions are saved for further use.
    """

    try:
        test_reader = csv.reader(open(test_set, "U"),
                                 delimiter=get_csv_delimiter(),
                                 lineterminator="\n")
    except IOError:
        sys.exit("Error: cannot read test %s" % test_set)

    headers = None
    exclude = []
    if test_set_header:
        headers = test_reader.next()
        # validate headers against model fields excluding objective_field,
        # that may be present or not
        fields_names = [fields.fields[fields.field_id(i)]
                        ['name'] for i in
                        sorted(fields.fields_by_column_number.keys())
                        if i != fields.field_column_number(objective_field)]
        headers = [unicode(header, "utf-8") for header in headers]
        exclude = [i for i in range(len(headers)) if not headers[i]
                   in fields_names]
        exclude.reverse()
        if len(exclude):
            if (len(headers) - len(exclude)):
                print (u"WARNING: predictions will be processed but some data"
                       u" might not be used. The used fields will be:\n\n%s"
                       u"\n\nwhile the headers found in the test file are:"
                       u"\n\n%s" %
                       (",".join(fields_names),
                        ",".join(headers))).encode("utf-8")
                for index in exclude:
                    del headers[index]
            else:
                raise Exception((u"No test field matches the model fields.\n"
                                 u"The expected fields are:\n\n%s\n\nwhile "
                                 u"the headers found in the test file are:\n\n"
                                 u"%s\n\nUse --no-test-header flag if first li"
                                 u"ne should not be interpreted as headers." %
                                 (",".join(fields_names),
                                  ",".join(headers))).encode("utf-8"))

    prediction_file = output
    output_path = u.check_dir(output)
    output = open(output, 'w', 0)
    number_of_tests = None
    if resume:
        number_of_tests = u.file_number_of_lines(test_set)
        if test_set_header:
            number_of_tests -= 1
    # Remote predictions: predictions are computed in bigml.com and stored
    # in a file named after the model in the following syntax:
    #     model_[id of the model]__predictions.csv
    # For instance,
    #     model_50c0de043b563519830001c2_predictions.csv
    if remote:
        remote_predict(models, headers, output_path, number_of_tests, resume,
                       verbosity, test_reader, exclude, fields, api,
                       prediction_file, method, tags, objective_field,
                       session_file, test_set_header, log, debug)
    # Local predictions: Predictions are computed locally using models' rules
    # with MultiModel's predict method
    else:
        message = u.dated("Creating local predictions.\n")
        u.log_message(message, log_file=session_file, console=verbosity)
        # For a small number of models, we build a MultiModel using all of
        # the given models and issue a combined prediction
        if len(models) < max_models:
            local_predict(models, headers, test_reader, exclude, fields,
                          method, objective_field, output, test_set_header)
        # For large numbers of models, we split the list of models in chunks
        # and build a MultiModel for each chunk, issue and store predictions
        # for each model and combine all of them eventually.
        else:
            local_batch_predict(models, headers, test_reader, exclude, fields,
                                resume, output_path, max_models,
                                number_of_tests, api, output,
                                verbosity, method, objective_field,
                                session_file, debug)
    output.close()

def compute_output(api, args, training_set, test_set=None, output=None,
                   objective_field=None,
                   description=None,
                   field_attributes=None,
                   types=None,
                   dataset_fields=None,
                   model_fields=None,
                   name=None, training_set_header=True,
                   test_set_header=True, model_ids=None,
                   votes_files=None, resume=False, fields_map=None):
    """ Creates one or more models using the `training_set` or uses the ids
    of previously created BigML models to make predictions for the `test_set`.

    """
    source = None
    dataset = None
    model = None
    models = None
    fields = None

    path = u.check_dir(output)
    session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG)
    csv_properties = {}
    # If logging is required, open the file for logging
    log = None
    if args.log_file:
        u.check_dir(args.log_file)
        log = args.log_file
        # If --clear_logs the log files are cleared
        if args.clear_logs:
            try:
                open(log, 'w', 0).close()
            except IOError:
                pass

    if (training_set or (args.evaluate and test_set)):
        if resume:
            resume, args.source = u.checkpoint(u.is_source_created, path,
                                               bigml.api, debug=args.debug)
            if not resume:
                message = u.dated("Source not found. Resuming.\n")
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)

    # If neither a previous source, dataset or model are provided.
    # we create a new one. Also if --evaluate and test data are provided
    # we create a new dataset to test with.
    data_set = None
    if (training_set and not args.source and not args.dataset and
            not args.model and not args.models):
        data_set = training_set
        data_set_header = training_set_header
    elif (args.evaluate and test_set and not args.source):
        data_set = test_set
        data_set_header = test_set_header

    if not data_set is None:

        source_args = {
            "name": name,
            "description": description,
            "category": args.category,
            "tags": args.tag,
            "source_parser": {"header": data_set_header}}
        message = u.dated("Creating source.\n")
        u.log_message(message, log_file=session_file, console=args.verbosity)
        source = api.create_source(data_set, source_args,
                                   progress_bar=args.progress_bar)
        source = api.check_resource(source, api.get_source)
        message = u.dated("Source created: %s\n" % u.get_url(source, api))
        u.log_message(message, log_file=session_file, console=args.verbosity)
        u.log_message("%s\n" % source['resource'], log_file=log)

        fields = Fields(source['object']['fields'],
                        source['object']['source_parser']['missing_tokens'],
                        source['object']['source_parser']['locale'])
        source_file = open(path + '/source', 'w', 0)
        source_file.write("%s\n" % source['resource'])
        source_file.write("%s\n" % source['object']['name'])
        source_file.flush()
        source_file.close()

    # If a source is provided, we retrieve it.
    elif args.source:
        message = u.dated("Retrieving source. %s\n" %
                          u.get_url(args.source, api))
        u.log_message(message, log_file=session_file, console=args.verbosity)
        source = api.get_source(args.source)

    # If we already have source, we check that is finished and extract the
    # fields, and update them if needed.
    if source:
        if source['object']['status']['code'] != bigml.api.FINISHED:
            message = u.dated("Retrieving source. %s\n" %
                              u.get_url(source, api))
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            source = api.check_resource(source, api.get_source)
        csv_properties = {'missing_tokens':
                          source['object']['source_parser']['missing_tokens'],
                          'data_locale':
                          source['object']['source_parser']['locale']}

        fields = Fields(source['object']['fields'], **csv_properties)
        update_fields = {}
        if field_attributes:
            for (column, value) in field_attributes.iteritems():
                update_fields.update({
                    fields.field_id(column): value})
            message = u.dated("Updating source. %s\n" %
                              u.get_url(source, api))
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            source = api.update_source(source, {"fields": update_fields})

        update_fields = {}
        if types:
            for (column, value) in types.iteritems():
                update_fields.update({
                    fields.field_id(column): {'optype': value}})
            message = u.dated("Updating source. %s\n" %
                              u.get_url(source, api))
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            source = api.update_source(source, {"fields": update_fields})

    if (training_set or args.source or (args.evaluate and test_set)):
        if resume:
            resume, args.dataset = u.checkpoint(u.is_dataset_created, path,
                                                bigml.api,
                                                debug=args.debug)
            if not resume:
                message = u.dated("Dataset not found. Resuming.\n")
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
    # If we have a source but not dataset or model has been provided, we
    # create a new dataset if the no_dataset option isn't set up. Also
    # if evaluate is set and test_set has been provided.
    if ((source and not args.dataset and not args.model and not model_ids and
            not args.no_dataset) or
            (args.evaluate and args.test_set and not args.dataset)):
        dataset_args = {
            "name": name,
            "description": description,
            "category": args.category,
            "tags": args.tag
        }

        if args.json_filter:
            dataset_args.update(json_filter=args.json_filter)
        elif args.lisp_filter:
            dataset_args.update(lisp_filter=args.lisp_filter)

        input_fields = []
        if dataset_fields:
            for name in dataset_fields:
                input_fields.append(fields.field_id(name))
            dataset_args.update(input_fields=input_fields)
        message = u.dated("Creating dataset.\n")
        u.log_message(message, log_file=session_file, console=args.verbosity)
        dataset = api.create_dataset(source, dataset_args)
        dataset = api.check_resource(dataset, api.get_dataset)
        message = u.dated("Dataset created: %s\n" % u.get_url(dataset, api))
        u.log_message(message, log_file=session_file, console=args.verbosity)
        u.log_message("%s\n" % dataset['resource'], log_file=log)
        dataset_file = open(path + '/dataset', 'w', 0)
        dataset_file.write("%s\n" % dataset['resource'])
        dataset_file.flush()
        dataset_file.close()

    # If a dataset is provided, let's retrieve it.
    elif args.dataset:
        message = u.dated("Retrieving dataset. %s\n" %
                          u.get_url(args.dataset, api))
        u.log_message(message, log_file=session_file, console=args.verbosity)
        dataset = api.get_dataset(args.dataset)

    # If we already have a dataset, we check the status and get the fields if
    # we hadn't them yet.
    if dataset:
        if dataset['object']['status']['code'] != bigml.api.FINISHED:
            message = u.dated("Retrieving dataset. %s\n" %
                              u.get_url(dataset, api))
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            dataset = api.check_resource(dataset, api.get_dataset)
        if not csv_properties:
            csv_properties = {'data_locale':
                              dataset['object']['locale']}
        if args.public_dataset:
            if not description:
                raise Exception("You should provide a description to publish.")
            public_dataset = {"private": False}
            if args.dataset_price:
                message = u.dated("Updating dataset. %s\n" %
                                  u.get_url(dataset, api))
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
                public_dataset.update(price=args.dataset_price)
            message = u.dated("Updating dataset. %s\n" %
                              u.get_url(dataset, api))
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            dataset = api.update_dataset(dataset, public_dataset)
        fields = Fields(dataset['object']['fields'], **csv_properties)

    # If we have a dataset but not a model, we create the model if the no_model
    # flag hasn't been set up.
    if (dataset and not args.model and not model_ids and not args.no_model):
        model_args = {
            "name": name,
            "description": description,
            "category": args.category,
            "tags": args.tag
        }
        if objective_field is not None:
            model_args.update({"objective_field":
                               fields.field_id(objective_field)})
        # If evaluate flag is on, we choose a deterministic sampling with 80%
        # of the data to create the model
        if args.evaluate:
            if args.sample_rate == 1:
                args.sample_rate = EVALUATE_SAMPLE_RATE
            seed = SEED
            model_args.update(seed=seed)

        input_fields = []
        if model_fields:
            for name in model_fields:
                input_fields.append(fields.field_id(name))
            model_args.update(input_fields=input_fields)

        if args.pruning and args.pruning != 'smart':
            model_args.update(stat_pruning=(args.pruning == 'statistical'))

        model_args.update(sample_rate=args.sample_rate,
                          replacement=args.replacement,
                          randomize=args.randomize)
        model_ids = []
        models = []
        if resume:
            resume, model_ids = u.checkpoint(u.are_models_created, path,
                                             args.number_of_models,
                                             bigml.api, debug=args.debug)
            if not resume:
                message = u.dated("Found %s models out of %s. Resuming.\n" %
                                  (len(model_ids),
                                   args.number_of_models))
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
            models = model_ids
            args.number_of_models -= len(model_ids)

        model_file = open(path + '/models', 'w', 0)
        for model_id in model_ids:
            model_file.write("%s\n" % model_id)
        last_model = None
        if args.number_of_models > 0:
            message = u.dated("Creating %s.\n" %
                              u.plural("model", args.number_of_models))
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            for i in range(1, args.number_of_models + 1):
                if i > args.max_parallel_models:
                    api.check_resource(last_model, api.get_model)
                model = api.create_model(dataset, model_args)
                u.log_message("%s\n" % model['resource'], log_file=log)
                last_model = model
                model_ids.append(model['resource'])
                models.append(model)
                model_file.write("%s\n" % model['resource'])
                model_file.flush()
            if args.number_of_models < 2 and args.verbosity:
                if model['object']['status']['code'] != bigml.api.FINISHED:
                    model = api.check_resource(model, api.get_model)
                    models[0] = model
                message = u.dated("Model created: %s.\n" %
                                  u.get_url(model, api))
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
        model_file.close()

    # If a model is provided, we retrieve it.
    elif args.model:
        message = u.dated("Retrieving model. %s\n" %
                          u.get_url(args.model, api))
        u.log_message(message, log_file=session_file, console=args.verbosity)
        model = api.get_model(args.model)

    elif args.models or args.model_tag:
        models = model_ids[:]

    if model_ids and test_set and not args.evaluate:
        model_id = ""
        if len(model_ids) == 1:
            model_id = model_ids[0]
        message = u.dated("Retrieving %s. %s\n" %
                          (u.plural("model", len(model_ids)),
                           u.get_url(model_id, api)))
        u.log_message(message, log_file=session_file, console=args.verbosity)
        if len(model_ids) < args.max_batch_models:
            models = []
            for model in model_ids:
                model = api.check_resource(model, api.get_model)
                models.append(model)
            model = models[0]
        else:
            model = api.check_resource(model_ids[0], api.get_model)
            models[0] = model

    # We check that the model is finished and get the fields if haven't got
    # them yet.
    if model and not args.evaluate and (test_set or args.black_box
                                        or args.white_box):
        if model['object']['status']['code'] != bigml.api.FINISHED:
            message = u.dated("Retrieving model. %s\n" %
                              u.get_url(model, api))
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            model = api.check_resource(model, api.get_model)
        if args.black_box:
            if not description:
                raise Exception("You should provide a description to publish.")
            model = api.update_model(model, {"private": False})
        if args.white_box:
            if not description:
                raise Exception("You should provide a description to publish.")
            public_model = {"private": False, "white_box": True}
            if args.model_price:
                message = u.dated("Updating model. %s\n" %
                                  u.get_url(model, api))
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
                public_model.update(price=args.model_price)
            if args.cpp:
                message = u.dated("Updating model. %s\n" %
                                  u.get_url(model, api))
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
                public_model.update(credits_per_prediction=args.cpp)
            model = api.update_model(model, public_model)
        if not csv_properties:
            csv_properties = {'data_locale':
                              model['object']['locale']}
        csv_properties.update(verbose=True)
        if args.user_locale:
            csv_properties.update(data_locale=args.user_locale)

        fields = Fields(model['object']['model']['fields'], **csv_properties)

    if model and not models:
        models = [model]

    if models and test_set and not args.evaluate:
        objective_field = models[0]['object']['objective_fields']
        if isinstance(objective_field, list):
            objective_field = objective_field[0]
        predict(test_set, test_set_header, models, fields, output,
                objective_field, args.remote, api, log,
                args.max_batch_models, args.method, resume, args.tag,
                args.verbosity, session_file, args.debug)

    # When combine_votes flag is used, retrieve the predictions files saved
    # in the comma separated list of directories and combine them
    if votes_files:
        model_id = re.sub(r'.*(model_[a-f0-9]{24})__predictions\.csv$',
                          r'\1', votes_files[0]).replace("_", "/")
        model = api.check_resource(model_id, api.get_model)
        local_model = Model(model)
        message = u.dated("Combining votes.\n")
        u.log_message(message, log_file=session_file,
                      console=args.verbosity)
        u.combine_votes(votes_files, local_model.to_prediction,
                        output, args.method)

    # If evaluate flag is on, create remote evaluation and save results in
    # json and human-readable format.
    if args.evaluate:
        if resume:
            resume, evaluation = u.checkpoint(u.is_evaluation_created, path,
                                              bigml.api,
                                              debug=args.debug)
            if not resume:
                message = u.dated("Evaluation not found. Resuming.\n")
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
        if not resume:
            evaluation_file = open(path + '/evaluation', 'w', 0)
            evaluation_args = {
                "name": name,
                "description": description,
                "tags": args.tag
            }
            if not fields_map is None:
                update_map = {}
                for (dataset_column, model_column) in fields_map.iteritems():
                    update_map.update({
                        fields.field_id(dataset_column):
                        fields.field_id(model_column)})
                evaluation_args.update({"fields_map": update_map})
            if not ((args.dataset or args.test_set)
                    and (args.model or args.models or args.model_tag)):
                evaluation_args.update(out_of_bag=True, seed=SEED,
                                       sample_rate=args.sample_rate)
            message = u.dated("Creating evaluation.\n")
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
            evaluation = api.create_evaluation(model, dataset, evaluation_args)
            u.log_message("%s\n" % evaluation['resource'], log_file=log)
            evaluation_file.write("%s\n" % evaluation['resource'])
            evaluation_file.flush()
            evaluation_file.close()
        message = u.dated("Retrieving evaluation. %s\n" %
                          u.get_url(evaluation, api))
        u.log_message(message, log_file=session_file, console=args.verbosity)
        evaluation = api.check_resource(evaluation, api.get_evaluation)
        evaluation_json = open(output + '.json', 'w', 0)
        evaluation_json.write(json.dumps(evaluation['object']['result']))
        evaluation_json.flush()
        evaluation_json.close()
        evaluation_txt = open(output + '.txt', 'w', 0)
        api.pprint(evaluation['object']['result'],
                   evaluation_txt)
        evaluation_txt.flush()
        evaluation_txt.close()

    # Workaround to restore windows console cp850 encoding to print the tree
    if sys.platform == "win32" and sys.stdout.isatty():
        import locale
        data_locale = locale.getlocale()
        if not data_locale[0] is None:
            locale.setlocale(locale.LC_ALL, (data_locale[0], "850"))
        message = (u"\nGenerated files:\n\n" +
                   unicode(u.print_tree(path, " "), "utf-8") + u"\n")
    else:
        message = "\nGenerated files:\n\n" + u.print_tree(path, " ") + "\n"
    u.log_message(message, log_file=session_file, console=args.verbosity)

def main(args=sys.argv[1:]):
    """Main process

    """
    for i in range(0, len(args)):
        if args[i].startswith("--"):
            args[i] = args[i].replace("_", "-")
    # If --clear-logs the log files are cleared
    if "--clear-logs" in args:
        for log_file in LOG_FILES:
            try:
                open(log_file, 'w', 0).close()
            except IOError:
                pass
    literal_args = args[:]
    for i in range(0, len(args)):
        if ' ' in args[i]:
            literal_args[i] = '"%s"' % args[i]
    message = "bigmler %s\n" % " ".join(literal_args)

    # Resume calls are not logged
    if not "--resume" in args:
        with open(COMMAND_LOG, "a", 0) as command_log:
            command_log.write(message)
        resume = False

    parser = create_parser(defaults=get_user_defaults(), constants={'NOW': NOW,
                           'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY})

    # Parses command line arguments.
    command_args = parser.parse_args(args)

    default_output = ('evaluation' if command_args.evaluate
                      else 'predictions.csv')
    if command_args.resume:
        debug = command_args.debug
        command = u.get_log_reversed(COMMAND_LOG,
                                     command_args.stack_level)
        args = shlex.split(command)[1:]
        output_dir = u.get_log_reversed(DIRS_LOG,
                                        command_args.stack_level)
        defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE)
        parser = create_parser(defaults=get_user_defaults(defaults_file),
                               constants={'NOW': NOW, 'MAX_MODELS': MAX_MODELS,
                                          'PLURALITY': PLURALITY})
        command_args = parser.parse_args(args)
        if command_args.predictions is None:
            command_args.predictions = ("%s%s%s" %
                                        (output_dir, os.sep,
                                         default_output))
        # Logs the issued command and the resumed command
        session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG)
        u.log_message(message, log_file=session_file)
        message = "\nResuming command:\n%s\n\n" % command
        u.log_message(message, log_file=session_file, console=True)
        try:
            defaults_handler = open(defaults_file, 'r')
            contents = defaults_handler.read()
            message = "\nUsing the following defaults:\n%s\n\n" % contents
            u.log_message(message, log_file=session_file, console=True)
            defaults_handler.close()
        except IOError:
            pass

        resume = True
    else:
        if command_args.predictions is None:
            command_args.predictions = ("%s%s%s" %
                                        (NOW, os.sep,
                                         default_output))
        if len(os.path.dirname(command_args.predictions).strip()) == 0:
            command_args.predictions = ("%s%s%s" %
                                        (NOW, os.sep,
                                         command_args.predictions))
        directory = u.check_dir(command_args.predictions)
        session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG)
        u.log_message(message + "\n", log_file=session_file)
        try:
            defaults_file = open(DEFAULTS_FILE, 'r')
            contents = defaults_file.read()
            defaults_file.close()
            defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE),
                                 'w', 0)
            defaults_copy.write(contents)
            defaults_copy.close()
        except IOError:
            pass
        with open(DIRS_LOG, "a", 0) as directory_log:
            directory_log.write("%s\n" % os.path.abspath(directory))

    if resume and debug:
        command_args.debug = True

    api_command_args = {
        'username': command_args.username,
        'api_key': command_args.api_key,
        'dev_mode': command_args.dev_mode,
        'debug': command_args.debug}

    api = bigml.api.BigML(**api_command_args)

    if (command_args.evaluate
        and not (command_args.training_set or command_args.source
                 or command_args.dataset)
        and not (command_args.test_set and (command_args.model or
                 command_args.models or command_args.model_tag))):
        parser.error("Evaluation wrong syntax.\n"
                     "\nTry for instance:\n\nbigmler --train data/iris.csv"
                     " --evaluate\nbigmler --model "
                     "model/5081d067035d076151000011 --dataset "
                     "dataset/5081d067035d076151003423 --evaluate")

    if command_args.objective_field:
        objective = command_args.objective_field
        try:
            command_args.objective_field = int(objective)
        except ValueError:
            pass

    output_args = {
        "api": api,
        "training_set": command_args.training_set,
        "test_set": command_args.test_set,
        "output": command_args.predictions,
        "objective_field": command_args.objective_field,
        "name": command_args.name,
        "training_set_header": command_args.train_header,
        "test_set_header": command_args.test_header,
        "args": command_args,
        "resume": resume,
    }

    # Reads description if provided.
    if command_args.description:
        description_arg = u.read_description(command_args.description)
        output_args.update(description=description_arg)
    else:
        output_args.update(description="Created using BigMLer")

    # Parses fields if provided.
    if command_args.field_attributes:
        field_attributes_arg = (
            u.read_field_attributes(command_args.field_attributes))
        output_args.update(field_attributes=field_attributes_arg)

    # Parses types if provided.
    if command_args.types:
        types_arg = u.read_types(command_args.types)
        output_args.update(types=types_arg)

    # Parses dataset fields if provided.
    if command_args.dataset_fields:
        dataset_fields_arg = map(lambda x: x.strip(),
                                 command_args.dataset_fields.split(','))
        output_args.update(dataset_fields=dataset_fields_arg)

    # Parses model input fields if provided.
    if command_args.model_fields:
        model_fields_arg = map(lambda x: x.strip(),
                               command_args.model_fields.split(','))
        output_args.update(model_fields=model_fields_arg)

    model_ids = []
    # Parses model/ids if provided.
    if command_args.models:
        model_ids = u.read_models(command_args.models)
        output_args.update(model_ids=model_ids)

    dataset_id = None
    # Parses dataset/id if provided.
    if command_args.datasets:
        dataset_id = u.read_dataset(command_args.datasets)
        command_args.dataset = dataset_id

    # Retrieve model/ids if provided.
    if command_args.model_tag:
        model_ids = (model_ids +
                     u.list_ids(api.list_models,
                                "tags__in=%s" % command_args.model_tag))
        output_args.update(model_ids=model_ids)

    # Reads a json filter if provided.
    if command_args.json_filter:
        json_filter = u.read_json_filter(command_args.json_filter)
        command_args.json_filter = json_filter

    # Reads a lisp filter if provided.
    if command_args.lisp_filter:
        lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
        command_args.lisp_filter = lisp_filter

    # Adds default tags unless that it is requested not to do so.
    if command_args.no_tag:
        command_args.tag.append('BigMLer')
        command_args.tag.append('BigMLer_%s' % NOW)

    # Checks combined votes method
    if (command_args.method and
            not command_args.method in COMBINATION_WEIGHTS.keys()):
        command_args.method = 0
    else:
        combiner_methods = dict([[value, key]
                                for key, value in COMBINER_MAP.items()])
        command_args.method = combiner_methods.get(command_args.method, 0)

    # Reads votes files in the provided directories.
    if command_args.votes_dirs:
        dirs = map(lambda x: x.strip(), command_args.votes_dirs.split(','))
        votes_path = os.path.dirname(command_args.predictions)
        votes_files = u.read_votes_files(dirs, votes_path)
        output_args.update(votes_files=votes_files)

    # Parses fields map if provided.
    if command_args.fields_map:
        fields_map_arg = u.read_fields_map(command_args.fields_map)
        output_args.update(fields_map=fields_map_arg)

    # Parses resources ids if provided.
    if command_args.delete:
        if command_args.predictions is None:
            path = NOW
        else:
            path = u.check_dir(command_args.predictions)
        session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG)
        message = u.dated("Retrieving objects to delete.\n")
        u.log_message(message, log_file=session_file,
                      console=command_args.verbosity)
        delete_list = []
        if command_args.delete_list:
            delete_list = map(lambda x: x.strip(),
                              command_args.delete_list.split(','))
        if command_args.delete_file:
            if not os.path.exists(command_args.delete_file):
                raise Exception("File %s not found" % command_args.delete_file)
            delete_list.extend([line for line
                                in open(command_args.delete_file, "r")])
        if command_args.all_tag:
            query_string = "tags__in=%s" % command_args.all_tag
            delete_list.extend(u.list_ids(api.list_sources, query_string))
            delete_list.extend(u.list_ids(api.list_datasets, query_string))
            delete_list.extend(u.list_ids(api.list_models, query_string))
            delete_list.extend(u.list_ids(api.list_predictions, query_string))
            delete_list.extend(u.list_ids(api.list_evaluations, query_string))
        # Retrieve sources/ids if provided
        if command_args.source_tag:
            query_string = "tags__in=%s" % command_args.source_tag
            delete_list.extend(u.list_ids(api.list_sources, query_string))
        # Retrieve datasets/ids if provided
        if command_args.dataset_tag:
            query_string = "tags__in=%s" % command_args.dataset_tag
            delete_list.extend(u.list_ids(api.list_datasets, query_string))
        # Retrieve model/ids if provided
        if command_args.model_tag:
            query_string = "tags__in=%s" % command_args.model_tag
            delete_list.extend(u.list_ids(api.list_models, query_string))
        # Retrieve prediction/ids if provided
        if command_args.prediction_tag:
            query_string = "tags__in=%s" % command_args.prediction_tag
            delete_list.extend(u.list_ids(api.list_predictions, query_string))
        # Retrieve evaluation/ids if provided
        if command_args.evaluation_tag:
            query_string = "tags__in=%s" % command_args.evaluation_tag
            delete_list.extend(u.list_ids(api.list_evaluations, query_string))
        message = u.dated("Deleting objects.\n")
        u.log_message(message, log_file=session_file,
                      console=command_args.verbosity)
        message = "\n".join(delete_list)
        u.log_message(message, log_file=session_file)
        u.delete(api, delete_list)
        if sys.platform == "win32" and sys.stdout.isatty():
            message = (u"\nGenerated files:\n\n" +
                       unicode(u.print_tree(path, " "), "utf-8") + u"\n")
        else:
            message = "\nGenerated files:\n\n" + u.print_tree(path, " ") + "\n"
        u.log_message(message, log_file=session_file,
                      console=command_args.verbosity)
    elif (command_args.training_set or command_args.test_set
          or command_args.source or command_args.dataset
          or command_args.datasets or command_args.votes_dirs):
        compute_output(**output_args)
    u.log_message("_" * 80 + "\n", log_file=session_file)

if __name__ == '__main__':
    main(sys.argv[1:])


from __future__ import print_function

import io
import pickle
import os.path
import sys

from googleapiclient.discovery import build

# If modifying these scopes, delete the file token.pickle.
from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload

SCOPES = ['https://www.googleapis.com/auth/drive.readonly']

TOKEN = 'travis_files/token.pickle'
DOWNLOAD_LINK =  "travis_files/download_link.txt"

class GoogleDriveService:

    def __init__(self):
        self.creds = self._auth()
        self.google_drive_service = build('drive', 'v3', credentials=self.creds)

    def _auth(self):
        """
        Authenticate google drive credentails.
        """
        creds = None
        # The file token.pickle stores the user's access and refresh tokens, and is
        # created automatically when the authorization flow completes for the first
        # time.
        if os.path.exists(TOKEN):
            with open(TOKEN, 'rb') as token:
                creds = pickle.load(token)
        else:
            sys.exit("can't find token.pickle")
        # If there are no (valid) credentials available, let the user log in.
        return creds

    def print_n_files(self, n):
        # Call the Drive v3 API
        results = self.google_drive_service.files().list(
            pageSize=n, fields="nextPageToken, files(id, name)").execute()
        items = results.get('files', [])

        if not items:
            print('No files found.')
        else:
            print('Files:')
            for item in items:
                print(u'{0} ({1})'.format(item['name'], item['id']))

    def download_result(self, commit_id):
        """
        Download the html file from google drive using the commit id
        :param commit_id: The connit id to download.
        :return: the html file and the results as a bool type.
        """
        file_id, file_name = self.find_file_id(commit_id)
        if not file_id:
            return False, False

        file = self.google_drive_service.files().get(fileId=file_id, fields='webContentLink').execute()
        print('Download test result at: %s' % file['webContentLink'])

        # create a text file with the download link (later to post as a comment)
        with open(DOWNLOAD_LINK, "w") as f:
            f.write(file['webContentLink'])

        file_request = self.google_drive_service.files().get_media(fileId=file_id)

        if file_request:
            # there are results
            fh = io.BytesIO()
            downloader = MediaIoBaseDownload(fh, file_request)
            done = False

            while done is False:
                status, done = downloader.next_chunk()
                print("Download %d%%." % int(status.progress() * 100))

            # os.mkdir('Travis_Results/%s' % pull_request_number)  # create a folder for the specific test result

            with io.open('%s' % file_name, 'wb') as f:
                fh.seek(0)
                f.write(fh.read())

            passed = file_name[
                     file_name.index('-') + 1: file_name.index('.')] == "True"  # extract the boolean from file name
            return True, passed

        else:
            # there are no results yet
            print("Didn't find the file id!")
            return False, False

    def find_file_id(self, commit_id):
        """
        Looking for the commit id in google drive
        :param commit_id: the commit id to look for.
        :return: The id and the file name as in google drive.
        """
        results = self.google_drive_service.files().list(
            fields="nextPageToken, files(id, name)").execute()
        files = results.get('files', [])
        # looking for the file with that name(what comes before the delimiter)
        filtered = list(filter(lambda item: item['name'].split('-')[0] == commit_id, files))
        if filtered:
            return filtered[0]['id'], filtered[0]['name']
        else:
            return False, False


"""Matrix assembling functions for B-spline IgA.

This module contains functions to assemble mass and stiffness matrices
for IgA with tensor product B-spline functions.

.. _gauss-asms:

Tensor product Gauss quadrature assemblers
------------------------------------------

Standard Gauss quadrature assemblers for mass and stiffness matrices.
They take one or two arguments:

- `kvs` (list of :class:`pyiga.bspline.KnotVector`):
  Describes the tensor product B-spline basis for which to assemble
  the matrix. One :class:`KnotVector` per coordinate direction.
  In the 1D case, a single :class:`pyiga.bspline.KnotVector` may be passed
  directly.
- `geo` (:class:`pyiga.geometry.BSplinePatch`; optional):
  Geometry transform, mapping from the parameter domain to the
  physical domain. If omitted, assume the identity map; a fast
  Kronecker product implementation is used in this case.

.. autofunction:: mass
.. autofunction:: stiffness

.. _fast-asms:

Fast low-rank assemblers
------------------------

Fast low-rank assemblers based on the paper
"A Black-Box Algorithm for Fast Matrix Assembly in Isogeometric Analysis".
They may achieve significant speedups over the classical Gauss assemblers,
in particular for fine discretizations and higher spline degrees.
They only work well if the geometry transform is rather smooth so that the
resulting matrix has relatively low (numerical) Kronecker rank.

They take the following additional arguments:

- `tol`: the stopping accuracy for the Adaptive Cross Approximation (ACA)
  algorithm
- `maxiter`: the maximum number of ACA iterations
- `skipcount`: terminate after finding this many successive near-zero pivots
- `tolcount`: terminate after finding this many successive pivots below the
  desired accuracy
- `verbose`: the amount of output to display. `0` is silent, `1` prints
  basic information, `2` prints detailed information

.. autofunction:: mass_fast
.. autofunction:: stiffness_fast

Right-hand sides
----------------

.. autofunction:: inner_products

Boundary and initial conditions
-------------------------------

.. autofunction:: compute_dirichlet_bc
.. autofunction:: compute_initial_condition_01
.. autofunction:: combine_bcs

.. autoclass:: RestrictedLinearSystem
    :members:

"""
import numpy as np
import scipy
import scipy.sparse
import itertools

from . import bspline
from . import assemble_tools
from . import assemblers
from . import fast_assemble_cy
from . import tensor
from . import operators
from . import utils
from . import geometry

from .quadrature import make_iterated_quadrature, make_tensor_quadrature
from .mlmatrix import MLStructure

################################################################################
# 1D assembling routines
################################################################################

def _assemble_element_matrices(nspans, nqp, vals1, vals2, qweights):
    assert nspans * nqp == vals1.shape[1]
    assert nspans * nqp == vals2.shape[1]
    assert qweights.shape == (vals1.shape[1],)
    n_act1,n_act2 = vals1.shape[0],vals2.shape[0]
    elMats = np.empty((nspans, n_act1, n_act2)) # contains one n_act1 x n_act2 element matrix per span
    for k in range(nspans):
        f1 = vals1[:,nqp*k:nqp*(k+1)]       # n_act1 x nqp
        f2 = vals2[:,nqp*k:nqp*(k+1)]       # n_act2 x nqp
        w = qweights[nqp*k:nqp*(k+1)]       # nqp-vector of weights
        elMats[k, :, :] = np.dot( f1, (f2 * w).transpose() )
    return elMats       # n_act1*nspans x n_act2

def _create_coo_1d(nspans, n_act1, n_act2=None):
    """Create COO indices for two sequentially numbered bases over `nspans` knot spans"""
    if n_act2 is None:
        n_act2 = n_act1     # if only one given, assume symmetry
    grid = np.mgrid[:n_act1, :n_act2] # 2 x n_act1 x n_act2 array which indexes element matrix
    I_ref = grid[0].ravel()          # slowly varying index, first basis
    J_ref = grid[1].ravel()          # fast varying index, second basis

    # first active basis function = index of knot span (independent of spline degree)
    first_act = np.repeat(range(nspans), n_act1*n_act2)
    I = first_act + np.tile(I_ref, nspans)
    J = first_act + np.tile(J_ref, nspans)
    return (I, J)

def _create_coo_1d_custom(nspans, n_act1, n_act2, first_act1, first_act2):
    """Create COO indices for two sequentially numbered bases over `nspans` knot spans"""
    grid = np.mgrid[:n_act1, :n_act2] # 2 x n_act1 x n_act2 array which indexes element matrix
    I_ref = grid[0].ravel()          # slowly varying index, first basis
    J_ref = grid[1].ravel()          # fast varying index, second basis

    I = np.repeat(first_act1, n_act1*n_act2) + np.tile(I_ref, nspans)
    J = np.repeat(first_act2, n_act1*n_act2) + np.tile(J_ref, nspans)
    return (I, J)

def _assemble_matrix(nspans, nqp, vals1, vals2, qweights):
    n_act1 = vals1.shape[0]
    n_act2 = vals2.shape[0]
    elMats = _assemble_element_matrices(nspans, nqp, vals1, vals2, qweights)
    (I, J) = _create_coo_1d(nspans, n_act1, n_act2)
    return scipy.sparse.coo_matrix((elMats.ravel(), (I, J))).tocsr()

def _assemble_matrix_custom(nspans, nqp, vals1, vals2, I, J, qweights):
    n_act1 = vals1.shape[0]
    n_act2 = vals2.shape[0]
    elMats = _assemble_element_matrices(nspans, nqp, vals1, vals2, qweights)
    return scipy.sparse.coo_matrix((elMats.ravel(), (I, J))).tocsr()

def bsp_mass_1d(knotvec):
    "Assemble the mass matrix for the B-spline basis over the given knot vector"""
    nspans = knotvec.numspans
    nqp = knotvec.p + 1
    q = make_iterated_quadrature(knotvec.kv[knotvec.p:-knotvec.p], nqp)
    vals = bspline.active_ev(knotvec, q[0])
    return _assemble_matrix(nspans, nqp, vals, vals, q[1])

def bsp_mass_1d_asym(knotvec1, knotvec2, quadgrid=None):
    """Assemble a mass matrix relating two B-spline bases. By default, uses the first knot vector for quadrature."""
    if quadgrid is None:
        quadgrid = np.unique(knotvec1.kv)

    # create iterated Gauss quadrature rule for each interval
    nqp = max(knotvec1.p, knotvec2.p) + 1
    nspans = len(quadgrid) - 1
    q = make_iterated_quadrature(quadgrid, nqp)
    assert len(q[0]) == nspans * nqp

    # evaluate basis functions at quadrature nodes
    vals1 = bspline.active_ev(knotvec1, q[0])
    vals2 = bspline.active_ev(knotvec2, q[0])

    first_points = q[0][::nqp]
    assert len(first_points) == nspans
    # map first_active_at over first quadrature points to get first active basis function index
    first_act1 = np.vectorize(knotvec1.first_active_at, otypes=(np.int,))(first_points)
    first_act2 = np.vectorize(knotvec2.first_active_at, otypes=(np.int,))(first_points)
    I,J = _create_coo_1d_custom(nspans, vals1.shape[0], vals2.shape[0], first_act1, first_act2)

    return _assemble_matrix_custom(nspans, nqp, vals1, vals2, I, J, q[1])

def bsp_mass_1d_weighted(knotvec, weightfunc):
    nspans = knotvec.numspans
    nqp = knotvec.p
    q = make_iterated_quadrature(knotvec.kv[knotvec.p:-knotvec.p], nqp)
    vals = bspline.active_ev(knotvec, q[0])
    weights = q[1] * weightfunc(q[0])
    return _assemble_matrix(nspans, nqp, vals, vals, weights)

def bsp_stiffness_1d(knotvec):
    "Assemble the Laplacian stiffness matrix for the B-spline basis over the given knot vector"""
    return bsp_mixed_deriv_biform_1d(knotvec, 1, 1)

def bsp_mixed_deriv_biform_1d(knotvec, du, dv):
    "Assemble the matrix for a(u,v)=(u^(du),v^(dv)) for the B-spline basis over the given knot vector"""
    nspans = knotvec.numspans
    nqp = knotvec.p
    q = make_iterated_quadrature(knotvec.kv[knotvec.p:-knotvec.p], nqp)
    derivs = bspline.active_deriv(knotvec, q[0], max(du, dv))
    return _assemble_matrix(nspans, nqp, derivs[dv, :, :], derivs[du, :, :], q[1])

def bsp_stiffness_1d_asym(knotvec1, knotvec2, quadgrid=None):
    """Assemble a stiffness matrix relating two B-spline bases. By default, uses the first knot vector for quadrature."""
    if quadgrid is None:
        quadgrid = np.unique(knotvec1.kv)

    # create iterated Gauss quadrature rule for each interval
    nqp = max(knotvec1.p, knotvec2.p) + 1
    nspans = len(quadgrid) - 1
    q = make_iterated_quadrature(quadgrid, nqp)
    assert len(q[0]) == nspans * nqp

    # evaluate derivatives of basis functions at quadrature nodes
    derivs1 = bspline.active_deriv(knotvec1, q[0], 1)[1, :, :]
    derivs2 = bspline.active_deriv(knotvec2, q[0], 1)[1, :, :]

    first_points = q[0][::nqp]
    assert len(first_points) == nspans
    # map first_active_at over first quadrature points to get first active basis function index
    first_act1 = np.vectorize(knotvec1.first_active_at, otypes=(np.int,))(first_points)
    first_act2 = np.vectorize(knotvec2.first_active_at, otypes=(np.int,))(first_points)
    I,J = _create_coo_1d_custom(nspans, derivs1.shape[0], derivs2.shape[0], first_act1, first_act2)

    return _assemble_matrix_custom(nspans, nqp, derivs1, derivs2, I, J, q[1])

################################################################################
# 2D/3D assembling routines (rely on Cython module)
################################################################################

def bsp_mass_2d(knotvecs, geo=None, format='csr'):
    if geo is None:
        (kv1, kv2) = knotvecs
        M1 = bsp_mass_1d(kv1)
        M2 = bsp_mass_1d(kv2)
        return scipy.sparse.kron(M1, M2, format=format)
    else:
        return assemble(
                assemblers.MassAssembler2D(knotvecs, geo),
                symmetric=True, format=format)

def bsp_stiffness_2d(knotvecs, geo=None, format='csr'):
    if geo is None:
        (kv1, kv2) = knotvecs
        M1 = bsp_mass_1d(kv1)
        M2 = bsp_mass_1d(kv2)
        K1 = bsp_stiffness_1d(kv1)
        K2 = bsp_stiffness_1d(kv2)
        return scipy.sparse.kron(K1, M2, format=format) + scipy.sparse.kron(M1, K2, format=format)
    else:
        return assemble(
                assemblers.StiffnessAssembler2D(knotvecs, geo),
                symmetric=True, format=format)

def bsp_mass_3d(knotvecs, geo=None, format='csr'):
    if geo is None:
        M = [bsp_mass_1d(kv) for kv in knotvecs]
        def k(A,B):
            return scipy.sparse.kron(A, B, format=format)
        return k(M[0], k(M[1], M[2]))
    else:
        return assemble(
                assemblers.MassAssembler3D(knotvecs, geo),
                symmetric=True, format=format)

def bsp_stiffness_3d(knotvecs, geo=None, format='csr'):
    if geo is None:
        MK = [(bsp_mass_1d(kv), bsp_stiffness_1d(kv)) for kv in knotvecs]
        def k(A,B):
            return scipy.sparse.kron(A, B, format=format)
        M12 = k(MK[1][0], MK[2][0])
        K12 = k(MK[1][1], MK[2][0]) + k(MK[1][0], MK[2][1])
        return k(MK[0][1], M12) + k(MK[0][0], K12)
    else:
        return assemble(
                assemblers.StiffnessAssembler3D(knotvecs, geo),
                symmetric=True, format=format)

################################################################################
# Assembling right-hand sides
################################################################################

def inner_products(kvs, f, f_physical=False, geo=None):
    """Compute the :math:`L_2` inner products between each basis
    function in a tensor product B-spline basis and the function `f`
    (i.e., the load vector).

    Args:
        kvs (seq): a sequence of :class:`pyiga.bspline.KnotVector`,
            representing a tensor product basis
        f: a function or :class:`pyiga.geometry.BSplineFunc` object
        f_physical (bool): whether `f` is given in physical coordinates.
            If `True`, `geo` must be passed as well.
        geo: a :class:`pyiga.geometry.BSplinePatch` which describes
            the integration domain; if not given, the integrals are
            computed in the parameter domain

    Returns:
        ndarray: the inner products as an array of size
        `kvs[0].ndofs x kvs[1].ndofs x ... x kvs[-1].ndofs`.
        Each entry corresponds to the inner product of the
        corresponding basis function with `f`.
        If `f` is not scalar, then each of its components is treated separately
        and the corresponding dimensions are appended to the end of the return
        value.
    """
    if isinstance(kvs, bspline.KnotVector):
        kvs = (kvs,)
    # compute quadrature rules
    nqp = max(kv.p for kv in kvs) + 1
    gaussgrid, gaussweights = make_tensor_quadrature([kv.mesh for kv in kvs], nqp)

    # evaluate function f on grid or transformed grid
    if f_physical:
        assert geo is not None, 'inner_products in physical domain requires geometry'
        fvals = utils.grid_eval_transformed(f, gaussgrid, geo)
    else:
        fvals = utils.grid_eval(f, gaussgrid)

    # multiply function values with quadrature weights
    fvals = tensor.apply_tprod(
              [operators.DiagonalOperator(gw) for gw in gaussweights], fvals)
    # if geometry was specified, multiply by abs(det(jac))
    if geo is not None:
        geo_jac = geo.grid_jacobian(gaussgrid)
        geo_det = np.abs(assemble_tools.determinants(geo_jac))
        # if f is not scalar, we simply add trivial dimensions on to the end
        extra_dims = fvals.ndim - geo_det.ndim
        if extra_dims > 0:
            geo_det.shape = geo_det.shape + (extra_dims * (1,))
        fvals *= geo_det
    # apply transposed spline collocation matrices (sum over Gauss nodes)
    Ct = [bspline.collocation(kvs[i], gaussgrid[i]).T
            for i in range(len(kvs))]
    return tensor.apply_tprod(Ct, fvals)

################################################################################
# Incorporating essential boundary conditions
################################################################################

def slice_indices(ax, idx, shape, ravel=False):
    """Return dof indices for a slice of a tensor product basis with size
    `shape`. The slice is taken across index `idx` on axis `ax`.

    The indices are returned either as a `N x dim` array of multiindices or,
    with `ravel=True`, as an array of sequential (raveled) indices.
    """
    shape = tuple(shape)
    if idx < 0:
        idx += shape[ax]     # wrap around
    axdofs = [range(n) for n in shape]
    axdofs[ax] = [idx]
    multi_indices = np.array(list(itertools.product(*axdofs)))
    if ravel:
        multi_indices = np.ravel_multi_index(multi_indices.T, shape)
    return multi_indices

def compute_dirichlet_bc(kvs, geo, bdspec, dir_func):
    """Compute indices and values for a Dirichlet boundary condition using
    interpolation.

    Args:
        kvs: a tensor product B-spline basis
        geo (:class:`pyiga.geometry.BSplinePatch`): the geometry transform
        bdspec: a pair `(axis, side)`. `axis` denotes the axis along
            which the boundary condition lies, and `side` is either
            0 for the "lower" boundary or 1 for the "upper" boundary.
        dir_func: a function which will be interpolated to obtain the
            Dirichlet boundary values. Assumed to be given in physical
            coordinates. If it is vector-valued, one Dirichlet dof is
            computed per component, and they are numbered according to
            the "blocked" matrix layout.

    Returns:
        A pair of arrays `(indices, values)` which denote the indices of the
        dofs within the tensor product basis which lie along the Dirichlet
        boundary and their computed values, respectively.
    """
    bdax, bdside = bdspec

    # get basis for the boundary face
    bdbasis = list(kvs)
    del bdbasis[bdax]

    # get boundary geometry and interpolate dir_func
    bdgeo = geo.boundary(bdax, bdside)
    from .approx import interpolate
    dircoeffs = interpolate(bdbasis, dir_func, geo=bdgeo)

    # compute sequential indices for eliminated dofs
    N = tuple(kv.numdofs for kv in kvs)
    bdindices = slice_indices(bdax, 0 if bdside==0 else -1, N, ravel=True)

    extra_dims = dircoeffs.ndim - len(bdbasis)
    if extra_dims == 0:
        return bdindices, dircoeffs.ravel()
    elif extra_dims == 1:
        # vector function; assume blocked vector discretization
        numcomp = dircoeffs.shape[-1]
        NN = np.prod(N)
        return combine_bcs(
            (bdindices + j*NN, dircoeffs[..., j].ravel())
                for j in range(numcomp))
    else:
        raise ValueError('invalid dimension of Dirichlet coefficients: %s' % dircoeffs.shape)

def compute_initial_condition_01(kvs, geo, bdspec, g0, g1, physical=True):
    r"""Compute indices and values for an initial condition including function
    value and derivative for a space-time discretization using interpolation.
    This only works for a space-time cylinder with constant (in time) geometry.
    To be precise, the space-time geometry transform `geo` should have the form

    .. math:: G(\vec x, t) = (\widetilde G(\vec x), t).

    Args:
        kvs: a tensor product B-spline basis
        geo (:class:`pyiga.geometry.BSplinePatch`): the geometry transform of
            the space-time cylinder
        bdspec: a pair `(axis, side)`. `axis` denotes the time axis of `geo`,
            and `side` is either 0 for the "lower" boundary or 1 for the
            "upper" boundary.
        g0: a function which will be interpolated to obtain the initial
            function values
        g1: a function which will be interpolated to obtain the initial
            derivatives.
        physical (bool): whether the functions `g0` and `g1` are given in
            physical (True) or parametric (False) coordinates. Physical
            coordinates are assumed by default.

    Returns:
        A pair of arrays `(indices, values)` which denote the indices of the
        dofs within the tensor product basis which lie along the initial face
        of the space-time cylinder and their computed values, respectively.
    """
    bdax, bdside = bdspec

    bdbasis = list(kvs)
    del bdbasis[bdax]

    bdgeo = geo.boundary(bdax, bdside) if physical else None
    from .approx import interpolate
    coeffs01 = np.stack((  # coefficients for 0th and 1st derivatives, respectively
        interpolate(bdbasis, g0, geo=bdgeo).ravel(),
        interpolate(bdbasis, g1, geo=bdgeo).ravel()
    ))

    # compute 2x2 matrix which maps the two boundary coefficients to 0-th and 1-st derivative
    # at the boundary (only two basis functions have contributions there!)
    if bdside == 0:
        bdcolloc = bspline.active_deriv(kvs[bdax], 0.0, 1)[:2, :2] # first two basis functions
    else:
        bdcolloc = bspline.active_deriv(kvs[bdax], 1.0, 1)[:2, -2:] # last two basis functions

    # note: this only works for a space-time cylinder with constant geometry!
    coll_coeffs = np.linalg.solve(bdcolloc, coeffs01)

    # compute indices for the two boundary slices
    N = tuple(kv.numdofs for kv in kvs)
    firstidx = (0 if bdside==0 else -2)
    bdindices = np.concatenate((
        slice_indices(bdax, firstidx,   N, ravel=True),
        slice_indices(bdax, firstidx+1, N, ravel=True)
    ))

    return bdindices, coll_coeffs.ravel()

def combine_bcs(bcs):
    """Given a sequence of `(indices, values)` pairs such as returned by
    :func:`compute_dirichlet_bc`, combine them into a single pair
    `(indices, values)`.

    Dofs which occur in more than one `indices` array take their
    value from an arbitrary corresponding `values` array.
    """
    bcs = list(bcs)
    indices = np.concatenate([ind for ind,_ in bcs])
    values  = np.concatenate([val for _,val in bcs])
    assert indices.shape == values.shape, 'Inconsistent BC sizes'

    uidx, lookup = np.unique(indices, return_index=True)
    return uidx, values[lookup]

class RestrictedLinearSystem:
    """Represents a linear system with some of its dofs eliminated.

    Args:
        A: the full matrix
        b: the right-hand side (may be 0)
        bcs: a pair of arrays `(indices, values)` which contain the
            indices and values, respectively, of dofs to be eliminated
            from the system

    Once constructed, the restricted linear system can be accessed through
    the following attributes:

    Attributes:
        A: the restricted matrix
        b: the restricted and updated right-hand side
    """
    def __init__(self, A, b, bcs):
        indices, values = bcs
        if np.isscalar(b):
            b = np.broadcast_to(b, A.shape[0])
        if np.isscalar(values):
            values = np.broadcast_to(values, indices.shape[0])

        I = scipy.sparse.eye(A.shape[0], format='csr')
        # compute mask which contains non-eliminated dofs
        mask = np.ones(A.shape[0], dtype=bool)
        mask[list(indices)] = False

        # TODO/BUG: this may require the indices to be in increasing order?
        self.R_free = I[mask]
        self.R_elim = I[np.logical_not(mask)]
        self.values = values

        self.A = self.restrict_matrix(A)
        self.b = self.restrict(b - A.dot(self.R_elim.T.dot(values)))

    def restrict(self, u):
        """Given a vector `u` containing all dofs, return its restriction to the free dofs."""
        return self.R_free.dot(u)

    def restrict_matrix(self, B):
        """Given a matrix `B` which operates on all dofs, return its restriction to the free dofs."""
        return self.R_free.dot(B).dot(self.R_free.T)

    def extend(self, u):
        """Given a vector `u` containing only the free dofs, pad it with zeros to all dofs."""
        return self.R_free.T.dot(u)

    def complete(self, u):
        """Given a solution `u` of the restricted linear system, complete it
        with the values of the eliminated dofs to a solution of the original
        system.
        """
        return self.extend(u) + self.R_elim.T.dot(self.values)

################################################################################
# Integration
################################################################################

def integrate(kvs, f, f_physical=False, geo=None):
    """Compute the integral of the function `f` over the geometry
    `geo` or a simple tensor product domain.

    Args:
        kvs (seq): a sequence of :class:`pyiga.bspline.KnotVector`;
            determines the parameter domain and the quadrature rule
        f: a function or :class:`pyiga.geometry.BSplineFunc` object
        f_physical (bool): whether `f` is given in physical coordinates.
            If `True`, `geo` must be passed as well.
        geo: a :class:`pyiga.geometry.BSplinePatch` which describes
            the integration domain; if not given, the integral is
            computed in the parameter domain

    Returns:
        float: the integral of `f` over the specified domain
    """
    if isinstance(kvs, bspline.KnotVector):
        kvs = (kvs,)
    # compute quadrature rules
    nqp = max(kv.p for kv in kvs) + 1
    gaussgrid, gaussweights = make_tensor_quadrature([kv.mesh for kv in kvs], nqp)

    # evaluate function f on grid or transformed grid
    if f_physical:
        assert geo is not None, 'integrate in physical domain requires geometry'
        fvals = utils.grid_eval_transformed(f, gaussgrid, geo)
    else:
        fvals = utils.grid_eval(f, gaussgrid)

    # multiply function values with quadrature weights
    fvals = tensor.apply_tprod(
              [operators.DiagonalOperator(gw) for gw in gaussweights], fvals)
    # if geometry was specified, multiply by abs(det(jac))
    if geo is not None:
        geo_jac = geo.grid_jacobian(gaussgrid)
        geo_det = np.abs(assemble_tools.determinants(geo_jac))
        fvals *= geo_det
    # sum over all coordinate axes (leave vector components intact, if any)
    return fvals.sum(axis=tuple(range(len(kvs))))

################################################################################
# Driver routines for assemblers
################################################################################

def assemble(asm, symmetric=False, format='csr'):
    kvs0, kvs1 = asm.kvs
    X = MLStructure.from_kvs(kvs0, kvs1).make_mlmatrix()

    if isinstance(asm, assemble_tools.BaseAssembler2D):
        X.data = assemble_tools.generic_assemble_core_2d(asm, X.structure.bidx, symmetric=symmetric)
    elif isinstance(asm, assemble_tools.BaseAssembler3D):
        X.data = assemble_tools.generic_assemble_core_3d(asm, X.structure.bidx, symmetric=symmetric)
    else:
        assert False, 'Unknown assembler type'
    if format == 'mlb':
        return X
    else:
        return X.asmatrix(format)

def assemble_vector(asm, symmetric=False, format='csr', layout='blocked'):
    assert layout in ('packed', 'blocked')

    kvs0, kvs1 = asm.kvs
    dim = len(kvs0)
    nc = asm.num_components()[::-1]  # reverse axes (u = kv0 = columns)
    struc = MLStructure.from_kvs(kvs0, kvs1).join(MLStructure.dense(nc))
    X = struc.make_mlmatrix()

    if dim == 2:
        X.data = assemble_tools.generic_assemble_core_vec_2d(asm, X.structure.bidx[:dim], symmetric)
    elif dim == 3:
        X.data = assemble_tools.generic_assemble_core_vec_3d(asm, X.structure.bidx[:dim], symmetric)
    else:
        assert False, 'dimension %d not implemented' % dim

    if layout == 'blocked':
        axes = (dim,) + tuple(range(dim))    # bring last axis to the front
        X = X.reorder(axes)
    if format == 'mlb':
        return X
    else:
        return X.asmatrix(format)

################################################################################
# Convenience functions
################################################################################

def _detect_dim(kvs):
    if isinstance(kvs, bspline.KnotVector):
        return 1
    else:
        return len(kvs)

def mass(kvs, geo=None, format='csr'):
    """Assemble a mass matrix for the given basis (B-spline basis
    or tensor product B-spline basis) with an optional geometry transform.
    """
    dim = _detect_dim(kvs)
    if geo:
        assert geo.dim == dim, "Geometry has wrong dimension"
    if dim == 1:
        assert geo is None, "Geometry map not supported for 1D assembling"
        return bsp_mass_1d(kvs)
    elif dim == 2:
        return bsp_mass_2d(kvs, geo, format)
    elif dim == 3:
        return bsp_mass_3d(kvs, geo, format)
    else:
        assert False, "Dimensions higher than 3 are currently not implemented."

def stiffness(kvs, geo=None, format='csr'):
    """Assemble a stiffness matrix for the given basis (B-spline basis
    or tensor product B-spline basis) with an optional geometry transform.
    """
    dim = _detect_dim(kvs)
    if geo:
        assert geo.dim == dim, "Geometry has wrong dimension"
    if dim == 1:
        assert geo is None, "Geometry map not supported for 1D assembling"
        return bsp_stiffness_1d(kvs)
    elif dim == 2:
        return bsp_stiffness_2d(kvs, geo, format)
    elif dim == 3:
        return bsp_stiffness_3d(kvs, geo, format)
    else:
        assert False, "Dimensions higher than 3 are currently not implemented."

def divdiv(kvs, geo=None, layout='blocked', format='csr'):
    dim = _detect_dim(kvs)
    if geo is None:
        geo = geometry.unit_cube(dim=dim)   # TODO: fast assembling for div-div?
    if dim == 2:
        asm = assemblers.DivDivAssembler2D(kvs, geo)
    elif dim == 3:
        asm = assemblers.DivDivAssembler3D(kvs, geo)
    else:
        assert False, 'dimension %d not implemented' % dim
    return assemble_vector(asm, symmetric=True, layout=layout, format=format)

def mass_fast(kvs, geo=None, tol=1e-10, maxiter=100, skipcount=3, tolcount=3, verbose=2):
    """Assemble a mass matrix for the given tensor product B-spline basis with
    an optional geometry transform, using the fast low-rank assembling
    algorithm.
    """
    if geo is None:
        # the default assemblers use Kronecker product assembling if no geometry present
        return mass(kvs)
    dim = _detect_dim(kvs)
    assert geo.dim == dim, "Geometry has wrong dimension"
    if dim == 1:
        assert False, "Geometry map not supported for 1D assembling"
    elif dim == 2:
        asm = assemblers.MassAssembler2D(kvs, geo)
    elif dim == 3:
        asm = assemblers.MassAssembler3D(kvs, geo)
    else:
        assert False, "Dimensions higher than 3 are currently not implemented."
    return fast_assemble_cy.fast_assemble(asm, kvs, tol, maxiter, skipcount, tolcount, verbose)

def stiffness_fast(kvs, geo=None, tol=1e-10, maxiter=100, skipcount=3, tolcount=3, verbose=2):
    """Assemble a stiffness matrix for the given tensor product B-spline basis
    with an optional geometry transform, using the fast low-rank assembling
    algorithm.
    """
    if geo is None:
        # the default assemblers use Kronecker product assembling if no geometry present
        return stiffness(kvs)
    dim = _detect_dim(kvs)
    assert geo.dim == dim, "Geometry has wrong dimension"
    if dim == 1:
        assert False, "Geometry map not supported for 1D assembling"
    elif dim == 2:
        asm = assemblers.StiffnessAssembler2D(kvs, geo)
    elif dim == 3:
        asm = assemblers.StiffnessAssembler3D(kvs, geo)
    else:
        assert False, "Dimensions higher than 3 are currently not implemented."
    return fast_assemble_cy.fast_assemble(asm, kvs, tol, maxiter, skipcount, tolcount, verbose)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug  7 00:34:30 2017

@author: sparshkoyarala
"""

# Importing the Libraries

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the Datasets

dataset = pd.read_csv('Position_Salaries.csv')
X = dataset.iloc[:,1:2].values
y = dataset.iloc[:,-1].values

# Plotting the Data to analyse

plt.scatter(X, y, color='red')
plt.show()

# Fitting Linear Regression

from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(X, y)

# Fitting Polynomial Regression

from sklearn.preprocessing import PolynomialFeatures

poly_reg = PolynomialFeatures(degree=4)
X_poly = poly_reg.fit_transform(X)
lin_reg_2 = LinearRegression()
lin_reg_2.fit(X_poly, y)

# Visualising Linear Regression

plt.scatter(X, y, color = 'red')
plt.plot(X, lin_reg.predict(X), color = 'blue')
plt.title('Linear Regression')
plt.xlabel('Level')
plt.ylabel('Salary')
plt.show()

# Visualising Polynomial Regression

X_grid = np.arange(min(X), max(X), 0.1)
X_grid = X_grid.reshape((len(X_grid), 1))

plt.scatter(X, y, color = 'red')
plt.plot(X_grid, lin_reg_2.predict(poly_reg.fit_transform(X_grid)), color = 'blue')
plt.title('Polynomial Regression')
plt.xlabel('Level')
plt.ylabel('Salary')
plt.show()

# Predicting Linear Reg for 6.5

lin_reg.predict(6.5)

# Predicting Poly Reg for 6.5

lin_reg_2.predict(poly_reg.fit_transform(6.5))


import json

import stripe

from django.urls import reverse
from django.views.generic import ListView, DetailView
from django.http import HttpResponseRedirect
from django.conf import settings
from django.views import View
from django.http import JsonResponse

from rest_framework import generics

from .models import Course
from .serializers import CourseSerializer

class CourseListAPIView(generics.ListAPIView):
    queryset = Course.objects.all()
    serializer_class = CourseSerializer

class CourseDetailAPIView(generics.RetrieveAPIView):
    queryset = Course.objects.all()
    serializer_class = CourseSerializer

#################
# direct charge #
#################

class CourseChargeView(View):

    def post(self, request, *args, **kwargs):
        stripe.api_key = settings.STRIPE_SECRET_KEY
        json_data = json.loads(request.body)
        course = Course.objects.filter(id=json_data['course_id']).first()
        fee_percentage = .01 * int(course.fee)
        try:
            customer = get_or_create_customer(
                # self.request.user.email,
                # pull the email off the request
                json_data['email'],
                json_data['token'],
                course.seller.stripe_access_token,
                course.seller.stripe_user_id,
            )
            charge = stripe.Charge.create(
                amount=json_data['amount'],
                currency='usd',
                customer=customer.id,
                description=json_data['description'],
                application_fee=int(json_data['amount'] * fee_percentage),
                stripe_account=course.seller.stripe_user_id,
            )
            if charge:
                return JsonResponse({'status': 'success'}, status=202)
        except stripe.error.StripeError as e:
            return JsonResponse({'status': 'error'}, status=500)

# helpers

def get_or_create_customer(email, token, stripe_access_token, stripe_account):
    stripe.api_key = stripe_access_token
    connected_customers = stripe.Customer.list()
    for customer in connected_customers:
        if customer.email == email:
            print(f'{email} found')
            return customer
    print(f'{email} created')
    return stripe.Customer.create(
        email=email,
        source=token,
        stripe_account=stripe_account,
    )


# -*- coding: utf-8 -*-
"""
Fields and widgets
"""
import json, copy

from django import forms
from django.core.urlresolvers import reverse

from django.forms.widgets import flatatt
from django.utils.html import escape
from django.utils.safestring import mark_safe

from djangocodemirror import settings_local
from djangocodemirror.config import ConfigManager

class CodeMirrorWidget(forms.Textarea):
    """
    Widget to add a CodeMirror or DjangoCodeMirror instance on a textarea
    Take the same arguments than ``forms.Textarea`` and accepts one suplementary
    optionnal arguments :

    * ``config_name`` name of the settings to use, a valid key name from
    ``settings.CODEMIRROR_SETTINGS``. Default to "default" that is the default config
    with minimal options;
    """
    def __init__(self, attrs=None, config_name='default', **kwargs):
        super(CodeMirrorWidget, self).__init__(attrs=attrs, **kwargs)
        self.config_name = config_name

    def init_editor_config(self):
        return ConfigManager(
            config_name=self.config_name,
        )

    def render(self, name, value, attrs=None):
        if not hasattr(self, "editor_config_manager"):
            self.editor_config_manager = self.init_editor_config()

        final_attrs = self.build_attrs(attrs, name=name)

        html = [super(CodeMirrorWidget, self).render(name, value, attrs)]
        # Append HTML for the Javascript settings just below the textarea
        if self.editor_config_manager.settings['embed_settings']:
            html.append(self._build_codemirror_settings(final_attrs))

        return mark_safe(u'\n'.join(html))

    def _build_codemirror_settings(self, final_attrs):
        """build HTML for the Javascript settings"""
        html = settings_local.DJANGOCODEMIRROR_FIELD_INIT_JS
        if self.editor_config_manager.settings['codemirror_only']:
            html = settings_local.CODEMIRROR_FIELD_INIT_JS
        return html.format(inputid=final_attrs['id'], settings=json.dumps(self.editor_config_manager.editor_config))

    @property
    def media(self):
        """
        Adds necessary files (Js/CSS) to the widget's medias
        """
        if not hasattr(self, "editor_config_manager"):
            self.editor_config_manager = self.init_editor_config()
        css, js = self.editor_config_manager.find_assets()
        return forms.Media(
            css={"all": css},
            js=js
        )


from lib import blocks
from lib import creature as cre

def test_the_creation_of_a_grid():
    # A grid holds the colors of the squares

    # We need a map to return the color for the grid. It starts off grey so that's all we need. It's a function in case we want to randomize it for fun
    color_map = {'grey': lambda: (1, 128, 1)}

    assert(len(blocks.create_grid(3, 3, color_map)) == 3)

def test_move_a_creature_around():
    # create a creature and a grid and make it move around

    # a creature is just a position right now
    creature = cre.create_creature()

    grid = blocks.create_grid(3, 3, _make_some_colors())

    # now move the creature a few times and see if it explodes
    for _ in range(10):
        grid, creature = blocks.move_creature(creature, grid, _make_some_colors())
    assert(creature)

def test_creature_moving_onto_food_heals(monkeypatch):
    creature = cre.create_creature()
    color_map = _make_some_colors()
    grid = blocks.create_grid(3, 3, _make_some_colors())
    food = cre.create_creature(0, 1)
    foods_map = {(0, 1): food}

    # put the food on the grid
    grid = blocks.set_food_onto_grid(grid, foods_map, color_map)

    # make the creature move to the right, where we put the food
    def move_right(creature):
        return 'right'
    monkeypatch.setattr(cre, '_ask_creature_where_to_move_to', move_right)

    grid, creature = blocks.move_creature(creature, grid, _make_some_colors())

    # eat
    creature, foods_map, new_food = cre.handle_eating(creature, foods_map)
    assert(creature['hp'] == 550)

def _make_some_colors():
    color_map = {
        'yellow': lambda: (1, 1, 1),
        'grey': lambda: (1, 128, 1),
        'blue': lambda: (1, 1, 1)
    }
    return color_map


from model import *
import sys
import cv2
import imutils

img = cv2.imread("./tmp.png")
img = cv2.resize(img, (28, 28))

img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

_, img = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY_INV)

cv2.imshow("window", img)
cv2.waitKey(0)

Model = DigitClassifier(28, 10)
print("loading...")
Model.load("./RS18_09model.pth")
print("loaded")

P = Model.Predict(np.array([img]))
print(P)


import typing
import json

import rx
import rx.operators as ops
import rxsci.io.file as file
import rxsci.framing.line as line

def load(skip=0, ignore_error=False):
    ''' Loads a json observable.

    The source observable must emit one JSON string per item
    The source must be an Observable.

    Args:
        skip: number of items to skip before parsing
        ignore_error: Ignore errors while parsing JSON

    Returns:
        An observable of dicts corresponding to the source json content.
    '''
    def load_json(i):
        try:
            i = json.loads(i)
        except Exception as e:
            if ignore_error is True:
                print(f"{e}: Ignoring this object.")
                return None
            else:
                raise e
        return i

    def _load(source):
        return source.pipe(
            ops.skip(skip),
            ops.map(load_json),
            ops.filter(lambda i: i is not None),
        )

    return _load

def load_from_file(
    filename,
    lines=True, skip=0,
    ignore_error=False, encoding=None
):
    ''' Loads a json file.

    This factory loads the provided file. The format of the returned observable
    depends on the *lines* parameter.

    Args:
        filename: Path of the file to read or a file object
        lines: Parse file as a JSON Line when set to True, as a single JSON object otherwise.
        skip: [Optional] Number of lines to skip before parsing
        encoding [Optional] Encoding used to parse the text content

    Returns:
        An observable of namedtuple items, where each key is a csv column
    '''

    if lines is True:
        return file.read(filename, size=64*1024, encoding=encoding).pipe(
            line.unframe(),
            load(skip=skip, ignore_error=ignore_error),
        )
    else:
        return file.read(filename, size=-1, encoding=encoding).pipe(
            load(skip=skip, ignore_error=ignore_error),
        )

def dump(newline='\n'):
    ''' dumps an observable to JSON.

    If several the source observable emits several items, then they are framed
    as JSON line.
    The source must be an Observable.

    Args:
        newline: [Optional] Character(s) used for end of line.

    Returns:
        An observable string items, where each item is a csv line.
    '''
    def _dump(source):
        def on_subscribe(observer, scheduler):

            def on_next(i):
                line = json.dumps(i)
                line += newline
                observer.on_next(line)

            return source.subscribe(
                on_next=on_next,
                on_completed=observer.on_completed,
                on_error=observer.on_error,
                scheduler=scheduler,
            )
        return rx.create(on_subscribe)

    return _dump

def dump_to_file(
    filename,
    newline='\n',
    encoding=None
):
    ''' dumps each item to a JSON file.

    The source must be an Observable.

    Args:
        filename: Path of the file to read or a file object
        newline: [Optional] Character(s) used for end of line.
        encoding [Optional] Encoding used to parse the text content

    Returns:
        An empty observable that completes on success when the source
        observable completes or completes on error if there is an error
        while writing the csv file.
    '''
    def _dump_to_file(source):
        mode = None
        if encoding is not None:
            mode = 'wb'
        return source.pipe(
            dump(newline=newline),
            ops.map(lambda i: i.encode(encoding) if encoding is not None else i),
            file.write(
                file=filename,
                mode=mode,
            ),
        )

    return _dump_to_file


import json
import time
import requests

import main

# 获取某个课的作业列表
def GetHomeworkList(auth, courseOpenId):
    reps = requests.post('http://api.hnscen.cn/mobile/api/GetHomeworkList',
                         {'auth_fix': auth, 'courseOpenId': courseOpenId}).text
    print(reps)
    homeworklistInfo = json.loads(reps)
    return homeworklistInfo

# 获取作业详情
def GetHomeworkInfo(auth, homeworkId):
    reps = requests.post('http://api.hnscen.cn/mobile/api/GetHomeworkInfo',
                         {'auth_fix': auth, 'homeworkId': homeworkId}).text
    homeworkInfo = json.loads(reps)
    return homeworkInfo

# 获取作业信息
def getPaper(UserId, WorkId):
    print(WorkId)
    reps = requests.get(
        'http://api.hnscen.cn/mobile/api/getPaper', params={'UserId': UserId, 'WorkId': WorkId}).text
    homeworkContextInfo = json.loads(reps)
    return homeworkContextInfo

def SaveObjectQuestion(Id, data):
    reps = requests.post('http://api.hnscen.cn/mobile/api/SaveObjectQuestion', {'Id': Id, 'data': str(data)})

def SaveWork(Id, data, CourseOpenId, PaperId):
    reps = requests.post('http://api.hnscen.cn/mobile/api/SaveWork',
                         {'Id': Id, 'data': str(data), 'Type': 2, 'CourseOpenId': CourseOpenId, 'IsSubmitWithoutAll': 1,
                          'PaperId': PaperId})
    return reps

if __name__ == "__main__":
    # 获取登陆返回的信息
    logininfo = main.login('username', 'password', None)
    time.sleep(0.1)
    # auth好像比较重要，每个方法都基本上用到了
    print(logininfo)
    auth = logininfo['auth']
    main.getuserinfo(auth)
    # 获取所有的课
    Course = main.MyCourse(auth)
    time.sleep(0.1)
    print('课程列表-----------------------')
    for index, i in enumerate(Course['course']):
        print(index, ':课程名字：' + i['name'] + '  进度：' + str(i['process']) + '%')
    print('请选择课程：')
    index = input()
    courseOpenId = Course['course'][int(index)]['courseOpenId']
    homeworkList = GetHomeworkList(auth, courseOpenId)
    for index, i in enumerate(homeworkList['data']):
        print(index, '作业名称：' + i['Name'])
    print('请选择作业：')
    index = input()
    print(homeworkList['data'][int(index)]['Id'])
    homeworkId = homeworkList['data'][int(index)]['Id']
    print(homeworkId)
    homeworkInfo = GetHomeworkInfo(auth, homeworkId)
    print(logininfo['userId'])
    homeworkContext = getPaper(logininfo['userId'], homeworkId)
    PaperId = homeworkContext['Paper']['Id']
    PaperPaperId = homeworkContext['Paper']['PaperId']
    data = []
    for index, i in enumerate(homeworkContext['Paper']['BigQuestions']):
        print(i['Title'])
        for index1, j in enumerate(i['StuQuestions']):
            AnswerRes = {
                "Bindex": index,
                "Qindex": index1,
                "StuAnswer": j['Answer'],
                "IsAssignmented": 1
            }
            SaveObjectQuestion(PaperId, AnswerRes)
            data.append(AnswerRes)
    print(SaveWork(PaperId, data, courseOpenId, PaperPaperId).text)


#!/usr/bin/env python2
# vim:fileencoding=utf-8

import gaetalk
import logging
import datetime
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.api import xmpp

class Userdeactive(webapp.RequestHandler):
  def get(self):
    self.response.headers['Content-Type'] = 'text/plain'
    for u in gaetalk.User.all():
      if u.jid.endswith('@gmail.com'):
        if u.avail != gaetalk.OFFLINE and 'fakeresouce' not in u.resources:
          if not xmpp.get_presence(u.jid):
            del u.resources[:]
            u.avail = gaetalk.OFFLINE
            u.last_offline_date = datetime.datetime.now()
            u.put()
            self.response.out.write(u.jid + ' should be offline.\n')
    self.response.out.write(u'OK.'.encode('utf-8'))

class Userdedup(webapp.RequestHandler):
  def get(self):
    users = {}
    for u in gaetalk.User.all():
      if u.jid in users:
        users[u.jid].append(u)
      else:
        users[u.jid] = [u]
    for k, v in users.items():
      if len(v) == 1:
        continue
      v.sort(key=lambda u: gaetalk.STATUS_LIST.index(u.avail))
      logging.error(' '.join([x.avail for x in v]))
      for i in v[1:]:
        l = gaetalk.Log(msg=u'删除重复用户', jid=i.jid,
                         nick=i.nick, type='misc')
        l.put()
        i.delete()
    self.response.out.write(u'OK.'.encode('utf-8'))

application = webapp.WSGIApplication(
  [
    ('/_admin/userdedup', Userdedup),
    ('/_admin/userdeactive', Userdeactive),
  ],
  debug=True)

def main():
  run_wsgi_app(application)

if __name__ == "__main__":
  main()


#%%
import numpy as np
from matplotlib import pyplot as plt
from scipy.signal import find_peaks
import time

# from soen_sim import input_signal, synapse, dendrite, neuron
from _plotting import plot_wr_comparison__synapse__spd_jj_test
from _functions import read_wr_data, chi_squared_error
from util import physical_constants
from soen_sim import input_signal, synapse
p = physical_constants()

plt.close('all')

#%%
# spike_times = [5e-9,55e-9,105e-9,155e-9,205e-9,255e-9,305e-9,355e-9,505e-9,555e-9,605e-9,655e-9,705e-9,755e-9,805e-9,855e-9]
spike_times = [5e-9]

dt = 0.01e-9
tf = 300e-9 # 1e-6

# create sim_params dictionary
sim_params = dict()
sim_params['dt'] = dt
sim_params['tf'] = tf

#%%
target_data_array = []
actual_data_array = []
error_array = []

#%% vary Isy
I_sy_vec = [39e-6] # [21e-6,39e-6] # [21e-6,27e-6,33e-6,39e-6]

num_files = len(I_sy_vec)
t_tot = time.time()
load_wr = True
for ii in range(num_files): # range(1): #

    print('\nvary Isy, ii = {} of {}\n'.format(ii+1,num_files))

    #load WR data
    if load_wr == True:
        # file_name = 'syn_1jj_Ispd20.00uA_trep50ns_Isy{:04.2f}uA_noSI_dt10.0ps_tsim1000ns.dat'.format(I_sy_vec[ii]*1e6)
        file_name = 'syn_1jj_Ispd20.00uA_Isy{:04.2f}uA_noSI_dt00.2ps_tsim1000ns.dat'.format(I_sy_vec[ii]*1e6)
        data_dict = read_wr_data('wrspice_data/test_data/1jj/'+file_name)
        target_drive = np.vstack((data_dict['time'],data_dict['L0#branch']))
        target_data = np.vstack((data_dict['time'],data_dict['L2#branch']))
        target_data_array.append(target_data)

    # find fluxon peaks
    time_vec = data_dict['time']
    I_sf_wr = data_dict['L2#branch']
    V_sf_wr = data_dict['v(4)']
    j_peaks, _ = find_peaks(V_sf_wr, height = 100e-6)

    # find inter-fluxon intervals and fluxon generation rates
    j_ifi = np.diff(time_vec[j_peaks])
    j_rate = 1/j_ifi

    # calculate average currents, voltages, and times
    V_sf_wr_avg = np.zeros([len(j_peaks)-1])
    I_sf_wr_avg = np.zeros([len(j_peaks)-1])
    time_avg = np.zeros([len(j_peaks)-1])

    for jj in range(len(j_peaks)-1):
        ind_vec = np.arange(j_peaks[jj],j_peaks[jj+1],1)
        V_sf_wr_avg[jj] = np.sum(V_sf_wr[ind_vec])/len(ind_vec)
        I_sf_wr_avg[jj] = np.sum(I_sf_wr[ind_vec])/len(ind_vec)
        time_avg[jj] = np.sum(time_vec[ind_vec])/len(ind_vec)

    # initialize input signal
    input_1 = input_signal('in', input_temporal_form = 'arbitrary_spike_train', spike_times = spike_times)

    # initialize synapse
    synapse_1 = synapse('sy', num_jjs = 1, synaptic_bias_currents = [I_sy_vec[ii]], input_signal_name = 'in', synapse_model_params = sim_params)

    synapse_1.run_sim()

    I_sf = synapse_1.I_sf
    V_sf = synapse_1.V_sf
    r_spd1 = synapse_1.r_spd1
    I_spd2 = I_sf - I_sy_vec[ii]
    j_sf_state = synapse_1.j_sf_state

    actual_drive = np.vstack((synapse_1.time_vec[:],I_spd2[:]))
    actual_data = np.vstack((synapse_1.time_vec[:],I_sf[:]))
    actual_data_array.append(actual_data)

    # error_drive = chi_squared_error(target_drive,actual_drive)
    # error_signal = chi_squared_error(target_data,actual_data)
    # error_array.append(error_signal)

    plot_wr_comparison__synapse__spd_jj_test(file_name,spike_times,target_drive,actual_drive,target_data,actual_data,file_name,V_sf_wr,j_peaks,V_sf_wr_avg,time_avg,V_sf)
    # plot_wr_comparison__synapse__spd_jj_test(file_name,spike_times,target_drive,actual_drive,target_data,actual_data,file_name,error_drive,error_signal,V_sf_wr,j_peaks,V_sf_wr_avg,time_avg,V_sf)

    # plot_wr_comparison__synapse__Isi_and_Isf('bias_lower all; J_sf criterion',spike_times,target_drive,actual_drive,target_data,actual_data,sf_data,synapse_1.I_c,synapse_1.I_reset,file_name,error_drive,error_signal)

elapsed = time.time() - t_tot
print('soen_sim duration = '+str(elapsed)+' s for vary I_sy')