import torch
import torch.nn as nn

from utils.operations import makeList

class Dense(nn.Module):
    """
        Implements Dense layer
        Linear -> Activation -> BatchNorm(optional)
    """
    def __init__(self, input_dim, output_dim, activation='relu', batch_norm=False):
        super(Dense, self).__init__()

        if isinstance(activation, str):
            if activation=='relu':
                activation = nn.ReLU()
            elif activation=='sigmoid':
                activation = nn.Sigmoid()
            elif activation=='leaky_relu':
                activation = nn.LeakyReLU()
            elif activation=='tanh':
                activation = nn.Tanh()
            else:
                raise AssertionError("Unknown activation {}, please provide with \
                    proper actiavtion function.".format(activation))

        self.linear = nn.Linear(input_dim, output_dim)
        self.activation = activation
        self.batch_norm = nn.BatchNorm1d() if batch_norm else None

    def forward(self, x):
        y = self.linear(x)
        y = self.activation(y)
        if self.batch_norm:
            y = self.batch_norm(y)
        return y

class DeepLayer(nn.Module):

    """
        Implements modulelist of Dense layers
        Dense -> Dense -> Dense -> ...
    """
    def __init__(self, size=512, n_layers=5, d_factor=2, activation='relu', bn=False):
        super(DeepLayer, self).__init__()

        if not isinstance(d_factor, list):
            d_factor = [d_factor]*n_layers

        self.size = size
        self.n_layers = n_layers
        self.layers = []
        for i in range(n_layers):
            self.layers.append(Dense(size, int(size//d_factor[i]), activation=activation, batch_norm=bn))
            size = int(size//d_factor[i])
        self.layers = nn.ModuleList(self.layers)

    def cuda():
        super(DeepLayer, self).cuda()
        for i in range(self.n_layers):
            self.layers[i] = self.layers[i].cuda()

    def forward(self, x):
        y = x
        for layer in self.layers:
            y = layer(y)
        return y

class CascadedDeep(nn.Module):
    """
        Implements cascaded deep layer
        Deep -> Deep -> Deep -> ...... -> Deep
    """
    def __init__(self, size, n_layers, d_factors,
                    activations='relu', bns=False):
        super(CascadedDeep, self).__init__()

        if not isinstance(activations, list):
            activations = [activations] * len(n_layers)
        if not isinstance(bns, list):
            bns = [bns] * len(n_layers)

        self.size, self.ret_size = size, size
        self.n_layers = n_layers
        self.d_factors = d_factors
        self.activations = activations
        self.bns = bns

        self.layers = []

        for layer_idx, (n_layer, d_factor, activation, bn) in \
            enumerate(zip(n_layers, d_factors, activations, bns)):

            layer = DeepLayer(
                size=size,
                n_layers=n_layer,
                d_factor=d_factor,
                activation=activation,
                bn=bn
            )
            self.layers.append(layer)
            self.add_module(
                "deeplayer_{}".format(layer_idx+1),
                layer
            )

            size = int(size // (d_factor ** n_layer))

        self.ret_size = size

    def cuda(self):
        super(CascadedDeep, self).cuda()
        self.layers = self.layers.cuda()

    def forward(self, x):
        y = x
        for layer in self.layers:
            y = layer(y)
        return y

    def outSize(self):
        return self.ret_size

class InceptionLayer(nn.Module):

    """
        Implements Inception Layer
        Args:
            n_branches : No of parallel branches
            size : Input size
            n_layers : (list / int) contains no of layers for each branch
            d_factors : (list / int) contains d_factor for each branch
            activations : (list / int) contains activations for each branch
            bns : (list / int) contains batch_norm for each branch
            out :
                'add' / 'sum' => add all the final outputs
                'cat' => concatenate all the final outputs
    """
    def __init__(self, n_branches, size, n_layers, d_factors,
                activations='relu', bns=False,
                out='add'):

        super(InceptionLayer, self).__init__()

        self.n_branches = n_branches
        self.size = size
        self.n_layers = makeList(n_layers, n_branches)
        self.d_factors = makeList(d_factors, n_branches)
        self.activations = makeList(activations, n_branches)
        self.bns = makeList(bns, n_branches)
        self.out = out

        self.branches = []

        for branch_idx, (d_factor, activation, bn) in enumerate(zip(self.d_factors, self.activations, self.bns)):
            branch_layers, size = [], self.size
            for i in range(self.n_layers[branch_idx]):
                branch_layers.append(Dense(size, int(size//d_factor), activation=activation, batch_norm=bn))
                size = int(size//d_factor)
            self.branches.append(nn.ModuleList(branch_layers))

        for branch_idx, branch in enumerate(self.branches):
            self.add_module(str(branch_idx), branch)

    def cuda():
        super(DeepLayer, self).cuda()
        for i in range(self.branches):
            self.branches[i] = self.branches[i].cuda()

    def forward(self, x):
        y = []
        for branch in self.branches:
            y_ = branch(x)
            y.append(y_)

        if self.out=='add' or self.out=='sum':
            y = sum(y)
        else:
            y = torch.cat(y, dim=0)

        return y


import load_data
import numpy as np
import scipy.sparse as sp
import pandas as pd
from sklearn.externals.joblib import Memory
memory = Memory('data/cache/')

nr_levels = {'a': 11, 'b': 91, 'c': 439, 'd': 36122}

#@memory.cache
def cat_matrices(path='data/', is_test=False, min_freq=0):
    path += 'testData.csv' if is_test else'trainingData.csv'
    if min_freq > 0:
        cat_to_nr, _ = load_data.freq_mapping(min_freq)
        tmp, _ = load_data.cat_mapping()
        cat_to_nr['ab'] = tmp['ab']
        cat_to_nr['bc'] = tmp['bc']
        cat_to_nr['abc'] = tmp['abc']
    else:
        cat_to_nr, _ = load_data.cat_mapping()
    n_lines = load_data.file_len(path)

    dok_cats = {cat: sp.lil_matrix((n_lines, len(cat_to_nr[cat])),
                dtype=np.float64) for cat in ['a', 'b', 'c', 'd']}
    unique_items = []

    with open(path) as f:
        for nr, line in enumerate(f):
            splited = line.split(',')
            unique_items.append(splited[-1].count(';') + 1)
            for click in splited[-1].split(';'):

                cs = click.split('/')[:-1]
                dok_cats['a'][nr, cat_to_nr['a'][cs[0]]] += 1
                if cs[1] in cat_to_nr['b']:
                    dok_cats['b'][nr, cat_to_nr['b'][cs[1]]] += 1
                else:
                    dok_cats['b'][nr, cat_to_nr['b']['unknown']] += 1
                if cs[2] in cat_to_nr['c']:
                    dok_cats['c'][nr, cat_to_nr['c'][cs[2]]] += 1
                else:
                    dok_cats['c'][nr, cat_to_nr['c']['unknown']] += 1
                if cs[3] in cat_to_nr['d']:
                    dok_cats['d'][nr, cat_to_nr['d'][cs[3]]] += 1
                else:
                    dok_cats['d'][nr, cat_to_nr['d']['unknown']] += 1

    if min_freq > 0:
        for cat in ['a', 'b', 'c', 'd']:
            dok_cats[cat + '_freq' + str(min_freq)] = dok_cats.pop(cat)
    return dok_cats

def get_range(id_, pad):
    max_ = {'A': 11, 'B': 91, 'C': 439, 'D': 33489}
    max_ = {'A': 11, 'B': 91, 'C': 439, 'D': 36122}
    pos = int(id_[1:])
    return range(max(0, pos - pad), min(max_[id_[0]], pos + pad))

def dummy_encoding(x, n_levels):
    """ assumes values are in [0, n_levels]
    """
    n_samples = len(x)
    n_features = n_levels
    X = sp.dok_matrix((n_samples, n_features), dtype=np.float64)
    for i in range(len(x)):
        X[i, x[i]] = 1
    return X.tocsc()

def neighbors(cats, doc, is_test):
    max_shift = doc['max_shift']
    results = {}
    for f in doc['shift_features']:
        X = cats[f].tocsr()
        n_samples, n_features = X.shape
        X_coll = None
        for dist in range(1, max_shift + 1):
            X_left, X_right, X_shift = matrix_shift(X, dist)
            if X_coll is None:
                X_coll = X_shift
            else:
                X_coll = X_coll + X_shift
                results[f + '1-' + str(dist)] = X_shift
            assert X_shift.shape == X.shape
            results[f + str(dist)] = X_shift
    return results

def matrix_shift(X, shift):
    n_samples = X.shape[0]
    i_shift = np.arange(n_samples) + shift
    padding = i_shift > n_samples - 1
    i_shift[padding] = 0
    X_right = X[i_shift, :]
    X_right[padding, :] = 0

    i_shift = np.arange(n_samples) - shift
    padding = i_shift < 0
    i_shift[padding] = 0
    X_left = X[i_shift, :]
    X_left[padding, :] = 0
    return X_left, X_right, X_left + X_right


from django.contrib.auth.models import AnonymousUser, User
from django.shortcuts import redirect

from Badges.models import CourseConfigParams
from Instructors.models import Courses
from Students.models import Student, StudentConfigParams

class CourseConfigMiddleware:
    ''' This middleware will prevent students (and instructors) from being able to access pages by url
        that should only be available if certain config settings are enabled

        The paths dictionary is a collection of url paths and course config/student config variables that
        should have a certain value. If the current config setting for a user doesn't match any of the
        values for that url, the user will be sent back to the course home page.
    '''
    def __init__(self, get_response):
        self.get_response = get_response
        # One-time configuration and initialization.
        self.paths = {
            '/oneUp/students/Announcements':
                {
                    'announcementsUsed': True,
                },
            '/oneUp/students/Leaderboard':
                {
                    'leaderboardUsed': True,
                    'skillLeaderboardDisplayed': True,
                    'gamificationUsed': True,
                },
            '/oneUp/students/LeaderboardInfo':
                {
                    'leaderboardUsed': True,
                    'gamificationUsed': True,
                },
            '/oneUp/badges/CourseBadges':
                {
                    'badgesUsed': True,
                    'gamificationUsed': True,
                },
            '/oneUp/students/VirtualCurrencyRules':
                {
                    'virtualCurrencyUsed': True,
                    'gamificationUsed': True,
                },
            '/oneUp/students/ChallengesWarmUpList':
                {
                    'warmupsUsed': True,
                },
            '/oneUp/students/ChallengesList':
                {
                    'seriousChallengesUsed': True,
                },
            '/oneUp/students/ActivityList':
                {
                    'activitiesUsed': True,
                },
            '/oneUp/students/CoursePerformance':
                {
                    'gradebookUsed': True,
                },
            '/oneUp/students/achievements':
                {
                    'displayAchievementPage': True,

                },
            '/oneUp/students/avatar':
                {
                    'avatarUsed': True,
                    'gamificationUsed': True,
                    'useCustomAvatar': False,
                },
            '/oneUp/students/AvatarEditor':
                {
                    'avatarUsed': True,
                    'gamificationUsed': True,
                    'useCustomAvatar': True,
                },
            '/oneUp/students/goalslist':
                {
                    'goalsUsed': True,
                    'gamificationUsed': True,
                },
            '/oneUp/chat/api':
                {
                    'chatUsed': True,
                },
            '/oneUp/students/VirtualCurrencyShop':
                {
                    'virtualCurrencyUsed': True,
                    'gamificationUsed': True,
                },
            '/oneUp/students/Transactions':
                {
                    'virtualCurrencyUsed': True,
                    'gamificationUsed': True,
                },
            '/oneUp/students/Callouts':
                {
                    'classmatesChallenges': True,
                    'gamificationUsed': True,
                },

            # TODO: Add links for instructors?
            '/oneUp/instructors/preferences':
                {
                    'gamificationUsed': True,
                },
            '/oneUp/badges/VirtualApplauseEarnRuleList':
                {
                    'gamificationUsed': True,
                    'applauseOn': True,
                },
            '/oneUp/badges/CreateVirtualApplauseRule':
                {
                    'gamificationUsed': True,
                    'applauseOn': True,
                },
        }
        self.reset_course_paths = ['/oneUp/students/StudentHome', '/oneUp/instructors/instructorHome']

        # not sure about caching these as we can have many users requesting at once
        self.course_config_params = None
        self.student_config_params = None
        self.current_course = None
        self.current_user = None

    def __call__(self, request):
        # Code to be executed for each request before
        # the view (and later middleware) are called.
        # print(f'PATH {request.path}')

        # Check if we are trying to go to a page that should have no course selected and delete the course session
        if request.path in self.reset_course_paths:
            self.course_config_params = None
            self.current_course = None
            if 'currentCourseID' in request.session:
                del request.session['currentCourseID']

        if request.user == AnonymousUser:
            self.student_config_params = None
            self.current_user = None
            self.course_config_params = None
            self.current_course = None

        # If request in paths to check and user is not fully logged out..
        if request.path in self.paths and not request.user == AnonymousUser:
            # Get current course
            if 'currentCourseID' in request.session:
                course = Courses.objects.get(pk=int(request.session['currentCourseID']))
            else:
                course = None

            # Get current student
            if 'userID' in request.GET:
                # This is a teacher viewing as a student
                stud = User.objects.get(username=request.GET['userID'])
                student = Student.objects.filter(user=stud).first()
            else:
                # This will also get the teacher student object (test student)
                student = Student.objects.filter(user=request.user).first()

            # Get the latest course and student config parameters
            self.update_params(course, student)
            # ccparams, scparams = self.update_params_constant(course, student)
            # print(self.course_config_params, self.student_config_params)
            # We check the configs fields to see if this request is valid
            if self.validate_request(request.path, ccparams=self.course_config_params, scparams=self.student_config_params) is False:
                if course:
                    if student and student.user.groups.filter(name='Teachers').exists() or not student:
                        return redirect('/oneUp/instructors/instructorCourseHome')
                    else:
                        return redirect('/oneUp/students/StudentCourseHome')
                else:
                    if student and student.user.groups.filter(name='Teachers').exists() or not student:
                        return redirect('/oneUp/instructors/instructorHome')
                    else:
                        return redirect('/oneUp/students/StudentHome')

        response = self.get_response(request)

        # Code to be executed for each request/response after
        # the view is called.

        return response

    def update_params(self, course, student):
        if course:
            self.course_config_params = CourseConfigParams.objects.get(courseID=course)
            self.current_course = course
        if course and student:
            self.student_config_params = StudentConfigParams.objects.get(courseID=course, studentID=student)
            self.current_user = student

        # print(self.__dict__)

    def validate_request(self, path, ccparams=None, scparams=None):
        ''' Check corresponding configuration fields for a path that
            should have a specific value
        '''
        fields_to_check = self.paths[path]

        for field, value in fields_to_check.items():
            if ccparams and hasattr(ccparams, field):
                if getattr(ccparams, field) is not value:
                    return False
            elif scparams and hasattr(scparams, field):
                if getattr(scparams, field) is not value:
                    return False
            else:
                return False

        return True

    def update_params_constant(self, course, student):
        ccparams = None
        scparams = None
        if course:
            ccparams = CourseConfigParams.objects.get(courseID=course)
            if student:
                scparams = StudentConfigParams.objects.get(courseID=course, studentID=student)

        return ccparams, scparams


import asyncio
import click
import csv

from gen3.tools.metadata.discovery import (
    publish_discovery_metadata,
    output_expanded_discovery_metadata,
    try_delete_discovery_guid,
    combine_discovery_metadata,
)
from gen3.tools.metadata.discovery_objects import (
    output_discovery_objects,
    publish_discovery_object_metadata,
    try_delete_discovery_objects,
    try_delete_discovery_objects_from_dict,
    is_valid_object_manifest,
)
from gen3.utils import get_or_create_event_loop_for_thread

@click.group()
def discovery():
    """Commands for reading and editing discovery metadata"""
    pass

@click.command()
@click.argument("file", required=False)
@click.option(
    "--default-file",
    "use_default_file",
    is_flag=True,
    help="Publishes {commons}-{guid_type}.tsv from current directory",
    show_default=True,
)
@click.option(
    "--omit-empty",
    "omit_empty",
    is_flag=True,
    help="omit fields from empty columns if set",
    show_default=True,
)
@click.option(
    "--guid-type",
    "guid_type",
    default="discovery_metadata",
    help=(
        "The value of this gets set as _guid_type in the root level metadata. "
        "discovery_metadata is the default that enables the Gen3 Discovery Page to visualize the results."
    ),
    show_default=True,
)
@click.option(
    "--guid_field",
    "guid_field",
    help=(
        'The column / field name within the metadata that will be used as GUIDs, if not specified, will try to find a column \ field named "guid" from the metadata.'
        "If that field doesn't exists in a certain metadata record, that record will be skipped from publishing."
    ),
    default=None,
    show_default=True,
)
@click.pass_context
def discovery_publish(ctx, file, use_default_file, omit_empty, guid_type, guid_field):
    """
    Run a discovery metadata ingestion on a given metadata TSV / JSON file with guid column / field.
    If [FILE] is omitted and --default-file not set, prompts for TSV / JSON file name.
    """
    if is_valid_object_manifest(file):
        click.confirm(
            "WARNING! It appears like you are attempting to publish discovery **objects** (based on the file provided) but you are attempting to use the non-object, dataset-level command `gen3 discovery publish`. Perhaps you meant to use `gen3 discovery objects publish`. Are you sure you want proceed with publishing the dataset-level discovery metadata using the object file?",
            abort=True,
        )

    auth = ctx.obj["auth_factory"].get()
    if not file and not use_default_file:
        file = click.prompt("Enter discovery metadata TSV / JSON file to publish")

    loop = get_or_create_event_loop_for_thread()
    endpoint = ctx.obj.get("endpoint")
    loop.run_until_complete(
        publish_discovery_metadata(
            auth,
            file,
            endpoint=endpoint,
            omit_empty_values=omit_empty,
            guid_type=guid_type,
            guid_field=guid_field,
        )
    )

@click.command()
@click.option(
    "--limit",
    "limit",
    help="max number of metadata records to fetch",
    default=500,
    show_default=True,
)
@click.option(
    "--agg",
    is_flag=True,
    help="use aggregate metadata service instead of the metadata service",
    show_default=True,
)
@click.option(
    "--guid_type",
    "guid_type",
    help="value of intended GUID type for query",
    default="discovery_metadata",
    show_default=True,
)
@click.option(
    "--output_format",
    "output_format",
    help="format of output file (can only be either tsv or json)",
    default="tsv",
    show_default=True,
)
@click.option(
    "--output_filename_suffix",
    "output_filename_suffix",
    help="additional suffix for the output file name",
    default="",
    show_default=True,
)
@click.pass_context
def discovery_read(ctx, limit, agg, guid_type, output_format, output_filename_suffix):
    """
    Download the metadata used to populate a commons' discovery page into a TSV or JSON file.
    Outputs the TSV / JSON filename with format {commons-url}-{guid_type}.tsv/.json
    If "output_filename_suffix" exists, file name will be something like {commons-url}-{guid_type}-{output_filename_suffix}
    """
    auth = ctx.obj["auth_factory"].get()
    loop = get_or_create_event_loop_for_thread()
    endpoint = ctx.obj.get("endpoint")
    output_file = loop.run_until_complete(
        output_expanded_discovery_metadata(
            auth,
            endpoint=endpoint,
            limit=limit,
            use_agg_mds=agg,
            guid_type=guid_type,
            output_format=output_format,
            output_filename_suffix=output_filename_suffix,
        )
    )

    click.echo(output_file)

@click.command()
@click.argument("file", required=False)
@click.option(
    "--output-filename",
    "output_filename",
    help="filename for final combined output",
    default="combined_discovery_metadata.tsv",
    show_default=True,
)
@click.option(
    "--max-number-discovery-records",
    "max_number_discovery_records",
    help="max number of metadata records to fetch. Initially defaulted to something very high. If you start to miss records, you may need to bump this up.",
    default=8192,
    type=int,
    show_default=True,
)
@click.option(
    "--discovery-column-to-map-on",
    "discovery_column_to_map_on",
    help="The column in the current discovery metadata to use to",
    default="guid",
    show_default=True,
)
@click.option(
    "--metadata-column-to-map",
    "metadata_column_to_map",
    help="The column in the provided metadata file to use to map/merge into the current Discovery metadata",
    default="guid",
    show_default=True,
)
@click.option(
    "--metadata-prefix",
    "metadata_prefix",
    help="Prefix to add to the column names in the provided metadata file before final output",
    default="",
    show_default=True,
)
@click.option(
    "--agg",
    is_flag=True,
    help="use aggregate metadata service instead of the metadata service",
    show_default=True,
)
@click.pass_context
def discovery_read_and_combine(
    ctx,
    file,
    output_filename,
    discovery_column_to_map_on,
    metadata_column_to_map,
    max_number_discovery_records,
    metadata_prefix,
    agg,
):
    """
    Combine provided metadata from file with current commons' discovery page metadata.
    Outputs a TSV with the original discovery metadata and provided metadata.
    """
    auth = ctx.obj["auth_factory"].get()
    loop = get_or_create_event_loop_for_thread()
    endpoint = ctx.obj.get("endpoint")
    current_discovery_metadata_file = loop.run_until_complete(
        output_expanded_discovery_metadata(
            auth, endpoint=endpoint, limit=max_number_discovery_records, use_agg_mds=agg
        )
    )

    output_file = combine_discovery_metadata(
        current_discovery_metadata_file,
        file,
        discovery_column_to_map_on,
        metadata_column_to_map,
        output_filename,
        metadata_prefix=metadata_prefix,
    )

    click.echo(f"{output_file}")

@click.command()
@click.argument("guid")
@click.pass_context
def discovery_delete(ctx, guid):
    """
    Delete all discovery metadata for the provided guid
    """
    auth = ctx.obj["auth_factory"].get()
    try_delete_discovery_guid(auth, guid)

discovery.add_command(discovery_read, name="read")
discovery.add_command(discovery_read_and_combine, name="combine")
discovery.add_command(discovery_publish, name="publish")
discovery.add_command(discovery_delete, name="delete")

@discovery.group()
def objects():
    """For ingesting dataset-level files"""
    pass

@click.command(
    help="""
    Outputs a TSV with populated information. [DATASET_GUIDS] argument is
    a variable number of datasets, e.g. 'phs000001.v1.p1.c1 phs000002.v1.p1.c1'
    """
)
@click.argument(
    "dataset_guids",
    nargs=-1,
)
@click.option(
    "--only-object-guids",
    help="""
    Outputs object guids to the command line in the following format:
        drs://dg.4503:943200c3-271d-4a04-a2b6-040272239a64
        drs://dg.4503:58818753-e8b9-4225-99bd-635a36ac41d9
        drs://dg.4503:72a9051e-5afa-4eb1-8faf-2fb28369f99b
        drs://dg.4503:034e406d-f8f1-4ee8-bbb3-3818473174a6
    Output can be piped into another command
    """,
    is_flag=True,
)
@click.option(
    "--template",
    help="""
    Output a TSV with required columns for `publish`. Will NOT contain any actual existing values for datasets or objects.
    If you want the full output of what's in the commons, don't supply this flag.
    """,
    is_flag=True,
)
@click.option(
    "--output_format",
    help="format of output file (can only be either tsv or json)",
    default="tsv",
    show_default=True,
)
@click.pass_context
def discovery_objects_read(
    ctx, dataset_guids, only_object_guids, template, output_format
):
    """
    Download the discovery objects from dataset(s) from a commons into TSV file.
    Outputs the TSV / JSON filename with format {commons-url}-discovery_objects.tsv
    If --only-object-guids is set, output object guids to the command line.
    If the --template option is set, output a TSV with just the required columns with
    format {commons-url}-discovery_objects-TEMPLATE.tsv
    """
    auth = ctx.obj["auth_factory"].get()
    loop = get_or_create_event_loop_for_thread()
    endpoint = ctx.obj.get("endpoint")
    output = loop.run_until_complete(
        output_discovery_objects(
            auth,
            dataset_guids=dataset_guids,
            endpoint=endpoint,
            template=template,
            output_format=output_format,
            only_object_guids=only_object_guids,
        )
    )
    if not only_object_guids:
        click.echo(output)
    else:
        for obj in output:
            click.echo(obj["guid"])

@click.command(
    help="""
    Takes a TSV as input and writes the specified objects GUIDs and defined metadata into Gen3's
    Metadata API, under a Discovery Metadata record for the dataset. The specified content from
    the TSV goes into an 'objects' block of the dataset's metadata.
    If dataset_guid already exists, update, if it doesn't already exist, create it.
    Use 'discovery objects read --template' to get a TSV with the minimum required columns
    to publish.
    """
)
@click.argument(
    "file",
    required=True,
)
@click.option(
    "--overwrite",
    help="""
    Replaces all guids in the 'objects' field of a dataset instead of appending (which is the default)
    """,
    is_flag=True,
)
@click.pass_context
def discovery_objects_publish(
    ctx,
    file,
    overwrite,
):
    """
    Run a discovery objects ingestion on a given metadata TSV file. If --overwrite is set, objects for
    each dataset are replaced instead of appeding by default.
    """
    auth = ctx.obj["auth_factory"].get()

    loop = get_or_create_event_loop_for_thread()
    endpoint = ctx.obj.get("endpoint")
    loop.run_until_complete(
        publish_discovery_object_metadata(
            auth,
            file,
            endpoint=endpoint,
            overwrite=overwrite,
        )
    )

@click.command(
    help="Delete objects related to datasets in TSV file or list of dataset_guids"
)
@click.argument(
    "guids",
    nargs=-1,
)
@click.option(
    "--file",
    help="""
    Delete objects from a file
    """,
    type=click.File("r"),
)
@click.pass_context
def discovery_objects_delete(
    ctx,
    guids,
    file,
):
    """
    Delete all discovery objects for the provided guid(s), or from the the file provided by the --file option.
    [GUIDS] argument is a variable number of dataset GUIDs, e.g. 'phs000001.v1.p1.c1 phs000002.v1.p1.c1'
    """
    auth = ctx.obj["auth_factory"].get()
    if guids:
        try_delete_discovery_objects(auth, guids)
    if file:
        if is_valid_object_manifest(file.name):
            delete_objs = {}
            tsv_reader = csv.DictReader(file, delimiter="\t")
            for row in tsv_reader:
                dataset_guid = row["dataset_guid"]
                guid = row["guid"]
                delete_objs.setdefault(dataset_guid, set()).add(guid)
            try_delete_discovery_objects_from_dict(auth, delete_objs)
        else:
            click.echo("Invalid objects file")

objects.add_command(discovery_objects_read, name="read")
objects.add_command(discovery_objects_publish, name="publish")
objects.add_command(discovery_objects_delete, name="delete")


# order = 5
# fs = 200.0      # sample rate, Hz
# cutoff = 5.0    # desired cutoff frequency of the filter, Hz
# IIR implemention
from scipy import integrate
from scipy.signal import butter, lfilter, freqz
from sklearn.decomposition import PCA
from scipy.signal import butter, lfilter
import os
import sklearn
import math
import numpy as np
def butter_lowpass(cutoff, fs, order=5):
    nyq = 0.5 * fs
    normal_cutoff = cutoff / nyq
    b, a = butter(order, normal_cutoff, btype='low', analog=False)
    return b, a

def butter_lowpass_filter(data, cutoff=5, fs=200, order=5):
    b, a = butter_lowpass(cutoff, fs, order=order)
    y = lfilter(b, a, data)
    return y
# to get physical characteristics
def get_user_propties(source_path='SisFall_dataset/physical_ch.txt'):
    f = open(source_path, mode='r', encoding='utf-8')
    datas = {}
    for line in f:
        line = line.split('|')
        line = [x.strip() for x in line]
        key = line[0]
        value = line[1:-1]
        for i in range(len(value) - 1):
            value[i] = float(value[i])
        if value[-1] == 'M':
            value[-1] = 0
        else:
            value[-1] = 1
        datas[key] = value
    f.close()
    return datas

# min-max normalization
def min_max(a):
    amin, amax = a.min(), a.max()
    a = (a - amin) / (amax - amin)
    return a

# return file names
def get_all_data(path):
    if not os.path.isdir(path):
        return []
    results = os.listdir(path)
    files = []
    for file in results:
        if file[0] in ['D', 'F']:
            files.append(file)
    files = [path + "/" + file for file in files]
    return files

# get data
def get_all_datas(path, datas):
    files = os.listdir(path)
    files = [path + "/" + file for file in files]
    for file in files:
        data = get_all_data(file)
        datas.extend(data)
    print(len(datas))

# process each line of file
def process_line(line):
    line = line.strip()
    line = line[:-1]
    nums = line.split(',')
    try:
        nums = [int(num) for num in nums]
    except:
        print('--',line)
    return nums

# feature extraction
def GetC1(datas):
    datan = np.array(datas)
    datan = datan ** 2
    datan = np.sum(datan, axis=1)
    datan = datan ** 0.5
    datan = np.mean(datan)
    return datan

def Get_std(datas):
    datan = np.array(datas)
    datan = np.std(datan, axis=0)
    horiz_std_mag9 = np.sqrt(datan[0]**2 + datan[2]**2)
    std_mag9 = np.sqrt(datan[0]**2 + datan[1]**2 + datan[2]**2)
    diff_std_mag9 = np.sqrt(datan[3]**2 + datan[4]**2 + datan[5]**2)
    horiz_std_mag2 = np.sqrt(datan[3]**2 + datan[5]**2)
    gyro_horiz_std_mag = np.sqrt(datan[6]**2 + datan[8]**2)
    gyro_std_mag = np.sqrt(datan[6]**2 + datan[7]**2 + datan[8]**2)
    datan = np.append([datan], [gyro_std_mag])
    datan = np.append([datan], [gyro_horiz_std_mag])
    datan = np.append([datan], [horiz_std_mag2])
    datan = np.append([datan], [diff_std_mag9])
    datan = np.append([datan], [std_mag9])
    datan = np.append([datan],[horiz_std_mag9])
    return datan

def Get_SigMagArea(datas):
    ntime = len(datas)
    sum = 0
    for i in range(3):
        sum += np.sum(datas[:,i])
    return sum/ntime

def Get_HorizSigMagArea(datas):
    ntime = len(datas)
    sum = 0
    for i in [0, 2]:
        sum += np.sum(datas[:, i])
    return sum / ntime

def Get_Amax(datas):
    datan = np.array(datas)
    datan = np.max(datan, axis=0)
    return datan

def Get_Amin(datas):
    datan = np.array(datas)
    datan = np.min(datan, axis=0)
    return datan

def Get_peak_diff(datas):
    datan = np.array(datas)
    return np.max(datan, axis=0) - np.min(datan, axis=0)

def Get_angle(datas):
    datan = np.array(datas)
    angle_from_horiz = np.arctan2(np.sqrt(datan[0] ** 2 + datan[2] ** 2), -datan[1]) * 180 / np.pi
    angle = np.append([np.max(angle_from_horiz)],[np.min(angle_from_horiz)])
    angle = np.append(angle, [np.mean(angle_from_horiz)])
    return angle

def GetC10(datas):
    f = lambda x: x
    N = len(datas)
    datan = np.array(datas)
    for i in range(9):
        if 1 % 3 == 2:
            continue
        datan[:, i] = datan[:, i] ** 2
    dataf = []
    for i in range(3):
        k = i * 3
        x = []
        for j in range(N):
            x.append(math.atan2((datan[j, k + 1] + datan[j, k]) ** 0.5, datan[j, k + 2]))
        x = np.mean(x)
        dataf.append(x)
    return np.array(dataf)

'''
file-->train data
a) file name-->train data: D means ADL, F means fall
b) cal avg
c) add extracted feature
d) add user info
'''
def get_one_data(path, pros):
    relaPath = path.split('/')[-1]
    proKey = path.split('/')[-2].strip()
    label = -1
    datas = []
    if relaPath[0] == 'D':
        label = 0
    elif relaPath[0] == 'F':
        label = 1
    else:
        return None
    f = open(path, 'r', encoding='utf-8')
    for line in f:
        key = process_line(line)
        if len(key) != 9:
            continue
        datas.append(key)
    f.close()
    tupleDatas = []

    # for i in range(0,len(datas)-256, 128):
    #     datan = datas[i:i+256]
    #     datan = np.array(datan)
    #     datatmp = np.array(datan)
    #     c1 = GetC1(datan)
    #     c10 = GetC10(datan)
    #     # for i in range(len(datas)):
    #         # datas[i,:] = butter_lowpass_filter(datas[i,:])
    #     datan = np.mean(datan, axis=0)
    #     datan = np.append([datan], [Get_SigMagArea(datatmp)])
    #     datan = np.append([datan], [Get_std(datatmp)])
    #     datan = np.append([datan], [Get_Amax(datatmp)])
    #     datan = np.append([datan], [Get_Amin(datatmp)])
    #
    #     datan = np.std(datan,axis = 0)
    #     datan = np.append([datan],[c1])
    #     datas = np.append([datas],[c10])
    #     tupleDatas.append((datan,label))

    datan = datas[:]
    datan = np.array(datan)
    datatmp = np.array(datan)
    c1 = GetC1(datan)
    c10 = GetC10(datan)
    datan = np.mean(datan, axis=0)
    datan = np.append([datan], [Get_SigMagArea(datatmp)])
    datan = np.append([datan], [Get_std(datatmp)])
    datan = np.append([datan], [Get_Amax(datatmp)])
    datan = np.append([datan], [Get_Amin(datatmp)])
    datan = np.append([datan], [Get_peak_diff(datatmp)])
    datan = np.append([datan], [Get_angle(datatmp)])
    datan = np.append([datan],[c1])
    # user = pros.get(proKey)
    # datan = np.append([datan], [user])
    tupleDatas.append((datan,label))

    return tupleDatas

def get_data_and_labels(trainDatas, testDatas, files):
    datas = []
    index = 0
    pros = get_user_propties()
    for file in files:
        data = get_one_data(file, pros)
        if data:
            datas.extend(data)
    length = len(datas)
    trainDatas.extend(datas[:int(length * 0.7)])
    testDatas.extend(datas[int(length * 0.7):])
    print('trainDatas', len(trainDatas))
    print('testDatas', len(testDatas))

def getXandY(datas):
    X = []
    Y = []
    for data in datas:
        x = data[0]
        Y.append(data[1])
        where_are_nan = np.isnan(x)
        where_are_inf = np.isinf(x)
        x[where_are_nan] = 0
        x[where_are_inf] = 0
        X.append(x)
    return X, Y

def norm_pro(trainX,testX):
    len1 = len(trainX)
    len2 = len(testX)
    X = []
    X.extend(trainX)
    X.extend(testX)
    X = np.array(X)
    print(X.shape)
    pca = PCA(n_components=20)
    print('begin pca', X.shape)
    X = pca.fit_transform(X)
    print('after pca',X.shape)
    return X[:len1],X[len1:]

def getProbability(trainY,testY):
    sumP1 = 0.0
    sumP2 = 0.0
    for y in trainY:
        if y == 1:
            sumP1 += 1
        else:
            sumP2 += 1
    for y  in testY:
        if y == 1:
            sumP1 += 1
        else:
            sumP2 += 1
    Pro1 = sumP1/(sumP1 + sumP2)
    Pro2 = sumP2/(sumP1 + sumP2)
    print('pro of two catagories', Pro1, Pro2)


#!/usr/bin/env python
# coding: utf-8

# In[4]:

# Import pathlib and set the file path
from pathlib import Path
import csv
csv_data = Path('./budget_data.csv')

# In[2]:

# Initialize list of records
records = []

# In[3]:

# Open the csv file as an object
with open(csv_data, 'r') as csvfile:
    # Pass in the csv file to the csv.reader() function
    #(with ',' as the delimiter/separator) and return the csvreader object
    csvreader = csv.reader(csvfile, delimiter=',')
    # Read the heder row first
    csv_header = next(csvreader)
    # Print the header
    #print(csv_header)

    #Append the column 'Average' to the header
    csv_header.append("Average")

    # Append the header to teh list of record
    records.append(csv_header)
    #print(csv_header)

    # Read each row of data after the header
    # for row in csvreader:

    net_profit = 0
    greatest_increase = 0
    date_greatincrease = str
    greatest_decrease = 0
    date_greatdecrease = str

    first_row = next(csvreader)
    month_count = 1
    net_profit = net_profit + int(first_row[1])
    previous_net = int(first_row[1])
    net_change_list = []

    #initialize greatest increase and decrease
    greatest_increase = int(first_row[1])
    date_greatincrease = str(first_row[0])
    greatest_decrease = int(first_row[1])
    date_greatdecrease = str(first_row[0])

    for row in csvreader:
        net_profit += int(row[1])
        month_count += 1
        net_change = int(row[1]) - previous_net
        previous_net = int(row[1])

        # Recording the change
        net_change_list = net_change_list + [net_change]
        month_change = month_count - 1

        #identififying the greatest increase and decrease
        if net_change > greatest_increase:
            greatest_increase = net_change
            date_greatincrease = str(row[0])
        elif net_change < greatest_decrease:
            greatest_decrease = net_change
            date_greatdecrease = str(row[0])

        # print(net_change)
average_month_change = round(sum(net_change_list) / month_change,2)
# printing results
print()
print("Financial Analysis")
print("-------------------------------")
print("Total Months: ", month_count)
print("Total: $",net_profit)
print("Average Change: $",average_month_change)
print("Greatest Increase in Profits: ", date_greatincrease, " ($",greatest_increase,")")
print("Greatest Decrease in Profits: ", date_greatdecrease, " ($",greatest_decrease,")")

# Export the results to text file

from pathlib import Path
import csv
pybank_analysis = Path('./pybank_analysis.txt')

with open(pybank_analysis, 'w') as txt_file:
    txt_file.write(f"Financial Analysis\n")
    txt_file.write(f"------------------------------\n")
    txt_file.write(f"Total Months: {month_count}\n")
    txt_file.write(f"Total: $ {net_profit}\n")
    txt_file.write(f"Average Change: $ {average_month_change}\n")
    txt_file.write(f"Greatest Increase in Profits:  {date_greatincrease} ($ {greatest_increase})\n")
    txt_file.write(f"Greatest Decrease in Profits:  {date_greatdecrease} ($ {greatest_decrease})\n")

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:


import asyncio

from unittest.mock import Mock

import pytest

import nono.domain.interfaces as interfaces
from nono.domain.purge import delete_last_messages

class MessageMock(interfaces.Message):
    def __init__(self, author, content):
        self._author = author
        self._content = content
        self.deleted = False

    @property
    def author(self):
        return self._author

    @property
    def content(self):
        return self._content

    async def delete(self):
        await asyncio.sleep(0)
        self.deleted = True

class ChannelMock(interfaces.Channel):
    def __init__(self, messages):
        self.messages = messages

    async def history(self, limit=None):
        limit = -1 * limit
        loop = asyncio.get_running_loop()
        future = loop.create_future()
        future.set_result(None)
        for message in self.messages[limit:]:
            await future
            yield message

@pytest.mark.asyncio
async def test_purge():
    admin = Mock(spec=interfaces.User)
    admin.id.return_value = 1
    legit = Mock(spec=interfaces.User)
    legit.id.return_value = 2
    spammer = Mock(spec=interfaces.User)
    spammer.id.return_value = 3

    messages = [
        MessageMock(legit, "hello"),
        MessageMock(spammer, "SPAM"),
        MessageMock(spammer, "SPAM"),
        MessageMock(spammer, "SPAM"),
        MessageMock(admin, "!purge 3"),
    ]
    channel = ChannelMock(messages)
    await delete_last_messages(channel, messages[-1], 3)

    for message in channel.messages:
        if message.author.id() == 3:
            assert message.deleted
        else:
            assert not message.deleted


from django.forms import ModelForm
from .models import Book, Review,Genre
from django import forms
from django.contrib.auth.models import User
from django.contrib.auth.forms import UserCreationForm
from django.forms.widgets import NumberInput

class Book_Form(ModelForm):
    genres = forms.ModelMultipleChoiceField(
        queryset=Genre.objects.all(),
        widget=forms.CheckboxSelectMultiple
    )
    class Meta:
        model = Book
        fields = ['title', 'author', 'genres', 'pub_date_book', 'summary']
        labels = {
            'pub_date_book': 'Publication date',
            'summary': 'Book summary'
        }
        widgets = {
            'pub_date_book': NumberInput(attrs={'type': 'date'}),
            # 'genres': forms.CheckboxSelectMultiple
        }

CHOICES = (
    ("1", 1),
    ("2", 2),
    ("3", 3),
    ("4", 4),
    ("5", 5)
)

class Review_Form(ModelForm):
    stars = forms.ChoiceField(choices=CHOICES, initial=0, widget=forms.RadioSelect())

    class Meta:
        model = Review
        fields = ['title', 'content', 'stars']
        labels = {
            'title': 'Review Title',
            'content': 'Review',
            'stars': 'Rating',
        }
        widgets = {
        }

class SignupForm(UserCreationForm):
    email = forms.EmailField(max_length=200, help_text='Required')

    class Meta:
        model = User
        fields = ('username', 'email', 'password1', 'password2')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#Adapted from https://gist.github.com/filipenf/2cc72af47e3570afaa9d3bf2e71658c3

import sys
import yaml
import argparse
from ansible.parsing.vault import VaultLib
from ansible.cli import CLI
from ansible import constants as C
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode

class VaultHelper():
    def __init__(self, vault_id):
        loader = DataLoader()
        vaults = [v for v in C.DEFAULT_VAULT_IDENTITY_LIST if v.startswith('{0}@'.format(vault_id))]
        if len(vaults) != 1:
            raise ValueError("'{0}' does not exist in ansible.cfg '{1}'".format(vault_id, C.DEFAULT_VAULT_IDENTITY_LIST))

        self.vault_id = vault_id
        vault_secret = CLI.setup_vault_secrets(
            loader=loader,
            vault_ids=vaults
        )
        self.vault = VaultLib(vault_secret)

    def convert_vault_to_strings(self, vault_data):
        decrypted = self.vault.decrypt(vault_data)
        d = yaml.load(decrypted, Loader=AnsibleLoader)
        self._encrypt_dict(d)
        return d

    def _encrypt_dict(self, d):
        for key in d:
            value = d[key]
            if isinstance(value, str):
                d[key] = AnsibleVaultEncryptedUnicode(
                    self.vault.encrypt(plaintext=value, vault_id=self.vault_id))
            elif isinstance(value, list):
                for item in value:
                    self._encrypt_dict(item)
            elif isinstance(value, dict):
                self._encrypt_dict(value)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--input-file',
                        help='File to read from',
                        required=True)
    parser.add_argument('--output-file',
                        help='File to to write to',
                        required=True)
    parser.add_argument('--vault-id',
                        help='Vault id used for the encryption',
                        required=True)
    args = parser.parse_args()
    original_secrets = open(args.input_file).read()
    vault = VaultHelper(args.vault_id)
    converted_secrets = vault.convert_vault_to_strings(original_secrets)

    with open(args.output_file, 'w+') as f:
        yaml.dump(converted_secrets, Dumper=AnsibleDumper, stream=f)

if __name__ == "__main__":
    main()


from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.svm import SVC
from sklearn.metrics import classification_report
from sklearn.datasets import make_classification
from sklearn.calibration import CalibratedClassifierCV
from classifiers import CoTrainingClassifier
import os

if __name__ == '__main__':
	N_SAMPLES = 25000
	N_FEATURES = 1000
	X, y = make_classification(n_samples=N_SAMPLES, n_features=N_FEATURES)

	y[:N_SAMPLES//2] = -1

	X_test = X[-N_SAMPLES//4:]
	y_test = y[-N_SAMPLES//4:]

	X_labeled = X[N_SAMPLES//2:-N_SAMPLES//4]
	y_labeled = y[N_SAMPLES//2:-N_SAMPLES//4]

	y = y[:-N_SAMPLES//4]
	X = X[:-N_SAMPLES//4]

	X1 = X[:,:N_FEATURES // 2]
	X2 = X[:, N_FEATURES // 2:]

	f = open("result.txt", "w+")

	print('Logistic')
	base_lr = LogisticRegression()
	base_lr.fit(X_labeled, y_labeled)
	y_pred = base_lr.predict(X_test)
	lr_result = classification_report(y_test, y_pred)
	print(lr_result)
	f.write("Logistic")
	f.write("\n======================================================\n")
	f.write(str(lr_result))
	f.write("\n======================================================\n\n")

	print('Logistic CoTraining')
	lg_co_clf = CoTrainingClassifier(LogisticRegression())
	lg_co_clf.fit(X1, X2, y)
	y_pred = lg_co_clf.predict(X_test[:, :N_FEATURES // 2], X_test[:, N_FEATURES // 2:])
	co_lr_result = classification_report(y_test, y_pred)
	print(co_lr_result)
	f.write("Logistic CoTraining")
	f.write("\n======================================================\n")
	f.write(str(co_lr_result))
	f.write("\n======================================================\n\n")

	print('SVM')
	base_svm = LinearSVC(dual=False)
	base_svm.fit(X_labeled, y_labeled)
	y_pred = base_svm.predict(X_test)
	svm_result = classification_report(y_test, y_pred)
	print(svm_result)
	f.write("SVM")
	f.write("\n======================================================\n")
	f.write(str(svm_result))
	f.write("\n======================================================\n\n")

	print('SVM CoTraining')
	svm = LinearSVC(dual=False)
	clf = CalibratedClassifierCV(svm)
	svm_co_clf = CoTrainingClassifier(clf, u=N_SAMPLES//10)
	svm_co_clf.fit(X1, X2, y)
	y_pred = svm_co_clf.predict(X_test[:, :N_FEATURES // 2], X_test[:, N_FEATURES // 2:])
	co_svm_result = classification_report(y_test, y_pred)
	print(co_svm_result)
	f.write("SVM CoTraining")
	f.write("\n======================================================\n")
	f.write(str(co_svm_result))
	f.write("\n======================================================\n\n")

	f.close()

	print("=== Process done, check result.txt file for the output. === \n")
	os.system("pause")


import logging
import functools

from omegaconf import open_dict
from omegaconf import OmegaConf
from omegaconf import DictConfig

from hydra._internal.config_search_path import ConfigSearchPath
from hydra._internal.pathlib import Path
from hydra.plugins import Launcher
from hydra.plugins import SearchPathPlugin
from hydra.plugins.common.utils import (
    configure_log,
    filter_overrides,
    run_job,
    setup_globals,
    HydraConfig,
)

import ray

log = logging.getLogger(__name__)

def get_key(cfg, key):
    if key == '':
        return cfg
    else:
        keys = key.split('.')
        if keys[0] in cfg:
            return get_key(getattr(cfg, keys[0]), '.'.join(keys[1:]))
        else:
            return False

def merge_kwargs(kwargs1, kwargs2):
    k1 = kwargs1 if isinstance(kwargs1, DictConfig) else OmegaConf.create(kwargs1)
    k2 = kwargs2 if isinstance(kwargs2, DictConfig) else OmegaConf.create(kwargs2)
    merged = OmegaConf.merge(k1,k2)
    return merged.to_container(resolve=True)

def pass_conf(f, cfg, key):
    item = get_key(cfg, key)
    if item:
        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            return f(*args, **merge_kwargs(item, kwargs))
        return wrapper
    else:
        return f

class RayLauncherSearchPathPlugin(SearchPathPlugin):
    """
    This plugin is allowing configuration files provided by the ExampleLauncher plugin to be discovered
    and used once the ExampleLauncher plugin is installed
    """

    def manipulate_search_path(self, search_path):
        assert isinstance(search_path, ConfigSearchPath)
        # Appends the search path for this plugin to the end of the search path
        search_path.append(
            "hydra-ray-launcher-badr", "pkg://hydra_plugins.ray_launcher_badr.conf"
        )

def launch(*args, **kwargs):
    setup_globals()
    run_job(*args, **kwargs)

class RayLauncher(Launcher):
    def __init__(self):
        self.config = None
        self.config_loader = None
        self.task_function = None

    def setup(self, config, config_loader, task_function):
        self.config = config
        self.config_loader = config_loader
        self.task_function = task_function

        if not ray.is_initialized(): pass_conf(ray.init, config, 'ray.init')()

    def launch(self, job_overrides):
        """
        :param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
        :return: an array of return values from run_job with indexes corresponding to the input list indexes.
        """
        configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)
        sweep_dir = Path(str(self.config.hydra.sweep.dir))
        sweep_dir.mkdir(parents=True, exist_ok=True)
        log.info(
            "Ray Launcher is launching {} jobs locally".format(
                len(job_overrides)
            )
        )
        log.info("Sweep output dir : {}".format(sweep_dir))
        runs = []

        for idx, overrides in enumerate(job_overrides):
            log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides))))
            sweep_config = self.config_loader.load_sweep_config(
                self.config, list(overrides)
            )
            with open_dict(sweep_config):
                # This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
                # In that case, it will not be available here because we are still in the main process.
                # but instead should be populated remotely before calling the task_function.
                sweep_config.hydra.job.id = idx
                sweep_config.hydra.job.num = idx
            HydraConfig().set_config(sweep_config)

            ray_remote_cfg = get_key(self.config, 'ray.remote')
            if ray_remote_cfg:
                run_job_ray = ray.remote(**ray_remote_cfg)(launch)
            else:
                run_job_ray = ray.remote(launch)

            ret = run_job_ray.remote(
                config=sweep_config,
                task_function=self.task_function,
                job_dir_key="hydra.sweep.dir",
                job_subdir_key="hydra.sweep.subdir",
            )

            runs.append(ret)
            configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)

        return [ray.get(run) for run in runs]


import RPi.GPIO as GPIO
import Adafruit_DHT
import argparse
import signal
import sys
from threading import Thread
from random import randint
from paho.mqtt.client import Client as mqtt
from time import sleep

# Parsing per disabilitare l'utilizzo del sensore di temperatura
# e abilitare valori randomici
parser = argparse.ArgumentParser(description='Argument parsing')
parser.add_argument('-r', '--random', action='store_true', help='Generate random values for temperature.')
args = parser.parse_args()

# CONNESSIONE MQTT
Broker=mqtt(client_id="raspberry")
Broker.connect("localhost", 1883)

# DOOR CONFIG
DOOR_LED = 19
DOOR_PIN = 18

# MAIN_LIGHT CONFIG
LIGHT_LED = 26
LIGHT_PIN = 23

# AIR_CONDITIONER CONFIG
HOT_AIR_PIN = 24
COLD_AIR_PIN = 25

# IMPOSTA I GPIO IN MODALITÀ BCM
GPIO.setmode(GPIO.BCM)

# ABILITAZIONE GPIO DOOR
GPIO.setup(DOOR_LED, GPIO.OUT)
GPIO.setup(DOOR_PIN, GPIO.IN, pull_up_down=GPIO.PUD_UP)

# ABILITAZIONE GPIO MAIN_LIGHT
GPIO.setup(LIGHT_LED, GPIO.OUT)
GPIO.setup(LIGHT_PIN, GPIO.IN, pull_up_down=GPIO.PUD_UP)

# ABILITAZIONE GPIO AIR_CONDITIONER
GPIO.setup(HOT_AIR_PIN, GPIO.OUT)
GPIO.setup(COLD_AIR_PIN, GPIO.OUT)

# VARIABILI DI STATO DOOR E MAIN_LIGHT
door_state = False
light_state = False

# CTRL+C HANDLER
def signal_handler(sig, frame):
    # PULIZIA DEL BUFFER GPIO
    GPIO.cleanup()
    sys.exit(0)

# Imposta il gestore del segnale di interruzione
signal.signal(signal.SIGINT, signal_handler)

def manage_temp(temperature):
    if temperature < 13:
        msg = "hot_air"
        GPIO.output(HOT_AIR_PIN, GPIO.HIGH)
        GPIO.output(COLD_AIR_PIN, GPIO.LOW)

    elif temperature > 23:
        msg = "cold_air"
        GPIO.output(HOT_AIR_PIN, GPIO.LOW)
        GPIO.output(COLD_AIR_PIN, GPIO.HIGH)
    else:
        msg = "off"
        GPIO.output(HOT_AIR_PIN, GPIO.LOW)
        GPIO.output(COLD_AIR_PIN, GPIO.LOW)

    return msg

def measure_temp():
    while True:
        if not args.random:
            try:
                # MISURA CON SENSORE DI TEMPERATURA
                _, temperature = Adafruit_DHT.read_retry(11, 4)
                msg = manage_temp(temperature)
            except:
                msg = "off"
        else:
            msg = manage_temp(randint(0, 40))

        Broker.publish(topic="temperature", payload=str(msg))
        sleep(2)

def door_button_callback(channel):
    global door_state

    if door_state:
        msg = "close"
        GPIO.output(DOOR_LED, GPIO.LOW)
        door_state = False

    else:
        msg = "open"
        GPIO.output(DOOR_LED, GPIO.HIGH)
        door_state = True

    Broker.publish(topic="door", payload=str(msg))

def light_button_callback(channel):
    global light_state

    if light_state:
        msg = "off"
        GPIO.output(LIGHT_LED, GPIO.LOW)
        light_state = False
    else:
        msg = "on"
        GPIO.output(LIGHT_LED, GPIO.HIGH)
        light_state = True

    Broker.publish(topic="main_light", payload=str(msg))

# AGGIUNTA EVENTI AL CLICK SUI TASTI
GPIO.add_event_detect(DOOR_PIN, GPIO.FALLING, callback=door_button_callback, bouncetime=200)
GPIO.add_event_detect(LIGHT_PIN, GPIO.FALLING, callback=light_button_callback, bouncetime=200)

# THREAD MISURA TEMPERATURA
temp_thread = Thread(target=measure_temp)
temp_thread.start()

try:
    while True:
        pass

finally:
    GPIO.cleanup()


"""this is a test code"""
from sklearn.cluster import KMeans
import gdal
import numpy as np
import matplotlib.pyplot as plt
import os

workspace_path = os.path.dirname(__file__)

raster_path = workspace_path + "/PL_PS_20200723T0742_ALL_Tile_0_0_qKSm9prB.tif"

dataset = gdal.Open(raster_path, gdal.GA_ReadOnly)
numpy_array = dataset.ReadAsArray().astype(np.float)
nbands = dataset.RasterCount
# Getting Dataset Information
print("Driver: {}/{}".format(dataset.GetDriver().ShortName, dataset.GetDriver().LongName))
print("Size is {} x {} x {}".format(dataset.RasterXSize,
                                    dataset.RasterYSize,
                                    dataset.RasterCount))

# create an empty array, each column of the empty array will hold one band of data from the image
# loop through each band in the image nad add to the data array
data = np.empty((dataset.RasterXSize*dataset.RasterYSize, nbands))
for i in range(1, nbands+1):
    band = dataset.GetRasterBand(i).ReadAsArray()
    data[:, i-1] = band.flatten()
print(data.shape)

# set up the kmeans classification, fit, and predict
km = KMeans(n_clusters=8)
km.fit(data)
km.predict(data)

# format the predicted classes to the shape of the original image
out_dat = km.labels_.reshape((dataset.RasterYSize, dataset.RasterXSize))
print(out_dat.shape)

# displaying the output
plt.figure(figsize=(20, 20))
plt.imshow(out_dat, cmap="hsv")
plt.show()


#
# Let us assume that an entrepreneur is interested in the wine making company and would like to buy its resources.
# The entrepreneur then needs to find out how much to pay for each unit of each of the resources, the pure-grape wines
# of 2010 A, B and C. This can be done by solving the dual version of the model that we will discuss next.

from pyscipopt import Model

model = Model("duality with wine factory example")
y1 = model.addVar(vtype="C", name="y1")
y2 = model.addVar(vtype="C", name="y2")
y3 = model.addVar(vtype="C", name="y3")

model.addCons(2 * y1 + y2 >= 15)
model.addCons(y1 + 2 * y2 >= 18)
model.addCons(y1 + y2 + y3 >= 30)
model.addCons(y1 >= 0)
model.addCons(y2 >= 0)
model.addCons(y3 >= 0)
model.setObjective(60 * y1 + 60 * y2 + 30 * y3)
model.optimize()
if model.getStatus() == "optimal":
    print("Optimal value: {}".format(model.getObjVal()))
    print("Solution:")
    print(" x = {}".format(model.getVal(y1)))
    print(" y = {}".format(model.getVal(y2)))
    print(" z = {}".format(model.getVal(y3)))
else:
    print("Problem could not be solved to optimality")


from lsst.utils import continueClass
import lsst.afw.image
from lsst.geom import Box2D

import pfs.datamodel.pfsDetectorMap

from .PolynomialDetectorMap import PolynomialDetectorMap
from .DetectorMapContinued import DetectorMap
from .PolynomialDistortion import PolynomialDistortion
from .SplinedDetectorMapContinued import SplinedDetectorMap
from .utils import headerToMetadata

__all__ = ("PolynomialDetectorMap",)

@continueClass  # noqa: F811 (redefinition)
class PolynomialDetectorMap:  # noqa: F811 (redefinition)
    @classmethod
    def fromDatamodel(cls, detMap):
        """Construct from the pfs.datamodel representation

        Parameters
        ----------
        detMap : `pfs.datamodel.PolynomialDetectorMap`
            datamodel representation of PolynomialDetectorMap.

        Returns
        -------
        self : `pfs.drp.stella.PolynomialDetectorMap`
            drp_stella representation of PolynomialDetectorMap.
        """
        if not isinstance(detMap, pfs.datamodel.PolynomialDetectorMap):
            raise RuntimeError(f"Wrong type: {detMap}")
        base = SplinedDetectorMap.fromDatamodel(detMap.base)
        distortion = PolynomialDistortion(detMap.order, Box2D(base.bbox), detMap.xCoeff, detMap.yCoeff)
        metadata = headerToMetadata(detMap.metadata)
        visitInfo = lsst.afw.image.VisitInfo(metadata)
        lsst.afw.image.stripVisitInfoKeywords(metadata)

        return cls(base, distortion, visitInfo, metadata)

    def toDatamodel(self, identity=None):
        """Convert to the pfs.datamodel representation

        Parameters
        ----------
        identity : `pfs.datamodel.CalibIdentity`, optional
            Identification of the calibration. Providing this is only necessary
            if you intend to write via the datamodel representation's ``write``
            method; other means of writing that provide a filename directly do
            not require providing an ``identity`` here.

        Returns
        -------
        detMap : `pfs.datamodel.PolynomialDetectorMap`
            Datamodel representation of PolynomialDetectorMap.
        """
        base = self.getBase().toDatamodel()
        distortion = self.getDistortion()

        metadata = self.metadata.deepCopy()
        if self.visitInfo is not None:
            lsst.afw.image.setVisitInfoMetadata(metadata, self.visitInfo)

        return pfs.datamodel.PolynomialDetectorMap(
            identity, pfs.datamodel.Box.fromLsst(self.bbox), base,
            distortion.getOrder(), distortion.getXCoefficients(), distortion.getYCoefficients(),
            metadata.toDict()
        )

DetectorMap.register(PolynomialDetectorMap)


import os
import subprocess
import uuid

class SimulatorInstanceManager(object):
    """
    Manages the running instance of the simulator
    """
    def __init__(self, simulator_path, simulator_log_path, delete_log_on_success):
        self.simulator_path = simulator_path
        self.simulator_log_path = simulator_log_path
        self.delete_log_on_success = delete_log_on_success
        self.simulation_execution_id = str(uuid.uuid4())

        self.simulator_exe_name = self.simulator_path.split('/')[-1]

        self.__kill_simulator_process(force=True)

        self.__simulator_popen_obj = self.__start_simulator()

    def get_simulator_pid(self):
        """
        NOTE: For some reason, 2 simulator processes seem to spawn...
              Not sure which PID gets returned.
              Or why there are two.
        """
        if self.__simulator_popen_obj is not None:
            return self.__simulator_popen_obj.pid
        return None

    def get_log_path(self):
        return os.path.join(self.simulator_log_path, '{0}.log'.format(self.simulation_execution_id)).replace('\\', '/')

    def is_simulation_alive(self):
        if (self.__simulator_popen_obj is None):
            return False
        return (self.__simulator_popen_obj.poll() == None)

    def finalize(self):
        if (self.__simulator_popen_obj is not None):
            self.__kill_simulator_process()
            self.__simulator_popen_obj = None

    def __start_simulator(self):
        args = [self.simulator_path, 'ABSLOG={0}'.format(self.get_log_path())]
        return subprocess.Popen(args)

    def __kill_simulator_process(self, force = False):
        if (not force and self.__simulator_popen_obj is None):
            return

        is_termination_graceful = False
        if (not force):
            is_termination_graceful = self.is_simulation_alive()

        # This doesn't seem to work.
        # self.__simulator_popen_obj.kill()

        # Time to use a bigger hammer
        args = ['taskkill', '/FI', 'IMAGENAME eq {0}'.format(self.simulator_exe_name), '/f']
        subprocess.call(args)

        if (is_termination_graceful and self.delete_log_on_success and os.path.exists(self.get_log_path())):
            os.remove(self.get_log_path())

        self.__simulator_popen_obj = None