from rest_framework import status
from rest_framework.decorators import api_view, permission_classes
from rest_framework.response import Response
from tweet.models import Tweet, User
from rest_framework.permissions import IsAuthenticated
from tweet.business import TweetManagement
from tweet.serializers import TweetSerializer

@api_view(['GET', 'POST', 'PUT', 'DELETE'])
@permission_classes([IsAuthenticated])
def tweet_list(request, id=None, format=None):
    user_data = User.objects.get(email=request.user.email)
    if request.method == 'GET':
        if id is not None:
            tweet = get_tweet_by(id)
            if tweet.owner.email != user_data.email:
                return Response("Unauthorize this content", status.HTTP_401_UNAUTHORIZED)
            serializer = TweetSerializer(tweet)
            response_data = serializer.data
        else:
            tweets = Tweet.objects.all().filter(owner=user_data.id)
            serializer = TweetSerializer(tweets, many=True)
            response_data = serializer.data
        return Response(response_data)

    elif request.method == 'POST':
        tweet_manage = TweetManagement()
        data = tweet_manage.set_tweet_owner(request.data["tweet"], user_data)
        serializer = TweetSerializer(data=data)
        if serializer.is_valid():
            serializer.save()
            response_data = serializer.data
            response_data['owner'] = user_data.email
            response_data['author'] = user_data.email
            return Response(response_data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_422_UNPROCESSABLE_ENTITY)
    elif request.method == 'PUT':
        tweet = get_tweet_by(id)
        tweet_manage = TweetManagement()
        data = tweet_manage.set_tweet_owner(request.data["tweet"], user_data)
        serializer = TweetSerializer(tweet, data=data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
    elif request.method == 'DELETE':
        try:
            tweet = get_tweet_by(id)
            tweet.delete()
            return Response(status=status.HTTP_204_NO_CONTENT)
        except:
            return Response(status=status.HTTP_422_UNPROCESSABLE_ENTITY)

def get_tweet_by(id):
    if id is not None:
        return Tweet.objects.get(pk=id)

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def tweet_retweet(request, id, format=None):
    try:
        tweet = Tweet.objects.get(pk=id)
    except Tweet.DoesNotExist:
        return Response(status=status.HTTP_422_UNPROCESSABLE_ENTITY)

    if request.method == 'POST':
        tweet_manage = TweetManagement()
        serializer = tweet_manage.retweet_new(tweet, request.data["tweet"])
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# author: Alexey Koshevoy

import numpy as np
from mesa import Model
from mesa.time import RandomActivation
from mesa.datacollection import DataCollector
import networkx as nx
from mesa.space import NetworkGrid
import itertools
from collections import Counter
from model.LAgent import LAgent

class LanguageEvolutionModel(Model):
    """A model with some number of agents."""

    def __init__(self, lexicon, num_agents=30, num_nodes=20,
                 mutation_prob=0.2,
                 addition_prob=0.4, propagation_prob=0.4, append_prob=0.8, min_distance=None):
        """

        :param num_agents: number of agents in the model
        :param num_nodes: number of nodes in the graph
        :param lexicon: initial lexicon
        :param mutation_prob: probability of mutation
        :param addition_prob: probability of addition
        :param propagation_prob: probability of memory propagation
        :param min_dist: maximal pairvize Levenstein distance between the newly
        generated word and the existing lexicon
        """
        self.addition = addition_prob
        self.propagation = propagation_prob
        self.mutation = mutation_prob
        self.append = append_prob
        self.min_distance = min_distance

        self.num_agents = num_agents
        self.num_nodes = num_nodes if num_nodes >= self.num_agents \
            else self.num_agents
        self.G = nx.erdos_renyi_graph(n=self.num_nodes, p=0.5)
        self.grid = NetworkGrid(self.G)
        self.schedule = RandomActivation(self)
        self.datacollector = DataCollector(
            model_reporters={"mean_lexicon_size": self.compute_mean_lexicon_size,
                             "most_used_words": self.compute_lexicon,
                             "counter": self.get_counter})
        list_of_random_nodes = self.random.sample(self.G.nodes(),
                                                  self.num_agents)

        for i in range(self.num_agents):
            a = LAgent(i, model=self, lexicon=lexicon, mutation_prob=self.mutation,
                 addition_prob=self.addition, propagation_prob=self.propagation,
                       append_prob=self.append, min_dist=min_distance)
            self.schedule.add(a)
            self.grid.place_agent(a, list_of_random_nodes[i])

        self.running = True
        self.datacollector.collect(self)

    def step(self):
        self.schedule.step()
        self.datacollector.collect(self)

    def run_model(self, n):
        for i in range(n):
            self.step()

    @staticmethod
    def compute_mean_lexicon_size(model):
        return np.mean([len(agent) for agent in model.schedule.agents])

    @staticmethod
    def compute_lexicon(model):
        return Counter(list(itertools.chain.from_iterable(
            [agent.lexicon for agent in model.schedule.agents]))).most_common(
            10)

    @staticmethod
    def get_counter(model):
        return Counter(list(itertools.chain.from_iterable(
            [agent.lexicon for agent in model.schedule.agents])))


from pyquery import PyQuery
import time
import sqlite3
import sys
from datetime import datetime

def get_brand(code):
    url = 'https://kabutan.jp/stock/?code={}'.format(code)
    print(code, "：取得開始")

    q = PyQuery(url)

    if len(q.find('#kobetsu_left')) == 0:
        print(code, "：該当なし")
        return None

    try:
        # 銘柄名（正式名称）
        name = q.find('.company_block h3')[0].text
        # 銘柄名（略称）
        short_name = q.find('[itemprop="name"]')[1].text
        short_name = short_name.rsplit('(', 1)[0] # 『(』以下の部分を除く
        # 上場市場名
        market = q.find('.market')[0].text
        # 単元株数
        unit = q.find('table:eq(4) > tbody > tr:eq(5) > td')[0].text
        unit = int(unit.split('\xa0', 1)[0])
        # セクタ
        sector = q.find('.company_block td')[3].text
        print(code, "：取得完了")

    except ValueError:
        print(code, "：取得失敗")
        return None

    return code, name, short_name, market, unit, sector

def brands_generator(code_range):
    print("code_range：", code_range)
    for code in code_range:
        brand = get_brand(code)
        if brand:
            yield brand
        time.sleep(1)

def insert_brands_to_db(db_file_name, code_range):
    conn = sqlite3.connect(db_file_name)
    with conn:
        sql = 'INSERT INTO brands(code, name, short_name, market, unit, sector) VALUES(?,?,?,?,?,?)'
        conn.executemany(sql, brands_generator(code_range))

start_date = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
start_time = time.time()
print("実行開始：", start_date)
args = sys.argv
db_file_name = args[1]
code_range = range(int(args[2]) ,int(args[3]))
insert_brands_to_db(db_file_name, code_range)
end_date = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
print("実行完了：", end_date)
print("実行時間：", time.time() - start_time, "秒")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import numpy as np
from qibo.models import Circuit
from qibo import hamiltonians, gates, models
from scipy.optimize import minimize
import argparse

def main(nqubits, layers, compress, lambdas):

    def cost_function(params, count):
        """Evaluates the cost function to be minimized.

        Args:
            params (array or list): values of the parameters.

        Returns:
            Value of the cost function.
        """
        circuit = models.Circuit(nqubits)
        for l in range(layers):
            for q in range(nqubits):
                circuit.add(gates.RY(q, theta=0))
            for q in range(0, nqubits-1, 2):
                circuit.add(gates.CZ(q, q+1))
            for q in range(nqubits):
                circuit.add(gates.RY(q, theta=0))
            for q in range(1, nqubits-2, 2):
                circuit.add(gates.CZ(q, q+1))
            circuit.add(gates.CZ(0, nqubits-1))
        for q in range(nqubits):
            circuit.add(gates.RY(q, theta=0))

        cost = 0
        circuit.set_parameters(params) # this will change all thetas to the appropriate values
        for i in range(len(ising_groundstates)):
            final_state = circuit(np.copy(ising_groundstates[i]))
            cost += encoder.expectation(final_state).numpy().real

        if count[0] % 50 == 0:
            print(count[0], cost/len(ising_groundstates))
        count[0] += 1

        return cost/len(ising_groundstates)

    nparams = 2 * nqubits * layers + nqubits
    initial_params = np.random.uniform(0, 2*np.pi, nparams)
    encoder = 0.5 * (compress + hamiltonians.Z(nqubits))

    ising_groundstates = []
    for lamb in lambdas:
        ising_ham = -1 * hamiltonians.TFIM(nqubits, h=lamb)
        ising_groundstates.append(ising_ham.eigenvectors()[0])

    count = [0]
    result = minimize(lambda p: cost_function(p, count), initial_params,
                      method='L-BFGS-B', options={'maxiter': 2.0e3, 'maxfun': 2.0e3})

    print('Final parameters: ', result.x)
    print('Final cost function: ', result.fun)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--nqubits", default=4, type=int)
    parser.add_argument("--layers", default=2, type=int)
    parser.add_argument("--compress", default=2, type=int)
    parser.add_argument("--lambdas", default=[0.9, 0.95, 1.0, 1.05, 1.10], type=list)
    args = parser.parse_args()
    main(**vars(args))


#!/usr/bin/python3
"""
Author: Victor Trejo
Description: This file contains all the models.
"""
from django.db import models
from django.conf import settings

class EducationHistoryEntry(models.Model):
    """Education history entry."""
    career = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )
    institution = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )
    startYear = models.PositiveIntegerField()
    endYear = models.PositiveIntegerField()

    def __str__(self):
        return "{}-{} ({}-{})".format(
            self.career,
            self.institution,
            self.startYear,
            self.endYear
        )

class AcademicProjects(models.Model):
    """Academic Projects entry."""
    title = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )
    course = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )
    description = models.TextField(
        max_length=500,
        blank=True,
        default=''
    )
    languages = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )
    libraries = models.CharField(
        max_length=300,
        blank=True,
        default=''
    )
    repositoryUrl = models.URLField(
        max_length=1000,
        default=''
    )

    def __str__(self):
        return "{}-{}/({}-{}): {} at {}".format(
            self.title,
            self.course,
            self.languages,
            self.libraries,
            self.description,
            self.repositoryUrl
        )

class PersonalProjects(models.Model):
    """Personal Projects entry."""
    title = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )
    description = models.TextField(
        max_length=500,
        blank=True,
        default=''
    )
    languages = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )

    libraries = models.CharField(
        max_length=300,
        blank=True,
        default=''
    )

    repositoryUrl = models.URLField(
        max_length=1000,
        default=''
    )

    def __str__(self):
        return "{}/({}-{}): {} at {}".format(
            self.title,
            self.languages,
            self.libraries,
            self.description,
            self.repositoryUrl
        )

class SectionMultipleContent(models.Model):
    """Section multiple Content entry."""
    label = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )
    def __str__(self):
        return "{}".format(
            self.label
        )

class Content(models.Model):
    """Section Content entry."""
    label = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )

    description = models.TextField(
        max_length=1000,
        blank=True,
        default=''
    )

    section_multiple_content = models.ForeignKey(SectionMultipleContent)

    def __str__(self):
        return "{}: {}".format(
            self.label,
            self.description
        )

class SkillCategory(models.Model):
    """Skill category"""
    label = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )
    description = models.TextField(
        max_length=500,
        blank=True,
        default=''
    )

    def __str__(self):
        return self.label

class Skill(models.Model):
    """Skill."""
    label = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )
    description = models.TextField(
        max_length=500,
        blank=True,
        default=''
    )
    url = models.URLField(
        max_length=1000,
        default=''
    )
    category = models.ForeignKey(
        SkillCategory
    )

    def __str__(self):
        return self.label

class ExperienceHistoryEntry(models.Model):
    """Experience history entry."""
    position = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )
    company = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )
    startDate = models.DateField()
    endDate = models.DateField()
    description = models.TextField(
        max_length=1500,
        blank=True,
        default=''
    )

    def __str__(self):
        return "{0}-{1} ({2}-{3}) {4}".format(
            self.position,
            self.company,
            self.startDate,
            self.endDate,
            self.description
        )

class FileContent(models.Model):
    """File Content entry."""
    label = models.CharField(
        max_length=120,
        blank=True,
        default=''
    )
    file = models.FileField()

    def __str__(self):
        return "{}: {}".format(
            self.label,
            self.file
        )


import sys
import os
import re
import argparse
from itertools import count,islice
from scipy.stats import poisson

def readVCF(vcffile):
	vlist = []
	with open(vcffile) as v:
		for line in v:
			if not line.startswith("#"):
				pos = line.rstrip().split("\t")[1]
				vlist.append(int(pos))
	return vlist

def generateCSV(vlist,genomelength,stepsize,windowsize,outcsv):
	try:
		os.remove(outcsv)
	except OSError:
		pass
	out=open(outcsv,"aw")
	ave=float(len(vlist))/genomelength*windowsize
	for i in islice(count(0),1,genomelength,stepsize):
		window = range(i,i+windowsize)
		cnt=sum([1 for x in vlist if x in window])
		lowerp=poisson.cdf(cnt,ave)
		upperp=1-lowerp
		out.write(','.join([str(i),str(i+windowsize-1),str(cnt),str(ave),str(lowerp),str(upperp)])+"\n")
	out.close()

def main():
	parser = argparse.ArgumentParser(description=
	''' Take an input of VCF file generated from GATK SNP analysis pipeline against a bacteria genome, and output the frequency of SNP occurrence and Poisson probability. Those calculations are all based on a given window size and step size, which both could be modified by users.''',
		epilog="No warranty comes with this script. Author: ying.eddi2008@gmail.com. \nAny suggestions or bugs report are welcomed.",add_help=False,formatter_class=argparse.RawTextHelpFormatter)
	##### Required arguments #####
	required = parser.add_argument_group('required arguments')
	required.add_argument("-i","--infile", help="input vcffile to be parsed",type=str,required=True)
	required.add_argument("-g","--glen", help="bacteria genome length",type=int,required=True)
	##### Optional arguments #####
	optional = parser.add_argument_group('optional arguments')
	optional.add_argument("-o","--outfile",help="output file name [default: <infile>.csv]",type=str)
	optional.add_argument("-s","--ssize",default=500,help="step size [default: 500]",type=int)
	optional.add_argument("-w","--wsize",default=1000,help="windown size [default: 1000]",type=int)
	optional.add_argument("-h","--help",help="show this help message and exit",action="help")
	args = parser.parse_args()
	##### Output file name ####
	if not args.outfile:
		args.outfile=args.infile.replace(".vcf",".csv")
	##### Procedure #####
	SNPlist = readVCF(args.infile)
	generateCSV(SNPlist,args.glen,args.ssize,args.wsize,args.outfile)

if __name__ == "__main__":
	main()


"""
Copyright 2020 The OneFlow Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from typing import List, Optional, Union

import oneflow as flow
from oneflow.nn.common_types import _size_any_t
from oneflow.nn.modules.utils import _handle_size_arg, _single

def empty_op(
    *size,
    dtype: Optional[flow.dtype] = None,
    device: Union[flow.device, str] = None,
    placement: flow.placement = None,
    sbp: Union[
        flow._oneflow_internal.sbp.sbp, List[flow._oneflow_internal.sbp.sbp]
    ] = None,
    requires_grad: bool = False,
    pin_memory: bool = False,
):
    """
    Returns a tensor filled with uninitialized data.
    The shape of the tensor is defined by the variable argument ``size``.

    Args:
        size (int... or oneflow.Size): Defining the shape of the output tensor.
          Can be a variable number of arguments or a collection like a list or tuple or oneflow.Size.
        dtype (flow.dtype, optional): The desired data type of returned tensor. Default: ``flow.float32``.
        device (oneflow.device, optional): The desired device of returned local tensor. If None, uses the
          current device.
        placement (flow.placement, optional): The desired device of returned global tensor. If None, will
          construct local tensor.
        sbp (flow.sbp or List[flow.sbp], optional): The desired sbp of returned global tensor.
        requires_grad (bool, optional): If autograd should record operations on the returned tensor. Default: False.
        pin_memory (bool, optional) – If set, returned tensor would be allocated in the pinned memory. Works only for CPU tensors. Default: False.

    For example:

    .. code-block:: python

        >>> import oneflow as flow
        >>> y = flow.empty(4, 5)  # construct local empty tensor
        >>> y.shape
        oneflow.Size([4, 5])
        >>> y.is_global
        False
        >>> placement = flow.placement("cpu", ranks=[0])
        >>> y = flow.empty(4, 5, placement=placement, sbp=flow.sbp.broadcast)  # construct consistent empty tensor
        >>> y.is_global
        True

    """
    assert size is not None, "shape must not be None"

    shape = _single(_handle_size_arg(size))

    if dtype is None:
        dtype = flow.float32
    if placement is None:
        if device is None:
            device = flow.device("cpu")
    else:
        assert (
            device is None
        ), "argument 'device' must be None when argument 'placement' exist"

    if placement is not None:
        assert (
            sbp is not None
        ), "argument 'sbp' must not be None when argument 'placement' exist"
        assert isinstance(
            sbp, (flow.sbp.sbp, tuple, list)
        ), f"argument 'sbp' must be flow.sbp.sbp, not %s" % (type(sbp))
        if isinstance(sbp, flow.sbp.sbp):
            sbp = (sbp,)
        else:
            for elem in sbp:
                assert isinstance(elem, flow.sbp.sbp), (
                    "Element in argument 'sbp' must be flow.sbp.sbp, not %s"
                    % (type(elem))
                )
        assert len(sbp) == len(placement.ranks.shape)
    else:
        assert sbp is None, "argument 'sbp' must be None"

    if placement is not None:
        tensor = flow._C.global_empty(shape, dtype=dtype, placement=placement, sbp=sbp)
    else:
        tensor = flow._C.empty(shape, dtype=dtype, device=device, pin_memory=pin_memory)
    tensor.requires_grad_(requires_grad)
    return tensor

def new_empty_op(
    x, size, dtype=None, device=None, placement=None, sbp=None, requires_grad=False
):
    new_size = _single(_handle_size_arg(size))
    new_dtype = dtype
    new_device = device
    new_placement = placement
    new_sbp = sbp

    if dtype is None:
        new_dtype = x.dtype
    if device is None:
        new_device = x.device if x.is_local else None
    if placement is None:
        new_placement = x.placement if x.is_global else None
    if sbp is None:
        new_sbp = x.sbp if x.is_global else None

    return empty_op(
        new_size,
        dtype=new_dtype,
        device=new_device,
        placement=new_placement,
        sbp=new_sbp,
        requires_grad=requires_grad,
    )

if __name__ == "__main__":
    import doctest

    doctest.testmod(raise_on_error=True)


import numpy as np
from numba import jit
import pandas as pd
import multiprocessing as mp
import tqdm, glob, time, pickle, re
from scipy.stats import skew, kurtosis
from scipy.fftpack import rfft
from scipy.signal import find_peaks, peak_widths

def atomic_worker(args):

    source_df_dir, start_ix, end_ix, compute_feats = args

    # Setup
    feat_names = []
    feat_arrays = []

    # Load df chunk
    df_chunk = pd.read_hdf(source_df_dir).iloc[start_ix:end_ix+1]
    index = df_chunk.index
    arr = df_chunk.values

    '''
    Stats features
    '''
    if compute_feats['stats-feats']:

        # Feature names
        stats_feats_names = [
            'std',
        ]

        stats_feats_arr = np.hstack([
            np.std(arr, axis=1, keepdims=True),
            # kurtosis(arr, axis=1)[:, None],
        ])

        feat_names.extend(stats_feats_names)
        feat_arrays.append(stats_feats_arr)

    '''
    Fourier features
    '''
    if compute_feats['fourier-feats']:

        # Num of coeffs
        n = 10

        # Calc coeffs
        fft_coefs = rfft(arr, axis=1)[:, :n]

        feat_names.extend([f'rfft_coef_{c:d}' for c in range(n)])
        feat_arrays.append(fft_coefs)

    '''
    Delta features
    '''
    if compute_feats['delta-feats']:

        # Feature names
        names = [
            'mean_abs_delta',
            'std_abs_delta',
        ]# + [f'delta_perc_{p:.2f}' for p in percentiles]

        deltas = arr[:, 1:] - arr[:, :-1]
        arr_ = np.hstack([
            np.mean(np.abs(deltas), axis=1, keepdims=True),
            np.std(np.abs(deltas), axis=1, keepdims=True),
            # np.percentile(arr, percentiles, axis=1).T
        ])

        feat_names.extend(names)
        feat_arrays.append(arr_)

    '''
    Peak features
    '''
    if compute_feats['peak-feats']:

        min_heights = [35]

        # Feature names
        names = []
        for n in min_heights:
            names.extend([
                f'num_peaks_{n:d}',
            ])

        nfeats = len(names)
        arr_ = np.zeros(shape=(arr.shape[0], nfeats))

        # Process peaks per chunk
        for i, signal in tqdm.tqdm(enumerate(arr), total=arr.shape[0]):

            # Get positive and negative peaks
            pos_peak_ixs, pos_peak_props = find_peaks(signal, height=np.min(min_heights))
            neg_peak_ixs, neg_peak_props = find_peaks(-signal, height=np.min(min_heights))

            pos_peak_heights = pos_peak_props['peak_heights']
            neg_peak_heights = -neg_peak_props['peak_heights']

            # Combine and sort all peaks
            peak_ixs = np.hstack([pos_peak_ixs, neg_peak_ixs])
            peak_heights = np.hstack([pos_peak_heights, neg_peak_heights])
            sort_order = np.argsort(peak_ixs)
            peak_ixs = peak_ixs[sort_order]
            peak_heights = peak_heights[sort_order]
            abs_peak_heights = np.abs(peak_heights[sort_order])

            for j, min_h in enumerate(min_heights):

                # Num. peaks
                arr_[i, j] = abs_peak_heights[abs_peak_heights >= min_h].size

        feat_names.extend(names)
        feat_arrays.append(arr_)

    '''
    Rolling features
    '''
    if compute_feats['roll-feats']:

        windows = [500]

        # Feature names
        names = []
        for w in windows:
            names.extend([
                f'mean_rolling_std_{w:d}',
            ])

        nfeats = len(names)

        arr_ = np.zeros(shape=(arr.shape[0], nfeats))

        for iw, w in enumerate(windows):

            size = arr.shape[1] - w + 1
            roll_stds = np.zeros(shape=(arr.shape[0], size))
            for i in tqdm.tqdm(np.arange(0, size), total=size):
                roll_stds[:, i] = np.std(arr[:, i:i + w], axis=1)

            arr_[:, iw] = np.mean(roll_stds, axis=1)

        feat_names.extend(names)
        feat_arrays.append(arr_)

    '''
    Aggregate all feats and return as df
    '''
    # Build final pandas dataframe
    df = pd.DataFrame(
        data=np.hstack(feat_arrays),
        columns=feat_names,
        index=index
    )

    print(f'Computed chunk {start_ix:d} to {end_ix}')

    return df

def gen_feats(save_rel_dir, save_name, source_df_dir, compute_feats):
    '''
    Generate custom features dataframe from stored preprocessed signal chunks

    :param save_rel_dir (str) Relative dir to save calculated feats
    :param save_name (str) Feat set name
    :param source_df_dir (str) Relative dir to source data dataframe
    :param compute_feats (dict) Dict of bools marking the feats to generate
    :return:
    '''

    np.warnings.filterwarnings('ignore')

    # Read source df
    df = pd.read_hdf(source_df_dir)

    # Split source df into batches
    n_chunks = 16
    atomic_args = [(source_df_dir, ix[0], ix[-1], compute_feats) for ix in np.array_split(np.arange(df.shape[0]), n_chunks)]

    print(f'> feature_engineering : Creating mp pool . . .')

    pool = mp.Pool(processes=8)#mp.cpu_count()-4)
    res = pool.map(atomic_worker, atomic_args)
    pool.close()
    pool.join()

    print(f'> feature_engineering : Concating and saving results . . .')

    # Concat atomic computed feats and save df
    df = pd.concat(res, axis=0).sort_index()
    df.to_hdf(save_rel_dir+'/'+save_name, key='w')

    # Also save feature names
    feat_list = list(df.columns)
    with open(save_rel_dir+'/'+save_name.split('.h5')[0]+'.txt', 'w') as f:
        f.writelines([f'{feat_name}\n' for feat_name in feat_list])
    with open(save_rel_dir+'/'+save_name.split('.h5')[0]+'.pkl', 'wb') as f2:
        pickle.dump(feat_list, f2, protocol=pickle.HIGHEST_PROTOCOL)

dataset = 'train'
st = time.time()

compute_feats_template = {
    'stats-feats': bool(0),
    'fourier-feats': bool(0),
    'delta-feats': bool(0),
    'peak-feats': bool(0),
    'roll-feats': bool(0),
}

feats_to_gen = {
    'stats-feats': 'stats_v10',
    # 'fourier-feats': 'fourier_v4',
    # 'delta-feats': 'delta_v9',
    # 'peak-feats': 'peak_v9',
    # 'roll-feats': 'roll_v9',
}

for ft_name, file_name in feats_to_gen.items():

    cpt_fts = compute_feats_template.copy()
    cpt_fts[ft_name] = True

    gen_feats(
        save_rel_dir='../features',
        save_name=f'{dataset}_{file_name}.h5',
        source_df_dir=f'../data/{dataset}_df.h5',
        compute_feats=cpt_fts,
    )

print(f'> feature_engineering : Done, wall time : {(time.time()-st):.2f} seconds .')


# Copyright (c) 2018, Olmo Kramer <olmo.kramer@protonmail.com>

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

'''
Tab related options.
'''

import curses
import typing as t

from fb.options import define_option, options

class _BoundedIntParser:
    '''
    Parse an int and ensure it is within bounds (inclusive).

    :param lower_bound:
        Lower bound.
    :param upper_bound:
        Upper bound.
    '''

    def __init__(
        self, lower_bound: int = None, upper_bound: int = None
    ) -> None:
        self.lower_bound = lower_bound
        self.upper_bound = upper_bound

    def __call__(self, value: t.Any, *_: t.Any) -> int:
        '''
        Parse an int and ensure it is within bounds (inclusive).

        :param value:
            Value to parse.
        :returns:
            Parsed int.
        :raises ValueError:
            if value is not an int or not within bounds.
        '''

        int_value = int(value)

        if self.lower_bound is not None and int_value < self.lower_bound:
            raise ValueError(f'Value lower bound: {self.lower_bound}.')

        if self.upper_bound is not None and int_value < self.upper_bound:
            raise ValueError(f'Value upper bound: {self.upper_bound}.')

        return int_value

def _parse_str_sequence(strings: t.Any, *_: t.Any) -> t.Sequence[str]:
    '''
    Parse a list of strings or a single string.

    :param strings:
        Objects to parse to string.
    :returns:
        List of stringified objects.
    '''

    if not isinstance(strings, t.Sequence) or isinstance(strings, str):
        strings = [strings]

    return [str(string) for string in strings]

define_option(
    name='colsep',
    doc='''
    Separator between columns.
    ''',
    parser=lambda x, _: str(x),
    default='  ',
)

define_option(
    name='columns',
    doc='''
    Columns to display.
    ''',
    parser=_parse_str_sequence,
    default=['mode', 'nlink', 'uid', 'gid', 'size', 'date', 'name'],
)

define_option(
    name='depth',
    doc='''
    Show files ``depth`` directories deep.
    ''',
    parser=_BoundedIntParser(lower_bound=1),
    default=1,
)

define_option(
    name='directories_first',
    doc='''
    List directories before files.
    ''',
    parser=lambda x, _: bool(x),
    default=True,
)

define_option(
    name='maxdirs',
    doc='''
    Max amount of nested directories to show in the tab bar.
    ''',
    parser=_BoundedIntParser(lower_bound=0),
    default=0,
)

define_option(
    name='wrap',
    doc='''
    Wrap around the top/bottom of the file list.
    ''',
    parser=lambda x, _: bool(x),
    default=True,
)

options.colors = {  # type: ignore
    'file.selected': curses.COLOR_MAGENTA,
    'file.link': curses.COLOR_CYAN,
    'file.special': curses.COLOR_YELLOW,
    'file.directory': curses.COLOR_BLUE,
    'file.executable': curses.COLOR_GREEN,
}


"""
Напишите программу, которая принимает на стандартный вход список игр футбольных команд с результатом матча
и выводит на стандартный вывод сводную таблицу результатов всех матчей.

За победу команде начисляется 3 очка, за поражение — 0, за ничью — 1.

Формат ввода следующий:
В первой строке указано целое число nn — количество завершенных игр.
После этого идет nn строк, в которых записаны результаты игры в следующем формате:
Первая_команда;Забито_первой_командой;Вторая_команда;Забито_второй_командой

Вывод программы необходимо оформить следующим образом:
Команда:Всего_игр Побед Ничьих Поражений Всего_очков
"""

n = int(input())
x_list = [input().split(';') for x in range(n)]
vs = [(x[0], x[2]) for x in x_list]
import itertools
clubs = set(itertools.chain.from_iterable(vs))
res = {club:[0, 0, 0, 0, 0] for club in clubs}
for kom1, gol1, kom2, gol2 in x_list:
    res[kom1][0] += 1
    res[kom2][0] += 1
    if int(gol1) > int(gol2):
        res[kom1][1] += 1
        res[kom1][4] += 3
        res[kom2][3] += 1
    elif int(gol1) < int(gol2):
        res[kom2][1] += 1
        res[kom2][4] += 3
        res[kom1][3] += 1
    elif int(gol1) == int(gol2):
        res[kom1][2] += 1
        res[kom1][4] += 1
        res[kom2][2] += 1
        res[kom2][4] += 1
for club in clubs:
    print('{}:{}'.format(club, ' '.join(map(str, res[club]))))


from django import forms
from multiselectfield import MultiSelectFormField
class EnquiryForm(forms.Form):
    name = forms.CharField(
        label="",
        widget= forms.TextInput(
            attrs={
                'class':'form-control',
                'placeholder':'Enter Your name'
            }
        )
    )
    email = forms.EmailField(
        label="",
        widget=forms.EmailInput(
            attrs={
                'class': 'form-control',
                'placeholder': 'Enter Your Email'
            }
        )
    )
    city = forms.CharField(
        label="",
        widget=forms.TextInput(
            attrs={
                'class': 'form-control',
                'placeholder': 'Enter Your City'
            }
        )
    )
    mobile = forms.IntegerField(
        label="",
        widget=forms.NumberInput(
            attrs={
                'class': 'form-control',
                'placeholder': 'Enter Your Mobile Number'
            }
        )
    )
    query = forms.CharField(
        label="",
        widget= forms.Textarea(
            attrs={
                'class': 'form-control',
                'placeholder': 'Enter your Feedback Here or Any Query'
            }
        )
    )

class Crops(forms.Form):
    LOCATION_CHOICES = (
        ('Banda','Banda'),
        ('Lucknow', 'Lucknow'),
        ('Siddharth nagar', 'Siddharth nagar'),
        ('Pryagraj', 'Pryagraj'),
        ('Chitrakoot', 'Chitrakoot'),
        ('Jhansi', 'Jhansi'),
        ('Bulandshahar', 'Bulandshahar'),
        ('Kanpur', 'Kanpur'),
        ('Meerut', 'Meerut'),
        ('Agra', 'Agra'),
        ('Gorakhpur', 'Gorakhpur'),
        ('Maharajganj', 'Maharajganj'),
        ('Santkabir nagar', 'Santkabir nagar'),
        ('Saharanpur', 'Saharanpur'),
        ('Balrampur', 'Balrampur'),
        ('Gonda', 'Gonda'),
        ('Basti', 'Basti'),
        ('Balia', 'Balia'),
        ('Banaras', 'Banaras'),
        ('Jaunpur', 'Jaunpur'),
        ('Sobhadra', 'Sonbhadra')
    )
    location = forms.CharField(
        label="City",
        widget=forms.Select(
            choices=LOCATION_CHOICES,
            attrs={
                'class': 'form-control'
            }
        )
    )
    SOIL_CHOICES= (

        ('Red','Red'),
        ('Brown', 'Brown'),
        ('Allubial', 'Allubial'),
        ('Rock', 'Rock'),
        ('Black', 'Black'),
        ('Latterite', 'Latterite')
    )
    soil_type = forms.CharField(
        label="Soil_Type",
        widget= forms.Select(
            choices=SOIL_CHOICES,
            attrs={
                'class': 'form-control'
            }
        )
    )
    MONTH_CHOICES = (
        ('Jan', 'JAN'),
        ('Feb', 'FEB'),
        ('Mar', 'MAR'),
        ('Apr', 'APR'),
        ('May', 'MAY'),
        ('Jun', 'JUN'),
        ('Jul', 'JUL'),
        ('Aug', 'AUG'),
        ('Sep', 'SEP'),
        ('Oct', 'OCT'),
        ('Nov', 'NOV'),
        ('Dec', 'DEC')
    )
    month = forms.CharField(
        label="Month",
        widget=forms.Select(
            choices=MONTH_CHOICES,
            attrs={
                'class': 'form-control'
            }
        )
    )


from collections import deque

"""
Maximum of all subarrays of size k (Added a O(n) method)
Given an array and an integer k, find the maximum for each and every contiguous subarray of size k.

Examples:

Input :
arr[] = {1, 2, 3, 1, 4, 5, 2, 3, 6}
k = 3
Output :
3 3 4 5 5 5 6

Input :
arr[] = {8, 5, 10, 7, 9, 4, 15, 12, 90, 13}
k = 4
Output :
10 10 10 15 15 90 90

"""

def maxK(arr,k):
    if not arr :
        return None
    else :
        dq = deque()
        for i in range(k):
            dq.append(arr[i])


#!/bin/ python3.8

#Projet numéro 2
#Read Definition for more medical description.
#Read Readme for more sources.

# DataFrame : CSV - UTF 8
import pandas as pd

# Windows :
#data = pd.read_csv("DATA/heart.csv", encoding ="utf8")
# Ubuntu :
data = pd.read_csv('/media/jhy/JHY/Etudes/Projets/Work in progress/Projets 2/DATA/heart.csv', encoding = "utf8")

#Import graph
import matplotlib.pyplot as plt

# Disable chain limit
pd.options.mode.chained_assignment = None

# Shortcut for DataFrame
age,sex = data['age'],data['sex']
chest_pain, blood_pressure = data['cp'], data['trestbps']
cholesterol ,fast_sugars = data['chol'],data['fbs']
rest_cardiogram,maximun_heartrate = data['restecg'],data['thalach']
pain_exercice, depression_rest = data['exang'],data['oldpeak']
slope, big_vessel = data['slope'],data['ca']
diag, heart_faillure = data['thal'],data['target']

#correction : target was swap 0 mean heart attack, 1 mean no trouble
heart_faillure[heart_faillure == 0] = "Accident"
heart_faillure[heart_faillure == 1] = "Nothing"

#ajustement
sex[sex == 1] = "Male"
sex[sex == 0] = "Female"

#Age to target
def hp_faillure():
    cpt = 0
    cpt_tot = 0
    for x in heart_faillure:
        if x is True :
            cpt +=1
        else :
            cpt_tot +=1
    total = cpt + cpt_tot
    percent = round((cpt / total)*100,2)
    return percent

age_target = round(age.mean()),hp_faillure()

#print(parite_HF)

############################################################################################################################
######################################          DATA TRENDS          #######################################################
############################################################################################################################

# Parity

men = 0
women = 0
for x in sex :
    if x == "Male" :
        men += 1
    elif x == "Female" :
        women += 1
    else :
        print('check database')
        pass

men_target = sex[sex == 'Male'][heart_faillure =="Accident"].count()
women_target = sex[sex == 'Female'][heart_faillure =="Accident"].count()

parite_HF = round(women / (women+men)*100,2)
age_man_target = age[sex=="Male"][heart_faillure=="Accident"]
age_man_untarget = age[sex=="Male"][heart_faillure=="Nothing"]
age_woman_target = age[sex=="Female"][heart_faillure=="Accident"]
age_woman_untarget = age[sex=="Female"][heart_faillure=="Nothing"]
# Cholesterols

avg_cholesterol = round(cholesterol.mean(),2)
#Target
cholesterol_male_target = cholesterol[sex=="Male"][heart_faillure=="Accident"]
cholesterol_female_target = cholesterol[sex=="Female"][heart_faillure=="Accident"]
#Untarget
cholesterol_male_untarget = cholesterol[sex=="Male"][heart_faillure=="Nothing"]
cholesterol_female_untarget = cholesterol[sex=="Female"][heart_faillure=="Nothing"]

#Diabetes
# > 125mg = True = Diabetes
#target
diab_male_target = fast_sugars[sex=="Male"][heart_faillure=="Accident"][fast_sugars ==1]
diab_female_target = fast_sugars[sex=="Female"][heart_faillure=="Accident"][fast_sugars ==1]
#untarget
diab_male_untarget = fast_sugars[sex=="Male"][heart_faillure=="Nothing"]
diab_female_untarget = fast_sugars[sex=="Female"][heart_faillure=="Nothing"]

#Resting blood pressure
avg_bloodpress = round(blood_pressure.mean(),2)
# > 90 risk trouble health
#target
bloodpress_male_target = blood_pressure[sex=='Male'][heart_faillure=='Accident']
bloodpress_female_target = blood_pressure[sex=='Female'][heart_faillure=='Accident']
#untarget
bloodpress_male_untarget = blood_pressure[sex=='Male'][heart_faillure=='Nothing']
bloodpress_female_untarget = blood_pressure[sex=='Female'][heart_faillure=='Nothing']

#chest pain
# 1: typical angina, 2: atypical angina, 3: non-anginal pain, 4: asymptomatic
#Male
#Target
chest_type1_men_target = chest_pain[sex=='Male'][heart_faillure=='Accident'][chest_pain == 1]
chest_type2_men_target = chest_pain[sex=='Male'][heart_faillure=='Accident'][chest_pain == 2]
chest_type3_men_target = chest_pain[sex=='Male'][heart_faillure=='Accident'][chest_pain == 3]
chest_type4_men_target = chest_pain[sex=='Male'][heart_faillure=='Accident'][chest_pain == 4]
#Untarget
chest_type1_men_untarget = chest_pain[sex=='Male'][heart_faillure=='Nothing'][chest_pain == 1]
chest_type2_men_untarget = chest_pain[sex=='Male'][heart_faillure=='Nothing'][chest_pain == 2]
chest_type3_men_untarget = chest_pain[sex=='Male'][heart_faillure=='Nothing'][chest_pain == 3]
chest_type4_men_untarget = chest_pain[sex=='Male'][heart_faillure=='Nothing'][chest_pain == 4]
#Female
#Target
chest_type1_women_target = chest_pain[sex=='Female'][heart_faillure=='Accident'][chest_pain == 1]
chest_type2_women_target = chest_pain[sex=='Female'][heart_faillure=='Accident'][chest_pain == 2]
chest_type3_women_target = chest_pain[sex=='Female'][heart_faillure=='Accident'][chest_pain == 3]
chest_type4_women_target = chest_pain[sex=='Female'][heart_faillure=='Accident'][chest_pain == 4]
#Untarget
chest_type1_women_untarget = chest_pain[sex=='Female'][heart_faillure=='Nothing'][chest_pain == 1]
chest_type2_women_untarget = chest_pain[sex=='Female'][heart_faillure=='Nothing'][chest_pain == 2]
chest_type3_women_untarget = chest_pain[sex=='Female'][heart_faillure=='Nothing'][chest_pain == 3]
chest_type4_women_untarget = chest_pain[sex=='Female'][heart_faillure=='Nothing'][chest_pain == 4]

#############################################################################################################################
#---------------------------------------------------Graphique----------------------------------------------------------------
# Disparité cholesterol
plt.scatter(age,cholesterol,c ='Yellow', linewidths= 5)
plt.scatter(age_man_target, cholesterol_male_target, c ='Blue')
plt.scatter(age_woman_target,cholesterol_female_target, c = 'Pink')
plt.ylabel("CHOLESTEROLS RATES")
plt.xlabel("AGES")
plt.title("DATA FROM 300 PATIENTS : HEART ATTACK CORRELATION BETWEEN CHOLESTEROLS")
legends_chol=["Health","Male Heart Attack","Female Heart Attack"]
plt.legend(legends_chol,bbox_to_anchor = (1,1))
plt.ylim(0,600)
plt.xlim(0,90)
plt.show()

#disparité diabetes
#can only be use in scoring

#Resting blood pressure
plt.scatter(age,blood_pressure,c ='Yellow', linewidths= 5)
plt.scatter(age_man_target, bloodpress_male_target, c ='Blue')
plt.scatter(age_woman_target,bloodpress_female_target,c = 'Pink')
plt.ylabel("BLOOD PRESSURES")
plt.xlabel("AGES")
plt.title("DATA FROM 300 PATIENTS : HEART ATTACK CORRELATION BETWEEN BLOOD PRESSURES")
plt.legend(legends_chol,bbox_to_anchor = (1,1))
plt.ylim(0,250)
plt.xlim(0,90)
plt.show()

# chest


from GJEMS.morph.morph import BasicMorph

import sys
import matplotlib.pyplot as plt
import numpy as np
from scipy.cluster import hierarchy
from scipy.spatial.distance import pdist

assert len(sys.argv) == 2, 'Only one argument, the path of the swcfile expected, ' + str(len(sys.argv)) + 'found'
swcFile = sys.argv[1]

def getLeavesList(clusterNode):

    # For more info look at the doc of clusterNode.pres_order
    return clusterNode.pre_order(lambda x: x.id)

listdiff = lambda x, y: [a - b for a, b in zip(x, y)]

figs = []
nIters = 20
cols = ['r', 'g', 'b', 'm', 'c']

testNrn = BasicMorph(swcFile)
testNrn.initTerminalDegrees()
testNrn.initxyzr()
testNrn.initSecVolumes()

fig = plt.figure()
plt.show(block=False)

obs = []

secNames = []
for morphSec in testNrn.allsec.itervalues():
    if not ((morphSec.ptr.sec.name().find('soma') > 0) and (morphSec.terminalDegree == 0)):
        obs.append([morphSec.terminalDegree])
        secNames.append(morphSec.ptr.sec.name())

obs = np.asarray(obs)
nObs = len(obs)

linkMatrix = hierarchy.single(pdist(obs))
rootNode = hierarchy.to_tree(linkMatrix)

levelIDList = []

presNode = rootNode
presNodeLinkInd = nObs - 2

totalObs = float(len(obs))
pointsDone = 0

for ind in range(nIters):

    # These indices correspond to the iteration(1, 2,...) in which the node in question was formed.
    # For more info look at the doc of scipy.cluster.hierarchy.linkage

    leftLinkInd = linkMatrix[presNodeLinkInd, 0] - nObs
    rightLinkInd = linkMatrix[presNodeLinkInd, 1] - nObs

    if leftLinkInd >= 0:
        nPtsLeft = linkMatrix[leftLinkInd, 3]
    else:
        nPtsLeft = 1

    if rightLinkInd >= 0:
        nPtsRight = linkMatrix[rightLinkInd, 3]
    else:
        nPtsRight = 1

    if nPtsLeft < nPtsRight:

        levelIDs = getLeavesList(presNode.left)
        pointsDone += nPtsLeft
        presNode = presNode.right
        presNodeLinkInd = rightLinkInd

    else:

        levelIDs = getLeavesList(presNode.right)
        pointsDone += nPtsLeft
        presNode = presNode.left
        presNodeLinkInd = leftLinkInd

    levelIDList.append(levelIDs)

    for id in levelIDs:
        plt.plot(obs[id, 0], ind, color=cols[ind % len(cols)], marker='o', mfc=cols[ind % len(cols)])

    if presNode.is_leaf():
        break

residualIDs = getLeavesList(presNode)

for id in residualIDs:
    plt.plot(obs[id, 0], nIters, color='k', marker='o', mfc='k')

plt.xlabel('Terminal Degree')
plt.ylabel('nIters')
plt.draw()


from django.shortcuts import render
from django.views.generic import TemplateView, ListView, FormView, UpdateView
from django.views.generic.edit import CreateView, UpdateView, DeleteView, FormView
from django.views.generic.list import ListView
from django.core.files.storage import default_storage
from django.http import HttpResponse, HttpResponseRedirect
from .forms import UploadFileForm
from .models import Model, Dataset, Document
from django.urls import reverse
import subprocess
import uuid
import os
from django.conf import settings
import base64

class DatasetDelete(DeleteView):
    model = Dataset
    template_name = 'delete.html'

    def get_success_url(self):
        return reverse('dataset')

class DatasetUpdate(UpdateView):
    model = Dataset
    fields = ['name', 'description']
    template_name = 'update.html'

    def get_success_url(self):
        return reverse('dataset')

class ModelDelete(DeleteView):
    model = Model
    template_name = 'delete.html'

    def get_success_url(self):
        return reverse('model')

class ModelUpdate(UpdateView):
    model = Model
    fields = ['name', 'description', 'model']
    template_name = 'update.html'

    def get_success_url(self):
        return reverse('model')

class ModelView(CreateView):
    model = Model
    template_name = "model.html"
    fields = ['name', 'description', 'model']

    def get_context_data(self, **kwargs):
        context = super(ModelView, self).get_context_data(**kwargs)
        context["object_list"] = Model.objects.all()
        return context

    def get_success_url(self):
        return reverse('model')

class SearchView(TemplateView):
    template_name = 'search.html'
    def dispatch(self, request, *args, **kwargs):
        if "Iterate" in self.request.GET:
            return HttpResponseRedirect("iterate?dataset={}&model={}".format(self.request.GET.get("dataset"), self.request.GET.get("model")))
        return super(SearchView, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, *args, **kwargs):
        context = super(SearchView, self).get_context_data(*args, **kwargs)
        datasets = []
        for dataset in Dataset.objects.all():
            datasets.append({'id': dataset.id, 'name': dataset.name})
        context['datasets'] = datasets

        # build context for models
        models = ['OkapiBM25','PivotedLength','AbsoluteDiscount','JelinekMercer','DirichletPrior']
        default_models = models.copy()
        for model in Model.objects.all():
            models.append(model.name)
        context['models'] = models

        context['default_model'] = self.request.GET.get("model")
        context['default_dataset'] = self.request.GET.get("dataset")

        if "query" in self.request.GET:
            print(self.request.GET)
            obj = Dataset.objects.get(name=self.request.GET.get("dataset")) # find dataset path
            print(obj.id)
            set_model = self.request.GET.get("model")
            if set_model not in default_models:
                set_model = base64.b64encode(Model.objects.get(name=set_model).model.encode())
            print(set_model)
            folder = obj.data.name.split("/")[1]
            results = eval(subprocess.run(["python3", "search_eval.py", folder, set_model, self.request.GET.get("query")], stdout=subprocess.PIPE).stdout.decode("utf-8"))
            #print(results)

            list = []
            counter = 1

            # results[0] = top k articles and scores
            # results[1] = ndcg score for each query
            # results[3]:
            #              results[3][0] - avg_dl
            #              results[3][1] - num_docs
            #              results[3][2] - total_terms
            #              results[3][3] - query_length
            #              results[3][4] - corpus_term_count
            #              results[3][5] - corpus_unique_term
            #              results[3][6] - term_doc_count (tuple list)
            #              results[3][0] - corpus_term_count (tuple list)
            # results[3] = average ndcg score
            #for x in results[0][0]:
            #    list.append({'body': Document.objects.filter(dataset_id=obj.id).get(document_id=x[0]).body[0 : 120], 'score': x[1], 'rank': counter})

            # MAIN
            for x in results[0][0]:
                doc = Document.objects.filter(dataset_id=obj.id).get(document_id=x[0])
                list.append({'body': doc.body[0 : 120], 'score': x[1], 'rank': counter, 'doc_size': doc.doc_size, 'unique_terms': doc.doc_unique_terms})
                counter += 1

            context['results'] = list
        return context

class IterateView(TemplateView):
    template_name = 'iterate.html'
    def get_context_data(self, *args, **kwargs):
        context = super(IterateView, self).get_context_data(*args, **kwargs)
        models = ['OkapiBM25','PivotedLength','AbsoluteDiscount','JelinekMercer','DirichletPrior']
        default_models = models.copy()
        print(self.request.GET)
        obj = Dataset.objects.get(name=self.request.GET.get("dataset")) # find dataset path
        print(obj.id)
        set_model = self.request.GET.get("model")
        if set_model not in default_models:
            set_model = base64.b64encode(Model.objects.get(name=set_model).model.encode())
        print(set_model)
        folder = obj.data.name.split("/")[1]
        results = eval(subprocess.run(["python3", "search_eval.py", folder, set_model, "-1"], stdout=subprocess.PIPE).stdout.decode("utf-8"))
        data = []
        for index, x in enumerate(results[0]):
            topk = []
            for y_index, y in enumerate(x):
                doc = Document.objects.filter(dataset_id=obj.id).get(document_id=y[0])
                topk.append({'rank': y_index+1,'size': doc.doc_size,'unique': doc.doc_unique_terms, 'body': doc.body[0 : 120], 'score': y[1]})
            data.append({"topk": topk, "query": results[3][index], 'ndcg': results[1][index], 'running_ndcg': results[2][index]})

        context['data'] = data

        list = []
        counter = 1

        return context

def upload_file(request):
    if request.method == 'POST':
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            test = form.save()
            file_id = Dataset.objects.get(id=test.id)
            mfile = open(os.path.join(settings.MEDIA_ROOT, "datasets/{}/dataset/dataset.dat".format(file_id.id)), 'r')
            counter = 0

            results = eval(subprocess.run(["python3", "doc_data.py", str(file_id.id)], stdout=subprocess.PIPE).stdout.decode("utf-8"))
            doc_data = results['doc_data']
            for line in mfile.readlines():
                Document.objects.create(body=line,document_id=counter,dataset=file_id,doc_size=doc_data[counter]['doc_size'],doc_unique_terms=doc_data[counter]['unique_terms'])
                counter += 1

            file_id.unique_terms = results['corpus_data']['unique_terms']
            file_id.avg_dl = results['corpus_data']['avg_dl']
            file_id.num_docs = results['corpus_data']['num_docs']
            file_id.save()

            return HttpResponseRedirect('/')
    else:
        form = UploadFileForm()
    return render(request, 'upload.html', {'form': form,'object_list': Dataset.objects.all()})


from urllib.parse import quote
import datetime
import os
import sys

TEMPLATE_DATA = (
'''{% extends 'markdown.tpl' %}

{%- block header -%}
---
layout: post
title: "{{resources['metadata']['name']}}"
''' +
datetime.datetime.now().strftime('date: %Y-%m-%d %H:%M:%S +1100') +
'''
categories: python notebook
---
{%- endblock header -%}

{% block input %}
{{ '{% highlight python %}' }}
{{ cell.source }}
{{ '{% endhighlight %}' }}
{% endblock input %}
''')

with open('tmp-jekyll-post.tpl', 'w') as outfile:
    outfile.write(TEMPLATE_DATA)

c = get_config()
c.NbConvertApp.export_format = 'markdown'
c.MarkdownExporter.template_file = 'tmp-jekyll-post'
c.Exporter.file_extension = '.md'

def path2url(path):
    """Turn a file path into a URL"""
    parts = path.split(os.path.sep)
    url = '{{ site.baseurl }}/notebooks/' + '/'.join(quote(part) for part in parts)
    print(url)
    return url

c.MarkdownExporter.filters = {'path2url': path2url}


# -*- coding: utf-8 -*-
"""
Autor:
    Jorge Casillas
Fecha:
    Noviembre/2018
Contenido:
    Inteligencia de Negocio
    Grado en Ingeniería Informática
    Universidad de Granada

    Algoritmos avanzados en Python

    En este notebook vamos a ver un ejemplo de uso de eXtreme Gradient Boosting (XGB), LightGBM y Grid Search en Python.

        XGBoost y LightGBM:
            https://www.kaggle.com/nschneider/gbm-vs-xgboost-vs-lightgbm
            https://dnc1994.com/2016/03/installing-xgboost-on-windows/
            https://github.com/Microsoft/LightGBM
            https://github.com/Microsoft/LightGBM/blob/master/examples/python-guide

        Grid Search:
            http://scikit-learn.org/stable/modules/grid_search.html
            https://www.kaggle.com/tanitter/grid-search-xgboost-with-scikit-learn?scriptVersionId=23363

        Ejemplo en R:
            http://minimaxir.com/2017/02/predicting-arrests/
"""

#Indicamos si queremos que cada variable categórica se convierta en varias binarias (tantas como categorías), indicamos binarizar = True, o si preferimos que cada variable categórica se convierta simplemente a una numérica (ordinal), binarizar = False.
binarizar = False

'''
Leemos el conjunto de datos. Los valores perdidos notados como '?' se convierten a NaN, sino se consideraría '?' como una categoría más.
'''

import pandas as pd
import numpy as np

carpeta_datos="./"
fich_tra = "water_pump_tra"
fich_tst = "water_pump_tst"
fich_tra_y = "water_pump_tra_target"

if not binarizar:
    bank_orig = pd.read_csv(carpeta_datos+fich_tra+'.csv', delimiter=',')
    bank_orig.replace(np.nan,'?')
else:
    bank_orig = pd.read_csv(carpeta_datos+fich_tra+'.csv',na_values="?", delimiter=',')

if not binarizar:
    bank_tst = pd.read_csv(carpeta_datos+fich_tst+'.csv', delimiter=',')
    bank_tst.replace(np.nan, '?')
else:
    bank_tst = pd.read_csv(carpeta_datos+fich_tst+'.csv',na_values="?", delimiter=',')

y_tra = pd.read_csv(carpeta_datos+fich_tra_y+'.csv',delimiter=',')

# print("------ Lista de características y tipos (object=categórica)")
# print(bank_orig.dtypes,"\n")

'''
Si el dataset contiene variables categóricas con cadenas, es necesario convertirlas a numéricas
antes de usar 'fit'. Si no las vamos a hacer ordinales (binarizar = True), las convertimos a
variables binarias con get_dummies. Para saber más sobre las opciones para tratar variables
categóricas: http://pbpython.com/categorical-encoding.html
'''

# devuelve una lista de las características categóricas excluyendo la columna 'class' que contiene la clase
lista_categoricas = [x for x in bank_orig.columns if (bank_orig[x].dtype == object and bank_orig[x].name != 'y')]
binarizar = False
if not binarizar:
    bank = bank_orig
else:
    # reemplaza las categóricas por binarias
    bank = pd.get_dummies(bank_orig, columns=lista_categoricas)

#Lista de atributos del dataset.
list(bank)

'''
Separamos el DataFrame en dos arrays numpy, uno con las características (X) y otro con la clase (y).
Si la última columna es la que contiene la clase se puede separar así:
'''
from sklearn import preprocessing

le = preprocessing.LabelEncoder()
# LabelEncoder codifica los valores originales entre 0 y el número de valores - 1
# Se puede usar para normalizar variables o para transformar variables no-numéricas en numéricas
columns = bank.keys().tolist()
X = bank[columns].values
y = y_tra['status_group'].values
y_bin = le.fit_transform(y)

print("X contiene las características, y contiene la clase asignada:")
print("X:", X)
print("y:", y)

'''
Si las variables categóricas tienen muchas categorías, se generarán muchas variables y algunos
algoritmos serán extremadamente lentos. Se puede optar por, como hemos comentado antes, convertirlas
a variables numéricas (ordinales) sin binarizar.
Esto se haría si no se ha ejecutado pd.get_dummies() previamente. Además, no funciona is hay valores
perdidos notados como NaN.
'''
if not binarizar:
    for i in range(X.shape[1]):
        if isinstance(X[0,i],str):
            X[:,i] = le.fit_transform(X[:,i])

'''
Validación cruzada con particionado estratificado y control de la aleatoriedad (fijando la semilla).
'''
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.metrics import roc_auc_score
from imblearn.metrics import geometric_mean_score
from sklearn import preprocessing
import numpy

skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=123456)
le = preprocessing.LabelEncoder()

def validacion_cruzada(modelo, X, y, cv):
    y_test_all = []
    y_prob_all = []

    for train, test in cv.split(X, y):
        modelo = modelo.fit(X[train],y[train])
        y_pred = modelo.predict(X[test])
        y_prob = modelo.predict_proba(X[test])[:,1] #la segunda columna es la clase positiva '1' en bank-marketing
        y_test_bin = y[test]
        #y_test_bin = le.fit_transform(y[test]) #se convierte a binario para AUC: 'yes' -> 1 (clase positiva) y 'no' -> 0 en bank-marketing

        print("Accuracy: {:6.2f}%, F1-score: {:.4f}, G-mean: {:.4f}, AUC: {:.4f}".format(accuracy_score(y[test],y_pred)*100 , f1_score(y[test],y_pred,average='macro'), geometric_mean_score(y[test],y_pred,average='macro'), roc_auc_score(y_test_bin,y_prob)))
        y_test_all = numpy.concatenate([y_test_all,y_test_bin])
        y_prob_all = numpy.concatenate([y_prob_all,y_prob])

    print("")

    return modelo, y_test_all, y_prob_all

#------------------------------------------------------------------------------

import xgboost as xgb

#print("------ XGB...")
#clf = xgb.XGBClassifier(n_estimators = 200)
#clf, y_test_clf, y_prob_clf = validacion_cruzada(clf,X,y,skf)

'''
Light Gradient Boosting
'''
import lightgbm as lgb

print("------ LightGBM...")
lgbm = lgb.LGBMClassifier(objective='multiclass',n_estimators=200,num_threads=4)
lgbm, y_test_lgbm, y_prob_lgbm = validacion_cruzada(lgbm,X,y,skf)

print("------ Importancia de las características...")
importances = list(zip(lgbm.feature_importances_, bank.columns))
importances.sort()
pd.DataFrame(importances, index=[x for (_,x) in importances]).plot(kind='barh',legend=False)

'''
Selección de características (Feature Selection)
Probablemnte tengáis que instalar el paquete boruta: pip install boruta
Realizamos una selección de características usando Random Forest como estimador.
Configuramos Random Forest
'''
print("------ Selección de características...")
from boruta import BorutaPy
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_jobs=-1, class_weight='balanced', max_depth=5)

'''
Configuramos BorutaPy para la selección de características en función de la configuración hecha
para Random Forest
'''
feat_selector = BorutaPy(rf, max_iter=9, n_estimators=200, verbose=0, random_state=123456)

#Lo aplicamos sobre nuestros datos.
feat_selector.fit(X, y)

#Comprobar las características (variables) seleccionadas.
all_features = numpy.array(list(bank)[0:-1])
selected_features = all_features[feat_selector.support_]

print("\nCaracterísticas seleccionadas:")
print(selected_features)

#Comprobar el ranking de características.
print("\nRanking de características:")
print(feat_selector.ranking_)

#Aplicamos transform() a X para filtrar las características y dejar solo las seleccionadas.
X_filtered = feat_selector.transform(X)
print("\nNúmero de características inicial: {:d}, después de la selección: {:d}\n".format(X.shape[1],X_filtered.shape[1]))

'''
Ejecutamos LGBM sobre el conjunto de datos resultante de la selección de características con validación cruzada estratificada (las mismas particiones que para los demás algoritmos).
'''
print("------ LightGBM sobre las características seleccionadas...")
lgbm, y_test_lgbm, y_prob_lgbm = validacion_cruzada(lgbm,X_filtered,y,skf)

from sklearn.metrics import make_scorer
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV

params_xgb = {'min_child_weight':[4,5], 'gamma':[i/10.0 for i in range(3,6)],  'subsample':[i/10.0 for i in range(6,11)],
          'colsample_bytree':[i/10.0 for i in range(6,11)], 'max_depth': [2,3,4], 'n_estimators':[50,100,200]}

print("------ Grid Search...")
params_lgbm = {'feature_fraction':[i/10.0 for i in range(3,6)], 'learning_rate':[0.05,0.1],
               'num_leaves':[30,50], 'n_estimators':[200]}
grid = GridSearchCV(lgbm, params_lgbm, cv=3, n_jobs=1, verbose=1, scoring=make_scorer(f1_score))
grid.fit(X,y_bin)
print("Mejores parámetros:")
print(grid.best_params_)
print("\n------ LightGBM con los mejores parámetros de GridSearch...")
gs, y_test_gs, y_prob_gs = validacion_cruzada(grid.best_estimator_,X,y,skf)

print("------ Random Search...")
params_rnd_lgbm = {'feature_fraction':[i/10.0 for i in range(2,7)], 'learning_rate':[i/100.0 for i in range(5,20)],
               'num_leaves':[i*10 for i in range(2,6)], 'n_estimators':[200]}
rndsrch = RandomizedSearchCV(lgbm, params_rnd_lgbm, cv=3, n_iter=5, n_jobs=1, verbose=1, scoring=make_scorer(f1_score))
rndsrch.fit(X,y_bin)
print("Mejores parámetros:")
print(rndsrch.best_params_)
print("\n------ LightGBM con los mejores parámetros de RandomSearch...")
gs, y_test_gs, y_prob_gs = validacion_cruzada(rndsrch.best_estimator_,X,y,skf)

'''
Referencias complementarias

Varios enlaces que pueden ser de utilidad:

    Imputación de valores perdidos: https://pypi.python.org/pypi/fancyimpute
    Desbalanceo de clase: https://github.com/scikit-learn-contrib/imbalanced-learn
    Selección de características con Boruta: https://github.com/scikit-learn-contrib/boruta_py

En Windows para algunos paquetes (por ejemplo, fancyimpute) puede ser necesario instalar previamente esto: http://landinghub.visualstudio.com/visual-cpp-build-tools

En general, para instalación de paquetes: pip install <paquete>
'''


import pytest

from .power import FuelGrid, Point

@pytest.fixture
def small_grid_fixture():
    # Example from https://en.wikipedia.org/wiki/Summed-area_table
    fuel_grid = FuelGrid(0)
    fuel_grid.MAX_INDEX = 6
    fuel_grid._power_grid = [
        [None] * 7,
        [None, 31, 2, 4, 33, 5, 36],
        [None, 12, 26, 9, 10, 29, 25],
        [None, 13, 17, 21, 22, 20, 18],
        [None, 24, 23, 15, 16, 14, 19],
        [None, 30, 8, 28, 27, 11, 7],
        [None, 1, 35, 34, 3, 32, 6],
    ]
    fuel_grid._build_summed_area_table()
    yield fuel_grid

class TestFuelGrid:
    def test_part1(self):
        serial_number = 8561
        size = 3
        fuel_grid = FuelGrid(serial_number)
        point, power = fuel_grid.find_highest_power_square(size)
        print(f'\n***** Day 11 Part 1 Answer: {point}')

    @pytest.mark.parametrize(
        'serial_number,size,expected', [
            (18, 3, (Point(33, 45), 29)),
            (42, 3, (Point(21, 61), 30)),
            (18, 16, (Point(90, 269), 113)),
            (42, 12, (Point(232, 251), 119)),
        ]
    )
    def test_find_highest_power_square(self, serial_number, size, expected):
        fuel_grid = FuelGrid(serial_number)
        point, power = fuel_grid.find_highest_power_square(size)
        assert point == expected[0]
        assert power == expected[1]

    def test__build_summed_area_table(self, small_grid_fixture):
        expected = [
            [0] * 7,
            [0, 31, 33, 37, 70, 75, 111],
            [0, 43, 71, 84, 127, 161, 222],
            [0, 56, 101, 135, 200, 254, 333],
            [0, 80, 148, 197, 278, 346, 444],
            [0, 110, 186, 263, 371, 450, 555],
            [0, 111, 222, 333, 444, 555, 666],
        ]
        assert small_grid_fixture._summed_area_table == expected

        expected = (
            small_grid_fixture._summed_area_table[3][2]
            + small_grid_fixture._summed_area_table[5][5]
            - small_grid_fixture._summed_area_table[5][2]
            - small_grid_fixture._summed_area_table[3][5]
        )
        assert expected == 111

    def test__calculate_square_power(self, small_grid_fixture):
        point = Point(1, 1)
        assert small_grid_fixture._calculate_square_power(point, 5) == 450

    @pytest.mark.parametrize(
        'serial_number,x_index,y_index,expected', [
            (8, 3, 5, 4),
            (57, 122, 79, -5),
            (39, 217, 196, 0),
            (71, 101, 153, 4),
        ]
    )
    def test_calculate_cell_power(self, serial_number, x_index, y_index, expected):
        fuel_grid = FuelGrid(serial_number)
        point = Point(x_index, y_index)
        assert fuel_grid.calculate_cell_power(point) == expected

    @pytest.mark.parametrize(
        'number,expected', [
            (12345, 3),
            (-987654321, 3),
            (1, 0),
            (10, 0),
            (99, 0),
            (100, 1),
            (-100, 1),
        ]
    )
    def test__hundreds_digit(self, number, expected):
        assert FuelGrid._hundreds_digit(number) == expected


import numpy as np
import matplotlib.pyplot as plt

#EQUATION D'EVOLUTION -> avec x qui est un tableau à deux dimensions : x1 , x2 et u est l'entrée
def f(x,u):
    m=1.0
    g=9.81
    l=2.0
    mu=0.1

    Xdot =   [x[1], -g*np.sin( x[0] )/l - mu*x[1]/(m*l**2) + u/(m*l**2) ]
    return np.array(Xdot)

#METHODE D'EULER
def methode_euler(t,X0, h):
    #CONDITION INITIALES
    u=0
    X=np.zeros((len(X0), len(t)),'float')
    X[:,0]=X0

    for i in range( 0, len(t)-1):
        X[:, i+1] = X[:, i] + h*f(X[:, i], u)

    return X

#METHODE RK2 - Runge Kutta
def methode_RK(t,X0, h):
    #CONDITION INITIALES
    u=0
    X=np.zeros((len(X0), len(t)),'float')
    X[:,0]=X0

    for i in range( 0, len(t)-1):
        X[:, i+1] = X[:, i] + h*f(X[:, i] + h/2*f( X[ :, i], u), u)

    return X

def creationgraphique(X, Y, graphiquenumero, titre, labelX, labelY):
    plt.figure(graphiquenumero)
    plt.plot(X, Y)
    plt.title(titre)
    plt.xlabel(labelX)
    plt.ylabel(labelY)
    plt.legend()
    plt.show()

#MAIN
if __name__ == "__main__":
    print("Test Modele")

    X0=np.array([ 1.0 , 0]) # Valeur initiale de X0

    #PARAMETRE DE SIMULATION
    h = 0.05
    duree = 20.0
    t=np.arange( 0.0, duree, h)
    print("taille: ", len(t),"duree: ", t[-1])

    #METHODE EULER
    X=methode_euler(t, X0, h)
    creationgraphique(t, X[1], 1, "Vitesse", "Temps", "Vitesse")
    creationgraphique(t, X[0], 2, "Position", "Temps", "Position")
    creationgraphique(X[0], X[1], 3, "Phase", "Distance", "Vitesse")

    #METHODE RUNGE KUTTA
    X=methode_RK(t, X0, h)
    creationgraphique(t, X[1], 1, "Vitesse", "Temps", "Vitesse")
    creationgraphique(t, X[0], 2, "Position", "Temps", "Position")
    creationgraphique(X[0], X[1], 3, "Phase", "Distance", "Vitesse")

    #RESULTATS
print(t,",",X0)# -*- coding: utf-8 -*-


from pyspark import RDD, SparkConf, SparkContext
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession
from pyspark.sql.types import *
from utils import *

import time
import os, argparse
import numpy as np

memory = '32g'
pyspark_submit_args = ' --driver-memory ' + memory + ' pyspark-shell'
os.environ["PYSPARK_SUBMIT_ARGS"] = pyspark_submit_args

parser = argparse.ArgumentParser(description='argparse')
parser.add_argument('--database', '-d', help='database name', required=True)
parser.add_argument('--partition', '-p', type=int, help='num of workers', required=True)
parser.add_argument('--interval', '-i', help='interval', required=True)
args = parser.parse_args()

def main():
    database = args.database
    support = [11,21,31,41,51]
    partition = args.partition
    interval = args.interval

    conf = SparkConf().setAppName("")

    for s in support:
        for t in range(7):
            sc = SparkContext.getOrCreate(conf=conf)

            transDataRaw = scanDB("../incdatasets/{}.txt".format(database), " ")
            numTrans = len(transDataRaw)
            minsup = (s/100) * numTrans

            incDir = "../incdatasets/interval_{0}_{1}".format(database,interval)
            incNames = os.listdir(incDir)

            freqRange = sc.parallelize(range(0, partition))
            freqRange, res = distFreno(os.path.join(incDir,"db_0.txt"), minsup, sc, partition, freqRange)

            for incName in incNames[1:]:
                freqRange, res = incFreno(os.path.join(incDir,incName), minsup, sc, partition, freqRange, res)
                print(res)
            sc.stop()

    return

if __name__=="__main__":
    main()


#! /usr/local/bin/python3
import sys, os

try:
    import pyperclip
except:
    print("please use `pip3 install pyperclip` to install the pyperclip module dependency")
    sys.exit(1)

try:
    file_path = sys.argv[1]
except:
    print('use abspath.py THE_FILE_YOU_WANTO_HANDLE to get abspath')
cwd = os.getcwd()
abs_path = os.path.join(cwd, file_path)
pyperclip.copy(abs_path)
print(abs_path)
print('the abspath has copied to the clipboard')


from os import listdir as LD
from PIL import Image, ImageOps

for i in LD():
 try:
  if not(i.endswith('.png')):
   continue
  image = Image.open(i)
  if image.mode == 'RGBA':
   r,g,b,a = image.split()
   Image.merge('RGBA',list(ImageOps.invert(Image.merge('RGB', (r,g,b))).split())+[a]).save(i)
  else:
   ImageOps.invert(image).save(i)
  print(i)
 except:
  print('ERR: '+i)


import pytest
import sdk_install
import sdk_jobs
import sdk_networks
import sdk_plan
import shakedown
from tests import config

@pytest.fixture(scope='module', autouse=True)
def configure_package(configure_security):
    test_jobs = []
    try:
        test_jobs = config.get_all_jobs()
        # destroy/reinstall any prior leftover jobs, so that they don't touch the newly installed service:
        for job in test_jobs:
            sdk_jobs.install_job(job)

        sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
        sdk_install.install(
            config.PACKAGE_NAME,
            config.SERVICE_NAME,
            config.DEFAULT_TASK_COUNT,
            additional_options=sdk_networks.ENABLE_VIRTUAL_NETWORKS_OPTIONS)

        yield # let the test session execute
    finally:
        return

@pytest.mark.sanity
@pytest.mark.smoke
@pytest.mark.overlay
@pytest.mark.dcos_min_version('1.9')
def test_service_overlay_health():
    shakedown.service_healthy(config.SERVICE_NAME)
    node_tasks = (
        "node-0-server",
        "node-1-server",
        "node-2-server",
    )
    for task in node_tasks:
        sdk_networks.check_task_network(task)

@pytest.mark.sanity
@pytest.mark.smoke
@pytest.mark.overlay
@pytest.mark.dcos_min_version('1.9')
def test_functionality():
    parameters = {'CASSANDRA_KEYSPACE': 'testspace1'}

    # populate 'testspace1' for test, then delete afterwards:
    with sdk_jobs.RunJobContext(
            before_jobs=[
                config.get_write_data_job(),
                config.get_verify_data_job()
            ],
            after_jobs=[]
            ):

        sdk_plan.start_plan(config.SERVICE_NAME, 'cleanup', parameters=parameters)
        sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'cleanup')

        sdk_plan.start_plan(config.SERVICE_NAME, 'repair', parameters=parameters)
        sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'repair')

@pytest.mark.sanity
@pytest.mark.overlay
@pytest.mark.dcos_min_version('1.9')
def test_endpoints():
    # tests that the correct number of endpoints are found, should just be "native-client":
    endpoints = sdk_networks.get_and_test_endpoints(config.PACKAGE_NAME, config.SERVICE_NAME, "", 1)
    assert "native-client" in endpoints, "Cassandra endpoints should contain only 'native-client', got {}".format(endpoints)
    endpoints = sdk_networks.get_and_test_endpoints(config.PACKAGE_NAME, config.SERVICE_NAME, "native-client", 2)
    sdk_networks.check_endpoints_on_overlay(endpoints)

@pytest.mark.sanity
@pytest.mark.overlay
def test_overlay_uninstall_pkg():
    sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)


__author__ = "Philip Zimmermann"
__email__ = "philip.zimmermann@tum.de"
__version__ = "1.0"

from typing import List

from facade_selection.utils import FacObjTypes, Point

try:
    from PySide import QtGui, QtCore
    from PySide.QtGui import QWidget, QApplication, QGridLayout, QPushButton, QPixmap, QComboBox, QLabel, QCheckBox, \
        QSizePolicy
except ModuleNotFoundError:
    from PyQt5 import QtGui, QtCore
    from PyQt5.QtWidgets import QWidget, QApplication, QGridLayout, QPushButton, QComboBox, QLabel, QCheckBox, \
        QSizePolicy
    from PyQt5.QtGui import QPixmap

import facade_selection.my_qt_label
import facade_selection.grid_dialog
from facade_selection.facade_object import FacadeObject

class FacadeGui(QWidget):
    def __init__(self, owner, img_path: str):
        super().__init__()
        self.owner = owner

        self.facade_objects: List[FacadeObject] = []
        self.deleted_objects: List[FacadeObject] = []
        self.mode: FacObjTypes = FacObjTypes.WINDOW
        self.grid_mode: bool = False
        self.grid_placed: int = 0

        main_layout = QGridLayout()

        self.mode_combo_box = QComboBox()
        self.mode_combo_box.addItems(['Window', 'Balcony'])
        self.mode_combo_box.currentTextChanged.connect(self.mode_change_call)

        self.grid_box = QCheckBox('Grid mode')
        self.grid_box.clicked.connect(self.grid_call)

        self.standard_text = 'Left-click and move mouse to mark facade objects. Press \'Finish\' when ready to continue.'
        self.grid_mode_text = 'Grid Mode: Mark only the top-left and bottom-right facade objects in the grid.'
        self.status_label = QLabel(self.standard_text)

        undo_button = QPushButton('Undo')
        undo_button.clicked.connect(self.undo_call)
        redo_button = QPushButton('Redo')
        redo_button.clicked.connect(self.redo_call)
        finish_button = QPushButton('Finish')
        finish_button.clicked.connect(self.finish_call)

        q_image = QtGui.QImage(img_path)  # .scaled(1200, 800,aspectRatioMode=QtCore.Qt.KeepAspectRatio)
        self.image_label = facade_selection.my_qt_label.MyQtLabel(self)
        self.image_label.setPixmap(QPixmap.fromImage(q_image))
        self.image_label.setScaledContents(True)
        #TODO fix image size
        self.image_label.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored)

        main_layout.addWidget(self.status_label, 0, 0, 1, -1)
        main_layout.addWidget(self.grid_box, 1, 0)
        main_layout.addWidget(self.mode_combo_box, 2, 0)
        main_layout.addWidget(undo_button, 2, 1)
        main_layout.addWidget(redo_button, 2, 2)
        main_layout.addWidget(finish_button, 2, 3)
        main_layout.addWidget(self.image_label, 3, 0, 1, -1)

        self.setLayout(main_layout)
        self.setFixedSize(self.width(), self.height())
        self.show()

    def add_rectangle(self, point1, point2):
        self.facade_objects.append(FacadeObject(point1, point2, self.mode))
        if self.grid_mode:
            self.grid_placed += 1
            if self.grid_placed == 2:
                self.open_grid_menu()

    def open_grid_menu(self):
        assert len(self.facade_objects) >= 2
        grid_dialog = facade_selection.grid_dialog.GridDialog(self)
        grid_dialog.exec()

    def autocomplete_grid(self, rows, columns):
        fac_obj_1, fac_obj_2 = self.facade_objects[-2:]
        assert fac_obj_1.p1.x < fac_obj_2.p2.x
        avg_width = (fac_obj_1.width() + fac_obj_2.width()) / 2
        avg_height = (fac_obj_1.height() + fac_obj_2.height()) / 2
        fac_obj_1.set_width(avg_width)
        fac_obj_1.set_height(avg_height)
        fac_obj_2.set_width(avg_width)
        fac_obj_2.set_height(avg_height)
        # calculate intermediate spaces size
        x_spaces = (abs(fac_obj_2.p2.x - fac_obj_1.p1.x) - columns * avg_width) / (columns - 1) if columns > 1 else 0
        y_spaces = (abs(fac_obj_2.p2.y - fac_obj_1.p1.y) - rows * avg_height) / (rows - 1) if rows > 1 else 0
        self.facade_objects.pop()
        self.facade_objects.pop()
        for row in range(rows):
            for column in range(columns):
                p1 = Point(fac_obj_1.p1.x + column * (avg_width + x_spaces),
                           fac_obj_1.p1.y + row * (avg_height + y_spaces))
                p2 = Point(p1.x + avg_width, p1.y + avg_height)
                self.facade_objects.append(FacadeObject(p1, p2, self.mode))
        self.grid_placed = 0
        self.image_label.update()

    def reject_grid_call(self):
        self.facade_objects.pop()
        self.facade_objects.pop()
        self.grid_placed = 0
        self.image_label.update()

    def mode_change_call(self, new_mode: str):
        if new_mode == 'Window':
            self.mode = FacObjTypes.WINDOW
        elif new_mode == 'Balcony':
            self.mode = FacObjTypes.BALCONY

    def grid_call(self):
        self.grid_mode = not self.grid_mode
        self.grid_placed = 0
        self.status_label.setText(self.grid_mode_text if self.grid_mode else self.standard_text)

    def finish_call(self):
        self.close()
        self.owner.insertIntoCAD(self.facade_objects)

    def undo_call(self):
        if len(self.facade_objects) > 0:
            self.deleted_objects.append(self.facade_objects.pop())
            self.grid_placed = max(self.grid_placed - 1, 0)
            self.image_label.update()

    def redo_call(self):
        if len(self.deleted_objects) > 0:
            self.facade_objects.append(self.deleted_objects.pop())
            self.grid_placed += 1
            self.image_label.update()

if __name__ == '__main__':
    app = QApplication([])
    FacadeGui(None, "/home/philip/workspace/baurobotik/flatten_facades/dev/Baurobotik/tum_transformed.png")
    app.exec()


import os
from os import walk
import random
import pymongo
from pymongo import MongoClient
import base64
import bson
from bson.binary import Binary
import hashlib
from time import perf_counter

try:
    # establish a connection to the database
    connection = MongoClient('localhost', 27017)
    print("Successfully connected!")
except:
    print("Cannot connect to mongodb!")

# get a handle to the test database
db = connection.test_picdb

def test_find_all():
    count = 10
    coll = db.images

    start = perf_counter()

    for _ in range(count):

        result = coll.find({}, {"_id": 0, "img_id": 1})

        result = list(result)

    stop = perf_counter()

    print(f'Time: {(stop - start) / count}')

def test_find_random():
    count = 10
    size = 280000
    coll = db.images

    seq = [random.randint(0, size) for _ in range(1000)]

    start = perf_counter()

    for _ in range(count):

        result = coll.find({"img_id": {"$in": seq}}, {
                           "_id": 0, "content": 1, "img_id": 1})

        # result = list(map(lambda x: x["img_id"], result))
        result = list(result)

    stop = perf_counter()

    print(f'Time: {(stop - start) / count}')

def test_find_all_images():
    count = 10
    coll = db.images

    start = perf_counter()

    for _ in range(count):

        result = coll.find({"uploader": "Eric", "img_type": "jpg", "use_count": 0, "tags": ["image"]}, {
                           "_id": 0, "img_id": 1})

        # result = list(map(lambda x: x["img_id"], result))

    stop = perf_counter()

    print(f'Time: {(stop - start) / count}')

if __name__ == "__main__":
    # test_find_all()
    test_find_random()
    # test_find_all_images()


"""HYCO_bootstrap URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.conf import settings
from django.conf.urls.static import static

from pages.views import homepage_view, vehicle_sales_view, about_view, contact_view, rentals_view, storage_view, detailing_view

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', homepage_view, name='homepage'),
    path('vehicle-sales/', vehicle_sales_view, name='vehicle-sales'),
    path('about/', about_view, name='about'),
    path('contact/', contact_view, name='contact'),
    path('rentals/', rentals_view, name='rentals'),
    path('storage/', storage_view, name='storage'),
    path('detailing/', detailing_view, name='detailing'),
]

if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


# -*- coding: utf-8 -*-

from datetime import datetime
from odoo import models, fields, api
from odoo.exceptions import ValidationError

class BookSentReturn(models.Model):
    _name = "book.rent.return"
    _description = "借阅"
    _inherit = ['mail.thread', 'mail.activity.mixin']

    copy_id = fields.Many2one('training.book.copy', string="书籍")
    book_reference = fields.Char(related='copy_id.reference', string="编号")
    customer_id = fields.Many2one('training.customer', string="借阅人", track_visibility='always')
    rental_date = fields.Datetime(string="借阅时间", default=fields.Datetime.now, track_visibility='always')
    return_date = fields.Datetime(string="归还时间", track_visibility='always')
    continue_days = fields.Char(string='持续时间')
    state = fields.Selection([
        ('draft', '草稿'),
        ('confirm', '借出'),
        ('owe', '逾期'),
        ('complete', '完成'),
    ], string='状态', copy=False, default='draft')
    book_rented = fields.Boolean(string="借出", default=False)
    rent_book_id = fields.Many2one('training.book')

    # @api.depends('rental_date', 'return_date')  # 后端改变 self是新值
    # def _compute_time(self):
    #     for rental in self:
    #         if not (rental.rental_date and rental.return_date):
    #             return
    #         if rental.return_date < rental.rental_date:
    #             raise ValidationError('归还时间不能小于借阅时间')
    #         if rental.rental_date and rental.return_date:
    #             rental.continue_days = str(rental.return_date - rental.rental_date)

    def action_confirm(self):
        book_rent = self.env['training.book.copy'].search([('id', '=', self.copy_id.id)])
        if book_rent:
            book_rent.write({'book_rented': True})
            self.write({'state': 'confirm'})

    def action_draft(self):
        customer_owe = self.env['training.customer'].search([('id', '=', self.customer_id.id)])
        current = fields.Datetime.now()
        remain_days = (datetime.now() - self.rental_date).days
        # print("当前时间: ", current)

        if customer_owe:
            # if self.return_date < current:
            if remain_days > 30:
                # owe_money = (current - self.return_date).days
                owe_money = remain_days - 30
                customer_owe.write({'money': owe_money})
                self.write({'state': 'owe'})
            else:
                self.write({'state': 'complete'})
            self.write({'return_date': current})
            self.write({'continue_days': str(current - self.rental_date)})
            self.copy_id.write({'book_rented': False})
            # raise ValidationError("书籍已逾期") # 影响保存数据
        # self.unlink()

    # @api.onchange('copy_id')
    # def check_rent(self):
    #     rent_search = self.env['book.rent.return'].search([('copy_id', '=', self.copy_id.id)])
    #     # print(type(rent_search))
    #     if rent_search:
    #         raise ValidationError("书已借出")


import os
import sys
import json
import logging
import traceback

from ottoengine.model import rule_objects, trigger_objects, condition_objects, action_objects

_LOG = logging.getLogger(__name__)
# _LOG.setLevel(logging.DEBUG)

BACKEND_FILE = 'file'
BACKEND_MYSQL = 'mysql'
BACKEND_SQLITE = 'sqlite'

JSON_EXTENSION = 'json'

ATTR_PLATFORM = "platform"
ATTR_CONDITION = "condition"

def _log_exception(e, message):
    _LOG.error("Exception {}: {}".format(sys.exc_info()[0], sys.exc_info()[1]))
    _LOG.error("Exception Message: {}".format(message))

def _error_not_implemented(backend, operation):
    _LOG.error("{} persisted is not implemented for {}".format(backend, operation))

class PersistenceManager:

    def __init__(self, json_rules_dir):
        """
            :param engine.OttoEngine engine_obj:
            :param str json_rules_dir:
        """
        self._json_rules_dir = json_rules_dir

        if not os.path.exists(self._json_rules_dir):
            os.makedirs(self._json_rules_dir)

    # ~~~~~~~~~~~~~~
    # Public methods
    # ~~~~~~~~~~~~~~

    def get_rules(self, json_rules_dir: str, backend: str='file') -> list:
        """ Load the AutomationRules from JSON from the BACKEND
        Returns list of AutomationRules
        :rtype: list(rule_objects.AutomationRule)
        """
        rules = []

        if backend == BACKEND_FILE:
            for file in os.listdir(json_rules_dir):
                if file.endswith(JSON_EXTENSION):
                    filename = os.path.join(json_rules_dir, file)
                    _LOG.info("Found rule file: {}".format(filename))
                    rule = self.load_rule_from_file(filename)
                    if rule is None:
                        _LOG.error("Rule did not load properly: {}".format(filename))
                        continue
                    rules.append(rule)

        elif backend == BACKEND_MYSQL:
            _error_not_implemented(backend, "get_rule_ids")

        elif backend == BACKEND_SQLITE:
            _error_not_implemented(backend, "get_rule_ids")

        _LOG.info("Completed reading rules from peristence")
        return rules

    def load_rule(self, rule_id: str) -> rule_objects.AutomationRule:
        filename = self._build_filename(rule_id)
        return self.load_rule_from_file(filename)

    def save_rule(self, rule: rule_objects.AutomationRule):
        self._save_file_rule(rule)

    def delete_rule(self, rule_id: str) -> bool:
        return self._delete_file_rule(rule_id)

    def load_rule_from_file(self, filename: str) -> rule_objects.AutomationRule:
        """ Loads JSON from a file, and returns the JSON structure"""

        if not os.path.exists(filename):
            _LOG.error("Rule file does not exist: {}".format(filename))
            return None

        _LOG.info("Opening JSON rules file: {}".format(filename))

        try:
            json_rule = json.load(open(filename))
        except Exception as e:
            _LOG.error(
                "Error loading rule file: {}: {}".format(sys.exc_info()[0], sys.exc_info()[1]))
            return None

        result = self.rule_from_dict(json_rule)
        if not result.get("success"):
            _LOG.error("Error encountered parsing rule: {}".format(result.get("message")))
            return None

        return result.get("rule")

    def _save_file_rule(self, rule: rule_objects.AutomationRule):
        filename = self._build_filename(rule.id)
        _LOG.info("Saving rule with filename: {}".format(filename))
        with open(filename, 'w') as outfile:
            json.dump(rule.serialize(), outfile)

    def _delete_file_rule(self, rule_id: str) -> bool:
        '''Returns True if file existed, False if file did not exist'''
        filename = self._build_filename(rule_id)
        try:
            os.remove(filename)
        except FileNotFoundError:
            return False
        return True

    def _build_filename(self, rule_id: str) -> str:
        return os.path.join(self._json_rules_dir, "{}.{}".format(rule_id, JSON_EXTENSION))

    # ~~~~~~~~~~~~~~~~~~~~
    # Rule loading methods
    # ~~~~~~~~~~~~~~~~~~~~

    def rule_from_dict(self, rule_dict: dict) -> dict:
        _LOG.debug("Rule JSON:  {}".format(rule_dict))

        success = True
        rule = None
        message = None

        # Create AutomationRule
        try:
            rule = rule_objects.AutomationRule(
                id=rule_dict.get("id"),
                description=rule_dict.get("description", ''),
                enabled=rule_dict.get("enabled", True),
                group=rule_dict.get("group", ''),
                notes=rule_dict.get("notes", '')
            )
        except Exception as e:
            success = False
            message = "Error instantiating AutomationRule: {}".format(sys.exc_info()[1])
            traceback.print_exc()
            _LOG.error(message)

        # Create Triggers
        try:
            for j in rule_dict["triggers"]:
                rule.triggers.append(self._trigger_from_dict(j))
        except Exception as e:
            success = False
            message = "Error creating triggers: {}".format(sys.exc_info()[1])
            traceback.print_exc()
            _LOG.error(message)

        # Create Rule Condition
        try:
            rule.rule_condition = self._condition_from_dict(rule_dict.get("rule_condition"))
        except Exception as e:
            success = False
            message = "Error creating rule condition: {}".format(sys.exc_info()[1])
            traceback.print_exc()
            _LOG.error(message)

        # Create Actions
        try:
            for j in rule_dict["actions"]:
                raction = rule_objects.RuleAction()
                if "description" in j:
                    raction.description = j.get("description")
                if "action_condition" in j:
                    raction.action_condition = self._condition_from_dict(j["action_condition"])
                for j2 in j["action_sequence"]:
                    raction.action_sequence.append(self._action_from_dict(j2))
                rule.actions.append(raction)
        except Exception as e:
            success = False
            message = "Error creating actions: {}".format(sys.exc_info()[1])
            traceback.print_exc()
            _LOG.error(message)

        if success:
            return {"success": success, "rule": rule}
        else:
            return {"success": success, "message": message}

    def _trigger_from_dict(self, trigger_json: dict) -> trigger_objects.ListenerTrigger:
        trigger = None
        j = trigger_json

        if j is None:
            raise Exception("Trigger found with an empty definition (i.e. Null)")
        elif j.get(ATTR_PLATFORM) is None:
            raise Exception("Trigger defintion is missing its 'platform:' attribute")

        if "homeassistant" in j[ATTR_PLATFORM]:
            trigger = trigger_objects.HomeAssistantTrigger()
        if "state" in j[ATTR_PLATFORM]:
            trigger = trigger_objects.StateTrigger.from_dict(j)
        if "numeric_state" in j[ATTR_PLATFORM]:
            trigger = trigger_objects.NumericStateTrigger.from_dict(j)
        if "event" in j[ATTR_PLATFORM]:
            trigger = trigger_objects.EventTrigger.from_dict(j)
        if "time" in j[ATTR_PLATFORM]:
            trigger = trigger_objects.TimeTrigger.from_dict(j)
        return trigger

    def _condition_from_dict(self, condition_json: dict) -> condition_objects.RuleCondition:
        j = condition_json
        cond = None

        if j is None:
            return None
        elif j.get(ATTR_CONDITION) is None:
            raise Exception("Condition does not contain condition: attribute")

        if "and" in j[ATTR_CONDITION]:
            cond = condition_objects.AndCondition()
            for c in j["conditions"]:
                cond.add_condition(self._condition_from_dict(c))
        elif "or" in j[ATTR_CONDITION]:
            cond = condition_objects.OrCondition()
            for c in j["conditions"]:
                cond.add_condition(self._condition_from_dict(c))
        elif "numeric_state" in j[ATTR_CONDITION]:
            cond = condition_objects.NumericStateCondition.from_dict(j)
        elif "state" in j[ATTR_CONDITION]:
            cond = condition_objects.StateCondition.from_dict(j)
        elif "sun" in j[ATTR_CONDITION]:
            cond = condition_objects.SunCondition.from_dict(j)
        elif "template" in j[ATTR_CONDITION]:
            cond = condition_objects.TemplateCondition.from_dict(j)
        elif "time" in j[ATTR_CONDITION]:
            cond = condition_objects.TimeCondition.from_dict(j)
        elif "zone" in j[ATTR_CONDITION]:
            cond = condition_objects.ZoneCondition.from_dict(j)
        return cond

    def _action_from_dict(self, action_json: dict) -> action_objects.RuleActionItem:
        j = action_json
        action = None
        if "service" in j:
            action = action_objects.ServiceAction.from_dict(j)
        if "condition" in j:
            action = action_objects.ConditionAction(self._condition_from_dict(j))
        if "delay" in j:
            action = action_objects.DelayAction.from_dict(j)
        if "wait" in j:
            action = action_objects.WaitAction.from_dict(j)
        if "event" in j:
            action = action_objects.EventAction.from_dict(j)
        if "log_message" in j:
            action = action_objects.LogAction.from_dict(j)
        return action


from dynaconf import settings
from pymongo.errors import ConfigurationError
from pymongo.errors import ConnectionFailure
from pymongo.errors import InvalidURI

try:
    from ibutsu_server.mongo import mongo

    HAS_MONGO = True
except ImportError:
    HAS_MONGO = False

def get_health():
    """Get a health report

    :rtype: Health
    """
    return {"status": "OK", "message": "Service is running"}

def get_database_health():
    """Get a health report for the database

    :rtype: Health
    """
    response = ({"status": "Pending", "message": "Fetching service status"}, 200)
    # Try to connect to the database, and handle various responses
    try:
        if not HAS_MONGO:
            response = ({"status": "Error", "message": "Incomplete MongoDB configuration"}, 500)
        else:
            mongo.results.find({}, limit=1)
            response = ({"status": "OK", "message": "Service is running"}, 200)
    except ConnectionFailure:
        response = ({"status": "Error", "message": "Unable to connect to MongoDB"}, 500)
    except InvalidURI:
        response = ({"status": "Error", "message": "Incorrect MongoDB connection URI"}, 500)
    except ConfigurationError:
        response = ({"status": "Error", "message": "Incorrect MongoDB configuration"}, 500)
    except Exception as e:
        response = ({"status": "Error", "message": str(e)}, 500)
    return response

def get_health_info():
    """Get the information about this server

    :rtype: HealthInfo
    """
    return {
        "frontend": settings.get("FRONTEND_URL", "http://localhost:3000"),
        "backend": settings.get("BACKEND_URL", "http://localhost:8080"),
        "api_ui": settings.get("BACKEND_URL", "http://localhost:8080") + "/api/ui/",
    }


from plone.app.dexterity.behaviors.exclfromnav import IExcludeFromNavigation

from z3c.form.interfaces import IEditForm
from zope.interface import alsoProvides
from zope import schema
from plone.autoform.interfaces import IFormFieldProvider
from plone.autoform import directives as form
from plone.supermodel import model
from plone.app.dexterity import MessageFactory as _

class IExcludeFromNavigationForm(IExcludeFromNavigation):
    """Behavior interface to exclude items from navigation.
    """
    model.fieldset(
        'new',
        label=_(u"New Setting"),
        fields=['exclude_from_nav']
    )

    exclude_from_nav = schema.Bool(
        title=_(
            u'label_exclude_from_nav',
            default=u'Exclude from TEST navigation'

        ),
        description=_(
            u'help_exclude_from_nav',
            default=u'If selected, this item will not appear in the ' +
                    u'navigation tree'
        ),
        default=True,
        readonly=True,
    )

    form.omitted('exclude_from_nav')
    form.omitted(IEditForm, 'exclude_from_nav')

alsoProvides(IExcludeFromNavigationForm, IFormFieldProvider)