import torch
import numpy as np
import pickle
from utils.process import *

class StandardScaler():
    """
    Standard the input
    """
    def __init__(self, mean, std):
        self.mean = mean
        self.std = std
    def transform(self, data):
        return (data - self.mean.to(data.device)) / self.std.to(data.device)
    def inverse_transform(self, data, dim):
        return (data * self.std[...,dim].item()) + self.mean[...,dim].item()

class DataLoader(object):
    def __init__(self, xs, ys, batch_size, pad_with_last_sample=True):
        """
        :param xs:
        :param ys:
        :param batch_size:
        :param pad_with_last_sample: pad with the last sample to make number of samples divisible to batch_size.
        """
        self.batch_size = batch_size
        self.current_ind = 0
        if pad_with_last_sample:
            num_padding = (batch_size - (len(xs) % batch_size)) % batch_size
            x_padding = np.repeat(xs[-1:], num_padding, axis=0)
            y_padding = np.repeat(ys[-1:], num_padding, axis=0)
            xs = np.concatenate([xs, x_padding], axis=0)
            ys = np.concatenate([ys, y_padding], axis=0)

        self.size = len(xs)
        self.num_batch = int(self.size // self.batch_size)
        self.xs = xs
        self.ys = ys

    def shuffle(self):
        permutation = np.random.permutation(self.size)
        xs, ys = self.xs[permutation], self.ys[permutation]
        self.xs = xs
        self.ys = ys

    def get_iterator(self):
        self.current_ind = 0
        def _wrapper():
            while self.current_ind < self.num_batch:
                start_ind = self.batch_size * self.current_ind
                end_ind = min(self.size, self.batch_size * (self.current_ind + 1))
                x_i = self.xs[start_ind: end_ind, ...]
                y_i = self.ys[start_ind: end_ind, ...]
                yield (x_i, y_i)
                self.current_ind += 1
        return _wrapper()

class pemsD:
    def __init__(self, num_nodes, path, adjpath, idpath=None):
        self.num_nodes = num_nodes
        self.path = path
        self.adjpath = adjpath
        self.idpath = idpath

    def load(self):
        data = np.load(self.path)
        return data['data']

    def load_graph(self):
        node2id = dict()
        if self.idpath is not None:
            file = open(self.idpath)
            id = 0
            for li in file:
                li = li.strip()
                node2id[int(li)] = id
                id += 1

        file = open(self.adjpath)
        nodes = [i for i in range(self.num_nodes)]
        dist = [0 for i in range(self.num_nodes)]

        srclist = []
        tgtlist = []
        dislist = []
        adj = torch.zeros(self.num_nodes,self.num_nodes)

        for li in file:
            li = li.strip().split(',')
            try:
                li = [float(t) for t in li]
            except Exception:
                continue
            if self.idpath is not None:
                src = int(node2id[li[0]])
                tgt = int(node2id[li[1]])
            else:
                src = int(li[0])
                tgt = int(li[1])
            adj[src,tgt] = li[2]
            srclist.append(src)
            tgtlist.append(tgt)
            dislist.append(li[2])

        return adj, nodes+srclist+tgtlist, nodes+tgtlist+srclist, dist+dislist+[t+1e9 for t in dislist]

    def load_graph1(self):
        node2id = dict()
        if self.idpath is not None:
            file = open(self.idpath)
            id = 0
            for li in file:
                li = li.strip()
                node2id[int(li)] = id
                id += 1

        file = open(self.adjpath)
        nodes = [i for i in range(self.num_nodes)]
        dist = [0 for i in range(self.num_nodes)]

        adj = torch.zeros(self.num_nodes,self.num_nodes)

        for li in file:
            li = li.strip().split(',')
            try:
                li = [float(t) for t in li]
            except Exception:
                continue
            if self.idpath is not None:
                src = int(node2id[li[0]])
                tgt = int(node2id[li[1]])
            else:
                src = int(li[0])
                tgt = int(li[1])
            if src!=tgt:
                adj[src,tgt] = li[2]
                #adj[tgt,src] = li[2]

        srclist = []
        tgtlist = []
        dislist = []
        # adj = torch.matmul(adj,adj)
        for i in range(self.num_nodes):
            for j in range(self.num_nodes):
                if adj[i, j] > 1e-9 and i!=j:
                    srclist.append(i)
                    tgtlist.append(j)
                    dislist.append(adj[i, j].item())

        # for i in range(self.num_nodes):
        #     for j in range(self.num_nodes):
        #         if adj[i, j] > 1e-9 and i!=j:
        #             srclist.append(j)
        #             tgtlist.append(i)
        #             dislist.append(1e9+adj[i, j].item())
        # print('# edges', len(srclist))
        return adj, nodes+srclist, nodes+tgtlist, dist+dislist

    def prcoess(self, savepath):
        data = {}
        x = self.load()
        x = x.transpose()
        x = torch.Tensor(x)
        length = x.shape[2]
        trainx = []
        trainy = []
        valx = []
        valy = []
        testx = []
        testy = []

        x = x.unsqueeze(dim=0)

        for i in range(int(length*0.6)-24):
            tx = x[...,i:i+12]
            ty = x[...,i+12:i+24]
            trainx.append(tx)
            trainy.append(ty)
        for i in range(int(length*0.6),int(length*0.8)-24):
            tx = x[...,i:i+12]
            ty = x[...,i+12:i+24]
            valx.append(tx)
            valy.append(ty)
        for i in range(int(length*0.8), length-24):
            tx = x[...,i:i+12]
            ty = x[...,i+12:i+24]
            testx.append(tx)
            testy.append(ty)

        trainx = torch.cat(trainx,dim=0)
        trainx = trainx.transpose(3,1)
        trainy = torch.cat(trainy,dim=0)
        trainy = trainy.transpose(3,1)

        valx = torch.cat(valx,dim=0)
        valx = valx.transpose(3,1)
        valy = torch.cat(valy,dim=0)
        valy = valy.transpose(3,1)

        testx = torch.cat(testx,dim=0)
        testx = testx.transpose(3,1)
        testy = torch.cat(testy,dim=0)
        testy = testy.transpose(3,1)

        data['x_train'] = trainx
        data['y_train'] = trainy
        data['x_val'] = valx
        data['y_val'] = valy
        data['x_test'] = testx
        data['y_test'] = testy

        data['adj'], srclist, tgtlist, distlist = self.load_graph1()
        file = open(savepath, "wb")
        pickle.dump(data, file)

def load_data(batch_size, path, device=None, normalize=True):
    file = open(path, "rb")
    data = pickle.load(file)
    mean = data['x_train'].mean(axis=(0, 1, 2), keepdims=True)
    std = data['x_train'].std(axis=(0, 1, 2), keepdims=True)
    if normalize:
        scaler = StandardScaler(mean=mean, std=std)
        for category in ['train', 'val', 'test']:
            data['x_' + category] = scaler.transform(data['x_' + category])
            data['y_' + category] = scaler.transform(data['y_' + category])
    else:
        scaler = StandardScaler(mean=0, std=1)

    data['train_loader'] = DataLoader(data['x_train'], data['y_train'], batch_size)
    data['val_loader'] = DataLoader(data['x_val'], data['y_val'], batch_size)
    data['test_loader'] = DataLoader(data['x_test'], data['y_test'], batch_size)
    data['scaler'] = scaler
    return data


from hatch.template import File
from hatch.utils.fs import Path
from hatchling.__about__ import __version__
from hatchling.metadata.spec import DEFAULT_METADATA_VERSION

from ..new.feature_no_src_layout import get_files as get_template_files
from .utils import update_record_file_contents

def get_files(**kwargs):
    metadata_directory = kwargs.get('metadata_directory', '')
    package_root = kwargs.get('package_root', '')

    files = []
    for f in get_template_files(**kwargs):
        if str(f.path) == 'LICENSE.txt':
            files.append(File(Path(metadata_directory, 'licenses', f.path), f.contents))

    pth_file_name = f"_{kwargs['package_name']}.pth"
    loader_file_name = f"_editable_impl_{kwargs['package_name']}.py"
    files.append(File(Path(pth_file_name), f"import _editable_impl_{kwargs['package_name']}"))
    files.append(
        File(
            Path(loader_file_name),
            f"""\
from editables.redirector import RedirectingFinder as F
F.install()
F.map_module({kwargs['package_name']!r}, {package_root!r})""",
        )
    )
    files.append(
        File(
            Path(metadata_directory, 'WHEEL'),
            f"""\
Wheel-Version: 1.0
Generator: hatchling {__version__}
Root-Is-Purelib: true
Tag: py2-none-any
Tag: py3-none-any
""",
        )
    )
    files.append(
        File(
            Path(metadata_directory, 'METADATA'),
            f"""\
Metadata-Version: {DEFAULT_METADATA_VERSION}
Name: {kwargs['project_name']}
Version: 0.0.1
License-File: LICENSE.txt
Requires-Dist: editables~=0.3
""",
        )
    )

    record_file = File(Path(metadata_directory, 'RECORD'), '')
    update_record_file_contents(record_file, files, generated_files={pth_file_name, loader_file_name})
    files.append(record_file)

    return files


import datetime  # to add timestamps on every block in blockchain
import hashlib  # library that is ued to hash the block
import json  # to communicate in json data
# Flask to implement webservices jsonify to see the jsop message/response
# request help us to connect all the nodes of the blockchain together froming the p2p network
from flask import Flask, jsonify, request
# it will help us to verify that all the blockchain have same blockhain or not http requests (used in replace_cahin)
import requests
from uuid import uuid4
from urllib.parse import urlparse

# Building a Blockchain

class Blockchain:

    def __init__(self):
        self.chain = []  # our main block chain
        # now we will create the list of transation which will record the all transactions
        self.transactions = []
        # create_block used to create the block in blockchain so it is executed only when the block is mined(meaning it has winnnig proof_of_work=proof)    proof=0 and previous_hash='0' for the genesis block
        self.create_block(proof=0, previous_hash='0')
        # nodes will contains the unique identifier of the address of all nodes in p2p network
        self.nodes = set()  # we have taken set() instead of list because we know that address are randomly generated by uuid4 to avoid duplicacy in it
    # part1

    def create_block(self, proof, previous_hash):
        block = {  # dictionary of python data structure
            'index': len(self.chain)+1,
            'timestamp': str(datetime.datetime.now()),
            'proof': proof,  # works like a nounce of block stops when we reach at or below the target
            'previous_hash': previous_hash,
            'transactions': self.transactions}
        self.transactions = []  # this need to be done bcoz we cant have duplicates lists of transactions in the further blocks so empty the transation that had been added in the block
        self.chain.append(block)
        return block

    def get_previous_block(self):
        return self.chain[-1]

    def proof_of_work(self, previous_proof):
        new_proof = 1
        check_proof = False
        while check_proof is False:
            hash_operation = hashlib.sha256(
                str(new_proof**2-previous_proof**2).encode()).hexdigest()
            if hash_operation[:4] == '0000':
                check_proof = True
            else:
                new_proof += 1
        return new_proof  # it is just a no. corresponding to the game solved by person is having a hash with trailing 4 zeroe's

    # hash of a block is created after generating block thats we have only use previous_hash because its already created
    def hash(self, block):
        encoded_block = json.dumps(block, sort_keys=True).encode()
        return hashlib.sha256(encoded_block).hexdigest()

    def is_chain_valid(self, chain):
        # reference of first block stored genesis block
        previous_block = chain[0]
        block_index = 1  # required for iteration
        while block_index < len(chain):
            block = chain[block_index]  # cuurent block
            # checking weather the refernce stored in property previus_hash is currently matched or not with the hash of previous block using hash function
            if block['previous_hash'] != self.hash(previous_block):
                return False
            previous_proof = previous_block['proof']
            proof = block['proof']
            # verfying the proof of block with the data proof and previous proof it is easy then creating the proof
            hash_operation = hashlib.sha256(
                str(proof**2 - previous_proof**2).encode()).hexdigest()
            # the more is zero's the more is harder to mine the block
            if hash_operation[:4] != '0000':
                return False
            previous_block = block
            block_index += 1
        return True

    # functions used to get add the transactions to the lists
    def add_transaction(self, senders, receiver, amount):
        self.transactions.append({
            'senders': senders,
            'receiver': receiver,
            'amount': amount
        })
        previous_block = self.get_previous_block()
        # +1 beacause before mining the transaction are added so new_block index will be +1 then previous
        return previous_block['index']+1
    # part-1 ends

    # part-3--> dealing with decentarlized application and transactions

    # this function allow us to add different nodes to chain

    def add_node(self, address):  # generating the decentarlized application
        # we need to parse the url before adding it
        parsed_url = urlparse(address)
        # .netloc gives us the unique identifier of the node address removing the unrequired part from it
        self.nodes.add(parsed_url.netloc)

    # this function help us to solve the problem of consensus protocols (competing chain)

    def replace_chain(self):
        # this variable help us to find the length of longest chain among different network
        max_length = len(self.chain)
        longest_chain = None
        network = self.nodes  # this variable will hold the address of all the nodes in network
        for node in network:
            # we know the nodes array will hold only the netlock value in nodes so we are going to use taht and make a request to that node check its length
            # using the requests library we make a requests to that node address ([f'http://{node}/get_chain']  -->  [f'http://127.0.0.5000/get_chain')]
            response = requests.get(f'http://{node}/get_chain')
            if response.status_code == 200:  # this ids the vode chaeck something is received in request
                length = response.json()['length']
                chain = response.json()['chain']
                if length > max_length and self.is_chain_valid(chain):
                    max_length = length
                    longest_chain = chain
            # this will happen in every node of network
        if longest_chain:
            # if this chain is shorter than otherit will be updated
            self.chain = longest_chain
            return True
            # if this chain is only longest in network than return false and no update
        return False
    # part-3 ends
# Mining our Blockchain

app = Flask(__name__)

# Creating a Blockchain
# creating the instance of blockchain
blockchain = Blockchain()

# Mining the blockchain
# create an random and unique address for the node on port 5000
# this is the address used by to send the whale coin when the miner mines the wahle coin
node_address = str(uuid4()).replace('-', '')

# part-2

@app.route('/mine_block', methods=['GET'])
def mine_block():
    previous_block = blockchain.get_previous_block()
    previous_proof = previous_block['proof']
    proof = blockchain.proof_of_work(previous_proof)
    previous_hash = blockchain.hash(previous_block)
    # miners price
    # usually the reciever public address is created when  user generate the wallet and mining pool send the coin after mining the block to miner address present in the bat file which is edited after downloading the software
    blockchain.add_transaction(node_address, 'Bhavjot', 1)
    # when created blockchain is called all the transactions performed will be inserted inside the current created block and when appended in transactions it will be again change to [] empty to avoid the duplicacy
    block = blockchain.create_block(proof, previous_hash)
    response = {'message': 'Congratulations, you just mined a block! 😈😈😈😈😈🤓🤓🤓',  # response is a json data
                'index': block['index'],
                'timestamp': block['timestamp'],
                'proof': block['proof'],
                'previous_hash': block['previous_hash'],
                'transactions': block['transactions']}
    return jsonify(response), 200

# getting all blocks in chain

@app.route('/get_chain', methods=['GET'])
def get_chain():
    response = {
        'chain': blockchain.chain,
        'length': len(blockchain.chain)
    }
    return jsonify(response), 200

# custom message

@app.route('/', methods=['GET'])
def custom_message():
    response = {
        'message': 'Congratulations you are on Whalecoin 🐳🐳🐳🐳🐳🐳'
    }
    return jsonify(response), 200

# part-2 ends
# creating the transactions

@app.route('/add_transactions', methods=['POST'])
def add_transaction():
    # this will help us to extract te post request made in postman like req.params.name in express
    json = request.get_json()
    # this will hep us to check that all the parameters are present or not for adding the transactions
    transaction_keys = ['sender', 'receiver', 'amount']
    if not all(key in json for key in transaction_keys):
        return 'Some elements of the transaction are missing', 400
    index = blockchain.add_transaction(
        json['sender'], json['receiver'], json['amount'])
    # when the block is mined all the transations in lists is added to block
    response = {'message': f'This transaction will be added to Block {index}'}
    return jsonify(response), 201

@app.route('/connect_node', methods=['POST'])
def connect_node():
    json = request.get_json()  # we will get request message send from postman
    # {'nodes':['http://127.0.0.1:5000','http://127.0.0.1:5001','http://127.0.0.1:5003',...]} when adding nodes using add_nodes 127.0.0.1:5001 it will be extracted using netloc
    nodes = json.get('nodes')
    if nodes is None:
        return "No node", 400
    for node in nodes:
        blockchain.add_node(node)  # add our nodes to network
    response = {'message': 'All the nodes are now connected. The Whalecoin 🐳🐳🐳🐳🐳🐳 Blockchain now contains the following nodes:',
                'total_nodes': list(blockchain.nodes)}
    return jsonify(response), 201

# Replacing the chain by the longest chain if needed
# this function will present in every node of blockchain and always checked so that the node remain upadatesd with other blockchains by hitiing replace_chain URL
@ app.route('/replace_chain', methods=['GET'])
def replace_chain():
    # using the above defined function in class
    is_chain_replaced = blockchain.replace_chain()
    if is_chain_replaced:  # means the current blockchain was the shortest one and it is replaced
        response = {'message': 'The nodes had different chains so the chain was replaced by the longest one.',
                    'new_chain': blockchain.chain}
    else:  # means the current blockchain was not the shortest one and it is not replaced
        response = {'message': 'All good. The chain is the largest one.',
                    'actual_chain': blockchain.chain}
    return jsonify(response), 200

# Running the app
# host= '0.0.0.0' specifies that it is available publicily
app.run(host='0.0.0.0', port=5001)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2010 Radim Rehurek <radimrehurek@seznam.cz>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""
Blei's LDA-C format.
"""

from __future__ import with_statement

from os import path
import logging

from gensim import interfaces, utils
from gensim.corpora import IndexedCorpus
from six.moves import xrange

logger = logging.getLogger('gensim.corpora.bleicorpus')

class BleiCorpus(IndexedCorpus):
    """
    Corpus in Blei's LDA-C format.

    The corpus is represented as two files: one describing the documents, and another
    describing the mapping between words and their ids.

    Each document is one line::

      N fieldId1:fieldValue1 fieldId2:fieldValue2 ... fieldIdN:fieldValueN

    The vocabulary is a file with words, one word per line; word at line K has an
    implicit ``id=K``.
    """

    def __init__(self, fname, fname_vocab=None):
        """
        Initialize the corpus from a file.

        `fname_vocab` is the file with vocabulary; if not specified, it defaults to
        `fname.vocab`.
        """
        IndexedCorpus.__init__(self, fname)
        logger.info("loading corpus from %s" % fname)

        if fname_vocab is None:
            fname_base, _ = path.splitext(fname)
            fname_dir = path.dirname(fname)
            for fname_vocab in [
                        utils.smart_extension(fname, '.vocab'),
                        utils.smart_extension(fname, '/vocab.txt'),
                        utils.smart_extension(fname_base, '.vocab'),
                        utils.smart_extension(fname_dir, '/vocab.txt'),
                        ]:
                if path.exists(fname_vocab):
                    break
            else:
                raise IOError('BleiCorpus: could not find vocabulary file')

        self.fname = fname
        with utils.smart_open(fname_vocab) as fin:
            words = [utils.to_unicode(word).rstrip() for word in fin]
        self.id2word = dict(enumerate(words))

    def __iter__(self):
        """
        Iterate over the corpus, returning one sparse vector at a time.
        """
        lineno = -1
        with utils.smart_open(self.fname) as fin:
            for lineno, line in enumerate(fin):
                yield self.line2doc(line)
        self.length = lineno + 1

    def line2doc(self, line):
        parts = utils.to_unicode(line).split()
        if int(parts[0]) != len(parts) - 1:
            raise ValueError("invalid format in %s: %s" % (self.fname, repr(line)))
        doc = [part.rsplit(':', 1) for part in parts[1:]]
        doc = [(int(p1), float(p2)) for p1, p2 in doc]
        return doc

    @staticmethod
    def save_corpus(fname, corpus, id2word=None, metadata=False):
        """
        Save a corpus in the LDA-C format.

        There are actually two files saved: `fname` and `fname.vocab`, where
        `fname.vocab` is the vocabulary file.

        This function is automatically called by `BleiCorpus.serialize`; don't
        call it directly, call `serialize` instead.
        """
        if id2word is None:
            logger.info("no word id mapping provided; initializing from corpus")
            id2word = utils.dict_from_corpus(corpus)
            num_terms = len(id2word)
        else:
            num_terms = 1 + max([-1] + id2word.keys())

        logger.info("storing corpus in Blei's LDA-C format into %s" % fname)
        with utils.smart_open(fname, 'wb') as fout:
            offsets = []
            for doc in corpus:
                doc = list(doc)
                offsets.append(fout.tell())
                parts = ["%i:%g" % p for p in doc if abs(p[1]) > 1e-7]
                fout.write(utils.to_utf8("%i %s\n" % (len(doc), ' '.join(parts))))

        # write out vocabulary, in a format compatible with Blei's topics.py script
        fname_vocab = utils.smart_extension(fname, '.vocab')
        logger.info("saving vocabulary of %i words to %s" % (num_terms, fname_vocab))
        with utils.smart_open(fname_vocab, 'wb') as fout:
            for featureid in xrange(num_terms):
                fout.write(utils.to_utf8("%s\n" % id2word.get(featureid, '---')))

        return offsets

    def docbyoffset(self, offset):
        """
        Return the document stored at file position `offset`.
        """
        with utils.smart_open(self.fname) as f:
            f.seek(offset)
            return self.line2doc(f.readline())

# endclass BleiCorpus


import numpy as np
import MeshProcess
from scipy.optimize import minimize
from scipy import spatial
import time

def heat_kernel_signature(evecs, evals, A, numTimes, ifNormalize=True):
    '''
    Compute a set of Heat Kernel Signatures (HKS)
    :param evecs: the eigenfunctions of the Laplacian-beltrami operator (n-by-k)
    :param evals: the corresponding eigenvalues (k-by-1)
    :param A: the area matrix of the shape S (n-by-n)
    :param numTimes: the number of descriptors (to control the scale)
    :param ifNormalize: if we want to normalize the descriptors w.r.t the surface area
    :return: a set of descriptors (n-by-numTimes)
    '''

    D = np.matmul(evecs.transpose(), np.matmul(A.toarray(), np.power(evecs, 2)))
    abs_evals = np.abs(evals)
    emin = abs_evals[1]
    emax = abs_evals[-1]
    t = np.array(np.linspace(start=emin, stop=emax, num=numTimes), ndmin=2)
    T = np.exp(-np.abs(np.matmul(evals, t)))
    hks = np.matmul(evecs, np.matmul(D, T))
    if ifNormalize:
        hks = descriptors_normalization(hks, A)
    return hks

def wave_kernel_signature(evecs, evals, A, numTimes, ifNormalize=True):
    '''
    Compute a set of Wave Kernel Signatures (WKS)
    :param evecs: the eigenfunctions of the Laplacian-beltrami operator (n-by-k)
    :param evals: the corresponding eigenvalues (k-by-1)
    :param A: the area matrix of the shape S (n-by-n)
    :param numTimes: the number of descriptors (to control the scale)
    :param ifNormalize: if we want to normalize the descriptors w.r.t the surface area
    :return: a set of descriptors (n-by-numTimes)
    '''
    num_eigs = evecs.shape[1]
    D = np.matmul(evecs.transpose(), np.matmul(A.toarray(), np.power(evecs, 2)))
    abs_evals = np.abs(evals)
    emin = np.log(abs_evals[1])
    emax = np.log(abs_evals[-1])
    s = 7*(emax - emin) / numTimes
    emin = emin + 2*s
    emax = emax - 2*s

    t = np.array(np.linspace(start=emin, stop=emax, num=numTimes), ndmin=2)
    tmp1 = np.tile(np.log(abs_evals), (1, numTimes))
    tmp2 = np.tile(t, (num_eigs, 1))
    T = np.exp(-np.power((tmp1 - tmp2), 2) / (2 * s ** 2))
    wks = np.matmul(evecs, np.matmul(D, T))
    if ifNormalize:
        wks = descriptors_normalization(wks, A)
    return wks

def descriptors_normalization(desc, A):
    '''
    Normalize the descriptors w.r.t. the mesh area
    :param desc: a set of descriptors defined on mesh S
    :param A: the area matrix of the mesh S
    :return: the normalized desc with the same size
    '''
    tmp1 = np.sqrt(np.matmul(desc.transpose(), np.matmul(A.toarray(), desc)).diagonal())
    tmp2 = np.tile(tmp1, (desc.shape[0], 1))
    desc_normalized = np.divide(desc, tmp2)
    return desc_normalized

def descriptors_projection(desc, B, A):
    '''
    Project the given descriptors into the given basis (for fMap computation)
    :param desc: a set of descriptors defined on mesh S
    :param B: the basis of mesh S for the projection (should be consistent with the fMap)
    :param A: the area matrix of the mesh S
    :return: a #LB-by-#desc matrix
    '''
    desc_projected = np.matmul(B.transpose(), np.matmul(A.toarray(), desc))
    return desc_projected

def descriptor_commutativity_operator(desc, B, A):
    '''
    Operator construction for descriptor preservation via commutativity
    See paper "Informative Descriptor Preservation via Commutativity for Shape Matching"
    by Dorian Nogneng, and Maks Ovsjanikov
    :param desc: a set of NORMALIZED descriptors
    :param B: the basis where the fMap lives in
    :param A: the area matrix of the corresponding shape
    :return: a LIST of operators to be preserved by a fMap via commutativity,
            each operator is a #LB-by-#LB matrix, there are #desc operators in total
    '''
    num_desc = desc.shape[1]
    num_basis = B.shape[1]
    CommOp = []
    for i in range(num_desc):
        tmp = np.tile(desc[:,i], [num_basis,1]).transpose()
        op = np.matmul(B.transpose(), np.matmul(A.toarray(), np.multiply(tmp, B)))
        CommOp.append(op)
    return CommOp

def descriptor_orientation_operator(desc, B, S):
    '''
    Operator construction for orientation preserving/reversing via commutativity
    See paper "Continuous and orientation-preserving correspondence via functional maps"
    by Jing Ren, Adrien Poulenard, Peter Wonka, and Maks Ovsjanikov
    :param desc: a set of NORMALIZED descriptors
    :param B: the basis where the fMap lives in
    :param S: the mesh where the descriptors computed from
    :return: a LIST of operators encoded with the orientation information
    '''
    num_desc = desc.shape[1]
    OrientOp = []
    for i in range(num_desc):
        f = desc[:, i]
        op = MeshProcess.compute_orientation_operator_from_a_descriptor(S, B, f)
        OrientOp.append(op)
    return OrientOp

def regularizer_laplacian_commutativity(C, eval_src, eval_tar):
    '''
    fMap regularizer: Laplacian-Beltrami operator commutativity term
    :param C: functional map: source -> target (matrix k2-by-k1)
    :param eval_src: eigenvalues of the source shape (length of k1)
    :param eval_tar: eigenvalues of the target shape (length of k2)
    :return: the objective and the gradient of this regularizer
    '''
    if len(eval_tar) != C.shape[0] or len(eval_src) != C.shape[1]:
        return -1
    else:
        Mask = np.power(np.tile(eval_tar / np.sum(eval_tar) * np.sum(eval_src), (1, len(eval_src))) -
                        np.tile(eval_src.transpose() / np.sum(eval_src) * np.sum(eval_tar), (len(eval_tar), 1)), 2)
        Mask = np.divide(Mask, np.sum(np.power(Mask, 2)))
        fval = 0.5 * np.sum(np.multiply(np.multiply(C, C), Mask))
        grad = np.multiply(C, Mask)
        return fval, grad

def regularizer_descriptor_preservation(C, Desc_src, Desc_tar):
    '''
    fMap regularizer: preserve the given corresponding descriptors (projected)
    :param C: functional map : source -> target
    :param Desc_src: the projected descriptors of the source shape
    :param Desc_tar: the projected descriptors of the target shape
    :return: the objective and the gradient of this regularizer
    '''
    if Desc_src.shape[1] != Desc_tar.shape[1]:
        return -1
    else:
        fval = 0.5 * np.sum(np.power(np.matmul(C, Desc_src) - Desc_tar, 2))
        grad = np.matmul(np.matmul(C, Desc_src) - Desc_tar, Desc_src.transpose())
        return fval, grad

def regularizer_operator_commutativity(C, Op_src, Op_tar, IfReversing=False):
    '''
    fMap regularizer: preserve the operators via commutativity
    the operators can be:
    - descriptor commutativity operator
    - orientation-preserving operator
    :param C: functional map: source -> target
    :param Op_src: a list of operators on the source shape
    :param Op_tar: the corresponding operators on the target shape
    :param IfReversing: if reversing the commutativity sign - used in orientation-reversing setting
    :return: the objective and the gradient of this regularizer
    '''
    if len(Op_src) != len(Op_tar):
        return -1
    else:
        fval = 0
        grad = np.zeros(C.shape)
        for i in range(len(Op_src)):
            X = Op_tar[i]
            Y = Op_src[i]
            if not IfReversing:
                fval += 0.5 * np.sum(np.power(np.matmul(X, C) - np.matmul(C, Y), 2))
                grad += (np.matmul(X.transpose(), np.matmul(X, C) - np.matmul(C, Y)) -
                         np.matmul(np.matmul(X, C) - np.matmul(C, Y), Y.transpose()))
            else: # used for orientation-reversing setting only!
                fval += 0.5 * np.sum(np.power(np.matmul(X, C) + np.matmul(C, Y), 2))
                grad += (np.matmul(X.transpose(), np.matmul(X, C) + np.matmul(C, Y)) +
                         np.matmul(np.matmul(X, C) + np.matmul(C, Y), Y.transpose()))
        return fval, grad

def compute_functional_map_from_descriptors(S1, S2, desc1, desc2, param):
    '''
    Given a pair of shapes and a set of corresponding descriptors, compute a functional map
    :param S1: the source shape
    :param S2: the target shape
    :param desc1: a set of descriptors defined on S1
    :param desc2: the corresponding descriptors defined on S2
    :param param: the paramters for functional map optimization
    :return: a functional map from S1 to S2
    '''
    k1 = param['fMap_size'][0]
    k2 = param['fMap_size'][1]
    weights = np.array([param['weight_descriptor_preservation'],
                        param['weight_laplacian_commutativity'],
                        param['weight_descriptor_commutativity'],
                        param['weight_descriptor_orientation']])

    B1 = S1.evecs[:, 0:k1]
    B2 = S2.evecs[:, 0:k2]
    Ev1 = S1.evals[0:k1]
    Ev2 = S2.evals[0:k2]

    Desc1 = descriptors_projection(desc1, B1, S1.A)
    Desc2 = descriptors_projection(desc2, B2, S2.A)

    CommOp1 = descriptor_commutativity_operator(desc1, B1, S1.A)
    CommOp2 = descriptor_commutativity_operator(desc2, B2, S2.A)

    OrientOp1 = descriptor_orientation_operator(desc1, B1, S1)
    OrientOp2 = descriptor_orientation_operator(desc2, B2, S2)

    # Construct the energy function

    funDesc = lambda C: regularizer_descriptor_preservation(C, Desc1, Desc2)[0]
    gradDesc = lambda C: regularizer_descriptor_preservation(C, Desc1, Desc2)[1]

    funCommLB = lambda C: regularizer_laplacian_commutativity(C, Ev1, Ev2)[0]
    gradCommLB = lambda C: regularizer_laplacian_commutativity(C, Ev1, Ev2)[1]

    funCommDesc = lambda C: regularizer_operator_commutativity(C, CommOp1, CommOp2)[0]
    gradCommDesc = lambda C: regularizer_operator_commutativity(C, CommOp1, CommOp2)[1]

    funOrient = lambda C: regularizer_operator_commutativity(C, OrientOp1, OrientOp2)[0]
    gradOrient = lambda C: regularizer_operator_commutativity(C, OrientOp1, OrientOp2)[1]

    myObj = lambda C: (weights[0] * funDesc(C) + weights[1] * funCommLB(C) +
                       weights[2] * funCommDesc(C) + weights[3] * funOrient(C))
    myGrad = lambda C: (weights[0] * gradDesc(C) + weights[1] * gradCommLB(C) +
                       weights[2] * gradCommDesc(C) + weights[3] * gradOrient(C))

    # the first column of the functional map is determinant (defined by the first Eigenfunctions)
    constDesc = np.sign(B1[0, 0] * B2[0, 0]) * np.sqrt(np.sum(S2.A) / np.sum(S1.A))
    first_col_C = np.zeros((k2, 1))
    first_col_C[0] = constDesc

    # i.e., the 2nd to the k1-th columns are the true variables to optimize
    myObj_new = lambda Cvar: myObj(np.concatenate((first_col_C, Cvar), axis=1))
    myGrad_new = lambda Cvar: myGrad(np.concatenate((first_col_C, Cvar), axis=1))[:, 1:k1]

    # vectorize the variable and the corresponding gradient for the solver
    vec_myObj_new = lambda vec_Cvar: myObj_new(np.reshape(vec_Cvar, (k2, k1 - 1)))
    vec_myGrad_new = lambda vec_Cvar: np.reshape(myGrad_new(np.reshape(vec_Cvar, (k2, k1 - 1))), (k2 * (k1 - 1), 1))

    # optimization
    vec_Cvar = np.zeros((k2 * (k1 - 1), 1))
    t_opt = time.time()
    print("Starting Functional Map Optimization...")
    res = minimize(vec_myObj_new, vec_Cvar, method='L-BFGS-B',
                   jac=vec_myGrad_new,
                   options={'disp': False})
    print("Optimization done in : %.2fs" % (time.time()-t_opt))
    # reshape the minimum to a matrix
    Copt = np.reshape(res.x, (k2, k1 - 1))
    # add the fixed first column back to get the complete functional map
    Copt_full = np.concatenate((first_col_C, Copt), axis=1)
    return Copt_full

def convert_functional_map_to_pointwise_map(C12, B1, B2):
    '''
    Pointwise map reconstruction
    :param C12: given functional map C12: S1 -> S2
    :param B1: the basis of S1
    :param B2: the basis of S2
    :return: T21: the pointwise map T21: S2 -> S1 (index 0-based)
    '''
    if C12.shape[0] != B2.shape[1] or C12.shape[1] != B1.shape[1]:
        return -1
    else:
        _, T21 = spatial.cKDTree(np.matmul(B1, C12.transpose())).query(B2, n_jobs=-1)
        return T21

def convert_pointwise_map_to_funcitonal_map(T12, B1, B2):
    '''
    Convert a pointwise map to a functional map
    :param T12: given pointwise map T12: S1 -> S2
    :param B1: the basis of S1
    :param B2: the basis of S2
    :return: C21: the corresponding functional map C21: S2 -> S1
    '''
    C21 = np.linalg.lstsq(B1, B2[T12, :], rcond=None)[0]
    return C21

# TODO: add projection to C12
def refine_fMap_icp(C12, B1, B2, num_iters=10):
    '''
    Regular Iterative Closest Point (ICP) to refine a functional map
    :param C12: initial functional map from S1 to S2
    :param B1: the basis of S1
    :param B2: the basis of S2
    :param num_iters: the number of iterations for refinement
    :return: C12_refined, T21_refined
    '''
    if C12.shape[0] != B2.shape[1] or C12.shape[1] != B1.shape[1]:
        return -1
    else:
        C12_refined = C12
        for i in range(num_iters):
            T21_refined = convert_functional_map_to_pointwise_map(C12_refined, B1, B2)
            C12_refined = convert_pointwise_map_to_funcitonal_map(T21_refined, B2, B1)
        return C12_refined, T21_refined

# TODO: add projection to C21
def refine_pMap_icp(T12, B1, B2, num_iters=10):
    '''
    Regular Iterative Closest Point (ICP) to refine a pointwise map
    :param T12: initial pointwise map from S1 to S2
    :param B1: the basis of S1
    :param B2: the basis of S2
    :param num_iters: the number of iterations for refinement
    :return: T12_refined, C21_refined
    '''
    T12_refined = T12
    for i in range(num_iters):
        C21_refined = convert_pointwise_map_to_funcitonal_map(T12_refined, B1, B2)
        T12_refined = convert_functional_map_to_pointwise_map(C21_refined, B2, B1)
    return T12_refined, C21_refined


import json
import os

from django.core.urlresolvers import reverse
from django.db import connections
from django.test import TestCase
from django.test.client import RequestFactory
from django.test.utils import override_settings

from healthcheck.contrib.django.status_endpoint import views

class StatusEndpointViewsTestCase(TestCase):
    urls = 'healthcheck.contrib.django.status_endpoint.urls'

    def setUp(self):
        self.factory = RequestFactory()

    @override_settings(
        STATUS_CHECK_DBS=True,
        STATUS_CHECK_FILES=('/etc/quiesce',)
    )
    def test_default_checks(self):
        request = self.factory.get(reverse(views.status))
        response = views.status(request)
        self.assertEqual(response.status_code, 200)

    @override_settings(
        STATUS_CHECK_DBS=True,
        STATUS_CHECK_FILES=()
    )
    def test_dont_check_files(self):
        request = self.factory.get(reverse(views.status))
        response = views.status(request)
        response_json = json.loads(response.content.decode())
        self.assertTrue(
            "quiesce file doesn't exist" not in response_json)
        self.assertTrue(
            'Django Databases Health Check' in response_json)
        db_names = response_json['Django Databases Health Check']['details']
        self.assertTrue(
            all(connection.alias in db_names
                for connection in connections.all()))
        self.assertEqual(response.status_code, 200)

    @override_settings(
        STATUS_CHECK_DBS=False,
        STATUS_CHECK_FILES=()
    )
    def test_no_checks_raises_200(self):
        request = self.factory.get(reverse(views.status))
        response = views.status(request)
        response = {
            'content': json.loads(response.content.decode()),
            'status': response.status_code,
        }

        expected_response = {
            'content': 'There were no checks.',
            'status': 200,
        }

        self.assertEqual(response, expected_response)

    @override_settings(
        STATUS_CHECK_DBS=False,
        STATUS_CHECK_FILES=('/usr/bin/env',)
    )
    def test_failed_check(self):
        request = self.factory.get(reverse(views.status))
        response = views.status(request)
        response = {
            'content': json.loads(response.content.decode()),
            'status': response.status_code,
        }

        expected_response = {
            'content': {
                "quiesce file doesn't exist": {
                    'details': {
                        '/usr/bin/env': 'FILE EXISTS'
                    },
                    'status': 'FAILED'
                }
            },
            'status': 500,
        }

        self.assertEqual(response, expected_response)


import dash
import dash_core_components as dcc
import dash_html_components as html
import dill
import pandas as pd
import plotly.graph_objs as go
import plotly.offline as py

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

report = dill.load(open("temp2", "rb"))

#add styling for the plots

app.layout = html.Div([

        html.H1(children='Results',
                style={
                    'textAlign': 'center'
                }
                ),

        html.Div(children='''
        Results of the optimisation process

        The dataset that was given as an input has been processed and multiple clustering algorithms have been applied.

        The smaller the error of he performance measure the better.
        ''', style={
            'textAlign': 'center'
        }),

    html.Div([
        dcc.Graph(
                figure=report.ranks
        )],
        style={'float': 'right',
               "width" : 700}),

    html.Div([
        dcc.Dropdown(
        id = "dropdown",
        options = [
            {"label": "K-Means Clustering", "value": "KMeans"},
            {"label": "Birch  Clustering", "value": "Birch"},
            {"label": "Graph-based spectral Clustering", "value": "SpectralClustering"},
            {"label": "Graph-based affinity Clustering", "value": "AffinityPropagation"},
            {"label": "Distribution-based Mean Shift Clustering", "value": "MeanShift"},
            {"label": "Agglomerative  Clustering", "value": "AgglomerativeClustering"},
            {"label": "Density Based  Clustering", "value": "DBSCAN"},
        ],
        value = "KMeans"
    ), dcc.RadioItems(
        id = "radio-items",
        options= [
            {"label": "3D Exploratory Plot", "value": "plot3D"},
            {"label": "2D Exploratory Plot", "value": "plot2D"}
        ],
        value = "plot2D",
        labelStyle={'display': 'inline-block'}
    )], style={'float': 'middle', 'display': 'inline-block'}),

    html.Div([
        dcc.Graph(
        id='scatter_plt'
    )], style = {"float": "left",
                 "width": 1000})
], style={'float': 'middle', 'display': 'inline-block'})

@app.callback(
    dash.dependencies.Output("scatter_plt", 'figure'),
    [dash.dependencies.Input('dropdown', 'value'),
     dash.dependencies.Input("radio-items", "value")])

def update_output(value_drp, value_radio):
    return report.tracker[value_drp][value_radio]

if __name__ == '__main__':
    app.run_server(debug= True, host = "localhost", port = 8000)


import pandas as pd
import numpy as np
import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import pickle
from sklearn import preprocessing
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials

#read an plk data
k = pd.read_pickle('old_items.plk') #to load 123.pkl back to the dataframe df

df=k
df.columns = df.columns.str.replace(' ', '_')
df = k.drop(['info','broker','Name_brocker','Prisutveckling'], axis = 1)

'''
function to clean data from scraper.
    Args:
        df: Data from scraper
'''
def rename_cloums(df):
    df['streetName']=df['streetName'].astype(str)
    df['price']=df['price'].astype(str).astype(float)
    df['Begärt_pris']=df['Begärt_pris'].astype(str).astype(float)
    df['ppk']=df['Pris_per_kvadratmeter'].astype(str).astype(float)
    df['Antal_rum']=df['Antal_rum'].str.replace('rum','')
    df['Antal_rum']=df['Antal_rum'].str.replace(',','.')
    df['Boarea']=df['Boarea'].str.replace(',','.')
    df['date_sold']=df['date_sold'].str.replace(' ','')
    df['date_sold']=df['date_sold'].str.replace('januari','-jan-')
    df['date_sold']=df['date_sold'].str.replace('february','-feb-')
    df['date_sold']=df['date_sold'].str.replace('december','-dec-')
    df['date_sold']=df['date_sold'].str.replace('november','-nov-')
    df['date_sold']=df['date_sold'].str.replace('oktober','-oct-')
    df['date_sold'] = pd.to_datetime(df['date_sold'])
    df['Driftskostnad'].astype(str).astype(float)
    df['Byggår']=df['Byggår'].str.replace('-','')
    df['Byggår'].astype(str)
    df=df.drop(['Pris_per_kvadratmeter'], axis = 1)
    #df['Prisutveckling'] =  pd.to_numeric(df['Prisutveckling'], errors='coerce')
    df['Antal_rum'] = df['Antal_rum'].astype(float)
    df['Boarea'] = df['Boarea'].astype(float)
    df['avgift'] = pd.to_numeric(df['Avgift/månad'], errors='coerce')
    df['Driftskostnad'] = df['Driftskostnad'].astype(float)
    df['Byggår'] = df['Byggår'].astype(float)
    df['ppk'] = df['ppk'].astype(float)
    df['dayofweek'] = df['date_sold'].dt.dayofweek
    df['quarter'] = df['date_sold'].dt.quarter
    df['month'] = df['date_sold'].dt.month
    #df['year'] = df['date_sold'].dt.year
    df['dayofyear'] = df['date_sold'].dt.dayofyear
    #df['dayofmonth'] = df['date_sold'].dt.day
    lbl = preprocessing.LabelEncoder()
    df['encode_street'] = lbl.fit_transform(df['streetName'].fillna('').astype(str))
    df.loc[df['area'].str.contains('Såld den'), 'area'] = 'NA'
    df['area']=df['area'].str.replace(' ','')
    df['area']=df['area'].str.replace('-','/')
    df['encode_area'] = lbl.fit_transform(df['area'].fillna('').astype(str))
    return df
df=rename_cloums(df)

'''
Create traing test data. use inside optimze and traning functions
    Args:
        df: Data from scraper
        size: splitt betewwn train/test, ex 0.2
'''
def create_traning_test_data(df,size):
    # Splitt data in X and Y streetName
    X=df.filter(items=['encode_street','encode_area','Begärt_pris','encode_name','Antal_rum','Boarea','avgift',
                       'Driftskostnad','byggår','dayofweek','quater','month','dayofyear'])
    #target data.
    y=df.filter(items=['price'])
    data_dmatrix = xgb.DMatrix(data=X,label=y)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=size, random_state=1234567)
    return X_train, X_test, y_train, y_test

'''
Definde grid for optimations serche
'''
space={'max_depth': hp.quniform("max_depth", 3, 10, 1),
       'gamma': hp.uniform('gamma',0,3),
       'reg_alpha' : hp.quniform('reg_alpha', 0,90,1),
       'reg_lambda' : hp.uniform('reg_lambda', 0,40),
       'colsample_bytree' : hp.uniform('colsample_bytree', 0.1,1),
       'min_child_weight' : hp.quniform('min_child_weight', 0, 16, 1),
       'n_estimators': hp.quniform('n_estimators', 300, 3000, 1),
       'learning_rate': hp.loguniform('learning_rate', np.log(0.0001), np.log(1.0)),
       'subsample': hp.uniform('subsample', 0.5, 1.0),
       'early_stopping_rounds': hp.quniform('early_stopping_rounds', 250, 500, 1)}

#######################################
#pre_train hyper optimation for xgboost
#######################################

'''
Prepper xg-boost omtpimatxzion for apparment data
    Args:
        df:DataFarme
        space: grid of hypoeroppt parameters
'''
def hyperparameter_tuning(space):
    reg=xgb.XGBRegressor(max_depth = int(space['max_depth']),
                         gamma = space['gamma'],
                         reg_alpha = int(space['reg_alpha']),
                         reg_lambda = space['reg_lambda'],
                         colsample_bytree = space['colsample_bytree'],
                         min_child_weight = int(space['min_child_weight']),
                         n_estimators = int(space['n_estimators']),
                         learning_rate = space['learning_rate'],
                         subsample = space['subsample'],
                         objective = 'reg:squarederror')

    X_train, X_test, y_train, y_test = create_traning_test_data(df,0.2)
    X_train=X_train.fillna(0)
    y_test=y_test.fillna(0)
    X_train=X_train.fillna(0)
    y_train=y_train.fillna(0)
    evaluation = [( X_train, y_train), ( X_test, y_test)]

    reg.fit(X_train, y_train,
            eval_set=evaluation, eval_metric="rmse",
            early_stopping_rounds=400,verbose=False)

    pred = reg.predict(X_test)
    mse= mean_squared_error(y_test, pred)
    rmse=np.sqrt(mean_squared_error( y_test,pred))
    #change the metric if you like
    return {'loss':rmse, 'status': STATUS_OK }

def run_optimaziation(hyperparameter_tuning,space,evals,save):
    trials = Trials()
    best = fmin(fn=hyperparameter_tuning,
                space=space,
                algo=tpe.suggest,
                max_evals=evals,
                trials=trials)
    print(best)
    if save:
        pickle.dump(best, open("best_param_xgb.p", "wb"))

    return best
#######################################
#optimation for xgboost
#######################################
'''
function to retring wight of the hyperoptimatsion
    Args:
        hyperparameter_tuning: fuction to optimize
        space: grid of hypoeroppt parameters
        evals: number of runs
        save: False/True statet what to save new params
'''
#best=run_optimaziation(hyperparameter_tuning,space,2,False)

'''
Train XG-Boost modeil with optimazet wights from histroul data set
    Arg:
        df: dataset what to predict on
        best: True if what to use best params
'''

def appartmet_model(df,best):
    if best:
        best = pickle.load(open("best_param_xgb.p", "rb"))

    X_train, X_test, y_train, y_test = create_traning_test_data(df,0.2)
    #fit moddel
    xgb_model_opt = xgb.XGBRegressor(objective='reg:squarederror',
                           learning_rate =best['learning_rate'],
                           n_estimators =int(best['n_estimators']),
                           max_depth = int(best['max_depth']),
                           gamma = best['gamma'],
                           reg_alpha = int(best['reg_alpha']),
                           min_child_weight=best['min_child_weight'],
                           colsample_bytree=best['colsample_bytree'])
    xgb_model_opt.fit(X_train, y_train,
            eval_set=[(X_train, y_train), (X_test, y_test)],
            early_stopping_rounds=10000,
            verbose=False)

    # calculate the auc score
    y_pred = xgb_model_opt.predict(X_test)
    feature_important = xgb_model_opt.get_booster().get_score(importance_type='weight')
    keys = list(feature_important.keys())
    values = list(feature_important.values())

    data_feuteres = pd.DataFrame(data=values, index=keys, columns=["score"]).sort_values(by = "score", ascending=False)
    print(data_feuteres)

    # save model to file
    pickle.dump(xgb_model_opt, open("xg_opti.pickle.dat", "wb"))
    print(xgb_model_opt)

    #y_pred = xgb_model.predict(X)
    data_pred = pd.DataFrame(y_pred)
    data_pred = data_pred.reset_index(drop=True)
    data_ytest = pd.DataFrame(y_test)
    data_ytest = data_ytest.reset_index(drop=True)

    result = pd.concat([data_ytest, data_pred], axis=1)
    result.columns = ['price', 'pred_price']
    result['RSME']=np.sqrt(mean_squared_error(result['price'],result['pred_price']))
    result['error'] = (abs(result['price']-result['pred_price'])/result['pred_price'])
    print(result['RSME'].mean())
    print(result['error'].mean())

    return result['error'].mean(), data_feuteres

error,data_feuteres=appartmet_model(df,True)
print('Error',error)
print('After',data_feuteres)

'''
run train moduel on DataFrame
    Arg:
        df: dataset what to predict on
        pre_train_model: Model from prives train model
'''

def run_model(df,pre_train_model):
    #load an pre train moduel
    if pre_train_model:
        loaded_model = pickle.load(open("xg_opti.pickle.dat", "rb"))
    # Splitt data in X and Y streetName
    X=df.filter(items=['encode_street','encode_area','Begärt_pris','encode_name','Antal_rum','Boarea','avgift',
                       'Driftskostnad','byggår','dayofweek','quater','month','dayofyear'])
    yhat = loaded_model.predict(X)
    return yhat

yhat=run_model(df.iloc[-1:],True)
print(df.iloc[-1:])
print(yhat)


import torch
from torch import nn
import torch.nn.functional as F
from fastai import *
from fastai.vision import *
from fastai.layers import *

MAX_LENGTH = 1000

class SpeechModel(Module):
    "Contains an Encoder and a Decoder"
    def __init__(self,
                 encoder_output_size,
                 lang,
                 device=None,
                 teacher_forcing_pct=0.5):

        # select device
        self.device = device
        if device is None:
            if torch.cuda.is_available():
                self.device = torch.device("cuda:0")

        self.lang = lang
        self.vocab_size = len(lang)
        self.teacher_forcing_pct = teacher_forcing_pct

        self.encoder = CNNEncoder(encoder_output_size)
        self.embedding = nn.Embedding(self.vocab_size, encoder_output_size)
        self.pos_enc = PositionalEncoding(encoder_output_size, max_len=MAX_LENGTH)

        self.decoder = TransformerDecoder(encoder_output_size)
        self.ffn = nn.Linear(encoder_output_size, self.vocab_size)

        # put model on device
        if self.device is not "cpu":
            self.cuda()

    def forward(self, spectrogram, l=MAX_LENGTH):
        """
        spectrogram shape: [B x C x W x H]
        """

        spectrogram = spectrogram.to(self.device)
        batch_size = spectrogram.size(0)

        # run encoder to transform audio
        # to a vector in latent space
        context = self.encoder(spectrogram)
        print("context:", context.shape)

        # prepare input tokens for decoder
        sos = self.lang.t2i["<SOS>"]
        pad = self.lang.t2i["<PAD>"]

        # TODO: the input is all empty...?
        input_tokens = torch.LongTensor([[sos] + [pad] * (l-1)] * batch_size).to(self.device)
        print("input_tokens:", input_tokens.shape)

        # embed input token
        embedded = self.embedding(input_tokens)

        # apply positional encoding
        embedded_pos_enc = self.pos_enc(embedded)
        embedded_pos_enc = embedded_pos_enc.permute(1, 0, 2)
        print("embedded_pos_enc:", embedded_pos_enc.shape)

        # feed into decoder
        output_decoder = self.decoder(embedded_pos_enc, context)
        print("output_decoder:", output_decoder.shape)

        # ffn net
        output_tokens = self.ffn(output_decoder)

        # reshape output to [B x T x D]
        output_tokens = output_tokens.permute(1, 0, 2)
        print("output_tokens:", output_tokens.shape)
        return output_tokens

    def decode_greedy(self, spectrogram, **kwargs):
        output_tokens = self.forward(spectrogram, **kwargs)
        output_tokens = torch.log_softmax(output_tokens, dim=-1)

        with torch.no_grad():
            output_tokens = output_tokens.cpu().numpy()
            print(output_tokens)

            out = []
            for batch in output_tokens:

                sentence = ""
                for t in batch:
                    idx = t.argmax()
                    sentence += self.lang.i2t[idx]

                out.append(sentence)
            return out

    def predict(self):
        steps = 0
        while not tokens[-1] == eos and not steps > MAX_LENGTH:

            # NO teacher forcing here
            input_token = tokens[-1]

            # embed input token
            embedded = self.embedding(input_token)

            # apply positional encoding
            embedded = self.pos_enc(embedded)

            # feed to decoder
            decoder_output = self.decoder(embedded, context)
            decoder_output = self.linear(decoder_output)
            decoder_output_sm = F.log_softmax(decoder_output, dim=-1)

            print("decoder_output_sm:", decoder_output_sm.shape)
            _, decoder_output_token = decoder_output_sm.max(0)

            # append to already created tokens
            print("decoder_output_token:", decoder_output_token.shape)
            tokens.append(decoder_output_token)

        return tokens

class CNNEncoder(Module):
    "Encodes an audio spectrogram into a latent state representation"
    def __init__(self, output_size, base_model=models.resnet34):
        self.output_size = output_size
        self.model = create_cnn_model(base_model, output_size)

    def forward(self, spectrogram):
        context = self.model(spectrogram)
        return context

class TransformerDecoder(Module):
    "TransformerDecoder Module"
    def __init__(self, input_size, n_heads=2, num_layers=2):
        self.decoder_layer = nn.TransformerDecoderLayer(input_size, n_heads)
        self.decoder = nn.TransformerDecoder(self.decoder_layer, num_layers)
        self.forward = self.decoder.forward

class PositionalEncoding(Module):

    def __init__(self, d_model, dropout=0.1, max_len=5000):
        self.dropout = nn.Dropout(p=dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0).transpose(0, 1)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + self.pe[:x.size(0), :]
        return self.dropout(x)


import smtplib
import ssl
from email.mime.text import MIMEText
from email.utils import formataddr
from email.mime.multipart import MIMEMultipart  # New line
from email.mime.base import MIMEBase  # New line
from email import encoders  # New line
from PIL import Image
import os
import time
import win32com.client as win32

def sender(receiver_emails, receiver_names, name_file):

    # User configuration
    sender_email = "pl@avtosojuz.ua"
    sender_name = "pl"

    receiver_emails = [receiver_emails]
    receiver_names = [receiver_names]
    filename2 = name_file

    # Email body
    email_body = """
{}
""".format("")

    for receiver_email, receiver_name in zip(receiver_emails, receiver_names):
            # Configurating user's info
            msg = MIMEMultipart()
            msg['To'] = formataddr((receiver_name, receiver_email))
            msg['From'] = formataddr((sender_name, sender_email))
            msg['Subject'] = '' + receiver_name

            msg.attach(MIMEText(email_body, 'html'))

            try:
                # Open PDF file in binary mode
                with open(filename2, "rb") as attachment:
                                part = MIMEBase("application", "octet-stream")
                                part.set_payload(attachment.read())

                # Encode file in ASCII characters to send by email
                encoders.encode_base64(part)

                # Add header as key/value pair to attachment part
                part.add_header(
                        "Content-Disposition",
                        f"attachment; filename= {filename2}",
                )
                msg.attach(part)
            except Exception as e:
                    break

            try:
                    # Creating a SMTP session | use 587 with TLS, 465 SSL and 25
                    server = smtplib.SMTP('10.10.10.47', 25)
                    # Encrypts the email
                    # We log in into our Google account
                    # Sending email from sender, to receiver with the email body
                    server.sendmail(sender_email, receiver_email, msg.as_string())
            except Exception as e:
                    break
            #finally:
                    #server.quit()
                   # time.sleep(20)

def add_attachments(to_address, filename):
    outlook = win32.Dispatch('outlook.application')
    mail = outlook.CreateItem(0)
    mail.To = '{}'.format(to_address)
    mail.Subject = ''
    mail.Body = ''
    mail.HTMLBody = '

'  # this field is optional

    # To attach a file to the email (optional):
    attachment = filename
    print(attachment)
    mail.Attachments.Add(attachment)
    mail.display()


from data import questions
from quiz import Question
from quiz import Quiz

question_bank = []
for question in questions:
    new_question = question['question']
    new_answer = question['correct_answer']
    current_question = Question(new_question, new_answer)
    question_bank.append(current_question)

quiz = Quiz(question_bank)

while quiz.still_has_questions():
    quiz.next_question()

print("You've completed the quiz")
print(f"Your final score is {quiz.score}/{quiz.question_number}")


import numpy as np
from collections import deque
import gym
from gym import spaces
import cv2
cv2.ocl.setUseOpenCL(False)

class NoopResetEnv(gym.Wrapper):
    def __init__(self, env, noop_max=30):
        gym.Wrapper.__init__(self, env)
        self.noop_max = noop_max
        self.noop_action = 0
        assert env.unwrapped.get_action_meanings()[0] == 'NOOP'

    def reset(self, **kwargs):
        self.env.reset(**kwargs)
        noops = self.unwrapped.np_random.randint(1, self.noop_max+1)
        for _ in range(noops):
            obs, _, done, _ = self.env.step(self.noop_action)
            if done:
                obs = self.env.reset(**kwargs)
        return obs

    def step(self, a):
        return self.env.step(a)

class FireResetEnv(gym.Wrapper):
    def __init__(self, env):
        gym.Wrapper.__init__(self, env)
        assert env.unwrapped.get_action_meanings()[1] == 'FIRE'

    def reset(self, **kwargs):
        self.env.reset(**kwargs)
        obs, _, done, _ = self.env.step(1)
        if done:
            self.env.reset(**kwargs)
        obs, _, done, _ = self.step(2)
        if done:
            self.env.reset(**kwargs)
        return obs

    def step(self, a):
        return self.env.step(a)

class MaxAndSkipEnv(gym.Wrapper):
    def __init__(self, env, skip=4):
        gym.Wrapper.__init__(self, env)
        self._skip = skip
        self._obs = deque([], maxlen=2)

    def reset(self, **kwargs):
        return self.env.reset(**kwargs)

    def step(self, a):
        total_reward = 0.0
        done = None
        for i in range(self._skip):
            obs, reward, done, info = self.env.step(a)
            if i == self._skip - 2: self._obs.append(obs)
            if i == self._skip - 1: self._obs.append(obs)
            total_reward += reward
            if done:
                break
        max_obs = np.array(self._obs).max(axis=0)
        return max_obs, total_reward, done, info

class EpisodicLifeEnv(gym.Wrapper):
    def __init__(self, env):
        gym.Wrapper.__init__(self, env)
        self.lives = 0
        self.was_real_done = True

    def reset(self, **kwargs):
        if self.was_real_done:
            obs = self.env.reset(**kwargs)
        else:
            obs, _, _, _ = self.env.step(0)
        self.lives = self.env.unwrapped.ale.lives()
        return obs

    def step(self, a):
        obs, reward, done, info = self.env.step(a)
        self.was_real_done = done
        lives = self.env.unwrapped.ale.lives()
        if lives < self.lives and lives > 0:
            done = True
        self.lives = lives
        return obs, reward, done, info

class WarpFrame(gym.ObservationWrapper):
    def __init__(self, env):
        gym.ObservationWrapper.__init__(self, env)
        self.width = 84
        self.height = 84
        self.observation_space = spaces.Box(low=0, high=255, shape=(self.height, self.width), dtype=np.uint8)

    def observation(self, frame):
        frame = frame.astype('float32')
        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
        frame = frame.astype('uint8')
        frame = cv2.resize(frame, (self.height, self.width), interpolation=cv2.INTER_AREA)
        return frame

def make_env(env_id, noop_max, skip=4):
    env = gym.make(env_id)
    assert 'NoFrameskip' in env.spec.id
    env = NoopResetEnv(env, noop_max=noop_max)
    env = MaxAndSkipEnv(env, skip=skip)
    env = EpisodicLifeEnv(env)
    if 'FIRE' in env.unwrapped.get_action_meanings():
        env = FireResetEnv(env)
    env = WarpFrame(env)
    return env


#!/usr/bin/env python
# coding: utf-8

# author: Jordan King

from __future__ import absolute_import
from __future__ import print_function

import numpy as np
import pandas as pd
import csv as csv
from scipy import spatial
from scipy import stats

from sklearn.cross_validation import KFold
from sklearn.cross_validation import train_test_split

from random import randint, uniform
import math as math

class Predictor:
    """
    Classifies model observations based on nearest neighbor search
    """

    def __init__(self):
        """
        sets the parameters for the Predictor:
        mode:
            kmxv - xval
            kmnn - predict and save predictions
        folds: for xval
        subset: percent of training data to use during xval or model fitting
        """
        self.mode = 'kmnn'

        self.models = [{'path': 'data/model1_train.csv', 'resolution':[192,288], 'y':1},
                        {'path': 'data/model2_train.csv', 'resolution':[128,256], 'y':2},
                        {'path': 'data/model3_train.csv', 'resolution':[160,320], 'y':3}]

        # self.models = [{'path': 'small_data/model1_train.csv', 'resolution':[99,1], 'y':1},
        #                 {'path': 'small_data/model2_train.csv', 'resolution':[99,1], 'y':2},
        #                 {'path': 'small_data/model3_train.csv', 'resolution':[99,1], 'y':3}]

        self.test_data = {'path': 'data/model_test.csv', 'resolution':[72,144], 'y':[]}
        self.nb_classes = len(self.models)

        self.subset = 1
        self.columns = 1274
        self.topn = 5

        self.folds = 5

        self.out_file = 'solutions/answersknnn.csv'

    def load_data(self):
        """
        returns X and y - data and target - as numpy arrays, y made categorical.
        """
        self.observations = [model['resolution'][0]*model['resolution'][1]*4 for model in self.models]

        # add a column for y values to shuffle
        self.X = np.empty([sum(self.observations), self.columns+1])

        idx = 0
        for m in range(len(self.models)):
            print('Loading model', m+1, '...')
            # print(self.models[m])
            self.X[idx:idx+self.observations[m], :self.columns] = pd.read_csv(self.models[m]['path'], delimiter=',', header=0, dtype='float32').values[:,1:self.columns+1]
            self.X[idx:idx+self.observations[m], self.columns] = self.models[m]['y']
            idx += self.observations[m]

        print('Finding mean observations...')
        for i in range(self.X.shape[0]/4):
            self.X[i,:] = np.average(self.X[i*4:(i*4)+4,:], axis=0)
            for l in range(2):
                correction = self.X[i,l] % 2.5
                if correction < 2.5/2:
                    correction = -1 * correction
                else:
                    correction = 2.5 - correction
                self.X[i,l] = self.X[i,l] + correction
        self.X = self.X[0:self.X.shape[0]/4,:]

        if self.subset != 1:
            np.random.shuffle(model['X'])
            self.X = self.X[0:int(self.subset*self.X.shape[0]):, ::]

        self.y = self.X[:,[self.columns]]
        self.X = self.X[:, :self.columns]

        print('Normalizing models...')
        self.observations = np.array(self.observations) / 4
        idx = 0
        for m in range(len(self.models)):
            model_min = np.amin(self.X[idx:idx+self.observations[m], 2:])
            model_max = np.amax(self.X[idx:idx+self.observations[m], 2:])
            self.X[idx:idx+self.observations[m], 2:] =  (self.X[idx:idx+self.observations[m], 2:] - model_min) / (model_max - model_min)
            idx += self.observations[m]

    def kmeans(self):
        """
        runs scipy kmeans
        """

        print('Loading test data...')
        X_test = pd.read_csv(self.test_data['path'], delimiter=',', header=0, dtype='float32').values[:,1:self.columns+1]

        print('Normalizing test data...')
        test_min = np.amin(X_test[:,2:])
        test_max = np.amax(X_test[:,2:])
        X_test[:,2:] = (X_test[:,2:] - test_min) / (test_max - test_min)

        predictions = np.empty([X_test.shape[0]], dtype = 'int32')

        print('Building tree...')
        kdtree = spatial.cKDTree(self.X[:,0:2], leafsize=10)

        print('Developing predictions...')
        for i in range(X_test.shape[0]):

            # find points within +- x degrees lat long of test point
            neighbors = kdtree.query_ball_point(X_test[i,0:2], 4)
            neighbor_data = np.empty([len(neighbors), 3])
            neighbor_data[:,0] = neighbors

            # Identify stats about these points
            for neighbor in neighbor_data:
                # find climate dist difference and class of neighbor
                neighbor[1] = np.linalg.norm(X_test[i,2:] - self.X[neighbor[0],2:])
                neighbor[2] = self.y[neighbor[0]]

            # sort the data by dist
            neighbor_data = neighbor_data[neighbor_data[:,1].argsort()]

            predictions[i] = neighbor_data[0,2]
            print('ID:', i, 'class:', predictions[i], 'dist:', neighbor_data[0,1])

            # topx = self.topn
            # while True:
            #     if neighbor_data[0,2] == neighbor_data[1,2] or neighbor_data[0,1] * 2 < neighbor_data[1,1]:
            #         predictions[i] = neighbor_data[0,2]
            #         print('No vote at:', i, 'with outcome:', predictions[i], 'and dist:', neighbor_data[0,1])
            #         break

            #     classes, counts = stats.mode(neighbor_data[:topx,2])
            #     if counts[0] > (topx/3)+1:
            #         predictions[i] = classes[0]

            #         if not counts[0] >= topx:
            #             print('Vote resolved at:', i, 'with outcome:', predictions[i], 'and votes:')
            #             for x in range(topx):
            #                 print('Class:', neighbor_data[x,2], 'Dist:', neighbor_data[x,1])
            #         break

            #     else:
            #         print('Disputed point with', topx, 'votes:', i)
            #         # print('Point:', X_test[i])
            #         for x in range(topx):
            #             print('Class:', neighbor_data[x,2], 'Dist:', neighbor_data[x,1])
            #             # print('Point:', self.X[neighbor_data[x,0]])
            #         topx+=2
        print(np.bincount(predictions))

        return predictions

    def save_predictions(self, predictions):
        """
        saves the predictions to file in a format that kaggle likes.
        """

        predictions_file = open(self.out_file, "wb")
        open_file_object = csv.writer(predictions_file, quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
        open_file_object.writerow(['id','model'])
        for i in range(0,predictions.shape[0]):
            open_file_object.writerow([i+1, predictions[i]])
        predictions_file.close()

    def run(self):
        """
        set up the test here!
        """

        print('Loading data...')
        self.load_data()

        if self.mode == 'kmnn':
            print('Beginning evaluation...')
            self.save_predictions(self.kmeans())

def main():
    p = Predictor()
    p.run()

if __name__ == '__main__':
    main()


from django import forms
from site1.models import Site1Calling,Site1Blogs,Site1Guestbook,Site1Feedback,Site1New,Site1Service

class Site1CallingForm(forms.ModelForm):
    name = forms.CharField( label="",widget=forms.TextInput(
            attrs={'class': 'form-control', 'placeholder': 'Как к Вам обращаться?'}
        ))
    phone = forms.CharField( label="",widget=forms.TextInput(
            attrs={'class': 'form-control', 'placeholder': 'На какой номер позвонить?'}
        ))
    time = forms.CharField( label="",widget=forms.TextInput(
            attrs={'class': 'form-control', 'placeholder': 'В какое время позвонить?'}
        ))
    class Meta:
        model = Site1Calling
        fields = ['name', 'phone', 'time']

class Blog2Form(forms.ModelForm):
	class Meta:
		model = Site1Blogs
		exclude = ['likes', 'views']

class GuestbookForm(forms.ModelForm):
	class Meta:
		model=Site1Guestbook
		fields='__all__'
	user=forms.CharField(max_length=20, label="Пользователь")
	content=forms.CharField(widget=forms.Textarea, label="Содержание")
	honeypot=forms.CharField(required=False, label="Ловушка для спамеров")

class FeedbackForm(forms.ModelForm):
    name = forms.CharField( label="",widget=forms.TextInput(
            attrs={'class': 'form-control', 'placeholder': 'Имя'}
        ))
    email = forms.EmailField( label="",widget=forms.EmailInput(
            attrs={'class': 'form-control', 'placeholder': 'Электронная почта'}
        ))
    message = forms.CharField( label="",widget=forms.Textarea(
            attrs={'class': 'form-control', 'placeholder': 'Сообщение'}
        ))
    class Meta:
        model = Site1Feedback
        fields = ['name', 'email', 'message']

class New2Form(forms.ModelForm):
	class Meta:
		model = Site1New
		fields = '__all__'

class ServiceForm(forms.ModelForm):
	class Meta:
		model = Site1Service
		fields = '__all__'


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('deal', '0012_auto_20151017_0854'),
    ]

    operations = [
        migrations.AlterField(
            model_name='deal',
            name='price',
            field=models.DecimalField(verbose_name='\u8ba2\u5355\u603b\u4ef7', default=0.0, help_text='', max_digits=20, decimal_places=2),
        ),
    ]


import numpy as np
import matplotlib.pyplot as plt
import os
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(9988)

def get_data():

    d = 300

    cov = [[4, 0], [0, 4]]
    mean1 = [-4, 0]
    x1 = np.random.multivariate_normal(mean1, cov, d)
    y1 = np.zeros([d,3])
    y1[:,0] = 1
    plt.plot(x1[:, 0], x1[:, 1],'bx')

    mean2 = [4, 0]
    x2 = np.random.multivariate_normal(mean2, cov, d)
    y2 = np.zeros([d,3])
    y2[:,1] = 1
    plt.plot(x2[:, 0], x2[:, 1], 'yx')

    mean3 = [0, 5]
    x3 = np.random.multivariate_normal(mean3, cov, d)
    y3 = np.zeros([d,3])
    y3[:,2] = 1
    plt.plot(x3[:, 0], x3[:, 1], 'rx')

    h = 200
    dataX = np.concatenate((x1[:h,:], x2[:h,:], x3[:h,:]), axis = 0)
    dataY = np.concatenate((y1[:h,:], y2[:h,:], y3[:h,:]), axis = 0)

    dist_val = 6.5

    def euclid_dist(i,j, mu):

        dist = np.sqrt((i-mu[0])*(i-mu[0]) + (j-mu[1])*(j-mu[1]))
        return dist

    count = 0
    while count< 600:
        i = np.random.uniform(-15, 15)
        j = np.random.uniform(-13, 17)

        d1 = euclid_dist(i,j,mean1)
        d2 = euclid_dist(i,j,mean2)
        d3 = euclid_dist(i,j,mean3)

        if d1>dist_val and d2>dist_val and d3>dist_val:
            dataX = np.concatenate( (dataX, [[i,j]]), axis=0 )
            dataY = np.concatenate( (dataY, [[0.33, 0.33, 0.33]]), axis=0 )
            count += 1

    plt.plot(dataX[h*3:, 0], dataX[h*3:, 1], 'k,')

    plt.plot(-20,20)
    plt.plot(20,-20)
    plt.plot(-20,-20)
    plt.plot(20,20)

    plt.xlabel("x")
    plt.ylabel("y")

    return dataX, dataY

get_data()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import time
import logging
import threading

def consumer(cond: threading.Condition):
    logging.debug("Starting consumer thread")
    with cond:
        cond.wait()
        logging.debug("Resource is available to consumer")

def producer(cond: threading.Condition):
    logging.debug("Starting producer thread")
    with cond:
        logging.debug("Making resource available")
        cond.notifyAll()

def main():
    logging.basicConfig(
        level=logging.DEBUG,
        format="%(asctime)s (%(threadName)-10s) %(message)s",
    )

    condition = threading.Condition()

    consumers = []
    for i in range(2):
        t = threading.Thread(target=consumer,
                             args=(condition,),
                             name="c" + str(i + 1),)
        consumers.append(t)

    p = threading.Thread(target=producer,
                         args=(condition,),
                         name="p",)

    for c in consumers:
        c.start()
        time.sleep(0.2)
    p.start()

if __name__ == '__main__':
    main()


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.macosx-10.9-x86_64/egg/obpds/units.py
# Compiled at: 2015-04-01 14:26:54
from pint import UnitRegistry
units = UnitRegistry()
m = units.m
cm = units.cm
cm3 = units.cm ** 3
um = units.um
nm = units.nm

def to_units(v, u):
    if not isinstance(v, units.Quantity):
        raise TypeError(('Missing units: {}').format(v))
    return v.to(u).magnitude


import numpy as np
import os
import sys
import pickle
import cv2
import json

global cascaded

def varnorm(img):
    img = np.asarray(img)
    # mean = np.sum(np.sum(img))/576
    mean = np.mean(img)
    # std = np.sqrt(mean**2 - np.sum(np.sum(np.multiply(img,img)))/576)
    std = np.std(img)
    # print("std",std)
    img = ((img-mean)/float(std))

    return img

file2 = open("Model_Files/cascade.pkl", "rb")
cascaded = pickle.load(file2)

def det_face(integimg,coor):

    global cascaded
    f_score = np.zeros(len(integimg))

    for jj, vv in enumerate(cascaded):
        stage_v = np.zeros(len(integimg))
        is_face = np.zeros(len(integimg))
        for j,v in enumerate(vv[0]):
            if v[2][2] == 0:
                ir = v[2][0][0]
                ic = v[2][0][1]
                iir = v[2][1][0]
                iic = v[2][1][1]
                if len(integimg)==1:
                    fv = integimg[ir-1,ic-1] - integimg[ir-1,iic] + 2*integimg[ir+((iir-ir)//2),iic] - integimg[iir,iic] + integimg[iir,ic-1] - 2*integimg[ir+((iir-ir)//2),ic-1]
                else:
                    fv = integimg[:,ir-1,ic-1] - integimg[:,ir-1,iic] + 2*integimg[:,ir+((iir-ir)//2),iic] - integimg[:,iir,iic] + integimg[:,iir,ic-1] - 2*integimg[:,ir+((iir-ir)//2),ic-1]
            elif v[2][2] == 1:
                ir = v[2][0][0]
                ic = v[2][0][1]
                iir = v[2][1][0]
                iic = v[2][1][1]
                if len(integimg)==1:
                    fv = -integimg[ir-1,ic-1] + 2*integimg[ir-1,ic+((iic-ic)//2)] - integimg[ir-1,iic] + integimg[iir,iic] - 2*integimg[iir,ic+((iic-ic)//2)] + integimg[iir,ic-1]
                else:
                    fv = -integimg[:,ir-1,ic-1] + 2*integimg[:,ir-1,ic+((iic-ic)//2)] - integimg[:,ir-1,iic] + integimg[:,iir,iic] - 2*integimg[:,iir,ic+((iic-ic)//2)] + integimg[:,iir,ic-1]
            elif v[2][2] == 2:
                ir = v[2][0][0]
                ic = v[2][0][1]
                iir = v[2][1][0]
                iic = v[2][1][1]
                if len(integimg)==1:
                    fv = -integimg[ir-1,ic-1] + 2*integimg[(ir+(iir-ir)//3),ic-1] - 2*integimg[(ir+2*((iir-ir)//3)+1),ic-1] + integimg[iir,ic-1] + integimg[ir-1,iic] - 2*integimg[(ir+(iir-ir)//3),iic] + 2*integimg[(ir+2*((iir-ir)//3)+1),iic] - integimg[iir,iic]
                else:
                    fv = -integimg[:,ir-1,ic-1] + 2*integimg[:,(ir+(iir-ir)//3),ic-1] - 2*integimg[:,(ir+2*((iir-ir)//3)+1),ic-1] + integimg[:,iir,ic-1] + integimg[:,ir-1,iic] - 2*integimg[:,(ir+(iir-ir)//3),iic] + 2*integimg[:,(ir+2*((iir-ir)//3)+1),iic] - integimg[:,iir,iic]
            elif v[2][2] == 3:
                ir = v[2][0][0]
                ic = v[2][0][1]
                iir = v[2][1][0]
                iic = v[2][1][1]
                if len(integimg) ==1:
                    fv = -integimg[ir-1,ic-1] + 2*integimg[ir-1,(ic+((iic-ic)//3))] - 2*integimg[ir-1,ic+(2*(iic-ic)//3)+1] + integimg[ir-1,iic] + integimg[iir,ic-1] - 2*integimg[iir,(ic+((iic-ic)//3))] + 2*integimg[iir,ic+(2*((iic-ic)//3))+1] - integimg[iir,iic]
                else:
                    fv = -integimg[:,ir-1,ic-1] + 2*integimg[:,ir-1,(ic+((iic-ic)//3))] - 2*integimg[:,ir-1,ic+(2*(iic-ic)//3)+1] + integimg[:,ir-1,iic] + integimg[:,iir,ic-1] - 2*integimg[:,iir,(ic+((iic-ic)//3))] + 2*integimg[:,iir,ic+(2*((iic-ic)//3))+1] - integimg[:,iir,iic]
            elif v[2][2] == 4:
                ir = v[2][0][0]
                ic = v[2][0][1]
                iir = v[2][1][0]
                iic = v[2][1][1]
                if len(integimg)==1:
                    fv = -integimg[ir-1,ic-1] + 2*integimg[ir-1,ic+((iic-ic)//2)] - integimg[ir-1,iic] + 2*integimg[ir+((iir-ir)//2),ic-1] - 4*integimg[ir+((iir-ir)//2),ic+((iic-ic)//2)] +2*integimg[ir+((iir-ir)//2),iic] - integimg[iir,ic-1] + 2* integimg[iir,ic+((iic-ic)//2)] - integimg[iir,iic]
                else:
                    fv = -integimg[:,ir-1,ic-1] + 2*integimg[:,ir-1,ic+((iic-ic)//2)] - integimg[:,ir-1,iic] + 2*integimg[:,ir+((iir-ir)//2),ic-1] - 4*integimg[:,ir+((iir-ir)//2),ic+((iic-ic)//2)] +2*integimg[:,ir+((iir-ir)//2),iic] - integimg[:,iir,ic-1] + 2* integimg[:,iir,ic+((iic-ic)//2)] - integimg[:,iir,iic]
            ffv = np.zeros(len(integimg))
            ffv[v[3]*fv > v[3]*v[4]*np.ones(len(integimg))] = 1
#             if v[3]*fv > v[3]*v[4]:
#                 ffv = 1
#             else:
#                 ffv = 0
#             alpha = np.log(1/v[1])
            alphas = np.ones(len(integimg))*np.log(1/v[1])

            stage_v = stage_v + np.multiply(alphas,ffv)
        th = np.ones(len(integimg))*vv[1]
        is_face[stage_v>th] = 1
#         print(stage_v.shape, th.shape, f_score.shape)
        f_score = f_score + (stage_v - th)
#         print(len(integimg), len(is_face))
        integimg = integimg[is_face==1]
        f_score = f_score[is_face==1]
        coor = coor[is_face==1]
#         print(len(integimg))
    return (integimg,f_score,coor)

folder = sys.argv[1]
# folder = "test_images"
rs = 4
final_jason = []
count = 0
for filename in os.listdir(folder):
    count += 1
    print("count", count)
    print(filename)
    img = cv2.imread(os.path.join(folder, filename), cv2.IMREAD_GRAYSCALE)

    img = cv2.resize(img, (0,0), fx=1/rs, fy=1/rs)
    num_r = img.shape[0]
    num_c = img.shape[1]
    # pr_fd_thr = 0
    # cr_fd_thr = 0
    scale = 5
    # scale = 1

    f_score_l = []
    coor_list = []
    swl = []
    while scale < 130:
        sub_win_len = int(scale*24/rs)
        print(scale)

        sub_win_arr = []
        coor = []
        for ir in range(num_r-(sub_win_len-1)):
            for ic in range(num_c-(sub_win_len-1)):
                sub_win = img[ir:ir+sub_win_len,ic:ic+sub_win_len]
                sub_win = cv2.resize(sub_win, (24,24))
                sub_win = varnorm(sub_win)
                sub_win = cv2.integral(sub_win)
                sub_win_arr.append(sub_win)
                coor.append((ir,ic))

        sub_win_arr = np.asarray(sub_win_arr)
    #     print(sub_win_arr.shape)
        coor = np.asarray(coor)
        if len(sub_win_arr) < 1:
            break
        (face_sw, f_score, coor) = det_face(sub_win_arr,coor)
        print(coor.shape,"coor shape")
        print(f_score.shape,"f_score shape")
        f_score_l += list(f_score)
        coor_list += list(coor)
        swl += [sub_win_len]*len(coor)
        scale = scale*5

        f_score_list = np.asarray(f_score_l)
        max_f_score = np.max(f_score_list)
        max_f_score_ind = np.argmax(f_score_list)
#         print(max_f_score_ind)
#         print(max_f_score)
        T = 0.965
        final_faces = []
        for ii,vv in enumerate(f_score_list):
            if vv/max_f_score > T:
                final_faces.append(ii)

        for hh in final_faces:
            final_jason.append({"iname": filename, "bbox": [int(coor_list[hh][0]*rs), int(coor_list[hh][1]*rs), int(swl[hh]*rs), int(swl[hh]*rs)]})
#             cv2.rectangle(img0,(coor_list[hh][1]*rs,coor_list[hh][0]*rs),(coor_list[hh][1]*rs+swl[hh]*rs,coor_list[hh][0]*rs+swl[hh]*rs),(255, 0, 0),1)
output_json = "results.json"
#dump json_list to result.json
with open(output_json, 'w') as f:
    json.dump(final_jason, f)
#         cv2.namedWindow('image', cv2.WINDOW_AUTOSIZE)
#         cv2.imshow('image', img0)
#         cv2.waitKey(5000)
#         cv2.destroyAllWindows()