#!/usr/bin/python
# -*- coding: utf-8 -*-

import pandas as pd
import sys
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from scipy.integrate import odeint
from math import factorial as fat
from scipy.optimize import fmin,differential_evolution
import math

# Bibliotecas proprias
from expmmq import *
from mix import *
from NovoModelo import *

#pathway to the data
path = '../../data/'

# global data
# ajustar pela viremia e anticorpos
dadosViremiaLog10 = pd.read_csv(path+'Viral_load.csv',',')
dadosAnticorposLog2 = pd.read_csv(path+'IgG_IgM.csv',',')
dadosIL6 = pd.read_csv(path+'IL6_ajuste.csv',',')

first_day = 0
'''
virus = np.power(10,dadosViremiaLog10['Viral_load'])
antibody_g = np.power(2,dadosAnticorposLog2['IgG'])-1
antibody_m = np.power(2,dadosAnticorposLog2['IgM'])-1

'''

virus = dadosViremiaLog10['Viral_load']
antibody_g = dadosAnticorposLog2['IgG']
antibody_m = dadosAnticorposLog2['IgM']
il6_data = dadosIL6['IL6(pg/mL)']

# ones in days to be considered and zero otherwise
mask_virus     =[0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
mask_antibodies=[0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0]
mask_cytokine  =[0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0]

execution_de =  []

def model(x):
    '''
    Params description:
    x[0]=>pi_v
    x[1]=>k_v3
    x[2]=>delta_Apm
    x[3]=>alpha_B
    x[4]=>delta_A_G
    x[5]=>delta_A_M
    x[6]=>pi_c_apm,
    x[7]=>pi_c_i,
    x[8]=>pi_c_tke,
    x[9]=>delta_c,
    x[10]=>k_apm
    x[11]=>V0
    x[12]=>Am0
    x[13]=>Ag0
    '''

    ts=range(len(virus))
    #       V0,   Ap0,Apm0,  Thn0,The0,  Tkn0,,Tke0,     B0, Ps0, Pl0, Bm0, A0_M, A0_G Ai C
    #P0 = [9.971841136161140184e+02, 1.0e6, 0.0, 1.0e6, 0.0, 5.0e5, 0.0, 1.25E5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
    V0 = x[8]
    Ap0 = 1.0e6#0.6e6
    Apm0 = 0.0
    Ai0=0
    C0=0
    Thn0 = (1.0e6)#*0.5
    The0 = 0.0  #### Convertendo de ul para ml
    Tkn0 = (1.0e3)*500.0#(1.0e3)*500.0
    Tke0 = 0.0
    B0 =  (1.0e3)*250.0#125000.0#
    Ps0 = 0.0
    Pl0 = 0.0
    Bm0 = 0.0
    A0_M = 0.0
    A0_G = 0.0
    P0 = [V0,Ap0,Apm0,Thn0,The0,Tkn0,Tke0,B0,Ps0,Pl0,Bm0,A0_M,A0_G,Ai0,C0]
    #P0 = [x[11], 1.0e6, 0.0, 1.0e6, 0.0, 5.0e5, 0.0, 1.25E5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]

    #            pi_v,  c_v1, c_v2, k_v1, k_v2, alpha_Ap, beta_Ap,k_ap1,k_ap2,delta_Apm, alpha_Tn,    pi_T,   k_te1,delta_te,alpha_B,     pi_B1,   pi_B2,
    #beta_S,   beta_L, beta_Bm,delta_S,    delta_L, gamma_M, k_bm1,    k_bm2, pi_AS,  pi_AL,delta_A_G, delta_A_M,       c11,     c12,  c13,  c14, Ap0, Thn0,
    #Tkn0

    '''
    model_args = (x[0], 2.63, 0.60, 9.783944251204167164e-02, 4.164999863476288964e-05, 2.50E-03, 5.5e-01, 0.8, 40.0,
    x[2], 2.17E-04, 1.0E-05, 1.0E-08,  0.0003,   x[3], 4.826E-06, 1.27E-8,
    0.000672, 5.61E-06, 1.0E-06,   2.0, 2.3976E-04, 9.75E-04, 1.0e-5, 2500.0, 0.002,0.00068,     x[4],
    x[5], 2.17E-04, 1.0E-07, 1.0E-08, 0.22, 1.0e6, 1.0e6, 5.0e5, 2.5E5, x[6], x[7], x[8], x[9], x[10], x[1])

    model_args = (1.2, 2.63, 0.60, 4.5e-05, 4.5e-05, 1.87E-06*0.4, 2.00E-03, 0.8, 40.0,
    8.14910996e+00, 2.17E-04, 1.9E-05, 1.0E-08, 0.1*0.003, 6.55248840e+01, 2.826E-06, 1.27E-08,
    0.000672, 5.61E-07, 1.0E-08, 1.5, 0.8, (1.95E-06)*500.0, 1.0e-5, 2500.0, 0.004, 0.0005, 8.00694162e-02,
    5.06889922e-00, 2.17E-04, 1.0E-04, 1.0E-08, 0.22,1.0e6, 1.0e6, 5.0e5, 2.5E5,0.000015,0.015,0.000015,0.1,0.000007,1.0e-6)

    model_args = (1.2, 2.63, 0.60, 4.5e-05, 4.5e-05, 1.87E-06*0.4, 2.00E-03, 0.8, 40.0,
    8.14910996e+00, 2.17E-04, 1.9E-05, 1.0E-08, 0.1*0.003, 6.55248840e+01, 2.826E-06, 1.27E-08,
    0.000672, 5.61E-07, 1.0E-08, 1.5, 0.8, (1.95E-06)*500.0, 1.0e-5, 2500.0, 0.004, 0.0005, 8.00694162e-02,
    5.06889922e-00, 2.17E-04, 1.0E-04, 1.0E-08, 0.22,1.0e6, 1.0e6, 5.0e5, 2.5E5,0.000015,0.015,0.000015,0.1,0.000007,1.0e-6)
    '''

    pi_v = x[0] #ajuste
    c_v1 = 2.63
    c_v2 = 0.60
    k_v1 = x[1]#4.5e-05 #ajustar
    k_v2 = x[2]#4.5e-05 #ajustar
    alpha_Ap = 1.87E-06*0.4
    beta_Ap = 2.00E-03
    k_ap1 = 0.8
    k_ap2 = 40.0

    delta_Apm = 8.14910996e+00 # ou ajuste carla 5.38E-01
    alpha_Tn =2.17E-04
    pi_T = x[3]#1.9E-05 #ajustar
    k_te1 = 1.0E-08
    delta_te = 0.0003
    alpha_B = x[9] #ajuste
    pi_B1 = x[10]#2.826E-06 #ajuste
    pi_B2 = 1.27E-8

    beta_S = 0.000672
    beta_L = 5.61E-06
    beta_Bm = 1.0E-06
    delta_S = 1.5
    delta_L = 0.8
    gamma_M = (1.95E-06)*500.0
    k_bm1 = 1.0e-5
    k_bm2 = 2500.0
    pi_AS = x[4]#0.004 #ajuste
    pi_AL = x[5]#0.0005 #ajuste
    delta_A_G = x[6] #ajuste
    delta_A_M = x[7] #ajuste
    c11 = 2.17E-04
    c12 = 1.0E-07
    c13 = 1.0E-08
    c14 = 0.22
    Ap0 = 1.0e6
    Thn0 = 1.0e6
    Tkn0 = 5.0e5
    B0 = 2.5E5
    pi_c_apm = 0.000015# x[11] #ajuste
    pi_c_i = 0.015#x[12] #ajuste
    pi_c_tke = 0.000015#x[13] #ajuste
    delta_c = 0.1#x[14]#ajuste
    k_apm = 0.000007#x[15]  #ajuste
    k_v3 = 1.0e-6#x[16]#ajuste

    model_args = (pi_v, c_v1, c_v2, k_v1, k_v2, alpha_Ap, beta_Ap, k_ap1, k_ap2,
    delta_Apm, alpha_Tn, pi_T, k_te1, delta_te, alpha_B, pi_B1, pi_B2,
    beta_S, beta_L, beta_Bm,delta_S, delta_L, gamma_M, k_bm1, k_bm2, pi_AS,
    pi_AL, delta_A_G, delta_A_M, c11, c12, c13, c14, Ap0, Thn0, Tkn0, B0,
    pi_c_apm, pi_c_i,pi_c_tke,delta_c, k_apm, k_v3)

    Ps= odeint(immune_response, P0, ts, args=(model_args))

    V=Ps[:,0] # virus
    A_m=Ps[:,11] # antibody
    A_g=Ps[:,12] # antibody
    il6 = Ps[:,14] #Citocine

    erro_V = 0;
    erro_IgG = 0;
    erro_IgM = 0;
    vnorm=2

    #Viremia error
    #virus represents experimental data, V the numerical one

    #virus_aux = np.multiply(virus, mask_virus)

    V_aux = np.multiply(V, mask_virus[first_day:])
    erro_V = np.linalg.norm(virus[first_day:]-V_aux, vnorm)/np.linalg.norm(virus[first_day:], vnorm)
    #print(np.linalg.norm(V_aux,vnorm))
    if (math.isnan(erro_V) or math.isinf(erro_V)):
        erro_V = 1e12

    #antibody G
    #print(len(antibody_g), len(mask_antibodies))
    #print(antibody_g)

    antibody_g_aux = np.multiply(antibody_g, mask_antibodies)
    IgG_aux = np.multiply(A_g, mask_antibodies[first_day:])
    erro_IgG = np.linalg.norm(antibody_g_aux[first_day:]-IgG_aux, vnorm)/np.linalg.norm(antibody_g_aux[first_day:], vnorm)
    #print(np.linalg.norm(antibody_g_aux[first_day:]-IgG_aux, vnorm))
    if (math.isnan(erro_IgG) or math.isinf(erro_IgG)):
        erro_IgG = 1e12

    #antibody M
    antibody_m_aux = np.multiply(antibody_m, mask_antibodies)
    IgM_aux = np.multiply(A_m, mask_antibodies[first_day:])
    erro_IgM = np.linalg.norm(antibody_m_aux[first_day:]-IgM_aux, vnorm)/np.linalg.norm(antibody_m_aux[first_day:], vnorm)
    #print(antibody_m_aux[first_day:]-IgM_aux)
    if (math.isnan(erro_IgM) or math.isinf(erro_IgM)):
        erro_IgM = 1e12

    #cytokine il-6
    il6data_aux = np.multiply(il6_data, mask_cytokine)
    il6_aux = np.multiply(il6, mask_cytokine[first_day:])
    erro_il6 = np.linalg.norm(il6data_aux[first_day:]-il6_aux, vnorm)/np.linalg.norm(il6data_aux[first_day:], vnorm)
    #print(antibody_m_aux[first_day:]-IgM_aux)
    if (math.isnan(erro_il6) or math.isinf(erro_il6)):
        erro_il6 = 1e12

    weight = 0.5
    erro = weight*erro_IgG + weight*erro_IgM + erro_V
    '''
    print("RELATIVE ERROR")
    print("Erro viremia: ", erro_V)
    print("Erro IgM: ", erro_IgM)
    print("Erro IgG: ", erro_IgG)
    '''
    #erro = erro_V

    return [erro, V, A_m, A_g,il6, ts, erro_V, erro_IgM, erro_IgG, erro_il6]

def model_adj(x):
    result = model(x)
    return result[0]

if __name__ == "__main__":

    #define os bounds para cada um dos parâmetros

    opt_de = False
    if opt_de:
        '''
        Params description:
        x[0]=>pi_v
        x[1]=>k_v3
        x[2]=>delta_Apm
        x[3]=>alpha_B
        x[4]=>delta_A_G
        x[5]=>delta_A_M
        x[6]=>pi_c_apm,
        x[7]=>pi_c_i,
        x[8]=>pi_c_tke,
        x[9]=>delta_c,
        x[10]=>k_apm
        x[11]=>V0
        beta_s
        beta_l
        pi_AS
        pi_AL

        bounds = [
        (1e-2,1e1),
        (1e-7,1),
        (1e-7,1),
        (1e-7,1),
        (1e-4,1e1),
        (1e-5,1e1),
        (1e-5,1e2),
        (1e-5,1e2),
        (1e-7,1),
        (1e-7,1),
        (1e-7,1),
        (1e-7,1),
        (1e-7,1e2),
        (0,1e5),
        (1e-7,1),
        (1e-7,1),
        (1e-7,1),
        (1e-7,1)
        ]
        '''
        bounds = [
        (1e-1,2),
        (1e-6,1e-4),
        (1e-6,1e-4),
        (1e-7,1e-4),
        (1e-5,1e1),
        (1e-4,1),
        (1e-3,1),
        (1e-2,1e1),
        (1e-5,1e3),
        (1e-2,1e3),
        (1e-7,1e-4)
        ]
        '''
        (1e-7,1),
        (1e-7,1e-3),
        (1e-3,1),
        (1e-3,1e-3),
        (1e-7,1e-4),
        (1e-7,1e-4),
        '''
        #chama a evolução diferencial que o result contém o melhor individuo
        result = differential_evolution(model_adj, bounds, strategy='best1bin', popsize=20, disp=True, workers=3)
        print('Params order: ')
        print ('V0, pi_v,k_v1, k_v2, delta_Apm, alpha_B, delta_A_G,delta_A_M')
        print(result.x)
        #saving the best offspring...
        np.savetxt('params_simple.txt',result.x)
        best=result.x
    else:
        best = np.loadtxt('params_simple.txt')

    #saving the samples for UQ
    #np.savetxt('execution_de_100_ge.txt',execution_de)

    erro, V, A_m, A_g, il6, ts, erro_V, erro_IgM, erro_IgG, erro_il6 = model(best)
    print("RELATIVE ERROR")
    print("Erro viremia: ", erro_V)
    print("Erro IgM: ", erro_IgM)
    print("Erro IgG: ", erro_IgG)
    print("Erro il6: ", erro_il6)

    fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, 1)
    fig.set_size_inches(12, 25)

    ax1.set_title('Viremia and Antibodies')

    #Plot active infected cases
    ax1.plot(ts, V, label='Viremia model', linewidth=4)
    ax1.plot(ts, virus[first_day:], 'o', label='data', linewidth=4)
    ax1.set_xlabel('day')
    ax1.set_ylabel('log10 (copies/ml)')
    ax1.legend()
    ax1.grid()

    #Plot IgG
    ax2.plot(ts, np.log2(A_g+1), label='IgG model', linewidth=4)
    ax2.plot(ts, np.log2(antibody_g[first_day:]+1), 'o', label='data', linewidth=4)
    ax2.set_xlabel('day')
    ax2.set_ylabel('log2(S/CO+1)')
    ax2.legend()
    ax2.grid()

    #Plot igM
    ax3.plot(ts, np.log2(A_m+1), label='igM model', linewidth=4)
    ax3.plot(ts, np.log2(antibody_m[first_day:]+1), 'o', label='data', linewidth=4)
    ax3.set_xlabel('day')
    ax3.set_ylabel('log2(S/CO+1)')
    ax3.legend()
    ax3.grid()

    #Plot IL-6
    ax4.plot(ts, il6, label='IL-6 model', linewidth=4)
    ax4.plot(ts,il6_data, 'o', label='data', linewidth=4)
    ax4.set_xlabel('day')
    ax4.set_ylabel('(pg/mL)')
    ax4.legend()
    ax4.grid()

    plt.show()


# -*- coding: utf-8 -*-
"""Module providing views for the folderish content page type"""
import os
from AccessControl import Unauthorized
from Acquisition import aq_inner
from Products.CMFPlone.utils import safe_unicode
from Products.Five.browser import BrowserView
from plone import api
from zope.component import getMultiAdapter
from zope.component import getUtility

from ade25.base.mailer import get_mail_template

from newe.sitecontent.inquiry.mailer import create_plaintext_message
from newe.sitecontent.inquiry.mailer import prepare_email_message
from newe.sitecontent.inquiry.mailer import send_mail

from newe.sitecontent.interfaces import IResponsiveImagesTool

from newe.sitecontent import _

class SaleItemView(BrowserView):
    """ Sales item view

    The sales item should normally be displayed in form of a preview
    card in the parent folders overview listing. The item view consists
    of a request quotation form
    """
    def __call__(self):
        return self.render()

    def update(self):
        self.errors = {}
        unwanted = ('_authenticator', 'form.button.Submit')
        required = (
            'email', 'subject', 'fullname', 'street', 'city',
            'postal_code'
        )
        required_boolean = ('privacy-policy-agreement', 'privacy-policy')
        if 'form.button.Submit' in self.request:
            authenticator = getMultiAdapter((self.context, self.request),
                                            name=u"authenticator")
            if not authenticator.verify():
                raise Unauthorized
            form = self.request.form
            form_data = {}
            form_errors = {}
            error_idx = 0
            for field_name in required_boolean:
                if field_name not in form:
                    form_errors[field_name] = self.required_field_error()
                    error_idx += 1
            for value in form:
                if value not in unwanted:
                    form_data[value] = safe_unicode(form[value])
                    if not form[value] and value in required:
                        form_errors[value] = self.required_field_error()
                        error_idx += 1
                    else:
                        error = {
                            'active': False,
                            'msg': form[value]
                        }
                        form_errors[value] = error
            if error_idx > 0:
                self.errors = form_errors
            else:
                self.send_inquiry(form)

    def render(self):
        self.update()
        return self.index()

    @staticmethod
    def required_field_error():
        translation_service = api.portal.get_tool(name="translation_service")
        error = {}
        error_msg = _(u"This field is required")
        error['active'] = True
        error['msg'] = translation_service.translate(
            error_msg,
            'newe.sitecontent',
            target_language=api.portal.get_default_language()
        )
        return error

    def has_image(self):
        context = aq_inner(self.context)
        try:
            lead_img = context.image
        except AttributeError:
            lead_img = None
        if lead_img is not None:
            return True
        return False

    def get_image_data(self, uuid):
        tool = getUtility(IResponsiveImagesTool)
        return tool.create(uuid)

    def default_value(self, error):
        value = ''
        if error['active'] is False:
            value = error['msg']
        return value

    def send_inquiry(self, data):
        translation_service = api.portal.get_tool(name="translation_service")
        context = aq_inner(self.context)
        subject = _(u"Sale Inquiry")
        mail_tpl = self._compose_message(data)
        mail_plain = create_plaintext_message(mail_tpl)
        msg = prepare_email_message(mail_tpl, mail_plain)
        default_email = api.portal.get_registry_record(
            'plone.email_from_address')
        recipient_email = getattr(context, 'email', default_email)
        recipients = [recipient_email, ]
        send_mail(
            msg,
            recipients,
            translation_service.utranslate(
                subject,
                'ade25.contacts',
                target_language=api.portal.get_default_language()
            )
        )
        next_url = context.absolute_url()
        msg = _(u"Thank you for your interest. Your message has been sent.")
        api.portal.show_message(message=msg, request=self.request)
        return self.request.response.redirect(next_url)

    def _compose_message(self, data):
        context = aq_inner(self.context)
        portal = api.portal.get()
        portal_url = portal.absolute_url()
        template_vars = {
            'email': data['email'],
            'fullname': data['fullname'],
            'city': data['city'],
            'postal_code': data['postal_code'],
            'street': data['street'],
            'phone': data['phone'],
            'subject': str(data['subject']),
            'message': data['comment'],
            'url': portal_url,
            'title': context.Title,
            'description': context.Description,
            'price': getattr(context, 'price', _('No price available')),
        }
        template_path = os.path.dirname(os.path.realpath(
            os.path.abspath(__file__)))
        template_name = 'sales-inquiry-mail.html'
        message = get_mail_template(template_name,
                                    template_path,
                                    template_vars)
        return message


# adapted from http://stackoverflow.com/questions/20716842/python-download-images-from-google-image-search
# this is much better: https://github.com/NikolaiT/incolumitas/blame/master/content/Programming/googlesearch-a-rapid-python-class-to-get-search-results.md

import re
import os
import argparse
import sys
import json
import logging
from bs4 import BeautifulSoup
import pandas
from urllib2 import urlopen, Request

FORMAT = "[%(levelname)s] %(filename)s::%(funcName)s():%(lineno)s | %(message)s"
logger = logging.getLogger()
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format=FORMAT)

IMAGESIZE = 'isz:lt,islt:qsvga' #larger than 400x300; good to remove icons
PHOTO = 'itp:photo'
COLOR = 'ic:color'
JPGONLY='ift:jpg'
#usage rights in excess of fair use / fair dealing
#https://support.google.com/websearch/answer/29508?hl=en
#n: standard fair use
#f: free to use or share non-commercially
#fc: free to use or share even commercially
#fm: free to use, share or modify non-commercially
#fmc:free to use, share or modify even commercially
LEGALTERMS = ['n','f','fc','fm','fmc']

def setLevel(level):
  logger = logging.getLogger()
  logger.setLevel(level)

def get_soup(url, header):
  return BeautifulSoup(urlopen(Request(url, headers=header)), 'html.parser')

def get_image(path, query, max_images, legal_term = 'n'):
  newpath = os.path.join(path, query)
  if not os.path.exists(newpath):
      os.makedirs(newpath)
  qry = query.split()
  qry = '+'.join(qry)
  url = "https://www.google.com.hk/search?q=" + qry + "&source=lnms&tbm=isch&num=100"
  url += '&tbs=' + JPGONLY + ',' + PHOTO + ',' + COLOR #augment with filters that give us only color photos in jpeg format
  if legal_term != 'n':
      #url += '&tbs=sur:' + legal_term
      url += ',sur:' + legal_term

  logging.info('attempting query for search term: ['+ query + '] url = ' + url)
  header = {'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) "
                          "'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36"}
  soup = get_soup(url, header)
  ActualImages = []
  for a in soup.find_all("div", {"class":"rg_meta"}):
    #link, Type = json.loads(a.text)["ou"], json.loads(a.text)["ity"]
    link = json.loads(a.text)["ou"]
    #ActualImages.append((link, Type))
    ActualImages.append(link)
  logging.info('found ' + str(len(ActualImages)) + ' images for this query...getting first ' + str(max_images) + ' images')
  i = 1
  #for (img, Type) in ActualImages:
  for img in ActualImages:
    try:
      req = Request(url=img, headers={'User-Agent': header})
      raw_img = urlopen(req).read()
      img_path = os.path.join(newpath, "img" + "_" + str(i) + ".jpg")
      #if len(Type) == 0:
      #  img_path = os.path.join(newpath, "img" + "_" + str(i) + ".jpg")
      #else:
      #  img_path = os.path.join(newpath, "img" + "_" + str(i) + "." + Type)
      logging.info('  attempting to save file:' + img_path)
      f = open(img_path, 'wb')
      f.write(raw_img)
      f.close()
      i += 1
      if i > max_images:
        break
    except Exception as e:
      logging.error("could not retrieve: " + img)
      logging.error(e)

if __name__ == '__main__':
  setLevel(level=logging.DEBUG)
  parser = argparse.ArgumentParser(description='Scrape Google images')
  parser.add_argument('-n', '--num_images', default=5, type=int, help='num images to save per query')
  parser.add_argument('-p', '--path', default='/mnt/scratch/flowers/images/', type=str, help='data root path')
  parser.add_argument('-l', '--legal', default='n', type=str, choices=LEGALTERMS, help='download based on legal use rights: {'+ ','.join(LEGALTERMS)  +'}')
  parser.add_argument('-f', '--file', default='./query.csv', type=str, help='the universe of flowers in csv format.  uses column `search_term`')
  args = parser.parse_args()

  univ = pandas.read_csv(args.file)
  try:
    for q in univ.search_term:
      get_image(args.path, q, args.num_images, args.legal)
  except KeyboardInterrupt:
    pass
  sys.exit()


import matplotlib.pyplot as plt
import os

##############################################################################################################################
# read data

def parse_percent(s):
    s = s.replace('%','')
    s = s.replace(',','')
    return float(s)/100

# read validation accuracies
validation_accuracies = []

experiments = ['solidcolor', 'reflection', 'simpleshape', 'matching']

for exp in experiments:
    fn = '%s.txt' % exp
    valaccs = []
    with open(fn, 'r+') as file:
        for line in file:
            train_acc = parse_percent(line.split()[6])
            valacc   = parse_percent(line.split()[9])
            valaccs.append(train_acc)

    name = fn.replace('.txt', '')
    validation_accuracies.append((name,valaccs))

##############################################################################################################################
# plot data

# setup
fig, ax = plt.subplots()

# labels
plt.title('CNN Training Progress')
plt.ylabel('training accuracy')
plt.xlabel('epoch')

# data
epochs = range(1, len(validation_accuracies[0][1])+1)
for name, valaccs in validation_accuracies:
    ax.plot(epochs, valaccs, label=name)
plt.axis([1, epochs[-1],  0, 1.05])

# legend
legend = ax.legend(loc='lower center') #fontsize='x-large')

# show
plt.show()


from django.conf.urls import url
from . import views

urlpatterns=[
    url(r'^$',views.movie_list,name='movie_list'),
    url(r'^(?P<m_id>[0-9]+)/$',views.movie_info,name='movie_info'),
    url(r'^(?P<m_id>[0-9]+)/sentiment/$',views.sentiment_classify,name='sentiment_classify'),
    url(r'^index/$',views.show_index),
    url(r'^hotfilm/$',views.hot_film),
]


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os.path as osp
import glob
import datetime
import matplotlib.pyplot as plt

import ddf.resource.common
import ddf.resource.base
import ddf.database.common
import ddf.database.base

from ddf.database import query
import logging

logging.getLogger().setLevel(logging.DEBUG)
dir_path = osp.normpath(osp.expanduser('~/ddf-demo-query'))
dir_path = osp.normpath(osp.normpath('c:/MAGNETUBE/mesures'))

db_path = osp.join(dir_path, 'db-ddf')

def save_db():
    fpaths = glob.glob(osp.join(dir_path, '**/*.tdms'), recursive=True)
    fpaths.sort(key=str.lower)
    dbframes = []
    for fpath in fpaths:
        raw = ddf.resource.common.auto_raw(fpath)
        num, rargs_num = ddf.resource.common.auto_num(raw)
        tmeta, rargs_tmeta = ddf.resource.common.auto_tmeta(raw, num)
        gs = ddf.database.base.Gsetting(num=rargs_num, tmeta=rargs_tmeta)
        dbframe = ddf.database.common.build_frame(num, raw, tmeta, gs)
        dbframes.append(dbframe)

    db = ddf.database.common.update(
            db=ddf.database.base.EMPTY_DATABASE,
            dbframes=dbframes,
            date=datetime.datetime(2018, 1, 1),
            )

    ddf.database.common.save(db, db_path)

try:
    db = ddf.database.common.load(db_path)
except FileNotFoundError:
    save_db()
    db = ddf.database.common.load(db_path)

material = ddf.resource.base.MATERIALS.AL.name
speed = 4.7
sensor_id = 'JD21'
acquisition = "ACQ_MAGNETUBE_0"

qp3 = query.find(
        db=db,
        acquisition=acquisition,
        qfilters=(
                query.QSet('sensor_id', sensor_id),
                query.QSet('material', material),
                query.QPoint('speed', speed, 1.0),
                ),
        )
q0 = query.find(
        db=db,
        acquisition=acquisition,
        qfilters=(
                query.QSet('sensor_id', sensor_id),
                query.QSet('material', material),
                query.QPoint('speed', 0.0, 1.0),
                ),
        )
qm3 = query.find(
        db=db,
        acquisition=acquisition,
        qfilters=(
                query.QSet('sensor_id', sensor_id),
                query.QSet('material', material),
                query.QPoint('speed', -speed, 1.0),
                ),
        )

qx = query.find(
        db=db,
        acquisition=acquisition,
        qfilters=(
                query.QSet('sensor_id', sensor_id),
                query.QSet('material', material),
                ),
        )

fig, (ax_DR, ax_DPhi) = plt.subplots(2, 1, sharex=True)

#ax_DR.scatter(qx.data['num_fp'], qx.data['num_DR'], c=qx.data['speed'])
ax_DR.scatter(qx.data['num_fp'], qx.data['num_DR'] - qx.data['num_DR_mean'], c=qx.data['speed'])

#ax_DPhi.scatter(qx.data['num_fp'], qx.data['num_DPhi'], c=qx.data['speed'])
ax_DPhi.scatter(qx.data['num_fp'], qx.data['li_DPhi'] - qx.data['li_DPhi_mean'], c=qx.data['speed'])

#ax_DR.scatter(qp3.data['num_fp'], qp3.data['num_DR'], color='#FF0000')
#ax_DR.scatter(q0.data['num_fp'], q0.data['num_DR'], color='#00FF00')
#ax_DR.scatter(qm3.data['num_fp'], qm3.data['num_DR'], color='#0000FF')
#ax_DPhi.scatter(qp3.data['num_fp'], qp3.data['num_DPhi'], color='#FF0000')
#ax_DPhi.scatter(q0.data['num_fp'], q0.data['num_DPhi'], color='#00FF00')
#ax_DPhi.scatter(qm3.data['num_fp'], qm3.data['num_DPhi'], color='#0000FF')

#ax_DR.scatter(qp3.data['num_fp'], qp3.data['li_DR'], color='#AA0000')
#ax_DR.scatter(q0.data['num_fp'], q0.data['li_DR'],color='#00AA00')
#ax_DR.scatter(qm3.data['num_fp'], qm3.data['li_DR'], color='#0000AA')
#ax_DPhi.scatter(qp3.data['num_fp'], qp3.data['li_DPhi'], color='#AA0000')
#ax_DPhi.scatter(q0.data['num_fp'], q0.data['li_DPhi'],color='#00AA00')
#ax_DPhi.scatter(qm3.data['num_fp'], qm3.data['li_DPhi'], color='#0000AA')

ax_DR.grid()
ax_DPhi.grid()

fig.tight_layout()


#!/usr/bin/env python
# coding: utf-8
from __future__ import absolute_import

from tornado import gen
import tornado.web

from .base import JSONHandler

class FeedsHandler(JSONHandler):

    def initialize(self, **kwargs):
        self.require_setting('feeds')
        self.require_setting('default_feed_refresh')
        self.feeds = self.settings['feeds']

    @gen.coroutine
    def post(self):
        url = self.get_argument('url')
        refresh = self.get_argument(
            'refresh', self.settings['default_feed_refresh']
        )
        feed = self.feeds.add(url, refresh)
        self.write(feed)

    def get(self, feed_ids=None, action='show'):
        ids = self.parse_ids(feed_ids)
        if action == 'show':
            for feed in self.feeds.get(*ids):
                self.write(feed)
        elif action == 'delete':
            for feed in self.feeds.delete(*ids):
                self.write(feed)


import qutip as qt
import numpy as np
import benchmarkutils

name = "timeevolution_mcwf_cavity"

samples = 2
evals = 100
cutoffs = range(50, 501, 50)

def setup(N):
    options = qt.Options()
    options.atol = 1e-8
    options.rtol = 1e-6
    options.num_cpus = 1
    return options

def f(N, options):
    kappa = 1.
    eta = 1.5
    wc = 1.8
    wl = 2.
    delta_c = wl - wc
    alpha0 = 0.3 - 0.5j
    tspan = np.linspace(0, 10, 11)

    a = qt.destroy(N)
    at = qt.create(N)
    n = at * a

    H = delta_c * n + eta * (a + at)
    J = [np.sqrt(kappa) * a]

    psi0 = qt.coherent(N, alpha0)
    exp_n = qt.mcsolve(H, psi0, tspan, J, [n], ntraj=evals, options=options).expect[0]
    return np.real(exp_n)

print("Benchmarking:", name)
print("Cutoff: ", end="", flush=True)
checks = {}
results = []
for N in cutoffs:
    print(N, "", end="", flush=True)
    options = setup(N)
    checks[N] = sum(f(N, options))
    t = benchmarkutils.run_benchmark(f, N, options, samples=samples, evals=1)
    results.append({"N": N, "t": t / evals})
print()

benchmarkutils.check(name, checks, eps=0.05)
benchmarkutils.save(name, results)


from __future__ import absolute_import
#--------------------------------------------------------------------------
# File and Version Information:
#  $Id$
#
# Description:
#   GUIDarkListItem ...
#------------------------------------------------------------------------

#--------------------------------
__version__ = "$Revision$"
#--------------------------------

#import os
import unicodedata

from PyQt5 import QtCore, QtGui, QtWidgets

from CalibManager.Frame     import Frame
from .ConfigParametersForApp import cp
from CalibManager.Logger                 import logger
from . import GlobalUtils          as     gu
from .FileNameManager        import fnm

from .GUIDarkListItemRun     import *
from .GUIDarkListItemAdd     import *

#------------------------------

#class GUIDarkListItem(QtGui.QWidget) :
class GUIDarkListItem(Frame) :
    """GUI sets the source dark run number, validity range, and starts calibration of pedestals"""

    char_expand    = cp.char_expand.strip()
    char_shrink    = cp.char_shrink.strip()
    #char_expand    = ' v'
    #char_expand    = cp.char_expand
    #char_expand    = u'\u25BC' # solid down-head triangle
    #char_shrink    = u'\u25B2' # solid up-head triangle
    #char_shrink    = u'\u25B6' # solid right-head triangle
    #char_expand    = u'\u25BD' # open down-head triangle
    #char_shrink    = u'\u25B7' # open right-head triangle
    #char_expand    = 'V' # solid down-head triangle
    #char_shrink    = '>' # solid right-head triangle

    def __init__(self, parent=None, str_run_num='0000', run_type='Type N/A', comment='', xtc_in_dir=True) :

        #self.t0_sec = time()
        Frame.__init__(self, parent, mlw=1, style=QtWidgets.QFrame.HLine | QtWidgets.QFrame.Sunken)
        #QtGui.QWidget.__init__(self, parent)

        self.parent = parent

        #self.setGeometry(100, 100, 740, 77)
        self.setGeometry(100, 100, 740, 35)
        self.setWindowTitle('GUI Dark List Item')
        #try : self.setWindowIcon(cp.icon_help)
        #except : pass

        self.list_of_runs    = None

        self.str_run_num = str_run_num # cp.str_run_number

        #self.str_run_number = cp.str_run_number # cp.str_run_number
        #self.calib_dir      = cp.calib_dir
        #self.det_name       = cp.det_name

        self.but_expand_shrink = QtWidgets.QPushButton(self.char_expand)

        self.gui_add = None
        self.gui_run = GUIDarkListItemRun(self, str_run_num, run_type, comment, xtc_in_dir)
        self.hboxTT = QtWidgets.QHBoxLayout()
        self.hboxTT.addSpacing(5)
        self.hboxTT.addWidget(self.but_expand_shrink)
        self.hboxTT.addWidget(self.gui_run)
        self.hboxTT.addStretch(1)

        self.hboxW = QtWidgets.QHBoxLayout()
        #self.hboxWW = QtGui.QHBoxLayout()
        ##self.hboxWW.addStretch(1)
        #self.hboxWW.addSpacing(5)
        #self.hboxWW.addLayout(self.hboxW)
        #self.hboxWW.addStretch(1)

        self.vbox = QtWidgets.QVBoxLayout()
        #self.vbox.addWidget(self.gui_run)
        self.vbox.addLayout(self.hboxTT)
        self.vbox.addLayout(self.hboxW)
        self.vbox.addStretch(1)

        self.setLayout(self.vbox)

        self.but_expand_shrink.clicked.connect(self.onButExpandShrink)

        self.showToolTips()

        self.setStyle()

        #print '\nConsumed time (sec) =', time()-self.t0_sec

    def showToolTips(self):
        self.but_expand_shrink.setToolTip('Expand/shrink additional information space for this run.')

    def setFieldsEnabled(self, is_enabled=True):
        logger.debug('Set fields enabled: %s' %  is_enabled, __name__)
        #self.lab_rnum .setEnabled(is_enabled)
        self.but_expand_shrink.setEnabled(is_enabled)
        self.gui_run.setFieldsEnabled(is_enabled)

    def setStyle(self):
        self.setMinimumSize(740,35)
        self.setStyleSheet (cp.styleBkgd)
        #self.setStyleSheet(cp.styleGreenish)
        #self.setStyleSheet (cp.styleYellowish)

        self.but_expand_shrink.setFixedSize(20,22)
        #self.but_expand_shrink.setStyleSheet(cp.styleButtonGood)
        self.but_expand_shrink.setStyleSheet(cp.stylePink)

        #self.lab_from.setStyleSheet(cp.styleLabel)
        #self.lab_rnum .setFixedWidth(80)
        #self.edi_from .setAlignment (QtCore.Qt.AlignRight)
        #if self.edi_from.isReadOnly() : self.edi_from.setStyleSheet (cp.styleEditInfo)

        self.layout().setContentsMargins(0,0,0,9)

    #def resizeEvent(self, e):
        #logger.debug('resizeEvent', __name__)
        #self.box_txt.setGeometry(self.contentsRect())
        #print 'GUIDarkListItem resizeEvent: %s' % str(self.size())
        #pass

    #def moveEvent(self, e):
        #logger.debug('moveEvent', __name__)
        #cp.posGUIMain = (self.pos().x(),self.pos().y())
        #pass

    def closeEvent(self, event):
        logger.debug('closeEvent', __name__)
        #self.saveLogTotalInFile() # It will be saved at closing of GUIMain

        try    : self.gui_add.close()
        except : pass

        try    : self.gui_run.close()
        except : pass

    def onClose(self):
        logger.info('onClose', __name__)
        self.close()

    def setStatusMessage(self):
        if cp.guistatus is None : return
        #cp.guistatus.setStatusMessage(msg)

    def onButExpandShrink(self):
        #logger.debug('onButExpandShrink', __name__)
        but = self.but_expand_shrink

        if but.text() == self.char_expand :
            #print 'Click on expand button'
            but.setText(self.char_shrink)
            self.onClickExpand()
            self.parent.onItemExpand(self)

        elif but.text() == self.char_shrink :
            #print 'Click on shrink button'
            but.setText(self.char_expand)
            self.onClickShrink()
            self.parent.onItemShrink(self)

        self.updateButtons()

    def onClickExpand(self):
        logger.debug('onClickExpand', __name__)
        #self.gui_add = QtGui.QLabel('Additional information')
        self.gui_add = GUIDarkListItemAdd(self, self.str_run_num)
        self.hboxW.addWidget(self.gui_add)

        self.setFixedHeight( self.height() + self.gui_add.height() )

        #self.gui_add.setMinimumHeight( self.height() + self.gui_add.height() )
        #self.hboxW.addStretch(1)
        #self.gui_add.setStyleSheet(cp.styleYellowish)
        #self.gui_run.setStyleSheet(cp.styleYellowish)

    def onClickShrink(self):
        logger.debug('onClickShrink', __name__)

        #self.gui_add.setVisible(False)
        self.hboxW.removeWidget(self.gui_add)

        try    : self.gui_add.close()
        except : pass

        try    : del self.gui_add
        except : pass

        self.gui_add = None

        self.gui_run.setStyleSheet(cp.styleBkgd)
        self.setFixedHeight(35)
        #self.resize(740,35)
        #self.updateGeometry()

    def getHeight(self):
        logger.debug('getHeight', __name__)
        h = self.gui_run.height()
        if self.gui_add is not None :
            h += self.gui_add.getHeight()
        return h + 10

    def updateButtons(self, str_run_type='', comment='', xtc_in_dir=True) :
        #logger.info('update', __name__)
        self.gui_run.updateButtons(str_run_type, comment, xtc_in_dir)

    def getStrRunNum(self) :
        return self.str_run_num

    def getRunNum(self) :
        return int(self.str_run_num)

#------------------------------

if __name__ == "__main__" :
    import sys
    app = QtWidgets.QApplication(sys.argv)
    w = GUIDarkListItem(parent=None, str_run_num='0016')
    w.setFieldsEnabled(True)
    w.show()
    app.exec_()

#------------------------------


from configurations import *
from models import obtain_model_cfg
from detectron2.config import get_cfg
import argparse
import os
from loaders import register_dataset
from detectron2.engine.hooks import PeriodicCheckpointer
from detectron2.engine import DefaultTrainer
from new_heads import *
from detectron2.utils.logger import setup_logger

dict_scheduler = {
    "multistep": ("WarmupMultiStepLR", 1000),
    "cosine": ("WarmupCosineLR", 1000),
    "multistep_nowarmup": ("WarmupMultiStepLR", 0)
}

def main(parser):
    # Reproducible results
    torch.manual_seed(0)

    # load dataset
    register_dataset(use_mots_challenge=parser.motschallenge==1)
    print("Dataset loaded...")

    cfg = get_cfg()
    # Basic config
    cfg = basic_configuration(cfg)
    # Modify configuration to use certain model
    cfg = obtain_model_cfg(cfg, parser.model_name, not parser.scratch)
    if parser.balanced_weights:
        if parser.focal_loss:
            cfg = modify_head_focalloss_class(cfg)
        else:
            cfg = modify_head_balanced_weight_class(cfg)

    cfg.INPUT.MASK_FORMAT = "bitmask"
    cfg.OUTPUT_DIR = f"/home/group01/W4_detectron2/{parser.experiment_name}"
    if not os.path.exists(cfg.OUTPUT_DIR):
        os.makedirs(cfg.OUTPUT_DIR)

    # SPECIFIC TO KITTI
    cfg.DATASETS.TRAIN = ("ds_train",)
    cfg.DATASETS.VAL = ("ds_val",)
    cfg.DATASETS.TEST = ("ds_val",)
    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 2 # for new ds

    # TRAINING PARAMS
    cfg.MODEL.BACKBONE.FREEZE_AT = parser.freeze_at # Where to freeze backbone layers to finetune
    cfg.DATALOADER.NUM_WORKERS = parser.workers
    cfg.SOLVER.IMS_PER_BATCH = parser.batch_size
    cfg.SOLVER.MAX_ITER = parser.max_steps
    cfg.SOLVER.WARMUP_ITERS = 0
    cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512

    # Min / max input size
    cfg.INPUT.MIN_SIZE_TRAIN = (parser.min_inp,)
    cfg.INPUT.MAX_SIZE_TRAIN = parser.max_inp

    # Cropping
    if parser.crop_size != 0:
        cfg.INPUT.CROP.ENABLED = True
        cfg.INPUT.CROP.SIZE = [parser.crop_size, parser.crop_size]

    # Lr scheduler
    if parser.lr_scheduler != "":
        cfg.SOLVER.STEPS = [3000, 4000]
        cfg.SOLVER.LR_SCHEDULER_NAME, cfg.SOLVER.WARMUP_ITERS = dict_scheduler[parser.lr_scheduler]

    cfg.MODEL.RPN.IOU_THRESHOLDS = [parser.min_iou, parser.max_iou]
    cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE = parser.bbox_reg_loss

    #cfg.SOLVER.NESTEROV = True
    #cfg.SOLVER.MOMENTUM = 0.8
    #cfg.SOLVER.LR_SCHEDULER_NAME, cfg.SOLVER.WARMUP_ITERS = ("WarmupCosineLR", 1000)
    cfg.SOLVER.BASE_LR = parser.lr
    #cfg.SOLVER.STEPS = [40000, 45000] # quick test

    # CHECKPOINTS AND EVAL
    cfg.TEST.EVAL_PERIOD = 5000
    cfg.SOLVER.CHECKPOINT_PERIOD = 2500
    setup_logger(output=cfg.OUTPUT_DIR, name="train")

    #cfg = prepare_dirs_experiment(cfg, parser.experiment_name, split=0)
    if parser.use_da:
        trainer = DataAugmTrainer(cfg)
    else:
        trainer = CustomDefaultTrainer(cfg)

    trainer = register_validation_loss_hook(cfg, trainer)
    trainer.build_hooks()
    trainer.build_writers()

    #checkpointer = PeriodicCheckpointer()
    #trainer.register_hooks([checkpointer])

    # Train...
    trainer.resume_or_load(resume=parser.resume) # if true => load last checkpoint if available (and start training from there)
    print("Trainer loaded and hooks registered...")
    trainer.train()
    print("Training finished...")
    #trainer.test()

def check_args():
    parser = argparse.ArgumentParser()

    #TODO: option to select one model or another
    parser.add_argument(
        "--experiment_name",
        type=str,
        default="test",
        help="experiment name",
    )

    parser.add_argument(
        "--resume",
        action="store_true",
        default=False,
        help="if model will be resumed from last checkpoint if exists",
    )

    parser.add_argument(
        "--use_da",
        action="store_true",
        default=False,
        help="use data augm",
    )

    parser.add_argument(
        "--balanced_weights",
        action="store_true",
        default=False,
        help="balance class imbalance",
    )

    parser.add_argument(
        "--scratch",
        action="store_true",
        default=False,
        help="whether to train from scratch",
    )

    parser.add_argument(
        "--focal_loss",
        action="store_true",
        default=False,
        help="focal loss",
    )

    parser.add_argument(
        "--lr",
        type=float,
        default=0.001,
        help="learning rate",
    )

    parser.add_argument(
        "--batch_size",
        type=int,
        default=4,
        help="batch_size",
    )

    parser.add_argument(
        "--workers",
        type=int,
        default=10,
        help="batch_size",
    )

    parser.add_argument(
        "--model_name",
        type=str,
        default="resnet_fpn",
        help="model name",
    )

    parser.add_argument(
        "--freeze_at",
        type=int,
        default=0,
        help="layers to freeze in backbone",
    )

    parser.add_argument(
        "--max_steps",
        type=int,
        default=15000,
        help="max steps to train",
    )

    parser.add_argument(
        "--motschallenge",
        type=int,
        default=1,
        help="include motschallenge. 0 = no, 1 = yes",
    )

    # Other config
    parser.add_argument(
        "--min_inp",
        type=int,
        default=800,
        help="min input",
    )

    parser.add_argument(
        "--max_inp",
        type=int,
        default=1333,
        help="max input",
    )

    parser.add_argument(
        "--crop_size",
        type=float,
        default=0,
        help="crop range",
    )

    parser.add_argument(
        "--lr_scheduler",
        type=str,
        default="",
        help="lr scheduler",
    )

    parser.add_argument(
        "--bbox_reg_loss",
        type=str,
        default="smooth_l1",
        help="bbox reg loss",
    )

    parser.add_argument(
        "--min_iou",
        type=float,
        default=0.3,
        help="min iou",
    )

    parser.add_argument(
        "--max_iou",
        type=float,
        default=0.7,
        help="max iou",
    )

    return parser.parse_args()

if __name__ == "__main__":
    parser = check_args()
    main(parser)


# -*- coding: utf-8 -*-
from django.core.management.base import BaseCommand
from django.db.models import Q
from django.utils import timezone

from ...models import Trip, TripUserRelated
from accounts.models import User

from operator import itemgetter

class Command(BaseCommand):
    """
    Сортировка похожих путешествий в ЛК
    Сортировка двух типов:
    1. Сортировка для самого пользователя осуществляется на основании трёх списков:
    - список путешествий, в которых больше всего участвует пользователей, с которыми пользователь участвовал в других путешествиях;
    - список путешествий, которые совпадают с предпочтительными местами пользователя (указанные в личном кабинете);
    - список путешествий, которые совпадают с предпочтительными датами пользователя (указанные в личном кабинете).
    В первую очередь отображаем результаты, включающие все три совпадения, затем по любым двум совпадениям, далее по одному.
    2. Сортировка для посетителей личного кабинета пользователя осуществляется по дате (по возрастанию)
    """

    def handle(self, *args, **options):
        for user in User.objects.all():
            trips = Trip.objects.filter(active=True)\
                                .filter(Q(organizers=user) | Q(members=user))
            user_trip_ids = []

            # список путешествий (см описание класса)
            trips1 = {}
            trips2 = []
            trips3 = []

            related_users = []  # все юзеры, с которыми мы когда-либо путешествовали
            for trip in trips:
                for organizer in trip.organizers.all():
                    if organizer != user and trip.creator != user:
                        related_users.append(organizer)
                for member in trip.members.all():
                    if member != user and trip.creator != user:
                        related_users.append(member)

            if related_users:
                trips = Trip.objects.filter(active=True, end_date__gt=timezone.now())\
                                    .exclude(Q(organizers=user) | Q(members=user) | Q(creator=user))
                for trip in trips:
                    for rel_user in related_users:
                        if rel_user in trip.organizers.all() or rel_user in trip.members.all():
                            if trip in trips1:
                                trips1[trip] += 1
                            else:
                                trips1[trip] = 1

            trips1 = sorted(trips1.items(), key=itemgetter(1), reverse=True)
            trips1 = [x[0] for x in trips1]

            for destination in user.user_places.all():
                trips = Trip.objects.filter(active=True,
                                            destinations=destination.name)\
                                    .exclude(Q(organizers=user) | Q(members=user) | Q(creator=user))
                trips2 = [x for x in trips]

            for date in user.user_dates.all():
                trips = Trip.objects.filter(active=True)\
                                    .filter(Q(start_date__lte=date.start_date) |
                                            Q(end_date__gte=date.end_date))\
                                    .exclude(Q(organizers=user) | Q(members=user) | Q(creator=user))
                trips3 = [x for x in trips]

            trips_level1 = list(set(trips1) & set(trips2) & set(trips3))
            for trip in trips_level1:
                try:
                    p = TripUserRelated.objects.get(trip=trip, user=user)
                    p.level = 1
                    p.save()
                except TripUserRelated.DoesNotExist:
                    p = TripUserRelated(trip=trip, user=user, level=1)
                    p.save()
                user_trip_ids.append(p.pk)

            trips_level2 = []
            if trips1 and trips2:
                trips_level2 = [x for x in list(set(trips1) & set(trips2)) if x not in trips_level1]
            elif trips2 and trips3:
                trips_level2 = [x for x in list(set(trips2) & set(trips3)) if x not in trips_level1]
            elif trips1 and trips3:
                trips_level2 = [x for x in list(set(trips1) & set(trips3)) if x not in trips_level1]
            for trip in trips_level2:
                try:
                    p = TripUserRelated.objects.get(trip=trip, user=user)
                    p.level = 2
                    p.save()
                except TripUserRelated.DoesNotExist:
                    p = TripUserRelated(trip=trip, user=user, level=2)
                    p.save()
                user_trip_ids.append(p.pk)

            trips_level3 = []
            if trips1:
                trips_level3 = [x for x in list(set(trips1)) if x not in list(set(trips_level1) | set(trips_level2))]
            elif trips2:
                trips_level3 = [x for x in list(set(trips2)) if x not in list(set(trips_level1) | set(trips_level2))]
            elif trips3:
                trips_level3 = [x for x in list(set(trips3)) if x not in list(set(trips_level1) | set(trips_level2))]
            for trip in trips_level3:
                try:
                    p = TripUserRelated.objects.get(trip=trip, user=user)
                    p.level = 3
                    p.save()
                except TripUserRelated.DoesNotExist:
                    p = TripUserRelated(trip=trip, user=user, level=3)
                    p.save()
                user_trip_ids.append(p.pk)

            TripUserRelated.objects.filter(user=user)\
                                   .exclude(pk__in=user_trip_ids)\
                                   .delete()


import matplotlib.pyplot as plt
import numpy as np
import os
import random
import pickle
import cv2
import random
import tensorflow as tf
from pathlib import Path
from sklearn.utils import shuffle
import tensorflow.keras
from tensorflow.keras import applications, layers, losses, optimizers, metrics
from tensorflow.keras.models import Sequential, Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow.keras.backend as K
from keras.layers.core import Lambda,Dense
import time

input = 224
path_data1 = "/home/fproenca/Tese/Data/Market/x.pickle"
path_data2 = "/home/fproenca/Tese/Data/Market/labels.pickle"
app = '/home/fproenca/Tese/M/junk/'
model_name = '/home/fproenca/Tese/Results/Market/MobileNet_Class_224.h5'
call_name = '/home/fproenca/Tese/Results/Market/triplet.h5'
path_data1_total = "/home/fproenca/Tese/Data/Market/x_total.pickle"
path_data2_total = "/home/fproenca/Tese/Data/Market/labels_total.pickle"
aug_val = 1
split_val = 11763
################################################################################
def get_mobile_net(x):
	model = load_model(model_name)
	embedding = Sequential(name="Embedding")
	for layer in model.layers[:-2]: # go through until last layer
	    embedding.add(layer)
	trainable = False
	for layer in embedding.layers:
 	 if layer.name == 'mobilenet_1.00_224':
 	   for i in layer.layers:
  	    if i.name == "conv1":
  	      trainable = True
  	    i.trainable = trainable
	embedding.add(Lambda(lambda x: K.l2_normalize(x,axis=1)))
	embedding.summary()
	feature_vectors = []
	i=0
	for xi in x:
	  feature_vectors.append(embedding.predict(np.expand_dims(xi, axis=0)))
	  i+=1
	for j in range(len(feature_vectors)):
		feature_vectors[j] = feature_vectors[j][0]
	feature_vectors = np.array(feature_vectors)
	return feature_vectors
################################################################################
def load_data():
  pickle_in = open(path_data1,"rb")
  x = pickle.load(pickle_in)
  x = get_mobile_net(x)
  pickle_in = open(path_data2,"rb")
  labels = pickle.load(pickle_in)
  return x,labels
################################################################################
data_augmentation = tf.keras.Sequential([
  layers.experimental.preprocessing.RandomFlip("horizontal"),
  layers.experimental.preprocessing.RandomRotation(0.1),
  layers.experimental.preprocessing.RandomZoom(0.1),
  layers.experimental.preprocessing.RandomContrast(0.15),
  layers.experimental.preprocessing.RandomTranslation(0.1,0.1,fill_mode='nearest',interpolation='bilinear'),
])

def aug_data(images,labels):
  aug_img = []
  labels_aug = []
  for idxA in range(len(images)-1):
      # grab the current image and label belonging to the current
      # iteration
      currentImage = images[idxA]
      image = tf.expand_dims(currentImage, 0)
      label = labels[idxA]
      for idxB in range(aug_val):
        augmented_image = data_augmentation(image)
        aug_img.append(augmented_image[0])
        labels_aug.append(label)
  print(len(labels_aug))
  return np.array(aug_img),np.array(labels_aug)
################################################################################
x, labels = load_data()

x_val = x[split_val:]
labels_val = labels[split_val:]
x_train = x[:split_val]
labels_train = labels[:split_val]

#aug_img,aug_label = aug_data(x_train,labels_train)
#x_train = np.concatenate((x_train, aug_img))
#labels_train = np.concatenate((labels_train, aug_label))

x_train_ =[]
x_val_ =[]

cnt = 0
for idx,image in enumerate(x_train):
  name = app + str(cnt) + 'i.npy'
  Image = np.save(name,image)
  x_train_.append(name)
  cnt +=1
for idx,image in enumerate(x_val):
  name = app + str(cnt) + 'i.npy'
  Image = np.save(name,image)
  x_val_.append(name)
  cnt +=1

x_val = x_val_
x_train = x_train_
################################################################################
def loss1(triplet_file):

  tripletImages=[]
  triplet = [np.load(triplet_file[0]) , np.load(triplet_file[1])]
  tripletImages.append(triplet)
  tripletImages = np.array(tripletImages)
  #ap_distance = np.linalg.norm(tripletImages[0,0]  - tripletImages[0,1])
  ap_distance = tf.reduce_sum(tf.square(tripletImages[0,0]- tripletImages[0,1]), -1)

  return ap_distance.numpy()

def siamese_calc(anchor, positive, negative):
  #ap_distance = tf.reduce_sum(tf.square(anchor - positive), -1)
  #print(ap_distance.numpy())
  ap_distance = np.linalg.norm(anchor  - positive)
  #print(ap_distance)
  #an_distance = tf.reduce_sum(tf.square(anchor - negative), -1)
  #print(an_distance.numpy())
  an_distance = np.linalg.norm(anchor  - negative)
  #print(an_distance)

  return (ap_distance.numpy(), an_distance.numpy())

def test_loss(ap_distance, an_distance):
  loss = ap_distance - an_distance
  #loss = max(loss + 1.0, 0.0)
  return loss

def make_triplets1(images, labels,flag):
  # initialize two empty lists to hold the (image, image) pairs and
	# labels to indicate if a pair is positive or negative
  tripletImages = []
  loss_vector = []
	# calculate the total number of classes present in the dataset
	# and then build a list of indexes for each class label that
	# provides the indexes for all examples with a given label
  numClasses = len(np.unique(labels))
  idx = [np.where(labels == i)[0] for i in range(0, numClasses+1)]
  cnt = 0
  aux_cnt = 0
  stri = '.npy'
  if flag == 1:
    stri = '_.npy'
  for idxA in range(len(images)-1):
      # grab the current image and label belonging to the current
      # iteration
      currentImage = images[idxA]
      label = labels[idxA]
      for idxB in range(idxA+1,len(images)):
        loss = 0.0
        posImage = images[idxB]
        label_ = labels[idxB]
        #print(idxB)
        #print(label_)
        if (label == label_):
            print("Current:  " + str(currentImage) + " || Positive:  " + str(posImage))
            negIdx = np.where(labels != label)[0]
            #print(len(negIdx))
            ap = [currentImage, posImage]
            ap_distance = loss1(ap)
            for s in negIdx:
              negImage = images[s]
              an = [currentImage,negImage]
              an_distance = loss1(an)
              loss = test_loss(ap_distance,an_distance)
              if (loss+0.7) > 0.0 :
                tripletImage = [currentImage, posImage, negImage]
              	#print(negImage)
                name = app + str(cnt) + stri
                np.save(name,tripletImage)
                tripletImages.append(name)
                loss_vector.append(loss)
                #print("Current:  " + str(currentImage) + " || Positive:  " + str(posImage) + "|| Negative:  " + str(negImage) + "|| LOSS:  " + str(loss))
                cnt +=1
            aux_cnt += 1
            print("ID: " +  str(label_) + "  Counter: " + str(aux_cnt))
  return (np.array(tripletImages),loss_vector)

triplet_val, val_loss = make_triplets1(x_val, labels_val,1)

pickle_out = open("/home/fproenca/Tese/Data/Market/triplet_val.pickle","wb")
pickle.dump(triplet_val, pickle_out)
pickle_out.close()

pickle_out = open("/home/fproenca/Tese/Data/Market/val_loss.pickle","wb")
pickle.dump(val_loss, pickle_out)
pickle_out.close()