import telebot
import crawler
from time import sleep

link = ''

bot = telebot.TeleBot('442709449:AAEZLaE3Ja3K_HToUfDEB67z4hab-NEGII8')
shop_name = ''

def show_keyboard(type):
    keyboard = telebot.types.ReplyKeyboardMarkup(resize_keyboard=True)
    if type is 'shops':
        keyboard.row('/stop')
        keyboard.row('Ашан', 'Дикси')
        return keyboard, 'Выберите магазин из списка:'
    elif type is 'options':
        keyboard.row('/stop')
        keyboard.row('Количество акций')
        keyboard.row('Список наименований')
        keyboard.row('Отслеживать изменения')
        return keyboard, 'Выберите опцию:'
    elif type is 'categoryDixy':
        keyboard.row('/stop')
        # keyboard.row('Все')
        keyboard.row('Овощи и фрукты', 'Консервы, соусы')
        keyboard.row('Кофе, чай', 'Детское питание')
        keyboard.row('Хлеб, торты', 'Молочная гастрономия')
        keyboard.row('Мясная гастрономия','Мясо, яйцо')
        keyboard.row('Кондитерские изделия')
        keyboard.row('Кулинария, заморозка, мороженое')
        keyboard.row('Напитки', 'Крупы, завтраки, специи')
        keyboard.row('Непродовольственные товары')
        return keyboard, 'Выберите категорию:'
    elif type is 'categoryAuchan':
        keyboard.row('/stop')
        # keyboard.row('Все')
        keyboard.row('Игрушки', 'Мебель')
        keyboard.row('Дача', 'Электроника')
        keyboard.row('Детям', 'Спорт')
        keyboard.row('Горящие цены')
        keyboard.row('Зоотовары','Кухня')
        keyboard.row('Детская одежда', 'Все остальное', 'DVD')
        return keyboard, 'Выберите категорию:'
    elif type is 'start_button':
        keyboard.row('/start')
        return keyboard
    elif type is 'stop_button':
        keyboard.row('/stop')
        return keyboard

def hide_keyboard():
    keyboard = telebot.types.ReplyKeyboardRemove()
    return keyboard, 'До встречи!'

@bot.message_handler(commands=['start'])
def handle_start(msg):
    bot.send_message(msg.from_user.id, show_keyboard('shops')[1], reply_markup=show_keyboard('shops')[0])

@bot.message_handler(commands=['stop'])
def handle_stop(msg):
    bot.send_message(msg.from_user.id, hide_keyboard()[1], reply_markup=hide_keyboard()[0])
    bot.send_message(msg.from_user.id, 'Ждем Вас снова!', reply_markup=show_keyboard('start_button'))

@bot.message_handler(content_types=['text'])
def handle_text(msg):
    global shop_name
    if msg.text == 'Ашан':
        shop_name = 'Ашан'
        bot.send_message(msg.from_user.id, '...', reply_markup=hide_keyboard()[0])
        bot.send_message(msg.from_user.id, show_keyboard('options')[1], reply_markup=show_keyboard('options')[0])
        link = crawler.auchan

    elif msg.text == 'Дикси':
        shop_name = 'Дикси'
        bot.send_message(msg.from_user.id, '...', reply_markup=hide_keyboard()[0])
        bot.send_message(msg.from_user.id, show_keyboard('options')[1], reply_markup=show_keyboard('options')[0])
        link = crawler.dixy

    elif msg.text == 'Количество акций':
        if shop_name == 'Ашан':
            bot.send_message(msg.from_user.id, crawler.get_number_of_sales(crawler.auchan))
        elif shop_name == 'Дикси':
            bot.send_message(msg.from_user.id, crawler.get_number_of_sales(crawler.dixy))

    elif msg.text == 'Список наименований':
        bot.send_message(msg.from_user.id, '...', reply_markup=hide_keyboard()[0])
        if shop_name == 'Ашан':
            bot.send_message(msg.from_user.id, show_keyboard('categoryAuchan')[1], reply_markup=show_keyboard('categoryAuchan')[0])
        elif shop_name == 'Дикси':
            bot.send_message(msg.from_user.id, show_keyboard('categoryDixy')[1], reply_markup=show_keyboard('categoryDixy')[0])

    elif msg.text == 'Отслеживать изменения':
        bot.send_message(msg.from_user.id, '...следим...', reply_markup=show_keyboard('stop_button'))
        if shop_name == 'Ашан':
            if crawler.notify_auchan() == 1:
                bot.send_message(msg.from_user.id, 'Есть обновления! Прекращаем отслеживание.', reply_markup=hide_keyboard()[0])
                bot.send_message(msg.from_user.id, 'Для возобновления введите команду снова.',  reply_markup=show_keyboard('start_button'))
                crawler.update_auchan()
        elif shop_name == 'Дикси':
            if crawler.notify_dixy() == 1:
                bot.send_message(msg.from_user.id, 'Есть обновления! Прекращаем отслеживание.', reply_markup=hide_keyboard()[0])
                bot.send_message(msg.from_user.id, 'Для возобновления введите команду снова.',  reply_markup=show_keyboard('start_button'))
                crawler.update_dixy()

    elif msg.text in crawler.d_item:
        reply_str = crawler.reply(msg.text, crawler.dixy_items)
        for message in reply_str.split('#'):
            bot.send_message(msg.from_user.id, message)

    elif msg.text + ' ' in crawler.a_item:
        reply_str = crawler.reply(msg.text, crawler.auchan_items)
        for message in reply_str.split('#'):
            bot.send_message(msg.from_user.id, message)

# Запускаем бота
bot.polling(none_stop=True)


import requests
import re
import os

if __name__ == '__main__':
    url = "https://www.qiushibaike.com/imgrank/"

    # make a folder to store images
    if not os.path.exists('../qiutuImages'):
        os.mkdir('../qiutuImages')

    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:84.0) Gecko/20100101 Firefox/84.0'
    }

    # use universal spider to retrieve html
    page_sc = requests.get(url=url, headers=headers).text

    #use focus spider to retrieve all images from that html

    # note:
    #

    #
    #
    #
    #

    ex = '
.*?<img src="(.*?)" alt.*?
'
    img_src_list = re.findall(ex, page_sc, re.S)
    # print(img_src_list)

    for img_origin_src in img_src_list:
        img_src = 'https:' + img_origin_src
        # got the image binary content
        img_bin = requests.get(img_src, headers=headers).content

        # generate name
        img_name = img_src.split('/')[-1]
        imgPath = '../qiutuImages/' + img_name + '.jpg'
        with open(imgPath, 'wb') as fp:
            fp.write(img_bin)
            print(img_name + ' downloaded')


from django.shortcuts import render
from django.http import Http404
from django.http import HttpResponse
from stack_it.models import Page
from django.views import View

class StackItView(View):
    def get(self, request, *args, **kwargs):
        self.object = self.get_object(request, **kwargs)
        return render(request, self.object.template_path, self.get_context_data())

    def get_object(self, request, *args, **kwargs):
        path = request.get_full_path()
        print(path)
        if len(path) > 0 and path[-1] != "/":
            path += "/"
        try:
            page = Page.objects.get(ref_full_path=path)
        except Page.DoesNotExist:
            raise Http404()
        return page

    def get_context_data(self, **kwargs):
        ctx = self.object.get_context_data(**kwargs)
        print(ctx)

        ctx.update({"page": self.object})
        return ctx


from typing import List
from math import ceil
from more_itertools import chunked
import uuid
from pathlib import Path
from mdparse.parser import transform_pre_rules, compose
from torch.nn.utils.rnn import pad_sequence
from fastai.text.transform import defaults
from fastai.core import PathOrStr, parallel
from fastai.basic_train import load_learner
from fastai.text import TextLMDataBunch as lmdb
from fastai.text.data import TokenizeProcessor
from torch import Tensor, tensor, split, cat as torch_cat
from numpy import concatenate as cat, stack
from torch.cuda import empty_cache
import pandas as pd
from tqdm.auto import tqdm
import numpy as np
import logging

def pass_through(x):
    """Avoid messages when the model is deserialized in fastai library."""
    return x

class InferenceWrapper:
    "Utility to aid with generating a document embedding from the Title and the Body of a GitHub Issue."

    def __init__(self,
                 model_path:PathOrStr,
                 model_file_name:PathOrStr):
        """Load the Learner object from model_path/model_file_name.
        Args:
          model_path: The path (directory) of the Learner object.
                      e.g., ./model_files
          model_file_name: The file name of the Learner object.
                           e.g., model.pkl
        """
        self.learn = load_learner(path=model_path, file=model_file_name)
        self.learn.model.eval()  # turn off dropout, etc. only need to do this after loading model.
        self.encoder = self.learn.model[0]
        self.pad_idx = self.learn.data.pad_idx
        self.model_tokenizer = [x.tokenizer for x in self.learn.data.processor if type(x) == TokenizeProcessor][0]
        self.path = Path(f'./inference_utils/{str(uuid.uuid4())}')
        self.vocab = self.learn.data.vocab

    @staticmethod
    def parse(x: str) -> str:
        """
        Pre-process the text (markdown annotation and cleanup) prior to tokenizing.

        This method is meant to be applied to the GitHub issue title and body seperately.
        """
        return compose(transform_pre_rules+defaults.text_pre_rules)(x)

    def numericalize_one(self, x:str) -> Tensor:
        """Convert text to a series of integers in preparation for inference."""
        return self.learn.data.one_item(x)[0]

    def _forward_pass(self, x:Tensor) -> Tensor:
        self.encoder.reset()
        return self.encoder.forward(x)[-1][-1].detach().cpu().numpy()

    def get_raw_features(self, x:str) -> Tensor:
        """
        Get features from encoder of the language model.

        Returns Tensor of the shape (1, sequence_length, ndim)
        """
        seq_ints = self.numericalize_one(x)
        self.encoder.reset() # so the hidden states reset between predictions

        return self.encoder.forward(seq_ints)[-1][-1]

    def get_pooled_features(self, x:str) -> Tensor:
        """
        Get concatenation of [mean, max, last] of last hidden state.

        Parameters
        ----------
        x: str
            this is the pre-processed string associated with the issue.
            If you have two seperate fields "title" and "body" you will want
            to pre-process these fields with the process_dict method before calling
            this.

        Returns
        -------
        Tensor
            This is an embedding in the form of a Tensor with the shape (1, 2400)
        """
        raw = self.get_raw_features(x)
        # return [mean, max, last] with size of (1, self.learn.emb_sz * 3)
        return torch_cat([raw.mean(dim=1), raw.max(dim=1)[0], raw[:,-1,:]], dim=-1)

    @classmethod
    def process_dict(cls, dfdict:dict) -> dict:
        """
        Transform the text in a dictionary containing these keys:
        - title: the title of the GitHub issue
        - body: the body, not including any comments

        This method will combine these two fields into one string wtih markup deleniating
        the title and body fields, as well as identify other artifacts that might occur
        in markdown (see https://github.com/machine-learning-apps/mdparse for references.)

        Parameters
        ---------
        dfdict: dict
            Example:  {'title': "This is the title", 'body': "This is the body"}

        Returns
        -------
        dict
            Example: {'text': 'xxxfldtitle This is the title xxxfldbody This is the body'}

        """
        assert 'title' in dfdict, 'Missing the field "title"'
        assert 'body' in dfdict, 'Missing the field "body"'
        title = dfdict['title']
        body = dfdict['body']
        try:
            text = 'xxxfldtitle '+ cls.parse(title) + ' xxxfldbody ' + cls.parse(body)
        except Exception as e:
            logging.error(f"Exception occurred in process_dict {e}")
            return {'text': 'xxxUnk'}
        return {'text': text}

    @classmethod
    def process_df(cls, dataframe:pd.DataFrame) -> pd.DataFrame:
        """Loop through a pandas DataFrame and create a single text field."""
        lst = []
        for d in tqdm(dataframe.to_dict(orient='rows'), desc="Tokenizing and parsing text:"):
            lst.append(cls.process_dict(d))

        df = pd.DataFrame(lst)
        return df

    def df_to_embedding(self, dataframe:pd.DataFrame, bs=100) -> np.ndarray:
        """
        Retrieve document embeddings for a dataframe with the columns `title` and `body`.
        Uses batching for effiecient computation, which is useful when you have many documents
        to retrieve embeddings for.

        Paramaters
        ----------
        dataframe: pandas.DataFrame
            Dataframe with columns `title` and `body`, which reprsent the Title and Body of a
            GitHub Issue.
        bs: int
            batch size for doing inference.  Set this variable according to your available GPU memory.
            The default is set to 200, which was stable on a Nvida-Tesla V-100.

        Returns
        -------
        numpy.ndarray
            An array with of shape (number of dataframe rows, 2400)
            This numpy array represents the latent features of the GitHub issues.

        Example
        -------
        >>> import pandas as pd
        >>> wrapper = InferenceWrapper(model_path='/path/to/model',
                                   model_file_name='model.pkl')
        # load 200 sample GitHub issues
        >>> testdf = pd.read_csv(f'https://bit.ly/2GDY5NY').head(200)
        >>> embeddings = wrapper.df_to_embedding(testdf)

        >>> embeddings.shape
        (200, 2400)
        """
        new_df = self.process_df(dataframe)
        # to get the benefit of batching similar length sequences together, have a minimum of 20 batches
        bs = min(bs, (len(new_df) // 20) + 1)

        # use the machinery of the data block to numericalize text in parallel
        data_lm = lmdb.from_df(path=self.path,
                               train_df=new_df.head(), # train_df gets sample data only
                               valid_df=new_df,
                               text_cols='text',
                               tokenizer=self.model_tokenizer,
                               vocab=self.vocab)

        # extract numericalized arrays and convert to pytorch
        docs = data_lm.valid_dl.x.items
        lengths = []
        numericalized_docs = []
        for arr in docs:
            numericalized_docs.append(tensor(arr).cuda()) # convert to torch.Tensor
            lengths.append(arr.shape[0])

        # sort the data by sequence length and assemble batches
        length_arr = np.array(lengths)
        len_mask = length_arr.argsort()
        len_mask_reversed = len_mask.argsort()
        ordered_features = [numericalized_docs[i] for i in len_mask]
        ordered_lengths = length_arr[len_mask]

        # perform model inference
        hidden_states_batched = []
        pooled_states = []
        i = 0
        total = len(numericalized_docs)
        while i < total:
            logging.info(f'Model inference: {i} / {total}')
            try:
                # pad the batch to the same length
                bp = pad_sequence(ordered_features[i:i+bs], batch_first=True, padding_value=self.pad_idx)
                # perform inference
                hidden_states = self._forward_pass(bp)
                empty_cache()
                # fetch the summary of the hidden states as the embedding
                pooled_states.append(self.batch_seq_pool(hidden_states, ordered_lengths[i:i+bs]))
                i += bs
            except RuntimeError as e:
                # encounter CUDA out of memory error
                if bs == 1:
                    logging.error(f'Batch size has been 1, can not feed the size {ordered_lengths[i]} to CUDA')
                    raise Exception(e)
                # decrease the batch size because of CUDA out of memory
                # not sure whether there is a good way to find a new batch size
                bs = bs // 2
                logging.info(f'CUDA out of memory, the new batch size is {bs}')
                empty_cache()

        # restore the original order of the data by unsorting
        pooled_states = cat(pooled_states)[len_mask_reversed, :]
        assert pooled_states.shape[0] == length_arr.shape[0] == len(dataframe)

        return pooled_states

    @classmethod
    def batch_seq_pool(cls, seq_emb:Tensor, lengths=List[int]):
        """
        Concatenate the mean, max and last hidden representations of a batch of sequences.

        Parameters
        ----------
        seq_emb: Tensor
            Tensor of shape (bs, sequence length, dimension)
            This tensor reprsents the hidden states of final layer of the encoder from a language model.
        lengths: List
            list of integers indicating the sequence lengths

        Returns
        -------
        Tensor
            Tensor of size (bs, 2400)
        """
        assert seq_emb.shape[0] == len(lengths), 'Number of elements in lengths should match the first dimension of seq_emb'

        # ignore information beyond the sequence length, which is padding
        embs = [seq_emb[i, :x, :] for i, x in enumerate(lengths)]

        # calculate the pooled features ignoring the padding
        features = [cat([emb.mean(axis=0), emb.max(axis=0), emb[-1,:]], axis=-1) for emb in embs]
        combined_features = stack(features)

        # check that the dimensionality of the document embedding is 3x the dimensionality of the
        # final hidden states of the encoder.
        assert combined_features.shape[-1] == (seq_emb.shape[-1] * 3)

        return combined_features


"""Fixed Database many to many relationship.
Many to many relationship must be between Database and ReportResult classes

Revision ID: 73b182e8bf5c
Revises: d21928bd8ebe
Create Date: 2016-03-03 01:19:47.075279

"""

# revision identifiers, used by Alembic.
revision = '73b182e8bf5c'
down_revision = 'd21928bd8ebe'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('database_results',
    sa.Column('database_id', sa.Integer(), nullable=True),
    sa.Column('result_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['database_id'], ['databases.id'], ),
    sa.ForeignKeyConstraint(['result_id'], ['results.id'], )
    )
    op.drop_table('database_report')
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('database_report',
    sa.Column('database_id', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.Column('report_id', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.ForeignKeyConstraint(['database_id'], ['databases.id'], name='database_report_database_id_fkey'),
    sa.ForeignKeyConstraint(['report_id'], ['reports.id'], name='database_report_report_id_fkey')
    )
    op.drop_table('database_results')
    ### end Alembic commands ###


from django.conf import settings
from django.conf.urls.static import static
from django.conf.urls.i18n import i18n_patterns
from django.contrib import admin
from django.urls import path, include

urlpatterns = [
    path('admin/', admin.site.urls),
    path('cart/', include('cart.urls', namespace='cart')),
    path('orders/', include('orders.urls', namespace='orders')),
    path('accounts/', include('allauth.urls')),
    path('pages/', include('django.contrib.flatpages.urls')),
    path('i18n/', include('django.conf.urls.i18n')),
    path("", include("MyShop.urls"))
]

urlpatterns += i18n_patterns(
    path('accounts/', include('allauth.urls')),
    path('pages/', include('django.contrib.flatpages.urls'))
)

if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import numpy as np
import cv2
import os
from PIL import Image, ImageEnhance

os.chdir("C:/01.work/01.python/998.data/02.nn/char_rnn/99.id_cd")
img = Image.open('easy_img/195.jpg')
#
enhancer = ImageEnhance.Contrast(img)
enhancer.enhance(0.0).save(
    "out_img/ImageEnhance_Contrast_000.jpg")
enhancer.enhance(0.25).save(
    "out_img/ImageEnhance_Contrast_025.jpg")
enhancer.enhance(0.5).save(
    "out_img/ImageEnhance_Contrast_050.jpg")
enhancer.enhance(0.75).save(
    "out_img/ImageEnhance_Contrast_075.jpg")
enhancer.enhance(1.0).save(
    "out_img/ImageEnhance_Contrast_100.jpg")
def kk():

    cap = cv2.VideoCapture('vtest.avi')
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    # fgbg = cv2.createBackgroundSubtractorGMG()
    fgbg = cv2.BackgroundSubtractorMOG2()
    os.chdir("C:/01.work/01.python/998.data/02.nn/char_rnn/99.id_cd")
    # image = Image.open("easy_img/195.jpg")
    cap = cv2.VideoCapture('easy_img/20180710_172823.mp4')

    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(5,5))
    fgbg = cv2.createBackgroundSubtractorKNN()

    while(1):
        ret, frame = cap.read()

        fgmask = fgbg.apply(frame)
        fgmask = cv2.morphologyEx(fgmask, cv2.MORPH_OPEN, kernel)

        cv2.imshow('frame',fgmask)
        k = cv2.waitKey(30) & 0xff
        if k == 27:
            break

    cap.release()
    cv2.destroyAllWindows()


r'''
test_pipelines_4.py

Test plotSETI.
'''

import os
from tempfile import gettempdir
import pytest
from turbo_seti import run_pipelines

TESTDIR = gettempdir() + '/pipeline_testing/'
PLOTDIR = TESTDIR + 'plots/'

def execute_one(counter, args):
    print("\n====test_pipelines_4 [{}]================== args: {}".format(counter, args))
    rc = run_pipelines.main(args)
    print("\n====test_pipelines_4 [{}]================== rc: {}".format(counter, rc))
    return rc

@pytest.mark.order(2)
def test_pipelines_4():
    print('\n===== test_pipelines_4: BEGIN =====')

    args = [TESTDIR, "-o", PLOTDIR, "-f", "1", "-s", "25.0", "-c", "on"]
    rc = execute_one(1, args)
    assert(rc == 0)

    args = [TESTDIR, "-o", PLOTDIR, "-f", "2", "-s", "25.0", "-c", "on"]
    rc = execute_one(2, args)
    assert(rc == 0)

    args = [TESTDIR, "-o", PLOTDIR, "-f", "3", "-s", "25.0", "-c", "on"]
    rc = execute_one(3, args)
    assert(rc == 0)

    args = [TESTDIR, "-o", PLOTDIR, "-c", "off"]
    rc = execute_one(4, args)
    assert(rc == 86)

    args = [TESTDIR, "-o", PLOTDIR, "-c", "complex", "-n", "Rubbish"]
    rc = execute_one(5, args)
    assert(rc == 86)

    args = [TESTDIR, "-o", PLOTDIR, "-c", "complex", "-n", "VOYAGER-1"]
    rc = execute_one(6, args)
    assert(rc == 0)

    print('\n===== test_pipelines_4: END =====')

if __name__ == '__main__':
    test_pipelines_4()


from django.contrib.auth import get_user_model
from django.test import Client, TestCase

from posts.models import Group, Post

User = get_user_model()

class PostModelTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create_user(username='AA')
        cls.test_group = Group.objects.create(title='tests')
        cls.post = Post.objects.create(
            text='Тестовый текст',
            pub_date=Post.pub_date,
            author=cls.user,
            group=cls.test_group
        )

    def setUp(self):
        # Создаем неавторизованный клиент
        self.guest_client = Client()
        # Создаем второй клиент
        self.authorized_client = Client()
        # Авторизуем пользователя
        self.authorized_client.force_login(self.user)

    def test_verbose_name(self):
        """verbose_name в полях совпадает с ожидаемым."""
        post = PostModelTest.post
        field_verboses = {
            'text': 'текст поста',
            'group': 'группа',
        }
        for value, expected in field_verboses.items():
            with self.subTest(value=value):
                self.assertEqual(
                    post._meta.get_field(value).verbose_name, expected)

    def test_help_text(self):
        """help_text в полях совпадает с ожидаемым."""
        post = PostModelTest.post
        field_help_texts = {
            'text': 'Напишите, о чем ваш пост',
            'group': 'Выберите группу из списка',
        }
        for value, expected in field_help_texts.items():
            with self.subTest(value=value):
                self.assertEqual(
                    post._meta.get_field(value).help_text, expected)

    def test_str_post(self):
        """правильно ли отображается
        значение поля __str__ в модели Post"""
        post = PostModelTest.post
        expected_object_name = post.text
        self.assertEquals(expected_object_name, str(post.text))

class GroupModelTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.group = Group.objects.create(
            title='Тестовый заголовок',
            slug='test-slug',
            description='Тестовое описание',
        )

    def test_str_group(self):
        """правильно ли отображается
        значение поля __str__ в модели Group"""
        group = GroupModelTest.group
        expected_object_name = group.title
        self.assertEquals(expected_object_name, str(group.title))


##---------------------------archivo principal -------------------------------##
##-----Jessica Castillo, Juan David Garcia, Juan Francisco Suescun------------##

## Importamos las librerias
from __future__ import print_function, division
import os
import pandas as pd
from skimage import io, transform
import numpy as np
import matplotlib.pyplot as plt
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
import pdb

import copy
import time
import argparse
import os.path as osp
from PIL import Image

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchvision import models, datasets, transforms
from torch.autograd import Variable
import torch.utils.model_zoo as model_zoo
from dataloader import AmazonDataset, Rescale
import model
import utils
from tensorboard_logger import configure, log_value
from sklearn.metrics import fbeta_score

# Argumentos

parser = argparse.ArgumentParser(description='PyTorch resnet18 for Image Multiclassification')
parser.add_argument("--model", type=str, default='AmazonSimpleNet', help="model: AmazonSimpleNet")
parser.add_argument('--batch-size', type=int, default=32, metavar='N',
                    help='input batch size for training (default: 32)')
parser.add_argument('--test-batch-size', type=int, default=128, metavar='N',
                    help='input batch size for testing (default: 128)')
parser.add_argument('--epochs', type=int, default=15, metavar='N',
                    help='number of epochs to train (default: 15)')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                    help='learning rate (default: 0.01)')
parser.add_argument("-v", action='store_true', help="verbose")
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                    help='SGD momentum (default: 0.5)')
parser.add_argument('--gamma', type=float, default=2, metavar='M',
                    help='learning rate decay factor (default: 0.5)')
parser.add_argument("--patience", type=int, default=10, help="early stopping patience")
parser.add_argument('--input_size', type=int, default=256, metavar='N',
                    help='Input image size (default: 256)')
parser.add_argument('--no_cuda', action='store_true', default=False,
                    help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',
                    help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                    help='how many batches to wait before '
                         'logging training status')
parser.add_argument('--save', type=str, default='model.pt',
                    help='file on which to save model weights')

args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
cuda = not args.no_cuda and torch.cuda.is_available() # use cuda

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)
kwargs = {'pin_memory': True} if args.cuda else {}

# -------------------------- LOADING THE DATA --------------------------
# Data augmentation and normalization for training
# Just normalization for validation

print("Initializing Datasets and Dataloaders...")
data_path = '/home/jlcastillo/Proyecto/Database/Dataset/train-jpg'
# Create training, validation and test datasets
train_dataset = AmazonDataset('csv/train.csv', data_path,'csv/labels.txt', transform = transforms.Compose([Rescale((args.input_size, args.input_size)), transforms.ToTensor()]))
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = args.batch_size, shuffle = True, num_workers = 4)

#Val
val_dataset = AmazonDataset('csv/val.csv', data_path,'csv/labels.txt',transform = transforms.Compose([Rescale((args.input_size, args.input_size)), transforms.ToTensor()]))
val_loader = torch.utils.data.DataLoader(train_dataset, batch_size = args.batch_size, shuffle = True, num_workers = 4)

# TEST call your dataset function def __init__(self, csv_file, data_path, transform=None)
test_dataset = AmazonDataset('csv/test.csv', data_path, 'csv/labels.txt',transform = transforms.Compose([Rescale((args.input_size, args.input_size)), transforms.ToTensor()]))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size = args.batch_size, shuffle = True, num_workers = 4)

# check the size of your datatset
dataset_sizes = {}
dataset_sizes['train'] = len(train_dataset)
dataset_sizes['val'] = len(val_dataset)
dataset_sizes['test'] = len(test_dataset)
print('Training dataset size:', dataset_sizes['train'])
print('Validation dataset size:', dataset_sizes['val'])
print('Test dataset size:', dataset_sizes['test'])

# -------------------------- MODEL --------------------------
## URL`s a los pesos
RESNET_18 = 'https://download.pytorch.org/models/resnet18-5c106cde.pth'
RESNET_101 = 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth'

"""
model = models.resnet18(num_classes=17)
#resnet18.classifier = [nn.Linear(resnet18.fc.in_features, 17)]

#resnet101 = models.resnet101(pretrained = True, progress = True)

for param in model.parameters():
    param.requires_grad = True

if args.cuda:
    model.cuda()

load_model = False
if osp.exists(args.save):
    with open(args.save, 'rb') as fp:
        state = torch.load(fp)
        model.load_state_dict(state)
        load_model = True
else:

    ## Cargamos los pesos pre entrenados a las redes
    state = model_zoo.load_url(RESNET_18)
     # eliminate fully connected layers weights (trained for 1000 categories)
    state = {x: state[x] for x in state if not x.startswith('fc')}

    # current weights (not the pretrained model)
    model_state = model.state_dict()
    # update state_dict with the pretrained model
    model_state.update(state)
    # load weights into the model
    model.load_state_dict(model_state)
"""
"""
state101 = model_zoo.load_url(RESNET_101)
# current weights (not the pretrained model)
model_state101 = resnet101.state_dict()
# update state_dict with the pretrained model
model_state101.update(state18)
# load weights into the model
resnet101.load_state_dict(model_state101)
"""
###### Para salvar modelos y logs ##################
# Setup folders for saved models and logs
if not os.path.exists('saved-models/'):
    os.mkdir('saved-models/')
if not os.path.exists('logs/'):
    os.mkdir('logs/')

# Setup tensorboard folders. Each run must have it's own folder. Creates
# a logs folder for each model and each run.
out_dir = 'logs/{}'.format(args.model)
if not os.path.exists(out_dir):
    os.mkdir(out_dir)
run = 0
current_dir = '{}/run-{}'.format(out_dir, run)
while os.path.exists(current_dir):
	run += 1
	current_dir = '{}/run-{}'.format(out_dir, run)
os.mkdir(current_dir)
logfile = open('{}/log.txt'.format(current_dir), 'w')
print(args, file=logfile)

# Tensorboard viz. tensorboard --logdir {yourlogdir}. Requires tensorflow.
configure(current_dir, flush_secs=5)

##########################################################################################
#Model names:
model_names = sorted(name for name in model.__dict__
    if name.startswith("Planet")
    and callable(model.__dict__[name]))

def train(net, loader, criterion, optimizer, verbose = False):
    net.train()
    running_loss = 0
    running_accuracy = 0

    for i, (X,y) in enumerate(loader):
        if args.cuda:
            X, y = X.cuda(), y.cuda()
        X, y = Variable(X), Variable(y)

        output = net(X)
        loss = criterion(output, y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        #pdb.set_trace()
        running_loss += loss.item()
        acc = utils.get_multilabel_accuracy(output, y)
        running_accuracy += acc
        if i%400 == 0 and verbose:
            pct = float(i+1)/len(loader)
            curr_loss = running_loss/(i+1)
            curr_acc = running_accuracy/(i+1)
            print('Complete: {:.2f}, Loss: {:.2f}, Accuracy: {:.4f}'.format(pct*100,
                        curr_loss, curr_acc))
    return running_loss/len(loader), running_accuracy/len(loader)

def validate(net, loader, criterion):
    net.eval()
    running_loss = 0
    running_accuracy = 0
    targets = torch.FloatTensor(0,17) # For fscore calculation
    predictions = torch.FloatTensor(0,17)
    for i, (X,y) in enumerate(loader):
        if args.cuda:
            X, y = X.cuda(), y.cuda()
        X, y = Variable(X, volatile=True), Variable(y)
        output = net(X)
        loss = criterion(output, y)
        acc = utils.get_multilabel_accuracy(output, y)
        targets = torch.cat((targets, y.cpu().data), 0)
        predictions = torch.cat((predictions,output.cpu().data), 0)
        running_loss += loss.item()
        running_accuracy += acc
    fscore = fbeta_score(targets.numpy(), predictions.numpy() > 0.23,
                beta=2, average='samples')
    return running_loss/len(loader), running_accuracy/len(loader), fscore

if __name__ == '__main__':
    net = model.__dict__[args.model]()
    criterion = torch.nn.BCELoss()

    if args.cuda:
        net, criterion = net.cuda(), criterion.cuda()
    # early stopping parameters
    patience = args.patience
    best_loss = 1e4

    # Print model to logfile
    print(net, file=logfile)

    # Change optimizer for finetuning
    if args.model=='AmazonSimpleNet':
        optimizer = optim.Adam(net.parameters())
    else:
        optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

    for e in range(args.epochs):
        start = time.time()
        train_loss, train_acc = train(net, train_loader, criterion, optimizer, args.v)
        val_loss, val_acc, fscore = validate(net, val_loader, criterion)
        end = time.time()

        # print stats
        stats ="""Epoch: {}\t train loss: {:.3f}, train acc: {:.3f}\t
                val loss: {:.3f}, val acc: {:.3f}\t fscore: {:.3f}\t
                time: {:.1f}s""".format( e, train_loss, train_acc, val_loss,
                val_acc, fscore, end-start)
        print(stats)
        print(stats, file=logfile)
        log_value('train_loss', train_loss, e)
        log_value('val_loss', val_loss, e)
        log_value('fscore', fscore, e)

        #early stopping and save best model
        if val_loss < best_loss:
            best_loss = val_loss
            patience = args.patience
            utils.save_model({
                'arch': args.model,
                'state_dict': net.state_dict()
            }, 'saved-models/{}-run-{}.pth.tar'.format(args.model, run))
        else:
            patience -= 1
            if patience == 0:
                print('Run out of patience!')
                break


"""
Module that performs a non-linear least squares minimization of the spectroscopic and/or astrometric data
using the lmfit package.
This module is developed with lmfit 0.9.14 and numpy 1.17.2, and requires emcee 3.0.0.

Author:
Matthias Fabry, Instituut voor Sterrekunde, KU Leuven, Belgium

"""
import os
import time

import lmfit as lm
import numpy as np

from modules.binary_system import System

RV1, RV2, AS = False, False, False
LAS, LRV = 0, 0

def LMminimizer(guess_dict: dict, datadict: dict, domcmc: bool, steps: int = 1000, as_weight: float = None):
    """
    Minimizes the provided data to a binary star model, with initial provided guesses and a search radius
    :param as_weight: weight to give to the astrometric data, optional.
    :param domcmc: boolean to indicate whether to do an MCMC posterior error estimation
    :param guess_dict: dictionary containing guesses and 'to-vary' flags for the 11 parameters
    :param datadict: dictionary containing observational data of RV and/or separations
    :param steps: integer giving the number of steps the MCMC should perform
    :return: result from the lmfit minimization routine. It is a MinimizerResult object.
    """

    # setup data for the solver
    hjds = dict()
    data = dict()
    errors = dict()
    # we need to store this on module level so the function to minimize knows quickly which data is included or not
    global RV1, RV2, AS
    RV1 = RV2 = AS = False
    global LAS, LRV
    LAS = LRV = 0
    try:
        hjds['RV1'] = datadict['RV1']['hjds']
        data['RV1'] = datadict['RV1']['RVs']
        errors['RV1'] = datadict['RV1']['errors']
        RV1 = True
        LRV += len(data['RV1'])
    except KeyError:
        pass
    try:
        hjds['RV2'] = datadict['RV2']['hjds']
        data['RV2'] = datadict['RV2']['RVs']
        errors['RV2'] = datadict['RV2']['errors']
        RV2 = True
        LRV += len(data['RV2'])
    except KeyError:
        pass
    try:
        hjds['AS'] = datadict['AS']['hjds']
        data['east'] = datadict['AS']['easts']
        data['north'] = datadict['AS']['norths']
        errors['east'] = datadict['AS']['easterrors']
        errors['north'] = datadict['AS']['northerrors']
        AS = True
        LAS += 2*len(data['east'])
    except KeyError:
        pass

    # setup Parameters object for the solver
    params = lm.Parameters()
    # populate with parameter data
    params.add_many(
        ('e', guess_dict['e'][0], guess_dict['e'][1], 0, 1 - 1e-5),
        ('i', guess_dict['i'][0], guess_dict['i'][1]),
        ('omega', guess_dict['omega'][0], guess_dict['omega'][1]),
        ('Omega', guess_dict['Omega'][0], guess_dict['Omega'][1]),
        ('t0', guess_dict['t0'][0], guess_dict['t0'][1]),
        ('p', guess_dict['p'][0], guess_dict['p'][1], 0),
        ('mt', guess_dict['mt'][0], guess_dict['mt'][1], 0),
        ('d', guess_dict['d'][0], guess_dict['d'][1], 0),
        ('k1', guess_dict['k1'][0], guess_dict['k1'][1], 0),
        ('gamma1', guess_dict['gamma1'][0], guess_dict['gamma1'][1]),
        ('k2', guess_dict['k2'][0], guess_dict['k2'][1], 0),
        ('gamma2', guess_dict['gamma2'][0], guess_dict['gamma2'][1])
    )

    # put e to a non zero value to avoid conditioning error in MCMC
    if params['e'].value < 1e-8:
        print('Warning: eccentricity is put to 1e-8 to avoid conditioning issues!')
        params['e'].set(value=1e-8)

    if RV1 and RV2:
        if not AS:
            params['d'].set(vary=False)
            params['i'].set(vary=False)
            params['Omega'].set(vary=False)
    elif RV1:
        params['k2'].set(vary=False)
        params['gamma2'].set(vary=False)
        params['d'].set(vary=False)
        if not AS:
            params['mt'].set(vary=False)
            params['i'].set(vary=False)
            params['Omega'].set(vary=False)
    elif AS:
        for key in 'k1', 'gamma1', 'k2', 'gamma2':
            params[key].set(vary=False)
    else:
        raise ValueError('No data supplied! Cannot minimize.')

    # build a minimizer object
    minimizer = lm.Minimizer(fcn2min, params, fcn_args=(hjds, data, errors, as_weight))
    print('Starting Minimization with {}{}{}...'.format('primary RV data, ' if RV1 else '',
                                                        'secondary RV data, ' if RV2 else '',
                                                        'astrometric data' if AS else ''))
    tic = time.time()
    result = minimizer.minimize()
    toc = time.time()
    print('Minimization Complete in {} s!\n'.format(toc - tic))
    lm.report_fit(result.params)
    print('\n')
    rms_rv1, rms_rv2, rms_as = 0, 0, 0
    system = System(result.params.valuesdict())
    if RV1:
        # weigh with number of points for RV1 data
        rms_rv1 = np.sqrt(
            np.sum((system.primary.radial_velocity_of_hjds(hjds['RV1']) - data['RV1']) ** 2) / len(data['RV1']))
    if RV2:
        # Same for RV2
        rms_rv2 = np.sqrt(
            np.sum((system.secondary.radial_velocity_of_hjds(hjds['RV2']) - data['RV2']) ** 2) / len(data['RV2']))
    if AS:
        # same for AS
        omc2E = np.sum((system.relative.east_of_hjds(hjds['AS']) - data['east']) ** 2)
        omc2N = np.sum((system.relative.north_of_hjds(hjds['AS']) - data['north']) ** 2)
        rms_as = np.sqrt(omc2E + omc2N / (len(data['east']) + len(data['north'])))
    if domcmc:
        mcminimizer = lm.Minimizer(fcn2min, params=result.params, fcn_args=(hjds, data, errors))
        print('Starting MCMC sampling using the minimized parameters...')
        tic = time.time()
        newresults = mcminimizer.emcee(workers=os.cpu_count(), steps=steps)
        toc = time.time()
        print('MCMC complete in {} s!\n'.format(toc - tic))
        lm.report_fit(newresults.params)
        print('\n')
        return newresults, rms_rv1, rms_rv2, rms_as
    return result, rms_rv1, rms_rv2, rms_as

def fcn2min(params, hjds, data, errors, weight=None):
    """
    Define the function to be minimized by the minimizer. It is simply to array of weighted distances from the model to
    the data, schematically:
        fun = array((data[hjd]-model[hjd])/error_on_data(hjd))
    The function will find out which data is omitted.
    :param weight: multiplicative weight to give to the astrometric points, optional. If None, no additional weight is
    applied
    :param params: Parameters object from the package lmfit, containing the 11 parameters to fit.
    :param hjds: dictionary of the days of the observations
    :param data: dictionary of the measurements, be it RV or AS data
    :param errors: dictionary of the errors on the measurements
    :return: array with the weighted errors of the data to the model defined by the parameters
    """
    # create the system belonging to the parameters
    system = System(params.valuesdict())

    if RV1:
        # Get weighted distance for RV1 data
        chisq_rv1 = ((system.primary.radial_velocity_of_hjds(hjds['RV1']) - data['RV1']) / errors['RV1'])
        if weight:
            chisq_rv1 *= (1-weight) * (LAS + LRV) / LRV
    else:
        # is RV1 not there, make empty list for this part of the data
        chisq_rv1 = np.asarray(list())
    if RV2:
        # Same for RV2
        chisq_rv2 = ((system.secondary.radial_velocity_of_hjds(hjds['RV2']) - data['RV2']) / errors['RV2'])
        if weight:
            chisq_rv2 *= (1 - weight) * (LAS + LRV) / LRV
    else:
        chisq_rv2 = np.asarray(list())
    if AS:
        # same for AS
        chisq_east = ((system.relative.east_of_hjds(hjds['AS']) - data['east']) / errors['east'])
        chisq_north = ((system.relative.north_of_hjds(hjds['AS']) - data['north']) / errors['north'])
        if weight:
            chisq_east *= weight * (LAS + LRV) / LAS
            chisq_north *= weight * (LAS + LRV) / LAS
    else:
        chisq_east = np.asarray(list())
        chisq_north = np.asarray(list())

    # concatentate the four parts (RV1, RV2, ASeast, ASnorth)
    res = np.concatenate((chisq_rv1, chisq_rv2, chisq_east, chisq_north))
    return res


#!/usr/bin/env python

from setuptools import setup, find_packages

readme = open('README.rst').read()

long_description = "%s" % readme

setup(
    name='pysparklines',
    version=0.9,
    description="pysparklines is a unicode sparkline generation library.",
    long_description=long_description,
    author="Brandon Whaley",
    author_email="redkrieg@gmail.com",
    url="https://github.com/RedKrieg/pysparklines",
    packages=find_packages(),
    entry_points={
        'console_scripts': [
            'sparkline = sparkline:main',
        ]
    },
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Console',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: BSD License',
        'Operating System :: Unix',
        'Operating System :: POSIX',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Topic :: Software Development',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
)


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 20 20:06:14 2018

@author: Figo
"""

plt_dir = 'D:/3_other_code/data/kaggle/visualization/filter/'

####对于在 ImageNet上预训练的 VGG16网络，其block3_conv1层第 0个过滤器激活的损失如下所示。######

#为过滤器的可视化定义损失张量
from keras.applications import VGG16
from keras import backend as K
import numpy as np
import matplotlib.pyplot as plt

model = VGG16(weights='imagenet',include_top=False)

'''
layer_name = 'block3_conv1'
filter_index = 0
layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])

# 为了实现梯度下降，我们需要得到损失相对于模型输入的梯度。
# 调用 gradients 返回的是一个张量列表（本例中列表长度为 1）。
# 因此，只保留第一个元素，它是一个张量
grads = K.gradients(loss, model.input)[0]

# 为了让梯度下降过程顺利进行，一个非显而易见的技巧是将梯度张量除以其 L2 范数（张量中
# 所有值的平方的平均值的平方根）来标准化。这就确保了输入图像的更新大小始终位于相同的范围。
# 做除法前加上 1e–5，以防不小心除以 0
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)

# 现在你需要一种方法：给定输入图像，它能够计算损失张量和梯度张量的值。
# 可以定义一个 Keras 后端函数来实现此方法： iterate 是一个函数，它将一个 Numpy 张量（表示为长度
# 为 1 的张量列表）转换为两个 Numpy 张量组成的列表，这两个张量分别是损失值和梯度值。
iterate = K.function([model.input], [loss, grads])
loss_value, grads_value = iterate([np.zeros((1, 150, 150, 3))])

#通过随机梯度下降让损失最大化
input_img_data = np.random.random((1, 150, 150, 3)) * 20 + 128
step = 1.
for i in range(40):
    # 计算损失值和梯度值
    loss_value, grads_value = iterate([input_img_data])
    # 沿着让损失最大化的方向调节输入图像
    input_img_data += grads_value * step

'''

# 得到的图像张量是形状为 (1, 150, 150, 3) 的浮点数张量，其取值可能不是 [0, 255] 区间内的整数。
# 因此，你需要对这个张量进行后处理，将其转换为可显示的图像。
def deprocess_image(x):
    # 对张量做标准化，使其均值为 0，标准差为 0.1
    x -= x.mean()
    x /= (x.std() + 1e-5)
    x *= 0.1
    # 将 x 裁切（clip）到 [0, 1] 区间
    x += 0.5
    x = np.clip(x, 0, 1)
    # 将 x 转换为 RGB 数组
    x *= 255
    x = np.clip(x, 0, 255).astype('uint8')
    return x

# 接下来，我们将上述代码片段放到一个 Python函数中，输入一个层的名称和一个过滤器索引，
# 它将返回一个有效的图像张量，表示能够将特定过滤器的激活最大化的模式。
# 生成过滤器可视化的函数
def generate_pattern(layer_name, filter_index, size=150):
    # 构建一个损失函数，将该层第 n 个过滤器的激活最大化
    layer_output = model.get_layer(layer_name).output
    loss = K.mean(layer_output[:, :, :, filter_index])
    # 计算这个损失相对于输入图像的梯度
    grads = K.gradients(loss, model.input)[0]
    # 标准化技巧：将梯度标准化
    grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
    # 返回给定输入图像的损失和梯度
    iterate = K.function([model.input], [loss, grads])
    # 从带有噪声的灰度图像开始
    input_img_data = np.random.random((1, size, size, 3)) * 20 + 128.
    # 运行 40 次梯度上升
    step = 1.
    for i in range(40):
        loss_value, grads_value = iterate([input_img_data])
        input_img_data += grads_value * step
    img = input_img_data[0]
    return deprocess_image(img)

# plt.imshow(generate_pattern('block3_conv1', 0))

# 我们只查看 VGG 每一层的前 64 个过滤器，并只查看每个卷积块的第一层（即 block1_conv1、
# block2_conv1、block3_conv1、 block4_ conv1、 block5_conv1）
# 我们将输出放在一个 8×8 的网格中，每个网格是一个 64 像素×64 像素的过滤器模式，
# 两个过滤器模式之间留有一些黑边

layer_names = ['block{}_conv1'.format(x) for x in range(1,6)]

for layer_name in layer_names:
    size = 64
    margin = 5
    results = np.zeros((8 * size + 7 * margin, 8 * size + 7 * margin, 3))
    for i in range(8):
        for j in range(8):
            filter_img = generate_pattern(layer_name, i + (j * 8), size=size)
            horizontal_start = i * size + i * margin
            horizontal_end = horizontal_start + size
            vertical_start = j * size + j * margin
            vertical_end = vertical_start + size
            results[horizontal_start: horizontal_end,vertical_start: vertical_end, :] = filter_img
    #print(results)
    # 要归一化，确保数值落在 0-1 之间
    results = results/255
    plt.figure(figsize=(20, 20))
    plt.imshow(results)
    plt.savefig(plt_dir + layer_name + '.png')


# Copyright (C) 2019 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>

"""Test on Cycle Task status update over import."""

import datetime
from collections import OrderedDict

import mock
import ddt

from ggrc import db
from ggrc.models import all_models
from ggrc.converters import errors

from integration import ggrc as ggrc_test
from integration.ggrc_workflows.models import factories
from integration.ggrc.models import factories as ggrc_factories

DENY_FINISHED_DATES_STATUSES_STR = ("<'Assigned' / 'In Progress' / "
                                    "'Declined' / 'Deprecated'>")
DENY_VERIFIED_DATES_STATUSES_STR = ("<'Assigned' / 'In Progress' / "
                                    "'Declined' / 'Deprecated' / 'Finished'>")

@ddt.ddt
class TestCycleTaskStatusUpdate(ggrc_test.TestCase):
  """Test cases for update status on import cycle tasks"""

  @classmethod
  def setUpClass(cls):
    """Setup special hooks to that test class."""
    cls._current_user_wfo_or_assignee = (
        all_models.CycleTaskGroupObjectTask.current_user_wfa_or_assignee
    )
    all_models.CycleTaskGroupObjectTask.current_user_wfa_or_assignee = (
        mock.MagicMock(return_value=True))

  @classmethod
  def tearDownClass(cls):
    all_models.CycleTaskGroupObjectTask.current_user_wfa_or_assignee = (
        cls._current_user_wfo_or_assignee
    )

  DEFAULT_TASK_STATUSES = [all_models.CycleTaskGroupObjectTask.ASSIGNED] * 3

  ALIAS = "Cycle Task"

  def setUp(self):
    with ggrc_factories.single_commit():
      self.workflow = factories.WorkflowFactory(
          status=all_models.Workflow.ACTIVE
      )
      self.cycle = factories.CycleFactory(workflow=self.workflow)
      self.group = factories.CycleTaskGroupFactory(
          cycle=self.cycle,
          context=self.cycle.workflow.context
      )
      self.tasks = []
      for ind, task_status in enumerate(self.DEFAULT_TASK_STATUSES):
        self.tasks.append(factories.CycleTaskGroupObjectTaskFactory(
            title='task{}'.format(ind),
            cycle=self.cycle,
            cycle_task_group=self.group,
            context=self.cycle.workflow.context,
            status=task_status,
        ))
    # Emulate that current user is assignee for all test CycleTasks.
    self.task_ids = [t.id for t in self.tasks]

  VERIFIED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.VERIFIED] * 3,
      "group_status": all_models.CycleTaskGroup.VERIFIED,
      "cycle_status": all_models.Cycle.VERIFIED,
      "workflow_status": all_models.Workflow.INACTIVE,
  }
  ASSIGNED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.ASSIGNED] * 3,
      "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
      "cycle_status": all_models.Cycle.IN_PROGRESS,
      "workflow_status": all_models.Workflow.ACTIVE
  }
  IN_PROGRESS_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.IN_PROGRESS] * 3,
      "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
      "cycle_status": all_models.Cycle.IN_PROGRESS,
      "workflow_status": all_models.Workflow.ACTIVE
  }
  DECLINED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.DECLINED] * 3,
      "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
      "cycle_status": all_models.Cycle.IN_PROGRESS,
      "workflow_status": all_models.Workflow.ACTIVE,
  }
  FINISHED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.FINISHED] * 3,
      "group_status": all_models.CycleTaskGroup.FINISHED,
      "cycle_status": all_models.Cycle.FINISHED,
      "workflow_status": all_models.Workflow.ACTIVE,
  }

  def _update_structure(self, structure=None):
    """Update structure of setuped objects."""
    if not structure:
      return
    structure = structure or {}
    start_statuses = structure.get("task_statuses") or []
    if start_statuses:
      for idx, status in enumerate(start_statuses):
        self.tasks[idx].status = status
    self.group.status = structure.get("group_status", self.group.status)
    self.cycle.status = structure.get("cycle_status", self.cycle.status)
    self.cycle.is_verification_needed = structure.get(
        "cycle_is_verification_needed",
        self.cycle.is_verification_needed)
    self.workflow.status = structure.get("workflow_status",
                                         self.workflow.status)
    db.session.commit()

  def _get_start_tasks_statuses(self, start_structure=None):
    start_statuses = (start_structure or {}).get("task_statuses", [])
    return (start_statuses + self.DEFAULT_TASK_STATUSES)[:3]

  def build_import_data(self, task_statuses):
    return [OrderedDict([("object_type", self.ALIAS),
                         ("Code*", self.tasks[idx].slug),
                         ("State", status)])
            for idx, status in enumerate(task_statuses)]

  @ddt.unpack
  @ddt.data(
      {"update_structure": VERIFIED_STRUCTURE},
      {"update_structure": IN_PROGRESS_STRUCTURE},
      {"update_structure": FINISHED_STRUCTURE},
      # verified few tasks
      {"update_structure": {
          "task_statuses": [all_models.CycleTaskGroupObjectTask.VERIFIED] * 2,
          "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
          "cycle_status": all_models.Cycle.IN_PROGRESS,
          "workflow_status": all_models.Workflow.ACTIVE,
      }},
      {"update_structure": DECLINED_STRUCTURE},
      {"update_structure": VERIFIED_STRUCTURE,
       "start_structure": IN_PROGRESS_STRUCTURE},
      {"update_structure": IN_PROGRESS_STRUCTURE,
       "start_structure": VERIFIED_STRUCTURE},
      {"update_structure": DECLINED_STRUCTURE,
       "start_structure": IN_PROGRESS_STRUCTURE},
      {"update_structure": IN_PROGRESS_STRUCTURE,
       "start_structure": DECLINED_STRUCTURE},
  )
  def test_update_status(self, update_structure, start_structure=None):
    """Simple update task status to {update_structure[task_statuses]}"""
    self._update_structure(start_structure)
    start_statuses = self._get_start_tasks_statuses(start_structure)
    task_count = len(all_models.CycleTaskGroupObjectTask.query.all())
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    task_statuses = update_structure["task_statuses"]
    group_status = update_structure["group_status"]
    cycle_status = update_structure["cycle_status"]
    workflow_status = update_structure["workflow_status"]
    response = self.import_data(*self.build_import_data(task_statuses))
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    group = self.tasks[0].cycle_task_group
    self.assertEqual(group_status, group.status)
    self.assertEqual(len(all_models.CycleTaskGroupObjectTask.query.all()),
                     task_count)
    self.assertEqual(cycle_status, group.cycle.status)
    self.assertEqual(workflow_status, group.cycle.workflow.status)

  @ddt.data(ASSIGNED_STRUCTURE,
            IN_PROGRESS_STRUCTURE,
            DECLINED_STRUCTURE)
  def test_update_to_verified(self, start_structure):
    """Update task status to verified from {0[task_statuses]} and back"""
    self._update_structure(start_structure)
    start_statuses = self._get_start_tasks_statuses(start_structure)
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    task_statuses = self.VERIFIED_STRUCTURE["task_statuses"]
    response = self.import_data(*self.build_import_data(task_statuses))
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    finished_dates = [t.finished_date for t in self.tasks]
    self.assertNotEqual([None] * len(self.tasks), finished_dates)
    verified_dates = [t.verified_date for t in self.tasks]
    self.assertNotEqual([None] * len(self.tasks), verified_dates)
    self.assertFalse(self.tasks[0].cycle.is_current)
    # wrong rollback  case (try to import from verified to finished)
    response = self.import_data(*self.build_import_data(start_statuses))
    line_error_tmpl_1 = errors.INVALID_STATUS_DATE_CORRELATION.format(
        date="Actual Verified Date",
        deny_states=DENY_VERIFIED_DATES_STATUSES_STR,
        line="{}"
    )
    line_error_tmpl_2 = errors.INVALID_STATUS_DATE_CORRELATION.format(
        date="Actual Finish Date",
        deny_states=DENY_FINISHED_DATES_STATUSES_STR,
        line="{}"
    )
    verfied_date_errors = {line_error_tmpl_1.format(i + 3)
                           for i in xrange(len(self.tasks))}
    finished_date_errors = {line_error_tmpl_2.format(i + 3)
                            for i in xrange(len(self.tasks))}
    self._check_csv_response(
        response,
        {
            "Cycle Task": {
                "row_errors": verfied_date_errors | finished_date_errors
            }
        }
    )
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # nothing change
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    self.assertEqual(finished_dates, [t.finished_date for t in self.tasks])
    self.assertEqual(verified_dates, [t.verified_date for t in self.tasks])
    self.assertFalse(self.tasks[0].cycle.is_current)
    # correct rollback
    rollback_data = self.build_import_data(start_statuses)
    # setup verified dates as empty
    for line in rollback_data:
      line["Actual Verified Date"] = "--"
      line["Actual Finish Date"] = "--"
    response = self.import_data(*rollback_data)
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # assert correct rollback
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    self.assertTrue(self.tasks[0].cycle.is_current)

  @ddt.data(1, 2, 3)
  def test_finished_to_verified(self, number_of_tasks):
    """Update {0} tasks status from finished from verified and back"""
    now = datetime.datetime.now().replace(microsecond=0)
    finished_date = now - datetime.timedelta(1)
    start_statuses = self._get_start_tasks_statuses(self.FINISHED_STRUCTURE)
    with ggrc_factories.single_commit():
      for task in self.tasks:
        task.finished_date = finished_date
    self._update_structure(self.FINISHED_STRUCTURE)
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    task_statuses = [
        all_models.CycleTaskGroupObjectTask.VERIFIED
    ] * number_of_tasks
    response = self.import_data(*self.build_import_data(task_statuses))
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    verified_dates = [t.verified_date for t in self.tasks][:number_of_tasks]
    self.assertNotEqual([None] * number_of_tasks, verified_dates)
    verified_dates += [t.verified_date for t in self.tasks][number_of_tasks:]
    self.assertEqual(
        [None] * (len(self.tasks) - number_of_tasks),
        [t.verified_date for t in self.tasks][number_of_tasks:])
    self.assertEqual(len(self.tasks) != number_of_tasks,
                     self.tasks[0].cycle.is_current)
    # wrong rollback  case (try to import from verified to finished)
    response = self.import_data(*self.build_import_data(start_statuses))
    line_error_tmpl = errors.INVALID_STATUS_DATE_CORRELATION.format(
        date="Actual Verified Date",
        deny_states=DENY_VERIFIED_DATES_STATUSES_STR,
        line="{}"
    )
    self._check_csv_response(
        response,
        {
            "Cycle Task": {
                "row_errors": {line_error_tmpl.format(idx + 3)
                               for idx in xrange(number_of_tasks)}
            }
        }
    )
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # nothing change
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual(len(self.tasks) != number_of_tasks,
                     self.tasks[0].cycle.is_current)
    self.assertEqual(
        verified_dates,
        [t.verified_date for t in self.tasks])
    # correct rollback
    rollback_data = self.build_import_data(start_statuses)
    # setup verified dates as empty
    for line in rollback_data:
      line["Actual Verified Date"] = "--"
    response = self.import_data(*rollback_data)
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # assert correct rollback
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    self.assertTrue(self.tasks[0].cycle.is_current)

  @staticmethod
  def __build_error_tmpl(error_tmpl, **context):
    """Create simple temaplte from base error tmpl."""
    if "line" not in context:
      context["line"] = "{}"
    return error_tmpl.format(**context)

  def __build_status_error_resp(self, key, error_tmpl, exception_statuses):
    """Return expected response dict based on sent arguments."""
    error_tmpl = self.__build_error_tmpl(error_tmpl,
                                         column_name="State",
                                         message="Invalid state '{}'")
    return {
        self.ALIAS: {
            key: {error_tmpl.format(3 + idx, status)
                  for idx, status in enumerate(exception_statuses)},
        }
    }

  def __assert_error_message(self,
                             sending_data,
                             expected_messages,
                             start_structure=None):
    """Assert validation error message on import cycle task statuses."""
    start_statuses = self._get_start_tasks_statuses(start_structure)

    group_status = self.group.status
    cycle_status = self.cycle.status
    workflow_status = self.workflow.status
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    response = self.import_data(*sending_data)
    self._check_csv_response(response, expected_messages)
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    group = self.tasks[0].cycle_task_group
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual(group_status, group.status)
    self.assertEqual(cycle_status, group.cycle.status)
    self.assertEqual(workflow_status, group.cycle.workflow.status)

  @ddt.data(
      VERIFIED_STRUCTURE["task_statuses"],
      DECLINED_STRUCTURE["task_statuses"],
  )
  def test_validation_error(self, exception_statuses):
    """Validation cycle task status update to {0} if no verification needed."""
    self._update_structure({"cycle_is_verification_needed": False})
    error_msgs = self.__build_status_error_resp("row_errors",
                                                errors.VALIDATION_ERROR,
                                                exception_statuses)
    send_data = self.build_import_data(exception_statuses)
    self.__assert_error_message(send_data, error_msgs)

  @ddt.unpack
  @ddt.data(
      {"start_structure": IN_PROGRESS_STRUCTURE,
       "exception_statuses": ["Absolutely Wrong Status"] * 3},
      {"start_structure": IN_PROGRESS_STRUCTURE,
       "exception_statuses": [""] * 3},
      {"exception_statuses": [""] * 3},
      {"exception_statuses": ["Absolutely Wrong Status"] * 3},
      {"start_structure": FINISHED_STRUCTURE,
       "exception_statuses": ["Absolutely Wrong Status"] * 3},
      {"start_structure": FINISHED_STRUCTURE,
       "exception_statuses": [""] * 3},
  )
  def test_simple_send_invalid_status(self,
                                      exception_statuses,
                                      start_structure=None):
    """Validation cycle task status update to {exception_statuses}."""
    self._update_structure(start_structure)
    warn_msgs = self.__build_status_error_resp("row_warnings",
                                               errors.WRONG_VALUE_CURRENT,
                                               exception_statuses)
    send_data = self.build_import_data(exception_statuses)
    self.__assert_error_message(send_data,
                                warn_msgs,
                                start_structure=start_structure)

  @ddt.data(
      ("Actual Finish Date", "Actual Verified Date"),
      ("Start Date", "Due Date"),
  )
  def test_for_date_compare_error(self, columns):
    """Validate task import data where {0[0]} bigger than {0[1]}."""
    self._update_structure(self.VERIFIED_STRUCTURE)
    today = datetime.date.today()
    dates = (today, today - datetime.timedelta(7))
    error_tmpl = self.__build_error_tmpl(errors.INVALID_START_END_DATES,
                                         start_date=columns[0],
                                         end_date=columns[1])
    # line format
    error_resp = {self.ALIAS: {"row_errors": {error_tmpl.format(3)}}}
    data_to_import = OrderedDict([("object_type", self.ALIAS),
                                  ("Code*", self.tasks[0].slug)])
    for column, date in zip(columns, dates):
      data_to_import[column] = date
    response = self.import_data(data_to_import)
    self._check_csv_response(response, error_resp)

  @ddt.data(
      ("Actual Finish Date", DENY_FINISHED_DATES_STATUSES_STR),
      ("Actual Verified Date", DENY_VERIFIED_DATES_STATUSES_STR),
  )
  @ddt.unpack
  def test_for_date_state_error(self, column, deny_states):
    """Validate task {0} not allowed for in {0} tasks."""
    self._update_structure(self.IN_PROGRESS_STRUCTURE)
    today = datetime.date.today()
    error_tmpl = self.__build_error_tmpl(
        errors.INVALID_STATUS_DATE_CORRELATION,
        date=column,
        deny_states=deny_states,
    )
    # line format
    error_resp = {self.ALIAS: {"row_errors": {error_tmpl.format(3)}}}
    data_to_import = OrderedDict([("object_type", self.ALIAS),
                                  ("Code*", self.tasks[0].slug),
                                  (column, today)])
    response = self.import_data(data_to_import)
    self._check_csv_response(response, error_resp)


# multi_task_all_variables_pytorch.py

import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os
from torch.autograd import Variable
import pandas as pd
import math
import sklearn.preprocessing as sk
from tensorboardX import SummaryWriter
import seaborn as sns
from sklearn.model_selection import KFold
from sklearn import metrics
from sklearn.feature_selection import VarianceThreshold
from sklearn.linear_model import Ridge
from sklearn.linear_model import RidgeCV
from sklearn.model_selection import train_test_split
import random
from torch.optim import lr_scheduler
from sklearn.metrics import r2_score
import torch.utils.data as Data
from torch.optim.lr_scheduler import LambdaLR
from torch.optim import lr_scheduler
from statistics import mean

def normalizedata(data):
    num_features = data.shape[1]

    mean = np.array([data[:,j].mean() for j in range(num_features)]).reshape(num_features)
    std = np.array([data[:,j].std() for j in range(num_features)]).reshape(num_features)

    for i in range(num_features):
        if float(std[i]) != 0:
            data[:, i] = (data[:, i] - float(mean[i])) * (1 / float(std[i]))
        else:
            data[:, i] = np.ones((data.shape[0]))
    return data

class MTLnet(nn.Module):
    def __init__(self):
        super(MTLnet, self).__init__()

        self.sharedlayer = nn.Sequential(
            nn.Linear(feature_size, shared_layer_size),
            nn.ReLU()
            # nn.Dropout()
        )
        # self.sharedlayer_1 = nn.Sequential(
        #     nn.Linear(shared_layer_size_1, shared_layer_size),
        #     nn.ReLU()
        #     # nn.Dropout()
        # )
        # self.sharedlayer_2 = nn.Sequential(
        #     nn.Linear(shared_layer_size_1, shared_layer_size),
        #     nn.ReLU()
        #     # nn.Dropout()
        # )
        self.shared_tower1 = nn.Sequential(
            nn.Linear(shared_layer_size, tower_h1),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h1, tower_h2),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h2, tower_h3),
            nn.ReLU()
        )
        self.tower1_1 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower1_2 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower1_3 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.shared_tower2 = nn.Sequential(
            nn.Linear(shared_layer_size, tower_h1),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h1, tower_h2),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h2, tower_h3),
            nn.ReLU()
             )
        self.tower2_1 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower2_2 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower2_3 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.shared_tower3 = nn.Sequential(
            nn.Linear(shared_layer_size, tower_h1),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h1, tower_h2),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h2, tower_h3),
            nn.ReLU()

             )

        self.tower3_1 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower3_2 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower3_3 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.shared_tower4 = nn.Sequential(
            nn.Linear(shared_layer_size, tower_h1),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h1, tower_h2),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h2, tower_h3),
            nn.ReLU()
             )
        self.tower4_1 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower4_2 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        self.tower4_3 = nn.Sequential(
            nn.Linear(tower_h3, tower_h4),
            nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h4, tower_h5),
            nn.ReLU(),
            # nn.Dropout(),
            # nn.Linear(tower_h2, tower_h2),
            # nn.ReLU(),
            # nn.Dropout(),
            nn.Linear(tower_h5, output_size)
        )
        # self.shared_tower5 = nn.Sequential(
        #     nn.Linear(shared_layer_size, tower_h1),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h1, tower_h2),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h2, tower_h3),
        #     nn.ReLU()
        #      )
        # self.tower5_1 = nn.Sequential(
        #     nn.Linear(tower_h3, tower_h4),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h4, tower_h5),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h5, output_size)
        # )
        # self.tower5_2 = nn.Sequential(
        #     nn.Linear(tower_h3, tower_h4),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h4, tower_h5),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h5, output_size)
        # )
        # self.tower5_3 = nn.Sequential(
        #     nn.Linear(tower_h3, tower_h4),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h4, tower_h5),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h5, output_size)
        # )

        # self.shared_tower6 = nn.Sequential(
        #     nn.Linear(shared_layer_size, tower_h1),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h1, tower_h2),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h2, tower_h3),
        #     nn.ReLU()
        #      )
        # self.tower6_1 = nn.Sequential(
        #     nn.Linear(tower_h3, tower_h4),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h4, tower_h5),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h5, output_size)
        # )
        # self.tower6_2 = nn.Sequential(
        #     nn.Linear(tower_h3, tower_h4),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h4, tower_h5),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h5, output_size)
        # )
        # self.tower6_3 = nn.Sequential(
        #     nn.Linear(tower_h3, tower_h4),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h4, tower_h5),
        #     nn.ReLU(),
        #     # nn.Dropout(),
        #     # nn.Linear(tower_h2, tower_h2),
        #     # nn.ReLU(),
        #     # nn.Dropout(),
        #     nn.Linear(tower_h5, output_size)
        # )
    def forward(self, x):
        h_shared = self.sharedlayer(x)

        # h_shared_1 = self.sharedlayer_1(h_shared)
        # h_shared_2 = self.sharedlayer_2(h_shared)

        out1 = self.shared_tower1(h_shared)
        out2 = self.shared_tower2(h_shared)
        out3 = self.shared_tower3(h_shared)
        out4 = self.shared_tower4(h_shared)
        # out5 = self.shared_tower5(h_shared)
        # out6 = self.shared_tower6(h_shared)

        out1_1 = self.tower1_1(out1)
        out1_2 = self.tower1_2(out1)
        out1_3 = self.tower1_3(out1)

        out2_1 = self.tower1_1(out2)
        out2_2 = self.tower1_2(out2)
        out2_3 = self.tower1_3(out2)

        out3_1 = self.tower1_1(out3)
        out3_2 = self.tower1_2(out3)
        out3_3 = self.tower1_3(out3)

        out4_1 = self.tower1_1(out4)
        out4_2 = self.tower1_2(out4)
        out4_3 = self.tower1_3(out4)

        # out5_1 = self.shared_tower1_1(out5)
        # out5_2 = self.shared_tower1_1(out5)
        # out5_3 = self.shared_tower1_1(out5)

        # out6_1 = self.shared_tower1_1(out6)
        # out6_2 = self.shared_tower1_1(out6)
        # out6_3 = self.shared_tower1_1(out6)
        return [out1_1, out1_2, out1_3, out2_1, out2_2, out2_3, out3_1, out3_2, out3_3, out4_1, out4_2, out4_3]

loss_func = torch.nn.MSELoss()  # this is for regression mean squared loss
def loss_function_multi(prediction, y):
    loss = []
    ls = 0
    for ye in range(y.shape[1]):
        loss.append(loss_func(prediction[ye], y[:,ye]))
        ls = ls + loss_func(prediction[ye], y[:,ye])
    return loss, (ls / 12)

loss = 0
def train(x, y, x_valid, y_valid, xtest, ytest):
    BATCH_SIZE = 50
    EPOCH = 500
    net = MTLnet()
    # x, y = Variable(torch.from_numpy(x).type(torch.FloatTensor)), Variable(torch.from_numpy(y).type(torch.FloatTensor))
    torch_dataset = Data.TensorDataset(x, y)

    learning_rate = 0.1
    optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

    loader = Data.DataLoader(
        dataset=torch_dataset,
        batch_size=BATCH_SIZE,
        shuffle=True, num_workers=2,)

    exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.1)

    # start training

    for epoch in range(EPOCH):
        for step, (batch_x, batch_y) in enumerate(loader): # for each training step

            b_x = Variable(batch_x)
            b_y = Variable(batch_y)
            # b_y = b_y.view(b_y.size()[0], 1)

            prediction = net(b_x)     # input x and predict based on x

            loss_list, loss = loss_function_multi(prediction, b_y)    # must be (1. nn output, 2. target)

            optimizer.zero_grad()   # clear gradients for next train
            loss.backward()         # backpropagation, compute gradients
            optimizer.step()        # apply gradients
        print('train Loss at step {0}: {1}'.format(epoch, loss))
        if epoch % 100 == 0:
            with torch.no_grad():
                yhat = net(x_valid)
                a,losshat =loss_function_multi(yhat, y_valid)
                print('validation Loss at step {0}: {1}'.format(epoch, losshat))
        exp_lr_scheduler.step()

    # xtest, ytest = Variable(torch.from_numpy(xtest).type(torch.FloatTensor)), Variable(torch.from_numpy(ytest).type(torch.FloatTensor))
    # ytest = ytest.view(ytest.size()[0], 1)
    predictionhat = net(xtest)     # input x and predict based on x
    a,loss = loss_function_multi(predictionhat, ytest)
    a,loss_test = loss_function_multi(predictionhat, ytest)
    print('test loss {0}'.format(loss_test))
    for i in range(ytest.shape[1]):
        print('test loss {0}'.format(a[i]))
    for i in range(ytest.shape[1]):
        print('test r2score {0}'.format(r2_score(ytest[:,i].data.numpy(), predictionhat[i].data.numpy())))

columns = []
response_columns= []

#SABLE
#LIMON
#ARGILE

# response_columns = ['SABLE', 'LIMON', 'ARGILE']
# response_columns = ['MVA', 'CONDHYD', 'DMP', 'POROTOT', 'MO']
response_columns = ['MVA', 'POROTOT1', 'PORODRAI1', 'CH_cm_h' ] #DMP
# response_columns=['PCMO']
columns = ['PM3', 'CEC', 'MNM3', 'CUM3'  ,'FEM3' ,'ALM3' ,'BM3'  ,'KM3'  ,'CAM3' ,'MGM3', 'ARGILE', 'SABLE', 'LIMON', 'CentreEp', 'PHSMP', 'PHEAU']

df1 = pd.read_csv('Couche_Inv1990tot.csv', usecols = ['IDEN2.x'] + ['IDEN3'] + ['GROUPE.x'] + ['Couche'] + columns + response_columns, encoding='latin-1')
df2 = pd.read_csv('Site_Inv1990.csv', usecols = ['IDEN2'] + ['xcoord', 'ycoord'], encoding='latin-1')
df3 = pd.read_csv('Champ_Inv1990.csv', usecols = ['IDEN3', 'Culture_1']  , encoding='latin-1')
# print(df1.columns)
# print(df2.columns)
# print(df3.columns)

# df = pd.merge(df1, df2, left_index=True, right_index=True, how='inner')
df4 = df1.merge(df2, left_on='IDEN2.x', right_on='IDEN2').reindex(columns=['IDEN2', 'IDEN3', 'GROUPE.x', 'Couche' ] + ['xcoord', 'ycoord'] + columns + response_columns)
df = df4.merge(df3, left_on='IDEN3', right_on='IDEN3')
# geopandas_(df)

# print('stop')
# print(df.columns)
# plot_features(df, columns, 'Alltogether')

# print(df['Culture_1'].unique())
# print(df['Culture_1'].value_counts())

# use pd.concat to join the new columns with your original dataframe
df = pd.concat([df,pd.get_dummies(df['Culture_1'])],axis=1)

# now drop the original 'country' column (you don't need it anymore)
df.drop(['Culture_1'],axis=1, inplace=True)
# print(df.columns)

columns = columns + ['xcoord', 'ycoord'] + ['1-prairie',
   '2-céréales', '3-maïs-grain', '4-pommes de terre', '5-maïs-ensilage',
   '6-autres']

from functools import reduce

for column in columns + response_columns:
	df[column].fillna((df[column].mean()), inplace=True)

cc = columns.copy()

df.dropna(inplace = True, subset=response_columns)
dfcouche1 = df[df['Couche'] == 1]
dfcouche2 = df[df['Couche'] == 2]
dfcouche3 = df[df['Couche'] == 3]

dfff = [0, 0 , 0]

print(dfcouche1['IDEN2'].count())
print(dfcouche1['IDEN2'].nunique())

print(dfcouche2['IDEN2'].count())
print(dfcouche2['IDEN2'].nunique())

print(dfcouche3['IDEN2'].count())

YY = []
for response in response_columns:

    dfff[0] =  dfcouche1
    dfff[1] =  dfcouche2[['IDEN2', response]]
    dfff[2] =  dfcouche3[['IDEN2', response]]
    print(dfcouche3[response].nunique())

    df_ = reduce(lambda  left,right: pd.merge(left,right,on=['IDEN2'],how='inner'), dfff)
    # df_ = dfcouche1.merge(dfcouche2[['IDEN2', 'MVA']], 'inner', on=['IDEN2'], suffixes=['_1', '_2'])
    # df_ = df_.merge(dfcouche3['IDEN2', 'MVA'], 'inner', on=['IDEN2'], suffixes=['_3'])
    df_.rename ({response + '_x': response + '_1', response + '_y': response + '_2', response : response + '_3'}, axis=1, inplace=True)

    print(df_.columns)
    # MVA = [df_['MVA_1'], df_['MVA_2'], df_['MVA_3']]

    # Y = np.array[([dfcouche1['MVA'], dfcouche2['MVA']])]
    Y1 = np.array([df_[response + '_1']]).flatten()
    Y2 = np.array([df_[response + '_2']]).flatten()
    Y3 = np.array([df_[response + '_3']]).flatten()

    YY.append(Y1)
    YY.append(Y2)
    YY.append(Y3)

    #YY is appending all the above

X = df_[columns].to_numpy()

X = normalizedata(X)

YY=np.array([np.array(xi) for xi in YY]).transpose()

# YY = normalizedata(YY)
print(X.shape)
print(YY.shape)
# print(YY.columns)

L = Y2.shape[0]
L1 = L - 100
L2 = L - 200
N = L

split = list(np.random.permutation(N))

X_train = X[split[0:L2],:]

YY_train = YY[split[0:L2], :]

X_valid = X[L2:L1,:]
YY_valid = YY[L2:L1, :]

X_test = X[L1:L,:]
YY_test = YY[L1:L,:]

X_train = torch.from_numpy(X_train)
X_train = X_train.float()

YY_train =  torch.from_numpy(YY_train)
YY_train =  YY_train.float()

X_valid = torch.from_numpy(X_valid)
X_valid = X_valid.float()

YY_valid = torch.from_numpy(YY_valid)
YY_valid = YY_valid.float()

X_test = torch.from_numpy(X_test)
X_test = X_test.float()

YY_test = torch.from_numpy(YY_test)
YY_test = YY_test.float()

print(X_train.shape)
print(X_valid.shape)
print(X_test.shape)

input_size, feature_size = X.shape
shared_layer_size_1 = 1024
shared_layer_size = 512
tower_h1 = 256
tower_h2 = 128
tower_h3 = 64
tower_h4 = 32
tower_h5 = 16

output_size = 1
LR = 0.01
epoch = 500
mb_size = 10

# l1D = loss_func(Yhat1, Y1_test.view(-1,1))
# l2D = loss_func(Yhat2, Y2_test.view(-1,1))
# l3D = loss_func(Yhat3, Y3_test.view(-1,1))

# print(r2_score(Y1_test.view(-1,1), Yhat1.data.numpy()))
# print(r2_score(Y2_test.view(-1,1), Yhat2.data.numpy()))
# print(r2_score(Y3_test.view(-1,1), Yhat3.data.numpy()))

# plt.plot(np.squeeze(costtr)[-100:], '-r',np.squeeze(costD)[-100:], '-b')
# plt.ylabel('total cost')
# plt.xlabel('iterations (per tens)')
# plt.show()

# plt.plot(np.squeeze(cost1tr)[-50:], '-r', np.squeeze(cost1D)[-50:], '-b')
# plt.ylabel('task 1 cost')
# plt.xlabel('iterations (per tens)')
# plt.show()

# plt.plot(np.squeeze(cost2tr)[-50:],'-r', np.squeeze(cost2D)[-50:],'-b')
# plt.ylabel('task 2 cost')
# plt.xlabel('iterations (per tens)')
# plt.show()

train(X_valid, YY_valid, X_valid, YY_valid, X_valid, YY_valid)


# coding: utf-8
# this code runs in Python 3.4.3 env

import urllib.request
import json

serviceurl = 'http://python-data.dr-chuck.net/geojson?'

while True:
    uni = input('Enter university: ')
    if len(uni) < 1 : uni = 'University of Utah'
    #	address = address.decode('utf8')
    print(uni)

    url = serviceurl + urllib.parse.urlencode({'sensor':'false', 'address': uni})

    print('Retrieving', url)
    with urllib.request.urlopen(url) as uh:
        data = uh.read()
    print('Retrieved',len(data),'characters')

    try:
        js = json.loads(data.decode())
    except:
        js = None
        print(json.dumps(js, indent=4))

    if "status" not in js or js['status'] != 'OK':
        print ('=====Failure To Retrieve=====')
        continue
    place_id = js["results"][0]["place_id"]
    print(place_id)


#! /usr/local/bin/python

from pathlib import Path
import pandas
import mysql.connector
from sqlalchemy import create_engine
import data.Database_Credentials as Database_Credentials

#Section: Create SQLAlchemy Engine
Database = 'Collections_Assessment_Warehouse_0.1'

Engine = create_engine(
    'mysql+mysqlconnector://' +
    Database_Credentials.Username + ':' +
    Database_Credentials.Password + '@' +
    Database_Credentials.Host + ':' + str(Database_Credentials.Post) + '/' +
    Database,
    echo=False # Should logging output be silenced?
)

#Section: Run SQL Query
SQL_Query = input("Enter the SQL statement to run (without newlines): ")

Query_Results = pandas.read_sql(
    sql=SQL_Query,
    con=Engine
)

#Section: Save Query Results to CSV
File_Name = input("Enter the name of the file (not including path or file extension info); don't use spaces or slashes. ")
File = str(Path('.', 'data', 'SQL_Output', File_Name + '.csv'))

Query_Results.to_csv(
    path_or_buf=File,
    index=False
)


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index),
    url(r'^processreg$', views.processreg),
    url(r'^processlog$', views.processlog),
    url(r'^logout$', views.logout),
    url(r'^success$', views.success),
]


#
# BSD 3-Clause License
#
# Copyright (c) 2017 xxxx
# All rights reserved.
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ============================================================================
#
import numpy as np
import logging
import pathlib
import xml.etree.ElementTree as ET
import cv2
import os

class VOCDataset:

    def __init__(self, root, transform=None, target_transform=None, is_test=False, keep_difficult=False, label_file=None):
        """Dataset for VOC data.
        Args:
            root: the root of the VOC2007 or VOC2012 dataset, the directory contains the following sub-directories:
                Annotations, ImageSets, JPEGImages, SegmentationClass, SegmentationObject.
        """
        self.root = pathlib.Path(root)
        self.transform = transform
        self.target_transform = target_transform
        if is_test:
            image_sets_file = self.root / "ImageSets/Main/test.txt"
        else:
            image_sets_file = self.root / "ImageSets/Main/trainval.txt"
        self.ids = VOCDataset._read_image_ids(image_sets_file)
        self.keep_difficult = keep_difficult

        # if the labels file exists, read in the class names
        label_file_name = self.root / "labels.txt"

        if os.path.isfile(label_file_name):
            class_string = ""
            with open(label_file_name, 'r') as infile:
                for line in infile:
                    class_string += line.rstrip()

            # classes should be a comma separated list

            classes = class_string.split(',')
            # prepend BACKGROUND as first class
            classes.insert(0, 'BACKGROUND')
            classes  = [ elem.replace(" ", "") for elem in classes]
            self.class_names = tuple(classes)
            logging.info("VOC Labels read from file: " + str(self.class_names))

        else:
            logging.info("No labels file, using default VOC classes.")
            self.class_names = ('BACKGROUND',
            'aeroplane', 'bicycle', 'bird', 'boat',
            'bottle', 'bus', 'car', 'cat', 'chair',
            'cow', 'diningtable', 'dog', 'horse',
            'motorbike', 'person', 'pottedplant',
            'sheep', 'sofa', 'train', 'tvmonitor')

        self.class_dict = {class_name: i for i, class_name in enumerate(self.class_names)}

    def __getitem__(self, index):
        image_id = self.ids[index]
        boxes, labels, is_difficult = self._get_annotation(image_id)
        if not self.keep_difficult:
            boxes = boxes[is_difficult == 0]
            labels = labels[is_difficult == 0]
        image = self._read_image(image_id)
        if self.transform:
            image, boxes, labels = self.transform(image, boxes, labels)
        if self.target_transform:
            boxes, labels = self.target_transform(boxes, labels)
        return image, boxes, labels

    def get_image(self, index):
        image_id = self.ids[index]
        image = self._read_image(image_id)
        if self.transform:
            image, _ = self.transform(image)
        return image

    def get_annotation(self, index):
        image_id = self.ids[index]
        return image_id, self._get_annotation(image_id)

    def __len__(self):
        return len(self.ids)

    @staticmethod
    def _read_image_ids(image_sets_file):
        ids = []
        with open(image_sets_file) as f:
            for line in f:
                ids.append(line.rstrip())
        return ids

    def _get_annotation(self, image_id):
        annotation_file = self.root / f"Annotations/{image_id}.xml"
        objects = ET.parse(annotation_file).findall("object")
        boxes = []
        labels = []
        is_difficult = []
        for object in objects:
            class_name = object.find('name').text.lower().strip()
            # we're only concerned with clases in our list
            if class_name in self.class_dict:
                bbox = object.find('bndbox')

                # VOC dataset format follows Matlab, in which indexes start from 0
                x1 = float(bbox.find('xmin').text) - 1
                y1 = float(bbox.find('ymin').text) - 1
                x2 = float(bbox.find('xmax').text) - 1
                y2 = float(bbox.find('ymax').text) - 1
                boxes.append([x1, y1, x2, y2])

                labels.append(self.class_dict[class_name])
                is_difficult_str = object.find('difficult').text
                is_difficult.append(int(is_difficult_str) if is_difficult_str else 0)

        return (np.array(boxes, dtype=np.float32),
                np.array(labels, dtype=np.int64),
                np.array(is_difficult, dtype=np.uint8))

    def _read_image(self, image_id):
        image_file = self.root / f"JPEGImages/{image_id}.jpg"
        image = cv2.imread(str(image_file))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        return image