from datetime import date
from typing import List

from implementation.market_app.models.api_models import ApartmentOfferAveragePrice, CompanyStatisticResult, \
    SaleOfferStatusUpdate
from implementation.market_app.models.postgres_models import Offer
from implementation.market_app.repositories.repository_dependencies import get_postgres_db

class PostgresOfferRepository:

    def __init__(self):
        self.connection = get_postgres_db()

    def create_table(self):
        cursor = self.connection.cursor()
        create_table_query = """
                CREATE TABLE IF NOT EXISTS offer (
                    id SERIAL PRIMARY KEY,
                    price DECIMAL(8,2) NOT NULL,
                    status VARCHAR(8) NOT NULL,
                    negotiable BOOLEAN NOT NULL,
                    description VARCHAR(300),
                    agency_fee DECIMAL(8,2),
                    creation_date DATE NOT NULL,
                    modification_date DATE,
                    apartment_id INTEGER REFERENCES apartment(id) ON DELETE CASCADE
                );
            """
        cursor.execute(create_table_query)
        self.connection.commit()

    def insert(self, sale_offer: Offer) -> Offer:
        with self.connection.cursor() as cur:
            cur.execute(
                "INSERT INTO offer (price, status, negotiable, description, agency_fee, creation_date,modification_date,apartment_id) VALUES (%s, %s, %s, "
                "%s, %s,%s,%s,%s) RETURNING id",
                (sale_offer.price, sale_offer.status, sale_offer.negotiable, sale_offer.description,sale_offer.agency_fee, sale_offer.creation_date,sale_offer.modification_date,sale_offer.apartment_id),
            )
            new_id = cur.fetchone()[0]
        self.connection.commit()
        return new_id

    def get_offers_by_city_and_address(self, city: str, street: str) -> List[Offer]:
        with self.connection.cursor() as cursor:
            cursor.execute(
                """
                SELECT * from offer where apartment_id in (SELECT apartment_id from apartment where address_id in
                (SELECT address_id from address where city = %s and street_name = %s));
                """, (city, street))
            offers = cursor.fetchall()
        return [
            Offer(offer[0], offer[1], offer[2], offer[3], offer[4],
                  offer[5], offer[6], offer[7], offer[8]) for offer
            in offers]

    def get_offers_by_city_and_price_range(self, city: str, min_price: float, max_price: float) -> List[Offer]:
        with self.connection.cursor() as cursor:
            cursor.execute(
                """
                SELECT * FROM offer as so
                JOIN apartment as ap on so.apartment_id = ap.id
                JOIN address as ad on ap.address_id = ad.id
                WHERE ad.city = %s and so.price BETWEEN %s and %s
                """, (city, min_price, max_price))
            offers = cursor.fetchall()
        return [
            Offer(offer[0], offer[1], offer[2], offer[3], offer[4],
                  offer[5], offer[6], offer[7], offer[9]) for offer
            in offers]

    def get_offers_by_company_name(self, company_name: str) -> List[Offer]:
        with self.connection.cursor() as cursor:
            cursor.execute(
                """
                SELECT * FROM offer as so
                JOIN apartment as ap on so.apartment_id = ap.id
                JOIN owner as ow on ap.owner_id = ow.id
                WHERE ow.company_name = %s
                """, (company_name,))
            offers = cursor.fetchall()
        return [
            Offer(offer[0], offer[1], offer[2], offer[3], offer[4],
                  offer[5], offer[6], offer[7], offer[8]) for offer
            in offers]

    ## chyba nie ma sensu?
    # def get_offers_by_city_and_price_and_id(self, city: str, price: float, id: int) -> List[
    #     SaleOffer]:
    #     with self.connection.cursor() as cursor:
    #         cursor.execute(
    #             """
    #             SELECT * FROM sale_offer as so
    #             JOIN apartment as ap on so.apartment_id = ap.id
    #             JOIN address as ad on ap.address_id = ad.id
    #             WHERE ad.city = %s and so.price = %s and so.id = %s
    #             """, (city, price, id))
    #         offers = cursor.fetchall()
    #         offers_list = []
    #     for offer in offers:
    #         sale_offer = SaleOffer(offer_id=offer['offer_id'],
    #                                price=offer['price'],
    #                                status=offer['status'],
    #                                negotiable=offer['negotiable'],
    #                                description=offer['description'],
    #                                agency_fee=offer['agency_fee'],
    #                                creation_date=offer['creation_date'],
    #                                modification_date=offer['modification_date'],
    #                                apartment_id=offer['apartment_id']
    #                                )
    #         offers_list.append(sale_offer)
    #         return offers_list

    def update_sale_offer_status(self, offer_id: int, update_info: SaleOfferStatusUpdate) -> Offer:
        with self.connection.cursor() as cursor:
            cursor.execute("""
                UPDATE offer
                SET status = %s
                WHERE id = %s
                RETURNING *
            """, (update_info.status, offer_id))
            self.connection.commit()
            updated_offer = cursor.fetchone()
        return Offer(*updated_offer)

    def get_average_price_by_city(self, city: str) -> List[ApartmentOfferAveragePrice]:
        with self.connection.cursor() as cursor:
            cursor.execute("""
                SELECT ad.city, AVG(so.price) as avg_price, AVG(so.price/ap.area) as avg_price_per_m2
                FROM offer as so
                JOIN apartment as ap on so.apartment_id = ap.id
                JOIN address as ad on ap.address_id = ad.id
                WHERE ad.city = %s
                GROUP BY ad.city
            """, (city,))
            result = cursor.fetchall()
        return [ApartmentOfferAveragePrice(city=row[0], avg_price=row[1], avg_price_per_m2=row[2]) for row in result]

    def get_companies_sales_statistics(self, company_name: str) -> List[CompanyStatisticResult]:
        with self.connection.cursor() as cursor:
            cursor.execute("""
                SELECT
                    ow.company_name as company_name,
                    COUNT(so.id) as total_sales,
                    SUM(so.price) as total_sales_price,
                    AVG(so.price) as average_sales_price
                FROM offer as so
                JOIN apartment as ap on so.apartment_id = ap.id
                JOIN owner as ow on ap.owner_id = ow.id
                WHERE ow.company_name = %s
                GROUP BY ow.company_name
            """, (company_name,))
            result = cursor.fetchall()
        return [CompanyStatisticResult(company_name=row[0], total_sales=row[1], total_sales_price=row[2],
                                       average_sales_price=row[3]) for row in result]

    def get_statistic_by_company(self, company_filter: str) -> List[CompanyStatisticResult]:
        with self.connection.cursor() as cursor:
            cursor.execute(
                """
                SELECT company_name,
                       AVG(price) as avg_price,
                       AVG(price/area) as avg_price_per_m2,
                       COUNT(*) as sales_offer_count
                FROM market_app.offers_by_company_name
                WHERE company_name = %s
                GROUP BY company_name
                """, (company_filter,))
            results = cursor.fetchall()
        return [CompanyStatisticResult(company_name=row[0],
                                       avg_price=row[1],
                                       avg_price_per_m2=row[2],
                                       sales_offer_count=row[3])
                for row in results]

    def delete(self, id: int) -> None:
        with self.connection.cursor() as cursor:
            cursor.execute("DELETE FROM offer WHERE id = %s", (id,))
            self.connection.commit()

    def update(self, id: int, price: float, status: str, negotiable: bool, description: str, agency_fee: float,
               creation_date: date, modification_date: date, apartment_id: int) -> Offer:
        with self.connection.cursor() as cursor:
            cursor.execute("""
                UPDATE offer
                SET price = %s, status = %s, negotiable = %s, description = %s, agency_fee = %s,
                creation_date = %s, modification_date = %s,
                apartment_id = %s
                WHERE id = %s
                """, (
                price, status, negotiable, description, agency_fee, creation_date, modification_date, apartment_id, id))
            self.connection.commit()

        updated_offer = self.get_by_id(id)
        return updated_offer

    def get_by_id(self, sale_offer_id: int) -> Offer | None:
        with self.connection.cursor() as cursor:
            cursor.execute("SELECT * FROM offer WHERE id = %s", (sale_offer_id,))
            result = cursor.fetchone()
        if result:
            return Offer(*result)
        else:
            return None

    def get_all(self) -> List[Offer]:
        with self.connection.cursor() as cursor:
            cursor.execute("SELECT * FROM offer")
            results = cursor.fetchall()
        return [Offer(*result) for result in results]


import pytest

import pyannote.core
from pyannote.core import Annotation
from pyannote.core import Segment
from pyannote.core import Timeline
from pyannote.metrics.diarization import DiarizationErrorRate
from pyannote.metrics.diarization import DiarizationPurity
from pyannote.metrics.diarization import DiarizationCoverage

import numpy.testing as npt

@pytest.fixture
def reference():
    reference = Annotation()
    reference[Segment(0, 10)] = 'A'
    reference[Segment(12, 20)] = 'B'
    reference[Segment(24, 27)] = 'A'
    reference[Segment(30, 40)] = 'C'
    return reference

@pytest.fixture
def reference_with_overlap():
    reference = Annotation()
    reference[Segment(0, 13)] = 'A'
    reference[Segment(12, 20)] = 'B'
    reference[Segment(24, 27)] = 'A'
    reference[Segment(30, 40)] = 'C'
    return reference

@pytest.fixture
def hypothesis():
    hypothesis = Annotation()
    hypothesis[Segment(2, 13)] = 'a'
    hypothesis[Segment(13, 14)] = 'd'
    hypothesis[Segment(14, 20)] = 'b'
    hypothesis[Segment(22, 38)] = 'c'
    hypothesis[Segment(38, 40)] = 'd'
    return hypothesis

def test_error_rate(reference, hypothesis):
    diarizationErrorRate = DiarizationErrorRate()
    error_rate = diarizationErrorRate(reference, hypothesis)
    npt.assert_almost_equal(error_rate, 0.5161290322580645, decimal=7)

def test_optimal_mapping(reference, hypothesis):
    diarizationErrorRate = DiarizationErrorRate()
    mapping = diarizationErrorRate.optimal_mapping(reference, hypothesis)
    assert mapping == {'a': 'A', 'b': 'B', 'c': 'C'}

def test_detailed(reference, hypothesis):
    diarizationErrorRate = DiarizationErrorRate()
    details = diarizationErrorRate(reference, hypothesis, detailed=True)

    confusion = details['confusion']
    npt.assert_almost_equal(confusion, 7.0, decimal=7)

    correct = details['correct']
    npt.assert_almost_equal(correct, 22.0, decimal=7)

    rate = details['diarization error rate']
    npt.assert_almost_equal(rate, 0.5161290322580645, decimal=7)

    false_alarm = details['false alarm']
    npt.assert_almost_equal(false_alarm, 7.0, decimal=7)

    missed_detection = details['missed detection']
    npt.assert_almost_equal(missed_detection, 2.0, decimal=7)

    total = details['total']
    npt.assert_almost_equal(total, 31.0, decimal=7)

def test_purity(reference, hypothesis):
    diarizationPurity = DiarizationPurity()
    purity = diarizationPurity(reference, hypothesis)
    npt.assert_almost_equal(purity, 0.8275, decimal=3)

def test_coverage(reference, hypothesis):
    diarizationCoverage = DiarizationCoverage()
    coverage = diarizationCoverage(reference, hypothesis)
    npt.assert_almost_equal(coverage, 0.7586, decimal=3)

def test_skip_overlap(reference_with_overlap, hypothesis):
    metric = DiarizationErrorRate(skip_overlap=True)
    total = metric(reference_with_overlap, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 32, decimal=3)

def test_leep_overlap(reference_with_overlap, hypothesis):
    metric = DiarizationErrorRate(skip_overlap=False)
    total = metric(reference_with_overlap, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 34, decimal=3)

def test_bug_16():
    reference = Annotation()
    reference[Segment(0, 10)] = 'A'
    hypothesis = Annotation()

    metric = DiarizationErrorRate(collar=1)
    total = metric(reference, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 9, decimal=3)

    metric = DiarizationErrorRate(collar=0)
    total = metric(reference, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 10, decimal=3)


# -*- coding: utf-8 -*-
# @Author: cyang
# @Date:   2020-09-13 10:44:28
# @Last Modified by:   cyang
# @Last Modified time: 2020-09-19 15:47:58

import os
import send2trash

SUFFIX = ('.torrent', '.txt', '.url', 'rar', '.jpg', '.mht', '.gif', '.png', '.mhtml', '.xltd', '.chm', '.bpm', '.apk', 'pdf')
# SUFFIX = ('.torrent', '.txt', '.url', 'rar', '.mht','.mhtml', '.xltd', '.chm', '.ass')
PATH = 'D:\迅雷下载'

def deleteFile(path):
	for folderName, subFolders, fileNames in os.walk(path):
		print('\n-----------------------------------')
		print('The current folder is ' + folderName)
		# print(subFolders, fileNames)
		if not os.listdir(folderName):
			print(folderName + ' is an empty folder, remove it')
			filePath = os.path.join(folderName)
			print('#send2trash#: ' + filePath)
			send2trash.send2trash(filePath)

		for subFolder in subFolders:
			print('SUBFOLDER of ' + folderName + ': ' + subFolder)
		for fileName in fileNames:
			print('FILE INSIDE ' + folderName + ': ' + fileName)

			#remove file to trash if its suffix is one of SUFFIX
			if fileName.endswith(SUFFIX):
				filePath = os.path.join(folderName, fileName)
				print('#send2trash#: ' + filePath)
				send2trash.send2trash(filePath)
			elif fileName.endswith('.mp4'):
				filePath = os.path.join(folderName, fileName)
				fileSize = os.path.getsize(filePath)
				fileSize = round(fileSize/float(1024*1024), 2)
				print('%s, %d MB' % (filePath, fileSize))

				if fileSize < 100:
					print('#send2trash#: ' + filePath)
					send2trash.send2trash(filePath)

if __name__ == '__main__':
	deleteFile(PATH)


import tkinter as tk
from tkinter import filedialog as fd
from PIL import Image, ImageTk

window = tk.Tk()
window.title("Image Viewer")
window.resizable(width=False, height=False)

images = []
cursor = 0

def open_image():
    global cursor
    filename = fd.askopenfilename(
        title="Select an image",
        initialdir="/",
        )

    file = Image.open(filename)

    if file.width > 1000:
        file = file.resize((1000, file.height * 1000 // file.width))
    elif file.height > 1000:
        file = file.resize((file.width * 1000 // file.height, 1000))

    img = ImageTk.PhotoImage(file)
    images.append(img)
    cursor = len(images) - 1
    display_image(img)

def display_image(img):
    global lbl_image

    if len(images) > 1:
        lbl_image.grid_forget()

    lbl_image = tk.Label(image=img)
    lbl_image.grid(row=1, column=0)
    lbl_index = tk.Label(text=f"{cursor + 1} / {len(images)}")
    lbl_index.grid(row=0, column=0)

def forward():
    global cursor
    if cursor < len(images) - 1:
        cursor += 1
        display_image(images[cursor])

def back():
    global cursor
    if cursor > 0:
        cursor -= 1
        display_image(images[cursor])

lbl_index = tk.Label()
lbl_image = tk.Label(text="Display an image here!", width=15, height=15)
frm_buttons = tk.Frame(master=window)

btn_back = tk.Button(
    master=frm_buttons,
    text="<<",
    command=back,
    )

btn_addImage = tk.Button(
    master=frm_buttons,
    text="Add Image",
    command=open_image,
    )

btn_forward = tk.Button(
    master=frm_buttons,
    text=">>",
    command=forward,
    )

lbl_index.grid(row=0, column=0)
lbl_image.grid(row=1, column=0)
frm_buttons.grid(row=2, column=0)
btn_back.grid(row=0, column=0)
btn_addImage.grid(row=0, column=1)
btn_forward.grid(row=0, column=2)

window.mainloop()


import os

import pandas as pd
from py._path.local import LocalPath

from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import MinMaxScaler

from src.run_pipeline import train, run_pipeline
from src.entities import (
    SplittingParams,
    TrainParams,
    PredictParams,
    ModelParams,
    TransformParams,
    ConfigParams,
)

def test_run_pipeline(
    tmpdir: LocalPath,
    dataset_path: str,
    target_col: str,
):

    params = ConfigParams(
        do_train=True,
        do_predict=True,
        bin_model_path=tmpdir.join("bin_model.json"),
        train_params=TrainParams(
            train_data_path=dataset_path,
            target_col=target_col,
            train_metric_path=tmpdir.join("metrics.json"),
            splitting_params=SplittingParams(val_size=0.2, random_state=239),
        ),
        transformer=TransformParams(
            builder="",
        ),
        model=ModelParams(
            params={},
            module="sklearn.tree.DecisionTreeClassifier",
        ),
        test_params=PredictParams(
            target_col=target_col,
            predict_metric_path=None,
            predict_data_path=dataset_path,
            out_predict_path=tmpdir.join("predicts.csv"),
        ),
    )

    run_pipeline(params)

def test_train(
    tmpdir: LocalPath,
    dataset_path: str,
    target_col: str,
):
    expected_metric_path = tmpdir.join("metrics.json")
    params = TrainParams(
        train_data_path=dataset_path,
        target_col=target_col,
        train_metric_path=expected_metric_path,
        splitting_params=SplittingParams(val_size=0.2, random_state=239),
    )
    pipeline = Pipeline(
        [("transformer", MinMaxScaler()), ("model", LogisticRegression())]
    )
    pipeline = train(pipeline, params)

    data = pd.read_csv(dataset_path)
    X = data.drop(columns=target_col)
    y = data[target_col]

    assert pipeline.score(X, y) > 0.8
    assert os.path.exists(params.train_metric_path)


from django.urls import path
from . import views
app_name = 'carts'
urlpatterns = [
    # 购物车管理
    path('carts/', views.CartsView.as_view(), name='info'),
    # 选择购物车商品
    path('carts/selection/', views.CartsSelectAllView.as_view()),
    # 简单购物车
    path('carts/simple/', views.CartsSimpleView.as_view()),
]


'''
term frequency and term term distribution based keyword extraction
'''

import re
import os
import nltk
from nltk.stem.porter import *
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from collections import defaultdict

class TFD(object):
    def __init__(self, data_dir, rank = 36):
        self.data_dir = data_dir
        self.stopwords = stopwords.words('english')
        self.stemmer = PorterStemmer()
        self.lemmatizer = WordNetLemmatizer()
        self.rank = rank

    def pre_process(self, text):
        text = re.sub(r'[^a-z-]+', ' ', text)
        text = re.split(r'[^a-z-]+', text)
        text = [word for word in text if word not in self.stopwords]
        text = [self.stemmer.stem(word) for word in text]
        text = [self.lemmatizer.lemmatize(word) for word in text]
        text = [word for word in text if len(word) > 1]
        return text

    def get_doc(self, f):
        self.doc = f.read().lower()
        self.doc = self.pre_process(self.doc)

    def get_key(self, f):
        self.key = []
        for line in f.readlines():
            line = self.pre_process(line.lower())
            line = ' '.join(line)
            self.key.append(line)

    def append_ngrams(self, tokens):
        temp = []
        bigram = [' '.join(tokens[i:i+2]) for i in xrange(len(tokens) - 1)]
        trigram = [' '.join(tokens[i:i+3]) for i in xrange(len(tokens) - 2)]
        fourgram = [' '.join(tokens[i:i+4]) for i in xrange(len(tokens) - 3)]
        temp = tokens + bigram + trigram + fourgram
        return temp

    def partition_doc(self, n = 5):
        self.d = dict()
        bin = len(self.doc) / n  #number of terms in each part
        for i in xrange(n):
            self.d[i] = self.doc[i*bin : (i+1)*bin]
        self.d[n-1] = self.doc[(i+1)*bin : ]
        for i in xrange(n):
            self.d[i] = self.append_ngrams(self.d[i])
        return self.d

    def get_term_frequency(self):
        self.tf = dict()
        tokens = self.append_ngrams(self.doc)
        for term in tokens:
            self.tf[term] = self.tf.get(term, 0) + 1
        return self.tf

    def get_term_distribution(self):
        self.td = dict()
        self.partition_doc()
        for term in set(self.append_ngrams(self.doc)):
            for key in self.d:
                if term in self.d[key]:
                    self.td[term] = self.td.get(term, 0) + 1
        return self.td

    def get_term_score(self):
        if self.method == 'tf':
            self.score = self.get_term_frequency()
        if self.method == 'td':
            self.score = self.get_term_distribution()
        if self.method == 'tfd':
            self.get_term_distribution()
            self.get_term_frequency()
            self.score = {key:self.tf[key] + self.td[key] for key in self.td}

    def get_ans(self, ans_file):
        with open(self.data_dir + ans_file, 'r') as f:
            self.ans = []
            for line in f.readlines():
                line = self.pre_process(line.lower())
                line = ' '.join(line)
                self.ans.append(line)
            self.ans = [word for word in self.ans if word in self.append_ngrams(self.doc)]

    def fit(self, filename, ans_file):
        self.get_keywords(filename, self.method)
        self.get_ans(ans_file)

    def get_keywords(self, filename, method):
        self.method = method
        with open(self.data_dir + filename, 'r') as f:
            self.get_doc(f)
            self.get_term_score()
            doclen = len(set(self.append_ngrams(self.doc)))
            self.kw = [items[0] for items in sorted(self.score.items(), key = lambda item: item[1])[-doclen/self.rank:]]
            return self.kw

    def get_accuracy_recall(self):
        hit = len([word for word in self.kw if word in self.ans])
        accuracy = hit*1.0 / len(self.kw)
        recall = hit*1.0 / len(self.ans)
        return accuracy, recall

    def get_accuracy_recall2(self, method):
        self.method = method
        accuracy = recall = 0.0
        nfiles = 0.0
        for filename in os.listdir(self.data_dir):
            if filename.endswith('txt'):
                nfiles += 1
                ans_file = filename[:-3] + 'key'
                self.fit(filename, ans_file)
                a, r = self.get_accuracy_recall()
                accuracy += a
                recall += r
        return accuracy/nfiles, recall/nfiles


import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import matplotlib.pyplot as plt
import torch.nn.functional as F
import re
def build_corpus(raw_corpus):
    corpus = []
    for sentence in raw_corpus:
        pattern = re.compile(r'[A-Za-z]+[\w^\']*|[\w^\']*[A-Za-z]+[\w^\']*')
        corpus.append(pattern.findall(sentence.lower()))
    return corpus

raw_corpus = ['i dance',
          'she cook for me',
          'he sing with my mon at home']

corpus = build_corpus(raw_corpus)

vocab = []
for sentence in corpus:
    for word in sentence:
        if word not in vocab and word:
            vocab.append(word.lower())
vocab.sort()

# print(vocab)
# print(word2idx)

END_SENT = "E"
START_SENT = "S"
PARSING = "P"

max_len = 0
for sentence in corpus:
    if len(sentence) > max_len:
        max_len = len(sentence)
# print(max_len)

def passing_word(corpus, max_len, START_SENT='S', END_SENT='E', PARSING='P'):
    for i in range(len(corpus)):
        len_sent = len(corpus[i])
        num_parsing = max_len - len_sent
        corpus[i].insert(0, START_SENT)
        for j in range(num_parsing):
            corpus[i].extend(PARSING)
        corpus[i].insert(len(corpus[i]), END_SENT)
    return corpus

corpus = passing_word(corpus, max_len)
# print(corpus)
max_len = max_len + 2
vocab.insert(0, END_SENT)
vocab.insert(0, START_SENT)
vocab.insert(0, PARSING)
word2idx = {word: i for i, word in enumerate(vocab)}
# print(vocab)
# print(word2idx)

def convert_to_idx(sentence, word2idx):
    sentence_idx = []
    for word in sentence:
        sentence_idx.append(word2idx[word])
    return sentence_idx
corpus_idx = []
for sentence in corpus:
    corpus_idx.append(convert_to_idx(sentence, word2idx))
# print(corpus)
# print(corpus_idx)
def outputs_shifted_right(sentence_idx):
    len_sentence = len(sentence_idx)
    outputs = np.zeros((len_sentence, max_len))
    for i in range(len_sentence):
        for j in range(len_sentence):
            if j <= i:
                outputs[i][j] = sentence_idx[j]
    outputs.tolist()
    return torch.LongTensor(outputs)

# print(outputs_shifted_right(corpus_idx[1]))

# build model
vocab_size = len(vocab)
src_len = 5
tgt_len = 5

d_model = 512  # Embedding Size
d_ff = 2048  # FeedForward dimension
d_k = d_v = 64  # dimension of K(=Q), V
n_layers = 6  # number of Encoder of Decoder Layer
n_heads = 8  # number of heads in Multi-Head Attention

def poition_encoder(n_position, d_model):
    def compute_angle(pos, i):
        return pos / np.power(10000, 2*i/d_model)
    sinusoid_table = np.zeros((n_position, d_model))
    for pos in range(n_position):
        for i in range(d_model):
            if i % 2 == 0:
                sinusoid_table[pos][i] = np.sin(compute_angle(pos, i))
            else:
                sinusoid_table[pos][i] = np.cos(compute_angle(pos, i))
    return torch.FloatTensor(sinusoid_table)

class ScaledDotProductAttention(nn.Module):
    def __init__(self):
        super(ScaledDotProductAttention, self).__init__()
    def forward(self, Q, K, V):
        scores = torch.matmul(Q, K.t()) / np.sqrt(d_k)
        attn = nn.Softmax(dim=-1)(scores)
        context = torch.matmul(attn, V)
        return context

class MultilHeadAttention(nn.Module):
    def __init__(self):
        super(MultilHeadAttention, self).__init__()
        self.W_Q = nn.Linear(d_model, d_k*n_heads)
        self.W_K = nn.Linear(d_model, d_k*n_heads)
        self.W_V = nn.Linear(d_model, d_v*n_heads)
    def forward(self, Q, K, V):
        residual, batch_size = Q, Q.size(0)
        Q_scores = self.W_Q(Q)
        K_scores = self.W_K(K)
        V_scores = self.W_V(V)
        Qs = Q_scores.view(batch_size, n_heads, d_k)
        Ks = K_scores.view(batch_size, n_heads, d_k)
        Vs = V_scores.view(batch_size, n_heads, d_v)
        contexts = []
        for i in range(n_heads):
            context= ScaledDotProductAttention()(Qs[:, i, :], Ks[:, i, :], Vs[:, i, :])
            contexts.append(context)
        context = torch.cat(contexts, dim=1)
        output = nn.Linear(n_heads*d_v, d_model)(context)
        return nn.LayerNorm(d_model)(output + residual)    # add & norm

class FeedForwardNet(nn.Module):
    def __init__(self, d_model, d_ff, dropout=0.1):
        super(FeedForwardNet, self).__init__()
        self.w_1 = nn.Linear(d_model, d_ff)
        self.w_2 = nn.Linear(d_ff, d_model)
        self.dropout = nn.Dropout(dropout)

    def forward(self, inputs):
        residual = inputs
        output = nn.ReLU()(self.w_1(inputs))
        output = self.w_2(output)
        return nn.LayerNorm(d_model)(output + residual)

class EncoderLayer(nn.Module):
    def __init__(self):
        super(EncoderLayer, self).__init__()
        self.enc_self_attn = MultilHeadAttention()
        self.enc_ff = FeedForwardNet(d_model, d_ff)
    def forward(self, enc_inputs):
        enc_outputs = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs)
        enc_outputs = self.enc_ff(enc_outputs)
        return enc_outputs

class DecoderLayer(nn.Module):
    def __init__(self):
        super(DecoderLayer, self).__init__()
        self.dec_self_attn = MultilHeadAttention()
        self.dec_enc_attn = MultilHeadAttention()
        self.dec_ff = FeedForwardNet(d_model, d_ff)
    def forward(self, dec_inputs, enc_outputs):
        dec_outputs = self.dec_self_attn(dec_inputs, dec_inputs, dec_inputs)
        dec_outputs = self.dec_enc_attn(enc_outputs, enc_outputs, dec_outputs)
        dec_outputs = self.dec_ff(dec_outputs)
        return dec_outputs

class Encoder(nn.Module):
    def __init__(self):
        super(Encoder, self).__init__()
        self.emb = nn.Embedding(vocab_size, d_model)
        self.pos_emb = nn.Embedding.from_pretrained(poition_encoder(max_len, d_model), freeze=True)
        self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])
    def forward(self, enc_inputs):
        enc_outputs = self.emb(enc_inputs)
        for layer in self.layers:
            enc_outputs = layer(enc_outputs)
        return enc_outputs

class Decoder(nn.Module):
    def __init__(self):
        super(Decoder, self).__init__()
        self.emb = nn.Embedding(vocab_size, d_model)
        self.pos_emb = nn.Embedding.from_pretrained(poition_encoder(max_len, d_model), freeze=True)
        self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])

    def forward(self, dec_inputs, enc_outputs):
        dec_outputs = self.emb(dec_inputs)
        for layer in self.layers:
            dec_outputs = layer(dec_outputs, enc_outputs)
        return dec_outputs

class Transformer(nn.Module):
    def __init__(self):
        super(Transformer, self).__init__()
        self.encoder = Encoder()
        self.decoder = Decoder()
        self.projection = nn.Linear(d_model, vocab_size, bias=False)

    def forward(self, enc_inputs, dec_inputs):
        enc_outputs = self.encoder(enc_inputs)
        dec_outputs = self.decoder(dec_inputs, enc_outputs)
        dec_logits = self.projection(dec_outputs)
        return F.log_softmax(dec_logits.view(-1, dec_logits.size(-1)), dim=-1)

model = Transformer()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

def showgraph(attn):
    attn = attn[-1].squeeze(0)[0]
    attn = attn.squeeze(0).data.numpy()
    fig = plt.figure(figsize=(n_heads, n_heads)) # [n_heads, n_heads]
    ax = fig.add_subplot(1, 1, 1)
    ax.matshow(attn, cmap='viridis')
    ax.set_xticklabels(['']+raw_corpus[0].split(), fontdict={'fontsize': 14}, rotation=90)
    ax.set_yticklabels(['']+raw_corpus[2].split(), fontdict={'fontsize': 14})
    plt.show()

n_epochs = 100

for epoch in range(n_epochs):
    optimizer.zero_grad()
    total_loss = 0
    for sentence in corpus_idx:
        enc_inputs = Variable(torch.LongTensor(sentence))
        outputs_sr = outputs_shifted_right(sentence)
        for i in range(len(sentence)-1):
            dec_inputs, target = outputs_sr[i], outputs_sr[i+1]
            dec_inputs = Variable(dec_inputs)
            outputs = model(enc_inputs, dec_inputs)
            loss = criterion(outputs, target.contiguous().view(-1))

            total_loss += loss
            loss.backward()
            optimizer.step()
    # print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(total_loss))

torch.save(model.state_dict(), 'models/transformer_v1.pkl')


import collectorbase
import matplotlib.dates as mdates
import datetime

class Loc(collectorbase.Collector):

    def __init__(self, **kwargs):
        self.code = 'code' in kwargs and kwargs['code'] or False
        self.comments = 'comments' in kwargs and kwargs['comments'] or False
        self.blanks = 'blanks' in kwargs and kwargs['blanks'] or False

    def collect(self, branch):
        x, y1, y2, y3 = [], [], [], []
        code, comments, blanks = 0, 0, 0
        for rev in branch.revisions:
            (c_code, c_comments, c_blanks) = rev.getLocDiff()

            code += c_code
            comments += c_comments
            blanks += c_blanks

            x.append(rev.commitdate)
            y1.append(code+comments+blanks)
            y2.append(code+comments)
            y3.append(code)

        self.data = [
            (x, y1, dict(label='blanks',color='#96afd6')),
            (x, y2, dict(label='comments',color='#597db9')),
            (x, y3, dict(label='code',color='#3f5578')),
        ]
        self.mode = 'fill'
        self.format_xdata = mdates.DateFormatter('%b %Y')
        self.xaxis_date = True
        self.output = 'loc-%s.png' % branch.name.replace('/', '-')

# Modeline for vim: set tw=79 et ts=4:


# sunnyspeed studio
# GitHub: https://github.com/sunnyspeedstudio/favorite-focal-length
# YouTube: https://www.youtube.com/sunnyspeedstudio
# bilibili: http://space.bilibili.com/36609936
# Tested using python 3 under mac and windows

import subprocess
import os
import operator
import datetime
import pygal
import pickle

# all video folders need to be scanned (the script will go through all subfolders)
videoFolders = [  'path1',
                'path2' ]
# videoFolders = ['/Volumes/MEDIA/Camera - Old/2016-03 - Sony A6300', '/Volumes/MEDIA/Camera - Old/2016-12 - Sony A6500', '/Volumes/MEDIA/Camera - Old/2019-03 - Sony A6400', '/Volumes/MEDIA/Camera - Current/2018-03 - Sony A7R III']
# all video extensions need to be scanned
videoExtensions = ('.mp4', '.MP4')
# exiftool path
exiftoolPath = 'exiftool'

# funcions:
def formatDuration(s):
    hour = int(s / 3600)
    minute = int((s - hour * 3600) / 60)
    second = s - hour * 3600 - minute * 60
    return str(hour) + ':' + str(minute) + ':' + str(second)

def drawBarChart(title, data):
    value = []
    label = []
    for x in data:
        if ('(time in seconds)' in title):
            label.append(str(x[0]) + ' (' + formatDuration(x[1]) + ' - ' + str("{:.2f}".format(float(x[1]) * 100 / totalDuration)) + '%)')
        else:
            label.append(str(x[0]) + ' (' + str("{:.2f}".format(float(x[1]) * 100 / totalVideo)) + '%)')

        value.append(x[1])

    bar_chart = pygal.HorizontalBar(legend_at_bottom=True)
    bar_chart.x_labels =label
    bar_chart.add(title, value)
    bar_chart.render_to_file(svgPath + '/' + title.lower().replace(' ', '_') + '.svg')

def printResult(title, data, sort):
    print('\n')
    print(title)
    # sorting
    if sort == 'value':
        data = sorted(data.items(), key=operator.itemgetter(1), reverse=False)
    else:
        data = data.items()
    # printing
    for x in data:
        if ('(time in seconds)' in title):
            print(x[0], '=', x[1], '-', "{:.2f}".format(float(x[1]) * 100 / totalDuration), '%')
        else:
            print(x[0], '=', x[1], '-', "{:.2f}".format(float(x[1]) * 100 / totalVideo), '%')
    # generate chart
    drawBarChart(title, data)

def svgObject(title):
    return '<object type="image/svg+xml" data="svg/' + title.lower().replace(' ', '_') + '.svg"></object>'

def createHTML():
    separator = ', '

    html = """

Scanned number of videos: %s

Total size: %s, average size: %s

Total duration: %s, average duration: %s

Scanned video folders: %s

Powered by sunnyspeed studio @ GitHub
        YouTube

    """%(totalVideo, fileSize, averageSize, formatDuration(totalDuration), formatDuration(averageDuration), separator.join(videoFolders))
    html += '
'
    html += '
' + svgObject('Camera') + '	' + svgObject('Camera (time in seconds)') + '
'
    html += '
' + svgObject('Duration') + '	' + svgObject('Duration (time in seconds)') + '
'
    html += '
' + svgObject('Bitrate') + '	' + svgObject('Bitrate (time in seconds)') + '
'
    html += '
' + svgObject('Frame Rate') + '	' + svgObject('Frame Rate (time in seconds)') + '
'
    html += '
' + svgObject('Resolution') + '	' + svgObject('Resolution (time in seconds)') + '
'
    html += '
' + svgObject('Created Day') + '	' + svgObject('Created Day (time in seconds)') + '
'
    html += '
' + svgObject('Created Hour') + '	' + svgObject('Created Hour (time in seconds)') + '
'
    html += '
'
    Html_file= open(htmlPath + "/output.html", "w")
    Html_file.write(html)
    Html_file.close()

# pickle file path
pickleFilePath = 'video.pkl'
# load saved videos info from pickle file to save time
# please delete local pickle file for a fresh new scan
scannedVideo = {}
if os.path.exists(pickleFilePath):
    scannedVideo = pickle.load(open('video.pkl', 'rb'))

# variables
totalVideo = 0
fileSize = 0
averageSize = 0
totalDuration = 0
averageDuration = 0

 # counters
duration = {   'Unknown':0,
                '>1h':0,
                '45m~1h':0,
                '30m~45m':0,
                '20m~30m':0,
                '10m~20m':0,
                '5m~10m':0,
                '1m~5m':0,
                '<1m':0 }
durationT = {   'Unknown':0,
                '>1h':0,
                '45m~1h':0,
                '30m~45m':0,
                '20m~30m':0,
                '10m~20m':0,
                '5m~10m':0,
                '1m~5m':0,
                '<1m':0 }
createHour = {  '23':0,
                '22':0,
                '21':0,
                '20':0,
                '19':0,
                '18':0,
                '17':0,
                '16':0,
                '15':0,
                '14':0,
                '13':0,
                '12':0,
                '11':0,
                '10':0,
                '09':0,
                '08':0,
                '07':0,
                '06':0,
                '05':0,
                '04':0,
                '03':0,
                '02':0,
                '01':0,
                '00':0 }
createHourT = {  '23':0,
                '22':0,
                '21':0,
                '20':0,
                '19':0,
                '18':0,
                '17':0,
                '16':0,
                '15':0,
                '14':0,
                '13':0,
                '12':0,
                '11':0,
                '10':0,
                '09':0,
                '08':0,
                '07':0,
                '06':0,
                '05':0,
                '04':0,
                '03':0,
                '02':0,
                '01':0,
                '00':0 }
createDay = {   'Sunday':0,
                'Saturday':0,
                'Friday':0,
                'Thursday':0,
                'Wednesday':0,
                'Tuesday':0,
                'Monday':0,}
createDayT = {   'Sunday':0,
                'Saturday':0,
                'Friday':0,
                'Thursday':0,
                'Wednesday':0,
                'Tuesday':0,
                'Monday':0,}
resolution = {}
resolutionT = {}
camera = {}
cameraT = {}
bitrate= {}
bitrateT= {}
frameRate = {}
frameRateT = {}

# scan through all video files
for videoFolder in videoFolders:
    for subfolder, folders, files in os.walk(videoFolder):
        for f in files:
            videoFile = subfolder + os.sep + f

            if videoFile.endswith(videoExtensions):
                exifInfo = {}
                if videoFile in scannedVideo.keys():    # this video was already scanned
                    exifInfo = scannedVideo[videoFile]
                else:
                    # prepare the exif holder
                    exifInfo['File Size'] = 'Unknown'               # not for charting

                    # the following strange numbers, such as 99999, is for handling the edge cases
                    # if the exifInfo is changed below, please delete pickle file to do a fresh new scan
                    exifInfo['Duration'] = 'Unknown'        # sony
                    exifInfo['Create Date'] = 'Unknown'             # break down to hour and day
                    exifInfo['Video Size'] = 'Unknown'
                    exifInfo['Device Model Name'] = 'Unknown'
                    exifInfo['Video Avg Bitrate'] = 'Unknown'
                    exifInfo['Video Avg Frame Rate'] = 'Unknown'

                    # retrieve exif from each video
                    process = subprocess.Popen([exiftoolPath, videoFile, '-api', 'largefilesupport=1'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
                    # create the exif array for the video
                    for line in process.stdout:
                        kv = line.strip().split(':', 1)
                        if kv[0].strip() in exifInfo:   # only save the tags we need
                            exifInfo[kv[0].strip()] = kv[1].strip()
                    process.kill()

                    # update scannedVideo with raw exif info, and save it back to pickle file
                    scannedVideo[videoFile] = exifInfo
                    pickle.dump(scannedVideo, open(pickleFilePath, "wb"))

                # special treatments on the exif data
                # 1. file size: remove MB in file size, eg: 8.3 MB
                kv = exifInfo['File Size'].split(' ')
                fileSize += float(kv[0])

                # 3. duration: from 0:01:17 to seconds
                durationInSecond = 0
                if ('s' in exifInfo['Duration']):
                    kv = exifInfo['Duration'].split('.')
                    durationInSecond = kv[0]
                else:
                    kv = exifInfo['Duration'].split(':')
                    durationInSecond = int(kv[0]) * 3600 + int(kv[1]) * 60 + int(kv[2])
                totalDuration += int(durationInSecond)

                # 5. hour and day: grab hour and day of the week from the date, eg: 2020:05:02 13:57:57.072-07:00
                kv = exifInfo['Create Date'].split(' ')
                kv1 = kv[0].split(':')
                d = datetime.datetime(int(kv1[0]), int(kv1[1]), int(kv1[2]))
                exifInfo['Create Day'] = d.strftime('%A')   # day of the week
                kv2 = kv[1].split(':')
                exifInfo['Create Hour'] = kv2[0]    # hour

                # counting
                t = int(durationInSecond)

                # duration
                if exifInfo['Duration'] == 'Unknown':
                    duration['Unknown'] += 1
                    durationT['Unknown'] += t
                else:
                    if int(durationInSecond) <= 60:
                        duration['<1m'] += 1
                        durationT['<1m'] += t
                    elif int(durationInSecond) > 60 and int(durationInSecond) <= 300:
                        duration['1m~5m'] += 1
                        durationT['1m~5m'] += t
                    elif int(durationInSecond) > 300 and int(durationInSecond) <= 600:
                        duration['5m~10m'] += 1
                        durationT['5m~10m'] += t
                    elif int(durationInSecond) > 600 and int(durationInSecond) <= 1200:
                        duration['10m~20m'] += 1
                        durationT['10m~20m'] += t
                    elif int(durationInSecond) > 1200 and int(durationInSecond) <= 1800:
                        duration['20m~30m'] += 1
                        durationT['20m~30m'] += t
                    elif int(durationInSecond) > 1800 and int(durationInSecond) <= 2700:
                        duration['30m~45m'] += 1
                        durationT['30m~45m'] += t
                    elif int(durationInSecond) > 2700 and int(durationInSecond) <= 3600:
                        duration['45m~1h'] += 1
                        durationT['45m~1h'] += t
                    elif int(durationInSecond) > 3600:
                        duration['>1h'] += 1
                        durationT['>1h'] += t

                # camera
                if exifInfo['Device Model Name'] in camera.keys():
                    camera[exifInfo['Device Model Name']] += 1
                    cameraT[exifInfo['Device Model Name']] += t
                else:
                    camera[exifInfo['Device Model Name']] = 1
                    cameraT[exifInfo['Device Model Name']] = t

                # create hour
                createHour[exifInfo['Create Hour']] += 1
                createHourT[exifInfo['Create Hour']] += t

                # create day
                createDay[exifInfo['Create Day']] += 1
                createDayT[exifInfo['Create Day']] += t

                # resolution
                if exifInfo['Video Size'] in resolution.keys():
                    resolution[exifInfo['Video Size']] += 1
                    resolutionT[exifInfo['Video Size']] += t
                else:
                    resolution[exifInfo['Video Size']] = 1
                    resolutionT[exifInfo['Video Size']] = t

                # bitrate
                if exifInfo['Video Avg Bitrate'] in bitrate.keys():
                    bitrate[exifInfo['Video Avg Bitrate']] += 1
                    bitrateT[exifInfo['Video Avg Bitrate']] += t
                else:
                    bitrate[exifInfo['Video Avg Bitrate']] = 1
                    bitrateT[exifInfo['Video Avg Bitrate']] = t

                # frame rate
                if exifInfo['Video Avg Frame Rate'] in frameRate.keys():
                    frameRate[exifInfo['Video Avg Frame Rate']] += 1
                    frameRateT[exifInfo['Video Avg Frame Rate']] += t
                else:
                    frameRate[exifInfo['Video Avg Frame Rate']] = 1
                    frameRateT[exifInfo['Video Avg Frame Rate']] = t

                # total video counter
                totalVideo += 1

                # print each video's exif info
                print(  totalVideo, ':',
                        videoFile, ':',
                        exifInfo['Duration'], ':',
                        exifInfo['Create Day'], ':',
                        exifInfo['Create Hour'], ':',
                        exifInfo['Video Size'], ':',
                        exifInfo['Device Model Name'], ':',
                        exifInfo['Video Avg Bitrate'], ':',
                        exifInfo['Video Avg Frame Rate'])

# print out final results
print('\nTotal scanned number of videos:', totalVideo)
if (totalVideo == 0):
    print('\nNo Video is found.')
else:
    averageSize = str("{:.1f}".format(float(fileSize / totalVideo))) + 'MB'
    if fileSize < 1024:
        fileSize = str(int(fileSize)) + 'MB'
    else:
        fileSize = str(int(fileSize / 1024)) + 'GB'
    print('Total size:', fileSize)
    print('Average size:', averageSize)
    print('Total duration:', formatDuration(totalDuration))
    averageDuration = int(totalDuration / totalVideo)
    print('Average duration:', formatDuration(averageDuration))

    # create a new folder to store charts and html
    htmlPath = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    os.makedirs(htmlPath)
    svgPath = htmlPath + '/svg'
    os.makedirs(svgPath)

    printResult('Camera', camera, 'value')
    printResult('Duration', duration, 'no')
    printResult('Created Hour', createHour, 'no')
    printResult('Created Day', createDay, 'no')
    printResult('Resolution', resolution, 'value')
    printResult('Bitrate', bitrate, 'value')
    printResult('Frame Rate', frameRate, 'value')

    printResult('Camera (time in seconds)', cameraT, 'value')
    printResult('Duration (time in seconds)', durationT, 'no')
    printResult('Created Hour (time in seconds)', createHourT, 'no')
    printResult('Created Day (time in seconds)', createDayT, 'no')
    printResult('Resolution (time in seconds)', resolutionT, 'value')
    printResult('Bitrate (time in seconds)', bitrateT, 'value')
    printResult('Frame Rate (time in seconds)', frameRateT, 'value')

    createHTML()

    print('\n\nThe result is saved in folder ' + htmlPath)

print('Thank you for using this tool!\n\n')


from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
from status import status_decode

base = declarative_base()

TTL = 15 * 60

class Request(base):
    __tablename__ = "requests"
    r_id = Column(Integer, primary_key=True)
    r_key = Column(String)
    code = Column(Integer)
    created = Column(Integer)
    status = Column(Integer)

    def __repr__(self):
        return f"{self.r_id}, {self.r_key}, {self.code}, {self.created}, {self.status}"

    def jsonify(self) -> dict:
        return {"r_key": self.r_key,
                "code": self.code,
                "created": self.created,
                "TTl": TTL,
                "status": status_decode(self.status)}


# ---------------------------------------------#
# This is the SVS image processing code from the example of openslide
# Some of the codes were referred to Nicolas's script.
#
# ---------------------------------------------#
import openslide
from openslide import open_slide, ImageSlide
from openslide.deepzoom import DeepZoomGenerator
from multiprocessing import Process, JoinableQueue
import numpy as np
import os
import sys
import simplejson as json

VIEWER_SLIDE_NAME = 'slide'

class TileWorker(Process):
    """A child process that generates and writes tiles."""

    def __init__(self, queue, slidepath, tile_size, overlap, limit_bounds, quality):
        Process.__init__(self, name='TileWorker')
        self.daemon = True
        self._queue = queue
        self._slidepath = slidepath
        self._tile_size = tile_size
        self._overlap = overlap
        self._limit_bounds = limit_bounds
        self._quality = quality
        self._slide = None

    def bk_detect(self, _img):
        """
        Writing the new function for the background detection;
        Remove the image patches which the background is more than 50% of the whole patches.
        """
        try:
            _gray = _img.convert('L')
            bw = _gray.point(lambda x: 0 if x < 220 else 1, 'F')
            arr = np.array(np.asarray(bw))
            avg_bkg = np.average(bw)
            if arr.shape[0] + arr.shape[1] - 2*self._tile_size == 4:
                return avg_bkg
            else:
                return 1

        except Exception as e:
            print(e)
            # print(level, address)
            return 1

    def run(self):
        self._slide = open_slide(self._slidepath)
        last_associated = None
        dz = self._get_dz()
        while True:
            data = self._queue.get()
            if data is None:
                self._queue.task_done()
                break
            associated, level, address, outfile = data
            if last_associated != associated:
                dz = self._get_dz(associated)
                last_associated = associated
            tile = dz.get_tile(level, address)
            bkg_val = self.bk_detect(tile)
            if bkg_val <= 0.5:
                tile.save(outfile, quality=self._quality)
            self._queue.task_done()

    def _get_dz(self, associated=None):
        if associated is not None:
            image = ImageSlide(self._slide.associated_images[associated])
        else:
            image = self._slide
        return DeepZoomGenerator(image, self._tile_size, self._overlap,
                                 limit_bounds=self._limit_bounds)

class DeepZoomImageTiler(object):
    """Handles generation of tiles and metadata for a single image."""

    def __init__(self, dz, basename, format, associated, queue, t_mag, slide):
        self._dz = dz
        self._basename = basename
        self._format = format
        self._associated = associated
        self._queue = queue
        self._processed = 0
        self._t_mag = int(t_mag)
        self._slide = slide

    def run(self):
        self._write_tiles_new()
        #self._write_dzi()

    def _write_tiles_new(self):
        _magnification = 20
        _tol = 2
        _factors = self._slide.level_downsamples
        _objective = None
        try:
            _objective = float(self._slide.properties[openslide.PROPERTY_NAME_OBJECTIVE_POWER])
        except Exception as e:
            print(e)
            print(self._basename + " - No Objective information found")
            return
        _available = tuple(_objective / x for x in _factors)
        for level in range(self._dz.level_count-1, -1, -1):
            _thisMag = int(_available[0] / pow(2, self._dz.level_count - (level + 1)))
            if self._t_mag != _thisMag:
                continue
            tile_dir = os.path.join("%s_files" % self._basename, str(_thisMag))
            if not os.path.exists(tile_dir):
                os.makedirs(tile_dir)
            cols, rows = self._dz.level_tiles[level]
            for row in range(rows):
                for col in range(cols):
                    tile_name = os.path.join(tile_dir, '%d_%d.%s' % (col, row, self._format))
                    if not os.path.exists(tile_name):
                        self._queue.put((self._associated, level, (col, row), tile_name))
                    self._tile_done()

    def _tile_done(self):
        self._processed += 1
        count, total = self._processed, self._dz.tile_count
        if count % 100 == 0 or count == total:
            print("Tiling %s: wrote %d/%d tiles" % (self._associated or 'slide',
                                                    count, total), end='\r', file=sys.stderr)
            if count == total:
                print(file=sys.stderr)

    def _write_dzi(self):
        with open('%s.dzi' % self._basename, 'w') as fh:
            fh.write(self.get_dzi())

    def get_dzi(self):
        return self._dz.get_dzi(self._format)

class DeepZoomStaticTiler(object):
    """Handles generation of tiles and metadata for all images in a slide."""

    def __init__(self, slidepath, basename, format, tile_size, overlap,
                 limit_bounds, quality, workers, with_viewer, tile_level):
        if with_viewer:
            # Check extra dependency before doing a bunch of work
            import jinja2
        self._slide = open_slide(slidepath)
        self._basename = basename
        self._format = format
        self._tile_size = tile_size
        self._overlap = overlap
        self._limit_bounds = limit_bounds
        self._queue = JoinableQueue(2 * workers)
        self._workers = workers
        self._with_viewer = with_viewer
        self._dzi_data = {}
        self._tile_level = tile_level
        for _i in range(workers):
            TileWorker(self._queue, slidepath, tile_size, overlap, limit_bounds, quality).start()

    def run(self):
        self._run_image()
        if self._with_viewer:
            for name in self._slide.associated_images:
                self._run_image(name)
            self._write_html()
            self._write_static()
        self._shutdown()

    def _run_image(self, associated=None):
        """Run a single image from self._slide."""
        if associated is None:
            image = self._slide
            if self._with_viewer:
                basename = os.path.join(self._basename, VIEWER_SLIDE_NAME)
            else:
                basename = self._basename
        else:
            image = ImageSlide(self._slide.associated_images[associated])
            basename = os.path.join(self._basename, self._slugify(associated))
        dz = DeepZoomGenerator(image, self._tile_size, self._overlap,
                               limit_bounds=self._limit_bounds)
        tiler = DeepZoomImageTiler(dz, basename, self._format, associated,
                                   self._queue, self._tile_level, image)
        tiler.run()
        self._dzi_data[self._url_for(associated)] = tiler.get_dzi()

    def _url_for(self, associated):
        if associated is None:
            base = VIEWER_SLIDE_NAME
        else:
            base = self._slugify(associated)
        return '%s.dzi' % base

    def _write_html(self):
        import jinja2
        env = jinja2.Environment(loader=jinja2.PackageLoader(__name__),
                                 autoescape=True)
        template = env.get_template('slide-multipane.html')
        associated_urls = dict((n, self._url_for(n)) for n in self._slide.associated_images)
        try:
            mpp_x = self._slide.properties[openslide.PROPERTY_NAME_MPP_X]
            mpp_y = self._slide.properties[openslide.PROPERTY_NAME_MPP_Y]
            mpp = (float(mpp_x) + float(mpp_y)) / 2
        except (KeyError, ValueError):
            mpp = 0
        # Embed the dzi metadata in the HTML to work around Chrome's
        # refusal to allow XmlHttpRequest from file:///, even when
        # the originating page is also a file:///
        data = template.render(slide_url=self._url_for(None), slide_mpp=mpp,
                               associated=associated_urls, properties=self._slide.properties,
                               dzi_data=json.dumps(self._dzi_data))
        with open(os.path.join(self._basename, 'index.html'), 'w') as fh:
            fh.write(data)

    def _write_static(self):
        basesrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'static')
        basedst = os.path.join(self._basename, 'static')
        self._copydir(basesrc, basedst)
        self._copydir(os.path.join(basesrc, 'images'), os.path.join(basedst, 'images'))

    def _copydir(self, src, dest):
        if not os.path.exists(dest):
            os.makedirs(dest)
        for name in os.listdir(src):
            srcpath = os.path.join(src, name)
            if os.path.isfile(srcpath):
                shutil.copy(srcpath, os.path.join(dest, name))

    @classmethod
    def _slugify(cls, text):
        text = normalize('NFKD', text.lower()).encode('ascii', 'ignore').decode()
        return re.sub('[^a-z0-9]+', '_', text)

    def _shutdown(self):
        for _i in range(self._workers):
            self._queue.put(None)
        self._queue.join()


from parser.exceptions import BaseParseError

import pytest

@pytest.mark.parametrize(
    ["code", "offset", "expected_line_num", "expected_column_num", "expected_line"],
    [
        ("", 0, 1, 1, ""),
        ("\n", 0, 1, 1, ""),
        ("\n", 1, 2, 1, ""),
        ("aaa", 3, 1, 4, "aaa"),
        ("aaa\n", 3, 1, 4, "aaa"),
        ("aaa\nbbb", 3, 1, 4, "aaa"),
        ("aaa\nbbb", 4, 2, 1, "bbb"),
        ("aaa\nbbb\n", 4, 2, 1, "bbb"),
        ("aaa\nbbb\n", 7, 2, 4, "bbb"),
        ("aaa\nbbb\n", 8, 3, 1, ""),
    ],
)
def test_base_parse_error(
    code: str,
    offset: int,
    expected_line_num: int,
    expected_column_num: int,
    expected_line: str,
) -> None:
    error = BaseParseError("", code, offset)
    assert error.get_line_column_numbers() == (expected_line_num, expected_column_num)
    assert error.get_line() == expected_line


# coding: utf8

"""
This file contains a set of functional tests designed to check the correct execution of the pipeline and the
different functions available in Clinica
"""

import warnings
from os import fspath
from pathlib import Path
from test.nonregression.testing_tools import (
    configure_paths,
    similarity_measure,
    similarity_measure_large_nifti,
)

import numpy as np
import pytest
from numpy.testing import assert_array_almost_equal

# Determine location for working_directory
warnings.filterwarnings("ignore")

@pytest.mark.fast
def test_dwi_b0_flirt(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        b0_flirt_pipeline,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIB0Flirt")
    b0_flirt = b0_flirt_pipeline(num_b0s=11)
    b0_flirt.inputs.inputnode.in_file = str(input_dir / "sub-01_ses-M000_dwi_b0.nii.gz")
    (tmp_path / "tmp").mkdir()
    b0_flirt.base_dir = str(tmp_path / "tmp")
    b0_flirt.run()

    out_file = fspath(
        tmp_path
        / "tmp"
        / "b0_coregistration"
        / "concat_ref_moving"
        / "merged_files.nii.gz"
    )
    ref_file = fspath(ref_dir / "merged_files.nii.gz")

    assert similarity_measure(out_file, ref_file, 0.99)

@pytest.mark.slow
def test_dwi_epi_pipeline(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        epi_pipeline,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIEPIPipeline")
    (tmp_path / "tmp").mkdir()
    epi = epi_pipeline(
        base_dir=str(base_dir),
        output_dir=str(tmp_path / "tmp"),
        ants_random_seed=42,
        use_double_precision=False,
        delete_cache=True,
    )
    epi.inputs.inputnode.t1_filename = str(input_dir / "sub-01_ses-M000_T1w.nii.gz")
    epi.inputs.inputnode.dwi_filename = str(input_dir / "sub-01_ses-M000_dwi.nii.gz")
    epi.inputs.inputnode.b_vectors_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bvec"
    )

    epi.run()

    out_file = fspath(
        tmp_path / "tmp" / "rotated_b_vectors" / "sub-01_ses-M000_dwi_rotated.bvec"
    )
    ref_file = fspath(ref_dir / "sub-01_ses-M000_dwi_rotated.bvec")
    out_bvecs = np.loadtxt(out_file)
    ref_bvecs = np.loadtxt(ref_file)

    assert_array_almost_equal(out_bvecs, ref_bvecs, decimal=2)

    out_file = fspath(
        tmp_path
        / "tmp"
        / "epi_corrected_dwi_image"
        / "Jacobian_image_maths_thresh_merged.nii.gz"
    )
    ref_file = fspath(ref_dir / "Jacobian_image_maths_thresh_merged.nii.gz")

    similarity_measure_large_nifti(out_file, ref_file, 0.95)

@pytest.mark.slow
def test_dwi_perform_ants_registration(cmdopt, tmp_path):
    from test.nonregression.testing_tools import similarity_measure

    import nibabel as nib

    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        perform_ants_registration,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIANTSRegistration"
    )
    (tmp_path / "tmp").mkdir()
    ants_registration = perform_ants_registration(
        base_dir=str(base_dir),
        output_dir=str(tmp_path / "tmp"),
        ants_random_seed=42,  # Set the random seed to avoid stochastic results (requires ants >= 2.3.0)
    )
    ants_registration.inputs.inputnode.t1_filename = str(
        input_dir / "sub-01_ses-M000_T1w.nii.gz"
    )
    ants_registration.inputs.inputnode.dwi_filename = str(
        input_dir / "sub-01_ses-M000_dwi.nii.gz"
    )
    ants_registration.inputs.inputnode.b_vectors_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bvec"
    )

    ants_registration.run()

    out_file = fspath(
        tmp_path / "tmp" / "epi_correction_deformation_field" / "transform1Warp.nii.gz"
    )
    ref_file = fspath(ref_dir / "transform1Warp.nii.gz")

    assert nib.load(out_file).shape == nib.load(ref_file).shape

    out_file = fspath(
        tmp_path / "tmp" / "epi_correction_image_warped" / "transformWarped.nii.gz"
    )
    ref_file = fspath(ref_dir / "transformWarped.nii.gz")

    assert similarity_measure(out_file, ref_file, 0.9)

    out_file = fspath(tmp_path / "tmp" / "merged_transforms" / "transform1Warp.nii.gz")
    ref_file = fspath(ref_dir / "merged_transform.nii.gz")

    assert nib.load(out_file).shape == nib.load(ref_file).shape

    out_file = fspath(
        tmp_path / "tmp" / "rotated_b_vectors" / "sub-01_ses-M000_dwi_rotated.bvec"
    )
    ref_file = fspath(ref_dir / "sub-01_ses-M000_dwi_rotated.bvec")
    out_bvecs = np.loadtxt(out_file)
    ref_bvecs = np.loadtxt(ref_file)

    assert_array_almost_equal(out_bvecs, ref_bvecs, decimal=2)

@pytest.mark.slow
def test_dwi_perform_dwi_epi_correction(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        perform_dwi_epi_correction,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIEPICorrection"
    )
    (tmp_path / "tmp").mkdir()
    epi_correction = perform_dwi_epi_correction(
        base_dir=str(base_dir),
        output_dir=str(tmp_path / "tmp"),
        delete_cache=True,
        use_double_precision=False,
    )
    epi_correction.inputs.inputnode.t1_filename = str(
        input_dir / "sub-01_ses-M000_T1w.nii.gz"
    )
    epi_correction.inputs.inputnode.dwi_filename = str(
        input_dir / "sub-01_ses-M000_dwi.nii.gz"
    )
    epi_correction.inputs.inputnode.merged_transforms = str(
        input_dir / "merged_transforms.nii.gz"
    )

    epi_correction.run(plugin="MultiProc", plugin_args={"n_procs": 8})

    out_file = fspath(
        tmp_path
        / "tmp"
        / "epi_corrected_dwi_image"
        / "Jacobian_image_maths_thresh_merged.nii.gz"
    )
    ref_file = fspath(ref_dir / "Jacobian_image_maths_thresh_merged.nii.gz")

    similarity_measure_large_nifti(out_file, ref_file, 0.95)

@pytest.mark.slow
def test_dwi_eddy_fsl(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        eddy_fsl_pipeline,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIEddyFSL")
    (tmp_path / "tmp").mkdir()
    eddy_fsl = eddy_fsl_pipeline(
        use_cuda=False,
        initrand=True,
        compute_mask=True,
        output_dir=str(tmp_path / "tmp"),
    )
    eddy_fsl.inputs.inputnode.total_readout_time = 0.0342002
    eddy_fsl.inputs.inputnode.phase_encoding_direction = "y-"
    eddy_fsl.inputs.inputnode.dwi_filename = str(
        input_dir / "sub-01_ses-M000_dwi.nii.gz"
    )
    eddy_fsl.inputs.inputnode.b_values_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bval"
    )
    eddy_fsl.inputs.inputnode.b_vectors_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bvec"
    )
    eddy_fsl.inputs.inputnode.reference_b0 = str(
        input_dir / "sub-01_ses-M000_dwi_b0.nii.gz"
    )

    eddy_fsl.run()

    out_file = fspath(tmp_path / "tmp" / "out_corrected" / "eddy_corrected.nii.gz")
    ref_file = fspath(ref_dir / "eddy_corrected.nii.gz")

    assert similarity_measure(out_file, ref_file, 0.97)

    out_file = fspath(
        tmp_path / "tmp" / "out_rotated_bvecs" / "eddy_corrected.eddy_rotated_bvecs"
    )
    ref_file = fspath(ref_dir / "eddy_corrected.eddy_rotated_bvecs")
    out_bvecs = np.loadtxt(out_file)
    ref_bvecs = np.loadtxt(ref_file)

    assert_array_almost_equal(out_bvecs, ref_bvecs, decimal=3)

@pytest.mark.slow
def test_dwi_preprocessing_using_t1(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIPreprocessingUsingT1"
    )
    run_dwi_preprocessing_using_t1(input_dir, tmp_dir, ref_dir, working_dir)

@pytest.mark.slow
def test_dwi_compute_reference_b0(cmdopt, tmp_path):
    """Test step 1 of pipeline DWIPreprocessingUsingPhaseDiff.

    This is a slow test. Expect approximately a 2.5 hours runtime.

    We use the same input files as for the DWIPreprocessingUsingPhaseDiff pipeline.

    However, since we are testing the step 1 in isolation, we don't have the
    BIDS querying logic available to us. The compute_reference_b0 pipeline is
    not BIDS-aware, it just expects three files:
        - the DWI image
        - the associated b-values
        - and the associated b-vectors

    These files are directly available in the input_dir (no BIDS structure, files
    are directly at the root level).

    It also needs the phase encoding direction and total readout time which are
    available in the JSON file (also available in the input_dir).
    In the main pipeline, the input node is performing the extraction of these
    metadata from the JSON file. Here, we perform this extraction directly with
    the `extract_metadata_from_json` and `bids_dir_to_fsl_dir`functions.

    The compute_reference_b0 workflow produces two outputs that are compared
    against their reference values:
        - The reference B0 volume
        - The brain mask computed on the B0 volume
    """
    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_workflows import (
        compute_reference_b0,
    )
    from clinica.utils.dwi import bids_dir_to_fsl_dir
    from clinica.utils.filemanip import (
        extract_metadata_from_json,
        handle_missing_keys_dwi,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIComputeReferenceB0"
    )
    (tmp_path / "tmp").mkdir()

    [total_readout_time, phase_encoding_direction] = extract_metadata_from_json(
        str(input_dir / "sub-01_ses-M000_dwi.json"),
        ["TotalReadoutTime", "PhaseEncodingDirection"],
        handle_missing_keys=handle_missing_keys_dwi,
    )
    phase_encoding_direction = bids_dir_to_fsl_dir(phase_encoding_direction)

    wf = compute_reference_b0(
        b_value_threshold=5.0,
        use_cuda=False,
        initrand=False,
        output_dir=str(tmp_path / "tmp"),
        name="compute_reference_b0",
    )
    wf.inputs.inputnode.b_values_filename = str(input_dir / "sub-01_ses-M000_dwi.bval")
    wf.inputs.inputnode.b_vectors_filename = str(input_dir / "sub-01_ses-M000_dwi.bvec")
    wf.inputs.inputnode.dwi_filename = str(input_dir / "sub-01_ses-M000_dwi.nii.gz")
    wf.inputs.inputnode.image_id = "sub-01_ses-M000"
    wf.inputs.inputnode.total_readout_time = total_readout_time
    wf.inputs.inputnode.phase_encoding_direction = phase_encoding_direction

    wf.run()

    for folder, filename in zip(
        ["reference_b0", "brainmask"],
        ["sub-01_ses-M000_avg_b0_brain.nii.gz", "brainmask.nii.gz"],
    ):
        out_file = fspath(tmp_path / "tmp" / folder / filename)
        ref_file = fspath(ref_dir / folder / filename)

        assert similarity_measure(out_file, ref_file, 0.99)

@pytest.mark.fast
def test_prepare_phasediff_fmap(cmdopt, tmp_path):
    """Test the pipeline prepare_phasediff_fmap which is part of
     step 2 of the pipeline DWIPreprocessingUsingPhaseDiff.

    This is a fast test which should run in less than a minute.
    """
    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_workflows import (
        prepare_phasediff_fmap,
    )
    from clinica.utils.filemanip import extract_metadata_from_json

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIPreparePhasediffFmap"
    )
    (tmp_path / "tmp").mkdir()

    [echo_time1, echo_time2] = extract_metadata_from_json(
        str(input_dir / "sub-01_ses-M000_phasediff.json"), ["EchoTime1", "EchoTime2"]
    )
    delta_echo_time = abs(echo_time2 - echo_time1)

    wf = prepare_phasediff_fmap(output_dir=str(tmp_path / "tmp"))
    wf.inputs.input_node.fmap_mask = str(
        input_dir / "sub-01_ses-M000_magnitude1_corrected_brain_mask.nii.gz"
    )
    wf.inputs.input_node.fmap_phasediff = str(
        input_dir / "sub-01_ses-M000_phasediff.nii.gz"
    )
    wf.inputs.input_node.fmap_magnitude = str(
        input_dir / "sub-01_ses-M000_magnitude1_corrected_brain.nii.gz"
    )
    wf.inputs.input_node.delta_echo_time = str(delta_echo_time)

    wf.run()

    out_filename = (
        "sub-01_ses-M000_phasediff_rads_unwrapped_radsec_fieldmap_demean_maths.nii.gz"
    )
    out_file = fspath(tmp_path / "tmp" / "calibrated_fmap" / out_filename)
    ref_file = fspath(ref_dir / "calibrated_fmap" / out_filename)

    assert similarity_measure(out_file, ref_file, 0.99)

@pytest.mark.fast
def test_dwi_calibrate_and_register_fmap(cmdopt, tmp_path):
    """Test step 2 of pipeline DWIPreprocessingUsingPhaseDiff.

    This is a fast test which should run in about 1 minute.
    """
    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_workflows import (
        calibrate_and_register_fmap,
    )
    from clinica.utils.filemanip import extract_metadata_from_json

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWICalibrateAndRegisterFmap"
    )
    (tmp_path / "tmp").mkdir()

    [echo_time1, echo_time2] = extract_metadata_from_json(
        str(input_dir / "sub-01_ses-M000_phasediff.json"), ["EchoTime1", "EchoTime2"]
    )
    delta_echo_time = abs(echo_time2 - echo_time1)

    wf = calibrate_and_register_fmap(output_dir=str(tmp_path / "tmp"))
    wf.inputs.inputnode.reference_b0 = str(
        input_dir / "sub-01_ses-M000_avg_b0_brain.nii.gz"
    )
    wf.inputs.inputnode.bias_magnitude_fmap = str(
        input_dir / "sub-01_ses-M000_magnitude1.nii.gz"
    )
    wf.inputs.inputnode.fmap_phasediff = str(
        input_dir / "sub-01_ses-M000_phasediff.nii.gz"
    )
    wf.inputs.inputnode.delta_echo_time = delta_echo_time

    wf.run()

    for folder, filename in zip(
        [
            "smooth_calibrated_fmap",
            "bet_magnitude_fmap_registered_onto_b0",
            "registered_calibrated_fmap",
        ],
        [
            "sub-01_ses-M000_phasediff_rads_unwrapped_radsec_fieldmap_demean_maths_flirt_smooth.nii.gz",
            "sub-01_ses-M000_magnitude1_corrected_brain_flirt.nii.gz",
            "sub-01_ses-M000_phasediff_rads_unwrapped_radsec_fieldmap_demean_maths_flirt.nii.gz",
        ],
    ):
        out_file = fspath(tmp_path / "tmp" / folder / filename)
        ref_file = fspath(ref_dir / folder / filename)

        assert similarity_measure(out_file, ref_file, 0.98)

@pytest.mark.slow
def test_dwi_preprocessing_using_phase_diff_field_map(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir,
        tmp_path,
        "DWIPreprocessingUsingPhaseDiffFieldmap",
    )
    run_dwi_preprocessing_using_phase_diff_field_map(
        input_dir, tmp_dir, ref_dir, working_dir
    )

@pytest.mark.slow
def test_dwi_dti(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIDTI")
    run_dwi_dti(input_dir, tmp_dir, ref_dir, working_dir)

@pytest.mark.slow
def test_dwi_connectome(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIConnectome")
    run_dwi_connectome(input_dir, tmp_dir, ref_dir, working_dir)

def run_dwi_preprocessing_using_t1(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_pipeline import (
        DwiPreprocessingUsingT1,
    )

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    parameters = {
        "initrand": True,
        "low_bval": 5,
        "use_cuda": False,
    }
    pipeline = DwiPreprocessingUsingT1(
        bids_directory=fspath(input_dir / "bids"),
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Assert
    out_file = fspath(
        caps_dir
        / "subjects"
        / "sub-PREVDEMALS0010025PG"
        / "ses-M000"
        / "dwi"
        / "preprocessing"
        / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-T1w_preproc.nii.gz"
    )
    ref_file = fspath(
        ref_dir / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-T1w_preproc.nii.gz"
    )

    assert similarity_measure_large_nifti(out_file, ref_file, 0.99)

def run_dwi_preprocessing_using_phase_diff_field_map(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import warnings

    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_pipeline import (
        DwiPreprocessingUsingPhaseDiffFMap,
    )

    warnings.filterwarnings("ignore")

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    parameters = {
        "initrand": True,
        "low_bval": 5,
        "use_cuda": False,
        "delete_cache": True,
    }
    pipeline = DwiPreprocessingUsingPhaseDiffFMap(
        bids_directory=fspath(input_dir / "bids"),
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Assert
    out_file = fspath(
        caps_dir
        / "subjects"
        / "sub-PREVDEMALS0010025PG"
        / "ses-M000"
        / "dwi"
        / "preprocessing"
        / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-b0_preproc.nii.gz"
    )
    ref_file = fspath(
        ref_dir / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-b0_preproc.nii.gz"
    )

    assert similarity_measure(out_file, ref_file, 0.95)

def run_dwi_dti(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import shutil

    import numpy as np
    import pandas as pds

    from clinica.pipelines.dwi_dti.dwi_dti_pipeline import DwiDti

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    # Copy necessary data from in to out
    shutil.copytree(input_dir / "caps", caps_dir, copy_function=shutil.copy)

    pipeline = DwiDti(
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Check files
    subject_id = "sub-PREVDEMALS0010025PG"
    maps = ["AD", "FA", "MD", "RD"]
    out_files = [
        fspath(
            caps_dir
            / "subjects"
            / subject_id
            / "ses-M000"
            / "dwi"
            / "dti_based_processing"
            / "atlas_statistics"
            / (
                subject_id
                + "_ses-M000_dwi_space-JHUDTI81_res-1x1x1_map-"
                + m
                + "_statistics.tsv"
            )
        )
        for m in maps
    ]
    ref_files = [
        fspath(
            ref_dir
            / (
                subject_id
                + "_ses-M000_dwi_space-JHUDTI81_res-1x1x1_map-"
                + m
                + "_statistics.tsv"
            )
        )
        for m in maps
    ]

    for i in range(len(out_files)):
        out_csv = pds.read_csv(out_files[i], sep="\t")
        out_mean_scalar = np.array(out_csv.mean_scalar)
        ref_csv = pds.read_csv(ref_files[i], sep="\t")
        ref_mean_scalar = np.array(ref_csv.mean_scalar)

        assert np.allclose(out_mean_scalar, ref_mean_scalar, rtol=0.025, equal_nan=True)

def run_dwi_connectome(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import shutil

    from clinica.pipelines.dwi_connectome.dwi_connectome_pipeline import DwiConnectome

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    # Copy necessary data from in to out
    shutil.copytree(input_dir / "caps", caps_dir, copy_function=shutil.copy)

    parameters = {"n_tracks": 1000}
    pipeline = DwiConnectome(
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Check files
    atlases = ["desikan", "destrieux"]
    subject_id = "sub-PREVDEMALS0010025PG"
    session_id = "ses-M000"

    out_fod_file = fspath(
        caps_dir
        / "subjects"
        / subject_id
        / session_id
        / "dwi"
        / "connectome_based_processing"
        / (subject_id + "_" + session_id + "_dwi_space-b0_model-CSD_diffmodel.nii.gz")
    )
    ref_fod_file = fspath(
        ref_dir
        / (subject_id + "_" + session_id + "_dwi_space-b0_model-CSD_diffmodel.nii.gz"),
    )

    out_parc_files = [
        fspath(
            caps_dir
            / "subjects"
            / subject_id
            / session_id
            / "dwi"
            / "connectome_based_processing"
            / (
                subject_id
                + "_"
                + session_id
                + "_dwi_space-b0_atlas-"
                + a
                + "_parcellation.nii.gz"
            )
        )
        for a in atlases
    ]
    ref_parc_files = [
        fspath(
            ref_dir
            / (
                subject_id
                + "_"
                + session_id
                + "_dwi_space-b0_atlas-"
                + a
                + "_parcellation.nii.gz"
            )
        )
        for a in atlases
    ]

    assert similarity_measure(out_fod_file, ref_fod_file, 0.97)

    for i in range(len(out_parc_files)):
        assert similarity_measure(out_parc_files[i], ref_parc_files[i], 0.955)


import functools
import itertools

from absl.testing import parameterized
import numpy as np

from tensorflow.compiler.tests import xla_test
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.platform import googletest

@parameterized.named_parameters(('32_bit_index', dtypes.int32),
                                ('64_bit_index', dtypes.int64))
class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase):
  def _testReduction(self,
                     tf_reduce_fn,
                     np_reduce_fn,
                     dtype,
                     test_inputs,
                     index_dtype,
                     rtol=1e-4,
                     atol=1e-4):
    """Tests that the output of 'tf_reduce_fn' matches numpy's output."""

    for test_input in test_inputs:
      with self.session() as sess:
        with self.test_scope():
          a = array_ops.placeholder(dtype)
          index = array_ops.placeholder(index_dtype)
          out = tf_reduce_fn(a, index)
        result = sess.run(out, {a: test_input, index: [0]})
        self.assertAllClose(
            result, np_reduce_fn(test_input, axis=0), rtol=rtol, atol=atol)

        result = sess.run(out, {a: test_input, index: [1]})
        self.assertAllClose(
            result, np_reduce_fn(test_input, axis=1), rtol=rtol, atol=atol)

        result = sess.run(out, {a: test_input, index: [-1]})
        self.assertAllClose(
            result, np_reduce_fn(test_input, axis=1), rtol=rtol, atol=atol)

        # MLIR bridge doesn't return the same error so it can't be matched
        # directly.
        if not test_util.is_mlir_bridge_enabled():
          with self.assertRaisesWithPredicateMatch(
              errors_impl.InvalidArgumentError, 'Invalid reduction dim'):
            sess.run(out, {a: test_input, index: [-33]})

          with self.assertRaisesWithPredicateMatch(
              errors_impl.InvalidArgumentError, 'Invalid reduction dim'):
            sess.run(out, {a: test_input, index: [2]})

  REAL_DATA = [
      np.zeros(shape=(2, 0)),
      np.zeros(shape=(0, 30)),
      np.arange(1, 7).reshape(2, 3),
      np.arange(-10, -4).reshape(2, 3),
      np.arange(-4, 2).reshape(2, 3),
  ]
  COMPLEX_DATA = [
      np.zeros(shape=(2, 0)).astype(np.complex64),
      np.zeros(shape=(0, 30)).astype(np.complex64),
      np.arange(1, 13, dtype=np.float32).view(np.complex64).reshape(2, 3),
      np.arange(-14, -2, dtype=np.float32).view(np.complex64).reshape(2, 3),
      np.arange(-4, 8, dtype=np.float32).view(np.complex64).reshape(2, 3),
  ]
  NONEMPTY_REAL_DATA = [x for x in REAL_DATA if np.size(x) > 0]
  NONEMPTY_COMPLEX_DATA = [x for x in COMPLEX_DATA if np.size(x) > 0]
  BOOL_DATA = [
      np.array([], dtype=np.bool_).reshape(2, 0),
      np.array([], dtype=np.bool_).reshape(0, 3),
      np.array([[False, True, False], [True, True, False]]),
  ]
  ONES = [np.ones([34000, 2])]

  def testReduceSumF32(self, index_dtype):
    self._testReduction(math_ops.reduce_sum, np.sum, np.float32, self.REAL_DATA,
                        index_dtype)

  def testReduceSumC64(self, index_dtype):
    self._testReduction(math_ops.reduce_sum, np.sum, np.complex64,
                        self.COMPLEX_DATA, index_dtype)

  def testReduceProdF32(self, index_dtype):
    self._testReduction(math_ops.reduce_prod, np.prod, np.float32,
                        self.REAL_DATA, index_dtype)

  def testReduceProdC64(self, index_dtype):
    self._testReduction(math_ops.reduce_prod, np.prod, np.complex64,
                        self.COMPLEX_DATA, index_dtype)

  def testReduceMin(self, index_dtype):

    def reference_min(dtype, inp, axis):
      """Wrapper around np.amin that returns +infinity for an empty input."""
      if inp.shape[axis] == 0:
        if np.issubdtype(dtype, np.floating):
          return np.full(inp.shape[0:axis] + inp.shape[axis + 1:], float('inf'))
        return np.full(inp.shape[0:axis] + inp.shape[axis + 1:],
                       np.iinfo(dtype).max)
      return np.amin(inp, axis)

    for dtype in set(self.all_types).intersection(
        [np.float32, np.int32, np.int64]):
      self._testReduction(math_ops.reduce_min,
                          functools.partial(reference_min, dtype), dtype,
                          self.REAL_DATA, index_dtype)

  def testReduceMax(self, index_dtype):

    def reference_max(dtype, inp, axis):
      """Wrapper around np.amax that returns -infinity for an empty input."""
      if inp.shape[axis] == 0:
        if np.issubdtype(dtype, np.floating):
          return np.full(inp.shape[0:axis] + inp.shape[axis + 1:],
                         float('-inf'))
        return np.full(inp.shape[0:axis] + inp.shape[axis + 1:],
                       np.iinfo(dtype).min)
      return np.amax(inp, axis)

    for dtype in set(self.all_types).intersection(
        [np.float32, np.int32, np.int64]):
      self._testReduction(math_ops.reduce_max,
                          functools.partial(reference_max, dtype), dtype,
                          self.REAL_DATA, index_dtype)

  def testReduceMeanF32(self, index_dtype):
    # TODO(phawkins): mean on XLA currently returns 0 instead of NaN when
    # reducing across zero inputs.
    self._testReduction(math_ops.reduce_mean, np.mean, np.float32,
                        self.NONEMPTY_REAL_DATA, index_dtype)

  def testReduceMeanF16(self, index_dtype):
    if np.float16 in self.all_types:
      self._testReduction(math_ops.reduce_mean, np.mean, np.float16, self.ONES,
                          index_dtype)

  def testReduceMeanC64(self, index_dtype):
    self._testReduction(math_ops.reduce_mean, np.mean, np.complex64,
                        self.NONEMPTY_COMPLEX_DATA, index_dtype)

  def testReduceAll(self, index_dtype):
    self._testReduction(math_ops.reduce_all, np.all, np.bool_, self.BOOL_DATA,
                        index_dtype)

  def testReduceAny(self, index_dtype):
    self._testReduction(math_ops.reduce_any, np.any, np.bool_, self.BOOL_DATA,
                        index_dtype)

  @test_util.disable_mlir_bridge('Error messages differ')
  def testReduceSumWithDuplicateAxes(self, index_dtype):
    with self.session() as sess:
      with self.test_scope():
        a = array_ops.placeholder(np.float32)
        index = array_ops.placeholder(np.int32)
        out = math_ops.reduce_sum(a, index)
      with self.assertRaisesWithPredicateMatch(
          errors_impl.InvalidArgumentError,
          'Axes contains duplicate dimension'):
        sess.run(out, {a: [10, 20, 30], index: [0, 0]})

class ReduceOpPrecisionTest(xla_test.XLATestCase):

  def _testReduceSum(self,
                     expected_result,
                     dtype,
                     test_inputs,
                     rtol=1e-3,
                     atol=1e-4):
    """Tests reduce sum on a list of input arrays.

    For each array in test_inputs, check that performing reduce sum on the array
    produces a value that is close to the expected result.

    Args:
      expected_result: the expected result.
      dtype: the data type of the reduce sum operation.
      test_inputs: a list of input arrays for the reduce sum operation.
      rtol: the relative error.
      atol: the absolute error.
    """

    for test_input in test_inputs:
      with self.session() as sess:
        with self.test_scope():
          a = array_ops.placeholder(dtype)
          index = array_ops.placeholder(dtypes.int32)
          out = math_ops.reduce_sum(a, index)
        result = sess.run(out, {
            a: np.array(test_input, dtype=dtype),
            index: [0]
        })
        # Compare the results using float32 type.
        self.assertAllClose(
            np.float32(result),
            np.float32(expected_result),
            rtol=rtol,
            atol=atol)

  def testReduceSumF16(self):
    """Tests the reduce sum of float16 doesn't lose too much precision."""

    if np.float16 not in self.all_types:
      return

    f16_max = np.finfo(np.float16).max
    self._testReduceSum(
        f16_max, np.float16,
        itertools.permutations([f16_max, f16_max, f16_max * (-1.0)], 3))

  def testReduceSumBF16(self):
    """Tests the reduce sum of bfloat16 doesn't lose too much precision."""

    if dtypes.bfloat16.as_numpy_dtype not in self.all_types:
      return

    bf16_max = np.float32(dtypes.bfloat16.max)
    f32_max = dtypes.float32.max
    value = min(bf16_max, f32_max - bf16_max) / 2
    self._testReduceSum(
        dtypes.bfloat16.as_numpy_dtype(value), dtypes.bfloat16.as_numpy_dtype,
        itertools.permutations([bf16_max, value, bf16_max * (-1.0)], 3))

if __name__ == '__main__':
  googletest.main()


from itertools import chain

from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework_api_key.permissions import HasAPIKey
from django.db.models.functions import Concat
from django.db.models import Prefetch, Min, Value

from .models import Offering, Retailer, Scrape, Laptop
from .serializers import LaptopSerializer

def flatten(listable):
    return list(chain(*listable))

def get_distinct_property(items, property):
    return flatten(items.values_list(property).order_by(property).distinct())

class GetOfferings(APIView):
    def get(self, request, format=None):
        laptops = Laptop.objects.prefetch_related(Prefetch('offerings', queryset=Offering.objects.filter(active=True).filter(ignored=False).order_by('price'))).filter(offerings__active=True)
        items = laptops.annotate(min_price=Min('offerings__price')).order_by('min_price')
        vendors = get_distinct_property(laptops, 'vendor__name')
        gpu_vendors = get_distinct_property(laptops, 'gpu_vendor__name')
        retailers = get_distinct_property(laptops, 'offerings__retailer__name')
        cpu_families = flatten(laptops.values_list(Concat('cpu_family__cpu_vendor__name', Value(' - '), 'cpu_family__name')).order_by('cpu_family__cpu_vendor__name', 'cpu_family__name').distinct())
        ram = get_distinct_property(laptops, 'ram')
        hdd = get_distinct_property(laptops, 'hdd')
        sizes = get_distinct_property(laptops, 'size')
        resolutions = get_distinct_property(laptops, 'resolution')
        return Response({
            'vendor': vendors,
            'retailer': retailers,
            'cpu_family': cpu_families,
            'ram': ram,
            'hdd': hdd,
            'gpu_vendor': gpu_vendors,
            'size': sizes,
            'resolution': resolutions,
            'items': LaptopSerializer(items, many=True).data,
        })

class Update(APIView):
    permission_classes = [HasAPIKey,]

    def post(self, request, format=None):
        items = request.data
        total_offerings_count = len(items)
        Offering.objects.update(active=False)
        new_offerings_count = 0
        for item in items:
            url = item.get('url')
            price = item.get('price')
            offering = Offering.objects.filter(url=url).first()
            if (offering is not None):
                offering.price = price
                offering.active = True
                offering.save()
                continue
            name = item.get('name')
            sku = item.get('sku')
            laptop = Laptop.objects.filter(sku=sku).first()
            retailer_string = item.get('retailer')
            retailer = Retailer.objects.get(code=retailer_string)
            offering = Offering(
                sku=sku,
                name=name,
                url=url,
                price=price,
                retailer=retailer,
                laptop=laptop,
            )

            offering.save()
            new_offerings_count += 1

        scrape_instance = Scrape(
            new_offerings=new_offerings_count,
            total_offerings=total_offerings_count
        )
        scrape_instance.save()
        return Response()