import torch as t
from torch.utils.data import Dataset, DataLoader
import os
from torchvision.io import read_image, decode_image
import random
import pandas as pd
import numpy as np
import json

class ImgPairDataset(Dataset):

    def __init__(self, path):
        super(ImgPairDataset, self).__init__()
        self.width = 1024
        self.height = 384

        lvl1_subfolders = [path]
        lvl2_subfolders = []
        for subfolder in lvl1_subfolders:
            lvl2_subfolder = [f.path for f in os.scandir(subfolder) if f.is_dir() and "season" in f.name]
            lvl2_subfolders.append(lvl2_subfolder)
        images_displacements = {}
        for subfolder in lvl2_subfolders:
            for subsubfolder in subfolder:
                os.path.join(subsubfolder, "displacements.txt")
                file = open(os.path.join(subsubfolder, "displacements.txt"), 'r')
                displacements = []
                for idx, line in enumerate(file):
                    try:
                        displacements.append(int(line.split(" ")[0]))
                    except:
                        # ! it means that data is in float form. because of other codes output
                        displacements.append(int(float(line.split(" ")[0])))
                images_displacements[subsubfolder] = displacements
        season_pairs = []
        for subsubfolder in lvl2_subfolders:
            res = [(a, b) for idx, a in enumerate(subsubfolder) for b in subsubfolder[idx + 1:]]
            season_pairs.extend(res)
        self.annotated_img_pairs = []
        for pair in season_pairs:
            generated_triplets = [(os.path.join(pair[0], str(idx).zfill(9)) + ".jpg",
                                   os.path.join(pair[1], str(idx).zfill(9)) + ".jpg",
                                   images_displacements[pair[1]][idx] - images_displacements[pair[0]][idx])
                                  for idx in range(len(images_displacements[pair[0]]))]
            self.annotated_img_pairs.extend(generated_triplets)

    def __len__(self):
        return len(self.annotated_img_pairs)

    def __getitem__(self, idx):
        # if random.random() > 0.0:
        a, b = 0, 1
        #    displacement = self.annotated_img_pairs[idx][2]
        # else:
        #    b, a = 0, 1
        displacement = -self.annotated_img_pairs[idx][2]

        source_img = read_image(self.annotated_img_pairs[idx][a]) / 255.0
        target_img = read_image(self.annotated_img_pairs[idx][b]) / 255.0
        return source_img, target_img, displacement

class CroppedImgPairDataset(ImgPairDataset):

    def __init__(self, crop_width, fraction, smoothness, path="/home/zdeeno/Documents/Datasets/grief_jpg",
                 transforms=None, dataset=None, allign_flag=False):
        super(CroppedImgPairDataset, self).__init__(path=path, dataset=dataset)
        self.crop_width = crop_width
        self.fraction = fraction
        self.smoothness = smoothness
        self.tr = transforms
        self.allign_flag = allign_flag

    def __getitem__(self, idx):
        source, target, displac = super(CroppedImgPairDataset, self).__getitem__(idx)
        displac = displac / 2
        if self.tr is not None:
            source = self.tr(source)
            target = self.tr(target)
        if self.allign_flag:
            source, target = self.allign(source, target, displac)
        cropped_target, crop_start = self.crop_img(target)
        if self.smoothness == 0:
            heatmap = self.get_heatmap(crop_start, displac)
        else:
            heatmap = self.get_smooth_heatmap(crop_start, displac)
        return source, cropped_target, heatmap

    def allign(self, source, target, displac):
        dsp = int(displac)
        mid = self.width // 2
        if dsp < 0:  # ! not sure about sign here // positive is displacement to right
            source = source[:, :, 0:dsp]
            target = target[:, :, -dsp:]
        else:
            source = source[:, :, dsp:]
            target = target[:, :, 0:-dsp]
        return source, target

    def crop_img(self, img):
        crop_start = random.randint(self.crop_width, self.width - 2 * self.crop_width - 1)
        return img[:, :, crop_start:crop_start + self.crop_width], crop_start

    def get_heatmap(self, crop_start, displacement):
        frac = self.width // self.fraction - 1
        heatmap = t.zeros(frac).long()
        idx = int((crop_start - displacement + self.crop_width // 2) / self.fraction)
        if 0 <= idx < 31:
            heatmap[idx] = 1
        return heatmap

    def get_smooth_heatmap(self, crop_start, displacement):
        surround = self.smoothness * 2
        frac = self.width // self.fraction - 1
        heatmap = t.zeros(frac + surround)
        idx = int((crop_start - displacement + self.crop_width // 2) / self.fraction) + 3
        idxs = t.tensor([-1, +1])
        for i in range(self.smoothness):
            for j in idx + i * idxs:
                if 0 <= j < heatmap.size(0):
                    heatmap[j] = 1 - i * (1 / self.smoothness)
        return heatmap[surround // 2:-surround // 2]

def test_annotations():
    data_name = "stromovka"
    dataset = ImgPairDataset(dataset=data_name)
    df = pd.read_csv("/home/zdeeno/Downloads/new_annot.csv")
    old_displac = np.zeros(500)
    new_displac = np.zeros(500)
    for pair in dataset.annotated_img_pairs:
        img_idx = int(pair[0].split("/")[-1].split(".")[0])
        old_displac[img_idx] = pair[2]
    for entry in df.iterrows():
        json_str = entry[1]["meta_info"].replace("'", "\"")
        entry_dict = json.loads(json_str)
        if entry_dict["dataset"] == data_name:
            img_idx = int(entry_dict["place"])
            diff = 0
            kp_dict1 = json.loads(entry[1]["kp-1"].replace("'", "\""))
            kp_dict2 = json.loads(entry[1]["kp-2"].replace("'", "\""))
            for kp1, kp2 in zip(kp_dict1, kp_dict2):
                diff += (kp1["x"] / 100) * 1024 - (kp2["x"] / 100) * 1024
            new_displac[img_idx] = diff // len(kp_dict1)
    old_displac = old_displac[new_displac != 0]
    new_displac = new_displac[new_displac != 0]
    return old_displac, new_displac

class RectifiedEULongterm(ImgPairDataset):

    def __init__(self, crop_width, fraction, smoothness, path=None):
        super(RectifiedEULongterm, self).__init__(path)
        self.crop_width = crop_width
        self.fraction = fraction
        self.smoothness = smoothness
        self.center_mask = 48
        self.flip = K.Hflip()

    def __getitem__(self, idx):
        source, target, displacement = super(RectifiedEULongterm, self).__getitem__(idx)
        # flipping with displacement?
        # source, target = self.augment(source, target)

        cropped_target, crop_start = self.crop_img(target, displacement)
        if self.smoothness == 0:
            heatmap = self.get_heatmap(crop_start)
        else:
            heatmap = self.get_smooth_heatmap(crop_start)
        return source, cropped_target, heatmap

    def set_crop_size(self, crop_size, smoothness=None):
        self.crop_width = crop_size
        if smoothness is None:
            self.smoothness = (crop_size // 8) - 1
        else:
            self.smoothness = smoothness

    def crop_img(self, img, displac):
        # crop - avoid asking for unavailable crop
        if displac >= 0:
            crops = [random.randint(0 + displac, int(self.width - self.crop_width - 1))]
        else:
            crops = [random.randint(0, int(self.width - self.crop_width - 1) + displac)]

        crop_start = random.choice(crops)
        crop_out = crop_start - displac
        # crop_start = random.randint(0, self.width - self.crop_width - 1)
        return img[:, :, crop_start:crop_start + self.crop_width], crop_out

    def get_heatmap(self, crop_start):
        frac = self.width // self.fraction
        heatmap = t.zeros(frac).long()
        idx = int((crop_start + self.crop_width // 2) * (frac / self.width))
        heatmap[idx] = 1
        heatmap[idx + 1] = 1
        return heatmap

    def get_smooth_heatmap(self, crop_start):
        surround = self.smoothness * 2
        frac = self.width // self.fraction
        heatmap = t.zeros(frac + surround)
        idx = int((crop_start + self.crop_width // 2) * (frac / self.width)) + self.smoothness
        heatmap[idx] = 1
        idxs = np.array([-1, +1])
        for i in range(1, self.smoothness + 1):
            indexes = list(idx + i * idxs)
            for j in indexes:
                if 0 <= j < heatmap.size(0):
                    heatmap[j] = 1 - i * (1 / (self.smoothness + 1))
        return heatmap[surround // 2:-surround // 2]

    def augment(self, source, target):
        # crop the logo - this for some reason makes the network diverge on evaluation set
        # source = source[:, 30:, :]
        # target = target[:, 30:, :]
        source[:, :32, -64:] = (t.randn((3, 32, 64)) / 4 + 0.5).clip(0.2, 0.8)
        if random.random() > 0.95:
            target = source.clone()
        if random.random() > 0.5:
            source = self.flip(source)
            target = self.flip(target)
        return source.squeeze(0), target

if __name__ == '__main__':
    # dataset = CroppedImgPairDataset(64, 8, 3)
    # a, b, c = dataset[0]
    # plot_samples(a, b, c)
    old, new = test_annotations()


from twilio.rest import TwilioRestClient
import send_text_config

config = send_text_config

client = TwilioRestClient(config.account_sid, config.auth_token)

message = client.messages.create(
    to= config.to_number,
    from_= config.from_number,
    body="Hello from Python!")

print(message.sid)


from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
import numpy as np

train_images = np.load('train_images.npy')
train_labels = np.load('train_labels.npy')
test_images = np.load('test_images.npy')
test_labels = np.load('test_labels.npy')

train_images_reshaped = np.reshape(train_images, (30000, 28, 28))
test_images_reshaped = np.reshape(test_images, (test_images.shape[0], 28, 28))

scaled_images = np.empty(shape=(30000, 28, 28))
scaled_images_test = np.empty(shape=(5000, 28, 28))
# In order to scale pixel values to [-1, 1], The MaxAbsScaler was used.
counter = 0
scaler = MinMaxScaler(feature_range=(-1, 1))

for x in train_images_reshaped:
    scaled_images[counter, :, :] = scaler.fit_transform(x)
    counter += 1

counter = 0
for x in test_images_reshaped:
    scaled_images_test[counter, :, :] = scaler.fit_transform(x)
    counter += 1

# In order to split 10 percent of the training data set to validation
X_train, X_validate, y_train, y_validate = train_test_split(scaled_images,
                                                    train_labels, test_size=0.1,
                                                    random_state=40, stratify=train_labels)
number_of_epoch = 20

arch_fav = (16, 128, 16)

average_loss_1 = np.zeros(number_of_epoch)
average_loss_01 = np.zeros(number_of_epoch)
average_loss_001 = np.zeros(number_of_epoch)

average_valid_scheduled = np.zeros(number_of_epoch)

average_valid_accuracy_1 = np.zeros(number_of_epoch)
average_valid_accuracy_01 = np.zeros(number_of_epoch)
average_valid_accuracy_001 = np.zeros(number_of_epoch)

counter_for_list = 0
arc_dict = {}

for x in range(0, 1):
    mlp_1 = MLPClassifier(hidden_layer_sizes=arch_fav, activation='relu',
                        solver='sgd', max_iter=1, shuffle=True, learning_rate_init=0.1, momentum=0.0)

    mlp_01 = MLPClassifier(hidden_layer_sizes=arch_fav, activation='relu',
                        solver='sgd', max_iter=1, shuffle=True, learning_rate_init=0.01, momentum=0.0)

    mlp_001 = MLPClassifier(hidden_layer_sizes=arch_fav, activation='relu',
                        solver='sgd', max_iter=1, shuffle=True, learning_rate_init=0.001, momentum=0.0)

    mlp_scheduled = MLPClassifier(hidden_layer_sizes=arch_fav, activation='relu',
                    solver='sgd', max_iter=1, shuffle=True, learning_rate_init=0.1, momentum=0.0)

    # Flatten input data
    nsamples, nx, ny = X_train.shape
    d2_train_dataset = X_train.reshape((nsamples, nx*ny))

    nsamples, nx, ny = X_validate.shape
    d2_validate_dataset = X_validate.reshape((nsamples, nx*ny))

    nsamples, nx, ny = scaled_images_test.shape
    d2_test_dataset = scaled_images_test.reshape((nsamples, nx*ny))

    valid_accuracy_1 = np.empty(shape=(number_of_epoch,))
    valid_accuracy_01 = np.empty(shape=(number_of_epoch,))
    valid_accuracy_001 = np.empty(shape=(number_of_epoch,))

    valid_accuracy_scheduled = np.empty(shape=(number_of_epoch))

    Loss_1 = np.empty(shape=(number_of_epoch,))
    Loss_01 = np.empty(shape=(number_of_epoch,))
    Loss_001 = np.empty(shape=(number_of_epoch,))

    # For loop to 100 epochs
    counter_10_epoch = 0
    for i in range(1, 201):
        # Shuffle training set after each epoch
        random_perm = np.random.permutation(X_train.shape[0])
        mini_batch_index = 0

        if i == 70:
            mlp_scheduled.set_params(learning_rate_init=0.01)

        if i == 120:
            mlp_scheduled.set_params(learning_rate_init=0.001)

        while True:
            indices = random_perm[mini_batch_index:mini_batch_index + 500]
            mlp_1.partial_fit(d2_train_dataset[indices], y_train[indices], np.unique(y_train))
            mlp_01.partial_fit(d2_train_dataset[indices], y_train[indices], np.unique(y_train))
            mlp_001.partial_fit(d2_train_dataset[indices], y_train[indices], np.unique(y_train))
            mlp_scheduled.partial_fit(d2_train_dataset[indices], y_train[indices], np.unique(y_train))
            mini_batch_index += 500

            if mini_batch_index >= X_train.shape[0]:
                break

        if i % 10 == 0:
            valid_accuracy_1[counter_10_epoch] = mlp_1.score(d2_validate_dataset, y_validate)
            valid_accuracy_01[counter_10_epoch] = mlp_01.score(d2_validate_dataset, y_validate)
            valid_accuracy_001[counter_10_epoch] = mlp_001.score(d2_validate_dataset, y_validate)
            valid_accuracy_scheduled[counter_10_epoch] = mlp_scheduled.score(d2_validate_dataset, y_validate)

            Loss_1[counter_10_epoch] = mlp_1.loss_
            Loss_01[counter_10_epoch] = mlp_01.loss_
            Loss_001[counter_10_epoch] = mlp_001.loss_
            counter_10_epoch += 1

    average_loss_1 = (average_loss_1 + Loss_1)
    average_loss_01 = (average_loss_01 + Loss_01)
    average_loss_001 = (average_loss_001 + Loss_001)

    average_valid_accuracy_1 = (average_valid_accuracy_1 + valid_accuracy_1)
    average_valid_accuracy_01 = (average_valid_accuracy_01 + valid_accuracy_01)
    average_valid_accuracy_001 = (average_valid_accuracy_001 + valid_accuracy_001)
    average_valid_scheduled = (average_valid_scheduled + valid_accuracy_scheduled)

arc_dict['name'] = 'arch_3'
arc_dict['loss_curve_1'] = average_loss_1
arc_dict['loss_curve_01'] = average_loss_01
arc_dict['loss_curve_001'] = average_loss_001
arc_dict['val_acc_curve_1'] = average_valid_accuracy_1
arc_dict['val_acc_curve_01'] = average_valid_accuracy_01
arc_dict['val_acc_curve_001'] = average_valid_accuracy_001

from utils import part4Plots
part4Plots(arc_dict, save_dir='', filename='', show_plot=True)

plt.plot(average_valid_scheduled)
plt.ylabel('Validation Accuracy')
plt.show()

print(mlp_scheduled.score(d2_train_dataset, y_train))


from collections import defaultdict

class Graph:
    def __init__(self, v_count):
        self.V = v_count
        self.graph = defaultdict(list)
        self.time = 0

    def add_edge(self, u, v):
        self.graph[u].append(v)
        self.graph[v].append(u)

    def ap_util(self, u, visited, disc, low, parent):
        # Perform normal dfs
        visited[u] = True
        disc[u] = self.time
        low[u] = self.time
        self.time += 1
        children = 0
        for v in self.graph[u]:
            if visited[v] == False:
                parent[v] = u
                children += 1
                if self.ap_util(v, visited, disc, low, parent):
                    return True
                # update low based on lowest in subtree
                low[u] = min(low[v], low[u])
                # if its a root node with 1+ childrens, its ap
                if parent[u] == -1 and children > 1:
                    return True
                # if its not a root node, with no backedge, its ap
                if parent[u] != -1 and low[v] >= disc[u]:
                    return True
            elif v != parent[u]:
                low[u] = min(low[u], disc[v])
        return False

    def is_bcg(self):
        visited = [False] * self.V
        disc = [float('inf')] * self.V
        low = [float('inf')] * self.V
        parent = [-1] * self.V

        # check for two conditions, if true its not biconnected
        # articulation point
        # connectivity
        if self.ap_util(0, visited, disc, low, parent):
            return False

        for i in range(self.V):
            if visited[i] == False:
                return False
        return True

if __name__ == "__main__":
    g1 = Graph(5)
    g1.add_edge(1, 0)
    g1.add_edge(0, 2)
    g1.add_edge(2, 1)
    g1.add_edge(0, 3)
    g1.add_edge(3, 4)
    g1.add_edge(2, 4)
    print(g1.is_bcg())
    g2 = Graph(3)
    g2.add_edge(0, 1)
    g2.add_edge(1, 2)
    print(g2.is_bcg())


import cv2
import os
import shutil
import torch
from Save_result import Tool

filepath = r'C:\Users\GÜNERARDAIŞIK\Desktop\labeler\images'
destination = r'C:\Users\GÜNERARDAIŞIK\Desktop\labeler\dest'

model = torch.hub.load(r'C:\yolov5', 'custom', path=r'C:\Users\GÜNERARDAIŞIK\Desktop\labeler\model\KAAN.pt', source='local')

for filename in os.listdir(filepath):
    if "." not in filename:
        continue
    ending = filename.split(".")[-1]
    if ending not in ["jpg", "gif", "png","jfif","JPG", 'txt']:
        continue
    img = cv2.imread('images/'+filename)
    filename=filename.split('.jpg')[0]

    img1, img2 = Tool.divide(img)
    cv2.imwrite(destination+'\\'+ filename +'1.jpg', img1)
    cv2.imwrite(destination+'\\'+ filename +'2.jpg', img2)
    results1 = model(img1)
    results2 = model(img2)
    results1.display(render=True)
    results2.display(render=True)
    # Save_result.get_values(results)
    lines = Tool.get_values(results=results1)
    lines2 = Tool.get_values(results=results2)

    Tool.save_txt(lines,destination+'\\'+filename+'1.txt')
    Tool.save_txt(lines2,destination+'\\'+filename+'2.txt')

    cv2.imshow('img1', results1.imgs[0])
    cv2.imshow('img2', results2.imgs[0])

    cv2.waitKey(0)


""" tools.py - cpfigure tools that do not depend on WX
"""

import io

import centrosome.cpmorphology
import imageio
import matplotlib
import matplotlib.figure
import matplotlib.transforms
import numpy

def figure_to_image(figure, *args, **kwargs):
    """Convert a figure to a numpy array"""
    #
    # Save the figure as a .PNG and then load it using imageio.imread
    #
    fd = io.BytesIO()
    kwargs = kwargs.copy()
    kwargs["format"] = "png"
    figure.savefig(fd, *args, **kwargs)
    image = imageio.imread(fd.getvalue())
    return image[:, :, :3]

def only_display_image(figure, shape):
    """Set up a figure so that the image occupies the entire figure

    figure - a matplotlib figure
    shape - i/j size of the image being displayed
    """
    assert isinstance(figure, matplotlib.figure.Figure)
    figure.set_frameon(False)
    ax = figure.axes[0]
    ax.set_axis_off()
    figure.subplots_adjust(0, 0, 1, 1, 0, 0)
    dpi = figure.dpi
    width = float(shape[1]) / dpi
    height = float(shape[0]) / dpi
    figure.set_figheight(height)
    figure.set_figwidth(width)
    bbox = matplotlib.transforms.Bbox(numpy.array([[0.0, 0.0], [width, height]]))
    transform = matplotlib.transforms.Affine2D(
        numpy.array([[dpi, 0, 0], [0, dpi, 0], [0, 0, 1]])
    )
    figure.bbox = matplotlib.transforms.TransformedBbox(bbox, transform)

def renumber_labels_for_display(labels):
    """Scramble the label numbers randomly to make the display more discernable

    The colors of adjacent indices in a color map are less discernable than
    those of far-apart indices. Nearby labels tend to be adjacent or close,
    so a random numbering has more color-distance between labels than a
    straightforward one
    """
    return centrosome.cpmorphology.distance_color_labels(labels)


from django.contrib.auth.decorators import login_required
from django.http import Http404
from django.shortcuts import render, get_object_or_404

from .models import Plan, Section, Activity
from cart.models import Order, CartItem

def list_plans(request):
    queryset = Plan.objects.all()
    context = {
        'queryset': queryset
    }
    return render(request, "plan-list.html", context)

@login_required
def plan_details(request, slug):
    plan = get_object_or_404(Plan, slug=slug)
    order_qs = Order.objects.filter(user=request.user)
    plan_in_cart = False
    if order_qs.exists():
        order = order_qs[0]
        order_item_qs = CartItem.objects.filter(plan=plan)
        if order_item_qs.exists():
            order_item = order_item_qs[0]
            if order_item in order.items.all():
                plan_in_cart = True
    context = {
        'plan': plan,
        'in_cart_item': plan_in_cart
    }
    return render(request, "plan-detail.html", context)

@login_required
def section_details(request, plan_slug, section_number):
    section_qs = Section.objects \
        .filter(plan__slug=plan_slug) \
        .filter(section_number=section_number)
    if section_qs.exists():
        context = {
            'section': section_qs[0],
        }
        return render(request, "section_detail.html", context)
    return Http404

@login_required
def activity_details(request, plan_slug, section_number, activity_number):
    activity_qs = Activity.objects \
        .filter(section__plan__slug=plan_slug) \
        .filter(section__section_number=section_number) \
        .filter(activity_number=activity_number)
    if activity_qs.exists():
        context = {
            'activity': activity_qs[0],
        }
        return render(request, "activity_detail.html", context)
    return Http404


import discord

from auth import setAuth
from discord.ext import commands
#https://gist.github.com/vbe0201/ade9b80f2d3b64643d854938d40a0a2d
from music import *

auth = setAuth()
token = auth[0]
prefix = auth[1]

banned_users = (0, 467518086262816769, 464930509101858826, 220250919093272576, 205557510256590848) #Trung has been banned.
debug = False

bot = commands.Bot(prefix, description='RoEW')
bot.add_cog(Music(bot))

#https://discordpy.readthedocs.io/en/latest/api.html#discord-models
#https://discordpy.readthedocs.io/en/latest/faq.html
#https://www.w3schools.com/python/python_ref_list.asp

@bot.event
async def on_ready():
    print(':mima:')
    print('Logged in as {0.user}'.format(bot))

@bot.event
async def on_message(message):
    if message.author == bot.user:
        return

    pieces = message.content.split()
    cmd = pieces[0]

    if message.content.startswith(prefix) and debug:
        log_channel = bot.get_channel(541002881688666133)
        await log_channel.send(message.author.name + "#" + message.author.discriminator + ' used the ' + cmd + ' command in ' + message.guild.name + '.')

    if banned_users.count(message.author.id) == 0:
        await bot.process_commands(message)
    else:
        if message.content.startswith(prefix):
            print('Banned user ' + message.author.name + "#" + message.author.discriminator + ' tried to use the ' + cmd + ' command in ' + message.guild.name + '.')

class Mima(commands.Cog):

    @bot.command(name='xfis')
    async def _xfis(ctx, *, arg: str):
        """
        Wish's personal mischief function.
        """
        if ctx.author.id == 123211683052257280:
            await ctx.send(arg)
            await ctx.message.delete()
        else:
            await ctx.send('denied')

    @bot.command(name='mine')
    async def _mine(ctx, diff: str="lunatic"):
        """
        Sets up a game of minesweeper. (Old Meema functions under reconstruction.)
        """
        await ctx.send('under construction')

    @bot.command(name='update')
    async def _update(ctx, *, arg: str):
        """
        Updates/refreshes the bot.  Only usable by Wish.
        """
        if ctx.author.id == 123211683052257280:
            await ctx.send('Rebooting...')
            bot.close()
        else:
            await ctx.send('denied')

#bot.add_cog(Mima(bot))

bot.run(token)


import glob
import os
import pandas as pd
import svgwrite
from svgwrite import mm

'''
Description: require python >= 3.6
height bar is read counts (normalized); blue is exon; green is known donor; yellow is known acceptor; purple is novel donor; orange is novel acceptor.
'''

# ==========parameters to be changed==========
# ==========parameters for adjusting images==========
xoffset = 30
yoffset = 30
exon_height = 3
xaxis_length = 150
yaxis_length = 150
# ==========parameters for adjusting images==========
min_junction_reads_threshold = 10
splice_junction_file_suffix = 'SJ.out.tab'
wanted = 'FBgn0033159'  # wanted string which is contained in last column of the GTF file
path_gtf_file = '/home/user/data/dscam_fruit_fly.gtf'  # can be a plain text or compressed file (.gz, .zip, .xz, .bz2)
path_splice_junction_folder = '/home/user/data/fruit_fly_sj_data'
path_output_svg_folder = '/home/user/data/fruit_fly_plotting'
# ==========parameters to be changed==========

def draw_sj_image(df_gtf, df_splice_junction, path_output_svg_file, img_title):
    df_exon_drawing = df_gtf[['start', 'end', 'info']].copy()
    df_exon_drawing['exon_number'] = df_exon_drawing['info'].str.extract(r'exon_number "(.*?)";')
    df_exon_drawing['exon_number_count'] = df_exon_drawing['exon_number'].map(df_exon_drawing['exon_number'].value_counts())
    df_exon_drawing['exon_number_cumulative_count'] = df_exon_drawing.groupby('exon_number').cumcount() + 1    # need to add 1
    df_exon_drawing['exon_numering'] = df_exon_drawing['exon_number']
    df_exon_drawing.loc[df_exon_drawing['exon_number_count'] > 1, 'exon_numering'] = df_exon_drawing.loc[df_exon_drawing['exon_number_count'] > 1, 'exon_number'].astype('str') + '.' + df_exon_drawing.loc[df_exon_drawing['exon_number_count'] > 1, 'exon_number_cumulative_count'].astype('str')

    min_coordinate = df_exon_drawing['start'].min()
    max_coordinate = df_exon_drawing['end'].max()
    global xoffset, yoffset, exon_height, xaxis_length, yaxis_length
    df_exon_drawing['scaled_start'] = (df_exon_drawing['start'] - min_coordinate) / (max_coordinate - min_coordinate) * xaxis_length
    df_exon_drawing['scaled_end'] = (df_exon_drawing['end'] - min_coordinate) / (max_coordinate - min_coordinate) * xaxis_length
    df_exon_drawing['scaled_width'] = df_exon_drawing['scaled_end'] - df_exon_drawing['scaled_start']
    # draw exons
    dwg = svgwrite.Drawing(path_output_svg_file, profile='tiny')
    for idx_label in df_exon_drawing.index:
        dwg.add(dwg.text(df_exon_drawing.loc[idx_label, 'exon_numering'], insert=(df_exon_drawing.loc[idx_label, 'scaled_start'] + xoffset, yoffset - 2), fill='black', font_size="3.5px"))
        dwg.add(dwg.rect(insert=(df_exon_drawing.loc[idx_label, 'scaled_start'] + xoffset, yoffset), size=(df_exon_drawing.loc[idx_label, 'scaled_width'], exon_height), stroke='black', stroke_width=0.1, fill='lightblue', opacity=0.7))
    # process data for drawing splice junction reads
    df_sj_drawing = df_splice_junction[['start', 'end', 'number of uniquely mapping reads crossing the junction']].copy()
    df_sj_drawing['donor_exon_end'] = df_sj_drawing['start'] - 1
    df_sj_drawing['acceptor_exon_start'] = df_sj_drawing['end'] + 1
    df_sj_drawing.loc[df_sj_drawing['donor_exon_end'].isin(df_exon_drawing['end']), 'is_known_donor'] = True
    df_sj_drawing['is_known_donor'] = df_sj_drawing['is_known_donor'].fillna(False)
    df_sj_drawing.loc[df_sj_drawing['acceptor_exon_start'].isin(df_exon_drawing['start']), 'is_known_acceptor'] = True
    df_sj_drawing['is_known_acceptor'] = df_sj_drawing['is_known_acceptor'].fillna(False)
    df_temp_donor = pd.merge(df_exon_drawing[['start', 'end']], df_sj_drawing[['donor_exon_end']].reset_index(), how='right', left_on=['end'], right_on=['donor_exon_end'])
    df_temp_donor = df_temp_donor.rename({'start': 'donor_exon_start'}, axis=1)
    df_temp_donor = df_temp_donor.set_index('index').drop(['end', 'donor_exon_end'], axis=1)
    df_sj_drawing = pd.concat([df_sj_drawing, df_temp_donor], axis=1)
    df_temp_acceptor = pd.merge(df_exon_drawing[['start', 'end']], df_sj_drawing[['acceptor_exon_start']].reset_index(), how='right', left_on=['start'], right_on=['acceptor_exon_start'])
    df_temp_acceptor = df_temp_acceptor.rename({'end': 'acceptor_exon_end'}, axis=1)
    df_temp_acceptor = df_temp_acceptor.set_index('index').drop(['start', 'acceptor_exon_start'], axis=1)
    df_sj_drawing = pd.concat([df_sj_drawing, df_temp_acceptor], axis=1)

    df_sj_drawing['scaled_donor_exon_start'] = (df_sj_drawing['donor_exon_start'] - min_coordinate) / (max_coordinate - min_coordinate) * xaxis_length
    df_sj_drawing['scaled_donor_exon_end'] = (df_sj_drawing['donor_exon_end'] - min_coordinate) / (max_coordinate - min_coordinate) * xaxis_length
    df_sj_drawing['scaled_donor_exon_width'] = df_sj_drawing['scaled_donor_exon_end'] - df_sj_drawing['scaled_donor_exon_start']
    df_sj_drawing['scaled_acceptor_exon_start'] = (df_sj_drawing['acceptor_exon_start'] - min_coordinate) / (max_coordinate - min_coordinate) * xaxis_length
    df_sj_drawing['scaled_acceptor_exon_end'] = (df_sj_drawing['acceptor_exon_end'] - min_coordinate) / (max_coordinate - min_coordinate) * xaxis_length
    df_sj_drawing['scaled_acceptor_exon_width'] = df_sj_drawing['scaled_acceptor_exon_end'] - df_sj_drawing['scaled_acceptor_exon_start']

    df_sj_drawing['donor_exon_reads'] = df_sj_drawing.groupby('donor_exon_end')[['number of uniquely mapping reads crossing the junction']].transform('sum')
    df_sj_drawing['acceptor_exon_reads'] = df_sj_drawing.groupby('acceptor_exon_start')[['number of uniquely mapping reads crossing the junction']].transform('sum')

    total_reads = df_splice_junction['number of uniquely mapping reads crossing the junction'].sum()
    df_sj_drawing['scaled_donor_exon_reads'] = df_sj_drawing['donor_exon_reads'] / total_reads * yaxis_length
    df_sj_drawing['scaled_acceptor_exon_reads'] = df_sj_drawing['acceptor_exon_reads'] / total_reads * yaxis_length

    # draw splice junction reads. known donor: green; known acceptor: yellow; novel donor: purple; novel acceptor: orange
    for idx_label in df_sj_drawing[df_sj_drawing['is_known_donor'] == True].index:
        dwg.add(dwg.rect(insert=(df_sj_drawing.loc[idx_label, 'scaled_donor_exon_start'] + xoffset, exon_height + yoffset + 1), size=(df_sj_drawing.loc[idx_label, 'scaled_donor_exon_width'], df_sj_drawing.loc[idx_label, 'scaled_donor_exon_reads']), fill='lightgreen', fill_opacity=0.5))
    for idx_label in df_sj_drawing[df_sj_drawing['is_known_acceptor'] == True].index:
        dwg.add(dwg.rect(insert=(df_sj_drawing.loc[idx_label, 'scaled_acceptor_exon_start'] + xoffset, exon_height + yoffset + 1), size=(df_sj_drawing.loc[idx_label, 'scaled_acceptor_exon_width'], df_sj_drawing.loc[idx_label, 'scaled_acceptor_exon_reads']), fill='yellow', fill_opacity=0.5))
    for idx_label in df_sj_drawing[df_sj_drawing['is_known_donor'] == False].index:
        dwg.add(dwg.line(start=(df_sj_drawing.loc[idx_label, 'scaled_donor_exon_end'] + xoffset, exon_height + yoffset + 1), end=(df_sj_drawing.loc[idx_label, 'scaled_donor_exon_end'] + xoffset, df_sj_drawing.loc[idx_label, 'scaled_donor_exon_reads'] + exon_height + yoffset + 1), stroke='purple', stroke_width='0.25', stroke_opacity=1.0))
    for idx_label in df_sj_drawing[df_sj_drawing['is_known_acceptor'] == False].index:
        dwg.add(dwg.line(start=(df_sj_drawing.loc[idx_label, 'scaled_acceptor_exon_start'] + xoffset, exon_height + yoffset + 1), end=(df_sj_drawing.loc[idx_label, 'scaled_acceptor_exon_start'] + xoffset, df_sj_drawing.loc[idx_label, 'scaled_acceptor_exon_reads'] + exon_height + yoffset + 1), stroke='orange', stroke_width='0.25', stroke_opacity=1.0))
    # add title
    font_size = 5
    title_xstart = xaxis_length / 2 + xoffset - len(img_title) / 2 * font_size * 0.5
    dwg.add(dwg.text(img_title, insert=(title_xstart, yoffset / 2), fill='black', font_size=font_size))
    dwg.save()

os.chdir(path_splice_junction_folder)
list_sj_files = glob.glob(f'*{splice_junction_file_suffix}')

gtf_header_name = ['chr', 'annotation_source', 'feature_type', 'start', 'end', 'score', 'strand', 'genomic_phase', 'info']
df_gtf = pd.read_csv(path_gtf_file, sep='\t', header=None, names=gtf_header_name, dtype={'chr': 'str', 'start': 'Int64', 'end': 'Int64', 'strand': 'str'}, low_memory=False)
df_gtf = df_gtf[df_gtf['feature_type'] == 'exon']
df_gtf = df_gtf[df_gtf['info'].str.contains(wanted)]
df_gtf = df_gtf.drop_duplicates(subset=['chr', 'start', 'end', 'strand'], keep='first')	 # drop duplicate exons except for the first occurrence
chr_name = list(set(df_gtf['chr']))[0]  # splice junction should be at the same chromosome, so do not need to iterate it
strand_name = list(set(df_gtf['strand']))[0]
if strand_name == '-':
    df_gtf['start'], df_gtf['end'] = -df_gtf['end'], -df_gtf['start']

df_all_sj = pd.DataFrame()  # save all splice junction data
splice_junction_header_name = [
    'chr', 'start', 'end', 'strand', 'intron_motif', 'is_annoted', 'number of uniquely mapping reads crossing the junction',
    'number of multi-mapping reads crossing the junction', 'maximum spliced alignment overhang']
for sj_file in list_sj_files:
    path_splice_junction_file = os.path.join(path_splice_junction_folder, sj_file)
    sj_filename_prefix = sj_file[:-len(splice_junction_file_suffix)].rstrip('.')
    path_output_svg_file = os.path.join(path_output_svg_folder, f'{sj_file}.svg')
    df_sj_data = pd.read_csv(path_splice_junction_file, sep='\t', header=None, names=splice_junction_header_name, dtype={'chr': 'str', 'start': 'Int64', 'end': 'Int64', 'strand': 'Int64'}, low_memory=False)
    df_sj_data['image_title'] = sj_filename_prefix
    df_sj_data['path_output_svg_file'] = path_output_svg_file
    df_all_sj = pd.concat([df_all_sj, df_sj_data], axis=0)
df_all_sj = df_all_sj.loc[df_all_sj['strand'] != 0]  # drop splice junctions on undefined strand
df_all_sj = df_all_sj.replace({'strand': {1: '+', 2: '-'}})
df_all_sj = df_all_sj[df_all_sj['strand'] == strand_name]
if strand_name == '-':
    df_all_sj['start'], df_all_sj['end'] = -df_all_sj['end'], -df_all_sj['start']
df_all_sj = df_all_sj[df_all_sj['number of uniquely mapping reads crossing the junction'] >= min_junction_reads_threshold]    # remove low junction reads
# select exons and splice junctions in the right ranges
df_all_sj = df_all_sj[(df_all_sj['chr'] == chr_name) & (df_all_sj['start'] > df_gtf['start'].min()) & (df_all_sj['end'] < df_gtf['end'].max())]
df_gtf = df_gtf[(df_gtf['end'] + 2 > df_all_sj['start'].min()) & (df_gtf['start'] < (df_all_sj['end'].max() + 2))]
df_all_sj = df_all_sj[(df_all_sj['chr'] == chr_name) & (df_all_sj['start'] > df_gtf['start'].min()) & (df_all_sj['end'] < df_gtf['end'].max())]
df_gtf = df_gtf.sort_values(['start', 'end'], ascending=True)
df_all_sj = df_all_sj.sort_values(['start', 'end'], ascending=True)

for path_output_svg_file in set(df_all_sj['path_output_svg_file'].tolist()):
    df_splice_junction = df_all_sj[df_all_sj['path_output_svg_file'] == path_output_svg_file].copy()
    img_title = df_splice_junction['image_title'].tolist()[0]
    df_splice_junction = df_splice_junction.drop(['image_title', 'path_output_svg_file'], axis=1)
    draw_sj_image(df_gtf, df_splice_junction, path_output_svg_file, img_title)


from django.shortcuts import render
from . import forms

# Create your views here.

def form(request):
    form = forms.MyNewForm()

    if request.method == 'POST':
        form = forms.MyNewForm(request.POST)

        if form.is_valid():
            form.save(commit=True)
        else:
            print('ERROR. FORM INVALID')

    return render(request,'formsapp/form_page.html',{'form':form})


from setuptools import setup, find_packages

with open("README.md", "r") as f:
    page_description = f.read()

with open("requirements.txt") as f:
    requirements = f.read().splitlines()

setup(
    name="math_metods",
    version="0.0.3",
    author="Marcio",
    author_email="marcioalexdias@gmail.com",
    description="Esta é a terceira tentativa de disponibilizar os methods",
    long_description=page_description,
    long_description_content_type="text/markdown",
    #url="my_github_repository_project_link"
    packages=find_packages(), # ou posso substituir por ['math_metods'],
    install_requires=requirements,
    python_requires='>=3.8',
)


from __future__ import print_function
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from train import train
from test import test
from net import Net
from torchvision import datasets, transforms
import wandb

def main():
    wandb.init()

    # Training settings
    parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
    parser.add_argument('--batch-size', type=int, default=64, metavar='N',
                        help='input batch size for training (default: 64)')
    parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
                        help='input batch size for testing (default: 1000)')
    parser.add_argument('--epochs', type=int, default=10, metavar='N',
                        help='number of epochs to train (default: 10)')
    parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                        help='learning rate (default: 0.01)')
    parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                        help='SGD momentum (default: 0.5)')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                        help='how many batches to wait before logging training status')
    args = parser.parse_args()
    use_cuda = not args.no_cuda and torch.cuda.is_available()

    # We load all of the arguments into config to save as hyperparameters
    wandb.config.update(args)

    torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
    train_loader = torch.utils.data.DataLoader(
        datasets.MNIST('../data', train=True, download=True,
                       transform=transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,))
                       ])),
        batch_size=args.batch_size, shuffle=True, **kwargs)
    test_loader = torch.utils.data.DataLoader(
        datasets.MNIST('../data', train=False, transform=transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,))
                       ])),
        batch_size=args.test_batch_size, shuffle=True, **kwargs)

    model = Net().to(device)
    optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

    # This magic line lets us save ther pytorch model and track all of the gradients and optionally parameters
    wandb.watch(model)

    for epoch in range(1, args.epochs + 1):
        train(args, model, device, train_loader, optimizer, epoch, wandb)
        test(args, model, device, test_loader, wandb)

if __name__ == '__main__':
    main()


from typing import List

from django.conf import settings
from django.core.management import BaseCommand, CommandError
from django.db import transaction
from django.utils.translation import gettext_lazy as _

from openzaak.components.documenten.models import EnkelvoudigInformatieObject

class Command(BaseCommand):
    help = "Check for any duplicate documents (with same identificatie, bronorganisatie and versie)."

    def add_arguments(self, parser):
        parser.add_argument(
            "--noinput",
            "--no-input",
            action="store_false",
            dest="interactive",
            help=_("Delete automatically any duplicates"),
        )

    def handle(self, *args, **options):
        if settings.CMIS_ENABLED:
            self.stdout.write(_("This command does not run with CMIS enabled."))
            return

        msg = _("Checking {count} records ...").format(
            count=EnkelvoudigInformatieObject.objects.count()
        )
        self.stdout.write(msg)

        duplicates = find_duplicate_eios()

        if len(duplicates) == 0:
            self.stdout.write(_("Found no duplicate records."))
            return

        msg = _("Found {count} duplicate records.").format(count=len(duplicates))
        self.stdout.write(msg)

        if not options["interactive"]:
            delete_duplicates(duplicates)
            self.stdout.write(self.style.SUCCESS(_("Deleted all the duplicates.")))
            return

        while True:
            option = self.get_option()

            if option == 3:
                self.stdout.write(_("Exiting."))
                return
            elif option == 2:
                try:
                    delete_duplicates(duplicates)
                    self.stdout.write(
                        self.style.SUCCESS(
                            _("Deleted {count} duplicate document(s).").format(
                                count=len(duplicates)
                            )
                        )
                    )
                    return
                except Exception as e:
                    self.stderr.write(
                        _("An error occurred. No documents were deleted.")
                    )
                    raise e
            elif option == 1:
                for document in duplicates:
                    self.stdout.write(
                        f"* RSIN: {document.bronorganisatie}, "
                        f"Identificatie: {document.identificatie}, "
                        f"UUID: {document.uuid}"
                    )
            else:
                raise CommandError(_("Invalid option chosen."))

    def get_option(self) -> int:
        message = _(
            "\n1) Show duplicate records\n"
            "2) Delete duplicate records (the oldest record(s) will be deleted).\n"
            "3) Quit\n\n"
            "Choose your action [1-3]:\n"
        )

        raw_value = input(message)
        try:
            value = int(raw_value)
        except ValueError:
            raise CommandError(_("Invalid option chosen."))

        return value

def find_duplicate_eios() -> List[EnkelvoudigInformatieObject]:
    all_documents = EnkelvoudigInformatieObject.objects.order_by("creatiedatum")

    already_seen = set()
    duplicates = []
    for document in all_documents:
        unique_data = (
            document.identificatie,
            document.bronorganisatie,
            document.versie,
        )
        if unique_data in already_seen:
            duplicates.append(document)
        else:
            already_seen.add(unique_data)

    return duplicates

@transaction.atomic
def delete_duplicates(duplicates: List) -> None:
    for document in duplicates:
        document.delete()
    return


import torch, math
from torch import nn
from torch.nn import functional as F
from op import FusedLeakyReLU, fused_leaky_relu, upfirdn2d, conv2d_gradfix

class PixelNorm(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, input):
        return input * torch.rsqrt(torch.mean(input ** 2, dim=1, keepdim=True) + 1e-8)

def make_kernel(k):
    k = torch.tensor(k, dtype=torch.float32)
    if k.ndim == 1:
        k = k[None, :] * k[:, None]
    k /= k.sum()
    return k

class Upsample(nn.Module):
    def __init__(self, kernel, factor=2):
        super().__init__()

        self.factor = factor
        kernel = make_kernel(kernel) * (factor ** 2)
        self.register_buffer("kernel", kernel)
        p = kernel.shape[0] - factor
        pad0 = (p + 1) // 2 + factor - 1
        pad1 = p // 2
        self.pad = (pad0, pad1)

    def forward(self, input):
        out = upfirdn2d(input, self.kernel, up=self.factor, down=1, pad=self.pad)
        return out

class Downsample(nn.Module):
    def __init__(self, kernel, factor=2):
        super().__init__()

        self.factor = factor
        kernel = make_kernel(kernel)
        self.register_buffer("kernel", kernel)
        p = kernel.shape[0] - factor
        pad0 = (p + 1) // 2
        pad1 = p // 2
        self.pad = (pad0, pad1)

    def forward(self, input):
        out = upfirdn2d(input, self.kernel, up=1, down=self.factor, pad=self.pad)
        return out

class Blur(nn.Module):
    def __init__(self, kernel, pad, upsample_factor=1):
        super().__init__()

        kernel = make_kernel(kernel)
        if upsample_factor > 1:
            kernel = kernel * (upsample_factor ** 2)
        self.register_buffer("kernel", kernel)
        self.pad = pad

    def forward(self, input):
        out = upfirdn2d(input, self.kernel, pad=self.pad)
        return out

# https://github.com/facebookresearch/semi-discrete-flow/blob/main/model_itemsets.py#L306
class L2MultiheadAttention(nn.Module):
    """ Kim et al. "The Lipschitz Constant of Self-Attention" https://arxiv.org/abs/2006.04710 """
    def __init__(self, embed_dim, num_heads):
        super(L2MultiheadAttention, self).__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.head_dim = embed_dim // num_heads
        assert (
            self.head_dim * num_heads == self.embed_dim
        ), "embed_dim must be divisible by num_heads"

        self.q_weight = nn.Parameter(torch.empty(embed_dim, num_heads, self.head_dim))
        self.v_weight = nn.Parameter(torch.empty(embed_dim, num_heads, self.head_dim))
        self.out_proj = nn.Linear(embed_dim, embed_dim)

        self._reset_parameters()

    def _reset_parameters(self):
        nn.init.zeros_(self.q_weight)
        nn.init.zeros_(self.v_weight)
        nn.init.zeros_(self.out_proj.weight)
        nn.init.zeros_(self.out_proj.bias)

    def forward(self, x):
        """
        Args:
            x: (T, N, D)
            attn_mask: (T, T) added to pre-softmax logits.
        """

        T, N, _ = x.shape

        q = torch.einsum("tbm,mhd->tbhd", x, self.q_weight)
        k = torch.einsum("tbm,mhd->tbhd", x, self.q_weight)
        squared_dist = (
            torch.einsum("tbhd,tbhd->tbh", q, q).unsqueeze(1)
            + torch.einsum("sbhd,sbhd->sbh", k, k).unsqueeze(0)
            - 2 * torch.einsum("tbhd,sbhd->tsbh", q, k)
        )
        attn_logits = -squared_dist / math.sqrt(self.head_dim)
        attn_weights = F.softmax(attn_logits, dim=1)  # (T, S, N, H)
        A = torch.einsum("mhd,nhd->hmn", self.q_weight, self.q_weight) / math.sqrt(
            self.head_dim
        )
        XA = torch.einsum("tbm,hmn->tbhn", x, A)
        PXA = torch.einsum("tsbh,sbhm->tbhm", attn_weights, XA)
        PXAV = torch.einsum("tbhm,mhd->tbhd", PXA, self.v_weight).reshape(
            T, N, self.embed_dim
        )
        return self.out_proj(PXAV)

class SelfAttention(nn.Module):
    def __init__(self, in_channels, embed_dim=None, num_heads=1):
        super().__init__()
        embed_dim = in_channels if embed_dim is None else embed_dim
        self.use_conv = embed_dim != in_channels
        if self.use_conv:
            self.to_input = nn.Conv2d(in_channels, embed_dim, 1, bias=True)
            self.to_output = nn.Conv2d(embed_dim, in_channels, 1, bias=True)
        self.l2attn = L2MultiheadAttention(embed_dim, num_heads)
        self.ff = nn.Sequential(
            nn.Linear(embed_dim, embed_dim),
            nn.GELU(),
            nn.Linear(embed_dim, embed_dim),
        )
        self.ln1 = nn.LayerNorm(embed_dim)
        self.ln2 = nn.LayerNorm(embed_dim)

    def forward(self, input):
        attn_input = self.to_input(input) if self.use_conv else input
        batch, c, h, w = attn_input.shape
        # input: [N, C, H, W] --> [N, H, W, C] --> [N, HWC]
        attn_input = attn_input.permute(0, 2, 3, 1).reshape(batch, h*w*c)
        attn_input = attn_input.reshape(batch, h*w, c)
        # [N, HW+1, C]
        norm_is = self.ln1(attn_input)
        out1 = self.l2attn(norm_is) + attn_input
        norm_out1 = self.ln2(out1)
        out2 = self.ff(norm_out1.view(-1, c)).view(batch, -1, c)
        output = out2 + out1
        # [N, HW, C]
        output = output[:, :h*w, :]
        output = output.reshape(batch, h, w, c).permute(0, 3, 1, 2)
        output = self.to_output(output) if self.use_conv else output
        return output

class TextEncoder(nn.Module):
    def __init__(self, in_dim, out_dim, num_heads=8):
        super().__init__()

        self.embedding = nn.Linear(in_dim, out_dim)
        self.l2attn = L2MultiheadAttention(out_dim, num_heads)
        self.ff = nn.Sequential(
            nn.GELU(),
            nn.Linear(out_dim, out_dim),
            nn.GELU(),
            nn.Linear(out_dim, out_dim),
        )
        self.ln1 = nn.LayerNorm(out_dim)
        self.ln2 = nn.LayerNorm(out_dim)

    def forward(self, text_embeds):
        text_embeds = self.embedding(text_embeds)
        out1 = self.l2attn(text_embeds)
        out1 = self.ln1(out1 + text_embeds)
        out2 = self.ff(out1)
        output = self.ln2(out2 + out1)
        return output

class CrossAttention(nn.Module):
    def __init__(self, in_channels, text_dim, embed_dim=None, num_heads=1, bias=False):
        super().__init__()

        self.in_channels = in_channels
        embed_dim = in_channels if embed_dim is None else embed_dim
        self.embed_dim = embed_dim
        self.use_conv = in_channels != embed_dim
        if self.use_conv:
            self.to_input = nn.Conv2d(in_channels, embed_dim, 1, bias=True)
            self.to_output = nn.Conv2d(embed_dim, in_channels, 1, bias=True)
        self.to_q = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.to_k = nn.Linear(text_dim, embed_dim, bias=bias)
        self.to_v = nn.Linear(text_dim, embed_dim, bias=bias)
        self.mha = nn.MultiheadAttention(embed_dim, num_heads, batch_first=True)
        self.ff = nn.Sequential(
            nn.Linear(embed_dim, embed_dim),
            nn.GELU(),
            nn.Linear(embed_dim, embed_dim),
        )
        self.ln1 = nn.LayerNorm(embed_dim)
        self.ln2 = nn.LayerNorm(embed_dim)

    def forward(self, image_embeds, text_embeds):
        ie = self.to_input(image_embeds) if self.use_conv else image_embeds
        batch, c, h, w = ie.shape
        # image_embeds: [N, C, H, W] --> [N, H, W, C] --> [N, HW, C]
        ie = ie.permute(0, 2, 3, 1).reshape(batch, h*w, c)
        norm_ie = self.ln1(ie)
        query = self.to_q(norm_ie)
        key = self.to_k(text_embeds)
        value = self.to_v(text_embeds)
        attn_output, attn_output_weights = self.mha(query, key, value)
        out1 = attn_output + ie
        out2 = self.ff(self.ln2(out1))
        # [N, HW, C]
        output = out2 + out1
        output = output.reshape(batch, h, w, c).permute(0, 3, 1, 2)
        output = self.to_output(output)if self.use_conv else output
        return output

class EqualConv2d(nn.Module):
    def __init__(
        self, in_channel, out_channel, kernel_size, stride=1, padding=0, bias=True,
    ):
        super().__init__()

        self.weight = nn.Parameter(
            torch.randn(out_channel, in_channel, kernel_size, kernel_size)
        )
        self.scale = 1 / math.sqrt(in_channel * kernel_size ** 2)

        self.stride = stride
        self.padding = padding

        if bias:
            self.bias = nn.Parameter(torch.zeros(out_channel))

        else:
            self.bias = None

    def forward(self, input):
        out = conv2d_gradfix.conv2d(
            input,
            self.weight * self.scale,
            bias=self.bias,
            stride=self.stride,
            padding=self.padding,
        )

        return out

    def __repr__(self):
        return (
            f"{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},"
            f" {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})"
        )

class EqualLinear(nn.Module):
    def __init__(
        self, in_dim, out_dim, bias=True, bias_init=0, lr_mul=1, activation=None,
    ):
        super().__init__()

        self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul))

        if bias:
            self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init))

        else:
            self.bias = None

        self.activation = activation

        self.scale = (1 / math.sqrt(in_dim)) * lr_mul
        self.lr_mul = lr_mul

    def forward(self, input):
        if self.activation:
            out = F.linear(input, self.weight * self.scale)
            out = fused_leaky_relu(out, self.bias * self.lr_mul)

        else:
            out = F.linear(
                input, self.weight * self.scale, bias=self.bias * self.lr_mul
            )

        return out

    def __repr__(self):
        return (
            f"{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})"
        )

class ModulatedConv2d(nn.Module):
    def __init__(
        self, in_channel, out_channel, kernel_size, style_dim, n_kernel=1, demodulate=True,
        upsample=False, downsample=False, blur_kernel=[1, 3, 3, 1],
    ):
        super().__init__()

        self.eps = 1e-8
        self.kernel_size = kernel_size
        self.in_channel = in_channel
        self.out_channel = out_channel
        self.upsample = upsample
        self.downsample = downsample
        self.n_kernel = n_kernel

        if upsample:
            factor = 2
            p = (len(blur_kernel) - factor) - (kernel_size - 1)
            pad0 = (p + 1) // 2 + factor - 1
            pad1 = p // 2 + 1

            self.blur = Blur(blur_kernel, pad=(pad0, pad1), upsample_factor=factor)

        if downsample:
            factor = 2
            p = (len(blur_kernel) - factor) + (kernel_size - 1)
            pad0 = (p + 1) // 2
            pad1 = p // 2

            self.blur = Blur(blur_kernel, pad=(pad0, pad1))

        fan_in = in_channel * kernel_size ** 2
        self.scale = 1 / math.sqrt(fan_in)
        self.padding = kernel_size // 2
        # conv weight shape: out_ch, in_ch, k_h, k_w
        self.weight = nn.Parameter(torch.randn((n_kernel, out_channel, in_channel, kernel_size, kernel_size)))
        if self.n_kernel != 1:
            self.affine = nn.Linear(style_dim, n_kernel)
        self.modulation = EqualLinear(style_dim, in_channel, bias_init=1)
        self.demodulate = demodulate

    def __repr__(self):
        return (
            f"{self.__class__.__name__}({self.in_channel}, {self.out_channel}, {self.kernel_size}, "
            f"upsample={self.upsample}, downsample={self.downsample})"
        )

    def forward(self, input, style):
        batch, in_channel, height, width = input.shape
        if self.n_kernel != 1:
            selection = self.affine(style).softmax(dim=-1)
            selection = selection.view(-1, self.n_kernel, 1, 1, 1, 1)
            ada_weight = (selection * self.weight).sum(dim=1)
        else:
            ada_weight = self.weight

        style = self.modulation(style).view(batch, 1, in_channel, 1, 1)
        weight = self.scale * ada_weight * style

        if self.demodulate:
            demod = torch.rsqrt(weight.pow(2).sum([2, 3, 4]) + 1e-8)
            weight = weight * demod.view(batch, self.out_channel, 1, 1, 1)

        weight = weight.view(
            batch * self.out_channel, in_channel, self.kernel_size, self.kernel_size
        )

        if self.upsample:
            input = input.reshape(1, batch * in_channel, height, width)
            weight = weight.view(
                batch, self.out_channel, in_channel, self.kernel_size, self.kernel_size
            )
            weight = weight.transpose(1, 2).reshape(
                batch * in_channel, self.out_channel, self.kernel_size, self.kernel_size
            )
            out = conv2d_gradfix.conv_transpose2d(
                input, weight, padding=0, stride=2, groups=batch
            )
            _, _, height, width = out.shape
            out = out.view(batch, self.out_channel, height, width)
            out = self.blur(out)

        elif self.downsample:
            input = self.blur(input)
            _, _, height, width = input.shape
            input = input.view(1, batch * in_channel, height, width)
            out = conv2d_gradfix.conv2d(
                input, weight, padding=0, stride=2, groups=batch
            )
            _, _, height, width = out.shape
            out = out.view(batch, self.out_channel, height, width)

        else:
            input = input.reshape(1, batch * in_channel, height, width)
            out = conv2d_gradfix.conv2d(
                input, weight, padding=self.padding, groups=batch
            )
            _, _, height, width = out.shape
            out = out.view(batch, self.out_channel, height, width)

        return out

class NoiseInjection(nn.Module):
    def __init__(self):
        super().__init__()

        self.weight = nn.Parameter(torch.zeros(1))

    def forward(self, image, noise=None):
        if noise is None:
            batch, _, height, width = image.shape
            noise = image.new_empty(batch, 1, height, width).normal_()

        return image + self.weight * noise

class ConstantInput(nn.Module):
    def __init__(self, channel, size=4):
        super().__init__()

        self.input = nn.Parameter(torch.randn(1, channel, size, size))

    def forward(self, input):
        batch = input.shape[0]
        out = self.input.repeat(batch, 1, 1, 1)

        return out

class StyledConv(nn.Module):
    def __init__(
        self, in_channel, out_channel, kernel_size, style_dim,
        n_kernel=1, upsample=False, blur_kernel=[1, 3, 3, 1], demodulate=True,
    ):
        super().__init__()

        self.conv = ModulatedConv2d(
            in_channel, out_channel, kernel_size, style_dim, n_kernel=n_kernel,
            upsample=upsample, blur_kernel=blur_kernel, demodulate=demodulate,
        )

        self.noise = NoiseInjection()
        self.activate = FusedLeakyReLU(out_channel)

    def forward(self, input, style, noise=None):
        out = self.conv(input, style)
        out = self.noise(out, noise=noise)
        out = self.activate(out)

        return out

class ToRGB(nn.Module):
    def __init__(self, in_channel, style_dim, upsample=True, blur_kernel=[1, 3, 3, 1]):
        super().__init__()

        if upsample:
            self.upsample = Upsample(blur_kernel)

        self.conv = ModulatedConv2d(in_channel, 3, 1, style_dim, demodulate=False)
        self.bias = nn.Parameter(torch.zeros(1, 3, 1, 1))

    def forward(self, input, style, skip=None):
        out = self.conv(input, style)
        out = out + self.bias

        if skip is not None:
            skip = self.upsample(skip)

            out = out + skip

        return out

class FromRGB(nn.Module):
    def __init__(self, in_channel, out_channel):
        super().__init__()
        self.convs = nn.Sequential(
            nn.Conv2d(in_channel, 256, 3, padding='same'),
            nn.LeakyReLU(),
            nn.Conv2d(256, out_channel, 1),
        )

    def forward(self, input):
        return self.convs(input)


# coding: utf-8

from bs4 import BeautifulSoup as bs
from requests import get

def carregar(url):
    return bs(get(url).text, "lxml")

def parse(soup):
    for caixa in soup.find_all("div", class_="Box-row"):
        base = caixa.find("a")
        texto = caixa.find("a").text
        if "#0" in texto:
            link = base.get("href")
            # print(texto)
            # print(link)
            new_url = "https://github.com%s" % link
            new_soup = carregar(new_url)
            new_soup.find_all("h3", class_="timeline-comment-header-text")[0]
            author = new_soup.find_all("a", class_="author")[0].text
            hour_comment = (
                new_soup.find_all("h3", class_="timeline-comment-header-text")[0].find("relative-time").get("datetime")
            )
            for value in new_soup.findAll("td", class_="comment-body"):
                print(value.text)
                print(author, " - ", hour_comment)
                print("*" * 66)

if __name__ == "__main__":
    url = "https://github.com/CollabCodeTech/forum-do-front-ao-end/issues"
    soup = carregar(url)
    parse(soup)


from numpy import array, cross, sqrt, vstack
from numpy.linalg import norm
from typing import List, Tuple
from google.protobuf.wrappers_pb2 import StringValue
from orbdetpy.rpc.conversion_pb2_grpc import ConversionStub
from orbdetpy.rpc.messages_pb2 import AnglesInput, DoubleArray, IntegerArray, TransformFrameInput
from orbdetpy.rpc.server import RemoteServer

def transform_frame(src_frame: int, time: float, pva: List[float], dest_frame: int)->List[float]:
    """Transform a state vector from one frame to another.

    Parameters
    ----------
    src_frame : Source reference frame; a constant from Frame.
    time : Offset in TT from J2000 epoch [s]. Give a list for bulk transforms.
    pva : State vector to transform, can be pos or pos+vel or pos+vel+acc. Provide a
          list of lists for bulk frame transforms.
    dest_frame : Destination reference frame; a constant from Frame.

    Returns
    -------
    State vector transformed to the destination frame.
    """

    if (isinstance(time, float) or isinstance(time, str)):
        single, time, pva = True, [time], [pva]
    else:
        single = False

    if (isinstance(time[0], float)):
        resp = _conversion_stub.transformFrame(TransformFrameInput(
            src_frame=src_frame, time=time, pva=[DoubleArray(array=x) for x in pva], dest_frame=dest_frame))
    else:
        resp = _conversion_stub.transformFrame(TransformFrameInput(
            src_frame=src_frame, UTC_time=time, pva=[DoubleArray(array=x) for x in pva], dest_frame=dest_frame))
    return(resp.array[0].array if (single) else resp.array)

def get_lvlh_rotation(state: List[float])->array:
    """Construct the inertial->LVLH rotation matrix.

    Parameters
    ----------
    state : Inertial state vector.

    Returns
    -------
    Inertial->LVLH frame rotation matrix.
    """

    r = array(state[:3])
    r /= norm(r)
    v = array(state[3:6])
    v /= norm(v)
    h = cross(r, v)
    h /= norm(h)
    return(vstack((r, cross(h, r), h)))

def azel_to_radec(time: float, az: float, el: float, lat: float, lon: float, alt: float, frame: int)->Tuple[float, float]:
    """Convert Azimuth/Elevation to Right Ascension/Declination.

    Parameters
    ----------
    time : Offset in TT from J2000 epoch [s].
    az : Azimuth [rad].
    el : Elevation [rad].
    lat : Observer WGS-84 latitude [rad].
    lon : Observer WGS-84 longitude [rad].
    alt : Observer height above WGS-84 reference ellipsoid [m].
    frame : Destination reference frame; Frame.GCRF or Frame.EME2000.

    Returns
    -------
    Right Ascension and Declination.
    """

    return(_conversion_stub.convertAzElToRaDec(AnglesInput(time=[time], angle1=[az], angle2=[el],
                                                           latitude=lat, longitude=lon, altitude=alt, frame=frame)).array)

def radec_to_azel(frame: int, time: float, ra: float, dec: float, lat: float, lon: float, alt: float)->Tuple[float, float]:
    """Convert Right Ascension/Declination to Azimuth/Elevation.

    Parameters
    ----------
    frame : Source reference frame; Frame.GCRF or Frame.EME2000.
    time : Offset in TT from J2000 epoch [s].
    ra : Right Ascension [rad].
    dec : Declination [rad].
    lat : Observer WGS-84 latitude [rad].
    lon : Observer WGS-84 longitude [rad].
    alt : Observer height above WGS-84 reference ellipsoid [m].

    Returns
    -------
    Azimuth and Elevation.
    """

    return(_conversion_stub.convertRaDecToAzEl(AnglesInput(time=[time], angle1=[ra], angle2=[dec],
                                                           latitude=lat, longitude=lon, altitude=alt, frame=frame)).array)

def lla_to_pos(time: float, lla: List[float])->List[float]:
    """Convert WGS-84 lat/lon/alt to Cartesian position.

    Parameters
    ----------
    time : Offset in TT from J2000 epoch [s]. Give a list for bulk conversions.
    lla : WGS-84 latitude, longitude, altitude. Provide a list of lists for bulk conversions.

    Returns
    -------
    ITRF geocentric Cartesian position [m].
    """

    if (isinstance(time, float) or isinstance(time, str)):
        single, time, lla = True, [time], [lla]
    else:
        single = False

    if (isinstance(time[0], float)):
        resp = _conversion_stub.convertLLAToPos(TransformFrameInput(time=time, pva=[DoubleArray(array=x) for x in lla]))
    else:
        resp = _conversion_stub.convertLLAToPos(TransformFrameInput(UTC_time=time, pva=[DoubleArray(array=x) for x in lla]))
    return(resp.array[0].array if (single) else resp.array)

def pos_to_lla(frame: int, time: float, pos: List[float])->List[float]:
    """Convert a Cartesian position to WGS-84 lat/lon/alt.

    Parameters
    ----------
    frame : Position reference frame; a constant from Frame.
    time : Offset in TT from J2000 epoch [s]. Give a list for bulk conversions.
    pos : Position vector [m]. Provide a list of lists for bulk conversions.

    Returns
    -------
    WGS-84 latitude [rad], longitude [rad], altitude [m].
    """

    if (isinstance(time, float) or isinstance(time, str)):
        single, time, pos = True, [time], [pos]
    else:
        single = False

    if (isinstance(time[0], float)):
        resp = _conversion_stub.convertPosToLLA(TransformFrameInput(src_frame=frame, time=time, pva=[DoubleArray(array=x) for x in pos]))
    else:
        resp = _conversion_stub.convertPosToLLA(TransformFrameInput(src_frame=frame, UTC_time=time, pva=[DoubleArray(array=x) for x in pos]))
    return(resp.array[0].array if (single) else resp.array)

def elem_to_pv(frame: int, time: float, sma: float, ecc: float, inc: float,
               raan: float, argp: float, anom: float, anom_type: int)->List[float]:
    """Convert osculating orbital elements to Cartesian state vector.

    Parameters
    ----------
    frame : Inertial reference frame; a constant from Frame.
    time : Offset in TT from J2000 epoch [s]. Give a list for bulk conversions.
    sma : Semi-major axis [m]. Give a list for bulk conversions.
    ecc : Eccentricity. Give a list for bulk conversions.
    inc : Inclination [rad]. Give a list for bulk conversions.
    raan : RA of ascending node [rad]. Give a list for bulk conversions.
    argp : Argument of perigee [rad]. Give a list for bulk conversions.
    anom : Anomaly angle [rad]. Give a list for bulk conversions.
    anom_type : Anomaly angle type; a constant from PositionAngle. Give a list for bulk conversions.

    Returns
    -------
    Cartesian state vector.
    """

    if (isinstance(time, float) or isinstance(time, str)):
        single, time, sma, ecc, inc, raan, argp, anom, anom_type = True, [time], [sma], [ecc], [inc], [raan], [argp], [anom], [anom_type]
    else:
        single = False

    if (isinstance(time[0], float)):
        resp = _conversion_stub.convertElemToPv(TransformFrameInput(src_frame=frame, time=time, pva=[
            DoubleArray(array=[a,e,i,W,w,n,t]) for a,e,i,W,w,n,t in zip(sma, ecc, inc, raan, argp, anom, anom_type)]))
    else:
        resp = _conversion_stub.convertElemToPv(TransformFrameInput(src_frame=frame, UTC_time=time, pva=[
            DoubleArray(array=[a,e,i,W,w,n,t]) for a,e,i,W,w,n,t in zip(sma, ecc, inc, raan, argp, anom, anom_type)]))
    return(resp.array[0].array if (single) else resp.array)

def pv_to_elem(frame: int, time: float, pv: List[float])->List[float]:
    """Convert Cartesian state vector to osculating orbital elements.

    Parameters
    ----------
    frame : Inertial reference frame; a constant from Frame.
    time : Offset in TT from J2000 epoch [s]. Give a list for bulk conversions.
    pv : Inertial Cartesian state vector. Provide a list of lists for bulk conversions.

    Returns
    -------
    SMA, eccentricity, inclination, RAAN, perigee argument, mean anomaly, true anomaly, eccentric anomaly
    """

    if (isinstance(time, float) or isinstance(time, str)):
        single, time, pv = True, [time], [pv]
    else:
        single = False

    if (isinstance(time[0], float)):
        resp = _conversion_stub.convertPvToElem(TransformFrameInput(src_frame=frame, time=time, pva=[DoubleArray(array=x) for x in pv]))
    else:
        resp = _conversion_stub.convertPvToElem(TransformFrameInput(src_frame=frame, UTC_time=time, pva=[DoubleArray(array=x) for x in pv]))
    return(resp.array[0].array if (single) else resp.array)

def get_UTC_string(j2000_offset: float, precision: int=3)->str:
    """Get ISO-8601 formatted UTC string corresponding to TT offset.

    Parameters
    ----------
    j2000_offset : Offset in TT from J2000 epoch [s] or list of offsets.
    precision : Desired precision. Defaults to 3, i.e. milliseconds.

    Returns
    -------
    ISO-8601 formatted UTC string or list of strings.
    """

    if (isinstance(j2000_offset, float)):
        return(_conversion_stub.getUTCString(DoubleArray(array=[float(precision), j2000_offset])).value)
    return(_conversion_stub.getUTCString(DoubleArray(array=[float(precision), *j2000_offset])).value.split(" "))

def get_J2000_epoch_offset(utc: str)->float:
    """Get TT offset corresponding to ISO-8601 formatted UTC string.

    Parameters
    ----------
    utc : ISO-8601 formatted UTC string or list of strings.

    Returns
    -------
    Offset in TT from J2000 epoch [s] or list of offsets.
    """

    resp = _conversion_stub.getJ2000EpochOffset(StringValue(value=utc if isinstance(utc, str) else " ".join(utc)))
    return(resp.array[0] if (len(resp.array) == 1) else resp.array)

def get_epoch_difference(from_epoch: int, to_epoch: int)->str:
    """Get the constant time difference between two epochs.

    Parameters
    ----------
    from_epoch : From epoch; a value from the Epoch enumeration.
    to_epoch : To epoch; a value from the Epoch enumeration.

    Returns
    -------
    to_epoch - from_epoch in seconds.
    """

    return(_conversion_stub.getEpochDifference(IntegerArray(array=[from_epoch, to_epoch])).value)

def ltr_to_matrix(lower_triangle: List[float])->List[List[float]]:
    """Construct a symmetric matrix from its lower triangle.

    Parameters
    ----------
    lower_triangle : Lower triangle of a symmetric matrix in row-major order.

    Returns
    -------
    Symmetric matrix as a list of lists or None on error.
    """

    m = (sqrt(8*len(lower_triangle) + 1) - 1)/2
    n = int(m)
    if (m == n):
        k, matrix = 0, [[0]*n for _ in range(n)]
        for i in range(n):
            for j in range(i + 1):
                matrix[i][j], matrix[j][i] = lower_triangle[k], lower_triangle[k]
                k += 1
        return(matrix)
    return(None)

if (__name__ != "__main__"):
    _conversion_stub = ConversionStub(RemoteServer.channel())


import json
import logging
from functools import reduce
from os.path import join
from typing import Tuple, Iterator, List, Dict, Any

import numpy
import openslide
import pandas
from pandas import DataFrame

from antilles.block import Field, Step, Block
from antilles.pipeline.annotate import annotate_slides
from antilles.project import Project
from antilles.utils.image import get_mpp_from_openslide
from antilles.utils.io import DAO
from antilles.utils.math import pol2cart

def calc_bbox(
    dims: Tuple[int, int], center: Tuple[int, int], angle: float, **kwargs
) -> Tuple[Tuple[int, int], Tuple[int, int]]:
    width, height = dims
    c_x, c_y = center
    span = kwargs.get("span", 90.0)
    radius_inner = kwargs.get("radius_inner", 400)
    radius_outer = kwargs.get("radius_outer", 800)

    n_divisions = 200
    n_div0 = int(n_divisions * float(span) / 360.0)
    n_div1 = n_divisions - n_div0

    hspan = span / 2.0
    angles0 = numpy.linspace(angle - hspan, angle + hspan, n_div0)
    angles1 = numpy.linspace(angle + hspan, angle + 360 - hspan, n_div1)

    min_dx, max_dx = None, None
    min_dy, max_dy = None, None

    for r, angles in zip([radius_outer, radius_inner], [angles0, angles1]):
        for a in angles:
            dx, dy = pol2cart(r, a)

            if min_dx is None:
                min_dx = dx
            if max_dx is None:
                max_dx = dx

            if min_dy is None:
                min_dy = dy
            if max_dy is None:
                max_dy = dy

            min_dx, max_dx = min(min_dx, dx), max(max_dx, dx)
            min_dy, max_dy = min(min_dy, dy), max(max_dy, dy)

    min_x = int(max(c_x + min_dx, 0))
    max_x = int(min(c_x + max_dx, width))
    dx = max_x - min_x

    min_y = int(max(c_y + min_dy, 0))
    max_y = int(min(c_y + max_dy, height))
    dy = max_y - min_y

    return (min_x, min_y), (dx, dy)

def get_extraction_sequence(settings: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
    for key in ["samples", "devices", "coords", "angles"]:
        if key not in settings.keys():
            raise ValueError(f"Key not found: {key}")

    regions = settings["coords"].to_dict("records")
    sample_angles = settings["angles"].to_dict("records")
    sample_angles = {s["sample"]: float(s["angle"]) for s in sample_angles}

    for region in regions:
        # get wells from payload
        device_name = next(
            s["device"] for s in settings["samples"] if s["name"] == region["sample"]
        )
        payload = next(
            d["payload"] for d in settings["devices"] if d["name"] == device_name
        )
        wells = next(l["wells"] for l in payload if l["level"] == region["level"])

        src = region["relpath"]

        for well in wells:
            fields = {
                "project": region["project"],
                "block": region["block"],
                "panel": region["panel"],
                "level": region["level"],
                "sample": region["sample"],
                "cohorts": region["cohorts"],
                "drug": well["drug"],
            }

            params = {
                "center": (int(region["center_x"]), int(region["center_y"])),
                "angle": sample_angles[region["sample"]] + well["angle"],
            }

            yield {"src": src, "fields": fields, "params": params}

def microns2pixels(dct: Dict[str, Any], keys: List[str], mpp: float):
    for key in keys:
        try:
            dct[key] /= mpp
        except TypeError:
            pass
    return dct

def get_filepath(step: Step, fields: Dict[str, Any], output_order: List[str]) -> str:
    for key in ["project", "block", "panel", "level", "sample", "drug"]:
        if key not in fields.keys():
            raise ValueError(f"Key not found: {key}")

    fields["level"] = "LVL" + str(fields["level"])

    dirpath = join(fields["project"], fields["block"], step.value)
    dirpath = join(dirpath, *(fields[o] for o in output_order))
    DAO.make_dir(dirpath)

    filename_order = ["project", "block", "panel", "level", "sample", "drug"]
    filename = "_".join(fields[f] for f in filename_order) + ".tif"
    filepath = join(dirpath, filename)

    fields["level"] = int(fields["level"][len("LVL") :])

    return filepath

def extract_image(src: str, dst: str, params: Dict[str, Any]) -> Dict[str, Any]:
    with openslide.OpenSlide(DAO.abs(src)) as obj:
        dims = obj.dimensions
        mpp = get_mpp_from_openslide(obj)
        params = microns2pixels(params, ["radius_inner", "radius_outer"], mpp)
        origin, size = calc_bbox(dims=dims, **params)

        image = obj.read_region(origin, 0, size)
        image = image.convert("RGB")
        image.save(DAO.abs(dst))

    cx = params["center"][0] - origin[0]
    cy = params["center"][1] - origin[1]

    r_init = 400 / mpp
    dx, dy = pol2cart(r_init, params["angle"])
    wx, wy = int(round(cx + dx)), int(round(cy + dy))

    return {"oxy": origin, "cxy": (cx, cy), "wxy": (wx, wy), "dims": size, "mpp": mpp}

def update_translate(df: DataFrame, using: DataFrame):
    buffer = 5

    cols = ["project", "block", "panel", "level", "sample", "drug"]
    for i, row in df.iterrows():
        ind = (row[col] == using[col] for col in cols)
        ind = reduce((lambda x, y: x & y), ind)
        using_row = using[ind]

        if len(using_row) == 1:
            oxy_old = using_row["origin_x"].values[0], using_row["origin_y"].values[0]
            cxy_old = using_row["center_x"].values[0], using_row["center_y"].values[0]
            wxy_old = using_row["well_x"].values[0], using_row["well_y"].values[0]

            oxy_new = row["origin_x"], row["origin_y"]
            diff_x = oxy_new[0] - oxy_old[0]
            diff_y = oxy_new[1] - oxy_old[1]

            cxy_new = cxy_old[0] - diff_x, cxy_old[1] - diff_y
            wxy_new = wxy_old[0] - diff_x, wxy_old[1] - diff_y
            cxy_new = max(cxy_new[0], buffer), max(cxy_new[1], buffer)
            wxy_new = max(wxy_new[0], buffer), max(wxy_new[1], buffer)

            df.loc[i, ["center_x"]] = cxy_new[0]
            df.loc[i, ["center_y"]] = cxy_new[1]
            df.loc[i, ["well_x"]] = wxy_new[0]
            df.loc[i, ["well_y"]] = wxy_new[1]
            df.loc[i, ["metadata"]] = using_row["metadata"].values[0]

    return df

class Extractor:
    def __init__(self, project: Project, block: Block):
        self.log = logging.getLogger(__name__)
        self.project = project
        self.block = block

    def adjust(self):
        coords = self.block.get(Field.IMAGES_COORDS)
        angles = self.block.get(Field.ANGLES_COARSE)

        annotate_slides(coords, angles)

        self.block.save(coords, Field.IMAGES_COORDS)
        self.block.save(angles, Field.ANGLES_COARSE)

    def extract(self, params: Dict[str, Any]) -> None:
        self.log.info("Extracting wedges ... ")
        self.block.clean()

        regions_prev = self.block.get(Field.IMAGES_COORDS_BOW)
        regions = self.extract_wedges(params["wedge"])
        regions = update_translate(regions, using=regions_prev)

        self.block.save(regions, Field.IMAGES_COORDS_BOW)
        self.log.info("Extracting wedges complete.")

    def extract_wedges(self, params: Dict[str, Any]) -> DataFrame:
        output_order = self.project.config["output_order"]

        settings = {
            "samples": self.block.samples,
            "devices": self.project.config["devices"],
            "coords": self.block.get(Field.IMAGES_COORDS),
            "angles": self.block.get(Field.ANGLES_COARSE),
        }

        regions = []
        regions_to_extract = get_extraction_sequence(settings)
        for region in regions_to_extract:
            src = region["src"]
            dst = get_filepath(Step.S1, region["fields"], output_order)
            props = extract_image(src, dst, {**params, **region["params"]})

            regions.append(
                {
                    **region["fields"],
                    **{
                        "relpath": dst,
                        "origin_x": props["oxy"][0],
                        "origin_y": props["oxy"][1],
                        "center_x": props["cxy"][0],
                        "center_y": props["cxy"][1],
                        "well_x": props["wxy"][0],
                        "well_y": props["wxy"][1],
                        "mpp": props["mpp"],
                        "width": props["dims"][0],
                        "height": props["dims"][1],
                        "metadata": json.dumps({}),
                    },
                }
            )

        columns = [
            "relpath",
            "project",
            "block",
            "panel",
            "level",
            "sample",
            "cohorts",
            "drug",
            "origin_x",
            "origin_y",
            "center_x",
            "center_y",
            "well_x",
            "well_y",
            "mpp",
            "metadata",
        ]
        regions = pandas.DataFrame(regions, columns=columns)
        return regions


#!/usr/bin/python

from motor2 import MotorBankBase
from io_read import InputBoard
from config import ArmConfig
from protoc.motor_command_pb2 import MotorMoveProto
from hand import Hand
from arm import Arm
import time
import logging
import math

bank = MotorBankBase()
io_reader = InputBoard()
io_reader.Start()
arm_config = ArmConfig()
arm_config.ApplyTares(bank, io_reader)
#   while True:
#       s = "\n------\n"
#       for i in range(22, 32, 2):
#           s += " PIN %d = %d\n" % (i, io_reader.GetPin(i))

#       logging.info("RPS = %f %s", io_reader.ReadsPerSec(), s)
#       time.sleep(0.2)
bank.Release()
bank.Rezero()
time.sleep(5.0)

hand = Hand(bank, io_reader, arm_config)
arm = Arm(bank, io_reader, arm_config)
#hand.Dispense()
#   if True:
#       hand.Grab()
#       bank.WaitDone()
#       time.sleep(5)
#       logging.info("Release now.")
#       hand.Release()
#       bank.WaitDone()
#   while True:
#       logging.info("Wait here forever.")
#       time.sleep(10)
hand.Calibrate()
#bank.WaitDone()
bank.base_motor.SetDisableAfterMoving(False)
arm.Rotate(-math.pi / 2)
bank.WaitDone()
arm.Calibrate()
bank.WaitDone()
#arm.RaiseFully()
#bank.WaitDone()
arm.GoToPositionHeight()
bank.WaitDone()
arm.Rotate(0.0)
bank.WaitDone()
hand.Release()
bank.WaitDone()
time.sleep(5.0)
#arm.Rotate(math.pi / 30)
bank.WaitDone()
arm.LowerOntoBottle()
bank.WaitDone()
#   #   #arm.Lower()
#   #   #arm.Rotate()
hand.Grab()
bank.WaitDone()
#   #   #   #hand.Hold()
time.sleep(5.0)
arm.RaiseFully()
bank.WaitDone()
hand.Dispense()
bank.WaitDone()
arm.SetWristToGrabPosition()
#   #   #   time.sleep(1.0)
#   #   bank.WaitDone()

#   #   arm.SetupLowerStop()
#   #   bank.Rezero()
arm.Lower()
bank.WaitDone()
#arm.Rotate(-math.pi / 25)
#arm.Rotate(math.pi / 30)
#bank.WaitDone()
hand.Release()
bank.WaitDone()
#arm.GoToReleasePositionHeight()
#bank.WaitDone()
#hand.Rotate(math.pi / 6)
arm.RaiseOverDroppedBottle()
bank.WaitDone()
logging.info("Raise fully")
arm.RaiseFully()
bank.WaitDone()
bank.Rezero()
time.sleep(5.0)
#   #   #   #hand.Hold()


"""
Adversarial in-processing algorithm by Zhang et al. (2018) [DOI: https://doi.org/10.1145/3278721.3278779]

Code adapted from https://github.com/choprashweta/Adversarial-Debiasing and
																https://github.com/abacusai/intraprocessing_debiasing
"""
import time

import logging

import copy

import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import optim

from models.networks_ChestXRay import (ChestXRayVGG16, ChestXRayResNet18)

from utils.evaluation import compute_empirical_bias, compute_accuracy_metrics

from models.networks_tabular import load_model
from utils.evaluation import get_valid_objective, get_test_objective, get_best_thresh

logger = logging.getLogger("Debiasing")

class Classifier(nn.Module):
	def __init__(self, config):
		super(Classifier, self).__init__()

		self.arch = config['default']['arch']

		if config['default']['arch'] == 'vgg':
			self.base_model = ChestXRayVGG16(pretrained=config['default']['pretrained'])
		elif config['default']['arch'] == 'resnet':
			self.base_model = ChestXRayResNet18(pretrained=config['default']['pretrained'])
		else:
			NotImplementedError('Architecture not supported!')

	def forward(self, t, training_mode=False):
		if training_mode:
			if self.arch == 'vgg':
				classifier_prev_output = torch.relu(self.base_model.vgg16(t))
			elif self.arch == 'resnet':
				classifier_prev_output = torch.relu(self.base_model.resnet18(t))

			classifier_output = self.base_model.out(classifier_prev_output)
			return classifier_output, classifier_prev_output
		else:
			if self.arch == 'vgg':
				classifier_prev_output = torch.relu(self.base_model.vgg16(t))
			elif self.arch == 'resnet':
				classifier_prev_output = torch.relu(self.base_model.resnet18(t))

			classifier_output = self.base_model.out(classifier_prev_output)
			return torch.sigmoid(classifier_output)

class Adversary(nn.Module):
	def __init__(self):
		super(Adversary, self).__init__()

		self.a1 = nn.Linear(1000, 512)
		self.a2 = nn.Linear(512, 1)

	def forward(self, input_ids):
		adversary = F.relu(self.a1(input_ids))
		adversary_output = self.a2(adversary)
		return adversary_output

def train_adversary(adv, clf, optimizer_adv, train_loader, loss_criterion, device, n_steps):
	adv_loss = 0
	steps = 0

	for i, data in enumerate(train_loader):
		if steps > n_steps:
			break
		# get the inputs and labels
		inputs, labels, attrs = data
		inputs = inputs.to(device)
		labels = labels.to(device)
		attrs = attrs.to(device)

		optimizer_adv.zero_grad()

		classifier_output, classifier_prev_output = clf(inputs, training_mode=True)
		adversary_output = adv(classifier_prev_output)
		# NOTE: this adversarial loss is only applicable for the EOD (see the original paper by Zhang et al.)
		adversary_loss = loss_criterion(torch.sigmoid(adversary_output[labels == 1])[:, 0], attrs[labels == 1].float())
		adversary_loss.backward()
		optimizer_adv.step()
		adv_loss += adversary_loss.item()
		steps += 1

	return adv

def train_classifier(clf, optimizer_clf, adv, train_loader, loss_criterion, lbda, device, n_steps):
	for i, data in enumerate(train_loader):
		if i > n_steps:
			break
		# get the inputs and labels
		inputs, labels, attrs = data
		inputs = inputs.to(device)
		labels = labels.to(device)
		attrs = attrs.to(device)

		optimizer_clf.zero_grad()

		classifier_output, classifier_prev_output = clf(inputs, training_mode=True)
		adversary_output = adv(classifier_prev_output)
		# NOTE: this adversarial loss is only applicable for the EOD (see the original paper by Zhang et al.)
		adversary_loss = loss_criterion(torch.sigmoid(adversary_output[labels == 1])[:, 0], attrs[labels == 1].float())
		classifier_loss = loss_criterion(torch.sigmoid(classifier_output)[:, 0], labels.float())
		total_classifier_loss = classifier_loss - lbda * adversary_loss
		total_classifier_loss.backward()

		optimizer_clf.step()

	return clf

def train_fair_ChestXRay_model(dataloaders, dataset_sizes, device, config, bias_metric='spd', batch_size=256,
							   num_epochs=25):
	since = time.time()

	model = Classifier(config=config)
	adversary = Adversary()

	model.to(device)
	adversary.to(device)

	optimizer_clf = optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-8)
	optimizer_adv = optim.AdamW(adversary.parameters(), lr=1e-4, weight_decay=1e-8)

	loss_criterion_clf = nn.BCELoss()
	loss_criterion_adv = nn.BCELoss()

	# Best model parameters
	best_model_wts = copy.deepcopy(model.state_dict())
	best_loss = np.Inf
	best_acc = 0
	best_bias = 0
	train_acc = []
	val_acc = []
	train_loss = []
	val_loss = []
	train_bias = []
	val_bias = []

	# Iterate over epochs
	for epoch in range(num_epochs):
		print('Epoch {}/{}'.format(epoch, num_epochs - 1))
		print('-' * 10)

		# Each epoch has a training and validation phase
		for phase in ['train', 'val']:
			if phase == 'train':
				model.train()
				adversary.train()

				for param in model.parameters():
					param.requires_grad = False
				adversary = train_adversary(adv=adversary, clf=model, optimizer_adv=optimizer_adv,
											train_loader=dataloaders['train'], loss_criterion=loss_criterion_adv,
											device=device, n_steps=100)

				for param in model.parameters():
					param.requires_grad = True

				for param in adversary.parameters():
					param.requires_grad = False

				model = train_classifier(clf=model, optimizer_clf=optimizer_clf, adv=adversary,
										 train_loader=dataloaders['train'], loss_criterion=loss_criterion_clf,
										 lbda=config['mitigating']['lbda'], device=device, n_steps=100)

				for param in adversary.parameters():
					param.requires_grad = True
			else:

				model.eval()
				adversary.eval()

				# Running parameters
				running_loss = 0.0
				running_corrects = 0
				preds_vec = np.zeros(dataset_sizes[phase])
				labels_vec = np.zeros(dataset_sizes[phase])
				probs_mat = np.zeros((dataset_sizes[phase]))
				priv = np.zeros(dataset_sizes[phase])
				cnt = 0

				# Iterate over data
				for inputs, labels, attrs in dataloaders[phase]:
					# Send inputs and labels to the device
					inputs = inputs.to(device)
					labels = labels.to(device).to(torch.float)
					attrs = attrs.to(device)

					# Zero the gradients
					optimizer_clf.zero_grad()

					# Forward
					with torch.set_grad_enabled(False):
						outputs, outputs_prev = model(inputs, training_mode=True)
						adversary_output = adversary(outputs_prev)
						preds = outputs[:, 0] > 0.5
						loss = loss_criterion_clf(torch.sigmoid(outputs)[:, 0], labels.float())
						if torch.sum(labels == 1) > 0:
							loss = loss - config['mitigating']['lbda'] * loss_criterion_adv(
								torch.sigmoid(adversary_output[labels == 1])[:, 0], attrs[labels == 1].float())
						probs_mat[cnt * batch_size:(cnt + 1) * batch_size] = outputs[:, 0].cpu().detach().numpy()
						preds_vec[cnt * batch_size:(cnt + 1) * batch_size] = preds.cpu().detach().numpy()
						labels_vec[cnt * batch_size:(cnt + 1) * batch_size] = labels.cpu().detach().numpy()
						priv[cnt * batch_size:(cnt + 1) * batch_size] = attrs.cpu().detach().numpy()

					# Statistics
					running_loss += loss.item() * inputs.size(0)
					running_corrects += torch.sum(preds == labels.data)

					# Increment
					cnt = cnt + 1

				# Get the predictive performance metrics
				auroc, avg_precision, balanced_acc, f1_acc = compute_accuracy_metrics(preds_vec, labels_vec)
				bias = compute_empirical_bias(preds_vec, labels_vec, priv, bias_metric)
				print('Bias:', bias)
				acc = running_corrects.double() / dataset_sizes[phase]

				# Save
				epoch_loss = running_loss / dataset_sizes[phase]
				epoch_acc = balanced_acc

				# Print
				print('{} Loss: {:.4f} Acc: {:.4f} AUROC: {:.4f} Avg. Precision: {:.4f} Balanced Accuracy: {:.4f} '
					  'f1 Score: {:.4f}'.format(phase, epoch_loss, acc, auroc, avg_precision, balanced_acc, f1_acc))

				# Save the accuracy and loss
				val_acc.append(epoch_acc)
				val_loss.append(epoch_loss)

				# Deep copy the model
				if epoch_loss < best_loss:
					best_acc = epoch_acc
					best_loss = epoch_loss
					best_model_wts = copy.deepcopy(model.state_dict())

	# Time
	time_elapsed = time.time() - since
	print('Training complete in {:.0f}m {:.0f}s'.format(
		time_elapsed // 60, time_elapsed % 60))
	print('Best val Loss: {:4f}'.format(best_loss))
	print('Best val Acc: {:4f}'.format(best_acc))
	print('Best val Bias: {:4f}'.format(best_bias))

	# Load the best model weights
	model.load_state_dict(best_model_wts)
	return model, train_acc, train_loss, val_acc, val_loss

def mitigating_debiasing(model_state_dict, data, config, device):
	"""
	In-processing adversarial debiasing for tabular data.
	Adapted from https://github.com/abacusai/intraprocessing_debiasing
	"""
	logger.info('Training Mitigating model.')
	actor = load_model(data.num_features, config.get('hyperparameters', {}))
	actor.load_state_dict(model_state_dict)
	actor.to(device)
	critic = nn.Sequential(
		nn.Linear(32, 32),
		nn.Dropout(0.2),
		nn.ReLU(),
		nn.Linear(32, 32),
		nn.Dropout(0.2),
		nn.ReLU(),
		nn.Linear(32, 32),
		nn.Dropout(0.2),
		nn.ReLU(),
		nn.Linear(32, 2),
		nn.Softmax()
	)
	critic.to(device)
	critic_optimizer = optim.Adam(critic.parameters())
	critic_loss_fn = torch.nn.BCELoss()

	actor_optimizer = optim.Adam(actor.parameters(), lr=config['mitigating']['lr'])
	actor_loss_fn = torch.nn.BCELoss()

	for epoch in range(config['mitigating']['epochs']):
		for param in critic.parameters():
			param.requires_grad = True
		for param in actor.parameters():
			param.requires_grad = False
		actor.eval()
		critic.train()
		for step in range(config['mitigating']['critic_steps']):
			critic_optimizer.zero_grad()
			indices = torch.randint(0, data.X_valid.size(0), (config['mitigating']['batch_size'],))
			cy_valid = data.y_valid_gpu[indices]
			cX_valid = data.X_valid_gpu[indices]
			cp_valid = data.p_valid_gpu[indices]
			with torch.no_grad():
				scores = actor(cX_valid)[:, 0].reshape(-1).cpu().numpy()

			res = critic(actor.trunc_forward(cX_valid))
			loss = critic_loss_fn(res[:, 0], cp_valid.type(torch.float32))
			loss.backward()
			train_loss = loss.item()
			critic_optimizer.step()
			if (epoch % 5 == 0) and (step % 100 == 0):
				logger.info(f'=======> Critic Epoch: {(epoch, step)} loss: {train_loss}')

		for param in critic.parameters():
			param.requires_grad = False
		for param in actor.parameters():
			param.requires_grad = True
		actor.train()
		critic.eval()
		for step in range(config['mitigating']['actor_steps']):
			actor_optimizer.zero_grad()
			indices = torch.randint(0, data.X_valid.size(0), (config['mitigating']['batch_size'],))
			cy_valid = data.y_valid_gpu[indices]
			cX_valid = data.X_valid_gpu[indices]
			cp_valid = data.p_valid_gpu[indices]

			cx_predict = actor(cX_valid)
			loss_pred = actor_loss_fn(cx_predict[:, 0], cy_valid)

			cp_predict = critic(actor.trunc_forward(cX_valid))
			loss_adv = critic_loss_fn(cp_predict[:, 0], cp_valid.type(torch.float32))

			for param in actor.parameters():
				try:
					lp = torch.autograd.grad(loss_pred, param, retain_graph=True)[0]
					la = torch.autograd.grad(loss_adv, param, retain_graph=True)[0]
				except RuntimeError:
					continue
				shape = la.shape
				lp = lp.flatten()
				la = la.flatten()
				lp_proj = (lp.T @ la) * la
				grad = lp - lp_proj - config['mitigating']['alpha'] * la
				grad = grad.reshape(shape)
				param.backward(grad)

			actor_optimizer.step()
			if (epoch % 5 == 0) and (step % 100 == 0):
				logger.info(f'=======> Actor Epoch: {(epoch, step)}')

		if epoch % 5 == 0:
			with torch.no_grad():
				scores = actor(data.X_valid_gpu)[:, 0].reshape(-1, 1).cpu().numpy()
				_, best_mit_obj = get_best_thresh(scores, np.linspace(0, 1, 1001), data, config,
												  margin=config['mitigating']['margin'])
				logger.info(f'Objective: {best_mit_obj}')

	logger.info('Finding optimal threshold for Mitigating model.')
	with torch.no_grad():
		scores = actor(data.X_valid_gpu)[:, 0].reshape(-1, 1).cpu().numpy()

	best_mit_thresh, _ = get_best_thresh(scores, np.linspace(0, 1, 1001), data, config,
										 margin=config['mitigating']['margin'])

	logger.info('Evaluating Mitigating model on best threshold.')
	with torch.no_grad():
		labels = (actor(data.X_valid_gpu)[:, 0] > best_mit_thresh).reshape(-1, 1).cpu().numpy()
	results_valid = get_valid_objective(labels, data, config)
	logger.info(f'Results: {results_valid}')

	with torch.no_grad():
		labels = (actor(data.X_test_gpu)[:, 0] > best_mit_thresh).reshape(-1, 1).cpu().numpy()
	results_test = get_test_objective(labels, data, config)

	return results_valid, results_test